Skip to main content

mj_controller/
server_runtime.rs

1//! The phone-oriented remote-control server: its HTTP surface, the controller
2//! actions phones request, and the concurrency limits that keep them safe.
3
4use std::net::{Ipv4Addr, SocketAddr};
5use std::path::PathBuf;
6use std::sync::Arc;
7use std::sync::atomic::{AtomicBool, Ordering};
8use std::time::{Duration, Instant};
9
10use anyhow::{Context, Result, bail};
11mod api;
12mod api_activity;
13
14use mj_core::config::{Config, HarnessProfile, PhoneConfig, is_bare_project_target};
15use mj_core::remote_git::{default_branch, display_url, resolve_repository};
16use mj_core::state::{MaterializedSession, ProjectSourceIdentity, SessionRecord, State};
17
18use crate::controller::Controller;
19use crate::quota::ProfileQuota;
20use crate::server::{
21    ActionOutcome, BackgroundTaskStopFailure, BackgroundTaskStopRequest, BrowserTranscript,
22    ControllerAction, ControllerRequest, MovePreparationRequest, PreflightFailure,
23    ReadReceiptRequest, ResumeQueueDisposition, ServerOptions, ViewerActivityDetails,
24    ViewerActivityKind, ViewerBackgroundTask, ViewerMoveRecovery, ViewerQueuedPrompt, ViewerQuota,
25    ViewerSnapshot, ViewerUserShell,
26};
27use crate::session_manager::{SessionManagerChannels, SessionManagerControl, new_command_id};
28use crate::tailscale::TailscaleTls;
29#[cfg(test)]
30use crate::targets::ProcessExecutor;
31use crate::targets::{CancellableProcessExecutor, CommandExecutor};
32use crate::worker_client::CredentialSyncCoordinator;
33use mj_core::relay::RelayCommand;
34use mj_core::workspace::WorkspaceRecord;
35
36use crate::controller::config_only_controller;
37use crate::daemon::{
38    CreateSessionControl, CreateSessionRequest, ResumeSessionRequest, RuntimeState,
39};
40use crate::pollers::{
41    CredentialSyncNotices, CredentialSyncSignalTracker, QUOTA_STALE_AFTER, QuotaRefreshBatch,
42    QuotaUpdate, apply_worker_record_update, credential_sync_targets, dashboard_worker_targets,
43    projected_queued_prompts, queued_prompt_projection, quota_refresh_profiles,
44    schedule_due_credential_syncs, spawn_quota_refresher,
45};
46
47#[derive(Debug, Clone)]
48pub struct ServerArgs {
49    bind: String,
50    tailscale_detect: bool,
51    tls_cert: Option<PathBuf>,
52    tls_key: Option<PathBuf>,
53}
54
55impl From<&PhoneConfig> for ServerArgs {
56    fn from(config: &PhoneConfig) -> Self {
57        Self {
58            bind: config.bind.clone(),
59            tailscale_detect: config.tailscale_detect,
60            tls_cert: config.tls_cert.clone(),
61            tls_key: config.tls_key.clone(),
62        }
63    }
64}
65
66const TAILSCALE_COMMAND_TIMEOUT: Duration = Duration::from_secs(120);
67const TAILSCALE_RENEW_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60);
68
69/// Transcript projection parses every stored ACP content chunk. Keep that work
70/// off the control task, and allow only a small number of projections to use
71/// the blocking pool at once so a burst of active sessions cannot turn the
72/// pool into an unbounded queue.
73const MAX_CONCURRENT_CONVERSATION_PROJECTIONS: usize = 2;
74const CONVERSATION_PROJECTION_CHANNEL_CAPACITY: usize = 16;
75
76#[derive(Debug, Clone, PartialEq, Eq)]
77struct ConversationProjectionKey {
78    ordinal: u64,
79    digest: String,
80}
81
82impl ConversationProjectionKey {
83    fn of(materialized: &MaterializedSession) -> Self {
84        Self {
85            ordinal: materialized.applied_event_ordinal,
86            digest: materialized.applied_event_digest.clone(),
87        }
88    }
89
90    /// Event ordinals are monotonic. A changed digest at one ordinal is also a
91    /// new projection, which preserves the integrity-repair path without
92    /// relying on string ordering for digests.
93    fn is_newer_than(&self, other: &Self) -> bool {
94        self.ordinal > other.ordinal
95            || (self.ordinal == other.ordinal && self.digest != other.digest)
96    }
97}
98
99struct ConversationProjectionRequest {
100    materialized: MaterializedSession,
101    key: ConversationProjectionKey,
102    generation: u64,
103}
104
105struct ConversationProjectionResult {
106    session_id: String,
107    key: ConversationProjectionKey,
108    generation: u64,
109    result: std::result::Result<BrowserTranscript, String>,
110}
111
112/// Owns the one-at-a-time/latest-state scheduling for each session. The
113/// control loop remains the sole owner of these maps; background tasks only
114/// return completed browser projections through `results`.
115struct ConversationProjectionDispatcher {
116    in_flight: std::collections::BTreeMap<String, (ConversationProjectionKey, u64)>,
117    pending: std::collections::BTreeMap<String, ConversationProjectionRequest>,
118    completed: std::collections::BTreeMap<String, ConversationProjectionKey>,
119    generations: std::collections::BTreeMap<String, u64>,
120    permits: Arc<tokio::sync::Semaphore>,
121    results: tokio::sync::mpsc::Sender<ConversationProjectionResult>,
122    shutdown: tokio_util::sync::CancellationToken,
123}
124
125impl ConversationProjectionDispatcher {
126    fn new(
127        results: tokio::sync::mpsc::Sender<ConversationProjectionResult>,
128        shutdown: tokio_util::sync::CancellationToken,
129    ) -> Self {
130        Self {
131            in_flight: std::collections::BTreeMap::new(),
132            pending: std::collections::BTreeMap::new(),
133            completed: std::collections::BTreeMap::new(),
134            generations: std::collections::BTreeMap::new(),
135            permits: Arc::new(tokio::sync::Semaphore::new(
136                MAX_CONCURRENT_CONVERSATION_PROJECTIONS,
137            )),
138            results,
139            shutdown,
140        }
141    }
142
143    #[cfg(test)]
144    fn with_permits(
145        results: tokio::sync::mpsc::Sender<ConversationProjectionResult>,
146        shutdown: tokio_util::sync::CancellationToken,
147        permits: usize,
148    ) -> Self {
149        let mut dispatcher = Self::new(results, shutdown);
150        dispatcher.permits = Arc::new(tokio::sync::Semaphore::new(permits));
151        dispatcher
152    }
153
154    /// Queue a session's newest durable view. At most one request is running
155    /// and one newer request is retained for any given session.
156    fn enqueue(&mut self, materialized: MaterializedSession) {
157        let session_id = materialized.session_id.clone();
158        let key = ConversationProjectionKey::of(&materialized);
159        if self
160            .completed
161            .get(&session_id)
162            .is_some_and(|completed| !key.is_newer_than(completed))
163        {
164            return;
165        }
166        let generation = *self.generations.entry(session_id.clone()).or_default();
167        if let Some((in_flight, in_flight_generation)) = self.in_flight.get(&session_id) {
168            if generation != *in_flight_generation || key.is_newer_than(in_flight) {
169                let replace = self.pending.get(&session_id).is_none_or(|pending| {
170                    pending.generation != generation || key.is_newer_than(&pending.key)
171                });
172                if replace {
173                    self.pending.insert(
174                        session_id,
175                        ConversationProjectionRequest {
176                            materialized,
177                            key,
178                            generation,
179                        },
180                    );
181                }
182            }
183            return;
184        }
185        self.in_flight
186            .insert(session_id.clone(), (key.clone(), generation));
187        self.start(ConversationProjectionRequest {
188            materialized,
189            key,
190            generation,
191        });
192    }
193
194    /// Finish one task and, when the session is still active, immediately
195    /// launch the newest coalesced request. Returning `None` means the task
196    /// failed or no longer belongs to the current in-flight request.
197    fn finish(
198        &mut self,
199        result: ConversationProjectionResult,
200        session_active: bool,
201    ) -> Option<(String, ConversationProjectionKey, BrowserTranscript)> {
202        let expected = self.in_flight.remove(&result.session_id);
203        if expected.as_ref() != Some(&(result.key.clone(), result.generation)) {
204            tracing::warn!(
205                session_id = %result.session_id,
206                "discarding an out-of-date browser transcript projection"
207            );
208            return None;
209        }
210        let session_id = result.session_id;
211        let key = result.key;
212        let current_generation = self
213            .generations
214            .get(&session_id)
215            .copied()
216            .unwrap_or_default();
217        let current = result.generation == current_generation;
218        let projected = match result.result {
219            Ok(transcript) if session_active && current => {
220                self.completed.insert(session_id.clone(), key.clone());
221                Some((session_id.clone(), key, transcript))
222            }
223            Ok(_) => {
224                // An inactive session, or a result from an earlier lifecycle
225                // generation, must not resurrect a conversation. Its next
226                // active update gets a fresh generation.
227                self.completed.remove(&session_id);
228                None
229            }
230            Err(error) => {
231                tracing::warn!(
232                    session_id = %session_id,
233                    "browser transcript projection failed: {error}"
234                );
235                None
236            }
237        };
238        if session_active {
239            if let Some(pending) = self.pending.remove(&session_id) {
240                self.enqueue(pending.materialized);
241            }
242        } else {
243            self.pending.remove(&session_id);
244            self.completed.remove(&session_id);
245        }
246        projected
247    }
248
249    /// Drop queued/completed state after a controller reload removes a
250    /// session. An in-flight task is allowed to finish; `finish` receives the
251    /// current active-state guard and discards its result.
252    fn forget(&mut self, session_id: &str) {
253        self.pending.remove(session_id);
254        self.completed.remove(session_id);
255        let generation = self.generations.entry(session_id.to_owned()).or_default();
256        *generation = generation.wrapping_add(1);
257    }
258
259    fn session_ids(&self) -> std::collections::BTreeSet<String> {
260        self.in_flight
261            .keys()
262            .chain(self.pending.keys())
263            .chain(self.completed.keys())
264            .cloned()
265            .collect()
266    }
267
268    fn start(&self, request: ConversationProjectionRequest) {
269        let session_id = request.materialized.session_id.clone();
270        let key = request.key;
271        let generation = request.generation;
272        let permits = Arc::clone(&self.permits);
273        let results = self.results.clone();
274        let shutdown = self.shutdown.clone();
275        tokio::spawn(async move {
276            let result = match tokio::select! {
277                _ = shutdown.cancelled() => return,
278                result = permits.acquire_owned() => result,
279            } {
280                Ok(permit) => {
281                    let projection = tokio::task::spawn_blocking(move || {
282                        mj_client::transcript::materialized_browser_transcript(
283                            &request.materialized,
284                        )
285                    })
286                    .await;
287                    drop(permit);
288                    projection
289                        .map_err(|error| format!("transcript projection task failed: {error}"))
290                }
291                Err(error) => Err(format!("transcript projection worker stopped: {error}")),
292            };
293            let message = ConversationProjectionResult {
294                session_id,
295                key,
296                generation,
297                result,
298            };
299            tokio::select! {
300                _ = shutdown.cancelled() => {}
301                result = results.send(message) => {
302                    if let Err(error) = result {
303                        tracing::debug!(%error, "browser transcript projection result dropped after server shutdown");
304                    }
305                }
306            }
307        });
308    }
309}
310
311struct ResolvedServerArgs {
312    bind: SocketAddr,
313    viewer_url: String,
314    tls_files: Option<(PathBuf, PathBuf)>,
315    tailscale: Option<TailscaleTls>,
316    fallback_reason: Option<String>,
317}
318
319async fn resolve_server_args(
320    args: ServerArgs,
321    termination: tokio_util::sync::CancellationToken,
322) -> Result<ResolvedServerArgs> {
323    let configured_bind: SocketAddr = args.bind.parse().context("parse web viewer bind address")?;
324    match (args.tls_cert, args.tls_key) {
325        (Some(cert), Some(key)) => {
326            let scheme = "https";
327            return Ok(ResolvedServerArgs {
328                bind: configured_bind,
329                viewer_url: format!("{scheme}://{configured_bind}"),
330                tls_files: Some((cert, key)),
331                tailscale: None,
332                fallback_reason: None,
333            });
334        }
335        (None, None) => {}
336        _ => bail!("web viewer TLS requires both a certificate and private key"),
337    }
338
339    if !args.tailscale_detect {
340        return Ok(loopback_server_args(
341            configured_bind,
342            Some("automatic Tailscale detection is disabled".into()),
343        ));
344    }
345
346    let tls_root = mj_core::config::data_dir().join("viewer");
347    let prepared = run_tailscale_blocking(termination.clone(), move |executor| {
348        crate::tailscale::prepare_tailscale_tls(&tls_root, executor)
349    })
350    .await;
351    match prepared {
352        Ok(tailscale) => {
353            let bind = tailscale_bind(configured_bind);
354            let viewer_url = format!(
355                "https://{}:{}",
356                tailscale.cert_domain(),
357                configured_bind.port()
358            );
359            Ok(ResolvedServerArgs {
360                bind,
361                viewer_url,
362                tls_files: Some((
363                    tailscale.cert_path().to_owned(),
364                    tailscale.key_path().to_owned(),
365                )),
366                tailscale: Some(tailscale),
367                fallback_reason: None,
368            })
369        }
370        Err(error) if termination.is_cancelled() => Err(error),
371        Err(error) => {
372            let reason = format!("{error:#}");
373            tracing::debug!(error = reason, "Tailscale HTTPS unavailable for web viewer");
374            Ok(loopback_server_args(configured_bind, Some(reason)))
375        }
376    }
377}
378
379fn tailscale_bind(configured_bind: SocketAddr) -> SocketAddr {
380    SocketAddr::from((Ipv4Addr::UNSPECIFIED, configured_bind.port()))
381}
382
383fn loopback_server_args(bind: SocketAddr, fallback_reason: Option<String>) -> ResolvedServerArgs {
384    ResolvedServerArgs {
385        bind,
386        viewer_url: format!("http://{bind}"),
387        tls_files: None,
388        tailscale: None,
389        fallback_reason,
390    }
391}
392
393async fn run_tailscale_blocking<T>(
394    termination: tokio_util::sync::CancellationToken,
395    operation: impl FnOnce(&CancellableProcessExecutor) -> Result<T> + Send + 'static,
396) -> Result<T>
397where
398    T: Send + 'static,
399{
400    let cancelled = Arc::new(AtomicBool::new(false));
401    let executor_cancelled = cancelled.clone();
402    let mut task = tokio::task::spawn_blocking(move || {
403        let executor = CancellableProcessExecutor::new(executor_cancelled)
404            .with_deadline(TAILSCALE_COMMAND_TIMEOUT);
405        operation(&executor)
406    });
407    tokio::select! {
408        result = &mut task => result.context("Tailscale background task panicked")?,
409        _ = termination.cancelled() => {
410            cancelled.store(true, Ordering::Release);
411            let _ = task.await;
412            bail!("Tailscale operation cancelled during web viewer shutdown")
413        }
414    }
415}
416
417fn spawn_tailscale_cert_renewer(
418    tailscale: TailscaleTls,
419    rustls: axum_server::tls_rustls::RustlsConfig,
420    termination: tokio_util::sync::CancellationToken,
421) -> tokio::task::JoinHandle<()> {
422    tokio::spawn(async move {
423        let mut interval = tokio::time::interval(TAILSCALE_RENEW_INTERVAL);
424        interval.tick().await;
425        loop {
426            tokio::select! {
427                _ = termination.cancelled() => return,
428                _ = interval.tick() => {}
429            }
430            let renewing = tailscale.clone();
431            let result = run_tailscale_blocking(termination.clone(), move |executor| {
432                renewing.renew(executor)
433            })
434            .await;
435            if let Err(error) = result {
436                if !termination.is_cancelled() {
437                    tracing::warn!(
438                        error = format!("{error:#}"),
439                        "Tailscale certificate renewal failed"
440                    );
441                }
442                continue;
443            }
444            if let Err(error) = rustls
445                .reload_from_pem_file(tailscale.cert_path(), tailscale.key_path())
446                .await
447            {
448                tracing::warn!(%error, "could not activate renewed Tailscale certificate");
449            }
450        }
451    })
452}
453
454const MAX_CONCURRENT_PHONE_ACTIONS: usize = 4;
455const MAX_CONCURRENT_BUNDLE_CREATIONS: usize = 4;
456const MAX_CONCURRENT_PREFLIGHTS: usize = 4;
457
458struct PhoneActionStarted {
459    action_id: u64,
460    session: SessionRecord,
461    published: tokio::sync::oneshot::Sender<std::result::Result<(), String>>,
462}
463
464/// The phone replies the control loop still owes.
465///
466/// A phone is answered as soon as its action is admitted, because provisioning,
467/// resume and close run for minutes and a request held open that long dies on a
468/// mobile network. `new` is the one action whose acceptance means more than
469/// admission: the phone has no session id until the provisional session is
470/// published, so its reply is parked here until the loop publishes it — or
471/// until the action ends without ever getting that far.
472#[derive(Default)]
473struct PendingActionReplies(
474    std::collections::BTreeMap<u64, tokio::sync::oneshot::Sender<ActionOutcome>>,
475);
476
477impl PendingActionReplies {
478    fn accept(
479        &mut self,
480        action_id: u64,
481        action: &ControllerAction,
482        reply: tokio::sync::oneshot::Sender<ActionOutcome>,
483    ) {
484        if matches!(action, ControllerAction::New { .. }) {
485            self.0.insert(action_id, reply);
486        } else {
487            if reply.send(ActionOutcome::accepted()).is_err() {
488                tracing::debug!(
489                    action_id,
490                    "phone action acceptance reply dropped after client disconnect"
491                );
492            }
493        }
494    }
495
496    fn resolve(&mut self, action_id: u64, outcome: ActionOutcome) {
497        if let Some(reply) = self.0.remove(&action_id)
498            && reply.send(outcome).is_err()
499        {
500            tracing::debug!(
501                action_id,
502                "phone action completion reply dropped after client disconnect"
503            );
504        }
505    }
506}
507
508/// Admission control for one phone action, run before any work starts so the
509/// answer to the phone never waits on the operation itself. Reports the session
510/// the action occupies, or the outcome that refuses it.
511fn admit_phone_action(
512    action: &ControllerAction,
513    running_actions: usize,
514    active_sessions: &mut std::collections::BTreeSet<String>,
515) -> std::result::Result<Option<String>, ActionOutcome> {
516    let closing = matches!(
517        action,
518        ControllerAction::Close { .. } | ControllerAction::ForceClose { .. }
519    );
520    if !closing && !phone_action_capacity_available(running_actions) {
521        return Err(ActionOutcome::Busy);
522    }
523    let session_id = controller_action_session_id(action);
524    if let Some(session_id) = &session_id
525        && !active_sessions.insert(session_id.clone())
526        && !closing
527    {
528        return Err(ActionOutcome::SessionBusy);
529    }
530    Ok(session_id)
531}
532
533struct ReadReceiptPersisted {
534    session_id: String,
535    result: std::result::Result<u64, String>,
536    reply: tokio::sync::oneshot::Sender<std::result::Result<(), String>>,
537}
538
539struct ControllerReloaded {
540    result: std::result::Result<Controller, String>,
541}
542
543struct BundleCreated {
544    result: std::result::Result<
545        crate::controller::QuickBundleCreation,
546        crate::controller::QuickBundleFailure,
547    >,
548    reply: tokio::sync::oneshot::Sender<std::result::Result<String, crate::server::BundleFailure>>,
549}
550
551struct MovePrepared {
552    result: std::result::Result<mj_core::state::MovePreparation, String>,
553    reply:
554        tokio::sync::oneshot::Sender<std::result::Result<mj_core::state::MovePreparation, String>>,
555}
556
557/// Loads durable controller state without occupying the phone control loop.
558/// The outer task observes blocking-task panics and reports a closed result
559/// channel instead of silently abandoning the refresh.
560fn spawn_controller_reload(completed: tokio::sync::mpsc::UnboundedSender<ControllerReloaded>) {
561    spawn_controller_reload_with(completed, Controller::load);
562}
563
564fn spawn_controller_reload_with(
565    completed: tokio::sync::mpsc::UnboundedSender<ControllerReloaded>,
566    load: impl FnOnce() -> Result<Controller> + Send + 'static,
567) {
568    tokio::spawn(async move {
569        let result = match tokio::task::spawn_blocking(load).await {
570            Ok(result) => result.map_err(|error| format!("{error:#}")),
571            Err(error) => Err(format!("controller reload task failed: {error}")),
572        };
573        if completed.send(ControllerReloaded { result }).is_err() {
574            tracing::debug!("controller reload completed after the phone control loop stopped");
575        }
576    });
577}
578
579fn request_controller_reload(
580    in_flight: &mut bool,
581    requested: &mut bool,
582    completed: &tokio::sync::mpsc::UnboundedSender<ControllerReloaded>,
583) {
584    if *in_flight {
585        *requested = true;
586    } else {
587        *in_flight = true;
588        spawn_controller_reload(completed.clone());
589    }
590}
591
592fn request_daemon_controller_reload(daemon_runtime: Arc<RuntimeState>, reason: &'static str) {
593    tokio::spawn(async move {
594        if let Err(error) = daemon_runtime.reload_controller().await {
595            tracing::warn!(
596                error = format!("{error:#}"),
597                reason,
598                "phone operation could not refresh dashboard controller state"
599            );
600        }
601    });
602}
603
604/// What one phone read receipt actually needs.
605#[derive(Debug, PartialEq, Eq)]
606#[cfg(test)]
607enum ReadReceiptPlan {
608    UnknownSession,
609    /// The cursor has not advanced, so the receipt needs no work at all.
610    AlreadyRead,
611    /// The cursor advanced: persist it, then refresh the snapshot.
612    Persist,
613}
614
615#[cfg(test)]
616fn plan_read_receipt(state: &State, session_id: &str, through: u64) -> ReadReceiptPlan {
617    let Some(session) = state.sessions.get(session_id) else {
618        return ReadReceiptPlan::UnknownSession;
619    };
620    if through > session.viewed_through_event_ordinal {
621        ReadReceiptPlan::Persist
622    } else {
623        ReadReceiptPlan::AlreadyRead
624    }
625}
626
627/// Record a persisted receipt in the in-memory projection, reporting whether
628/// the cursor moved. That is exactly when the snapshot revision has to move,
629/// so surfaces showing unread state refresh and nothing else does.
630#[cfg(test)]
631fn apply_read_receipt(state: &mut State, session_id: &str, receipt: u64) -> bool {
632    let Some(session) = state.sessions.get_mut(session_id) else {
633        return false;
634    };
635    if receipt <= session.viewed_through_event_ordinal {
636        return false;
637    }
638    session.viewed_through_event_ordinal = receipt;
639    true
640}
641
642#[derive(Clone)]
643struct PhoneActionControl {
644    cancelled: Arc<AtomicBool>,
645    create: Option<CreateSessionControl>,
646}
647
648impl PhoneActionControl {
649    fn for_action(action: &ControllerAction) -> Self {
650        let create =
651            matches!(action, ControllerAction::New { .. }).then(CreateSessionControl::default);
652        let cancelled = create.as_ref().map_or_else(
653            || Arc::new(AtomicBool::new(false)),
654            |control| control.cancelled.clone(),
655        );
656        Self { cancelled, create }
657    }
658
659    fn request_cancel(&self) -> bool {
660        let accepted = self.create.as_ref().map_or_else(
661            || {
662                self.cancelled
663                    .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
664                    .is_ok()
665            },
666            |control| control.request_cancel(),
667        );
668        if accepted {
669            self.cancelled.store(true, Ordering::Release);
670        }
671        accepted
672    }
673
674    #[cfg(test)]
675    fn grant_new_commit(&self) -> bool {
676        self.create
677            .as_ref()
678            .is_some_and(|control| control.grant_commit())
679    }
680}
681
682pub async fn run_server(
683    args: ServerArgs,
684    termination: tokio_util::sync::CancellationToken,
685    worker: SessionManagerChannels,
686    daemon_runtime: Arc<RuntimeState>,
687    mut workspace_updates: tokio::sync::watch::Receiver<Vec<WorkspaceRecord>>,
688) -> Result<()> {
689    let resolved = resolve_server_args(args, termination.clone()).await?;
690    let bind = resolved.bind;
691    let mut controller = Controller::load()?;
692    let mut daemon_revisions = daemon_runtime.revisions();
693    daemon_revisions.borrow_and_update();
694    let mut phone_workspaces = workspace_updates.borrow_and_update().clone();
695    let mut quotas = std::collections::BTreeMap::new();
696    let subagent_quota_reports = Arc::new(std::sync::Mutex::new(quotas.clone()));
697    let (quota_profiles_tx, mut quota_updates_rx) = spawn_quota_refresher();
698    let mut quota_batch = QuotaRefreshBatch::default();
699    let mut published_quota_profiles = std::collections::BTreeMap::new();
700    republish_quota_profiles(
701        &controller,
702        &mut published_quota_profiles,
703        &mut quota_batch,
704        &quota_profiles_tx,
705    );
706    let mut revision = daemon_runtime.allocate_revision();
707    let mut conversations = std::collections::BTreeMap::new();
708    let mut queued_prompts = projected_queued_prompts(&controller)?;
709    let mut active_user_shells = std::collections::BTreeMap::new();
710    let mut pending_elicitations = std::collections::BTreeMap::new();
711    let mut prompt_images = std::collections::BTreeSet::new();
712    let mut operational = std::collections::BTreeMap::new();
713    let mut materialized_activity = load_materialized_activity(&controller).await?;
714    let mut project_sources = PhoneProjectSources::default();
715    let (records, lifecycles) = daemon_runtime.session_projection();
716    controller.state.sessions = records;
717    let mut operations = lifecycles
718        .iter()
719        .map(|view| (view.session_id.clone(), viewer_operation(view)))
720        .collect::<std::collections::BTreeMap<_, _>>();
721    let mut move_recoveries = ViewerMoveRecoveries::new();
722    let (move_recovery_tx, mut move_recovery_rx) =
723        tokio::sync::mpsc::unbounded_channel::<Result<ViewerMoveRecoveries, String>>();
724    let mut move_recovery_load_in_flight = false;
725    let mut launch_failures = Vec::new();
726    // What the capacity poller last said, per probe target. The projection is
727    // built from this on every publish rather than being accumulated, so a
728    // target that disappears from the configuration disappears from the page.
729    let mut capacity_state: std::collections::BTreeMap<String, PhoneCapacity> =
730        std::collections::BTreeMap::new();
731    let (capacity_targets_tx, capacity_triggers_tx, mut capacity_updates_rx) =
732        crate::pollers::spawn_dashboard_capacity_poller();
733    let (snapshot_tx, snapshot_rx) = tokio::sync::watch::channel(viewer_snapshot(
734        &controller,
735        &phone_workspaces,
736        &quotas,
737        &PhoneSessionViews {
738            conversations: &conversations,
739            queued_prompts: &queued_prompts,
740            active_user_shells: &active_user_shells,
741            pending_elicitations: &pending_elicitations,
742            prompt_images: &prompt_images,
743            operational: &operational,
744            materialized_activity: &materialized_activity,
745            project_sources: &project_sources,
746            operations: &operations,
747            move_recoveries: &move_recoveries,
748            capacity: &viewer_capacity(&capacity_state),
749            launch_failures: &launch_failures,
750            reviews: &review_views(&daemon_runtime),
751        },
752        revision,
753    ));
754    let (conversation_tx, conversation_rx) = tokio::sync::watch::channel(conversations.clone());
755    let (action_tx, mut action_rx) = tokio::sync::mpsc::channel(32);
756    let (bundle_tx, mut bundle_rx) = tokio::sync::mpsc::channel(16);
757    let (receipt_tx, mut receipt_rx) = tokio::sync::mpsc::channel(32);
758    let (preflight_tx, mut preflight_rx) = tokio::sync::mpsc::channel(32);
759    let (move_preparation_tx, mut move_preparation_rx) = tokio::sync::mpsc::channel(32);
760    let (client_state_tx, mut client_state_rx) = tokio::sync::mpsc::channel(64);
761    let (dictation_tx, mut dictation_rx) =
762        tokio::sync::mpsc::channel::<crate::dictation::DictationRequest>(8);
763    let (background_task_stop_tx, mut background_task_stop_rx) =
764        tokio::sync::mpsc::channel::<BackgroundTaskStopRequest>(32);
765    let SessionManagerChannels {
766        targets: worker_targets_tx,
767        control: worker_commands_tx,
768        updates: mut worker_updates_rx,
769        shutdown: worker_shutdown,
770    } = worker;
771    worker_targets_tx.send_replace(dashboard_worker_targets(&controller));
772    publish_capacity_targets(&controller, &capacity_targets_tx, &mut capacity_state);
773    let mut credential_sync = CredentialSyncCoordinator::spawn();
774    let credential_sync_handle = credential_sync.handle();
775    credential_sync_handle.set_targets(credential_sync_targets(&controller));
776    let mut credential_sync_signals = CredentialSyncSignalTracker::default();
777    let mut credential_sync_notices = CredentialSyncNotices::default();
778    // Captured before `options` is moved into the server.
779    let options_session_ttl = crate::server::default_session_ttl();
780    let activity_snapshots = snapshot_rx.clone();
781    let mut options = ServerOptions::new(
782        bind,
783        snapshot_rx,
784        conversation_rx,
785        crate::server::ServerRequests {
786            action_tx,
787            bundle_tx,
788            receipt_tx,
789            preflight_tx,
790            move_preparation_tx,
791            client_state_tx,
792            dictation_tx,
793        },
794    )?;
795    options.set_background_task_stop_tx(background_task_stop_tx);
796    options.shutdown = termination.clone();
797    // Session cookies are stateless, so a per-process key would sign every
798    // phone out on every restart. Delete the key file to sign them out on
799    // purpose.
800    let cookie_key_path = crate::server::cookie_key_path();
801    options.set_cookie_key(crate::server::load_or_create_cookie_key(&cookie_key_path)?)?;
802    // The documented `/api/v1` surface authenticates with a persisted bearer
803    // token and drives sessions through the daemon-side backend.
804    options.set_api_token(crate::server::load_or_create_api_token(
805        &crate::server::api_token_path(),
806    )?);
807    let api_runtime = daemon_runtime.clone();
808    let api_backend = Arc::new(
809        api::ApiBackend::new(
810            worker_commands_tx.client(),
811            Arc::new(move |session_id: &str| api_runtime.session_state(session_id)),
812            daemon_runtime.clone(),
813        )
814        .with_quota_reports(subagent_quota_reports.clone()),
815    );
816    options.set_subagent_backend(api_backend.clone());
817    let renewal_cancellation = termination.child_token();
818    let mut renewal_task = None;
819    if let Some((cert, key)) = resolved.tls_files {
820        let rustls = axum_server::tls_rustls::RustlsConfig::from_pem_file(cert, key)
821            .await
822            .context("load web viewer TLS certificate")?;
823        options.set_tls_config(rustls.clone());
824        if let Some(tailscale) = resolved.tailscale {
825            renewal_task = Some(spawn_tailscale_cert_renewer(
826                tailscale,
827                rustls,
828                renewal_cancellation.clone(),
829            ));
830        }
831    } else if bind.ip().is_loopback() {
832        options.secure_cookie = false;
833    } else {
834        anyhow::bail!("non-loopback web viewer requires TLS");
835    }
836    let fallback_reason = resolved.fallback_reason;
837    let qr_login_url = if fallback_reason.is_none() && resolved.viewer_url.starts_with("https://") {
838        let encoded = url::form_urlencoded::byte_serialize(options.login_token().as_bytes())
839            .collect::<String>();
840        Some(format!(
841            "{}/auth/login?token={encoded}",
842            resolved.viewer_url.trim_end_matches('/')
843        ))
844    } else {
845        None
846    };
847    let ready = crate::server::WebViewerAccess::Ready {
848        viewer_url: resolved.viewer_url,
849        viewer_code: options.viewer_code().to_owned(),
850        qr_login_url,
851        fallback_reason,
852    };
853
854    let serve = crate::web_viewer::serve(options, ready, &daemon_runtime.web_viewer, |access| {
855        daemon_runtime.publish_web_access(access);
856    });
857    let conversation_projection_shutdown = termination.child_token();
858    let control = async {
859        let mut credential_tick = tokio::time::interval(Duration::from_millis(250));
860        // Stored viewer state expires with the authentication that created it.
861        // The sweep is hourly rather than on every request, because it is
862        // housekeeping and nothing waits for it.
863        let mut prune_tick = tokio::time::interval(Duration::from_secs(60 * 60));
864        let client_state_retention = options_session_ttl;
865        let (action_done_tx, mut action_done_rx) = tokio::sync::mpsc::unbounded_channel::<(
866            u64,
867            Option<String>,
868            std::result::Result<(), String>,
869        )>();
870        let (action_started_tx, mut action_started_rx) =
871            tokio::sync::mpsc::unbounded_channel::<PhoneActionStarted>();
872        let (receipt_done_tx, mut receipt_done_rx) =
873            tokio::sync::mpsc::unbounded_channel::<ReadReceiptPersisted>();
874        let (controller_reload_tx, mut controller_reload_rx) =
875            tokio::sync::mpsc::unbounded_channel::<ControllerReloaded>();
876        let (bundle_done_tx, mut bundle_done_rx) =
877            tokio::sync::mpsc::unbounded_channel::<BundleCreated>();
878        let (move_prepared_tx, mut move_prepared_rx) =
879            tokio::sync::mpsc::unbounded_channel::<MovePrepared>();
880        let mut dictation_jobs = tokio::task::JoinSet::new();
881        let mut bundle_jobs = tokio::task::JoinSet::new();
882        let mut preflight_jobs = tokio::task::JoinSet::new();
883        let mut move_preparation_jobs = tokio::task::JoinSet::new();
884        let mut move_recovery_jobs = tokio::task::JoinSet::new();
885        let mut background_task_stop_jobs = tokio::task::JoinSet::new();
886        let mut background_task_stop_open = true;
887        let mut controller_reload_in_flight = false;
888        let mut controller_reload_requested = false;
889        let mut controller_reload_invalidated = false;
890        let mut pending_action_errors = std::collections::BTreeMap::<String, String>::new();
891        let mut active_actions = std::collections::BTreeSet::new();
892        let mut closing_actions = std::collections::BTreeMap::<String, u64>::new();
893        let mut next_action_id = 0_u64;
894        let mut action_cancellations = std::collections::BTreeMap::<u64, PhoneActionControl>::new();
895        let mut action_sessions = std::collections::BTreeMap::<u64, String>::new();
896        let mut action_replies = PendingActionReplies::default();
897        let mut launch_workspaces = std::collections::BTreeMap::new();
898        let mut subagent_jobs = tokio::task::JoinSet::new();
899        let mut subagent_completion_jobs = tokio::task::JoinSet::new();
900        let mut active_subagent_requests = std::collections::BTreeSet::new();
901        let (conversation_projection_tx, mut conversation_projection_rx) =
902            tokio::sync::mpsc::channel(CONVERSATION_PROJECTION_CHANNEL_CAPACITY);
903        let mut conversation_projections = ConversationProjectionDispatcher::new(
904            conversation_projection_tx,
905            conversation_projection_shutdown.clone(),
906        );
907        let mut quota_updates_open = true;
908        // A feed that ends is not a reason to exit quietly: the phone server
909        // exists to follow sessions, so losing that feed is a named failure
910        // rather than a silent success.
911        let mut failure: Option<anyhow::Error> = None;
912        request_move_recovery_reload(
913            &move_recovery_tx,
914            &mut move_recovery_load_in_flight,
915            &mut move_recovery_jobs,
916        );
917        macro_rules! publish_snapshot {
918            ($revision:expr) => {
919                let (records, lifecycles) = daemon_runtime.session_projection();
920                controller.state.sessions = records;
921                for (session_id, error) in &pending_action_errors {
922                    if let Some(session) = controller.state.sessions.get_mut(session_id)
923                        && session.last_error.is_none()
924                    {
925                        session.last_error = Some(error.clone());
926                    }
927                }
928                operations = lifecycles.iter()
929                    .map(|view| (view.session_id.clone(), viewer_operation(view)))
930                    .collect();
931                if let Err(error) = snapshot_tx.send(viewer_snapshot(
932                    &controller,
933                    &phone_workspaces,
934                    &quotas,
935                    &PhoneSessionViews {
936                        conversations: &conversations,
937                        queued_prompts: &queued_prompts,
938                        active_user_shells: &active_user_shells,
939                        pending_elicitations: &pending_elicitations,
940                        prompt_images: &prompt_images,
941                        operational: &operational,
942                        materialized_activity: &materialized_activity,
943                        project_sources: &project_sources,
944                        operations: &operations,
945                        move_recoveries: &move_recoveries,
946                        capacity: &viewer_capacity(&capacity_state),
947                        launch_failures: &launch_failures,
948                        reviews: &review_views(&daemon_runtime),
949                    },
950                    $revision,
951                )) {
952                    tracing::debug!(revision = $revision, %error, "phone snapshot delivery failed; no viewer is subscribed");
953                }
954            };
955        }
956        loop {
957            project_sources.synchronize(&controller);
958            tokio::select! {
959                _ = termination.cancelled() => break,
960                request = background_task_stop_rx.recv(), if background_task_stop_open => {
961                    let Some(request) = request else {
962                        background_task_stop_open = false;
963                        tracing::warn!("background-task stop request feed closed while the phone server was running");
964                        continue;
965                    };
966                    let session_control = worker_commands_tx.clone();
967                    background_task_stop_jobs.spawn(async move {
968                        let result = match session_control.session(&request.session_id).await {
969                            Ok(session) => session
970                                .client()
971                                .stop_background_task(request.background_task_id.clone())
972                                .await
973                                .map_err(|error| {
974                                    tracing::warn!(
975                                        session_id = %request.session_id,
976                                        background_task_id = %request.background_task_id,
977                                        %error,
978                                        "provider rejected background-task stop"
979                                    );
980                                    BackgroundTaskStopFailure::Provider
981                                }),
982                            Err(error) => {
983                                tracing::warn!(
984                                    session_id = %request.session_id,
985                                    %error,
986                                    "could not resolve live session for background-task stop"
987                                );
988                                Err(BackgroundTaskStopFailure::SessionUnavailable)
989                            }
990                        };
991                        if request.reply.send(result).is_err() {
992                            tracing::debug!(
993                                session_id = %request.session_id,
994                                background_task_id = %request.background_task_id,
995                                "background-task stop result dropped after viewer disconnected"
996                            );
997                        }
998                    });
999                }
1000                completed = background_task_stop_jobs.join_next(), if !background_task_stop_jobs.is_empty() => {
1001                    if let Some(Err(error)) = completed {
1002                        tracing::error!(%error, "background-task stop task failed unexpectedly");
1003                    }
1004                }
1005                move_reloaded = move_recovery_rx.recv() => {
1006                    let Some(result) = move_reloaded else {
1007                        failure = feed_stopped(
1008                            termination.is_cancelled(),
1009                            "the Move recovery projection stopped while the phone server was running",
1010                        );
1011                        break;
1012                    };
1013                    move_recovery_load_in_flight = false;
1014                    match result {
1015                        Ok(recoveries) => {
1016                            move_recoveries = recoveries;
1017                            revision = daemon_runtime.allocate_revision();
1018                            publish_snapshot!(revision);
1019                        }
1020                        Err(error) => tracing::warn!(%error, "could not refresh Move recovery projection"),
1021                    }
1022                }
1023                resolved = project_sources.jobs.join_next(), if !project_sources.jobs.is_empty() => {
1024                    match resolved {
1025                        Some(Ok(resolved)) => project_sources.complete(resolved),
1026                        Some(Err(error)) => {
1027                            failure = Some(anyhow::anyhow!("web project source task failed: {error}"));
1028                            break;
1029                        }
1030                        None => unreachable!("project source jobs were not empty"),
1031                    }
1032                    revision = daemon_runtime.allocate_revision();
1033                    publish_snapshot!(revision);
1034                }
1035                changed = daemon_revisions.changed() => {
1036                    if changed.is_err() {
1037                        failure = feed_stopped(
1038                            termination.is_cancelled(),
1039                            "the daemon stopped publishing runtime revisions to the phone server",
1040                        );
1041                        break;
1042                    }
1043                    daemon_revisions.borrow_and_update();
1044                    revision = daemon_runtime.allocate_revision();
1045                    publish_snapshot!(revision);
1046                    request_controller_reload(
1047                        &mut controller_reload_in_flight,
1048                        &mut controller_reload_requested,
1049                        &controller_reload_tx,
1050                    );
1051                }
1052                changed = workspace_updates.changed() => {
1053                    if changed.is_err() {
1054                        failure = feed_stopped(
1055                            termination.is_cancelled(),
1056                            "the daemon stopped publishing workspaces to the phone server",
1057                        );
1058                        break;
1059                    }
1060                    phone_workspaces = workspace_updates.borrow_and_update().clone();
1061                    revision = daemon_runtime.allocate_revision();
1062                    publish_snapshot!(revision);
1063                }
1064                update = capacity_updates_rx.recv() => {
1065                    let Some(update) = update else {
1066                        failure = feed_stopped(termination.is_cancelled(), "the capacity poller stopped while the phone server was running");
1067                        break;
1068                    };
1069                    if let Some(entry) = capacity_state.get_mut(&update.target_id) {
1070                        entry.refreshing = false;
1071                        entry.sampled_at_epoch_seconds = Some(update.sampled_at_epoch_seconds);
1072                        match update.result {
1073                            Ok(usage) => {
1074                                // A fleet with nothing running reports no
1075                                // figures, and that is an answer rather than a
1076                                // failure.
1077                                entry.on_demand = usage.is_none();
1078                                entry.usage = usage;
1079                                entry.failed = false;
1080                            }
1081                            // The last good reading stays on screen beside the
1082                            // failure: one failed probe is not a reason to
1083                            // forget what the machine was doing.
1084                            Err(_) => entry.failed = true,
1085                        }
1086                    }
1087                    revision = daemon_runtime.allocate_revision();
1088                    publish_snapshot!(revision);
1089                }
1090                update = quota_updates_rx.recv(), if quota_updates_open => {
1091                    match update {
1092                        Some(QuotaUpdate::Report(outcome)) => {
1093                            if outcome.credentials_changed {
1094                                credential_sync_handle
1095                                    .sync_profile_now(&outcome.report.profile_id, None);
1096                            }
1097                            quotas.insert(outcome.report.profile_id.clone(), outcome.report.clone());
1098                            subagent_quota_reports
1099                                .lock()
1100                                .expect("sub-agent quota reports lock poisoned")
1101                                .insert(outcome.report.profile_id.clone(), outcome.report);
1102                            revision = daemon_runtime.allocate_revision();
1103                            publish_snapshot!(revision);
1104                        }
1105                        Some(QuotaUpdate::Refreshing { .. } | QuotaUpdate::Finished { .. }) => {}
1106                        None => {
1107                            quota_updates_open = false;
1108                            tracing::warn!("quota refresher stopped while the phone server is running");
1109                        }
1110                    }
1111                }
1112                projected = conversation_projection_rx.recv() => {
1113                    let Some(projected) = projected else {
1114                        failure = feed_stopped(
1115                            termination.is_cancelled(),
1116                            "the browser transcript projection feed stopped",
1117                        );
1118                        break;
1119                    };
1120                    let session_id = projected.session_id.clone();
1121                    let session_active = controller
1122                        .state
1123                        .sessions
1124                        .get(&session_id)
1125                        .is_some_and(|session| session.state.is_active());
1126                    if !session_active {
1127                        // Invalidate a late result before it can be applied or
1128                        // launch another queued projection. A session that is
1129                        // resumed later gets a new generation from its next
1130                        // worker snapshot.
1131                        conversation_projections.forget(&session_id);
1132                    }
1133                    if let Some((session_id, _key, transcript)) =
1134                        conversation_projections.finish(projected, session_active)
1135                    {
1136                        conversations.insert(session_id, transcript);
1137                        revision = daemon_runtime.allocate_revision();
1138                        conversation_tx.send_replace(conversations.clone());
1139                        publish_snapshot!(revision);
1140                    } else if !session_active && conversations.remove(&session_id).is_some() {
1141                        // Controller reload normally removes inactive rows
1142                        // first, but this also covers a worker result racing
1143                        // that reload and keeps the viewer from seeing a
1144                        // conversation for a dead session.
1145                        revision = daemon_runtime.allocate_revision();
1146                        conversation_tx.send_replace(conversations.clone());
1147                        publish_snapshot!(revision);
1148                    }
1149                    // SessionManagerUpdates has a synchronous pending fast
1150                    // path. Yield after each completion so a hot stream of
1151                    // updates cannot monopolize this runtime worker.
1152                    tokio::task::yield_now().await;
1153                }
1154                update = worker_updates_rx.recv() => {
1155                    let Some(update) = update else {
1156                        failure = feed_stopped(termination.is_cancelled(), "the session manager stopped; the phone server can no longer follow sessions");
1157                        break;
1158                    };
1159                    if let Some(snapshot) = update.view.snapshot.as_ref()
1160                        && let Some(session) = controller.state.sessions.get(&update.session_id)
1161                        && let Some(signal) = snapshot.latest_credential_sync_signal.clone()
1162                    {
1163                        credential_sync_signals.observe(
1164                            &update.session_id,
1165                            &session.last_profile,
1166                            signal,
1167                        );
1168                    }
1169                    schedule_due_credential_syncs(
1170                        &mut credential_sync_signals,
1171                        &credential_sync_handle,
1172                        Instant::now(),
1173                    );
1174                    apply_worker_record_update(&mut controller, &update);
1175                    if let Some(snapshot) = update.view.snapshot {
1176                        for request in snapshot.subagent_requests.iter().cloned() {
1177                            let identity = (update.session_id.clone(), request.request_id.clone());
1178                            if !active_subagent_requests.insert(identity.clone()) {
1179                                continue;
1180                            }
1181                            let backend = api_backend.clone();
1182                            let runtime = daemon_runtime.clone();
1183                            let parent_session_id = update.session_id.clone();
1184                            subagent_jobs.spawn(async move {
1185                                let result = backend
1186                                    .execute_subagent_tool(parent_session_id.clone(), request)
1187                                    .await;
1188                                let outcome = async {
1189                                    backend
1190                                        .deliver_subagent_result(parent_session_id.clone(), &result)
1191                                        .await?;
1192                                    let handle = runtime
1193                                        .workspace_session_handle(&parent_session_id)
1194                                        .await?;
1195                                    let mut lease = handle.lease_connection().await?;
1196                                    lease
1197                                        .connection_mut()
1198                                        .complete_subagent_request(result)
1199                                        .await?;
1200                                    lease.release();
1201                                    anyhow::Ok(())
1202                                }
1203                                .await;
1204                                (identity, outcome)
1205                            });
1206                        }
1207                        if let Some(relation) = controller.state.subagents.get_mut(&update.session_id)
1208                            && matches!(snapshot.materialized.execution, mj_core::state::MaterializedExecutionState::Idle)
1209                            && let Some(outcome) = snapshot.materialized.last_turn_outcome.as_ref()
1210                            && relation.delivered_turn != Some(outcome.completed_ordinal)
1211                        {
1212                            let turn = outcome.completed_ordinal;
1213                            let output = snapshot
1214                                .materialized
1215                                .transcript
1216                                .iter()
1217                                .rev()
1218                                .find_map(|item| match &item.body {
1219                                    mj_core::transcript::TranscriptBody::Agent { chunks, .. }
1220                                        if item.position >= outcome.turn_start_position.unwrap_or(0) =>
1221                                    {
1222                                        Some(mj_core::transcript::materialized_chunks_text(chunks))
1223                                    }
1224                                    _ => None,
1225                                })
1226                                .unwrap_or_else(|| "The sub-agent completed without a final text response.".to_owned());
1227                            let child_id = relation.child_session_id.clone();
1228                            let parent_id = relation.parent_session_id.clone();
1229                            let task_name = relation.task_name.clone();
1230                            let outcome_name = format!("{:?}", outcome.outcome).to_lowercase();
1231                            relation.delivered_turn = Some(turn);
1232                            let backend = api_backend.clone();
1233                            subagent_completion_jobs.spawn(async move {
1234                                let result = async {
1235                                    backend
1236                                        .deliver_subagent_completion(
1237                                            parent_id,
1238                                            &child_id,
1239                                            &task_name,
1240                                            turn,
1241                                            &outcome_name,
1242                                            &output,
1243                                        )
1244                                        .await?;
1245                                    tokio::task::spawn_blocking({
1246                                        let child_id = child_id.clone();
1247                                        move || crate::database::mark_subagent_turn_delivered(&child_id, turn)
1248                                    })
1249                                    .await??;
1250                                    anyhow::Ok(())
1251                                }
1252                                .await;
1253                                (child_id, turn, result)
1254                            });
1255                        }
1256                        if snapshot.operational.native_session_is_ready()
1257                            && operational.get(&update.session_id).is_none_or(|old: &mj_core::relay::RelayOperationalState| old.config_options != snapshot.operational.config_options)
1258                            && let Some(session) = controller.state.sessions.get(&update.session_id)
1259                            && matches!(session.target, Some(mj_core::state::TargetLocator::LocalBare { .. } | mj_core::state::TargetLocator::SshBare { .. } | mj_core::state::TargetLocator::AwsEc2 { .. }))
1260                            && let Some(build) = snapshot.worker_build.clone()
1261                        {
1262                            let profile = session.last_profile.clone();
1263                            let state = snapshot.operational.clone();
1264                            tokio::spawn(async move {
1265                                if let Err(error) = crate::controller::profile_config::observe(profile, build, state).await {
1266                                    tracing::warn!(%error, "could not cache observed profile choices");
1267                                }
1268                            });
1269                        }
1270                        let materialized = snapshot.materialized;
1271                        let operational_state = snapshot.operational;
1272                        materialized_activity.insert(
1273                            update.session_id.clone(),
1274                            materialized.last_activity_at_ms,
1275                        );
1276                        let queued = queued_prompt_projection(&materialized);
1277                        let pending = materialized.pending_elicitations.clone();
1278                        let active_shells = operational_state.active_user_shells.clone();
1279                        let prompt_images_supported =
1280                            agent_accepts_prompt_images(&operational_state);
1281                        active_user_shells.insert(
1282                            update.session_id.clone(),
1283                            active_shells,
1284                        );
1285                        conversation_projections.enqueue(materialized);
1286                        queued_prompts.insert(
1287                            update.session_id.clone(),
1288                            queued,
1289                        );
1290                        pending_elicitations.insert(
1291                            update.session_id.clone(),
1292                            pending,
1293                        );
1294                        if prompt_images_supported {
1295                            prompt_images.insert(update.session_id.clone());
1296                        } else {
1297                            prompt_images.remove(&update.session_id);
1298                        }
1299                        operational.insert(
1300                            update.session_id.clone(),
1301                            operational_state,
1302                        );
1303                        revision = daemon_runtime.allocate_revision();
1304                        conversation_tx.send_replace(conversations.clone());
1305                        publish_snapshot!(revision);
1306                    }
1307                    // The session update receiver can return pending entries
1308                    // without touching Tokio's budgeted receive operation.
1309                    // Give HTTP/TLS tasks a scheduling opportunity after each
1310                    // update even when the worker is publishing continuously.
1311                    tokio::task::yield_now().await;
1312                }
1313                completed = subagent_jobs.join_next(), if !subagent_jobs.is_empty() => {
1314                    match completed {
1315                        Some(Ok((identity, Ok(())))) => {
1316                            active_subagent_requests.remove(&identity);
1317                        }
1318                        Some(Ok((identity, Err(error)))) => {
1319                            active_subagent_requests.remove(&identity);
1320                            tracing::warn!(
1321                                parent_session_id = %identity.0,
1322                                request_id = %identity.1,
1323                                error = %format!("{error:#}"),
1324                                "sub-agent tool request failed"
1325                            );
1326                        }
1327                        Some(Err(error)) => tracing::warn!(%error, "sub-agent tool task panicked"),
1328                        None => {}
1329                    }
1330                }
1331                completed = subagent_completion_jobs.join_next(), if !subagent_completion_jobs.is_empty() => {
1332                    match completed {
1333                        Some(Ok((_, _, Ok(())))) => {}
1334                        Some(Ok((child_id, turn, Err(error)))) => {
1335                            if let Some(relation) = controller.state.subagents.get_mut(&child_id)
1336                                && relation.delivered_turn == Some(turn)
1337                            {
1338                                relation.delivered_turn = None;
1339                            }
1340                            tracing::warn!(%child_id, turn, error = %format!("{error:#}"), "could not deliver sub-agent completion");
1341                        }
1342                        Some(Err(error)) => tracing::warn!(%error, "sub-agent completion task panicked"),
1343                        None => {}
1344                    }
1345                }
1346                _ = prune_tick.tick() => {
1347                    // Only rows whose client id names a phone are considered:
1348                    // a terminal client's place in a conversation is not the
1349                    // phone's to expire.
1350                    tokio::spawn(async move {
1351                        let pruned = tokio::task::spawn_blocking(move || {
1352                            crate::database::prune_phone_client_state(client_state_retention)
1353                        })
1354                        .await;
1355                        match pruned {
1356                            Ok(Ok(0)) => {}
1357                            Ok(Ok(rows)) => tracing::debug!(rows, "pruned expired phone viewer state"),
1358                            Ok(Err(error)) => tracing::warn!(%error, "could not prune phone viewer state"),
1359                            Err(error) => tracing::warn!(%error, "phone viewer state pruning task failed"),
1360                        }
1361                    });
1362                }
1363                _ = credential_tick.tick() => {
1364                    schedule_due_credential_syncs(
1365                        &mut credential_sync_signals,
1366                        &credential_sync_handle,
1367                        Instant::now(),
1368                    );
1369                    while let Some(result) = credential_sync.try_result() {
1370                        crate::pollers::log_credential_sync_actions(&result);
1371                        let harness = controller
1372                            .config
1373                            .profiles
1374                            .get(&result.profile_id)
1375                            .map(|profile| profile.kind);
1376                        if let Some(notice) = credential_sync_notices.notice(&result, harness) {
1377                            eprintln!("Mjolnir: {notice}");
1378                        }
1379                    }
1380                }
1381                request = dictation_rx.recv() => {
1382                    let Some(request) = request else {
1383                        failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering dictation requests");
1384                        break;
1385                    };
1386                    let paths = controller.state.sessions.get(&request.session_id).map(|session| {
1387                        crate::dictation::auth_paths(&controller.config, &session.last_profile)
1388                    });
1389                    dictation_jobs.spawn(crate::dictation::execute(
1390                        request, paths, termination.clone(),
1391                    ));
1392                }
1393                job = dictation_jobs.join_next(), if !dictation_jobs.is_empty() => {
1394                    if let Some(Err(error)) = job {
1395                        tracing::warn!(%error, "web dictation task failed");
1396                    }
1397                }
1398                stored = client_state_rx.recv() => {
1399                    let Some(stored) = stored else {
1400                        failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering viewer state requests");
1401                        break;
1402                    };
1403                    // Every one of these touches SQLite, so each runs on its
1404                    // own task. A composer autosaving on a debounce must never
1405                    // be able to stall the loop that follows sessions.
1406                    let workspace_of = |session_id: &str| {
1407                        controller
1408                            .state
1409                            .sessions
1410                            .get(session_id)
1411                            .map(|session| session.workspace_id.clone())
1412                    };
1413                    let bundle_of = |session_id: &str| {
1414                        controller
1415                            .state
1416                            .sessions
1417                            .get(session_id)
1418                            .map(|session| session.bundle_id.clone())
1419                    };
1420                    match stored {
1421                        crate::server::ClientStateRequest::Read { client_id, session_id, reply } => {
1422                            let workspace = workspace_of(&session_id);
1423                            tokio::spawn(async move {
1424                                let answer = tokio::task::spawn_blocking(move || {
1425                                    let workspace = workspace.context("unknown session")?;
1426                                    let state = crate::database::client_session_state(
1427                                        &client_id, &workspace, &session_id,
1428                                    )?;
1429                                    anyhow::Ok(crate::server::ViewerClientState {
1430                                        draft: state.draft,
1431                                        through_event_ordinal: state.through_event_ordinal,
1432                                    })
1433                                })
1434                                .await;
1435                                reply.send(flatten_stored(answer)).ok();
1436                            });
1437                        }
1438                        crate::server::ClientStateRequest::SaveDraft { client_id, session_id, draft, reply } => {
1439                            let workspace = workspace_of(&session_id);
1440                            tokio::spawn(async move {
1441                                let answer = tokio::task::spawn_blocking(move || {
1442                                    let workspace = workspace.context("unknown session")?;
1443                                    crate::database::persist_client_draft(
1444                                        &client_id, &workspace, &session_id, &draft,
1445                                    )
1446                                })
1447                                .await;
1448                                reply.send(flatten_stored(answer)).ok();
1449                            });
1450                        }
1451                        crate::server::ClientStateRequest::MarkWorkspaceRead { client_id, workspace_id, reply } => {
1452                            let sessions = controller
1453                                .state
1454                                .sessions
1455                                .values()
1456                                .filter(|session| session.workspace_id == workspace_id)
1457                                .map(|session| (session.id.clone(), session.viewed_through_event_ordinal))
1458                                .collect::<Vec<_>>();
1459                            tokio::spawn(async move {
1460                                let answer = tokio::task::spawn_blocking(move || {
1461                                    for (session_id, through) in sessions {
1462                                        // A receipt that would move backwards
1463                                        // is not an error; it is a session this
1464                                        // viewer had already read past.
1465                                        crate::database::persist_read_receipt(
1466                                            &client_id, &workspace_id, &session_id, through,
1467                                        )
1468                                        .ok();
1469                                    }
1470                                    anyhow::Ok(())
1471                                })
1472                                .await;
1473                                reply.send(flatten_stored(answer)).ok();
1474                            });
1475                        }
1476                        crate::server::ClientStateRequest::History { session_id, query, scope, reply } => {
1477                            let bundle = bundle_of(&session_id);
1478                            tokio::spawn(async move {
1479                                let answer = tokio::task::spawn_blocking(move || {
1480                                    let bundle = bundle.context("unknown session")?;
1481                                    let scope = match scope.as_str() {
1482                                        "session" => crate::database::HistoryScope::Session,
1483                                        "all" => crate::database::HistoryScope::All,
1484                                        _ => crate::database::HistoryScope::Project,
1485                                    };
1486                                    let found = crate::database::search_prompts_bounded(
1487                                        &session_id,
1488                                        &bundle,
1489                                        scope,
1490                                        &query,
1491                                        crate::server::MAX_HISTORY_MATCHES,
1492                                    )?;
1493                                    anyhow::Ok(crate::server::ViewerPromptHistory {
1494                                        entries: found
1495                                            .entries
1496                                            .into_iter()
1497                                            .map(|entry| entry.text)
1498                                            .collect(),
1499                                        truncated: found.truncated,
1500                                    })
1501                                })
1502                                .await;
1503                                reply.send(flatten_stored(answer)).ok();
1504                            });
1505                        }
1506                    }
1507                }
1508                bundle = bundle_rx.recv(), if bundle_jobs.len() < MAX_CONCURRENT_BUNDLE_CREATIONS => {
1509                    let Some(crate::server::BundleRequest { source, reply }) = bundle else {
1510                        failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering bundle requests");
1511                        break;
1512                    };
1513                    // Repository canonicalization and config persistence both
1514                    // touch the filesystem. Keep them off this loop, and
1515                    // report a panic as a failed request rather than dropping
1516                    // the browser's reply.
1517                    let done = bundle_done_tx.clone();
1518                    let daemon_runtime = daemon_runtime.clone();
1519                    bundle_jobs.spawn(async move {
1520                        let result = daemon_runtime
1521                            .create_quick_bundle(source)
1522                            .await;
1523                        if let Err(error) = done.send(BundleCreated { result, reply }) {
1524                            tracing::debug!(%error, "bundle creation finished after the server stopped");
1525                        }
1526                    });
1527                }
1528                bundle_done = bundle_done_rx.recv() => {
1529                    let Some(BundleCreated { result, reply }) = bundle_done else {
1530                        failure = feed_stopped(termination.is_cancelled(), "the bundle creation pipeline stopped while the phone server was running");
1531                        break;
1532                    };
1533                    match result {
1534                        Ok(created) => {
1535                            let bundle_id = created.bundle_id;
1536                            // Other config sections may have changed while
1537                            // this request was in flight. Publish only the
1538                            // bundle this transaction created; a full fresh
1539                            // config is requested below and must not make a
1540                            // later completion hide another completed bundle.
1541                            let Some(bundle) = created.config.bundles.get(&bundle_id) else {
1542                                tracing::error!(%bundle_id, "bundle creation returned a config without its bundle");
1543                                if reply.send(Err(crate::server::BundleFailure::Controller)).is_err() {
1544                                    tracing::debug!("bundle creation failure reply dropped after client disconnect");
1545                                }
1546                                continue;
1547                            };
1548                            controller
1549                                .config
1550                                .bundles
1551                                .insert(bundle_id.clone(), bundle.clone());
1552                            // A reload started before this save may still be
1553                            // queued. It must not hide a bundle after we have
1554                            // acknowledged it as available to the browser.
1555                            controller_reload_invalidated |= controller_reload_in_flight;
1556                            revision = daemon_runtime.allocate_revision();
1557                            publish_snapshot!(revision);
1558                            request_daemon_controller_reload(
1559                                daemon_runtime.clone(),
1560                                "new bundle publication",
1561                            );
1562                            if reply.send(Ok(bundle_id)).is_err() {
1563                                tracing::debug!("bundle creation reply dropped after client disconnect");
1564                            }
1565                        }
1566                        Err(error) => {
1567                            let failure = match error {
1568                                crate::controller::QuickBundleFailure::InvalidSource(
1569                                    detail,
1570                                ) => {
1571                                    tracing::debug!(error = %detail, "phone bundle source was invalid");
1572                                    crate::server::BundleFailure::InvalidSource
1573                                }
1574                                crate::controller::QuickBundleFailure::Persistence(
1575                                    detail,
1576                                ) => {
1577                                    tracing::warn!(error = %detail, "phone bundle creation failed");
1578                                    crate::server::BundleFailure::Controller
1579                                }
1580                            };
1581                            if reply.send(Err(failure)).is_err() {
1582                                tracing::debug!("bundle creation failure reply dropped after client disconnect");
1583                            }
1584                        }
1585                    }
1586                }
1587                bundle_job = bundle_jobs.join_next(), if !bundle_jobs.is_empty() => {
1588                    if let Some(Err(error)) = bundle_job {
1589                        tracing::warn!(%error, "bundle creation task panicked");
1590                    }
1591                }
1592                preflight = preflight_rx.recv(), if preflight_jobs.len() < MAX_CONCURRENT_PREFLIGHTS => {
1593                    let Some(crate::server::PreflightRequest {
1594                        bundle_id,
1595                        target_id,
1596                        project_directory,
1597                        mut reply,
1598                        remote_repairs,
1599                    }) = preflight else {
1600                        failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering preflight requests");
1601                        break;
1602                    };
1603                    // Reading a working tree's status or validating a project
1604                    // directory touches the disk, so it runs on its own task
1605                    // rather than on the loop that has to stay responsive to
1606                    // every other feed.
1607                    let config = controller.config.clone();
1608                    let project_validation = project_directory.is_some();
1609                    let task_termination = termination.clone();
1610                    preflight_jobs.spawn(async move {
1611                        let cancelled = Arc::new(AtomicBool::new(false));
1612                        let cancellation_guard = ProcessCancellationGuard(cancelled.clone());
1613                        let mut blocking = tokio::task::spawn_blocking(move || {
1614                            run_new_preflight_with_cancellation(
1615                                config,
1616                                bundle_id,
1617                                target_id,
1618                                project_directory,
1619                                cancelled,
1620                                remote_repairs,
1621                            )
1622                        });
1623                        let answer = tokio::select! {
1624                            biased;
1625                            _ = task_termination.cancelled() => None,
1626                            _ = reply.closed() => None,
1627                            answer = &mut blocking => Some(answer),
1628                        };
1629                        let Some(answer) = answer else {
1630                            drop(cancellation_guard);
1631                            match blocking.await {
1632                                Err(error) => tracing::warn!(%error, "cancelled phone preflight task failed"),
1633                                Ok(Err(error)) => tracing::debug!(%error, "phone preflight cancelled"),
1634                                Ok(Ok(_)) => {}
1635                            }
1636                            return;
1637                        };
1638                        let answer = match answer {
1639                            Ok(Ok(answer)) => Ok(answer),
1640                            Ok(Err(error)) => {
1641                                tracing::debug!(
1642                                    error = %error,
1643                                    project_validation,
1644                                    "phone preflight check failed"
1645                                );
1646                                Err(if project_validation {
1647                                    PreflightFailure::Validation
1648                                } else {
1649                                    PreflightFailure::InvalidRepository(format!("{error:#}"))
1650                                })
1651                            }
1652                            Err(error) => {
1653                                tracing::warn!(%error, "phone preflight task failed");
1654                                Err(PreflightFailure::Controller(format!(
1655                                    "preflight task failed: {error}"
1656                                )))
1657                            }
1658                        };
1659                        if reply.send(answer).is_err() {
1660                            tracing::debug!("phone preflight reply dropped after client disconnect");
1661                        }
1662                    });
1663                }
1664                preflight_job = preflight_jobs.join_next(), if !preflight_jobs.is_empty() => {
1665                    if let Some(Err(error)) = preflight_job {
1666                        tracing::warn!(%error, "phone preflight task panicked");
1667                    }
1668                }
1669                preparation = move_preparation_rx.recv() => {
1670                    let Some(MovePreparationRequest { selection, reply }) = preparation else {
1671                        failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering move preparation requests");
1672                        break;
1673                    };
1674                    // Preparation can inspect archives, target prerequisites,
1675                    // and harness capabilities. Keep it supervised and away
1676                    // from this feed loop so another browser can still read
1677                    // snapshots while a move form is open.
1678                    let done = move_prepared_tx.clone();
1679                    let daemon_runtime = daemon_runtime.clone();
1680                    move_preparation_jobs.spawn(async move {
1681                        let result = daemon_runtime
1682                            .prepare_move_session(selection)
1683                            .await
1684                            .map_err(|error| format!("{error:#}"));
1685                        if let Err(error) = done.send(MovePrepared { result, reply }) {
1686                            tracing::debug!(%error, "move preparation finished after the server stopped");
1687                        }
1688                    });
1689                }
1690                prepared = move_prepared_rx.recv() => {
1691                    let Some(MovePrepared { result, reply }) = prepared else {
1692                        failure = feed_stopped(termination.is_cancelled(), "the move preparation pipeline stopped while the phone server was running");
1693                        break;
1694                    };
1695                    if reply.send(result).is_err() {
1696                        tracing::debug!("move preparation reply dropped after client disconnect");
1697                    }
1698                }
1699                move_preparation_job = move_preparation_jobs.join_next(), if !move_preparation_jobs.is_empty() => {
1700                    if let Some(Err(error)) = move_preparation_job {
1701                        tracing::warn!(%error, "move preparation task failed");
1702                    }
1703                }
1704                move_recovery_job = move_recovery_jobs.join_next(), if !move_recovery_jobs.is_empty() => {
1705                    if let Some(Err(error)) = move_recovery_job {
1706                        move_recovery_load_in_flight = false;
1707                        tracing::warn!(%error, "Move recovery projection task failed");
1708                    }
1709                }
1710                receipt = receipt_rx.recv() => {
1711                    let Some(ReadReceiptRequest { client_id, session_id, through, reply }) = receipt else {
1712                        failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering read receipts");
1713                        break;
1714                    };
1715                    match controller.state.sessions.get(&session_id) {
1716                        None => {
1717                            if reply.send(Err("unknown session".into())).is_err() {
1718                                tracing::debug!(%session_id, "unknown-session read receipt reply dropped after client disconnect");
1719                            }
1720                        }
1721                        Some(session) => {
1722                            let workspace_id = session.workspace_id.clone();
1723                            let done = receipt_done_tx.clone();
1724                            let persisted_session_id = session_id.clone();
1725                            tokio::spawn(async move {
1726                                let joined = tokio::task::spawn_blocking(move || {
1727                                    crate::database::persist_read_receipt(
1728                                        &client_id,
1729                                        &workspace_id,
1730                                        &persisted_session_id,
1731                                        through,
1732                                    )
1733                                })
1734                                .await;
1735                                let result = match joined {
1736                                    Ok(result) => result.map_err(|error| format!("{error:#}")),
1737                                    Err(error) => Err(format!("phone read receipt task failed: {error}")),
1738                                };
1739                                if let Err(error) = done.send(ReadReceiptPersisted { session_id, result, reply }) {
1740                                    tracing::debug!(%error, "phone read receipt finished after the server stopped");
1741                                }
1742                            });
1743                        }
1744                    }
1745                }
1746                persisted = receipt_done_rx.recv() => {
1747                    let Some(ReadReceiptPersisted { session_id, result, reply }) = persisted else { continue };
1748                    match result {
1749                        Ok(receipt) => {
1750                            let _ = receipt;
1751                            if reply.send(Ok(())).is_err() {
1752                                tracing::debug!(%session_id, "phone read receipt reply dropped after client disconnect");
1753                            }
1754                        }
1755                        Err(error) => {
1756                            tracing::warn!(%session_id, "could not persist a phone read receipt: {error}");
1757                            if reply.send(Err(error)).is_err() {
1758                                tracing::debug!(%session_id, "failed phone read receipt reply dropped after client disconnect");
1759                            }
1760                        }
1761                    }
1762                }
1763                action = action_rx.recv() => {
1764                    let Some(request) = action else {
1765                        failure = feed_stopped(termination.is_cancelled(), "the phone HTTP server stopped delivering actions");
1766                        break;
1767                    };
1768                    // A refresh nudges a poller this loop owns. It takes no
1769                    // session slot and starts no lifecycle work, so it is
1770                    // answered here rather than admitted as an action.
1771                    match &request.action {
1772                        ControllerAction::RefreshCapacity { target_id } => {
1773                            let known = capacity_state.contains_key(target_id);
1774                            // One queued nudge refreshes every target. Do not
1775                            // block the consumer while readings wait for it.
1776                            let accepted = known && match capacity_triggers_tx.try_send(()) {
1777                                Ok(()) | Err(tokio::sync::mpsc::error::TrySendError::Full(())) => true,
1778                                Err(tokio::sync::mpsc::error::TrySendError::Closed(())) => {
1779                                    tracing::warn!("phone capacity refresh rejected: poller stopped");
1780                                    false
1781                                }
1782                            };
1783                            if known {
1784                                if let Some(entry) = capacity_state.get_mut(target_id) {
1785                                    entry.refreshing = accepted;
1786                                    if !accepted {
1787                                        entry.failed = true;
1788                                    }
1789                                }
1790                                revision = daemon_runtime.allocate_revision();
1791                                publish_snapshot!(revision);
1792                            }
1793                            let outcome = if accepted {
1794                                ActionOutcome::accepted()
1795                            } else {
1796                                ActionOutcome::Failed
1797                            };
1798                            if request.reply.send(outcome).is_err() {
1799                                tracing::debug!(%target_id, "phone capacity refresh reply dropped after client disconnect");
1800                            }
1801                            tokio::task::yield_now().await;
1802                            continue;
1803                        }
1804                        ControllerAction::RefreshQuota { profile_id } => {
1805                            let known = controller.config.enabled_profile(profile_id).is_some();
1806                            if known {
1807                                // The refresher works from a generation-stamped
1808                                // batch, so a new generation is how one is asked
1809                                // for again rather than a per-profile trigger.
1810                                quota_batch.generation = quota_batch.generation.saturating_add(1);
1811                                quota_batch.profiles = quota_refresh_profiles(&controller);
1812                                quota_profiles_tx.send_replace(quota_batch.clone());
1813                            }
1814                            let outcome = if known {
1815                                ActionOutcome::accepted()
1816                            } else {
1817                                ActionOutcome::Failed
1818                            };
1819                            if request.reply.send(outcome).is_err() {
1820                                tracing::debug!(%profile_id, "phone quota refresh reply dropped after client disconnect");
1821                            }
1822                            tokio::task::yield_now().await;
1823                            continue;
1824                        }
1825                        _ => {}
1826                    }
1827                    if let ControllerAction::Cancel { session_id } = &request.action {
1828                        let outcome = if request_phone_action_cancellation(
1829                            session_id,
1830                            &action_sessions,
1831                            &action_cancellations,
1832                        ) {
1833                            daemon_runtime.cancel_lifecycle_if_active(session_id);
1834                            ActionOutcome::accepted()
1835                        } else {
1836                            ActionOutcome::NotCancellable
1837                        };
1838                        if request.reply.send(outcome).is_err() {
1839                            tracing::debug!(%session_id, "phone cancellation reply dropped after client disconnect");
1840                        }
1841                        tokio::task::yield_now().await;
1842                        continue;
1843                    }
1844                    if let ControllerAction::Close { session_id } = &request.action {
1845                        if closing_actions.contains_key(session_id) {
1846                            if request.reply.send(ActionOutcome::accepted()).is_err() { tracing::debug!(%session_id, "repeated close reply dropped"); }
1847                            continue;
1848                        }
1849                        request_phone_action_cancellation(session_id, &action_sessions, &action_cancellations);
1850                        daemon_runtime.request_close(session_id);
1851                    }
1852                    // A force close runs even while a graceful close for the
1853                    // same session is still in flight; that stuck close is
1854                    // exactly what it is meant to take over.
1855                    if let ControllerAction::ForceClose { session_id } = &request.action {
1856                        request_phone_action_cancellation(session_id, &action_sessions, &action_cancellations);
1857                        daemon_runtime.request_close(session_id);
1858                    }
1859                    let session_id = match admit_phone_action(
1860                        &request.action,
1861                        action_cancellations.len(),
1862                        &mut active_actions,
1863                    ) {
1864                        Ok(session_id) => session_id,
1865                        Err(refusal) => {
1866                            if request.reply.send(refusal).is_err() {
1867                                tracing::debug!("phone action refusal reply dropped after client disconnect");
1868                            }
1869                            tokio::task::yield_now().await;
1870                            continue;
1871                        }
1872                    };
1873                    let ControllerRequest { action, reply } = request;
1874                    let done = action_done_tx.clone();
1875                    let session_control = worker_commands_tx.clone();
1876                    let daemon_runtime = daemon_runtime.clone();
1877                    let started = action_started_tx.clone();
1878                    next_action_id = next_action_id.wrapping_add(1).max(1);
1879                    let action_id = next_action_id;
1880                    if let ControllerAction::Close { session_id } | ControllerAction::ForceClose { session_id } = &action { closing_actions.insert(session_id.clone(), action_id); }
1881                    if let ControllerAction::New { workspace_id, .. } = &action {
1882                        let workspace_id = if workspace_id.is_empty() && phone_workspaces.len() == 1 {
1883                            phone_workspaces[0].id.clone()
1884                        } else {
1885                            workspace_id.clone()
1886                        };
1887                        launch_workspaces.insert(action_id, workspace_id);
1888                    }
1889                    let control = PhoneActionControl::for_action(&action);
1890                    action_cancellations.insert(action_id, control.clone());
1891                    if let Some(session_id) = &session_id {
1892                        action_sessions.insert(action_id, session_id.clone());
1893                    }
1894                    action_replies.accept(action_id, &action, reply);
1895                    let runtime = tokio::runtime::Handle::current();
1896                    tokio::spawn(async move {
1897                        let joined = tokio::task::spawn_blocking(move || {
1898                            let result = (|| -> Result<()> {
1899                                if control.cancelled.load(Ordering::Acquire) {
1900                                    bail!("phone action cancelled");
1901                                }
1902                                let mut operation_controller = Controller::load()?;
1903                                let executor =
1904                                    CancellableProcessExecutor::new(control.cancelled.clone());
1905                                runtime.block_on(apply_phone_action(
1906                                    &mut operation_controller,
1907                                    PhoneActionServices {
1908                                        sessions: &session_control,
1909                                        daemon_runtime: &daemon_runtime,
1910                                    },
1911                                    action,
1912                                    &executor,
1913                                    action_id,
1914                                    &started,
1915                                    &control,
1916                                ))
1917                            })();
1918                            result.map_err(|error| format!("{error:#}"))
1919                        })
1920                        .await;
1921                        let result = match joined {
1922                            Ok(result) => result,
1923                            Err(error) => Err(format!("phone action task failed: {error}")),
1924                        };
1925                        if let Err(error) = done.send((action_id, session_id, result)) {
1926                            tracing::debug!(action_id, %error, "phone action finished after the server stopped");
1927                        }
1928                    });
1929                }
1930                started = action_started_rx.recv() => {
1931                    let Some(started) = started else {
1932                        tokio::task::yield_now().await;
1933                        continue;
1934                    };
1935                    let started_session_id = started.session.id.clone();
1936                    let publication = if !action_cancellations.contains_key(&started.action_id) {
1937                        Err("phone action completed before its provisional session was published".into())
1938                    } else {
1939                        track_started_phone_session(
1940                            &mut controller.state,
1941                            &mut active_actions,
1942                            &mut action_sessions,
1943                            started.action_id,
1944                            started.session,
1945                        )
1946                    };
1947                    if publication.is_ok() {
1948                        revision = daemon_runtime.allocate_revision();
1949                        publish_snapshot!(revision);
1950                        request_daemon_controller_reload(
1951                            daemon_runtime.clone(),
1952                            "new session publication",
1953                        );
1954                    };
1955                    if publication.is_err()
1956                        && let Some(control) = action_cancellations.get(&started.action_id)
1957                    {
1958                        control.request_cancel();
1959                    }
1960                    // The phone asked for a session, and now there is one to
1961                    // point at: that is what its request was waiting for.
1962                    action_replies.resolve(
1963                        started.action_id,
1964                        if publication.is_ok() {
1965                            ActionOutcome::Accepted {
1966                                session_id: Some(started_session_id),
1967                            }
1968                        } else {
1969                            ActionOutcome::Failed
1970                        },
1971                    );
1972                    if started.published.send(publication).is_err() {
1973                        tracing::debug!(action_id = started.action_id, "phone new-session publication reply dropped after client disconnect");
1974                    }
1975                }
1976                completed = action_done_rx.recv() => {
1977                    let Some((action_id, session_id, result)) = completed else {
1978                        failure = feed_stopped(termination.is_cancelled(), "the phone action pipeline stopped reporting completions");
1979                        break;
1980                    };
1981                    action_cancellations.remove(&action_id);
1982                    let session_id = action_sessions.remove(&action_id).or(session_id);
1983                    if closing_actions.values().any(|closing_id| *closing_id == action_id) && let Some(id) = &session_id { daemon_runtime.clear_close_request(id); }
1984                    closing_actions.retain(|_, closing_id| *closing_id != action_id);
1985                    if let Some(session_id) = &session_id && !action_sessions.values().any(|active| active == session_id) {
1986                        active_actions.remove(session_id);
1987                    }
1988                    // A `new` that failed before publishing a session never
1989                    // reached the arm that answers it, so its phone is still
1990                    // waiting for a reply it can act on.
1991                    action_replies.resolve(action_id, ActionOutcome::Failed);
1992                    if let Some(workspace_id) = launch_workspaces.remove(&action_id)
1993                        && result.is_err()
1994                        && !session_id.as_ref().is_some_and(|id| closing_actions.contains_key(id))
1995                    {
1996                        record_launch_failure(
1997                            &mut launch_failures,
1998                            action_id,
1999                            workspace_id,
2000                            session_id.clone(),
2001                        );
2002                        revision = daemon_runtime.allocate_revision();
2003                        publish_snapshot!(revision);
2004                    }
2005                    if let Err(error) = &result {
2006                        tracing::warn!(action_id, %error, "phone action failed");
2007                    }
2008                    record_action_result(
2009                        &mut pending_action_errors,
2010                        session_id.as_deref(),
2011                        &result,
2012                    );
2013                    request_controller_reload(
2014                        &mut controller_reload_in_flight,
2015                        &mut controller_reload_requested,
2016                        &controller_reload_tx,
2017                    );
2018                    request_move_recovery_reload(
2019                        &move_recovery_tx,
2020                        &mut move_recovery_load_in_flight,
2021                        &mut move_recovery_jobs,
2022                    );
2023                    request_daemon_controller_reload(
2024                        daemon_runtime.clone(),
2025                        "phone action completion",
2026                    );
2027                }
2028                reloaded = controller_reload_rx.recv() => {
2029                    let Some(ControllerReloaded { result }) = reloaded else {
2030                        failure = feed_stopped(
2031                            termination.is_cancelled(),
2032                            "the controller reload pipeline stopped while the phone server was running",
2033                        );
2034                        break;
2035                    };
2036                    controller_reload_in_flight = false;
2037                    if std::mem::take(&mut controller_reload_invalidated) {
2038                        if let Err(error) = &result {
2039                            tracing::warn!(%error, "superseded controller reload failed");
2040                        }
2041                        controller_reload_requested = false;
2042                        request_controller_reload(
2043                            &mut controller_reload_in_flight,
2044                            &mut controller_reload_requested,
2045                            &controller_reload_tx,
2046                        );
2047                        continue;
2048                    }
2049                    match result {
2050                        Ok(mut reloaded) => {
2051                            for (session_id, error) in &pending_action_errors {
2052                                if let Some(session) = reloaded.state.sessions.get_mut(session_id)
2053                                    && session.last_error.is_none()
2054                                {
2055                                    session.last_error = Some(error.clone());
2056                                }
2057                            }
2058                            controller = reloaded;
2059                            quotas.retain(|id, _| controller.config.enabled_profile(id).is_some());
2060                            subagent_quota_reports
2061                                .lock()
2062                                .expect("sub-agent quota reports lock poisoned")
2063                                .retain(|id, _| controller.config.enabled_profile(id).is_some());
2064                            worker_targets_tx.send_replace(dashboard_worker_targets(&controller));
2065                            publish_capacity_targets(
2066                                &controller,
2067                                &capacity_targets_tx,
2068                                &mut capacity_state,
2069                            );
2070                            credential_sync_handle.set_targets(credential_sync_targets(&controller));
2071                            republish_quota_profiles(
2072                                &controller,
2073                                &mut published_quota_profiles,
2074                                &mut quota_batch,
2075                                &quota_profiles_tx,
2076                            );
2077                            queued_prompts.retain(|session_id, _| {
2078                                controller.state.sessions.contains_key(session_id)
2079                            });
2080                            pending_elicitations.retain(|session_id, _| {
2081                                controller.state.sessions.contains_key(session_id)
2082                            });
2083                            prompt_images.retain(|session_id| {
2084                                controller.state.sessions.contains_key(session_id)
2085                            });
2086                            operational.retain(|session_id, _| {
2087                                controller.state.sessions.contains_key(session_id)
2088                            });
2089                            materialized_activity.retain(|session_id, _| {
2090                                controller.state.sessions.contains_key(session_id)
2091                            });
2092                            request_move_recovery_reload(
2093                                &move_recovery_tx,
2094                                &mut move_recovery_load_in_flight,
2095                                &mut move_recovery_jobs,
2096                            );
2097                            conversations.retain(|id, _| {
2098                                controller.state.sessions.get(id).is_some_and(|session| session.state.is_active())
2099                            });
2100                            for session_id in conversation_projections.session_ids() {
2101                                if !controller
2102                                    .state
2103                                    .sessions
2104                                    .get(&session_id)
2105                                    .is_some_and(|session| session.state.is_active())
2106                                {
2107                                    conversation_projections.forget(&session_id);
2108                                }
2109                            }
2110                            revision = daemon_runtime.allocate_revision();
2111                            conversation_tx.send_replace(conversations.clone());
2112                            publish_snapshot!(revision);
2113                        }
2114                        Err(error) => {
2115                            tracing::warn!(%error, "completed phone operation could not reload controller state");
2116                        }
2117                    }
2118                    if controller_reload_requested {
2119                        controller_reload_requested = false;
2120                        controller_reload_in_flight = true;
2121                        spawn_controller_reload(controller_reload_tx.clone());
2122                    }
2123                }
2124            }
2125        }
2126        // Stop provider requests before the HTTP request channels disappear.
2127        dictation_jobs.shutdown().await;
2128        // Bundle jobs are supervised so shutdown never leaves a detached
2129        // request task behind holding the config mutation lock.
2130        bundle_jobs.shutdown().await;
2131        // Preflight jobs own cancellation guards for their blocking Git
2132        // probes. Aborting them here signals those probes before the server's
2133        // request channels disappear.
2134        preflight_jobs.shutdown().await;
2135        // Preparation tasks may be inspecting an archive or probing a target;
2136        // abort and drain them before the HTTP server's channels disappear.
2137        move_preparation_jobs.shutdown().await;
2138        move_recovery_jobs.shutdown().await;
2139        // Every exit stops in-flight work, whether it was asked for or forced.
2140        crate::controller::profile_config::cancel_all();
2141        for control in action_cancellations.values() {
2142            control.request_cancel();
2143        }
2144        match failure {
2145            Some(failure) => Err(failure),
2146            None => Ok::<(), anyhow::Error>(()),
2147        }
2148    };
2149    let result = tokio::select! {
2150        result = api_activity::record_activity_stream(activity_snapshots) => result.context("native API activity recorder stopped"),
2151        result = serve => result,
2152        result = control => result,
2153    };
2154    conversation_projection_shutdown.cancel();
2155    renewal_cancellation.cancel();
2156    if let Some(task) = renewal_task
2157        && let Err(error) = task.await
2158    {
2159        tracing::warn!(%error, "Tailscale certificate renewal task failed");
2160    }
2161    worker_shutdown
2162        .shutdown()
2163        .await
2164        .context("shut down phone server session manager")?;
2165    result?;
2166    Ok(())
2167}
2168
2169/// Why the control loop is stopping because one of its feeds ended.
2170///
2171/// During shutdown every feed ends, and that is the plan. At any other time it
2172/// means the phone server has lost the machinery it exists to drive, so it
2173/// says which feed and exits non-zero instead of reporting success.
2174fn feed_stopped(shutting_down: bool, reason: &'static str) -> Option<anyhow::Error> {
2175    (!shutting_down).then(|| anyhow::anyhow!(reason))
2176}
2177
2178/// Whether a session's agent said it accepts image content in prompts. An
2179/// agent that has not answered `initialize` yet has advertised nothing, so the
2180/// phone is not offered controls the agent may refuse.
2181fn agent_accepts_prompt_images(operational: &mj_core::relay::RelayOperationalState) -> bool {
2182    operational
2183        .agent_capabilities
2184        .as_ref()
2185        .is_some_and(|capabilities| capabilities.prompt_capabilities.image)
2186}
2187
2188fn controller_action_session_id(action: &ControllerAction) -> Option<String> {
2189    match action {
2190        ControllerAction::New { .. } => None,
2191        ControllerAction::Prompt { session_id, .. }
2192        | ControllerAction::RunShell { session_id, .. }
2193        | ControllerAction::CancelShell { session_id, .. }
2194        | ControllerAction::Close { session_id }
2195        | ControllerAction::ForceClose { session_id }
2196        | ControllerAction::Resume { session_id, .. }
2197        | ControllerAction::Open { session_id }
2198        | ControllerAction::Cancel { session_id }
2199        | ControllerAction::RemoveQueuedPrompt { session_id, .. }
2200        | ControllerAction::RespondElicitation { session_id, .. }
2201        | ControllerAction::Rename { session_id, .. }
2202        | ControllerAction::CancelTurn { session_id }
2203        | ControllerAction::SetConfig { session_id, .. }
2204        | ControllerAction::SetPlanMode { session_id, .. }
2205        | ControllerAction::StartReview { session_id }
2206        | ControllerAction::ResolveReview { session_id, .. } => Some(session_id.clone()),
2207        ControllerAction::Move { request } => {
2208            Some(request.preparation.selection.session_id.clone())
2209        }
2210        // A refresh belongs to a profile or a target rather than a session, so
2211        // it takes no session slot and cannot be refused as session-busy.
2212        ControllerAction::RefreshQuota { .. } | ControllerAction::RefreshCapacity { .. } => None,
2213    }
2214}
2215
2216/// Flatten a joined blocking result into the answer a phone channel carries.
2217fn flatten_stored<T>(
2218    joined: std::result::Result<Result<T>, tokio::task::JoinError>,
2219) -> std::result::Result<T, String> {
2220    match joined {
2221        Ok(Ok(value)) => Ok(value),
2222        Ok(Err(error)) => Err(format!("{error:#}")),
2223        Err(error) => Err(format!("viewer state task failed: {error}")),
2224    }
2225}
2226
2227/// Cancel the shared subprocess executor when its request owner goes away.
2228struct ProcessCancellationGuard(Arc<AtomicBool>);
2229
2230impl Drop for ProcessCancellationGuard {
2231    fn drop(&mut self) {
2232        self.0.store(true, Ordering::Release);
2233    }
2234}
2235
2236/// Perform the preflight for a new session.
2237///
2238/// This runs on the blocking task owned by the phone server. Bare targets
2239/// need the same directory-and-Git-HEAD validation as the dashboard. Isolated
2240/// targets resolve every bundle repository to a network source and report the
2241/// exact fetch branch and publication destinations. Local working-tree
2242/// contents are never copied for this path.
2243#[cfg(test)]
2244fn run_new_preflight(
2245    config: Config,
2246    bundle_id: String,
2247    target_id: String,
2248    project_directory: Option<PathBuf>,
2249) -> Result<crate::server::PreflightNew> {
2250    run_new_preflight_with_cancellation(
2251        config,
2252        bundle_id,
2253        target_id,
2254        project_directory,
2255        Arc::new(AtomicBool::new(false)),
2256        Vec::new(),
2257    )
2258}
2259
2260fn run_new_preflight_with_cancellation(
2261    config: Config,
2262    bundle_id: String,
2263    target_id: String,
2264    project_directory: Option<PathBuf>,
2265    cancelled: Arc<AtomicBool>,
2266    remote_repairs: Vec<mj_core::local_git::LocalRemoteRepair>,
2267) -> Result<crate::server::PreflightNew> {
2268    let executor =
2269        CancellableProcessExecutor::new(cancelled).with_deadline(Duration::from_secs(30));
2270    if !remote_repairs.is_empty() {
2271        let target = config.targets.get(&target_id).context("unknown target")?;
2272        anyhow::ensure!(
2273            !is_bare_project_target(target) && project_directory.is_none(),
2274            "remote repair requires an isolated target"
2275        );
2276        let bundle = config.bundles.get(&bundle_id).context("unknown bundle")?;
2277        mj_core::local_git::apply_repository_remote_repairs(bundle, &remote_repairs, &executor)?;
2278    }
2279    run_new_preflight_with_executor(config, bundle_id, target_id, project_directory, &executor)
2280}
2281
2282fn run_new_preflight_with_executor(
2283    config: Config,
2284    bundle_id: String,
2285    target_id: String,
2286    project_directory: Option<PathBuf>,
2287    executor: &impl CommandExecutor,
2288) -> Result<crate::server::PreflightNew> {
2289    let target_is_bare = config
2290        .targets
2291        .get(&target_id)
2292        .with_context(|| format!("unknown target template {target_id:?}"))
2293        .map(is_bare_project_target)?;
2294    if target_is_bare {
2295        let directory =
2296            project_directory.context("project directory is required for a bare target")?;
2297        let controller = config_only_controller(config);
2298        let directory = controller.resolve_project_directory(&target_id, &directory, executor)?;
2299        let managed_worktree =
2300            controller.managed_worktree_options(&target_id, &directory, executor)?;
2301        return Ok(crate::server::PreflightNew {
2302            managed_worktree,
2303            project_directory: Some(directory),
2304            remote_repairs: Vec::new(),
2305            dirty_repositories: Vec::new(),
2306            remote_repositories: Vec::new(),
2307            local_changes_excluded: false,
2308        });
2309    }
2310    if project_directory.is_some() {
2311        bail!("project directory is unsupported for this target");
2312    }
2313
2314    let bundle = config.bundles.get(&bundle_id).context("unknown bundle")?;
2315    let repairs = mj_core::local_git::repository_remote_repairs(bundle, executor)?;
2316    if !repairs.is_empty() {
2317        return Ok(crate::server::PreflightNew {
2318            managed_worktree: Default::default(),
2319            project_directory: None,
2320            remote_repairs: repairs,
2321            dirty_repositories: Vec::new(),
2322            remote_repositories: Vec::new(),
2323            local_changes_excluded: true,
2324        });
2325    }
2326    let remote_repositories = bundle
2327        .repositories
2328        .iter()
2329        .map(|repository| {
2330            let source = resolve_repository(repository, executor)
2331                .with_context(|| format!("repository {:?}", repository.id))?;
2332            let default_branch = default_branch(&source, executor)
2333                .with_context(|| format!("repository {:?}", repository.id))?;
2334            Ok(crate::server::PreflightRepository {
2335                id: repository.id.clone(),
2336                fetch_url: display_url(&source.fetch_url),
2337                default_branch,
2338                push_urls: source
2339                    .push_urls
2340                    .iter()
2341                    .map(|url| display_url(url))
2342                    .collect(),
2343            })
2344        })
2345        .collect::<Result<Vec<_>>>()?;
2346    Ok(crate::server::PreflightNew {
2347        managed_worktree: Default::default(),
2348        project_directory: None,
2349        remote_repairs: Vec::new(),
2350        dirty_repositories: Vec::new(),
2351        remote_repositories,
2352        local_changes_excluded: true,
2353    })
2354}
2355
2356fn phone_action_capacity_available(active_actions: usize) -> bool {
2357    active_actions < MAX_CONCURRENT_PHONE_ACTIONS
2358}
2359
2360/// Point the quota refresher at the profiles the configuration currently
2361/// defines, alongside the worker-poll and credential-sync targets that are
2362/// rebuilt from the same reload. A profile added to `config.toml` while the
2363/// server runs otherwise reaches the snapshot but never the refresher, and
2364/// reads "quota unavailable" until the next restart.
2365///
2366/// Sending a batch restarts every profile's refresh, which spawns a harness
2367/// process per profile, so the batch travels only when the profiles changed.
2368/// Reports whether it did.
2369fn republish_quota_profiles(
2370    controller: &Controller,
2371    published: &mut std::collections::BTreeMap<String, HarnessProfile>,
2372    batch: &mut QuotaRefreshBatch,
2373    profiles_tx: &tokio::sync::watch::Sender<QuotaRefreshBatch>,
2374) -> bool {
2375    if *published == controller.config.profiles {
2376        return false;
2377    }
2378    published.clone_from(&controller.config.profiles);
2379    batch.generation = batch.generation.saturating_add(1);
2380    batch.profiles = quota_refresh_profiles(controller);
2381    profiles_tx.send_replace(batch.clone());
2382    true
2383}
2384
2385fn request_phone_action_cancellation(
2386    session_id: &str,
2387    action_sessions: &std::collections::BTreeMap<u64, String>,
2388    action_cancellations: &std::collections::BTreeMap<u64, PhoneActionControl>,
2389) -> bool {
2390    let control = action_sessions
2391        .iter()
2392        .find_map(|(action_id, active_session_id)| {
2393            (active_session_id == session_id)
2394                .then(|| action_cancellations.get(action_id))
2395                .flatten()
2396        });
2397    if let Some(control) = control {
2398        return control.request_cancel();
2399    }
2400    false
2401}
2402
2403fn track_started_phone_session(
2404    state: &mut State,
2405    active_actions: &mut std::collections::BTreeSet<String>,
2406    action_sessions: &mut std::collections::BTreeMap<u64, String>,
2407    action_id: u64,
2408    session: SessionRecord,
2409) -> std::result::Result<(), String> {
2410    let session_id = session.id.clone();
2411    if !active_actions.insert(session_id.clone()) {
2412        return Err("another operation is already running for the new session".into());
2413    }
2414    action_sessions.insert(action_id, session_id.clone());
2415    state.sessions.insert(session_id, session);
2416    Ok(())
2417}
2418
2419/// Carry a finished action's failure into the session projection, and clear it
2420/// once a later action for the same session succeeds.
2421///
2422/// Nothing is waiting on the request any more, so a failure the action itself
2423/// did not record would reach no one but this process's stderr; the overlay
2424/// keeps it visible through every later durable reload, where the snapshot's
2425/// `has_error` takes it to the phone. Clearing on success matters just as
2426/// much: the overlay has no other expiry, so one transient failure would
2427/// otherwise badge the session as errored for the daemon's whole lifetime.
2428fn record_action_result(
2429    pending_action_errors: &mut std::collections::BTreeMap<String, String>,
2430    session_id: Option<&str>,
2431    result: &std::result::Result<(), String>,
2432) {
2433    let Some(session_id) = session_id else {
2434        return;
2435    };
2436    match result {
2437        Err(error) => {
2438            pending_action_errors.insert(session_id.to_owned(), error.clone());
2439        }
2440        Ok(()) => {
2441            pending_action_errors.remove(session_id);
2442        }
2443    }
2444}
2445
2446/// Retain safe notices even if provisioning removed its provisional session.
2447/// This bounded, daemon-lifetime history survives durable controller reloads.
2448fn record_launch_failure(
2449    failures: &mut Vec<crate::server::ViewerLaunchFailure>,
2450    action_id: u64,
2451    workspace_id: String,
2452    session_id: Option<String>,
2453) {
2454    failures.push(crate::server::ViewerLaunchFailure {
2455        id: format!("{}-{action_id}", std::process::id()),
2456        workspace_id,
2457        session_id,
2458    });
2459    if failures.len() > 16 {
2460        failures.remove(0);
2461    }
2462}
2463
2464struct PhoneActionServices<'a> {
2465    sessions: &'a SessionManagerControl,
2466    daemon_runtime: &'a Arc<RuntimeState>,
2467}
2468
2469async fn apply_phone_action(
2470    controller: &mut Controller,
2471    services: PhoneActionServices<'_>,
2472    action: ControllerAction,
2473    _executor: &(impl CommandExecutor + Sync),
2474    action_id: u64,
2475    started: &tokio::sync::mpsc::UnboundedSender<PhoneActionStarted>,
2476    control: &PhoneActionControl,
2477) -> Result<()> {
2478    match action {
2479        ControllerAction::New {
2480            workspace_id,
2481            profile_id,
2482            bundle_id,
2483            target_id,
2484            title,
2485            project_directory,
2486            create_managed_worktree,
2487            mjolnir_subagents,
2488            dirty_ack: _dirty_ack,
2489        } => {
2490            let workspace_id = if workspace_id.is_empty() {
2491                let workspaces = crate::database::list_workspaces()?;
2492                match workspaces.as_slice() {
2493                    [workspace] => workspace.id.clone(),
2494                    [] => bail!("create a workspace before starting a phone session"),
2495                    _ => bail!("phone session creation requires a workspace_id"),
2496                }
2497            } else {
2498                workspace_id
2499            };
2500            // A phone that supplies no title gets the one the terminal would
2501            // have derived, so a session started from either surface reads the
2502            // same way in both.
2503            let title = title.unwrap_or_else(|| {
2504                let project = project_directory
2505                    .as_ref()
2506                    .and_then(|path| path.file_name())
2507                    .map(|name| name.to_string_lossy().into_owned())
2508                    .unwrap_or_else(|| bundle_id.clone());
2509                format!("{project} via {profile_id}")
2510            });
2511            let session_title_override = Some(title.clone());
2512            // Isolated creation always starts from the network default branch;
2513            // the legacy field remains accepted on the wire for old phones but
2514            // cannot opt local commits or dirty files into a new session.
2515            let allow_dirty_local = false;
2516            let (published, publication) = tokio::sync::oneshot::channel();
2517            let registered = services
2518                .daemon_runtime
2519                .start_create_session_controlled(
2520                    CreateSessionRequest {
2521                        create_managed_worktree,
2522                        mjolnir_subagents,
2523                        initial_prompt: None,
2524                        workspace_id,
2525                        profile_id,
2526                        bundle_id,
2527                        project_directory,
2528                        target_template_id: target_id,
2529                        additional_mounts: Vec::new(),
2530                        allow_dirty_local,
2531                        resource_allocation: None,
2532                        title,
2533                        session_title_override,
2534                    },
2535                    control
2536                        .create
2537                        .clone()
2538                        .expect("New action has a daemon create control"),
2539                    publication,
2540                )
2541                .await?;
2542            let registered_session_id = registered.session.id.clone();
2543            started
2544                .send(PhoneActionStarted {
2545                    action_id,
2546                    session: registered.session,
2547                    published,
2548                })
2549                .map_err(|_| anyhow::anyhow!("phone server stopped before publishing session"))?;
2550            services
2551                .daemon_runtime
2552                .wait_create_session(&registered_session_id)
2553                .await
2554        }
2555        ControllerAction::Prompt {
2556            session_id,
2557            text,
2558            images,
2559        } => {
2560            services
2561                .sessions
2562                .wait_for_session(&session_id, Duration::from_secs(5))
2563                .await?
2564                .submit(
2565                    new_command_id("phone-prompt")?,
2566                    RelayCommand::Prompt {
2567                        prompt: phone_prompt_blocks(text, images),
2568                    },
2569                )
2570                .await?;
2571            Ok(())
2572        }
2573        ControllerAction::RunShell {
2574            session_id,
2575            command,
2576        } => {
2577            services
2578                .sessions
2579                .wait_for_session(&session_id, Duration::from_secs(5))
2580                .await?
2581                .submit(
2582                    new_command_id("phone-shell")?,
2583                    RelayCommand::RunUserShell { command },
2584                )
2585                .await?;
2586            Ok(())
2587        }
2588        ControllerAction::CancelShell {
2589            session_id,
2590            shell_command_id,
2591        } => {
2592            services
2593                .sessions
2594                .wait_for_session(&session_id, Duration::from_secs(5))
2595                .await?
2596                .submit(
2597                    new_command_id("phone-cancel-shell")?,
2598                    RelayCommand::CancelUserShell { shell_command_id },
2599                )
2600                .await?;
2601            Ok(())
2602        }
2603        ControllerAction::Close { session_id } => {
2604            services.daemon_runtime.close_session(session_id).await
2605        }
2606        ControllerAction::ForceClose { session_id } => {
2607            services
2608                .daemon_runtime
2609                .force_destroy_session(session_id)
2610                .await
2611        }
2612        ControllerAction::Resume {
2613            session_id,
2614            workspace_id,
2615            profile_id,
2616            target_id,
2617            queue,
2618            additional_mounts,
2619            resource_allocation,
2620        } => services
2621            .daemon_runtime
2622            .resume_session(ResumeSessionRequest {
2623                session_id,
2624                workspace_id,
2625                profile_id,
2626                target_template_id: target_id,
2627                additional_mounts,
2628                resource_allocation,
2629                discard_queue: queue == ResumeQueueDisposition::Discard,
2630                repository_preflight: None,
2631            })
2632            .await
2633            .map(|_| ()),
2634        ControllerAction::Move { request } => {
2635            let outcome = services.daemon_runtime.move_session(request).await?;
2636            match outcome.outcome.as_str() {
2637                "completed" | "unchanged" => Ok(()),
2638                "cancelled" | "failed" => {
2639                    // The daemon keeps detailed diagnostics in its durable
2640                    // operation record. Only its safe recovery guidance is
2641                    // copied into the phone action error, where the normal
2642                    // failed-action path records a visible session error.
2643                    let recovery = outcome.recovery.unwrap_or_else(|| {
2644                        "Inspect the session and retry Move or Resume with previous settings."
2645                            .into()
2646                    });
2647                    bail!("Move {}: {recovery}", outcome.outcome)
2648                }
2649                status => bail!("Move returned an unknown outcome: {status}"),
2650            }
2651        }
2652        ControllerAction::Open { .. } => Ok(()),
2653        ControllerAction::Cancel { .. } => {
2654            bail!("cancel actions must be handled by the phone control loop")
2655        }
2656        ControllerAction::RemoveQueuedPrompt {
2657            session_id,
2658            queue_id,
2659        } => {
2660            services
2661                .sessions
2662                .session(&session_id)
2663                .await?
2664                .submit(
2665                    new_command_id("phone-remove-prompt")?,
2666                    RelayCommand::RemoveQueuedPrompt {
2667                        queued_command_id: queue_id,
2668                    },
2669                )
2670                .await?;
2671            Ok(())
2672        }
2673        ControllerAction::RespondElicitation {
2674            session_id,
2675            elicitation_id,
2676            response,
2677        } => {
2678            services
2679                .sessions
2680                .session(&session_id)
2681                .await?
2682                .respond_elicitation(elicitation_id, response)
2683                .await
2684        }
2685        ControllerAction::Rename { session_id, title } => {
2686            controller.rename_session(&session_id, &title)?;
2687            Ok(())
2688        }
2689        ControllerAction::StartReview { session_id } => {
2690            // The refusal is a sentence for the person holding the phone --
2691            // "prompts are queued", "set [review] profile in config.toml" --
2692            // so it travels as the error text of this action.
2693            services
2694                .daemon_runtime
2695                .review_host()
2696                .start(&session_id, true)
2697                .await
2698                .map_err(|refusal| anyhow::anyhow!("{refusal}"))?;
2699            Ok(())
2700        }
2701        ControllerAction::ResolveReview {
2702            session_id,
2703            resolution,
2704        } => {
2705            let resolution = crate::server::resolution_from_name(&resolution)
2706                .context("a review is resolved by forward, dismiss, or cancel")?;
2707            services
2708                .daemon_runtime
2709                .review_host()
2710                .resolve(&session_id, resolution)
2711                .await
2712                .map_err(|error| anyhow::anyhow!("{error}"))?;
2713            Ok(())
2714        }
2715        ControllerAction::CancelTurn { session_id } => {
2716            services
2717                .sessions
2718                .session(&session_id)
2719                .await?
2720                .submit(new_command_id("phone-cancel-turn")?, RelayCommand::Cancel)
2721                .await?;
2722            Ok(())
2723        }
2724        ControllerAction::SetConfig {
2725            session_id,
2726            key,
2727            value,
2728        } => {
2729            services
2730                .sessions
2731                .session(&session_id)
2732                .await?
2733                .submit(
2734                    new_command_id("phone-set-config")?,
2735                    RelayCommand::SetConfig { key, value },
2736                )
2737                .await?;
2738            Ok(())
2739        }
2740        ControllerAction::SetPlanMode { session_id, active } => {
2741            // Which call turns plan mode on is a fact about the harness, so it
2742            // is asked of the shared decision rather than decided here or, far
2743            // worse, in the browser.
2744            let harness_kind = controller
2745                .state
2746                .sessions
2747                .get(&session_id)
2748                .with_context(|| format!("unknown session {session_id}"))?
2749                .harness_kind;
2750            let handle = services.sessions.session(&session_id).await?;
2751            let operational = handle
2752                .view()
2753                .snapshot
2754                .map(|snapshot| snapshot.operational)
2755                .context("the session has not reported what it supports yet")?;
2756            let facts = mj_core::acp::AcpSessionFacts::from_operational(
2757                harness_kind,
2758                &operational.config,
2759                &operational.config_options,
2760                operational.modes.as_ref(),
2761            );
2762            let command = match facts.plan_control(active) {
2763                Ok(mj_core::acp::PlanControl::SetConfig { key, value }) => {
2764                    RelayCommand::SetConfig { key, value }
2765                }
2766                Ok(mj_core::acp::PlanControl::SetSessionMode { mode_id }) => {
2767                    RelayCommand::SetSessionMode { mode_id }
2768                }
2769                Err(reason) => bail!("{reason}"),
2770            };
2771            handle
2772                .submit(new_command_id("phone-plan-mode")?, command)
2773                .await?;
2774            Ok(())
2775        }
2776        // Refreshes are handled by the phone control loop, which owns the
2777        // pollers they nudge.
2778        ControllerAction::RefreshQuota { .. } | ControllerAction::RefreshCapacity { .. } => {
2779            bail!("refresh actions must be handled by the phone control loop")
2780        }
2781    }
2782}
2783
2784/// Inputs that can change a session's source without changing its ID.
2785#[derive(Clone, PartialEq, Eq)]
2786struct ProjectSourceKey {
2787    directory: Option<PathBuf>,
2788    worktree: Option<mj_core::state::ManagedWorktree>,
2789    target: Option<mj_core::config::TargetTemplate>,
2790    fallback: ProjectSourceIdentity,
2791}
2792
2793impl ProjectSourceKey {
2794    fn of(session: &SessionRecord, config: &Config) -> Self {
2795        Self {
2796            directory: session.project_directory.clone(),
2797            worktree: session.managed_worktree.clone(),
2798            target: config.targets.get(&session.target_template_id).cloned(),
2799            fallback: session.project_source(config),
2800        }
2801    }
2802}
2803
2804struct ProjectSourceEntry {
2805    key: ProjectSourceKey,
2806    source: Option<ProjectSourceIdentity>,
2807    retry_at: Option<Instant>,
2808    cancelled: Arc<AtomicBool>,
2809}
2810
2811struct ProjectSourceResolved {
2812    cancelled: Arc<AtomicBool>,
2813    session_id: String,
2814    key: ProjectSourceKey,
2815    result: Result<ProjectSourceIdentity, String>,
2816}
2817
2818/// Git/SSH probes run independently of snapshot publication and are bounded
2819/// and cancelled when their inputs disappear or the server shuts down.
2820#[derive(Default)]
2821struct PhoneProjectSources {
2822    entries: std::collections::BTreeMap<String, ProjectSourceEntry>,
2823    jobs: tokio::task::JoinSet<ProjectSourceResolved>,
2824}
2825
2826impl PhoneProjectSources {
2827    fn synchronize(&mut self, controller: &Controller) {
2828        self.entries.retain(|id, entry| {
2829            let keep = controller.state.sessions.get(id).is_some_and(|session| {
2830                session.project_directory.is_some()
2831                    && entry.key == ProjectSourceKey::of(session, &controller.config)
2832            });
2833            if !keep {
2834                entry.cancelled.store(true, Ordering::Release);
2835            }
2836            keep
2837        });
2838        for session in controller.state.sessions.values() {
2839            if self.jobs.len() >= 8 {
2840                break;
2841            }
2842            if session.project_directory.is_none()
2843                || self.entries.get(&session.id).is_some_and(|entry| {
2844                    entry
2845                        .retry_at
2846                        .is_none_or(|deadline| Instant::now() < deadline)
2847                })
2848            {
2849                continue;
2850            }
2851            let key = ProjectSourceKey::of(session, &controller.config);
2852            let cancelled = Arc::new(AtomicBool::new(false));
2853            self.entries.insert(
2854                session.id.clone(),
2855                ProjectSourceEntry {
2856                    key: key.clone(),
2857                    source: None,
2858                    retry_at: None,
2859                    cancelled: cancelled.clone(),
2860                },
2861            );
2862            let source_controller = Controller {
2863                config: controller.config.clone(),
2864                state: State {
2865                    sessions: [(session.id.clone(), session.clone())]
2866                        .into_iter()
2867                        .collect(),
2868                    ..State::default()
2869                },
2870            };
2871            let session_id = session.id.clone();
2872            self.jobs.spawn_blocking(move || {
2873                let executor = CancellableProcessExecutor::new(cancelled.clone())
2874                    .with_deadline(Duration::from_secs(8));
2875                let result = source_controller
2876                    .resolve_session_project_source(&session_id, &executor)
2877                    .map_err(|error| format!("{error:#}"));
2878                ProjectSourceResolved {
2879                    cancelled,
2880                    session_id,
2881                    key,
2882                    result,
2883                }
2884            });
2885        }
2886    }
2887
2888    fn complete(&mut self, resolved: ProjectSourceResolved) {
2889        let Some(entry) = self.entries.get_mut(&resolved.session_id) else {
2890            return;
2891        };
2892        if entry.key != resolved.key || !Arc::ptr_eq(&entry.cancelled, &resolved.cancelled) {
2893            return;
2894        }
2895        match resolved.result {
2896            Ok(source) => entry.source = Some(source),
2897            Err(error) => {
2898                tracing::warn!(session_id = %resolved.session_id, %error, "could not resolve web project source");
2899                entry.retry_at = Some(Instant::now() + Duration::from_secs(30));
2900            }
2901        }
2902    }
2903
2904    fn source(&self, session: &SessionRecord, config: &Config) -> Option<&ProjectSourceIdentity> {
2905        self.entries
2906            .get(&session.id)
2907            .filter(|entry| entry.key == ProjectSourceKey::of(session, config))
2908            .and_then(|entry| entry.source.as_ref())
2909    }
2910}
2911
2912impl Drop for PhoneProjectSources {
2913    fn drop(&mut self) {
2914        for entry in self.entries.values() {
2915            entry.cancelled.store(true, Ordering::Release);
2916        }
2917    }
2918}
2919
2920/// The live, per-session projections the phone snapshot layers on top of the
2921/// controller's durable state. They arrive from relay snapshots rather than
2922/// from disk, so they travel together instead of as separate arguments.
2923struct PhoneSessionViews<'a> {
2924    conversations: &'a std::collections::BTreeMap<String, crate::server::BrowserTranscript>,
2925    queued_prompts: &'a std::collections::BTreeMap<String, Vec<mj_core::relay::QueuedPrompt>>,
2926    active_user_shells:
2927        &'a std::collections::BTreeMap<String, Vec<mj_core::relay::ActiveUserShell>>,
2928    pending_elicitations:
2929        &'a std::collections::BTreeMap<String, Vec<mj_core::elicitation::ElicitationRequest>>,
2930    /// Sessions whose agent advertised image support in prompts.
2931    prompt_images: &'a std::collections::BTreeSet<String>,
2932    /// What each managed session's relay last reported. This is where the
2933    /// projection learns what the agent can do, rather than guessing from the
2934    /// durable record, which knows only what was configured.
2935    operational: &'a std::collections::BTreeMap<String, mj_core::relay::RelayOperationalState>,
2936    /// Durable activity watermarks delivered with the materialized worker
2937    /// snapshots. Keeping this in the control-loop cache avoids a database
2938    /// read while rendering each viewer snapshot.
2939    materialized_activity: &'a std::collections::BTreeMap<String, Option<i64>>,
2940    project_sources: &'a PhoneProjectSources,
2941    /// Lifecycle operations running now, keyed by session.
2942    operations: &'a std::collections::BTreeMap<String, crate::server::ViewerOperation>,
2943    /// Durable Move records, projected without diagnostics or checkpoint paths.
2944    move_recoveries: &'a std::collections::BTreeMap<String, ViewerMoveRecovery>,
2945    /// The most recent capacity reading per probe target.
2946    capacity: &'a [crate::server::ViewerTargetCapacity],
2947    launch_failures: &'a [crate::server::ViewerLaunchFailure],
2948    /// Reviews the daemon is running, keyed by session. The phone renders the
2949    /// same review the terminal does, from the same host.
2950    reviews: &'a std::collections::BTreeMap<String, crate::review_host::RuntimeReviewView>,
2951}
2952
2953/// What the phone server remembers about one probe target between readings.
2954///
2955/// The last good reading is kept beside any failure, because one failed probe
2956/// is not a reason to forget what a machine was doing a minute ago; the phone
2957/// is told both, and says so.
2958#[derive(Debug, Clone)]
2959struct PhoneCapacity {
2960    target: crate::targets::DeploymentCapacityTarget,
2961    usage: Option<crate::targets::DeploymentCapacityUsage>,
2962    on_demand: bool,
2963    sampled_at_epoch_seconds: Option<u64>,
2964    refreshing: bool,
2965    failed: bool,
2966}
2967
2968/// How old a reading may be before the page says so.
2969const CAPACITY_STALE_AFTER: Duration = Duration::from_secs(120);
2970
2971/// Tell the poller which targets to probe, and keep the state map in step.
2972fn publish_capacity_targets(
2973    controller: &Controller,
2974    targets_tx: &tokio::sync::watch::Sender<Vec<crate::targets::DeploymentCapacityTarget>>,
2975    state: &mut std::collections::BTreeMap<String, PhoneCapacity>,
2976) {
2977    let targets = controller.deployment_capacity_targets();
2978    state.retain(|id, _| targets.iter().any(|target| target.id == *id));
2979    for target in &targets {
2980        state
2981            .entry(target.id.clone())
2982            .and_modify(|entry| entry.target = target.clone())
2983            .or_insert_with(|| PhoneCapacity {
2984                target: target.clone(),
2985                usage: None,
2986                on_demand: false,
2987                sampled_at_epoch_seconds: None,
2988                // A target with no reading yet is loading, not idle.
2989                refreshing: true,
2990                failed: false,
2991            });
2992    }
2993    if targets_tx.borrow().as_slice() != targets.as_slice() {
2994        targets_tx.send_replace(targets);
2995    }
2996}
2997
2998/// Project the capacity readings for the phone.
2999fn viewer_capacity(
3000    state: &std::collections::BTreeMap<String, PhoneCapacity>,
3001) -> Vec<crate::server::ViewerTargetCapacity> {
3002    let now = std::time::SystemTime::now()
3003        .duration_since(std::time::UNIX_EPOCH)
3004        .unwrap_or_default()
3005        .as_secs();
3006    state
3007        .values()
3008        .map(|entry| {
3009            let usage = entry.usage.as_ref();
3010            crate::server::ViewerTargetCapacity {
3011                id: entry.target.id.clone(),
3012                label: entry.target.host.clone(),
3013                target_ids: entry.target.target_ids.clone(),
3014                cpu_percent: usage.and_then(|usage| usage.cpu_percent),
3015                memory_used_bytes: usage.map(|usage| usage.memory_used_bytes),
3016                memory_total_bytes: usage.map(|usage| usage.memory_total_bytes),
3017                logical_cores: usage.map(|usage| usage.logical_cores),
3018                disk_total_bytes: usage.and_then(|usage| usage.disk_total_bytes),
3019                // A fleet reports how many machines it is running; a plain host
3020                // has no such count and says nothing rather than zero.
3021                virtual_machines: matches!(
3022                    entry.target.kind,
3023                    crate::targets::DeploymentCapacityKind::AwsFleet
3024                )
3025                .then(|| u64::from(!entry.on_demand)),
3026                sampled_at_epoch_seconds: entry.sampled_at_epoch_seconds,
3027                refreshing: entry.refreshing,
3028                stale: entry.sampled_at_epoch_seconds.is_some_and(|sampled| {
3029                    now.saturating_sub(sampled) > CAPACITY_STALE_AFTER.as_secs()
3030                }),
3031                has_error: entry.failed,
3032            }
3033        })
3034        .collect()
3035}
3036
3037/// Turn one lifecycle operation into the projection a phone follows.
3038fn viewer_operation(view: &crate::daemon::RuntimeLifecycleView) -> crate::server::ViewerOperation {
3039    use crate::server::{ViewerOperationKind, ViewerOperationStage};
3040
3041    crate::server::ViewerOperation {
3042        // RuntimeLifecycle owns the identity. A session can have consecutive
3043        // operations, and a client must be able to retire a late response from
3044        // the previous one without mistaking it for the current operation.
3045        id: view.operation_id.clone(),
3046        session_id: view.session_id.clone(),
3047        kind: match view.kind {
3048            crate::daemon::RuntimeLifecycleKind::Create => ViewerOperationKind::Create,
3049            crate::daemon::RuntimeLifecycleKind::Resume => ViewerOperationKind::Resume,
3050            crate::daemon::RuntimeLifecycleKind::Move => ViewerOperationKind::Move,
3051            // Stop, destroy, and retained cleanup remain distinct so a phone
3052            // can describe which part of teardown owns the session.
3053            crate::daemon::RuntimeLifecycleKind::Close
3054            | crate::daemon::RuntimeLifecycleKind::ForceStop => ViewerOperationKind::Stop,
3055            crate::daemon::RuntimeLifecycleKind::DestroyStopped
3056            | crate::daemon::RuntimeLifecycleKind::ForceDestroy => ViewerOperationKind::Destroy,
3057            crate::daemon::RuntimeLifecycleKind::Cleanup => ViewerOperationKind::Cleanup,
3058        },
3059        started_at_epoch_seconds: view.started_at_epoch_seconds,
3060        stages: view
3061            .active_stages
3062            .iter()
3063            .map(|(stage, started_at)| ViewerOperationStage {
3064                label: stage.label(),
3065                started_at_epoch_seconds: *started_at,
3066            })
3067            .collect(),
3068        notice: view.notice.clone(),
3069        cancellable: view.cancellable,
3070    }
3071}
3072
3073/// What the phone may do with one session.
3074///
3075/// Everything here is a fact the controller holds and the browser cannot:
3076/// whether the session manager is driving this session, what the agent said it
3077/// supports, and whether a lifecycle operation already owns it.
3078fn session_capabilities(
3079    session: &crate::server::ViewerSession,
3080    operational: Option<&mj_core::relay::RelayOperationalState>,
3081    operation: Option<&crate::server::ViewerOperation>,
3082    facts: Option<&mj_core::acp::AcpSessionFacts>,
3083) -> crate::server::ViewerSessionCapabilities {
3084    use crate::server::ViewerLifecycleCategory;
3085
3086    let live = session.lifecycle == ViewerLifecycleCategory::Live;
3087    // A session the manager is not driving cannot be talked to, whatever its
3088    // durable state says.
3089    let attached = operational.is_some();
3090    // A failed close/destroy can leave its durable record in an intermediate
3091    // state after the lifecycle owner has gone away. Keep the status card
3092    // selectable, but do not turn the failure into an endless mutation lock:
3093    // the published Stop capability is the recovery action.
3094    let transition_busy = session.transitioning && !session.has_error;
3095    let busy = operation.is_some() || transition_busy;
3096    // A failed move can retain a live destination while queue admission is
3097    // incomplete. The durable move hold owns all mutating session controls in
3098    // that interval; retry Move is the one intentional exception.
3099    let partial_move_queue = session.move_recovery.as_ref().is_some_and(|recovery| {
3100        recovery.queue_admission_started && !recovery.queue_admission_finished
3101    });
3102    let mutation_busy = busy || partial_move_queue;
3103    let idle = operational
3104        .is_some_and(|state| state.execution == mj_core::relay::RelayExecutionState::Idle);
3105    crate::server::ViewerSessionCapabilities {
3106        open: session.conversation_available
3107            && !session.transitioning
3108            && session.lifecycle == ViewerLifecycleCategory::Live,
3109        prompt: live && attached && !mutation_busy,
3110        run_shell: live && attached && !mutation_busy,
3111        cancel_turn: live
3112            && !mutation_busy
3113            && operational.is_some_and(|state| {
3114                state.active_prompt.is_some() || state.capacity_retry.is_some()
3115            }),
3116        cancel_operation: operation.is_some_and(|operation| operation.cancellable),
3117        // Stopping a session that is already stopping asks for something that
3118        // is happening; resuming one that is running asks for a second copy.
3119        stop: session.lifecycle.is_dashboard_visible() && !mutation_busy,
3120        rename: !session.transitioning,
3121        resume: !session.lifecycle.is_dashboard_visible() && !mutation_busy,
3122        move_session: live && !busy,
3123        set_config: live && attached && facts.is_some() && !mutation_busy,
3124        // Plan mode is a turn boundary: the terminal offers it only while the
3125        // agent is idle, and the phone must not be looser.
3126        set_plan_mode: live
3127            && !mutation_busy
3128            && idle
3129            && facts.is_some_and(mj_core::acp::AcpSessionFacts::supports_plan_mode),
3130    }
3131}
3132
3133/// The ACP content blocks one phone prompt becomes: its text, then each
3134/// attached image as the image block the prompt path already carries.
3135fn phone_prompt_blocks(
3136    text: String,
3137    images: Vec<crate::server::ViewerPromptImage>,
3138) -> Vec<agent_client_protocol::schema::v1::ContentBlock> {
3139    use agent_client_protocol::schema::v1::{ContentBlock, ImageContent, TextContent};
3140
3141    let mut prompt = Vec::with_capacity(images.len() + 1);
3142    if !text.is_empty() {
3143        prompt.push(ContentBlock::Text(TextContent::new(text)));
3144    }
3145    prompt.extend(images.into_iter().map(|image| match image.attachment {
3146        Some(reference) => reference.content_block(),
3147        None => ContentBlock::Image(ImageContent::new(image.data_base64, image.mime_type)),
3148    }));
3149    prompt
3150}
3151
3152/// The Mjolnir commands a phone may offer for one session.
3153///
3154/// The list is built here, from what this session can actually do, and
3155/// published: the browser used to keep its own copy, which is how `/review`
3156/// was missing from the phone while the terminal had it.
3157fn phone_commands(
3158    session: &crate::server::ViewerSession,
3159    operational: Option<&mj_core::relay::RelayOperationalState>,
3160) -> Vec<crate::server::ViewerMjCommand> {
3161    use crate::server::ViewerCommandSource;
3162    use agent_client_protocol::schema::v1::AvailableCommandInput;
3163
3164    let command =
3165        |name: &str, description: &str, argument: Option<&str>| crate::server::ViewerMjCommand {
3166            name: name.to_owned(),
3167            description: description.to_owned(),
3168            source: ViewerCommandSource::Mj,
3169            argument: argument.map(str::to_owned),
3170        };
3171    let mut commands = vec![
3172        command("help", "show available Mjolnir and agent commands", None),
3173        command(
3174            "detach",
3175            "leave the conversation without stopping the worker",
3176            None,
3177        ),
3178    ];
3179    let option = |key: &str| {
3180        session
3181            .config_options
3182            .iter()
3183            .any(|option| option.key == key)
3184    };
3185    if option("model") {
3186        commands.push(command("model", "change the active model", Some("value")));
3187        commands.push(command("fast", "toggle Codex Fast mode", None));
3188    }
3189    if option("effort") {
3190        commands.push(command(
3191            "effort",
3192            "change the active reasoning effort",
3193            Some("value"),
3194        ));
3195    }
3196    if session.plan_mode_active.is_some() && session.capabilities.set_plan_mode {
3197        commands.push(command("plan", "toggle plan mode", Some("message")));
3198        commands.push(command(
3199            "implement",
3200            "leave plan mode and implement",
3201            Some("instruction"),
3202        ));
3203    }
3204    if session.capabilities.prompt || session.turn_review.is_some() {
3205        commands.push(command(
3206            "review",
3207            "review the finished turn now, or report how review is configured",
3208            Some("status"),
3209        ));
3210    }
3211    // These names are handled by Mjolnir even when the corresponding control
3212    // is unavailable for this session. An agent cannot claim one and turn a
3213    // locally interpreted slash command into a misleading palette entry.
3214    let reserved = [
3215        "help",
3216        "detach",
3217        "model",
3218        "fast",
3219        "effort",
3220        "plan",
3221        "implement",
3222        "review",
3223    ];
3224    for advertised in operational
3225        .into_iter()
3226        .flat_map(|state| state.available_commands.iter())
3227    {
3228        let name = advertised.name.trim();
3229        if name.is_empty()
3230            || reserved
3231                .iter()
3232                .any(|local| name.eq_ignore_ascii_case(local))
3233            || commands
3234                .iter()
3235                .any(|existing| existing.name.eq_ignore_ascii_case(name))
3236        {
3237            continue;
3238        }
3239        let argument = advertised.input.as_ref().and_then(|input| match input {
3240            AvailableCommandInput::Unstructured(input) => {
3241                let hint = input.hint.trim();
3242                (!hint.is_empty()).then(|| hint.to_owned())
3243            }
3244            _ => None,
3245        });
3246        commands.push(crate::server::ViewerMjCommand {
3247            name: name.to_owned(),
3248            description: advertised.description.trim().to_owned(),
3249            source: ViewerCommandSource::Agent,
3250            argument,
3251        });
3252    }
3253    commands
3254}
3255
3256/// The open reviews, keyed by session, for one snapshot.
3257fn review_views(
3258    daemon_runtime: &Arc<RuntimeState>,
3259) -> std::collections::BTreeMap<String, crate::review_host::RuntimeReviewView> {
3260    daemon_runtime
3261        .review_host()
3262        .views()
3263        .into_iter()
3264        .map(|review| (review.session_id.clone(), review))
3265        .collect()
3266}
3267
3268type ViewerMoveRecoveries = std::collections::BTreeMap<String, crate::server::ViewerMoveRecovery>;
3269
3270/// Refresh durable Move records away from the phone event loop. A Move can
3271/// finish after its initiating request disconnects, so active lifecycle views
3272/// alone are not enough to render Retry move or Resume with previous settings.
3273fn request_move_recovery_reload(
3274    completed: &tokio::sync::mpsc::UnboundedSender<Result<ViewerMoveRecoveries, String>>,
3275    in_flight: &mut bool,
3276    jobs: &mut tokio::task::JoinSet<()>,
3277) {
3278    if *in_flight {
3279        return;
3280    }
3281    *in_flight = true;
3282    let completed = completed.clone();
3283    jobs.spawn(async move {
3284        let result = match tokio::task::spawn_blocking(|| {
3285            crate::database::load_move_operations()
3286                .map(|operations| {
3287                    operations
3288                        .into_iter()
3289                        .filter_map(|operation| {
3290                            crate::server::ViewerMoveRecovery::from_operation(&operation)
3291                                .map(|recovery| (operation.selection.session_id.clone(), recovery))
3292                        })
3293                        .collect()
3294                })
3295                .map_err(|error| format!("{error:#}"))
3296        })
3297        .await
3298        {
3299            Ok(result) => result,
3300            Err(error) => Err(format!("move recovery projection task failed: {error}")),
3301        };
3302        let _ = completed.send(result);
3303    });
3304}
3305
3306fn viewer_snapshot(
3307    controller: &Controller,
3308    workspaces: &[mj_core::workspace::WorkspaceRecord],
3309    quotas: &std::collections::BTreeMap<String, ProfileQuota>,
3310    views: &PhoneSessionViews<'_>,
3311    revision: u64,
3312) -> ViewerSnapshot {
3313    let PhoneSessionViews {
3314        conversations,
3315        reviews,
3316        queued_prompts,
3317        active_user_shells,
3318        pending_elicitations,
3319        prompt_images,
3320        operational,
3321        materialized_activity,
3322        project_sources,
3323        operations,
3324        move_recoveries,
3325        capacity,
3326        launch_failures,
3327    } = views;
3328    let mut snapshot =
3329        ViewerSnapshot::from_config_state(&controller.config, &controller.state, revision);
3330    snapshot.launch_failures = launch_failures.to_vec();
3331    snapshot.workspaces = workspaces
3332        .iter()
3333        .map(|workspace| crate::server::ViewerWorkspace {
3334            id: workspace.id.clone(),
3335            name: workspace.name.clone(),
3336        })
3337        .collect();
3338    let now = std::time::SystemTime::now()
3339        .duration_since(std::time::UNIX_EPOCH)
3340        .unwrap_or_default()
3341        .as_secs();
3342    for profile in &mut snapshot.profiles {
3343        let Some(quota) = quotas.get(&profile.id) else {
3344            continue;
3345        };
3346        profile.quota = Some(ViewerQuota {
3347            summary: quota.compact(),
3348            windows: quota
3349                .windows
3350                .iter()
3351                .map(|window| crate::server::ViewerQuotaWindow {
3352                    label: window.label.clone(),
3353                    // The controller reports headroom; a bar fills as a limit
3354                    // is consumed, so the phone is given the complement.
3355                    percent_used: window
3356                        .remaining_percent
3357                        .map(|left| 100_u8.saturating_sub(left)),
3358                    resets_at: window.resets.clone(),
3359                    projects_exhaustion_before_reset: crate::quota::projects_exhaustion(
3360                        window,
3361                        quota.refreshed_at_epoch_seconds,
3362                    ),
3363                })
3364                .collect(),
3365            resets_at: quota
3366                .windows
3367                .iter()
3368                .find_map(|window| window.resets.clone()),
3369            stale: now.saturating_sub(quota.refreshed_at_epoch_seconds)
3370                > QUOTA_STALE_AFTER.as_secs(),
3371            refreshed_at_epoch_seconds: quota.refreshed_at_epoch_seconds,
3372            has_error: quota.error.is_some(),
3373        });
3374    }
3375    for session in &mut snapshot.sessions {
3376        session.move_recovery = move_recoveries.get(&session.id).cloned();
3377        if let Some(record) = controller.state.sessions.get(&session.id)
3378            && let Some(source) = project_sources.source(record, &controller.config)
3379        {
3380            session.set_project_source(source);
3381        }
3382        session.last_activity_at_ms = materialized_activity.get(&session.id).copied().flatten();
3383        session.queued_prompts = queued_prompts
3384            .get(&session.id)
3385            .into_iter()
3386            .flatten()
3387            .map(|prompt| ViewerQueuedPrompt {
3388                id: prompt.id.clone(),
3389                text: prompt.text.clone(),
3390                created_at: prompt.created_at_ms.to_string(),
3391            })
3392            .collect();
3393        session.active_user_shells = active_user_shells
3394            .get(&session.id)
3395            .into_iter()
3396            .flatten()
3397            .map(|shell| ViewerUserShell {
3398                id: shell.command_id.clone(),
3399                command: shell.command.clone(),
3400                started_at_ms: shell.started_at_ms,
3401            })
3402            .collect();
3403        session.background_tasks = operational
3404            .get(&session.id)
3405            .map(|state| {
3406                state
3407                    .background_commands
3408                    .iter()
3409                    .map(|task| ViewerBackgroundTask {
3410                        id: task.id.clone(),
3411                        command: task.command.clone(),
3412                        started_at_ms: task.started_at_ms,
3413                        can_stop: task.can_stop,
3414                    })
3415                    .collect()
3416            })
3417            .unwrap_or_default();
3418        session.pending_elicitations = pending_elicitations
3419            .get(&session.id)
3420            .cloned()
3421            .unwrap_or_default();
3422        session.prompt_images_supported = prompt_images.contains(&session.id);
3423        session.operation = operations.get(&session.id).cloned();
3424        // Runtime ownership takes precedence over the durable record. Move
3425        // can still look Running while its old conversation is no longer a
3426        // valid destination, and stopped cleanup/destroy operations have no
3427        // live lifecycle category of their own.
3428        if let Some(operation) = session.operation.as_ref()
3429            && operation.kind.transition_kind().is_some()
3430        {
3431            session.transitioning = true;
3432        }
3433        let live = operational.get(&session.id);
3434        let facts = live.map(|state| {
3435            mj_core::acp::AcpSessionFacts::from_operational(
3436                controller
3437                    .state
3438                    .sessions
3439                    .get(&session.id)
3440                    .map_or(mj_core::config::HarnessKind::Codex, |record| {
3441                        record.harness_kind
3442                    }),
3443                &state.config,
3444                &state.config_options,
3445                state.modes.as_ref(),
3446            )
3447        });
3448        if let Some(state) = live {
3449            session.latest_event_ordinal = state.latest_ordinal;
3450            session.chat_phase = match state.execution {
3451                mj_core::relay::RelayExecutionState::Idle => crate::server::ViewerChatPhase::Idle,
3452                mj_core::relay::RelayExecutionState::Running => {
3453                    crate::server::ViewerChatPhase::Running
3454                }
3455                mj_core::relay::RelayExecutionState::Closing => {
3456                    crate::server::ViewerChatPhase::Closing
3457                }
3458                mj_core::relay::RelayExecutionState::Closed => {
3459                    crate::server::ViewerChatPhase::Closed
3460                }
3461            };
3462            session.config_options = crate::server::viewer_config_options(
3463                &state.config_options,
3464                facts
3465                    .as_ref()
3466                    .expect("live operational state always has ACP session facts"),
3467            );
3468            // Share activity classification with the terminal. The browser
3469            // retains its detailed turn/step/background clock presentation.
3470            let turn_started_at_ms = state
3471                .active_prompt
3472                .as_ref()
3473                .map(|prompt| prompt.started_at_ms)
3474                .or_else(|| state.harness_turn.map(|turn| turn.started_at_ms));
3475            let turn_started_at = turn_started_at_ms
3476                .and_then(|started_at_ms| u64::try_from(started_at_ms).ok())
3477                .map(|started_at_ms| started_at_ms / 1_000);
3478            session.capacity_retry = state.capacity_retry.clone();
3479            let activity = mj_client::usage_format::SessionActivity::of(state);
3480            let activity_details =
3481                activity.details(turn_started_at_ms, state.current_step_started_at_ms);
3482            session.activity_details = Some(viewer_activity_details(&activity_details));
3483            session.is_idle = controller
3484                .state
3485                .sessions
3486                .get(&session.id)
3487                .is_some_and(|record| record.state == mj_core::state::SessionState::Running)
3488                && session.operation.is_none()
3489                && activity.is_idle(turn_started_at);
3490            session.activity = mj_client::usage_format::format_activity_columns(
3491                now,
3492                turn_started_at,
3493                state
3494                    .current_step_started_at_ms
3495                    .and_then(|value| u64::try_from(value).ok()),
3496                &activity,
3497            )
3498            .join("  ")
3499            .trim()
3500            .to_owned();
3501        }
3502        session.plan_mode_active = facts
3503            .as_ref()
3504            .filter(|facts| facts.supports_plan_mode())
3505            .map(mj_core::acp::AcpSessionFacts::plan_mode_active);
3506        session.turn_review = reviews
3507            .get(&session.id)
3508            .map(crate::server::ViewerTurnReview::from_runtime);
3509        session.capabilities =
3510            session_capabilities(session, live, operations.get(&session.id), facts.as_ref());
3511        session.available_commands = phone_commands(session, live);
3512        if let Some(transcript) = conversations.get(&session.id) {
3513            session.conversation_available = true;
3514            if !session.transitioning {
3515                let mut lines = transcript
3516                    .entries
3517                    .iter()
3518                    .flat_map(|entry| {
3519                        entry
3520                            .lines
3521                            .iter()
3522                            .enumerate()
3523                            .filter_map(move |(index, line)| {
3524                                let line = line.trim();
3525                                (!line.is_empty()).then(|| {
3526                                    if index == 0 {
3527                                        format!("{}: {line}", entry.label)
3528                                    } else {
3529                                        line.to_owned()
3530                                    }
3531                                })
3532                            })
3533                    })
3534                    .collect::<Vec<_>>();
3535                session.preview = lines.split_off(lines.len().saturating_sub(4));
3536            }
3537        }
3538        // `conversation_available` is only known after the transcript loop
3539        // above, so the capability that depends on it is settled here.
3540        session.capabilities.open = session.conversation_available && !session.transitioning;
3541    }
3542    snapshot.capacity = capacity.to_vec();
3543    snapshot
3544}
3545
3546fn viewer_activity_details(
3547    details: &mj_client::usage_format::SessionActivityDetails,
3548) -> ViewerActivityDetails {
3549    ViewerActivityDetails {
3550        kind: match details.kind {
3551            mj_client::usage_format::SessionActivityKind::Turn => ViewerActivityKind::Turn,
3552            mj_client::usage_format::SessionActivityKind::Step => ViewerActivityKind::Step,
3553            mj_client::usage_format::SessionActivityKind::Background => {
3554                ViewerActivityKind::Background
3555            }
3556            mj_client::usage_format::SessionActivityKind::Idle => ViewerActivityKind::Idle,
3557            mj_client::usage_format::SessionActivityKind::Lifecycle
3558            | mj_client::usage_format::SessionActivityKind::Goal => ViewerActivityKind::Lifecycle,
3559        },
3560        turn_started_at_ms: details.turn_started_at_ms,
3561        step_started_at_ms: details.step_started_at_ms,
3562        background_started_at_ms: details.background_started_at_ms,
3563        idle_since_ms: details.idle_since_ms,
3564        label: details.label.clone(),
3565    }
3566}
3567
3568/// Seed the viewer's in-memory activity cache from the durable projection
3569/// before publishing its first snapshot. Later worker snapshots update this
3570/// cache without adding a database read to the render path.
3571async fn load_materialized_activity(
3572    controller: &Controller,
3573) -> Result<std::collections::BTreeMap<String, Option<i64>>> {
3574    let session_ids = controller
3575        .state
3576        .sessions
3577        .keys()
3578        .cloned()
3579        .collect::<Vec<_>>();
3580    tokio::task::spawn_blocking(move || {
3581        session_ids
3582            .into_iter()
3583            .map(|session_id| {
3584                crate::database::load_materialized_session_summary(&session_id).map(|summary| {
3585                    (
3586                        session_id,
3587                        summary.and_then(|summary| summary.last_activity_at_ms),
3588                    )
3589                })
3590            })
3591            .collect()
3592    })
3593    .await
3594    .context("materialized activity startup task failed")?
3595}
3596
3597#[cfg(test)]
3598mod tests {
3599    use super::*;
3600    use crate::pollers::QUOTA_REFRESH_INTERVAL;
3601    use std::collections::BTreeMap;
3602
3603    use agent_client_protocol::schema::v1::{
3604        SessionConfigOption, SessionConfigOptionCategory, SessionConfigSelectOption,
3605        SessionConfigSelectOptions,
3606    };
3607    use mj_core::config::{
3608        CONFIG_VERSION, Config, HarnessKind, ProjectBundle, ProjectRepository, TargetTemplate,
3609    };
3610    use mj_core::state::SessionState;
3611
3612    #[test]
3613    fn viewer_config_options_publish_current_advertised_values() {
3614        let make_options = |model, effort| {
3615            vec![
3616                SessionConfigOption::select(
3617                    "model_selector",
3618                    "Model",
3619                    model,
3620                    SessionConfigSelectOptions::Ungrouped(vec![
3621                        SessionConfigSelectOption::new("sonnet", "Claude Sonnet"),
3622                        SessionConfigSelectOption::new("opus", "Claude Opus"),
3623                    ]),
3624                )
3625                .category(SessionConfigOptionCategory::Model),
3626                SessionConfigOption::select(
3627                    "reasoning_effort",
3628                    "Effort",
3629                    effort,
3630                    SessionConfigSelectOptions::Ungrouped(vec![
3631                        SessionConfigSelectOption::new("high", "High"),
3632                        SessionConfigSelectOption::new("max", "Maximum"),
3633                    ]),
3634                ),
3635            ]
3636        };
3637        let options = make_options("sonnet", "high");
3638        let defaults = mj_core::acp::AcpSessionFacts::from_operational(
3639            HarnessKind::Claude,
3640            &BTreeMap::new(),
3641            &options,
3642            None,
3643        );
3644        let projected = crate::server::viewer_config_options(&options, &defaults);
3645        assert_eq!(
3646            projected
3647                .iter()
3648                .map(|option| (option.key.as_str(), option.current.as_deref()))
3649                .collect::<Vec<_>>(),
3650            [("model", Some("sonnet")), ("effort", Some("high"))]
3651        );
3652
3653        let options = make_options("opus", "max");
3654        let updated = mj_core::acp::AcpSessionFacts::from_operational(
3655            HarnessKind::Claude,
3656            &BTreeMap::new(),
3657            &options,
3658            None,
3659        );
3660        let projected = crate::server::viewer_config_options(&options, &updated);
3661        assert_eq!(projected[0].current.as_deref(), Some("opus"));
3662        assert_eq!(projected[1].current.as_deref(), Some("max"));
3663    }
3664
3665    #[tokio::test]
3666    async fn explicit_tls_takes_precedence_over_tailscale_detection() {
3667        let resolved = resolve_server_args(
3668            ServerArgs {
3669                bind: "0.0.0.0:4443".into(),
3670                tailscale_detect: true,
3671                tls_cert: Some(PathBuf::from("configured-cert.pem")),
3672                tls_key: Some(PathBuf::from("configured-key.pem")),
3673            },
3674            tokio_util::sync::CancellationToken::new(),
3675        )
3676        .await
3677        .unwrap();
3678
3679        assert_eq!(resolved.bind, "0.0.0.0:4443".parse().unwrap());
3680        assert_eq!(resolved.viewer_url, "https://0.0.0.0:4443");
3681        assert_eq!(
3682            resolved.tls_files,
3683            Some((
3684                PathBuf::from("configured-cert.pem"),
3685                PathBuf::from("configured-key.pem")
3686            ))
3687        );
3688        assert!(resolved.tailscale.is_none());
3689        assert!(resolved.fallback_reason.is_none());
3690    }
3691
3692    #[tokio::test]
3693    async fn disabling_detection_keeps_the_viewer_on_loopback() {
3694        let resolved = resolve_server_args(
3695            ServerArgs {
3696                bind: "127.0.0.1:4765".into(),
3697                tailscale_detect: false,
3698                tls_cert: None,
3699                tls_key: None,
3700            },
3701            tokio_util::sync::CancellationToken::new(),
3702        )
3703        .await
3704        .unwrap();
3705
3706        assert_eq!(resolved.bind, "127.0.0.1:4765".parse().unwrap());
3707        assert_eq!(resolved.viewer_url, "http://127.0.0.1:4765");
3708        assert!(
3709            resolved
3710                .fallback_reason
3711                .unwrap()
3712                .contains("detection is disabled")
3713        );
3714    }
3715
3716    fn bare_preflight_config() -> Config {
3717        let mut config = Config::default();
3718        config
3719            .targets
3720            .insert("raw".into(), TargetTemplate::LocalBare);
3721        config
3722    }
3723
3724    #[test]
3725    fn move_recovery_projection_exposes_safe_retry_settings_only() {
3726        let operation = mj_core::state::MoveOperation {
3727            source_checkpoint_only: false,
3728            operation_id: "move-1".into(),
3729            selection: mj_core::state::MoveSelection {
3730                clear_resource_allocation: true,
3731                session_id: "session-1".into(),
3732                profile_id: Some("destination-profile".into()),
3733                target_template_id: Some("destination-target".into()),
3734                additional_mounts: Some(vec![crate::targets::AdditionalMount {
3735                    source: "/destination/source".into(),
3736                    destination: "/destination/target".into(),
3737                    read_only: true,
3738                }]),
3739                resource_allocation: None,
3740            },
3741            source_profile_id: "source-profile".into(),
3742            source_target_template_id: "source-target".into(),
3743            source_target: None,
3744            source_native_session_id: Some("private-native-id".into()),
3745            source_additional_mounts: vec![crate::targets::AdditionalMount {
3746                source: "/source/source".into(),
3747                destination: "/source/target".into(),
3748                read_only: false,
3749            }],
3750            source_resource_allocation: Some(
3751                mj_core::state::SessionResourceAllocation::Container {
3752                    cpus: 2,
3753                    memory_bytes: 4096,
3754                },
3755            ),
3756            destination_target: None,
3757            destination_native_session_id: None,
3758            destination_store_id: None,
3759            configuration_fingerprint: "private-fingerprint".into(),
3760            checkpoint: None,
3761            recovery_session: None,
3762            queue: mj_core::state::ResumeQueueDisposition::Start,
3763            phase: mj_core::state::MovePhase::Cancelled,
3764            queue_admission_started: false,
3765            queue_admission_finished: false,
3766            cancellation_requested: true,
3767            created_at: "now".into(),
3768            updated_at: "now".into(),
3769            error: Some("private path and token".into()),
3770        };
3771        let recovery = ViewerMoveRecovery::from_operation(&operation).unwrap();
3772        assert_eq!(recovery.phase, "cancelled");
3773        assert_eq!(recovery.source_profile_id, "source-profile");
3774        assert!(recovery.clear_resource_allocation);
3775        assert_eq!(recovery.source_additional_mounts.len(), 1);
3776        assert!(recovery.source_resource_allocation.is_some());
3777        assert_eq!(recovery.destination_additional_mounts.len(), 1);
3778        assert!(recovery.destination_resource_allocation.is_none());
3779        let json = serde_json::to_string(&recovery).unwrap();
3780        assert!(!json.contains("private-native-id"));
3781        assert!(!json.contains("private-fingerprint"));
3782        assert!(!json.contains("private path and token"));
3783    }
3784
3785    #[test]
3786    fn new_preflight_rejects_a_bare_project_without_a_git_head() {
3787        let error = run_new_preflight(
3788            bare_preflight_config(),
3789            "hel".into(),
3790            "raw".into(),
3791            Some(PathBuf::from("/definitely/not/a/project")),
3792        )
3793        .expect_err("a missing project directory must fail preflight");
3794
3795        assert!(
3796            error
3797                .to_string()
3798                .contains("project directory does not exist or is not a directory")
3799        );
3800    }
3801
3802    #[test]
3803    fn new_preflight_accepts_a_git_project_for_a_bare_target() {
3804        let directory = std::env::current_dir().expect("the test has a working directory");
3805        let answer = run_new_preflight(
3806            bare_preflight_config(),
3807            "hel".into(),
3808            "raw".into(),
3809            Some(directory.clone()),
3810        )
3811        .expect("the repository running the test has a valid Git HEAD");
3812
3813        assert!(answer.dirty_repositories.is_empty());
3814        assert_eq!(answer.project_directory, Some(directory));
3815    }
3816
3817    #[test]
3818    fn new_preflight_requires_network_sources_for_isolated_targets() {
3819        let mut config = Config::default();
3820        config.targets.insert(
3821            "podman".into(),
3822            TargetTemplate::LocalPodman {
3823                container: mj_core::config::ContainerTemplate {
3824                    image: "test-image".into(),
3825                    pull_policy: Default::default(),
3826                    platform: None,
3827                    cpus: None,
3828                    memory: None,
3829                    environment: Default::default(),
3830                    workspace_storage: Default::default(),
3831                },
3832            },
3833        );
3834        config.bundles.insert(
3835            "hel".into(),
3836            ProjectBundle {
3837                primary_repo: "hel".into(),
3838                repositories: vec![ProjectRepository {
3839                    id: "hel".into(),
3840                    github: None,
3841                    local: Some(PathBuf::from("/definitely/not/a/repository")),
3842                    destination: "hel".into(),
3843                    git_ref: None,
3844                }],
3845            },
3846        );
3847        let error = run_new_preflight(config, "hel".into(), "podman".into(), None)
3848            .expect_err("an isolated bundle cannot use a local source");
3849        assert!(error.to_string().contains("repository"));
3850    }
3851
3852    #[test]
3853    fn a_phone_prompt_becomes_its_text_then_its_images() {
3854        use agent_client_protocol::schema::v1::ContentBlock;
3855
3856        let image = |data: &str| crate::server::ViewerPromptImage {
3857            attachment: None,
3858            data_base64: data.into(),
3859            mime_type: "image/png".into(),
3860            width: 32,
3861            height: 24,
3862        };
3863        let blocks = phone_prompt_blocks(
3864            "look at this".into(),
3865            vec![image("aW1hZ2U="), image("c2Vjb25k")],
3866        );
3867        let ContentBlock::Text(text) = &blocks[0] else {
3868            panic!("the prompt leads with its text");
3869        };
3870        assert_eq!(text.text, "look at this");
3871        let ContentBlock::Image(first) = &blocks[1] else {
3872            panic!("each attachment travels as an image block");
3873        };
3874        assert_eq!(first.data, "aW1hZ2U=");
3875        assert_eq!(first.mime_type, "image/png");
3876        assert!(matches!(blocks[2], ContentBlock::Image(_)));
3877        assert_eq!(blocks.len(), 3);
3878
3879        // An image needs no words with it, and an empty text block would be a
3880        // message the user never wrote.
3881        let images_only = phone_prompt_blocks(String::new(), vec![image("aW1hZ2U=")]);
3882        assert_eq!(images_only.len(), 1);
3883        assert!(matches!(images_only[0], ContentBlock::Image(_)));
3884    }
3885
3886    #[test]
3887    fn image_prompts_are_offered_only_after_the_agent_advertises_them() {
3888        use agent_client_protocol::schema::v1::AgentCapabilities;
3889        use mj_core::relay::{RelayExecutionState, RelayOperationalState};
3890
3891        let operational = |agent_capabilities| RelayOperationalState {
3892            goal: Default::default(),
3893            capacity_retry: None,
3894            activity_turn_started_at_ms: None,
3895            session_id: "session-1".into(),
3896            store_id: None,
3897            idle_since_ms: None,
3898            execution: RelayExecutionState::Idle,
3899            latest_ordinal: 0,
3900            latest_digest: String::new(),
3901            acknowledged_through: 0,
3902            acknowledged_digest: String::new(),
3903            recovery_floor_ordinal: 0,
3904            recovery_floor_digest: String::new(),
3905            native_session_id: None,
3906            checkpoint_only: false,
3907            acp_ready: None,
3908            agent_capabilities,
3909            agent_info: None,
3910            steering_supported: None,
3911            config_options: Vec::new(),
3912            modes: None,
3913            available_commands: Vec::new(),
3914            config: std::collections::BTreeMap::new(),
3915            active_prompt: None,
3916            queued_prompts: Vec::new(),
3917            active_user_shells: Vec::new(),
3918            active_agent_terminals: Vec::new(),
3919            checkpoint_barrier: None,
3920            checkpoint_ready: None,
3921            last_acp_activity_at_ms: None,
3922            current_step_started_at_ms: None,
3923            foreground_tool_started_at_ms: None,
3924            harness_turn: None,
3925            last_harness_turn_started_ordinal: None,
3926            background_commands: Vec::new(),
3927            background_work_known: None,
3928        };
3929
3930        // A session whose agent has not answered `initialize` has advertised
3931        // nothing, so the phone is not offered a control the agent may refuse.
3932        assert!(!agent_accepts_prompt_images(&operational(None)));
3933        assert!(!agent_accepts_prompt_images(&operational(Some(Box::new(
3934            AgentCapabilities::default()
3935        )))));
3936        let mut capabilities = AgentCapabilities::default();
3937        capabilities.prompt_capabilities.image = true;
3938        assert!(agent_accepts_prompt_images(&operational(Some(Box::new(
3939            capabilities
3940        )))));
3941    }
3942
3943    #[test]
3944    fn phone_snapshot_projects_capability_gated_and_agent_commands_with_provenance() {
3945        use agent_client_protocol::schema::v1::{
3946            AvailableCommand, AvailableCommandInput, SessionMode, SessionModeState,
3947            UnstructuredCommandInput,
3948        };
3949        use mj_core::relay::{RelayExecutionState, RelayOperationalState};
3950
3951        use crate::server::ViewerCommandSource;
3952
3953        let mut controller = controller_with_profiles(&["claude"]);
3954        let mut record = phone_session("session-1", 0);
3955        record.harness_kind = HarnessKind::Claude;
3956        record.last_profile = "claude".into();
3957        record.state = SessionState::Running;
3958        controller.state.sessions.insert(record.id.clone(), record);
3959        let operational = RelayOperationalState {
3960            goal: Default::default(),
3961            capacity_retry: None,
3962            activity_turn_started_at_ms: None,
3963            session_id: "session-1".into(),
3964            store_id: None,
3965            idle_since_ms: None,
3966            execution: RelayExecutionState::Idle,
3967            latest_ordinal: 0,
3968            latest_digest: String::new(),
3969            acknowledged_through: 0,
3970            acknowledged_digest: String::new(),
3971            recovery_floor_ordinal: 0,
3972            recovery_floor_digest: String::new(),
3973            native_session_id: None,
3974            checkpoint_only: false,
3975            acp_ready: None,
3976            agent_capabilities: None,
3977            agent_info: None,
3978            steering_supported: None,
3979            config_options: Vec::new(),
3980            modes: Some(SessionModeState::new(
3981                "default",
3982                vec![
3983                    SessionMode::new("default", "Default"),
3984                    SessionMode::new("plan", "Plan"),
3985                ],
3986            )),
3987            available_commands: vec![
3988                AvailableCommand::new("inspect", " Inspect the workspace ").input(
3989                    AvailableCommandInput::Unstructured(UnstructuredCommandInput::new(" query ")),
3990                ),
3991                AvailableCommand::new("Review", "agent collision"),
3992                AvailableCommand::new("INSPECT", "duplicate agent command"),
3993            ],
3994            config: std::collections::BTreeMap::new(),
3995            active_prompt: None,
3996            queued_prompts: Vec::new(),
3997            active_user_shells: Vec::new(),
3998            active_agent_terminals: Vec::new(),
3999            checkpoint_barrier: None,
4000            checkpoint_ready: None,
4001            last_acp_activity_at_ms: None,
4002            current_step_started_at_ms: None,
4003            foreground_tool_started_at_ms: None,
4004            harness_turn: None,
4005            last_harness_turn_started_ordinal: None,
4006            background_commands: Vec::new(),
4007            background_work_known: None,
4008        };
4009        let mut operational = std::collections::BTreeMap::from([("session-1".into(), operational)]);
4010        let materialized_activity =
4011            std::collections::BTreeMap::from([("session-1".into(), Some(7_777_i64))]);
4012        let project = |operational: &std::collections::BTreeMap<String, RelayOperationalState>| {
4013            viewer_snapshot(
4014                &controller,
4015                &[],
4016                &std::collections::BTreeMap::new(),
4017                &PhoneSessionViews {
4018                    conversations: &std::collections::BTreeMap::new(),
4019                    queued_prompts: &std::collections::BTreeMap::new(),
4020                    active_user_shells: &std::collections::BTreeMap::new(),
4021                    pending_elicitations: &std::collections::BTreeMap::new(),
4022                    prompt_images: &std::collections::BTreeSet::new(),
4023                    operational,
4024                    materialized_activity: &materialized_activity,
4025                    project_sources: &PhoneProjectSources::default(),
4026                    operations: &std::collections::BTreeMap::new(),
4027                    move_recoveries: &std::collections::BTreeMap::new(),
4028                    capacity: &[],
4029                    launch_failures: &[],
4030                    reviews: &std::collections::BTreeMap::new(),
4031                },
4032                1,
4033            )
4034        };
4035        let snapshot = project(&operational);
4036        let session = &snapshot.sessions[0];
4037
4038        assert_eq!(session.display_location, "podman");
4039        assert_eq!(session.last_activity_at_ms, Some(7_777));
4040        assert_eq!(
4041            session.activity_details,
4042            Some(crate::server::ViewerActivityDetails {
4043                kind: ViewerActivityKind::Idle,
4044                turn_started_at_ms: None,
4045                step_started_at_ms: None,
4046                background_started_at_ms: None,
4047                idle_since_ms: None,
4048                label: None,
4049            })
4050        );
4051
4052        assert!(session.capabilities.prompt);
4053        assert!(session.capabilities.set_plan_mode);
4054        assert_eq!(
4055            session
4056                .available_commands
4057                .iter()
4058                .map(|command| (command.name.as_str(), command.source))
4059                .collect::<Vec<_>>(),
4060            vec![
4061                ("help", ViewerCommandSource::Mj),
4062                ("detach", ViewerCommandSource::Mj),
4063                ("plan", ViewerCommandSource::Mj),
4064                ("implement", ViewerCommandSource::Mj),
4065                ("review", ViewerCommandSource::Mj),
4066                ("inspect", ViewerCommandSource::Agent),
4067            ]
4068        );
4069        let inspect = session.available_commands.last().unwrap();
4070        assert_eq!(inspect.description, "Inspect the workspace");
4071        assert_eq!(inspect.argument.as_deref(), Some("query"));
4072
4073        assert!(session.is_idle);
4074        assert_eq!(session.activity, "[idle]");
4075        let state = operational.get_mut("session-1").unwrap();
4076        state
4077            .background_commands
4078            .push(mj_core::relay::BackgroundCommand {
4079                id: "background-1".into(),
4080                started_at_ms: 1_000,
4081                command: "background check".into(),
4082                can_stop: true,
4083            });
4084        let background = project(&operational);
4085        assert_eq!(
4086            background.sessions[0].background_tasks,
4087            vec![ViewerBackgroundTask {
4088                id: "background-1".into(),
4089                command: "background check".into(),
4090                started_at_ms: 1_000,
4091                can_stop: true,
4092            }]
4093        );
4094        assert!(!background.sessions[0].is_idle);
4095        assert!(background.sessions[0].activity.starts_with("BG "));
4096        let state = operational.get_mut("session-1").unwrap();
4097        state.background_commands.clear();
4098        // A phase flag alone can be stale; a current SDK step proves work.
4099        state.execution = RelayExecutionState::Running;
4100        let stale_running = project(&operational);
4101        assert!(stale_running.sessions[0].is_idle);
4102        operational
4103            .get_mut("session-1")
4104            .unwrap()
4105            .current_step_started_at_ms = Some(1_000);
4106        let running = project(&operational);
4107        assert!(!running.sessions[0].is_idle);
4108        assert_ne!(running.sessions[0].activity, "[idle]");
4109        operational.get_mut("session-1").unwrap().execution = RelayExecutionState::Idle;
4110        let idle_again = project(&operational);
4111        assert!(idle_again.sessions[0].is_idle);
4112        assert_eq!(idle_again.sessions[0].activity, "[idle]");
4113        let unknown = project(&std::collections::BTreeMap::new());
4114        assert!(!unknown.sessions[0].is_idle);
4115        assert!(unknown.sessions[0].activity.is_empty());
4116        assert!(unknown.sessions[0].activity_details.is_none());
4117    }
4118
4119    #[test]
4120    fn tailscale_listener_preserves_the_configured_port() {
4121        assert_eq!(
4122            tailscale_bind("127.0.0.1:4765".parse().unwrap()),
4123            "0.0.0.0:4765".parse().unwrap()
4124        );
4125    }
4126
4127    fn controller_with_profiles(ids: &[&str]) -> Controller {
4128        Controller {
4129            config: Config {
4130                subagents: Default::default(),
4131                version: CONFIG_VERSION,
4132                sessions_side: Default::default(),
4133                advanced: Default::default(),
4134                show_stopped_sessions: false,
4135                newer_config_version: None,
4136                spinner: Default::default(),
4137                theme: Default::default(),
4138                phone: Default::default(),
4139                review: Default::default(),
4140                legacy_startup: (),
4141                profiles: ids
4142                    .iter()
4143                    .map(|id| {
4144                        (
4145                            (*id).to_owned(),
4146                            HarnessProfile {
4147                                enabled: true,
4148                                context_window_bytes: None,
4149                                kind: HarnessKind::Codex,
4150                                home: PathBuf::from("/home/agent").join(id),
4151                                environment: std::collections::BTreeMap::new(),
4152                            },
4153                        )
4154                    })
4155                    .collect(),
4156                bundles: std::collections::BTreeMap::new(),
4157                targets: std::collections::BTreeMap::new(),
4158            },
4159            state: State::default(),
4160        }
4161    }
4162
4163    fn snapshot_with_project_sources(
4164        controller: &Controller,
4165        sources: &PhoneProjectSources,
4166    ) -> ViewerSnapshot {
4167        viewer_snapshot(
4168            controller,
4169            &[],
4170            &Default::default(),
4171            &PhoneSessionViews {
4172                conversations: &Default::default(),
4173                queued_prompts: &Default::default(),
4174                active_user_shells: &Default::default(),
4175                pending_elicitations: &Default::default(),
4176                prompt_images: &Default::default(),
4177                operational: &Default::default(),
4178                materialized_activity: &Default::default(),
4179                project_sources: sources,
4180                operations: &Default::default(),
4181                move_recoveries: &Default::default(),
4182                capacity: &[],
4183                launch_failures: &[],
4184                reviews: &Default::default(),
4185            },
4186            1,
4187        )
4188    }
4189
4190    #[tokio::test]
4191    async fn phone_projects_resolve_origins_and_discard_results_after_location_changes() {
4192        let root = tempfile::tempdir().unwrap();
4193        let mut controller = controller_with_profiles(&["codex"]);
4194        controller
4195            .config
4196            .targets
4197            .insert("local".into(), TargetTemplate::LocalBare);
4198        for (id, origin) in [
4199            ("first-checkout", "git@github.com:BrokkAi/hel.git"),
4200            ("second-checkout", "https://github.com/BrokkAi/hel.git"),
4201        ] {
4202            let directory = root.path().join(id);
4203            std::fs::create_dir(&directory).unwrap();
4204            for args in [vec!["init"], vec!["remote", "add", "origin", origin]] {
4205                let command = crate::targets::CommandSpec::new(
4206                    "git",
4207                    ["-C".to_owned(), directory.to_string_lossy().into_owned()]
4208                        .into_iter()
4209                        .chain(args.into_iter().map(str::to_owned)),
4210                );
4211                assert_eq!(ProcessExecutor.execute(&command).unwrap().status, 0);
4212            }
4213            let mut record = phone_session(id, 0);
4214            record.project_directory = Some(directory);
4215            record.target_template_id = "local".into();
4216            controller.state.sessions.insert(id.into(), record);
4217        }
4218        let mut sources = PhoneProjectSources::default();
4219        sources.synchronize(&controller);
4220        assert_eq!(sources.jobs.len(), 2);
4221        tokio::time::timeout(Duration::from_secs(10), async {
4222            while let Some(result) = sources.jobs.join_next().await {
4223                sources.complete(result.unwrap());
4224            }
4225        })
4226        .await
4227        .unwrap();
4228        let snapshot = snapshot_with_project_sources(&controller, &sources);
4229        assert_eq!(
4230            snapshot.sessions[0].project_key,
4231            snapshot.sessions[1].project_key
4232        );
4233        assert!(
4234            snapshot
4235                .sessions
4236                .iter()
4237                .all(|session| session.project_label == "hel")
4238        );
4239        assert!(
4240            !serde_json::to_string(&snapshot)
4241                .unwrap()
4242                .contains(&root.path().to_string_lossy().to_string())
4243        );
4244        sources.synchronize(&controller);
4245        assert!(
4246            sources.jobs.is_empty(),
4247            "unchanged inputs reuse the resolved origin"
4248        );
4249
4250        let previous = &sources.entries["first-checkout"];
4251        let late = ProjectSourceResolved {
4252            cancelled: previous.cancelled.clone(),
4253            session_id: "first-checkout".into(),
4254            key: previous.key.clone(),
4255            result: Ok(ProjectSourceIdentity::git_remote("old/wrong").unwrap()),
4256        };
4257        controller
4258            .state
4259            .sessions
4260            .get_mut("first-checkout")
4261            .unwrap()
4262            .project_directory = None;
4263        let snapshot = snapshot_with_project_sources(&controller, &sources);
4264        assert_ne!(
4265            snapshot.sessions[0].project_key,
4266            snapshot.sessions[1].project_key
4267        );
4268        sources.synchronize(&controller);
4269        assert!(late.cancelled.load(Ordering::Acquire));
4270        sources.complete(late);
4271        assert!(!sources.entries.contains_key("first-checkout"));
4272        assert_eq!(snapshot.sessions[0].project_label, "project");
4273    }
4274
4275    #[test]
4276    fn capacity_target_publication_skips_unchanged_targets_and_preserves_readings() {
4277        let mut controller = controller_with_profiles(&[]);
4278        controller
4279            .config
4280            .targets
4281            .insert("raw".into(), TargetTemplate::LocalBare);
4282        let (targets_tx, mut targets_rx) = tokio::sync::watch::channel(Vec::new());
4283        let mut state = std::collections::BTreeMap::new();
4284
4285        publish_capacity_targets(&controller, &targets_tx, &mut state);
4286        assert!(targets_rx.has_changed().expect("target sender is alive"));
4287        assert_eq!(targets_rx.borrow_and_update().len(), 1);
4288
4289        let usage = crate::targets::DeploymentCapacityUsage {
4290            cpu_percent: Some(37),
4291            memory_used_bytes: 3,
4292            memory_total_bytes: 4,
4293            logical_cores: 8,
4294            disk_total_bytes: Some(5),
4295        };
4296        let local = state.get_mut("local").expect("local capacity state");
4297        local.usage = Some(usage.clone());
4298        local.on_demand = true;
4299        local.sampled_at_epoch_seconds = Some(42);
4300        local.refreshing = false;
4301
4302        publish_capacity_targets(&controller, &targets_tx, &mut state);
4303        assert!(!targets_rx.has_changed().expect("target sender is alive"));
4304        let local_capacity = viewer_capacity(&state)
4305            .into_iter()
4306            .find(|capacity| capacity.id == "local")
4307            .expect("local viewer capacity");
4308        assert_eq!(local_capacity.cpu_percent, usage.cpu_percent);
4309        assert_eq!(
4310            local_capacity.memory_used_bytes,
4311            Some(usage.memory_used_bytes)
4312        );
4313        assert_eq!(local_capacity.logical_cores, Some(usage.logical_cores));
4314        assert_eq!(local_capacity.sampled_at_epoch_seconds, Some(42));
4315
4316        controller
4317            .config
4318            .targets
4319            .insert("second-local".into(), TargetTemplate::LocalBare);
4320        publish_capacity_targets(&controller, &targets_tx, &mut state);
4321        assert!(targets_rx.has_changed().expect("target sender is alive"));
4322        assert_eq!(targets_rx.borrow_and_update().len(), 1);
4323        assert_eq!(state["local"].usage, Some(usage.clone()));
4324
4325        controller.config.targets.insert(
4326            "fleet".into(),
4327            TargetTemplate::AwsEc2 {
4328                aws_profile: None,
4329                region: "us-east-1".into(),
4330                launch_template: "hel-runson".into(),
4331                launch_template_version: None,
4332                ssh_user: "ubuntu".into(),
4333                address_source: Default::default(),
4334                identity_file: None,
4335                ssh_args: Vec::new(),
4336            },
4337        );
4338        publish_capacity_targets(&controller, &targets_tx, &mut state);
4339        assert!(targets_rx.has_changed().expect("target sender is alive"));
4340        assert_eq!(targets_rx.borrow_and_update().len(), 2);
4341        assert!(state.contains_key("aws:fleet"));
4342        assert_eq!(state["local"].usage, Some(usage.clone()));
4343
4344        controller.config.targets.remove("fleet");
4345        publish_capacity_targets(&controller, &targets_tx, &mut state);
4346        assert!(targets_rx.has_changed().expect("target sender is alive"));
4347        assert_eq!(targets_rx.borrow_and_update().len(), 1);
4348        assert!(!state.contains_key("aws:fleet"));
4349        assert_eq!(state["local"].usage, Some(usage));
4350    }
4351
4352    fn prompt_action() -> ControllerAction {
4353        ControllerAction::Prompt {
4354            session_id: "session-1".into(),
4355            text: "ship it".into(),
4356            images: Vec::new(),
4357        }
4358    }
4359
4360    fn new_action() -> ControllerAction {
4361        ControllerAction::New {
4362            mjolnir_subagents: None,
4363            create_managed_worktree: None,
4364            workspace_id: String::new(),
4365            profile_id: "codex".into(),
4366            bundle_id: "project".into(),
4367            target_id: "podman".into(),
4368            title: Some("Phone launch".into()),
4369            project_directory: None,
4370            dirty_ack: Vec::new(),
4371        }
4372    }
4373
4374    fn phone_session(id: &str, viewed_through_event_ordinal: u64) -> SessionRecord {
4375        SessionRecord {
4376            mjolnir_subagents: None,
4377            create_managed_worktree: None,
4378            workspace_id: mj_core::workspace::DEFAULT_WORKSPACE_ID.to_owned(),
4379            archived: false,
4380            container_cpus: None,
4381            container_memory: None,
4382            id: id.into(),
4383            title: "Phone launch".into(),
4384            harness_kind: mj_core::config::HarnessKind::Codex,
4385            last_profile: "codex".into(),
4386            bundle_id: "project".into(),
4387            project_directory: None,
4388            managed_worktree: None,
4389            target_template_id: "podman".into(),
4390            resource_allocation: None,
4391            additional_mounts: Vec::new(),
4392            state: SessionState::Provisioning,
4393            target: None,
4394            native_session_id: None,
4395            acp_session_title: None,
4396            session_title_override: Some("Phone launch".into()),
4397            created_at: "2026-08-14T00:00:00Z".into(),
4398            updated_at: "2026-08-14T00:00:00Z".into(),
4399            viewed_through_event_ordinal,
4400            draft_input: String::new(),
4401            last_error: None,
4402            last_checkpoint_error: None,
4403            checkpoint: None,
4404        }
4405    }
4406
4407    #[tokio::test]
4408    async fn controller_reload_does_not_block_the_phone_control_loop() {
4409        let (completed_tx, mut completed_rx) = tokio::sync::mpsc::unbounded_channel();
4410        let (started_tx, started_rx) = std::sync::mpsc::channel();
4411        let (release_tx, release_rx) = std::sync::mpsc::channel();
4412        let loaded = controller_with_profiles(&[]);
4413        let release = std::thread::spawn(move || {
4414            started_rx.recv().unwrap();
4415            std::thread::sleep(Duration::from_millis(250));
4416            release_tx.send(()).unwrap();
4417        });
4418
4419        let started = Instant::now();
4420        spawn_controller_reload_with(completed_tx, move || {
4421            started_tx.send(()).unwrap();
4422            release_rx.recv().unwrap();
4423            Ok(loaded)
4424        });
4425        assert!(
4426            started.elapsed() < Duration::from_millis(100),
4427            "scheduling a controller reload occupied the control loop for {:?}",
4428            started.elapsed()
4429        );
4430
4431        let completed = tokio::time::timeout(Duration::from_secs(1), completed_rx.recv())
4432            .await
4433            .expect("background reload timed out")
4434            .expect("background reload channel closed");
4435        assert!(completed.result.is_ok());
4436        release.join().unwrap();
4437    }
4438
4439    #[test]
4440    fn read_receipt_only_persists_and_refreshes_when_the_cursor_advances() {
4441        let session_id = "0123456789abcdef0123456789abcdef";
4442        let mut state = State::default();
4443        state
4444            .sessions
4445            .insert(session_id.into(), phone_session(session_id, 5));
4446
4447        // The viewer re-posts its cursor after every refresh; a repeat must
4448        // not reach the database and must not move the revision.
4449        assert_eq!(
4450            plan_read_receipt(&state, session_id, 5),
4451            ReadReceiptPlan::AlreadyRead
4452        );
4453        assert_eq!(
4454            plan_read_receipt(&state, session_id, 4),
4455            ReadReceiptPlan::AlreadyRead
4456        );
4457        assert_eq!(
4458            plan_read_receipt(&state, "missing", 9),
4459            ReadReceiptPlan::UnknownSession
4460        );
4461        assert_eq!(
4462            plan_read_receipt(&state, session_id, 9),
4463            ReadReceiptPlan::Persist
4464        );
4465
4466        assert!(apply_read_receipt(&mut state, session_id, 9));
4467        assert_eq!(state.sessions[session_id].viewed_through_event_ordinal, 9);
4468        assert!(!apply_read_receipt(&mut state, session_id, 9));
4469        assert!(!apply_read_receipt(&mut state, session_id, 7));
4470        assert!(!apply_read_receipt(&mut state, "missing", 9));
4471        assert_eq!(
4472            plan_read_receipt(&state, session_id, 9),
4473            ReadReceiptPlan::AlreadyRead
4474        );
4475    }
4476
4477    #[tokio::test]
4478    async fn an_admitted_action_answers_its_phone_before_the_work_runs() {
4479        let mut replies = PendingActionReplies::default();
4480        let (reply, answer) = tokio::sync::oneshot::channel();
4481
4482        replies.accept(1, &prompt_action(), reply);
4483
4484        // No completion has been reported, and the phone already has its
4485        // answer: holding it until the action finished is what mobile
4486        // networks time out on.
4487        assert_eq!(answer.await.unwrap(), ActionOutcome::accepted());
4488    }
4489
4490    #[tokio::test]
4491    async fn a_new_action_answers_once_its_provisional_session_is_published() {
4492        let mut replies = PendingActionReplies::default();
4493        let (reply, mut answer) = tokio::sync::oneshot::channel();
4494
4495        replies.accept(7, &new_action(), reply);
4496        assert!(
4497            matches!(
4498                answer.try_recv(),
4499                Err(tokio::sync::oneshot::error::TryRecvError::Empty)
4500            ),
4501            "a new session has no id to report before it is published"
4502        );
4503
4504        replies.resolve(7, ActionOutcome::accepted());
4505        assert_eq!(answer.await.unwrap(), ActionOutcome::accepted());
4506    }
4507
4508    #[tokio::test]
4509    async fn a_new_action_that_never_publishes_still_answers_its_phone() {
4510        let mut replies = PendingActionReplies::default();
4511        let (reply, answer) = tokio::sync::oneshot::channel();
4512        replies.accept(7, &new_action(), reply);
4513
4514        // Registration failed before the session reached the loop, which is
4515        // the completion path rather than the publication path.
4516        replies.resolve(7, ActionOutcome::Failed);
4517
4518        assert_eq!(answer.await.unwrap(), ActionOutcome::Failed);
4519        // A second resolution is a no-op, so a completion after a publication
4520        // cannot overwrite the answer already sent.
4521        replies.resolve(7, ActionOutcome::accepted());
4522    }
4523
4524    #[test]
4525    fn force_close_is_admitted_like_close() {
4526        let mut active = std::collections::BTreeSet::from(["session-1".to_owned()]);
4527        let force_close = ControllerAction::ForceClose {
4528            session_id: "session-1".into(),
4529        };
4530        // A full action pool and a session already busy with a stuck close
4531        // are both exactly when a force close has to get through.
4532        assert_eq!(
4533            admit_phone_action(&force_close, MAX_CONCURRENT_PHONE_ACTIONS, &mut active),
4534            Ok(Some("session-1".into()))
4535        );
4536        assert_eq!(
4537            controller_action_session_id(&force_close),
4538            Some("session-1".to_owned())
4539        );
4540    }
4541
4542    #[test]
4543    fn close_is_admitted_while_provisioning_occupies_a_full_action_pool() {
4544        let mut active = std::collections::BTreeSet::from(["session-1".to_owned()]);
4545        let close = ControllerAction::Close {
4546            session_id: "session-1".into(),
4547        };
4548        assert_eq!(
4549            admit_phone_action(&close, MAX_CONCURRENT_PHONE_ACTIONS, &mut active),
4550            Ok(Some("session-1".into()))
4551        );
4552        assert_eq!(
4553            admit_phone_action(&prompt_action(), 0, &mut active),
4554            Err(ActionOutcome::SessionBusy)
4555        );
4556    }
4557
4558    #[test]
4559    fn a_refused_action_reports_the_reason_the_phone_can_act_on() {
4560        let mut active = std::collections::BTreeSet::new();
4561
4562        assert_eq!(
4563            admit_phone_action(&prompt_action(), 0, &mut active),
4564            Ok(Some("session-1".to_owned()))
4565        );
4566        assert_eq!(
4567            admit_phone_action(&prompt_action(), 1, &mut active),
4568            Err(ActionOutcome::SessionBusy)
4569        );
4570        assert_eq!(
4571            admit_phone_action(&new_action(), MAX_CONCURRENT_PHONE_ACTIONS, &mut active),
4572            Err(ActionOutcome::Busy)
4573        );
4574        // A refusal must not consume the session slot it did not take.
4575        assert_eq!(active.len(), 1);
4576        assert_eq!(admit_phone_action(&new_action(), 1, &mut active), Ok(None));
4577    }
4578
4579    #[test]
4580    fn a_feed_that_ends_outside_shutdown_names_the_failure() {
4581        assert!(feed_stopped(true, "the session manager stopped").is_none());
4582        let failure = feed_stopped(false, "the session manager stopped").expect("named failure");
4583        assert!(failure.to_string().contains("session manager"));
4584    }
4585
4586    #[test]
4587    fn a_profile_added_while_the_server_runs_reaches_the_quota_refresher() {
4588        let (profiles_tx, profiles_rx) = tokio::sync::watch::channel(QuotaRefreshBatch::default());
4589        let mut published = std::collections::BTreeMap::new();
4590        let mut batch = QuotaRefreshBatch::default();
4591        let controller = controller_with_profiles(&["codex"]);
4592
4593        assert!(republish_quota_profiles(
4594            &controller,
4595            &mut published,
4596            &mut batch,
4597            &profiles_tx
4598        ));
4599        assert_eq!(
4600            profiles_rx
4601                .borrow()
4602                .profiles
4603                .iter()
4604                .map(|profile| profile.profile_id.clone())
4605                .collect::<Vec<_>>(),
4606            vec!["codex".to_owned()]
4607        );
4608        let first_generation = profiles_rx.borrow().generation;
4609
4610        // Every finished action reloads the configuration; an unchanged one
4611        // must not restart a harness process per profile.
4612        assert!(!republish_quota_profiles(
4613            &controller,
4614            &mut published,
4615            &mut batch,
4616            &profiles_tx
4617        ));
4618        assert_eq!(profiles_rx.borrow().generation, first_generation);
4619
4620        let grown = controller_with_profiles(&["claude", "codex"]);
4621        assert!(republish_quota_profiles(
4622            &grown,
4623            &mut published,
4624            &mut batch,
4625            &profiles_tx
4626        ));
4627        assert_eq!(
4628            profiles_rx
4629                .borrow()
4630                .profiles
4631                .iter()
4632                .map(|profile| profile.profile_id.clone())
4633                .collect::<Vec<_>>(),
4634            vec!["claude".to_owned(), "codex".to_owned()]
4635        );
4636        assert!(profiles_rx.borrow().generation > first_generation);
4637    }
4638
4639    #[test]
4640    fn a_quota_reads_stale_only_once_its_next_refresh_is_overdue() {
4641        let controller = controller_with_profiles(&["codex"]);
4642        let now = std::time::SystemTime::now()
4643            .duration_since(std::time::UNIX_EPOCH)
4644            .unwrap()
4645            .as_secs();
4646        let quota_refreshed = |age: Duration| {
4647            let quotas = std::collections::BTreeMap::from([(
4648                "codex".to_owned(),
4649                ProfileQuota {
4650                    profile_id: "codex".into(),
4651                    harness: HarnessKind::Codex,
4652                    windows: Vec::new(),
4653                    extra: None,
4654                    error: None,
4655                    refreshed_at_epoch_seconds: now - age.as_secs(),
4656                },
4657            )]);
4658            viewer_snapshot(
4659                &controller,
4660                &[],
4661                &quotas,
4662                &PhoneSessionViews {
4663                    conversations: &std::collections::BTreeMap::new(),
4664                    queued_prompts: &std::collections::BTreeMap::new(),
4665                    active_user_shells: &std::collections::BTreeMap::new(),
4666                    pending_elicitations: &std::collections::BTreeMap::new(),
4667                    prompt_images: &std::collections::BTreeSet::new(),
4668                    operational: &std::collections::BTreeMap::new(),
4669                    materialized_activity: &std::collections::BTreeMap::new(),
4670                    project_sources: &PhoneProjectSources::default(),
4671                    operations: &std::collections::BTreeMap::new(),
4672                    move_recoveries: &std::collections::BTreeMap::new(),
4673                    capacity: &[],
4674                    launch_failures: &[],
4675                    reviews: &std::collections::BTreeMap::new(),
4676                },
4677                1,
4678            )
4679            .profiles[0]
4680                .quota
4681                .as_ref()
4682                .expect("the profile carries its quota")
4683                .stale
4684        };
4685
4686        // A reading taken one refresh interval ago is exactly what a healthy
4687        // refresher produces, so it must not be labelled stale.
4688        assert!(!quota_refreshed(QUOTA_REFRESH_INTERVAL));
4689        assert!(!quota_refreshed(QUOTA_STALE_AFTER));
4690        assert!(quota_refreshed(QUOTA_STALE_AFTER + Duration::from_secs(1)));
4691    }
4692
4693    #[test]
4694    fn phone_action_capacity_is_bounded() {
4695        assert!(phone_action_capacity_available(
4696            MAX_CONCURRENT_PHONE_ACTIONS - 1
4697        ));
4698        assert!(!phone_action_capacity_available(
4699            MAX_CONCURRENT_PHONE_ACTIONS
4700        ));
4701    }
4702
4703    #[test]
4704    fn started_phone_session_is_visible_and_mapped_before_provisioning() {
4705        let session_id = "0123456789abcdef0123456789abcdef";
4706        let session = phone_session(session_id, 0);
4707        let mut state = State::default();
4708        let mut active_actions = std::collections::BTreeSet::new();
4709        let mut action_sessions = std::collections::BTreeMap::new();
4710
4711        track_started_phone_session(
4712            &mut state,
4713            &mut active_actions,
4714            &mut action_sessions,
4715            7,
4716            session,
4717        )
4718        .unwrap();
4719
4720        assert_eq!(state.sessions[session_id].state, SessionState::Provisioning);
4721        assert_eq!(state.sessions[session_id].display_title(), "Phone launch");
4722        assert!(active_actions.contains(session_id));
4723        assert_eq!(
4724            action_sessions.get(&7).map(String::as_str),
4725            Some(session_id)
4726        );
4727    }
4728
4729    #[test]
4730    fn failed_launch_notice_survives_session_rollback_and_history_is_bounded() {
4731        let controller = controller_with_profiles(&["codex"]);
4732        let mut failures = Vec::new();
4733        for index in 0..20 {
4734            record_launch_failure(
4735                &mut failures,
4736                index,
4737                format!("workspace-{index}"),
4738                Some(format!("session-{index}")),
4739            );
4740        }
4741        let snapshot = viewer_snapshot(
4742            &controller,
4743            &[],
4744            &std::collections::BTreeMap::new(),
4745            &PhoneSessionViews {
4746                conversations: &std::collections::BTreeMap::new(),
4747                queued_prompts: &std::collections::BTreeMap::new(),
4748                active_user_shells: &std::collections::BTreeMap::new(),
4749                pending_elicitations: &std::collections::BTreeMap::new(),
4750                prompt_images: &std::collections::BTreeSet::new(),
4751                operational: &std::collections::BTreeMap::new(),
4752                materialized_activity: &std::collections::BTreeMap::new(),
4753                project_sources: &PhoneProjectSources::default(),
4754                operations: &std::collections::BTreeMap::new(),
4755                move_recoveries: &std::collections::BTreeMap::new(),
4756                capacity: &[],
4757                launch_failures: &failures,
4758                reviews: &std::collections::BTreeMap::new(),
4759            },
4760            1,
4761        );
4762        assert!(snapshot.sessions.is_empty());
4763        assert_eq!(failures.len(), 16);
4764        assert_eq!(failures[0].workspace_id, "workspace-4");
4765        assert_eq!(failures[15].workspace_id, "workspace-19");
4766        assert_ne!(failures[0].id, failures[1].id);
4767        assert_eq!(
4768            failures[15].session_id.as_deref(),
4769            Some("session-19"),
4770            "a wait on that session has to be able to recognize its own launch failure"
4771        );
4772        let json = serde_json::to_value(snapshot).unwrap();
4773        assert_eq!(json["launch_failures"][15]["workspace_id"], "workspace-19");
4774        assert_eq!(json["launch_failures"][15].as_object().unwrap().len(), 3);
4775    }
4776
4777    #[test]
4778    fn a_later_successful_action_clears_a_session_s_recorded_failure() {
4779        let mut pending = std::collections::BTreeMap::new();
4780
4781        record_action_result(&mut pending, Some("session-1"), &Err("relay hiccup".into()));
4782        assert_eq!(
4783            pending.get("session-1").map(String::as_str),
4784            Some("relay hiccup")
4785        );
4786
4787        record_action_result(&mut pending, Some("session-2"), &Ok(()));
4788        record_action_result(&mut pending, Some("session-1"), &Ok(()));
4789        assert!(
4790            pending.is_empty(),
4791            "the overlay has no other expiry, so a stale error would badge the session forever"
4792        );
4793
4794        // A completion with no session cannot clear or record anything.
4795        record_action_result(&mut pending, None, &Err("orphaned".into()));
4796        assert!(pending.is_empty());
4797    }
4798
4799    #[test]
4800    fn phone_cancel_targets_the_matching_background_action() {
4801        let first = PhoneActionControl {
4802            cancelled: Arc::new(AtomicBool::new(false)),
4803            create: None,
4804        };
4805        let second = PhoneActionControl {
4806            cancelled: Arc::new(AtomicBool::new(false)),
4807            create: None,
4808        };
4809        let action_sessions =
4810            std::collections::BTreeMap::from([(1, "session-1".into()), (2, "session-2".into())]);
4811        let cancellations =
4812            std::collections::BTreeMap::from([(1, first.clone()), (2, second.clone())]);
4813
4814        assert!(request_phone_action_cancellation(
4815            "session-2",
4816            &action_sessions,
4817            &cancellations,
4818        ));
4819        assert!(!first.cancelled.load(Ordering::Acquire));
4820        assert!(second.cancelled.load(Ordering::Acquire));
4821        assert!(!request_phone_action_cancellation(
4822            "missing",
4823            &action_sessions,
4824            &cancellations,
4825        ));
4826    }
4827
4828    #[test]
4829    fn phone_new_cancel_and_running_commit_have_one_atomic_winner() {
4830        for _ in 0..100 {
4831            let create = CreateSessionControl::default();
4832            let control = PhoneActionControl {
4833                cancelled: create.cancelled.clone(),
4834                create: Some(create),
4835            };
4836            let cancelling = control.clone();
4837            let committing = control.clone();
4838            let (cancelled, committed) = std::thread::scope(|scope| {
4839                let cancel = scope.spawn(move || cancelling.request_cancel());
4840                let commit = scope.spawn(move || committing.grant_new_commit());
4841                (cancel.join().unwrap(), commit.join().unwrap())
4842            });
4843
4844            assert_ne!(cancelled, committed);
4845            assert_eq!(control.cancelled.load(Ordering::Acquire), cancelled);
4846            assert!(!control.request_cancel());
4847            assert!(!control.grant_new_commit());
4848        }
4849    }
4850
4851    fn materialized_at(ordinal: u64) -> MaterializedSession {
4852        let mut materialized = MaterializedSession::empty("session-1");
4853        materialized.applied_event_ordinal = ordinal;
4854        materialized.applied_event_digest = format!("digest-{ordinal}");
4855        materialized
4856    }
4857
4858    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
4859    async fn browser_projection_enqueue_does_not_wait_for_a_blocking_permit() {
4860        let (results, mut completed) = tokio::sync::mpsc::channel(1);
4861        let shutdown = tokio_util::sync::CancellationToken::new();
4862        let mut dispatcher = ConversationProjectionDispatcher::with_permits(results, shutdown, 0);
4863
4864        dispatcher.enqueue(materialized_at(1));
4865        let (progress, progress_done) = tokio::sync::oneshot::channel();
4866        tokio::spawn(async move {
4867            progress.send(()).expect("control task is still alive");
4868        });
4869
4870        tokio::time::timeout(Duration::from_millis(100), progress_done)
4871            .await
4872            .expect("control progress was starved by transcript projection")
4873            .expect("progress task did not report");
4874        assert!(completed.try_recv().is_err());
4875    }
4876
4877    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
4878    async fn browser_projection_keeps_only_the_newest_pending_snapshot() {
4879        let (results, mut completed) = tokio::sync::mpsc::channel(4);
4880        let shutdown = tokio_util::sync::CancellationToken::new();
4881        let mut dispatcher = ConversationProjectionDispatcher::with_permits(results, shutdown, 0);
4882
4883        dispatcher.enqueue(materialized_at(1));
4884        dispatcher.enqueue(materialized_at(3));
4885        dispatcher.enqueue(materialized_at(2));
4886        assert_eq!(dispatcher.pending["session-1"].key.ordinal, 3);
4887
4888        dispatcher.permits.add_permits(1);
4889        let first = tokio::time::timeout(Duration::from_secs(1), completed.recv())
4890            .await
4891            .expect("first projection did not complete")
4892            .expect("projection result channel closed");
4893        assert_eq!(first.key.ordinal, 1);
4894        assert!(dispatcher.finish(first, true).is_some());
4895
4896        let second = tokio::time::timeout(Duration::from_secs(1), completed.recv())
4897            .await
4898            .expect("coalesced projection did not complete")
4899            .expect("projection result channel closed");
4900        assert_eq!(second.key.ordinal, 3);
4901        assert!(dispatcher.finish(second, true).is_some());
4902        assert!(dispatcher.pending.is_empty());
4903        assert!(dispatcher.in_flight.is_empty());
4904    }
4905
4906    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
4907    async fn forgotten_projection_cannot_repopulate_after_a_same_cursor_resume() {
4908        let (results, mut completed) = tokio::sync::mpsc::channel(4);
4909        let shutdown = tokio_util::sync::CancellationToken::new();
4910        let mut dispatcher = ConversationProjectionDispatcher::with_permits(results, shutdown, 0);
4911        let snapshot = materialized_at(7);
4912
4913        dispatcher.enqueue(snapshot.clone());
4914        dispatcher.forget("session-1");
4915        dispatcher.enqueue(snapshot);
4916        dispatcher.permits.add_permits(1);
4917
4918        let old = tokio::time::timeout(Duration::from_secs(1), completed.recv())
4919            .await
4920            .expect("old projection did not complete")
4921            .expect("projection result channel closed");
4922        assert!(dispatcher.finish(old, true).is_none());
4923
4924        let current = tokio::time::timeout(Duration::from_secs(1), completed.recv())
4925            .await
4926            .expect("resumed projection did not complete")
4927            .expect("projection result channel closed");
4928        assert!(dispatcher.finish(current, true).is_some());
4929        assert!(dispatcher.in_flight.is_empty());
4930    }
4931}