Skip to main content

sapphire_framework_server/sync/
mod.rs

1//! Replication, from the app server's side.
2//!
3//! One `Replica` per synced workspace, registered with the bridge, driven by announcements
4//! from it and by local edits.
5//!
6//! See `docs/superpowers/specs/2026-09-16-process-architecture-design.md` §4.2.
7
8use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use grain_id::GrainId;
13use sapphire_backend::protocol as proto;
14use sapphire_bridge_api::{
15    IncomingParams, RegisterParams, WorkspaceRegistration, WorkspacesResult,
16};
17use sapphire_bridge_client::BridgeClient;
18use sapphire_ipc::ManagedBy;
19use sapphire_sync::{PathUpdate, PauseReason, Replica, ReplicaConfig, ScanOutcome, SystemClock};
20use sapphire_workspace::{AppContext, Workspace};
21use tokio::sync::{Mutex, OnceCell, broadcast, mpsc};
22
23pub mod id;
24mod live;
25mod methods;
26#[cfg(any(test, feature = "test-util"))]
27pub mod testing;
28mod topology;
29mod watch;
30
31pub use id::{SYNC_ID_FILE, WORKSPACE_MAP_FILE, sync_id, sync_id_path};
32pub use live::{DIAL_BACKOFF_MAX, OPEN_TIMEOUT};
33pub use methods::sync_router;
34/// What [`SyncRuntime::peers`] answers with.
35pub use sapphire_bridge_api::{PeerInfo, PeersResult};
36pub use watch::{DEBOUNCE, Watcher};
37
38use live::{LivePeers, dial_backoff};
39
40use crate::error::{Error, Result};
41
42use crate::host::WorkspaceHost;
43
44/// What `sync.status` reports.
45#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
46pub struct SyncStatus {
47    /// Whether this workspace is synced at all.
48    pub enabled: bool,
49    /// Its identity across devices, when it is.
50    pub workspace_id: Option<GrainId>,
51    /// How many devices the workgroup has, this host excluded.
52    pub peers: usize,
53    /// Why replication is paused, if it is — a missing root or marker directory.
54    pub paused: Option<String>,
55    /// The last failure, if any.
56    pub last_error: Option<String>,
57    /// Whether the bridge is reachable. `false` is not an outage: the app server works.
58    pub bridge_available: bool,
59    /// How the workspace is wired to its peers, by the roles the bridge last reported.
60    pub topology: proto::Topology,
61    /// How far embedding has got, when this host embeds (#188).
62    pub embedding: Option<proto::EmbeddingProgress>,
63}
64
65/// One workspace this runtime syncs.
66struct Synced {
67    /// Its identity across devices, shared with every peer.
68    workspace_id: GrainId,
69    /// Its replica store. Held open because the store is a redb database.
70    replica: Arc<Mutex<Replica>>,
71    /// Why replication is paused, from the last scan.
72    paused: Option<PauseReason>,
73    /// The last failure, if any.
74    last_error: Option<String>,
75}
76
77/// How old a vector file no live content needs must be before the primary device removes
78/// it: a vector can arrive before its file, and a day is far longer than that window.
79const STALE_VECTOR_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
80
81/// Texts per embedding request in a server pass: small, so a search query waiting on the
82/// bridge's one worker is never stuck behind a long backfill (#188).
83const PASS_BATCH: usize = 4;
84
85/// How the primary device paces its backfill (#188).
86#[derive(Clone, Copy, Debug)]
87pub struct BackfillTiming {
88    /// How old another device's version must be before the primary embeds it: its author
89    /// has embedded it by then, or will not.
90    pub grace: std::time::Duration,
91    /// How often the primary device runs a pass with nothing prompting it.
92    pub interval: std::time::Duration,
93}
94
95impl Default for BackfillTiming {
96    fn default() -> Self {
97        BackfillTiming {
98            grace: std::time::Duration::from_secs(10 * 60),
99            interval: std::time::Duration::from_secs(10 * 60),
100        }
101    }
102}
103
104/// How long [`SyncRuntime::keep_connected`] waits before its first retry after the bridge
105/// could not be reached. Each failure doubles it, up to [`RECONNECT_BACKOFF_MAX`].
106pub const RECONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_millis(500);
107
108/// The longest [`SyncRuntime::keep_connected`] waits between two attempts to reach the
109/// bridge: a bridge that comes up is found within this long.
110pub const RECONNECT_BACKOFF_MAX: std::time::Duration = std::time::Duration::from_secs(30);
111
112/// The wait before reconnection attempt `attempt` (0-based).
113fn reconnect_backoff(attempt: u32) -> std::time::Duration {
114    RECONNECT_BACKOFF
115        .saturating_mul(1u32 << attempt.min(16))
116        .min(RECONNECT_BACKOFF_MAX)
117}
118
119/// What a connection attempt to the bridge yields.
120type Connecting = std::pin::Pin<
121    Box<dyn std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>> + Send>,
122>;
123
124/// How a [`SyncRuntime`] reaches the bridge, on start and whenever it has lost it (#244).
125///
126/// An attempt answers `Ok(None)` when no bridge is running — the runtime then tries again
127/// later — and an error for anything else, such as a bridge of another API version.
128#[derive(Clone)]
129pub struct BridgeConnector(Arc<dyn Fn() -> Connecting + Send + Sync>);
130
131impl BridgeConnector {
132    /// Connect with `connect`.
133    pub fn new<F, Fut>(connect: F) -> BridgeConnector
134    where
135        F: Fn() -> Fut + Send + Sync + 'static,
136        Fut: std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>>
137            + Send
138            + 'static,
139    {
140        BridgeConnector(Arc::new(move || Box::pin(connect())))
141    }
142
143    /// The host's own bridge, at its standard endpoint, as a client of `kind` at
144    /// `version`.
145    pub fn standard(kind: &'static str, version: &'static str) -> BridgeConnector {
146        BridgeConnector::new(move || async move {
147            match BridgeClient::connect(kind, version).await {
148                Ok(client) => Ok(Some(client)),
149                Err(sapphire_ipc::Error::NotRunning(_)) => Ok(None),
150                Err(err) => Err(err),
151            }
152        })
153    }
154
155    /// The bridge listening at `endpoint`: for a fixture that runs its own.
156    pub fn at(
157        endpoint: sapphire_ipc::Endpoint,
158        kind: &'static str,
159        version: &'static str,
160    ) -> BridgeConnector {
161        BridgeConnector::new(move || {
162            let endpoint = endpoint.clone();
163            async move { BridgeClient::connect_at(&endpoint, kind, version).await }
164        })
165    }
166
167    async fn connect(&self) -> sapphire_ipc::Result<Option<BridgeClient>> {
168        (self.0)().await
169    }
170}
171
172impl std::fmt::Debug for BridgeConnector {
173    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
174        f.write_str("BridgeConnector")
175    }
176}
177
178/// What [`SyncRuntime::reclaim`] found of a replica `disable` let go of.
179enum Reclaimed {
180    /// Still held: here it is.
181    Open(Arc<Mutex<Replica>>),
182    /// Let go of since, so its store may be closing still.
183    Closing,
184    /// Nothing to take back.
185    Closed,
186}
187
188/// How often, and how far apart, `enable` tries to open a store that may still be closing.
189const CLOSING_RETRIES: u32 = 50;
190const CLOSING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(20);
191
192/// Replication for one application's workspaces.
193pub struct SyncRuntime {
194    ctx: &'static AppContext,
195    /// The bridge connection, while there is one. Swapped when the runtime reconnects
196    /// (#244); [`run`](SyncRuntime::run) follows it.
197    bridge: tokio::sync::watch::Sender<Option<Arc<BridgeClient>>>,
198    /// How to reach the bridge again; `None` for a runtime given one connection to use.
199    connector: Option<BridgeConnector>,
200    /// Workspaces asked to sync while the bridge was away, with their ids. Enabled once
201    /// it is back.
202    ///
203    /// Also the lock that orders an `enable` against a reconnection: whoever holds it sees
204    /// the bridge slot as the other left it, so a workspace is either enabled now or left
205    /// here for the reconnection to enable — never neither.
206    pending: Mutex<HashMap<PathBuf, GrainId>>,
207    /// The announcements of a connection [`attach`](SyncRuntime::attach) just made,
208    /// subscribed to before it registered: the bridge announces the streams that waited for
209    /// this server as soon as it registers (#244), and a subscription made afterwards by
210    /// [`run`](SyncRuntime::run) would miss them.
211    subscribed: std::sync::Mutex<Option<(Arc<BridgeClient>, broadcast::Receiver<IncomingParams>)>>,
212    exe_path: PathBuf,
213    managed_by: ManagedBy,
214    synced: Mutex<HashMap<PathBuf, Synced>>,
215    /// The replicas [`disable`](SyncRuntime::disable) let go of, by canonical root, for as
216    /// long as something else still holds one: a scan, a session or the first dial that
217    /// began before. The store is a redb database, open until its last holder drops it, so
218    /// a re-enable meanwhile could not open it again; [`enable`](SyncRuntime::enable)
219    /// takes the open one back instead. [`release`](SyncRuntime::release) puts nothing
220    /// here: its store is about to be moved or deleted.
221    stopped: std::sync::Mutex<HashMap<PathBuf, std::sync::Weak<Mutex<Replica>>>>,
222    /// This host's device id. Asked of the bridge once, on the first [`enable`], and never
223    /// again: a second registration with an empty workspace list would briefly clear this
224    /// application's routes.
225    ///
226    /// [`enable`]: SyncRuntime::enable
227    device_id: OnceCell<GrainId>,
228    /// The file watcher, once [`watch_changes`] has started it.
229    ///
230    /// The runtime owns it because the spec puts it here: the sync runtime is a watcher and
231    /// a replica per registered root, and the two must agree on which roots are synced.
232    /// [`enable`] and [`disable`] keep them in step.
233    ///
234    /// [`watch_changes`]: SyncRuntime::watch_changes
235    /// [`enable`]: SyncRuntime::enable
236    /// [`disable`]: SyncRuntime::disable
237    watcher: OnceCell<Arc<Watcher>>,
238    /// The app server that owns these workspaces' caches, once it has told us.
239    ///
240    /// A replication session writes files, and the search index is not part of a session: the
241    /// index belongs to the app server, which holds the workspace open. Reaching it through
242    /// the host is what keeps a received file and its index moving together, and it is the
243    /// *same* [`WorkspaceState`](sapphire_workspace::WorkspaceState) the server serves from —
244    /// opening a second one would collide on the retrieve store's exclusive lock.
245    ///
246    /// Absent for a runtime nobody has wired to an app server (a test stub driving the
247    /// replica directly). Nothing then indexes, which is correct: nothing owns an index.
248    host: OnceCell<Arc<WorkspaceHost>>,
249    /// Serialises re-indexing, so two sessions finishing at once do not both sweep the
250    /// workspace through the same store.
251    reindexing: Mutex<()>,
252    /// Roots with an embedding pass waiting to run (#187). A request for a root already
253    /// waiting is folded into that pass.
254    embed_wanted: Mutex<std::collections::HashSet<PathBuf>>,
255    /// Serialises embedding passes: one model, one queue.
256    embedding: Mutex<()>,
257    /// Roots whose embedding pass is running now, for `sync.status`.
258    embed_running: Mutex<std::collections::HashSet<PathBuf>>,
259    /// The primary device's backfill pacing (#188).
260    backfill: std::sync::Mutex<BackfillTiming>,
261    /// The bridge an embedding pass asks for its model; `None` is the host's standard one.
262    bridge_endpoint: Option<sapphire_ipc::Endpoint>,
263    /// The live session table of every synced workspace, keyed by canonical root.
264    ///
265    /// One table per workspace rather than one for the runtime: a session is about one
266    /// workspace, and the two storm rules are per-workspace rules. A root that is not
267    /// synced has none, so a dial or a push for it is a no-op rather than an error.
268    live: Mutex<HashMap<PathBuf, Arc<LivePeers>>>,
269    /// The bridge's last answer to `peers`, roles included. The inbound guard and
270    /// [`is_primary`](SyncRuntime::is_primary) read it rather than asking again.
271    last_peers: Mutex<Option<PeersResult>>,
272    /// How long a dial waits for the bridge to open a stream; [`OPEN_TIMEOUT`] by default.
273    open_timeout: std::time::Duration,
274    /// The first dials [`enable`](SyncRuntime::enable) leaves running. Each pins the
275    /// runtime, so [`stop_dials`](SyncRuntime::stop_dials) ends them with the server.
276    first_dials: std::sync::Mutex<tokio::task::JoinSet<()>>,
277    /// The rest of the runtime's background work: inbound sessions, the readers of live
278    /// sessions, and embedding passes. Each pins the runtime — and through it the replica
279    /// stores, which are redb databases locked until the last pin drops — so
280    /// [`stop_background`](SyncRuntime::stop_background) ends them with the server (#235).
281    background: std::sync::Mutex<tokio::task::JoinSet<()>>,
282}
283
284impl SyncRuntime {
285    /// A runtime for `ctx`'s application, syncing through `bridge`.
286    ///
287    /// It has no way to reach the bridge again: when the connection closes, sync stops until
288    /// the runtime is given a [`connector`](Self::with_connector).
289    pub fn new(
290        ctx: &'static AppContext,
291        bridge: Arc<BridgeClient>,
292        exe_path: PathBuf,
293        managed_by: ManagedBy,
294    ) -> SyncRuntime {
295        SyncRuntime::build(ctx, Some(bridge), None, exe_path, managed_by)
296    }
297
298    /// A runtime for `ctx`'s application that reaches the bridge with `connector`, and again
299    /// whenever it loses it (#244).
300    ///
301    /// Not connected yet: [`connect`](Self::connect) makes the first attempt, and
302    /// [`keep_connected`](Self::keep_connected) the rest. [`AppServer::run`] does both.
303    ///
304    /// [`AppServer::run`]: crate::AppServer::run
305    pub fn connecting(
306        ctx: &'static AppContext,
307        connector: BridgeConnector,
308        exe_path: PathBuf,
309        managed_by: ManagedBy,
310    ) -> SyncRuntime {
311        SyncRuntime::build(ctx, None, Some(connector), exe_path, managed_by)
312    }
313
314    /// Reach the bridge again with `connector` once the connection closes.
315    pub fn with_connector(mut self, connector: BridgeConnector) -> SyncRuntime {
316        self.connector = Some(connector);
317        self
318    }
319
320    fn build(
321        ctx: &'static AppContext,
322        bridge: Option<Arc<BridgeClient>>,
323        connector: Option<BridgeConnector>,
324        exe_path: PathBuf,
325        managed_by: ManagedBy,
326    ) -> SyncRuntime {
327        SyncRuntime {
328            ctx,
329            bridge: tokio::sync::watch::Sender::new(bridge),
330            connector,
331            pending: Mutex::new(HashMap::new()),
332            subscribed: std::sync::Mutex::new(None),
333            exe_path,
334            managed_by,
335            synced: Mutex::new(HashMap::new()),
336            stopped: std::sync::Mutex::new(HashMap::new()),
337            device_id: OnceCell::new(),
338            watcher: OnceCell::new(),
339            host: OnceCell::new(),
340            reindexing: Mutex::new(()),
341            embed_wanted: Mutex::new(std::collections::HashSet::new()),
342            embedding: Mutex::new(()),
343            embed_running: Mutex::new(std::collections::HashSet::new()),
344            backfill: std::sync::Mutex::new(BackfillTiming::default()),
345            bridge_endpoint: None,
346            live: Mutex::new(HashMap::new()),
347            last_peers: Mutex::new(None),
348            open_timeout: OPEN_TIMEOUT,
349            first_dials: std::sync::Mutex::new(tokio::task::JoinSet::new()),
350            background: std::sync::Mutex::new(tokio::task::JoinSet::new()),
351        }
352    }
353
354    /// The bridge, while connected.
355    fn bridge(&self) -> Result<Arc<BridgeClient>> {
356        match &*self.bridge.borrow() {
357            Some(bridge) if !bridge.is_closed() => Ok(Arc::clone(bridge)),
358            _ => Err(Error::BridgeDisconnected),
359        }
360    }
361
362    /// Whether the runtime has a live connection to the bridge.
363    pub fn is_connected(&self) -> bool {
364        self.bridge().is_ok()
365    }
366
367    /// Run `task` in the background, tracked so the server's shutdown ends it.
368    fn spawn_background(&self, task: impl std::future::Future<Output = ()> + Send + 'static) {
369        let mut background = self.background.lock().expect("background tasks");
370        // Reap the finished ones, so the set holds only what is running.
371        while background.try_join_next().is_some() {}
372        background.spawn(task);
373    }
374
375    /// Give up on opening a stream to a peer after `timeout` rather than [`OPEN_TIMEOUT`]:
376    /// for tests, which cannot wait that long for a peer that never answers.
377    pub fn with_open_timeout(mut self, timeout: std::time::Duration) -> SyncRuntime {
378        self.open_timeout = timeout;
379        self
380    }
381
382    /// Tell the runtime which app server owns these workspaces' caches.
383    ///
384    /// Called by [`AppServer::sync`](crate::AppServer::sync). First writer wins: a runtime
385    /// belongs to the server that wired it.
386    /// Embed through the bridge at `endpoint` rather than the host's standard one: for a
387    /// fixture running several hosts in one process.
388    pub fn with_bridge_endpoint(mut self, endpoint: sapphire_ipc::Endpoint) -> SyncRuntime {
389        self.bridge_endpoint = Some(endpoint);
390        self
391    }
392
393    /// Pace the primary device's backfill differently: for tests, which cannot wait ten
394    /// minutes.
395    pub fn set_backfill_timing(&self, timing: BackfillTiming) {
396        *self.backfill.lock().expect("backfill timing") = timing;
397    }
398
399    fn backfill_timing(&self) -> BackfillTiming {
400        *self.backfill.lock().expect("backfill timing")
401    }
402
403    pub(crate) fn set_host(&self, host: Arc<WorkspaceHost>) {
404        let _ = self.host.set(host);
405    }
406
407    /// Start syncing `root`, returning its identity across devices.
408    ///
409    /// Idempotent: enabling an already-synced workspace returns the same id and changes
410    /// nothing.
411    pub async fn enable(self: &Arc<Self>, root: &Path) -> Result<GrainId> {
412        let key = root.canonicalize().map_err(Error::Io)?;
413        let watch_key = key.clone();
414        let workspace_id = sync_id(self.ctx.app_name, &key)?;
415
416        // Without a bridge there is no device id to open the replica with, and nobody to
417        // register with: keep the workspace for the reconnection to enable (#244).
418        {
419            let mut pending = self.pending.lock().await;
420            if self.bridge().is_err() {
421                pending.insert(key, workspace_id);
422                return Err(Error::BridgeDisconnected);
423            }
424            pending.remove(&key);
425        }
426
427        // The map stays locked across opening the replica, so two simultaneous `enable`s for
428        // one root cannot both open it — the store is a redb database, and it is opened once.
429        {
430            let mut synced = self.synced.lock().await;
431            if let Some(existing) = synced.get(&key) {
432                return Ok(existing.workspace_id);
433            }
434            // A replica disabled a moment ago may still be open; it is this root's, with this
435            // runtime's one device id, so it is taken back rather than opened a second time.
436            let replica = match self.reclaim(&key) {
437                Reclaimed::Open(replica) => replica,
438                reclaimed => {
439                    let device_id = self.device_id().await?;
440                    let state_dir = self.state_dir(&key)?;
441                    std::fs::create_dir_all(&state_dir).map_err(Error::Io)?;
442                    let config =
443                        ReplicaConfig::new(self.ctx.app_name, key.clone(), device_id, &state_dir);
444                    // The last holder let go between the two: its store closes as the replica
445                    // drops, just after it can no longer be taken back. Give it that moment.
446                    let mut tries = match reclaimed {
447                        Reclaimed::Closing => CLOSING_RETRIES,
448                        _ => 0,
449                    };
450                    let replica = loop {
451                        match Replica::open(config.clone(), Arc::new(SystemClock)) {
452                            Ok(replica) => break replica,
453                            Err(_) if tries > 0 => {
454                                tries -= 1;
455                                tokio::time::sleep(CLOSING_RETRY_DELAY).await;
456                            }
457                            Err(err) => return Err(Error::Sync(err.to_string())),
458                        }
459                    };
460                    Arc::new(Mutex::new(replica))
461                }
462            };
463            synced.insert(
464                key.clone(),
465                Synced {
466                    workspace_id,
467                    replica,
468                    paused: None,
469                    last_error: None,
470                },
471            );
472        }
473
474        // Watch the new root too, so an edit made outside the server from now on is seen.
475        // Not fatal if it fails: the app server's own writes take the exact path in
476        // `handlers.rs`, and the next `watch_changes` re-reads the root set anyway.
477        if let Some(watcher) = self.watcher.get()
478            && let Err(err) = watcher.watch(&watch_key)
479        {
480            tracing::warn!(root = %watch_key.display(), "could not watch: {err}");
481        }
482
483        // Register only once the replica is open, so a registration never names a workspace
484        // that cannot serve a session.
485        self.reregister().await?;
486
487        // Dial now, so enabling a workspace converges it rather than waiting for the next
488        // local edit to trigger a session. This is what catches a host up when it returns:
489        // the peer that has been running is the one the returning host has to ask, and no
490        // watcher on either side fires for a file that was never written locally.
491        //
492        // Best-effort: a workspace is enabled whether or not any peer is up, and a failure
493        // here is not a failure to enable. A session is symmetric, so dialing is enough —
494        // this side sends its own view, and the peer answers with what this side lacks.
495        //
496        // In the background, so the answer does not wait on it: a peer that is offline
497        // holds a dial up to `open_timeout`, longer than a client waits for `sync.enable`
498        // (#228).
499        //
500        // Scan first: a file already on disk when the workspace is enabled — a folder that
501        // existed before sync was turned on, or an edit made while the server was down —
502        // fires no watcher, so without this it is never recorded and the session has
503        // nothing to send. Logged, not fatal: the session still fetches what peers have.
504        let runtime = Arc::clone(self);
505        let mut dials = self.first_dials.lock().expect("first dials");
506        // Reap the finished ones, so the set holds only what is running.
507        while dials.try_join_next().is_some() {}
508        dials.spawn(async move {
509            if let Err(err) = runtime.scan(&key).await {
510                tracing::warn!(root = %key.display(), "the first scan after enabling failed: {err}");
511            }
512            if let Err(err) = runtime.sync_now(&key).await {
513                tracing::warn!(root = %key.display(), "the first session after enabling failed: {err}");
514            }
515        });
516        Ok(workspace_id)
517    }
518
519    /// Stop syncing `root`. Files and the sync id stay, so re-enabling rejoins the same
520    /// workspace rather than creating a second one.
521    ///
522    /// Something that began before — a scan, a session, the first dial — may still hold the
523    /// replica open; a re-enable meanwhile takes it back (see [`reclaim`](Self::reclaim)).
524    pub async fn disable(&self, root: &Path) -> Result<()> {
525        let (replica, unregistered) = self.stop(root).await;
526        if let Some(replica) = replica {
527            let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
528            let mut stopped = self.stopped.lock().expect("stopped replicas");
529            // Forget the ones nothing holds any more, so the map stays small.
530            stopped.retain(|_, replica| replica.strong_count() > 0);
531            stopped.insert(key, Arc::downgrade(&replica));
532        }
533        unregistered
534    }
535
536    /// Stop syncing `root`, like [`disable`](Self::disable), then wait up to `limit` until
537    /// nothing holds its replica store open any more — a session, a scan or an embedding
538    /// pass that started before may still have it. A store about to be moved or deleted
539    /// must be closed first.
540    ///
541    /// Only a store still in use at `limit` is an error. The bridge not hearing of it is
542    /// logged, not returned: the store is closed all the same, and the next registration
543    /// replaces the bridge's view.
544    pub async fn release(&self, root: &Path, limit: std::time::Duration) -> Result<()> {
545        let (replica, unregistered) = self.stop(root).await;
546        if let Err(err) = unregistered {
547            tracing::warn!(root = %root.display(), "stopping sync: {err}");
548        }
549        let Some(replica) = replica else {
550            return Ok(());
551        };
552        let deadline = tokio::time::Instant::now() + limit;
553        while Arc::strong_count(&replica) > 1 {
554            if tokio::time::Instant::now() >= deadline {
555                return Err(Error::Sync(format!(
556                    "the replica of {} is still in use; try again",
557                    root.display()
558                )));
559            }
560            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
561        }
562        Ok(())
563    }
564
565    /// Stop syncing `root`, handing back its replica — `None` when it was not synced — and
566    /// how telling the bridge went.
567    async fn stop(&self, root: &Path) -> (Option<Arc<Mutex<Replica>>>, Result<()>) {
568        // A root that no longer exists cannot be canonicalized, but the keys of `synced` are
569        // canonical already, so the path as given (a registry root) still names the entry.
570        let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
571        self.pending.lock().await.remove(&key);
572        let removed = self.synced.lock().await.remove(&key);
573        if removed.is_some()
574            && let Some(watcher) = self.watcher.get()
575        {
576            watcher.unwatch(&key);
577        }
578        // Any live session for a workspace this server no longer holds is closed: it would
579        // otherwise keep pushing files for a workspace nobody asked about any more.
580        if let Some(peers) = self.live.lock().await.remove(&key) {
581            peers.drop_connections().await;
582        }
583        let Some(removed) = removed else {
584            return (None, Ok(()));
585        };
586        let unregistered = async {
587            // A bridge that is gone has forgotten this server's registrations already, and
588            // the reconnection registers only what is still synced.
589            let Ok(bridge) = self.bridge() else {
590                return Ok(());
591            };
592            bridge
593                .unregister(removed.workspace_id)
594                .await
595                .map_err(|e| Error::Bridge(e.to_string()))?;
596            self.reregister().await
597        }
598        .await;
599        (Some(removed.replica), unregistered)
600    }
601
602    /// The workspaces the workgroup knows about.
603    ///
604    /// Read straight from the bridge, so it reflects every device's registrations. The
605    /// bridge being down is not an empty answer — that would read as "nothing to map" — so
606    /// it is an error, and `sync.map` reports it.
607    pub async fn workspaces(&self) -> Result<WorkspacesResult> {
608        self.bridge()?
609            .workspaces()
610            .await
611            .map_err(|e| Error::Bridge(e.to_string()))
612    }
613
614    /// Place a workgroup workspace at a directory of this host: write its identity and
615    /// map there, returning the canonical root. [`Current::map`](crate::Current::map)
616    /// then selects it and syncs it.
617    ///
618    /// `selector` is the workspace's name or id, as [`workspaces`] lists it; `dir` must
619    /// exist. Its marker is made when it has none, and nothing else: the marker's
620    /// `config.toml` is the application's, and arrives with the first sync — a local one
621    /// written first would meet it as a conflict (#253).
622    ///
623    /// [`workspaces`]: SyncRuntime::workspaces
624    pub async fn place(&self, selector: &str, dir: &Path) -> Result<PathBuf> {
625        let workgroup = self.workspaces().await?;
626        let wanted = workgroup
627            .workspaces
628            .iter()
629            // A name is the user-facing handle; the id is the exact one. Two workspaces
630            // may not share a name, so a name names at most one.
631            .find(|w| w.name == selector || w.workspace_id.to_string() == selector)
632            .ok_or_else(|| Error::UnknownWorkspaceName(selector.to_owned()))?;
633
634        // Refuse another application's workspace: this server syncs its own application's
635        // workspaces, and `Workspace::from_root` would later refuse the directory anyway.
636        if wanted.app_name != self.ctx.app_name {
637            return Err(Error::WrongApp {
638                name: wanted.name.clone(),
639                app_name: wanted.app_name.clone(),
640            });
641        }
642
643        let root = dir.canonicalize().map_err(Error::Io)?;
644        let marker = root.join(format!(".{}", self.ctx.app_name));
645        if !marker.is_dir() {
646            std::fs::create_dir(&marker).map_err(Error::Io)?;
647        }
648        let workspace = Workspace::from_root(self.ctx, &root)?;
649        if workspace.root != root {
650            return Err(Error::UnknownWorkspace(root, self.ctx.app_name));
651        }
652
653        // A directory already syncing under another identity is not remapped: the sync id
654        // is a device's word that it is the same workspace, and rewriting it silently
655        // would make two hosts disagree about what they share.
656        {
657            let synced = self.synced.lock().await;
658            if let Some(entry) = synced.get(&root)
659                && entry.workspace_id != wanted.workspace_id
660            {
661                return Err(Error::SyncId(format!(
662                    "{} already syncs as {}, not {}",
663                    root.display(),
664                    entry.workspace_id,
665                    wanted.workspace_id
666                )));
667            }
668        }
669
670        // The identity arrives from the workgroup, not from this host: two hosts that map
671        // the same workspace must sync as one, so the id is the one the workgroup lists,
672        // written before `enable` reads it. A different id already on disk is refused
673        // above; an absent one is written here.
674        let marker = workspace.marker_dir();
675        let id_path = marker.join(crate::sync::id::SYNC_ID_FILE);
676        match std::fs::read_to_string(&id_path) {
677            Ok(existing) if existing.trim() != wanted.workspace_id.to_string() => {
678                return Err(Error::SyncId(format!(
679                    "{} already holds identity {}, not {}",
680                    root.display(),
681                    existing.trim(),
682                    wanted.workspace_id
683                )));
684            }
685            Err(_) => {
686                std::fs::write(&id_path, format!("{}\n", wanted.workspace_id))
687                    .map_err(Error::Io)?;
688            }
689            Ok(_) => {}
690        }
691
692        // The map is what tells this application, on a later start, which workgroup
693        // workspace a directory is.
694        let map_path = root
695            .join(format!(".{}", self.ctx.app_name))
696            .join(WORKSPACE_MAP_FILE);
697        let workspace_id = wanted.workspace_id.to_string();
698        std::fs::write(&map_path, format!("{workspace_id}\n")).map_err(Error::Io)?;
699        Ok(root)
700    }
701
702    /// What `sync.status` answers with.
703    pub async fn status(&self, root: &Path) -> SyncStatus {
704        let bridge = self.bridge().ok();
705        let bridge_available = match &bridge {
706            Some(bridge) => bridge.status().await.is_ok(),
707            None => false,
708        };
709        let answer = match &bridge {
710            Some(_) => self.peers().await.ok(),
711            None => None,
712        };
713        let peers = answer
714            .as_ref()
715            .map_or(0, |p| p.peers.len().saturating_sub(1));
716        let Ok(key) = root.canonicalize() else {
717            return SyncStatus {
718                enabled: false,
719                workspace_id: None,
720                peers,
721                paused: None,
722                last_error: None,
723                bridge_available,
724                topology: proto::Topology::Mesh,
725                embedding: None,
726            };
727        };
728        let synced = self.synced.lock().await;
729        match synced.get(&key) {
730            Some(entry) => SyncStatus {
731                enabled: true,
732                workspace_id: Some(entry.workspace_id),
733                peers,
734                paused: entry.paused.map(|r| format!("{r:?}")),
735                last_error: entry.last_error.clone(),
736                bridge_available,
737                topology: topology::topology(
738                    answer
739                        .as_ref()
740                        .and_then(|p| p.roles_for(entry.workspace_id)),
741                ),
742                embedding: self.embedding_progress(&key).await,
743            },
744            None => {
745                drop(synced);
746                // Asked to sync while the bridge was away: it is enabled, waiting for the
747                // bridge, and `bridge_available` says why nothing moves.
748                let waiting = self.pending.lock().await.get(&key).copied();
749                SyncStatus {
750                    enabled: waiting.is_some(),
751                    workspace_id: waiting,
752                    peers,
753                    paused: None,
754                    last_error: None,
755                    bridge_available,
756                    topology: proto::Topology::Mesh,
757                    embedding: None,
758                }
759            }
760        }
761    }
762
763    /// How far `key`'s embedding has got, when this host embeds it. Reads the open
764    /// workspace's index; never opens one, so a status call stays cheap.
765    async fn embedding_progress(&self, key: &Path) -> Option<proto::EmbeddingProgress> {
766        let host = self.host.get()?;
767        let backend = host.open_backend(key)?;
768        let state = backend.state();
769        state.embedder()?;
770        let info = state.db_info().ok()?;
771        Some(proto::EmbeddingProgress {
772            vectors: info.vector_count,
773            pending: info.pending_count,
774            running: self.embed_running.lock().await.contains(key),
775        })
776    }
777
778    /// Bring the replica's view of the files up to date.
779    ///
780    /// Called straight after the app server's own writes, and by the watcher for everything
781    /// else. A scan that finds nothing is cheap; a scan that is skipped loses an edit until
782    /// the next one.
783    pub async fn scan(&self, root: &Path) -> Result<()> {
784        let Ok(key) = root.canonicalize() else {
785            return Ok(());
786        };
787        let replica = {
788            let synced = self.synced.lock().await;
789            match synced.get(&key) {
790                Some(entry) => Arc::clone(&entry.replica),
791                None => return Ok(()),
792            }
793        };
794        let (outcome, updates) = {
795            let mut replica = replica.lock().await;
796            let outcome = replica.scan().map_err(|e| Error::Sync(e.to_string()))?;
797            // What the scan committed, as path updates, so it can go out on the sessions
798            // that are already open. This is the whole point of the exact-path scan in
799            // `handlers.rs`: a write through the server reaches a peer live, not at the next
800            // dial.
801            let updates = match &outcome {
802                ScanOutcome::Scanned(report) => {
803                    // The unit of replication is the path state (design doc section 2.4):
804                    // the `seen` a peer prunes against is the *path's* version vector - the
805                    // context the recorded entry was built on plus its own dot - never the
806                    // whole replica vv. Sending the global vv here made receivers prune
807                    // causally-related versions they still held: a conflict copy on the
808                    // first remote edit, and tombstones pruned so deletes stopped
809                    // propagating (issue #157).
810                    report
811                        .recorded
812                        .iter()
813                        .map(|entry| PathUpdate {
814                            path: entry.path.clone(),
815                            versions: vec![entry.clone()],
816                            seen: {
817                                let mut seen = entry.context.clone();
818                                seen.add_dot(&entry.dot);
819                                seen
820                            },
821                        })
822                        .collect()
823                }
824                ScanOutcome::Paused(_) => Vec::new(),
825            };
826            (outcome, updates)
827        };
828        {
829            let mut synced = self.synced.lock().await;
830            if let Some(entry) = synced.get_mut(&key) {
831                entry.paused = match outcome {
832                    ScanOutcome::Paused(reason) => Some(reason),
833                    ScanOutcome::Scanned(_) => None,
834                };
835            }
836        }
837        // After the map is unlocked: a push awaits every peer, and the table must not be
838        // held behind a slow one.
839        self.after_commit(&key, updates).await;
840        Ok(())
841    }
842
843    /// Bring the search index up to date with files a session has written.
844    ///
845    /// A replication session moves files, not index rows: the index belongs to the app
846    /// server, and the sync core is deliberately free of it. So the side that *applied* a
847    /// change re-indexes, which is what makes a file that arrived from a peer searchable
848    /// without a manual reindex — the claim the whole architecture rests on.
849    ///
850    /// Incremental, not a full rebuild: the workspace's mtime/size stamps mean only what
851    /// actually changed is read back. A file the session wrote has a new stamp, so it is
852    /// picked up; one that was already indexed and untouched is skipped.
853    ///
854    /// Best-effort by contract: the caller's session has already succeeded, and a failure
855    /// here must not be reported as the session's. It is logged and the next session, local
856    /// edit or explicit `workspace.reindex` tries again.
857    async fn reindex(&self, root: &Path) {
858        let Some(host) = self.host.get() else {
859            return;
860        };
861        let host = Arc::clone(host);
862        let root = root.to_owned();
863        // One sweep at a time: two sessions that finish together would otherwise both walk
864        // the workspace through the same retrieve store.
865        let _serialised = self.reindexing.lock().await;
866        // Opening the workspace is itself blocking work, and it is where the exclusive
867        // retrieve-store lock is taken, so it happens here rather than inside the sweep.
868        let backend = match host.backend_at(&root).await {
869            Ok(backend) => backend,
870            Err(err) => {
871                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
872                return;
873            }
874        };
875        let state = Arc::clone(backend.state());
876        let result = tokio::task::spawn_blocking(move || state.sync_retrieve()).await;
877        match result {
878            Ok(Ok(report)) => {
879                tracing::debug!(root = %root.display(), ?report, "re-indexed after a session");
880            }
881            Ok(Err(err)) => {
882                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
883            }
884            Err(err) => {
885                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
886            }
887        }
888    }
889
890    /// Run an embedding pass over `root` in the background, after the pass already
891    /// waiting for it if there is one (#187).
892    ///
893    /// The pass reads the vector files that arrived, and computes the missing ones this
894    /// host is responsible for: the files whose winning version this replica wrote, and —
895    /// on the primary device — every other one. It never holds up the caller.
896    pub fn request_embed(self: &Arc<Self>, root: &Path) {
897        let Ok(key) = root.canonicalize() else {
898            return;
899        };
900        let runtime = Arc::clone(self);
901        self.spawn_background(async move {
902            if !runtime.embed_wanted.lock().await.insert(key.clone()) {
903                return;
904            }
905            let _one_at_a_time = runtime.embedding.lock().await;
906            // Taken off the list as the pass starts: a change during the pass asks again.
907            runtime.embed_wanted.lock().await.remove(&key);
908            runtime.embed_running.lock().await.insert(key.clone());
909            runtime.embed_pass(&key).await;
910            runtime.embed_running.lock().await.remove(&key);
911        });
912    }
913
914    /// One embedding pass over `key`. Best-effort, like the re-index: logged, never fatal.
915    async fn embed_pass(&self, key: &Path) {
916        let Some(host) = self.host.get() else {
917            return;
918        };
919        let (workspace_id, replica) = {
920            let synced = self.synced.lock().await;
921            match synced.get(key) {
922                Some(entry) => (entry.workspace_id, Arc::clone(&entry.replica)),
923                None => return,
924            }
925        };
926        let backend = match host.backend_at(key).await {
927            Ok(backend) => backend,
928            Err(err) => {
929                tracing::warn!(root = %key.display(), "embedding pass: {err}");
930                return;
931            }
932        };
933        let state = Arc::clone(backend.state());
934        if let Err(err) = state
935            .refresh_embedder_at(self.bridge_endpoint.clone())
936            .await
937        {
938            tracing::warn!(root = %key.display(), "embedding pass: {err}");
939            return;
940        }
941        if state.embedder().is_none() {
942            return;
943        }
944        let primary = self.is_primary(workspace_id).await;
945        // The replica's view, read once: which content each path has, and which of those
946        // versions this replica wrote.
947        let grace_ms = self.backfill_timing().grace.as_millis() as u64;
948        let now_ms = std::time::SystemTime::now()
949            .duration_since(std::time::UNIX_EPOCH)
950            .map_or(0, |d| d.as_millis() as u64);
951        let (mine, settled, live) = {
952            let replica = replica.lock().await;
953            let me = replica.replica_id();
954            let states = match replica.states() {
955                Ok(states) => states,
956                Err(err) => {
957                    tracing::warn!(root = %key.display(), "embedding pass: {err}");
958                    return;
959                }
960            };
961            let mut mine = HashMap::new();
962            // Other devices' versions old enough for the primary to fill in.
963            let mut settled = HashMap::new();
964            let mut live = std::collections::HashSet::new();
965            for (path, path_state) in states {
966                let winner = path_state.winner();
967                let Some(hash) = winner.content.hash() else {
968                    continue;
969                };
970                let hash = hash.to_hex();
971                if winner.dot.replica == me {
972                    mine.insert(path, hash.clone());
973                } else if now_ms.saturating_sub(winner.hlc.wall_ms) >= grace_ms {
974                    settled.insert(path, hash.clone());
975                }
976                live.insert(hash);
977            }
978            (mine, settled, live)
979        };
980        let root = key.to_owned();
981        let result = tokio::task::spawn_blocking(move || {
982            // The author embeds its own versions at once; the primary device fills in other
983            // devices' versions once they have settled, racing nobody.
984            let policy = |rel: &str, hash: &str| {
985                mine.get(rel).is_some_and(|h| h.as_str() == hash)
986                    || (primary && settled.get(rel).is_some_and(|h| h.as_str() == hash))
987            };
988            let embedded = state.embed_pending_with(&policy, PASS_BATCH)?;
989            let removed = if primary {
990                state.remove_stale_vectors(&live, STALE_VECTOR_AGE)
991            } else {
992                0
993            };
994            Ok::<_, sapphire_workspace::Error>((embedded, removed))
995        })
996        .await;
997        match result {
998            Ok(Ok((embedded, removed))) if embedded > 0 || removed > 0 => tracing::info!(
999                root = %root.display(),
1000                embedded,
1001                removed,
1002                "embedding pass"
1003            ),
1004            Ok(Ok(_)) => {}
1005            Ok(Err(err)) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
1006            Err(err) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
1007        }
1008    }
1009
1010    /// Open a session with every peer that will take one.
1011    pub async fn sync_now(self: &Arc<Self>, root: &Path) -> Result<()> {
1012        let Ok(key) = root.canonicalize() else {
1013            return Ok(());
1014        };
1015        let workspace_id = {
1016            let synced = self.synced.lock().await;
1017            match synced.get(&key) {
1018                Some(entry) => entry.workspace_id,
1019                None => return Ok(()),
1020            }
1021        };
1022        let peers = self.peers().await?;
1023        let me = self.device_id().await?;
1024
1025        // The same direction rule the dial loop follows: a host only dials peers with a
1026        // greater device id. Dialling the other way from here would reintroduce the
1027        // dial-both-ways deadlock the loop was cured of, just on a different trigger.
1028        // Inside a star, a pair with neither hub in it does not link at all.
1029        let roles = peers.roles_for(workspace_id).cloned();
1030        // Every peer at once, so one that is offline costs the others nothing (#228). On
1031        // this task, not spawned: cancelling the caller — the server's shutdown aborting the
1032        // watcher — must drop every dial with it, since each one pins the runtime.
1033        let mut dials: Vec<Dial<'_>> = Vec::new();
1034        for peer in peers
1035            .peers
1036            .iter()
1037            .filter(|p| topology::link(me, p.device_id, roles.as_ref()) == topology::Link::Dial)
1038            .cloned()
1039            .collect::<Vec<_>>()
1040        {
1041            // A peer that already has a live session is caught up by it: a second exchange
1042            // on the same workspace would be redundant, and a second stream between the same
1043            // pair is exactly what the tie-break cannot settle.
1044            if let Some(live) = self.live_peers(&key).await
1045                && live.has(&peer.device_id).await
1046            {
1047                continue;
1048            }
1049            // One dial per peer at a time. `sync_now` runs beside the dial loop, and two
1050            // overlapping outbound dials to one peer are the one case the tie-break rule
1051            // cannot resolve — each end would see its own stream as `outbound`. Whoever
1052            // arrives second simply leaves it to the next pass. The table is made here if
1053            // the dial loop has not yet: the first dial after enabling must not depend on
1054            // a tick that has not fired.
1055            let Some((_, table)) = self.live_peers_or_create(&key).await else {
1056                continue;
1057            };
1058            if !table.begin_dial(peer.device_id).await {
1059                continue;
1060            }
1061            let key = &key;
1062            dials.push(Box::pin(async move {
1063                // A peer that does not host this workspace refuses, which is normal and cheap.
1064                let result = self
1065                    .sync_one(key, &peer.device_id, peer.name.clone(), workspace_id)
1066                    .await;
1067                table.end_dial(peer.device_id).await;
1068                if let Err(err) = result {
1069                    tracing::debug!(peer = %peer.name, "no session: {err}");
1070                }
1071            }));
1072        }
1073        join_all(dials).await;
1074        Ok(())
1075    }
1076
1077    /// Dial one peer and adopt the stream as a live session.
1078    ///
1079    /// The one-shot path (`run_session`) would close the stream the moment the exchange
1080    /// finished — while the accepting peer keeps its half open and pushes into it, a session
1081    /// that dies with nothing reading it. So the dialled side is adopted live too, and both
1082    /// ends of a pair agree that a session stays open.
1083    async fn sync_one(
1084        self: &Arc<Self>,
1085        key: &Path,
1086        device: &GrainId,
1087        name: String,
1088        workspace_id: GrainId,
1089    ) -> Result<()> {
1090        let stream = self.open_stream(workspace_id, *device).await?;
1091        self.adopt_live_session(stream, key, *device, workspace_id)
1092            .await
1093            .map_err(|err| {
1094                tracing::warn!(peer = %name, "a live session could not be started: {err}");
1095                err
1096            })
1097    }
1098
1099    /// Ask the bridge for a stream to `device`, giving up after `open_timeout`.
1100    async fn open_stream(
1101        &self,
1102        workspace_id: GrainId,
1103        device: GrainId,
1104    ) -> Result<sapphire_ipc::RawStream> {
1105        match tokio::time::timeout(
1106            self.open_timeout,
1107            self.bridge()?.open_stream(workspace_id, device),
1108        )
1109        .await
1110        {
1111            Ok(opened) => Ok(opened?),
1112            Err(_) => Err(Error::Bridge(format!(
1113                "no stream to {device} within {} s; it may be offline",
1114                self.open_timeout.as_secs_f32()
1115            ))),
1116        }
1117    }
1118
1119    /// Answer the bridge's announcements, following the connection across reconnections
1120    /// (#244). Runs until the task is aborted.
1121    pub async fn run(self: Arc<Self>) -> Result<()> {
1122        let mut slot = self.bridge.subscribe();
1123        loop {
1124            let bridge = slot.borrow_and_update().clone();
1125            match bridge {
1126                Some(bridge) if !bridge.is_closed() => self.answer(&bridge, &mut slot).await,
1127                // Wait for a connection. The runtime holds the sender, so this never ends.
1128                _ => {
1129                    if slot.changed().await.is_err() {
1130                        return Ok(());
1131                    }
1132                }
1133            }
1134        }
1135    }
1136
1137    /// Answer `bridge`'s announcements until it closes or another connection replaces it.
1138    async fn answer(
1139        self: &Arc<Self>,
1140        bridge: &Arc<BridgeClient>,
1141        slot: &mut tokio::sync::watch::Receiver<Option<Arc<BridgeClient>>>,
1142    ) {
1143        let early = self
1144            .subscribed
1145            .lock()
1146            .expect("subscribed")
1147            .take_if(|(b, _)| Arc::ptr_eq(b, bridge));
1148        let mut incoming = match early {
1149            Some((_, incoming)) => incoming,
1150            None => bridge.incoming(),
1151        };
1152        loop {
1153            let received = tokio::select! {
1154                received = incoming.recv() => received,
1155                () = bridge.closed() => return,
1156                _ = slot.changed() => return,
1157            };
1158            let announcement = match received {
1159                Ok(a) => a,
1160                Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
1161                    tracing::warn!(missed = n, "fell behind on bridge announcements");
1162                    continue;
1163                }
1164                Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1165            };
1166
1167            let found = {
1168                let synced = self.synced.lock().await;
1169                synced
1170                    .iter()
1171                    .find(|(_, s)| s.workspace_id == announcement.workspace_id)
1172                    .map(|(root, s)| (root.clone(), Arc::clone(&s.replica)))
1173            };
1174            // `replica` is still what the accepted session shares, and `root` is where its
1175            // files and its index live.
1176            let Some((root, replica)) = found else {
1177                // The bridge routed to us for a workspace we no longer hold. Not fatal:
1178                // ignore it and let the ticket expire.
1179                tracing::debug!(
1180                    workspace = %announcement.workspace_id,
1181                    "an announcement for a workspace this server does not hold"
1182                );
1183                continue;
1184            };
1185
1186            let bridge = Arc::clone(bridge);
1187            let workspace_id = announcement.workspace_id;
1188            let peer = announcement.peer_device_id;
1189            let driver = Arc::clone(self);
1190            self.spawn_background(async move {
1191                let stream = match bridge.accept_stream(announcement.ticket).await {
1192                    Ok(stream) => stream,
1193                    Err(err) => {
1194                        tracing::warn!("could not claim an announced stream: {err}");
1195                        return;
1196                    }
1197                };
1198                if driver.should_skip(workspace_id, peer).await {
1199                    tracing::debug!(%peer, "dropped an inbound session: both ends are outside this workspace's star");
1200                    return;
1201                }
1202                // Live, not one-shot: the peer that dialled is keeping this session open and
1203                // pushing what it commits, so this side has to keep reading. A session that
1204                // closed after the exchange would leave the dialler pushing into a stream
1205                // nobody reads.
1206                let (_, session) = match sapphire_framework_session::open_live_session(
1207                    stream,
1208                    Arc::clone(&replica),
1209                    workspace_id,
1210                )
1211                .await
1212                {
1213                    Ok(started) => started,
1214                    Err(err) => {
1215                        tracing::warn!("an inbound session failed: {err}");
1216                        return;
1217                    }
1218                };
1219                let Some((key, table)) = driver.live_peers_or_create(&root).await else {
1220                    return;
1221                };
1222                // The peer dialled: this stream survives only if the *peer* has the lower
1223                // device id. A stream that loses is dropped, which closes it, and the
1224                // connection ends tidily rather than being left half-read.
1225                let Some(updates) = table.insert(peer, session, false).await else {
1226                    return;
1227                };
1228                table.catch_up(peer, &replica).await;
1229                // This is the receiving side: the files are on disk now and nobody else will
1230                // index them.
1231                driver.reindex(&key).await;
1232                driver.request_embed(&key);
1233                driver.spawn_reader(table, updates, key, peer);
1234            });
1235        }
1236    }
1237
1238    /// Make the first attempt to reach the bridge, when the runtime has a
1239    /// [`BridgeConnector`] and is not connected yet.
1240    ///
1241    /// No bridge running is not an error: the server serves without sync, and
1242    /// [`keep_connected`](Self::keep_connected) picks the bridge up once it starts. Any
1243    /// other failure — a bridge of another API version, say — is, and stops the start.
1244    pub async fn connect(self: &Arc<Self>) -> Result<()> {
1245        let Some(connector) = &self.connector else {
1246            return Ok(());
1247        };
1248        if self.is_connected() {
1249            return Ok(());
1250        }
1251        match connector.connect().await? {
1252            Some(client) => self.attach(client).await,
1253            None => tracing::warn!(
1254                "no sapphire-bridge is running; serving without sync until it starts"
1255            ),
1256        }
1257        Ok(())
1258    }
1259
1260    /// Stay connected to the bridge: wait for the connection to close, then reach the
1261    /// bridge again with the runtime's [`BridgeConnector`], backing off from
1262    /// [`RECONNECT_BACKOFF`] to [`RECONNECT_BACKOFF_MAX`] while it cannot (#244).
1263    ///
1264    /// Runs until the task is aborted, which is what the app server's shutdown does with
1265    /// it. Without a connector it only marks the runtime disconnected once the connection
1266    /// closes, and returns.
1267    pub async fn keep_connected(self: Arc<Self>) {
1268        let mut attempt = 0;
1269        loop {
1270            if let Ok(bridge) = self.bridge() {
1271                bridge.closed().await;
1272                tracing::warn!("lost the connection to sapphire-bridge; reconnecting");
1273                self.bridge.send_if_modified(|slot| {
1274                    let same = slot.as_ref().is_some_and(|b| Arc::ptr_eq(b, &bridge));
1275                    if same {
1276                        *slot = None;
1277                    }
1278                    same
1279                });
1280                attempt = 0;
1281                continue;
1282            }
1283            let Some(connector) = &self.connector else {
1284                return;
1285            };
1286            match connector.connect().await {
1287                Ok(Some(client)) => {
1288                    tracing::info!("connected to sapphire-bridge");
1289                    self.attach(client).await;
1290                    attempt = 0;
1291                    continue;
1292                }
1293                Ok(None) => tracing::debug!("no sapphire-bridge is running yet"),
1294                // Logged once per outage, not once per attempt.
1295                Err(err) if attempt == 0 => {
1296                    tracing::warn!("could not connect to sapphire-bridge: {err}")
1297                }
1298                Err(err) => tracing::debug!("could not connect to sapphire-bridge: {err}"),
1299            }
1300            tokio::time::sleep(reconnect_backoff(attempt)).await;
1301            attempt = attempt.saturating_add(1);
1302        }
1303    }
1304
1305    /// Start using `client`, and bring the bridge up to date: it knows nothing of this
1306    /// server yet, whether it has just started or this server has.
1307    async fn attach(self: &Arc<Self>, client: BridgeClient) {
1308        // Swapped under the pending lock, so an `enable` racing this either sees the new
1309        // connection or leaves its workspace for the drain below.
1310        let client = Arc::new(client);
1311        *self.subscribed.lock().expect("subscribed") =
1312            Some((Arc::clone(&client), client.incoming()));
1313        let waiting = {
1314            let mut pending = self.pending.lock().await;
1315            self.bridge.send_replace(Some(client));
1316            std::mem::take(&mut *pending)
1317        };
1318        // What was synced before the connection was lost.
1319        if !self.synced.lock().await.is_empty()
1320            && let Err(err) = self.reregister().await
1321        {
1322            tracing::warn!("could not register with sapphire-bridge: {err}");
1323        }
1324        // What was asked for while it was away. `enable` registers and dials each one.
1325        for root in waiting.into_keys() {
1326            if let Err(err) = self.enable(&root).await {
1327                tracing::warn!(root = %root.display(), "could not start sync: {err}");
1328            }
1329        }
1330    }
1331
1332    /// Every synced root, for the watcher.
1333    pub async fn roots(&self) -> Vec<PathBuf> {
1334        self.synced.lock().await.keys().cloned().collect()
1335    }
1336
1337    /// The live session table for `root`, if this runtime syncs it.
1338    ///
1339    /// `None` is not an error: a dial, a push or a drop for a workspace this server does not
1340    /// hold is a no-op, and every caller below treats it that way.
1341    async fn live_peers(&self, root: &Path) -> Option<Arc<LivePeers>> {
1342        let key = root.canonicalize().ok()?;
1343        self.live.lock().await.get(&key).cloned()
1344    }
1345
1346    /// The live session table for `root`, creating it if this runtime syncs it.
1347    ///
1348    /// The two locks are taken one after the other, never nested: `synced` to decide whether
1349    /// the workspace is ours at all, then `live` to get or make its table.
1350    async fn live_peers_or_create(&self, root: &Path) -> Option<(PathBuf, Arc<LivePeers>)> {
1351        let key = root.canonicalize().ok()?;
1352        if !self.synced.lock().await.contains_key(&key) {
1353            return None;
1354        }
1355        // A table is built by whoever walks the workspace first, and it needs this host's
1356        // device id — the tie-break between two simultaneous streams is decided from it.
1357        let me = self.device_id().await.ok()?;
1358        let mut live = self.live.lock().await;
1359        let peers = Arc::clone(
1360            live.entry(key.clone())
1361                .or_insert_with(|| Arc::new(LivePeers::new(me))),
1362        );
1363        Some((key, peers))
1364    }
1365
1366    /// Push `updates` to every open session for `root`.
1367    ///
1368    /// Called with what a scan recorded. `from` is always `None` here: a local commit came
1369    /// from this host, so no session is excluded. The forwarding path, which does name a
1370    /// source, is [`LivePeers::fan_out`](live::LivePeers::fan_out) called directly by the
1371    /// reader side of a session.
1372    ///
1373    /// Best-effort by contract: the commit already happened, and a peer that is gone is the
1374    /// dialer's problem, not the writer's.
1375    pub async fn after_commit(&self, root: &Path, updates: Vec<PathUpdate>) {
1376        if updates.is_empty() {
1377            return;
1378        }
1379        if let Some(peers) = self.live_peers(root).await {
1380            peers.fan_out(&updates, None).await;
1381        }
1382    }
1383
1384    /// The devices with an open live session for `root`.
1385    ///
1386    /// Used by tests to wait for the dialer; empty for a workspace this server does not hold.
1387    pub async fn live_session_devices(&self, root: &Path) -> Vec<GrainId> {
1388        match self.live_peers(root).await {
1389            Some(peers) => peers.devices().await,
1390            None => Vec::new(),
1391        }
1392    }
1393
1394    /// End the first dials [`enable`](SyncRuntime::enable) left running, waiting until
1395    /// they are gone.
1396    ///
1397    /// For the app server's shutdown: each one pins the runtime, and with it the replica
1398    /// stores, which are redb databases that stay locked — a dial outliving its server
1399    /// would lock the next one out.
1400    pub async fn stop_dials(&self) {
1401        let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1402        dials.abort_all();
1403        while dials.join_next().await.is_some() {}
1404    }
1405
1406    /// End the runtime's background work — inbound sessions, live-session readers and
1407    /// embedding passes — and wait until it has let go.
1408    ///
1409    /// The server's shutdown calls this, so a server restarted in the same process can open
1410    /// the replica stores again at once rather than whenever the last straggler finished
1411    /// (#235). An embedding pass already inside its blocking section runs that section to
1412    /// the end; it holds the workspace's index, never a replica.
1413    pub async fn stop_background(&self) {
1414        let mut background =
1415            std::mem::take(&mut *self.background.lock().expect("background tasks"));
1416        background.abort_all();
1417        while background.join_next().await.is_some() {}
1418    }
1419
1420    /// Wait for the first dials [`enable`](SyncRuntime::enable) left running to finish on
1421    /// their own: for a test that must not race them.
1422    #[cfg(test)]
1423    async fn settle_dials(&self) {
1424        let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1425        while dials.join_next().await.is_some() {}
1426    }
1427
1428    /// Close every open live session, as if the connections had been cut.
1429    ///
1430    /// Called for a workspace being disabled, and by tests that want to watch the dialer
1431    /// rebuild what was dropped.
1432    pub async fn drop_connections(&self) {
1433        let tables: Vec<Arc<LivePeers>> = self.live.lock().await.values().cloned().collect();
1434        for peers in tables {
1435            peers.drop_connections().await;
1436        }
1437    }
1438
1439    /// Keep a live session open to every peer, for every synced workspace.
1440    ///
1441    /// One walk per [`DIAL_INTERVAL`](live::DIAL_INTERVAL): for each synced workspace, ask
1442    /// the bridge who is in the workgroup, and open a session to every peer that is connected
1443    /// and does not already have one.
1444    ///
1445    /// A peer that cannot be reached is retried on a per-peer exponential backoff capped at
1446    /// [`DIAL_BACKOFF_MAX`] — one failed attempt per walk, not one per spin — and the counter
1447    /// resets when a session opens. An unreachable peer is therefore no obstacle to the
1448    /// others: its dial fails, the walk moves on, and the next workspace is dialled.
1449    ///
1450    /// Runs until the task is aborted, which is what the app server's shutdown does with it.
1451    /// It is allowed to fail like [`run`](SyncRuntime::run): only sync stops, and the app
1452    /// server keeps serving files (spec §10).
1453    pub async fn dial_loop(self: Arc<Self>) -> Result<()> {
1454        let mut tick = tokio::time::interval(live::DIAL_INTERVAL);
1455        // A missed tick is a late walk, not a burst of them: the interval is a pace, not a
1456        // quota to make up.
1457        tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1458        let mut waiting: HashMap<GrainId, std::time::Instant> = HashMap::new();
1459        let mut failures: HashMap<GrainId, u32> = HashMap::new();
1460        // The workspaces this host was the primary device of at the last walk.
1461        let mut primary_of: std::collections::HashSet<GrainId> = Default::default();
1462        // When the primary device last ran a backfill pass with nothing prompting it.
1463        let mut last_backfill = std::time::Instant::now();
1464        loop {
1465            tick.tick().await;
1466            let roots = self.roots().await;
1467            if roots.is_empty() {
1468                continue;
1469            }
1470            let peers = match self.peers().await {
1471                Ok(peers) => peers,
1472                // The bridge is down or has no workgroup: nothing to dial, and the next walk
1473                // asks again. Not an error for the caller — the app server is still serving.
1474                Err(err) => {
1475                    tracing::debug!("no peers to dial: {err}");
1476                    continue;
1477                }
1478            };
1479            let me = match self.device_id().await {
1480                Ok(me) => me,
1481                Err(err) => {
1482                    tracing::warn!("no device id, so nobody to dial: {err}");
1483                    continue;
1484                }
1485            };
1486            // A host that has just become a workspace's primary device owes it the vectors
1487            // nobody else computes (#187): files that arrived before the election settled
1488            // would otherwise wait for the next change.
1489            let ids: Vec<(PathBuf, GrainId)> = self
1490                .synced
1491                .lock()
1492                .await
1493                .iter()
1494                .map(|(root, entry)| (root.clone(), entry.workspace_id))
1495                .collect();
1496            // And every so often anyway: an author that never embedded leaves files only the
1497            // primary will fill in, and nothing else would prompt it (#188).
1498            let backfill_due = last_backfill.elapsed() >= self.backfill_timing().interval;
1499            if backfill_due {
1500                last_backfill = std::time::Instant::now();
1501            }
1502            for (root, workspace_id) in ids {
1503                let primary = peers
1504                    .roles_for(workspace_id)
1505                    .is_some_and(|r| r.primary == Some(me));
1506                if !primary {
1507                    primary_of.remove(&workspace_id);
1508                } else if primary_of.insert(workspace_id) || backfill_due {
1509                    self.request_embed(&root);
1510                }
1511            }
1512            let now = std::time::Instant::now();
1513            // Dial only peers whose device id is greater than ours. Every device dialling
1514            // every other can deadlock: two hosts that dial each other at once each hold
1515            // their replica's lock across their own exchange and wait for the other's
1516            // hello, which the other's dialler holds the lock for. With one direction per
1517            // pair the wait chain is a DAG — a higher-id host never dials, so it never
1518            // holds its lock out while a lower-id host's exchange needs it. The lower host
1519            // dials within `DIAL_INTERVAL`, and the exchange is symmetric, so nothing is
1520            // lost; the tie-break is left as the safety net it is.
1521            for root in roots {
1522                let Some((key, table)) = self.live_peers_or_create(&root).await else {
1523                    continue;
1524                };
1525                let workspace_id = {
1526                    let synced = self.synced.lock().await;
1527                    match synced.get(&key) {
1528                        Some(entry) => entry.workspace_id,
1529                        None => continue,
1530                    }
1531                };
1532                let roles = peers.roles_for(workspace_id);
1533                // Entering a star closes what it no longer needs. The hubs carry it all —
1534                // once one of them is reachable from here (#192).
1535                if topology::may_close_skipped(me, roles, &table.devices().await) {
1536                    table
1537                        .retain(|d| topology::link(me, *d, roles) != topology::Link::Skip)
1538                        .await;
1539                }
1540                for peer in peers.peers.iter().filter(|p| {
1541                    p.connected && topology::link(me, p.device_id, roles) == topology::Link::Dial
1542                }) {
1543                    let device = peer.device_id;
1544                    if table.has(&device).await {
1545                        // Already talking; a redial would drop a working session.
1546                        failures.remove(&device);
1547                        waiting.remove(&device);
1548                        continue;
1549                    }
1550                    if let Some(until) = waiting.get(&device)
1551                        && now < *until
1552                    {
1553                        // Backing off after a failure; the walk skips it until then.
1554                        continue;
1555                    }
1556                    // The same claim `sync_now` takes: this walk and it can overlap, and a
1557                    // double dial in one direction is the one race the tie-break cannot
1558                    // settle. Skipping is free — the next pass, or `sync_now` itself, dials.
1559                    if !table.begin_dial(device).await {
1560                        continue;
1561                    }
1562                    let opened = self.open_stream(workspace_id, device).await;
1563                    if opened.is_err() {
1564                        table.end_dial(device).await;
1565                    }
1566                    match opened {
1567                        Ok(stream) => {
1568                            let adopted = self
1569                                .adopt_live_session(stream, &root, device, workspace_id)
1570                                .await;
1571                            // The claim is out either way: the attempt is over, and a
1572                            // session that died mid-exchange is re-dialled next pass.
1573                            table.end_dial(device).await;
1574                            match adopted {
1575                                Ok(()) => {
1576                                    failures.remove(&device);
1577                                    waiting.remove(&device);
1578                                }
1579                                Err(err) => {
1580                                    tracing::debug!(
1581                                        peer = %peer.name,
1582                                        err = %err,
1583                                        "a live session could not be started"
1584                                    );
1585                                    let count = failures.entry(device).or_insert(0);
1586                                    *count = count.saturating_add(1);
1587                                    waiting.insert(device, now + dial_backoff(*count));
1588                                }
1589                            }
1590                        }
1591                        Err(err) => {
1592                            // Normal and cheap: a peer that does not host this workspace
1593                            // refuses, and one that is unreachable cannot be opened.
1594                            tracing::debug!("no stream to a peer: {err}");
1595                            let count = failures.entry(device).or_insert(0);
1596                            *count = count.saturating_add(1);
1597                            waiting.insert(device, now + dial_backoff(*count));
1598                        }
1599                    }
1600                }
1601            }
1602        }
1603    }
1604
1605    /// Take ownership of an opened stream: run the exchange, then keep it live.
1606    ///
1607    /// The session is stored under `device` and its reader pumped until the stream ends. A
1608    /// half-open or refused stream is not stored; the dialer retries it with backoff.
1609    async fn adopt_live_session(
1610        self: &Arc<Self>,
1611        stream: sapphire_ipc::RawStream,
1612        root: &Path,
1613        device: GrainId,
1614        workspace_id: GrainId,
1615    ) -> Result<()> {
1616        let replica = self.replica_of(root).await?;
1617        let (_, session) = sapphire_framework_session::open_live_session(
1618            stream,
1619            Arc::clone(&replica),
1620            workspace_id,
1621        )
1622        .await
1623        .map_err(|e| Error::Sync(e.to_string()))?;
1624        let Some((key, table)) = self.live_peers_or_create(root).await else {
1625            return Ok(());
1626        };
1627        // This stream was dialled by us, so it is the one the pair keeps if this host's
1628        // device id is the lower of the two. If it lost the tie-break the session is
1629        // dropped here, which closes the stream, and the surviving one is already in the
1630        // table.
1631        let Some(updates) = table.insert(device, session, true).await else {
1632            return Ok(());
1633        };
1634        table.catch_up(device, &replica).await;
1635        // The session wrote files during the exchange; the index has to catch up before the
1636        // workspace is searchable again.
1637        self.reindex(&key).await;
1638        self.request_embed(&key);
1639        self.spawn_reader(Arc::clone(&table), updates, key, device);
1640        Ok(())
1641    }
1642
1643    /// Forward what one session receives, and re-index it.
1644    ///
1645    /// This is the loop that makes A → S → B work: a batch that arrived on S's session with A
1646    /// is applied (the session crate does that before publishing it), the workspace behind
1647    /// `key` is re-indexed, and the batch is handed to every *other* session whose peer lacks
1648    /// it — `from` is the device it came from, so it never goes back to A.
1649    fn spawn_reader(
1650        self: &Arc<Self>,
1651        table: Arc<LivePeers>,
1652        mut updates: broadcast::Receiver<Vec<PathUpdate>>,
1653        key: PathBuf,
1654        from: GrainId,
1655    ) {
1656        let driver = Arc::clone(self);
1657        self.spawn_background(async move {
1658            loop {
1659                match updates.recv().await {
1660                    Ok(batch) => {
1661                        // The files are on disk now (the session guarantees it before
1662                        // publishing), so the index is behind them until the sweep below.
1663                        driver.reindex(&key).await;
1664                        driver.request_embed(&key);
1665                        table.fan_out(&batch, Some(from)).await;
1666                    }
1667                    // A burst bigger than the channel: the newest is kept and the gap is
1668                    // covered by the next session's exchange. Losing a batch here cannot lose
1669                    // an entry — the peer's vector does not cover it, so the next dial or push
1670                    // carries it again.
1671                    Err(broadcast::error::RecvError::Lagged(missed)) => {
1672                        tracing::debug!(
1673                            device = %from,
1674                            missed,
1675                            "fell behind on a live session; the next exchange catches up"
1676                        );
1677                    }
1678                    // The session closed. Its table entry is reaped by the next fan-out or
1679                    // dial pass, and the dialer opens a fresh one.
1680                    Err(broadcast::error::RecvError::Closed) => {
1681                        return;
1682                    }
1683                }
1684            }
1685        });
1686    }
1687
1688    /// The replica [`disable`](Self::disable) let go of for `key`, if something still holds it
1689    /// open.
1690    fn reclaim(&self, key: &Path) -> Reclaimed {
1691        let stopped = self.stopped.lock().expect("stopped replicas").remove(key);
1692        match stopped {
1693            None => Reclaimed::Closed,
1694            Some(replica) => replica
1695                .upgrade()
1696                .map_or(Reclaimed::Closing, Reclaimed::Open),
1697        }
1698    }
1699
1700    /// The replica of a synced `root`.
1701    async fn replica_of(&self, root: &Path) -> Result<Arc<Mutex<Replica>>> {
1702        let key = root.canonicalize().map_err(Error::Io)?;
1703        let synced = self.synced.lock().await;
1704        match synced.get(&key) {
1705            Some(entry) => Ok(Arc::clone(&entry.replica)),
1706            None => Err(Error::UnknownWorkspace(key, self.ctx.app_name)),
1707        }
1708    }
1709
1710    /// Watch the synced roots and, on a debounced report, scan and dial.
1711    ///
1712    /// This is the safety net for everything the app server did not do itself — a file
1713    /// edited in an editor, a `git checkout`. The exact path is [`scan`] called from
1714    /// `handlers.rs` right after a write the server made. A scan that finds nothing is
1715    /// cheap; a scan that is skipped loses an edit until the next one.
1716    ///
1717    /// Returns when the reporting channel closes — that is, when the [`Watcher`] this
1718    /// runtime holds is dropped, which happens when the runtime itself is.
1719    ///
1720    /// [`scan`]: SyncRuntime::scan
1721    pub async fn watch_changes(self: Arc<Self>) -> Result<()> {
1722        let (tx, mut rx) = mpsc::channel(64);
1723        let watcher = Arc::new(Watcher::start(self.roots().await, tx)?);
1724        let _ = self.watcher.set(Arc::clone(&watcher));
1725
1726        // A root enabled between `roots()` above and the `set` just now was not in the
1727        // starting set, and `enable` could not reach a watcher that did not exist yet.
1728        // Watching the current set again is idempotent on `notify`'s side.
1729        for root in self.roots().await {
1730            if let Err(err) = watcher.watch(&root) {
1731                tracing::warn!(root = %root.display(), "could not watch: {err}");
1732            }
1733        }
1734
1735        while let Some(root) = rx.recv().await {
1736            // Logged, not fatal: a failed scan or dial leaves the replica behind, and the
1737            // next report tries again. Neither may take the app server down (spec §10).
1738            if let Err(err) = self.scan(&root).await {
1739                tracing::warn!(root = %root.display(), "scan after a local edit failed: {err}");
1740            }
1741            self.request_embed(&root);
1742            if let Err(err) = self.sync_now(&root).await {
1743                tracing::warn!(root = %root.display(), "dial after a local edit failed: {err}");
1744            }
1745        }
1746        Ok(())
1747    }
1748
1749    /// Tell the bridge the complete current set. A registration is not a delta.
1750    async fn reregister(&self) -> Result<()> {
1751        let workspaces: Vec<WorkspaceRegistration> = {
1752            let synced = self.synced.lock().await;
1753            synced
1754                .iter()
1755                .map(|(root, entry)| WorkspaceRegistration {
1756                    workspace_id: entry.workspace_id,
1757                    root: root.clone(),
1758                })
1759                .collect()
1760        };
1761        self.bridge()?
1762            .register(RegisterParams {
1763                app_name: self.ctx.app_name.to_owned(),
1764                exe_path: self.exe_path.clone(),
1765                managed_by: wire_managed_by(self.managed_by),
1766                workspaces,
1767            })
1768            .await
1769            .map_err(|e| Error::Bridge(e.to_string()))?;
1770        Ok(())
1771    }
1772
1773    /// Whether the last roles the bridge reported say this pair does not link.
1774    ///
1775    /// Reads the cached device id only: asking the bridge here would register with an empty
1776    /// workspace list, racing `enable`'s registration. `false` (accept) when it is unknown.
1777    async fn should_skip(&self, workspace_id: GrainId, peer: GrainId) -> bool {
1778        let Some(&me) = self.device_id.get() else {
1779            return false;
1780        };
1781        let cached = self.last_peers.lock().await;
1782        let roles = cached.as_ref().and_then(|p| p.roles_for(workspace_id));
1783        topology::link(me, peer, roles) == topology::Link::Skip
1784    }
1785
1786    /// Whether this host is `workspace_id`'s primary device, by the last roles the
1787    /// bridge reported. `false` when the bridge has not answered yet, or when this host's
1788    /// device id is not known yet (no workspace was enabled). Reads the cached id only:
1789    /// asking the bridge here would register with an empty workspace list, racing
1790    /// `enable`'s registration.
1791    pub async fn is_primary(&self, workspace_id: GrainId) -> bool {
1792        let Some(&me) = self.device_id.get() else {
1793            return false;
1794        };
1795        let cached = self.last_peers.lock().await;
1796        cached
1797            .as_ref()
1798            .and_then(|p| p.roles_for(workspace_id))
1799            .is_some_and(|r| r.primary == Some(me))
1800    }
1801
1802    /// This host's device id, once the bridge has told it (#250).
1803    ///
1804    /// `None` until this runtime first registered — no workspace was enabled yet, or the
1805    /// bridge was not connected when one was. Never asks the bridge: a registration made
1806    /// only to learn the id would carry an empty workspace list, and briefly clear this
1807    /// application's routes.
1808    pub fn this_device(&self) -> Option<GrainId> {
1809        self.device_id.get().copied()
1810    }
1811
1812    /// The workgroup's devices, this host's included, and the roles the bridge assigns per
1813    /// workspace (#250). Asked of the bridge each time; [`is_primary`](Self::is_primary)
1814    /// reads the answer afterwards.
1815    ///
1816    /// [`Error::BridgeDisconnected`] while there is no connection to the bridge.
1817    pub async fn peers(&self) -> Result<PeersResult> {
1818        let peers = self
1819            .bridge()?
1820            .peers()
1821            .await
1822            .map_err(|e| Error::Bridge(e.to_string()))?;
1823        *self.last_peers.lock().await = Some(peers.clone());
1824        Ok(peers)
1825    }
1826
1827    /// This host's device id, asked of the bridge once.
1828    async fn device_id(&self) -> Result<GrainId> {
1829        if let Some(id) = self.device_id.get() {
1830            return Ok(*id);
1831        }
1832        let result = self
1833            .bridge()?
1834            .register(RegisterParams {
1835                app_name: self.ctx.app_name.to_owned(),
1836                exe_path: self.exe_path.clone(),
1837                managed_by: wire_managed_by(self.managed_by),
1838                workspaces: Vec::new(),
1839            })
1840            .await
1841            .map_err(|e| Error::Bridge(e.to_string()))?;
1842        let _ = self.device_id.set(result.device_id);
1843        Ok(result.device_id)
1844    }
1845
1846    /// `<workspace cache dir>/sync/`.
1847    fn state_dir(&self, root: &Path) -> Result<PathBuf> {
1848        let workspace = Workspace::from_root(self.ctx, root)?;
1849        Ok(workspace.cache_dir().join("sync"))
1850    }
1851}
1852
1853/// One dial of [`SyncRuntime::sync_now`].
1854type Dial<'a> = std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>>;
1855
1856/// Run `dials` concurrently on the calling task, until every one has finished.
1857async fn join_all(mut dials: Vec<Dial<'_>>) {
1858    std::future::poll_fn(|cx| {
1859        dials.retain_mut(|dial| dial.as_mut().poll(cx).is_pending());
1860        if dials.is_empty() {
1861            std::task::Poll::Ready(())
1862        } else {
1863            std::task::Poll::Pending
1864        }
1865    })
1866    .await
1867}
1868
1869/// The bridge API's copy of how this server was started, for [`RegisterParams`].
1870fn wire_managed_by(managed_by: ManagedBy) -> sapphire_bridge_api::ManagedBy {
1871    match managed_by {
1872        ManagedBy::Service => sapphire_bridge_api::ManagedBy::Service,
1873        ManagedBy::Spawned => sapphire_bridge_api::ManagedBy::Spawned,
1874    }
1875}
1876
1877#[cfg(test)]
1878mod tests {
1879    use super::*;
1880    use crate::sync::testing::StubBridge;
1881    use crate::test_support;
1882    use sapphire_workspace::AppKind;
1883    use std::ffi::OsString;
1884
1885    static CTX: AppContext = AppContext::new("sapphire-synctest");
1886
1887    /// The env vars `CTX.init` reads.
1888    const DIR_VARS: [&str; 3] = [
1889        "SAPPHIRE_SYNCTEST_CACHE_DIR",
1890        "SAPPHIRE_SYNCTEST_DATA_DIR",
1891        "SAPPHIRE_SYNCTEST_CONFIG_DIR",
1892    ];
1893
1894    /// Points the context's directories at the test's scratch tree, and restores the
1895    /// previous values when dropped — including while unwinding from a panic.
1896    struct EnvGuard {
1897        previous: [Option<OsString>; 3],
1898        _lock: std::sync::MutexGuard<'static, ()>,
1899    }
1900
1901    impl Drop for EnvGuard {
1902        fn drop(&mut self) {
1903            // SAFETY: `self._lock` still serialises the environment; it is dropped only
1904            // after this method returns.
1905            for (name, previous) in DIR_VARS.iter().zip(self.previous.iter_mut()) {
1906                match previous.take() {
1907                    Some(value) => unsafe { std::env::set_var(name, value) },
1908                    None => test_support::remove(name),
1909                }
1910            }
1911        }
1912    }
1913
1914    /// Everything one test needs, held for the test's whole body.
1915    ///
1916    /// The context is a `static` shared by this binary, so its directories are resolved from
1917    /// the environment by whichever test initialises it first. The environment lock is
1918    /// therefore held for the whole test, as in the crate's other test modules, and `_tmp`
1919    /// is declared before `_env` so the scratch tree is gone only once the environment no
1920    /// longer points at it.
1921    struct Fixture {
1922        _tmp: tempfile::TempDir,
1923        _env: EnvGuard,
1924        root: PathBuf,
1925        stub: StubBridge,
1926        runtime: Arc<SyncRuntime>,
1927    }
1928
1929    impl Fixture {
1930        /// A second workspace root beside the first, marker directory and all.
1931        fn second_root(&self) -> PathBuf {
1932            let root = self._tmp.path().join("ws2");
1933            std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
1934            root.canonicalize().unwrap()
1935        }
1936    }
1937
1938    /// How long the fixture's runtime waits on a stream to a peer.
1939    const FIXTURE_OPEN_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
1940
1941    /// `n` peers this host dials (a greater device id), as `bridge.peers` lists them.
1942    fn dialled_peers(me: GrainId, n: usize) -> Vec<sapphire_bridge_api::PeerInfo> {
1943        std::iter::repeat_with(GrainId::random)
1944            .filter(|id| *id > me)
1945            .take(n)
1946            .enumerate()
1947            .map(|(i, device_id)| sapphire_bridge_api::PeerInfo {
1948                device_id,
1949                name: format!("offline-{i}"),
1950                node_id: format!("node-{i}"),
1951                connected: false,
1952                priority: 1,
1953                availability: None,
1954            })
1955            .collect()
1956    }
1957
1958    #[tokio::test(flavor = "multi_thread")]
1959    async fn enabling_answers_at_once_with_a_peer_that_never_answers() {
1960        // #228: the first dial waited on a peer that was offline, past the client's limit.
1961        let f = fixture().await;
1962        let _hung = f.stub.hang_streams().await;
1963        f.stub.set_peers(dialled_peers(f.stub.device_id, 1));
1964
1965        let started = std::time::Instant::now();
1966        f.runtime.enable(&f.root).await.unwrap();
1967        assert!(
1968            started.elapsed() < FIXTURE_OPEN_TIMEOUT,
1969            "enable waited {:?} on the dial",
1970            started.elapsed()
1971        );
1972    }
1973
1974    #[tokio::test(flavor = "multi_thread")]
1975    async fn peers_that_never_answer_are_dialled_together_and_given_up_on() {
1976        let f = fixture().await;
1977        let _hung = f.stub.hang_streams().await;
1978        f.runtime.enable(&f.root).await.unwrap();
1979        // The first dial, with nobody to dial yet, must be over before the peers appear:
1980        // one still running would claim them, and `sync_now` would skip them all.
1981        f.runtime.settle_dials().await;
1982        f.stub.set_peers(dialled_peers(f.stub.device_id, 3));
1983
1984        let started = std::time::Instant::now();
1985        f.runtime.sync_now(&f.root).await.unwrap();
1986        let took = started.elapsed();
1987        assert!(took >= FIXTURE_OPEN_TIMEOUT, "gave up early: {took:?}");
1988        assert!(
1989            took < FIXTURE_OPEN_TIMEOUT * 2,
1990            "three dials one after another: {took:?}"
1991        );
1992    }
1993
1994    async fn fixture() -> Fixture {
1995        let (tmp, env, root) = scratch();
1996        let (stub, client) = StubBridge::start_in(tmp.path().join("run")).await;
1997        let runtime = Arc::new(
1998            SyncRuntime::new(&CTX, client, "/bin/true".into(), ManagedBy::Service)
1999                .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
2000        );
2001        Fixture {
2002            _tmp: tmp,
2003            _env: env,
2004            root,
2005            stub,
2006            runtime,
2007        }
2008    }
2009
2010    /// The bridge connections a [`connecting`] fixture's connector hands out, in order;
2011    /// empty is "no bridge is running".
2012    type Bridges = Arc<std::sync::Mutex<std::collections::VecDeque<BridgeClient>>>;
2013
2014    /// A fixture whose runtime reaches the bridge through a connector, and has not yet.
2015    /// `f.stub`'s client is not handed out: push it to the returned queue for that.
2016    async fn connecting() -> (Fixture, Bridges, BridgeClient) {
2017        let (tmp, env, root) = scratch();
2018        let (stub, client) = StubBridge::start_client_in(tmp.path().join("run")).await;
2019        let bridges: Bridges = Arc::default();
2020        let connector = BridgeConnector::new({
2021            let bridges = Arc::clone(&bridges);
2022            move || {
2023                let next = bridges.lock().unwrap().pop_front();
2024                async move { Ok(next) }
2025            }
2026        });
2027        let runtime = Arc::new(
2028            SyncRuntime::connecting(&CTX, connector, "/bin/true".into(), ManagedBy::Service)
2029                .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
2030        );
2031        let f = Fixture {
2032            _tmp: tmp,
2033            _env: env,
2034            root,
2035            stub,
2036            runtime,
2037        };
2038        (f, bridges, client)
2039    }
2040
2041    /// Wait up to ten seconds for `done`.
2042    async fn eventually(what: &str, mut done: impl AsyncFnMut() -> bool) {
2043        let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
2044        while !done().await {
2045            assert!(tokio::time::Instant::now() < deadline, "never: {what}");
2046            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
2047        }
2048    }
2049
2050    #[tokio::test(flavor = "multi_thread")]
2051    async fn a_runtime_started_without_a_bridge_syncs_once_one_starts() {
2052        // #244: a server that started before its bridge never synced until restarted.
2053        let (f, bridges, client) = connecting().await;
2054        f.runtime.connect().await.unwrap();
2055        assert!(!f.runtime.is_connected());
2056
2057        let err = f.runtime.enable(&f.root).await.unwrap_err();
2058        assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2059        let status = f.runtime.status(&f.root).await;
2060        assert!(status.enabled, "enabled, waiting for the bridge");
2061        assert!(!status.bridge_available);
2062        let id = status.workspace_id.expect("its id");
2063
2064        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2065        bridges.lock().unwrap().push_back(client);
2066        eventually("the waiting workspace is registered", async || {
2067            f.stub.last_workspaces() == vec![id]
2068        })
2069        .await;
2070        assert_eq!(f.runtime.roots().await, vec![f.root.clone()]);
2071        assert!(f.runtime.status(&f.root).await.bridge_available);
2072        keeping.abort();
2073    }
2074
2075    #[tokio::test(flavor = "multi_thread")]
2076    async fn a_runtime_that_loses_its_bridge_registers_with_the_next_one() {
2077        let (f, bridges, client) = connecting().await;
2078        bridges.lock().unwrap().push_back(client);
2079        f.runtime.connect().await.unwrap();
2080        assert!(f.runtime.is_connected());
2081        let id = f.runtime.enable(&f.root).await.unwrap();
2082        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2083
2084        f.stub.disconnect();
2085        eventually("the runtime notices", async || !f.runtime.is_connected()).await;
2086        let err = f.runtime.workspaces().await.unwrap_err();
2087        assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2088
2089        let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2090        bridges.lock().unwrap().push_back(client);
2091        eventually("the synced workspace is registered again", async || {
2092            next.last_workspaces() == vec![id]
2093        })
2094        .await;
2095        assert!(f.runtime.is_connected());
2096        keeping.abort();
2097    }
2098
2099    #[tokio::test(flavor = "multi_thread")]
2100    async fn this_device_is_known_once_a_workspace_is_enabled_and_never_asked_for() {
2101        let f = fixture().await;
2102        assert_eq!(f.runtime.this_device(), None);
2103        assert!(
2104            f.stub.seen.lock().unwrap().registrations.is_empty(),
2105            "asking must not register"
2106        );
2107
2108        f.runtime.enable(&f.root).await.unwrap();
2109        assert_eq!(f.runtime.this_device(), Some(f.stub.device_id));
2110    }
2111
2112    #[tokio::test(flavor = "multi_thread")]
2113    async fn peers_answers_the_workgroup_from_the_bridge() {
2114        let f = fixture().await;
2115        let listed = dialled_peers(f.stub.device_id, 2);
2116        f.stub.set_peers(listed.clone());
2117
2118        let peers = f.runtime.peers().await.unwrap();
2119        let ids: Vec<_> = peers.peers.iter().map(|p| p.device_id).collect();
2120        assert_eq!(ids, listed.iter().map(|p| p.device_id).collect::<Vec<_>>());
2121    }
2122
2123    #[tokio::test(flavor = "multi_thread")]
2124    async fn peers_says_so_while_the_bridge_is_away() {
2125        let (f, _bridges, _client) = connecting().await;
2126        let err = f.runtime.peers().await.unwrap_err();
2127        assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2128    }
2129
2130    #[tokio::test(flavor = "multi_thread")]
2131    async fn disabling_while_the_bridge_is_away_forgets_the_waiting_workspace() {
2132        let (f, bridges, client) = connecting().await;
2133        f.runtime.enable(&f.root).await.unwrap_err();
2134        f.runtime.disable(&f.root).await.unwrap();
2135        assert!(!f.runtime.status(&f.root).await.enabled);
2136
2137        bridges.lock().unwrap().push_back(client);
2138        f.runtime.connect().await.unwrap();
2139        assert!(f.runtime.roots().await.is_empty());
2140        assert!(f.stub.last_workspaces().is_empty());
2141    }
2142
2143    #[tokio::test(flavor = "multi_thread")]
2144    async fn announcements_are_answered_from_a_reconnected_bridge() {
2145        let (f, bridges, client) = connecting().await;
2146        bridges.lock().unwrap().push_back(client);
2147        f.runtime.connect().await.unwrap();
2148        let id = f.runtime.enable(&f.root).await.unwrap();
2149        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2150        let answering = tokio::spawn(Arc::clone(&f.runtime).run());
2151
2152        f.stub.disconnect();
2153        let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2154        bridges.lock().unwrap().push_back(client);
2155        eventually("reconnected", async || next.last_workspaces() == vec![id]).await;
2156
2157        // The announcement loop follows the new connection: an announcement on it is
2158        // claimed, which with no data endpoint fails — and the loop lives on.
2159        next.announce(id, "ticket").await;
2160        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2161        assert!(!answering.is_finished(), "the announcement loop ended");
2162        answering.abort();
2163        keeping.abort();
2164    }
2165
2166    /// The scratch tree, the environment pointed at it, and a workspace root in it.
2167    fn scratch() -> (tempfile::TempDir, EnvGuard, PathBuf) {
2168        let lock = test_support::lock();
2169        let tmp = tempfile::tempdir().unwrap();
2170        let previous = DIR_VARS.map(std::env::var_os);
2171        // SAFETY (via `test_support::set`): `lock` serialises every read and write of the
2172        // process environment in this test binary, and it is held until `drop` has restored
2173        // the old values.
2174        for (name, dir) in DIR_VARS
2175            .iter()
2176            .zip(["cache", "data", "config"].map(|cat| tmp.path().join(cat)))
2177        {
2178            test_support::set(name, &dir);
2179        }
2180        let env = EnvGuard {
2181            previous,
2182            _lock: lock,
2183        };
2184        CTX.init(AppKind::Server);
2185
2186        let root = tmp.path().join("ws");
2187        std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
2188        let root = root.canonicalize().unwrap();
2189        (tmp, env, root)
2190    }
2191
2192    #[tokio::test(flavor = "multi_thread")]
2193    async fn enabling_registers_the_workspace_with_the_bridge() {
2194        let f = fixture().await;
2195
2196        let id = f.runtime.enable(&f.root).await.unwrap();
2197        assert_eq!(f.stub.last_workspaces(), vec![id]);
2198    }
2199
2200    #[tokio::test(flavor = "multi_thread")]
2201    async fn enabling_twice_is_idempotent_and_keeps_the_same_id() {
2202        let f = fixture().await;
2203
2204        let first = f.runtime.enable(&f.root).await.unwrap();
2205        let second = f.runtime.enable(&f.root).await.unwrap();
2206        assert_eq!(first, second);
2207        assert_eq!(
2208            f.stub.last_workspaces(),
2209            vec![first],
2210            "still exactly one workspace"
2211        );
2212    }
2213
2214    #[tokio::test(flavor = "multi_thread")]
2215    async fn a_registration_carries_the_complete_current_list() {
2216        let f = fixture().await;
2217        let second_root = f.second_root();
2218
2219        let a = f.runtime.enable(&f.root).await.unwrap();
2220        let b = f.runtime.enable(&second_root).await.unwrap();
2221
2222        let mut seen = f.stub.last_workspaces();
2223        seen.sort();
2224        let mut want = vec![a, b];
2225        want.sort();
2226        assert_eq!(seen, want, "registration is the whole set, not a delta");
2227    }
2228
2229    #[tokio::test(flavor = "multi_thread")]
2230    async fn no_later_registration_clears_this_apps_routes() {
2231        // The device id is asked of the bridge once, on the first `enable`, with an empty
2232        // workspace list — the one moment that is safe, because no route exists yet. Every
2233        // later registration must name the complete set: one with an empty list would
2234        // briefly clear this application's routes, and a peer asking for a workspace this
2235        // server owns would be told nobody has it.
2236        let f = fixture().await;
2237        let second_root = f.second_root();
2238
2239        f.runtime.enable(&f.root).await.unwrap();
2240        f.runtime.enable(&second_root).await.unwrap();
2241        f.runtime.sync_now(&f.root).await.unwrap();
2242        f.runtime.disable(&f.root).await.unwrap();
2243
2244        let seen = f.stub.seen.lock().expect("stub");
2245        let first_full = seen
2246            .registrations
2247            .iter()
2248            .position(|r| !r.workspaces.is_empty())
2249            .expect("enabling registers the workspace");
2250        let emptied_afterwards = seen.registrations[first_full + 1..]
2251            .iter()
2252            .filter(|r| r.workspaces.is_empty())
2253            .count();
2254        assert_eq!(
2255            emptied_afterwards, 0,
2256            "a registration after the first workspace must still name the whole set"
2257        );
2258    }
2259
2260    #[tokio::test(flavor = "multi_thread")]
2261    async fn disabling_unregisters_and_leaves_the_files_alone() {
2262        let f = fixture().await;
2263        std::fs::write(f.root.join("keep.md"), "content").unwrap();
2264
2265        let id = f.runtime.enable(&f.root).await.unwrap();
2266        f.runtime.disable(&f.root).await.unwrap();
2267
2268        assert!(f.stub.last_workspaces().is_empty());
2269        assert!(
2270            f.root.join("keep.md").exists(),
2271            "disabling sync must not touch files"
2272        );
2273        assert!(
2274            crate::sync::id::sync_id_path("sapphire-synctest", &f.root).exists(),
2275            "the sync id stays, so re-enabling rejoins the same workspace"
2276        );
2277        let _ = id;
2278    }
2279
2280    #[tokio::test(flavor = "multi_thread")]
2281    async fn re_enabling_after_disabling_reuses_the_identity() {
2282        let f = fixture().await;
2283
2284        let first = f.runtime.enable(&f.root).await.unwrap();
2285        f.runtime.disable(&f.root).await.unwrap();
2286        let again = f.runtime.enable(&f.root).await.unwrap();
2287
2288        assert_eq!(first, again);
2289    }
2290
2291    #[tokio::test(flavor = "multi_thread")]
2292    async fn status_reports_disabled_for_a_workspace_that_was_never_enabled() {
2293        let f = fixture().await;
2294
2295        let status = f.runtime.status(&f.root).await;
2296        assert!(!status.enabled);
2297        assert!(status.workspace_id.is_none());
2298    }
2299
2300    #[tokio::test(flavor = "multi_thread")]
2301    async fn disabling_something_that_was_never_enabled_is_not_an_error() {
2302        let f = fixture().await;
2303        f.runtime
2304            .disable(&f.root)
2305            .await
2306            .expect("disabling twice is fine");
2307    }
2308
2309    #[tokio::test(flavor = "multi_thread")]
2310    async fn a_file_already_on_disk_is_recorded_on_enable() {
2311        // No watcher fires for a file written before the workspace was enabled; without a
2312        // scan on enable it was never recorded, and every session sent nothing.
2313        let f = fixture().await;
2314        std::fs::write(f.root.join("a.md"), "content").unwrap();
2315        f.runtime.enable(&f.root).await.unwrap();
2316        f.runtime.settle_dials().await;
2317
2318        let replica = f.runtime.replica_of(&f.root).await.unwrap();
2319        let states = replica.lock().await.states().unwrap();
2320        assert!(
2321            states.iter().any(|(path, _)| path == "a.md"),
2322            "a file on disk before enable must be recorded: {states:?}"
2323        );
2324    }
2325
2326    #[tokio::test(flavor = "multi_thread")]
2327    async fn re_enabling_while_the_replica_is_still_held_takes_it_back() {
2328        // The first scan after enabling runs in the background; a disable and a re-enable
2329        // can both land while it still holds the store, which redb opens only once.
2330        let f = fixture().await;
2331        f.runtime.enable(&f.root).await.unwrap();
2332        f.runtime.settle_dials().await;
2333        let held = f.runtime.replica_of(&f.root).await.unwrap();
2334
2335        f.runtime.disable(&f.root).await.unwrap();
2336        f.runtime
2337            .enable(&f.root)
2338            .await
2339            .expect("a re-enable must not open the store a second time");
2340
2341        let again = f.runtime.replica_of(&f.root).await.unwrap();
2342        assert!(Arc::ptr_eq(&held, &again), "the open replica is taken back");
2343    }
2344
2345    #[tokio::test(flavor = "multi_thread")]
2346    async fn re_enabling_while_the_replica_is_still_closing_waits_for_it() {
2347        // The last holder let go just before the re-enable: the replica can no longer be
2348        // taken back, and its store closes a moment later, as it drops.
2349        let f = fixture().await;
2350        f.runtime.enable(&f.root).await.unwrap();
2351        f.runtime.settle_dials().await;
2352        let config = {
2353            let replica = f.runtime.replica_of(&f.root).await.unwrap();
2354            replica.lock().await.config().clone()
2355        };
2356        f.runtime.disable(&f.root).await.unwrap();
2357
2358        // The store, open still, with nothing left to take back.
2359        let closing = Replica::open(config, Arc::new(SystemClock)).unwrap();
2360        let key = f.root.canonicalize().unwrap();
2361        f.runtime
2362            .stopped
2363            .lock()
2364            .unwrap()
2365            .insert(key, std::sync::Weak::new());
2366        let closed = tokio::spawn(async move {
2367            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2368            drop(closing);
2369        });
2370
2371        f.runtime
2372            .enable(&f.root)
2373            .await
2374            .expect("a re-enable must wait for a store that is closing");
2375        closed.await.unwrap();
2376    }
2377
2378    #[tokio::test(flavor = "multi_thread")]
2379    async fn a_workspace_whose_root_vanished_is_reported_as_paused_not_deleted() {
2380        let f = fixture().await;
2381        std::fs::write(f.root.join("a.md"), "content").unwrap();
2382        f.runtime.enable(&f.root).await.unwrap();
2383        f.runtime.scan(&f.root).await.unwrap();
2384
2385        // The drive is unmounted: the marker directory goes with it.
2386        std::fs::remove_dir_all(f.root.join(".sapphire-synctest")).unwrap();
2387        let _ = f.runtime.scan(&f.root).await;
2388
2389        let status = f.runtime.status(&f.root).await;
2390        assert!(
2391            status.paused.is_some(),
2392            "a missing root must pause, not replicate as a mass deletion"
2393        );
2394    }
2395
2396    #[tokio::test(flavor = "multi_thread")]
2397    async fn a_directory_that_is_not_a_workspace_cannot_be_enabled() {
2398        let f = fixture().await;
2399        let plain = f._tmp.path().join("plain");
2400        std::fs::create_dir_all(&plain).unwrap();
2401
2402        assert!(f.runtime.enable(&plain).await.is_err());
2403    }
2404
2405    #[tokio::test(flavor = "multi_thread")]
2406    async fn a_write_through_the_workspace_router_is_committed_to_the_replica() {
2407        // The brief's "main path": a write the app server itself made is scanned straight
2408        // after, not left to the watcher's debounce. Asserted against the replica's own
2409        // state, which is the only thing that can tell the two paths apart — reading the
2410        // file back would pass even if the scan never ran.
2411        let f = fixture().await;
2412        f.runtime.enable(&f.root).await.unwrap();
2413
2414        let host = Arc::new(crate::WorkspaceHost::new(&CTX));
2415        host.select(&f.root, false).await.unwrap();
2416        let current = Arc::new(crate::Current::new(
2417            &CTX,
2418            host,
2419            Some(Arc::clone(&f.runtime)),
2420            crate::selection::SelectionFile::at(f._tmp.path().join("workspace.toml")),
2421        ));
2422        let router = Arc::new(crate::workspace_router(current));
2423        let (client_conn, server_conn) = sapphire_ipc::Connection::pair();
2424        tokio::spawn(async move {
2425            let info = sapphire_ipc::ServerInfo {
2426                version: "0.0.0".into(),
2427                api: sapphire_backend::protocol::API_VERSION,
2428                app_api: None,
2429                pid: std::process::id(),
2430                managed_by: ManagedBy::Spawned,
2431            };
2432            let _ = sapphire_ipc::serve(server_conn, router, "sapphire-synctest", info).await;
2433        });
2434        let info = sapphire_ipc::ClientInfo {
2435            kind: "test".into(),
2436            version: "0.0.0".into(),
2437            api: sapphire_backend::protocol::API_VERSION,
2438            app_api: None,
2439            pid: std::process::id(),
2440        };
2441        let (client, _) = sapphire_ipc::Client::handshake(client_conn, "sapphire-synctest", info)
2442            .await
2443            .unwrap();
2444
2445        let _: sapphire_backend::protocol::Ack = client
2446            .call(
2447                sapphire_backend::protocol::WRITE_FILE,
2448                sapphire_backend::protocol::ContentParams {
2449                    path: PathBuf::from("through-the-server.md"),
2450                    content: "written by the server".into(),
2451                },
2452            )
2453            .await
2454            .unwrap();
2455
2456        // `scan` runs before the reply is written, so by the time the call returns the
2457        // replica must already know the path.
2458        let synced = f.runtime.synced.lock().await;
2459        let entry = synced.get(&f.root).expect("enabled above");
2460        let replica = entry.replica.lock().await;
2461        let state = replica
2462            .state("through-the-server.md")
2463            .expect("a readable state");
2464        assert!(
2465            state.is_some(),
2466            "a write through the server must be scanned into the replica, not left to the watcher"
2467        );
2468    }
2469
2470    #[tokio::test(flavor = "multi_thread")]
2471    async fn an_announcement_for_an_unknown_workspace_is_ignored_not_fatal() {
2472        let f = fixture().await;
2473        f.runtime.enable(&f.root).await.unwrap();
2474
2475        let driver = Arc::clone(&f.runtime);
2476        let handle = tokio::spawn(async move { driver.run().await });
2477
2478        f.stub
2479            .announce(grain_id::GrainId::random(), "no-such-ticket")
2480            .await;
2481        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
2482
2483        assert!(
2484            !handle.is_finished(),
2485            "one bad announcement must not stop the runtime"
2486        );
2487        handle.abort();
2488    }
2489}