Skip to main content

sapphire_framework_server/sync/
mod.rs

1//! Replication, from the app server's side.
2//!
3//! One `Replica` per synced workspace, registered with the bridge, driven by announcements
4//! from it and by local edits.
5//!
6//! See `docs/superpowers/specs/2026-09-16-process-architecture-design.md` §4.2.
7
8use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use grain_id::GrainId;
13use sapphire_backend::protocol as proto;
14use sapphire_bridge_api::{
15    IncomingParams, RegisterParams, WorkspaceRegistration, WorkspacesResult,
16};
17use sapphire_bridge_client::BridgeClient;
18use sapphire_ipc::ManagedBy;
19use sapphire_sync::{PathUpdate, PauseReason, Replica, ReplicaConfig, ScanOutcome, SystemClock};
20use sapphire_workspace::{AppContext, Workspace};
21use tokio::sync::{Mutex, OnceCell, broadcast, mpsc};
22
23pub mod id;
24mod live;
25mod methods;
26#[cfg(any(test, feature = "test-util"))]
27pub mod testing;
28mod topology;
29mod watch;
30
31pub use id::{SYNC_ID_FILE, WORKSPACE_MAP_FILE, sync_id, sync_id_path};
32pub use live::{DIAL_BACKOFF_MAX, OPEN_TIMEOUT};
33pub use methods::sync_router;
34/// What [`SyncRuntime::peers`] answers with.
35pub use sapphire_bridge_api::{PeerInfo, PeersResult};
36pub use watch::{DEBOUNCE, Watcher};
37
38use live::{LivePeers, dial_backoff};
39
40use crate::error::{Error, Result};
41use crate::settings::Settings;
42
43use crate::host::WorkspaceHost;
44
45/// What `sync.status` reports.
46#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
47pub struct SyncStatus {
48    /// Whether this workspace is synced at all.
49    pub enabled: bool,
50    /// Its identity across devices, when it is.
51    pub workspace_id: Option<GrainId>,
52    /// How many devices the workgroup has, this host excluded.
53    pub peers: usize,
54    /// Why replication is paused, if it is — a missing root or marker directory.
55    pub paused: Option<String>,
56    /// The last failure, if any.
57    pub last_error: Option<String>,
58    /// Whether the bridge is reachable. `false` is not an outage: the app server works.
59    pub bridge_available: bool,
60    /// How the workspace is wired to its peers, by the roles the bridge last reported.
61    pub topology: proto::Topology,
62    /// How far embedding has got, when this host embeds (#188).
63    pub embedding: Option<proto::EmbeddingProgress>,
64}
65
66/// One workspace this runtime syncs.
67struct Synced {
68    /// Its identity across devices, shared with every peer.
69    workspace_id: GrainId,
70    /// Its replica store. Held open because the store is a redb database.
71    replica: Arc<Mutex<Replica>>,
72    /// Why replication is paused, from the last scan.
73    paused: Option<PauseReason>,
74    /// The last failure, if any.
75    last_error: Option<String>,
76}
77
78/// How old a vector file no live content needs must be before the primary device removes
79/// it: a vector can arrive before its file, and a day is far longer than that window.
80const STALE_VECTOR_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
81
82/// Texts per embedding request in a server pass: small, so a search query waiting on the
83/// bridge's one worker is never stuck behind a long backfill (#188).
84const PASS_BATCH: usize = 4;
85
86/// How the primary device paces its backfill (#188).
87#[derive(Clone, Copy, Debug)]
88pub struct BackfillTiming {
89    /// How old another device's version must be before the primary embeds it: its author
90    /// has embedded it by then, or will not.
91    pub grace: std::time::Duration,
92    /// How often the primary device runs a pass with nothing prompting it.
93    pub interval: std::time::Duration,
94}
95
96impl Default for BackfillTiming {
97    fn default() -> Self {
98        BackfillTiming {
99            grace: std::time::Duration::from_secs(10 * 60),
100            interval: std::time::Duration::from_secs(10 * 60),
101        }
102    }
103}
104
105/// How long [`SyncRuntime::keep_connected`] waits before its first retry after the bridge
106/// could not be reached. Each failure doubles it, up to [`RECONNECT_BACKOFF_MAX`].
107pub const RECONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_millis(500);
108
109/// The longest [`SyncRuntime::keep_connected`] waits between two attempts to reach the
110/// bridge: a bridge that comes up is found within this long.
111pub const RECONNECT_BACKOFF_MAX: std::time::Duration = std::time::Duration::from_secs(30);
112
113/// The wait before reconnection attempt `attempt` (0-based), backing off from `first`.
114fn reconnect_backoff(first: std::time::Duration, attempt: u32) -> std::time::Duration {
115    first
116        .saturating_mul(1u32 << attempt.min(16))
117        .min(RECONNECT_BACKOFF_MAX.max(first))
118}
119
120/// What a connection attempt to the bridge yields.
121type Connecting = std::pin::Pin<
122    Box<dyn std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>> + Send>,
123>;
124
125/// How a [`SyncRuntime`] reaches the bridge, on start and whenever it has lost it (#244).
126///
127/// An attempt answers `Ok(None)` when no bridge is running — the runtime then tries again
128/// later — and an error for anything else, such as a bridge of another API version.
129#[derive(Clone)]
130pub struct BridgeConnector(Arc<dyn Fn() -> Connecting + Send + Sync>);
131
132impl BridgeConnector {
133    /// Connect with `connect`.
134    pub fn new<F, Fut>(connect: F) -> BridgeConnector
135    where
136        F: Fn() -> Fut + Send + Sync + 'static,
137        Fut: std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>>
138            + Send
139            + 'static,
140    {
141        BridgeConnector(Arc::new(move || Box::pin(connect())))
142    }
143
144    /// The host's own bridge, at its standard endpoint, as a client of `kind` at
145    /// `version`.
146    pub fn standard(kind: &'static str, version: &'static str) -> BridgeConnector {
147        BridgeConnector::new(move || async move {
148            match BridgeClient::connect(kind, version).await {
149                Ok(client) => Ok(Some(client)),
150                Err(sapphire_ipc::Error::NotRunning(_)) => Ok(None),
151                Err(err) => Err(err),
152            }
153        })
154    }
155
156    /// The bridge listening at `endpoint`: for a fixture that runs its own.
157    pub fn at(
158        endpoint: sapphire_ipc::Endpoint,
159        kind: &'static str,
160        version: &'static str,
161    ) -> BridgeConnector {
162        BridgeConnector::new(move || {
163            let endpoint = endpoint.clone();
164            async move { BridgeClient::connect_at(&endpoint, kind, version).await }
165        })
166    }
167
168    /// One attempt: `None` when no bridge is running.
169    pub(crate) async fn connect(&self) -> sapphire_ipc::Result<Option<BridgeClient>> {
170        (self.0)().await
171    }
172}
173
174impl std::fmt::Debug for BridgeConnector {
175    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
176        f.write_str("BridgeConnector")
177    }
178}
179
180/// What [`SyncRuntime::reclaim`] found of a replica `disable` let go of.
181enum Reclaimed {
182    /// Still held: here it is.
183    Open(Arc<Mutex<Replica>>),
184    /// Let go of since, so its store may be closing still.
185    Closing,
186    /// Nothing to take back.
187    Closed,
188}
189
190/// How often, and how far apart, `enable` tries to open a store that may still be closing.
191const CLOSING_RETRIES: u32 = 50;
192const CLOSING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(20);
193
194/// Replication for one application's workspaces.
195pub struct SyncRuntime {
196    ctx: &'static AppContext,
197    /// The bridge connection, while there is one. Swapped when the runtime reconnects
198    /// (#244); [`run`](SyncRuntime::run) follows it.
199    bridge: tokio::sync::watch::Sender<Option<Arc<BridgeClient>>>,
200    /// How to reach the bridge again; `None` for a runtime given one connection to use.
201    connector: Option<BridgeConnector>,
202    /// Woken by [`reconnect_now`](SyncRuntime::reconnect_now): someone wants the bridge
203    /// now, so [`keep_connected`](SyncRuntime::keep_connected) cuts its backoff short.
204    reconnect_now: tokio::sync::Notify,
205    /// Workspaces asked to sync while the bridge was away, with their ids. Enabled once
206    /// it is back.
207    ///
208    /// Also the lock that orders an `enable` against a reconnection: whoever holds it sees
209    /// the bridge slot as the other left it, so a workspace is either enabled now or left
210    /// here for the reconnection to enable — never neither.
211    pending: Mutex<HashMap<PathBuf, GrainId>>,
212    /// The announcements of a connection [`attach`](SyncRuntime::attach) just made,
213    /// subscribed to before it registered: the bridge announces the streams that waited for
214    /// this server as soon as it registers (#244), and a subscription made afterwards by
215    /// [`run`](SyncRuntime::run) would miss them.
216    subscribed: std::sync::Mutex<Option<(Arc<BridgeClient>, broadcast::Receiver<IncomingParams>)>>,
217    exe_path: PathBuf,
218    managed_by: ManagedBy,
219    synced: Mutex<HashMap<PathBuf, Synced>>,
220    /// The replicas [`disable`](SyncRuntime::disable) let go of, by canonical root, for as
221    /// long as something else still holds one: a scan, a session or the first dial that
222    /// began before. The store is a redb database, open until its last holder drops it, so
223    /// a re-enable meanwhile could not open it again; [`enable`](SyncRuntime::enable)
224    /// takes the open one back instead. [`release`](SyncRuntime::release) puts nothing
225    /// here: its store is about to be moved or deleted.
226    stopped: std::sync::Mutex<HashMap<PathBuf, std::sync::Weak<Mutex<Replica>>>>,
227    /// This host's device id. Asked of the bridge once, on the first [`enable`], and never
228    /// again: a second registration with an empty workspace list would briefly clear this
229    /// application's routes.
230    ///
231    /// [`enable`]: SyncRuntime::enable
232    device_id: OnceCell<GrainId>,
233    /// The file watcher, once [`watch_changes`] has started it.
234    ///
235    /// The runtime owns it because the spec puts it here: the sync runtime is a watcher and
236    /// a replica per registered root, and the two must agree on which roots are synced.
237    /// [`enable`] and [`disable`] keep them in step.
238    ///
239    /// [`watch_changes`]: SyncRuntime::watch_changes
240    /// [`enable`]: SyncRuntime::enable
241    /// [`disable`]: SyncRuntime::disable
242    watcher: OnceCell<Arc<Watcher>>,
243    /// The app server that owns these workspaces' caches, once it has told us.
244    ///
245    /// A replication session writes files, and the search index is not part of a session: the
246    /// index belongs to the app server, which holds the workspace open. Reaching it through
247    /// the host is what keeps a received file and its index moving together, and it is the
248    /// *same* [`WorkspaceState`](sapphire_workspace::WorkspaceState) the server serves from —
249    /// opening a second one would collide on the retrieve store's exclusive lock.
250    ///
251    /// Absent for a runtime nobody has wired to an app server (a test stub driving the
252    /// replica directly). Nothing then indexes, which is correct: nothing owns an index.
253    host: OnceCell<Arc<WorkspaceHost>>,
254    /// The app server's settings, whose workspace scope is a file in the selected
255    /// workspace's marker: read again when a session or a local edit changes the files.
256    settings: std::sync::Mutex<Option<Settings>>,
257    /// Serialises re-indexing, so two sessions finishing at once do not both sweep the
258    /// workspace through the same store.
259    reindexing: Mutex<()>,
260    /// Roots with an embedding pass waiting to run (#187). A request for a root already
261    /// waiting is folded into that pass.
262    embed_wanted: Mutex<std::collections::HashSet<PathBuf>>,
263    /// Serialises embedding passes: one model, one queue.
264    embedding: Mutex<()>,
265    /// Roots whose embedding pass is running now, for `sync.status`.
266    embed_running: Mutex<std::collections::HashSet<PathBuf>>,
267    /// The primary device's backfill pacing (#188).
268    backfill: std::sync::Mutex<BackfillTiming>,
269    /// The bridge an embedding pass asks for its model; `None` is the host's standard one.
270    bridge_endpoint: Option<sapphire_ipc::Endpoint>,
271    /// The live session table of every synced workspace, keyed by canonical root.
272    ///
273    /// One table per workspace rather than one for the runtime: a session is about one
274    /// workspace, and the two storm rules are per-workspace rules. A root that is not
275    /// synced has none, so a dial or a push for it is a no-op rather than an error.
276    live: Mutex<HashMap<PathBuf, Arc<LivePeers>>>,
277    /// The bridge's last answer to `peers`, roles included. The inbound guard and
278    /// [`is_primary`](SyncRuntime::is_primary) read it rather than asking again.
279    last_peers: Mutex<Option<PeersResult>>,
280    /// How long a dial waits for the bridge to open a stream; [`OPEN_TIMEOUT`] by default.
281    open_timeout: std::time::Duration,
282    /// The first wait of [`keep_connected`](SyncRuntime::keep_connected)'s backoff;
283    /// [`RECONNECT_BACKOFF`] but for tests.
284    reconnect_first: std::time::Duration,
285    /// The first dials [`enable`](SyncRuntime::enable) leaves running. Each pins the
286    /// runtime, so [`stop_dials`](SyncRuntime::stop_dials) ends them with the server.
287    first_dials: std::sync::Mutex<tokio::task::JoinSet<()>>,
288    /// The rest of the runtime's background work: inbound sessions, the readers of live
289    /// sessions, and embedding passes. Each pins the runtime — and through it the replica
290    /// stores, which are redb databases locked until the last pin drops — so
291    /// [`stop_background`](SyncRuntime::stop_background) ends them with the server (#235).
292    background: std::sync::Mutex<tokio::task::JoinSet<()>>,
293}
294
295impl SyncRuntime {
296    /// A runtime for `ctx`'s application, syncing through `bridge`.
297    ///
298    /// It has no way to reach the bridge again: when the connection closes, sync stops until
299    /// the runtime is given a [`connector`](Self::with_connector).
300    pub fn new(
301        ctx: &'static AppContext,
302        bridge: Arc<BridgeClient>,
303        exe_path: PathBuf,
304        managed_by: ManagedBy,
305    ) -> SyncRuntime {
306        SyncRuntime::build(ctx, Some(bridge), None, exe_path, managed_by)
307    }
308
309    /// A runtime for `ctx`'s application that reaches the bridge with `connector`, and again
310    /// whenever it loses it (#244).
311    ///
312    /// Not connected yet: [`connect`](Self::connect) makes the first attempt, and
313    /// [`keep_connected`](Self::keep_connected) the rest. [`AppServer::run`] does both.
314    ///
315    /// [`AppServer::run`]: crate::AppServer::run
316    pub fn connecting(
317        ctx: &'static AppContext,
318        connector: BridgeConnector,
319        exe_path: PathBuf,
320        managed_by: ManagedBy,
321    ) -> SyncRuntime {
322        SyncRuntime::build(ctx, None, Some(connector), exe_path, managed_by)
323    }
324
325    /// Reach the bridge again with `connector` once the connection closes.
326    pub fn with_connector(mut self, connector: BridgeConnector) -> SyncRuntime {
327        self.connector = Some(connector);
328        self
329    }
330
331    fn build(
332        ctx: &'static AppContext,
333        bridge: Option<Arc<BridgeClient>>,
334        connector: Option<BridgeConnector>,
335        exe_path: PathBuf,
336        managed_by: ManagedBy,
337    ) -> SyncRuntime {
338        SyncRuntime {
339            ctx,
340            bridge: tokio::sync::watch::Sender::new(bridge),
341            connector,
342            reconnect_now: tokio::sync::Notify::new(),
343            pending: Mutex::new(HashMap::new()),
344            subscribed: std::sync::Mutex::new(None),
345            exe_path,
346            managed_by,
347            synced: Mutex::new(HashMap::new()),
348            stopped: std::sync::Mutex::new(HashMap::new()),
349            device_id: OnceCell::new(),
350            watcher: OnceCell::new(),
351            host: OnceCell::new(),
352            settings: std::sync::Mutex::new(None),
353            reindexing: Mutex::new(()),
354            embed_wanted: Mutex::new(std::collections::HashSet::new()),
355            embedding: Mutex::new(()),
356            embed_running: Mutex::new(std::collections::HashSet::new()),
357            backfill: std::sync::Mutex::new(BackfillTiming::default()),
358            bridge_endpoint: None,
359            live: Mutex::new(HashMap::new()),
360            last_peers: Mutex::new(None),
361            open_timeout: OPEN_TIMEOUT,
362            reconnect_first: RECONNECT_BACKOFF,
363            first_dials: std::sync::Mutex::new(tokio::task::JoinSet::new()),
364            background: std::sync::Mutex::new(tokio::task::JoinSet::new()),
365        }
366    }
367
368    /// The bridge, while connected.
369    fn bridge(&self) -> Result<Arc<BridgeClient>> {
370        match &*self.bridge.borrow() {
371            Some(bridge) if !bridge.is_closed() => Ok(Arc::clone(bridge)),
372            _ => Err(Error::BridgeDisconnected),
373        }
374    }
375
376    /// Whether the runtime has a live connection to the bridge.
377    pub fn is_connected(&self) -> bool {
378        self.bridge().is_ok()
379    }
380
381    /// The bridge connection, while there is an open one: what the relay forwards over.
382    pub(crate) fn connected_bridge(&self) -> Option<Arc<BridgeClient>> {
383        self.bridge().ok()
384    }
385
386    /// Have [`keep_connected`](Self::keep_connected) try the bridge now rather than when its
387    /// backoff next ends, and start its backoff over: for the relay, whose client may just
388    /// have started the bridge. A no-op while connected, bar one early attempt after the
389    /// next loss, which makes one at once anyway.
390    pub(crate) fn reconnect_now(&self) {
391        self.reconnect_now.notify_one();
392    }
393
394    /// The bridge slot, followed: changes each time the runtime connects or loses the
395    /// connection.
396    pub(crate) fn bridge_changes(&self) -> tokio::sync::watch::Receiver<Option<Arc<BridgeClient>>> {
397        self.bridge.subscribe()
398    }
399
400    /// Run `task` in the background, tracked so the server's shutdown ends it.
401    fn spawn_background(&self, task: impl std::future::Future<Output = ()> + Send + 'static) {
402        let mut background = self.background.lock().expect("background tasks");
403        // Reap the finished ones, so the set holds only what is running.
404        while background.try_join_next().is_some() {}
405        background.spawn(task);
406    }
407
408    /// Give up on opening a stream to a peer after `timeout` rather than [`OPEN_TIMEOUT`]:
409    /// for tests, which cannot wait that long for a peer that never answers.
410    pub fn with_open_timeout(mut self, timeout: std::time::Duration) -> SyncRuntime {
411        self.open_timeout = timeout;
412        self
413    }
414
415    /// Back off from `first` rather than [`RECONNECT_BACKOFF`] while the bridge cannot be
416    /// reached: for tests of a backoff that has grown long, which cannot wait for it to.
417    #[cfg(test)]
418    pub(crate) fn with_reconnect_backoff(mut self, first: std::time::Duration) -> SyncRuntime {
419        self.reconnect_first = first;
420        self
421    }
422
423    /// Tell the runtime which app server owns these workspaces' caches.
424    ///
425    /// Called by [`AppServer::sync`](crate::AppServer::sync). First writer wins: a runtime
426    /// belongs to the server that wired it.
427    /// Embed through the bridge at `endpoint` rather than the host's standard one: for a
428    /// fixture running several hosts in one process.
429    pub fn with_bridge_endpoint(mut self, endpoint: sapphire_ipc::Endpoint) -> SyncRuntime {
430        self.bridge_endpoint = Some(endpoint);
431        self
432    }
433
434    /// Pace the primary device's backfill differently: for tests, which cannot wait ten
435    /// minutes.
436    pub fn set_backfill_timing(&self, timing: BackfillTiming) {
437        *self.backfill.lock().expect("backfill timing") = timing;
438    }
439
440    fn backfill_timing(&self) -> BackfillTiming {
441        *self.backfill.lock().expect("backfill timing")
442    }
443
444    pub(crate) fn set_host(&self, host: Arc<WorkspaceHost>) {
445        let _ = self.host.set(host);
446    }
447
448    /// Read `settings` again whenever a workspace's files change under this runtime — a
449    /// session wrote them, or someone edited them — so workspace keys follow the marker's
450    /// `config.toml`. Replaced, not kept, by a later call (a restarted server's).
451    pub(crate) fn set_settings(&self, settings: Option<Settings>) {
452        *self.settings.lock().unwrap_or_else(|e| e.into_inner()) = settings;
453    }
454
455    /// Read the settings again after the files under a workspace changed. Cheap — two
456    /// small files — and holding no lock past the read, so it runs on every change.
457    fn refresh_settings(&self) {
458        let settings = self
459            .settings
460            .lock()
461            .unwrap_or_else(|e| e.into_inner())
462            .clone();
463        if let Some(settings) = settings
464            && settings.refresh().is_err()
465        {
466            // The error may quote the file; the change itself went through.
467            tracing::warn!("could not read the settings again after a change to the files");
468        }
469    }
470
471    /// Start syncing `root`, returning its identity across devices.
472    ///
473    /// Idempotent: enabling an already-synced workspace returns the same id and changes
474    /// nothing.
475    pub async fn enable(self: &Arc<Self>, root: &Path) -> Result<GrainId> {
476        let key = root.canonicalize().map_err(Error::Io)?;
477        let watch_key = key.clone();
478        let workspace_id = sync_id(self.ctx.app_name, &key)?;
479
480        // Without a bridge there is no device id to open the replica with, and nobody to
481        // register with: keep the workspace for the reconnection to enable (#244).
482        {
483            let mut pending = self.pending.lock().await;
484            if self.bridge().is_err() {
485                pending.insert(key, workspace_id);
486                return Err(Error::BridgeDisconnected);
487            }
488            pending.remove(&key);
489        }
490
491        // The map stays locked across opening the replica, so two simultaneous `enable`s for
492        // one root cannot both open it — the store is a redb database, and it is opened once.
493        {
494            let mut synced = self.synced.lock().await;
495            if let Some(existing) = synced.get(&key) {
496                return Ok(existing.workspace_id);
497            }
498            // A replica disabled a moment ago may still be open; it is this root's, with this
499            // runtime's one device id, so it is taken back rather than opened a second time.
500            let replica = match self.reclaim(&key) {
501                Reclaimed::Open(replica) => replica,
502                reclaimed => {
503                    let device_id = self.device_id().await?;
504                    let state_dir = self.state_dir(&key)?;
505                    std::fs::create_dir_all(&state_dir).map_err(Error::Io)?;
506                    let config =
507                        ReplicaConfig::new(self.ctx.app_name, key.clone(), device_id, &state_dir);
508                    // The last holder let go between the two: its store closes as the replica
509                    // drops, just after it can no longer be taken back. Give it that moment.
510                    let mut tries = match reclaimed {
511                        Reclaimed::Closing => CLOSING_RETRIES,
512                        _ => 0,
513                    };
514                    let replica = loop {
515                        match Replica::open(config.clone(), Arc::new(SystemClock)) {
516                            Ok(replica) => break replica,
517                            Err(_) if tries > 0 => {
518                                tries -= 1;
519                                tokio::time::sleep(CLOSING_RETRY_DELAY).await;
520                            }
521                            Err(err) => return Err(Error::Sync(err.to_string())),
522                        }
523                    };
524                    Arc::new(Mutex::new(replica))
525                }
526            };
527            synced.insert(
528                key.clone(),
529                Synced {
530                    workspace_id,
531                    replica,
532                    paused: None,
533                    last_error: None,
534                },
535            );
536        }
537
538        // Watch the new root too, so an edit made outside the server from now on is seen.
539        // Not fatal if it fails: the app server's own writes take the exact path in
540        // `handlers.rs`, and the next `watch_changes` re-reads the root set anyway.
541        if let Some(watcher) = self.watcher.get()
542            && let Err(err) = watcher.watch(&watch_key)
543        {
544            tracing::warn!(root = %watch_key.display(), "could not watch: {err}");
545        }
546
547        // Register only once the replica is open, so a registration never names a workspace
548        // that cannot serve a session.
549        self.reregister().await?;
550
551        // Dial now, so enabling a workspace converges it rather than waiting for the next
552        // local edit to trigger a session. This is what catches a host up when it returns:
553        // the peer that has been running is the one the returning host has to ask, and no
554        // watcher on either side fires for a file that was never written locally.
555        //
556        // Best-effort: a workspace is enabled whether or not any peer is up, and a failure
557        // here is not a failure to enable. A session is symmetric, so dialing is enough —
558        // this side sends its own view, and the peer answers with what this side lacks.
559        //
560        // In the background, so the answer does not wait on it: a peer that is offline
561        // holds a dial up to `open_timeout`, longer than a client waits for `sync.enable`
562        // (#228).
563        //
564        // Scan first: a file already on disk when the workspace is enabled — a folder that
565        // existed before sync was turned on, or an edit made while the server was down —
566        // fires no watcher, so without this it is never recorded and the session has
567        // nothing to send. Logged, not fatal: the session still fetches what peers have.
568        let runtime = Arc::clone(self);
569        let mut dials = self.first_dials.lock().expect("first dials");
570        // Reap the finished ones, so the set holds only what is running.
571        while dials.try_join_next().is_some() {}
572        dials.spawn(async move {
573            if let Err(err) = runtime.scan(&key).await {
574                tracing::warn!(root = %key.display(), "the first scan after enabling failed: {err}");
575            }
576            if let Err(err) = runtime.sync_now(&key).await {
577                tracing::warn!(root = %key.display(), "the first session after enabling failed: {err}");
578            }
579        });
580        Ok(workspace_id)
581    }
582
583    /// Stop syncing `root`. Files and the sync id stay, so re-enabling rejoins the same
584    /// workspace rather than creating a second one.
585    ///
586    /// Something that began before — a scan, a session, the first dial — may still hold the
587    /// replica open; a re-enable meanwhile takes it back (see [`reclaim`](Self::reclaim)).
588    pub async fn disable(&self, root: &Path) -> Result<()> {
589        let (replica, unregistered) = self.stop(root).await;
590        if let Some(replica) = replica {
591            let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
592            let mut stopped = self.stopped.lock().expect("stopped replicas");
593            // Forget the ones nothing holds any more, so the map stays small.
594            stopped.retain(|_, replica| replica.strong_count() > 0);
595            stopped.insert(key, Arc::downgrade(&replica));
596        }
597        unregistered
598    }
599
600    /// Stop syncing `root`, like [`disable`](Self::disable), then wait up to `limit` until
601    /// nothing holds its replica store open any more — a session, a scan or an embedding
602    /// pass that started before may still have it. A store about to be moved or deleted
603    /// must be closed first.
604    ///
605    /// Only a store still in use at `limit` is an error. The bridge not hearing of it is
606    /// logged, not returned: the store is closed all the same, and the next registration
607    /// replaces the bridge's view.
608    pub async fn release(&self, root: &Path, limit: std::time::Duration) -> Result<()> {
609        let (replica, unregistered) = self.stop(root).await;
610        if let Err(err) = unregistered {
611            tracing::warn!(root = %root.display(), "stopping sync: {err}");
612        }
613        let Some(replica) = replica else {
614            return Ok(());
615        };
616        let deadline = tokio::time::Instant::now() + limit;
617        while Arc::strong_count(&replica) > 1 {
618            if tokio::time::Instant::now() >= deadline {
619                return Err(Error::Sync(format!(
620                    "the replica of {} is still in use; try again",
621                    root.display()
622                )));
623            }
624            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
625        }
626        Ok(())
627    }
628
629    /// Stop syncing `root`, handing back its replica — `None` when it was not synced — and
630    /// how telling the bridge went.
631    async fn stop(&self, root: &Path) -> (Option<Arc<Mutex<Replica>>>, Result<()>) {
632        // A root that no longer exists cannot be canonicalized, but the keys of `synced` are
633        // canonical already, so the path as given (a registry root) still names the entry.
634        let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
635        self.pending.lock().await.remove(&key);
636        let removed = self.synced.lock().await.remove(&key);
637        if removed.is_some()
638            && let Some(watcher) = self.watcher.get()
639        {
640            watcher.unwatch(&key);
641        }
642        // Any live session for a workspace this server no longer holds is closed: it would
643        // otherwise keep pushing files for a workspace nobody asked about any more.
644        if let Some(peers) = self.live.lock().await.remove(&key) {
645            peers.drop_connections().await;
646        }
647        let Some(removed) = removed else {
648            return (None, Ok(()));
649        };
650        let unregistered = async {
651            // A bridge that is gone has forgotten this server's registrations already, and
652            // the reconnection registers only what is still synced.
653            let Ok(bridge) = self.bridge() else {
654                return Ok(());
655            };
656            bridge
657                .unregister(removed.workspace_id)
658                .await
659                .map_err(|e| Error::Bridge(e.to_string()))?;
660            self.reregister().await
661        }
662        .await;
663        (Some(removed.replica), unregistered)
664    }
665
666    /// The workspaces the workgroup knows about.
667    ///
668    /// Read straight from the bridge, so it reflects every device's registrations. The
669    /// bridge being down is not an empty answer — that would read as "nothing to map" — so
670    /// it is an error, and `sync.map` reports it.
671    pub async fn workspaces(&self) -> Result<WorkspacesResult> {
672        self.bridge()?
673            .workspaces()
674            .await
675            .map_err(|e| Error::Bridge(e.to_string()))
676    }
677
678    /// Place a workgroup workspace at a directory of this host: write its identity and
679    /// map there, returning the canonical root. [`Current::map`](crate::Current::map)
680    /// then selects it and syncs it.
681    ///
682    /// `selector` is the workspace's name or id, as [`workspaces`] lists it; `dir` must
683    /// exist. Its marker is made when it has none, and nothing else: the marker's
684    /// `config.toml` is the application's, and arrives with the first sync — a local one
685    /// written first would meet it as a conflict (#253).
686    ///
687    /// [`workspaces`]: SyncRuntime::workspaces
688    pub async fn place(&self, selector: &str, dir: &Path) -> Result<PathBuf> {
689        let workgroup = self.workspaces().await?;
690        let wanted = workgroup
691            .workspaces
692            .iter()
693            // A name is the user-facing handle; the id is the exact one. Two workspaces
694            // may not share a name, so a name names at most one.
695            .find(|w| w.name == selector || w.workspace_id.to_string() == selector)
696            .ok_or_else(|| Error::UnknownWorkspaceName(selector.to_owned()))?;
697
698        // Refuse another application's workspace: this server syncs its own application's
699        // workspaces, and `Workspace::from_root` would later refuse the directory anyway.
700        if wanted.app_name != self.ctx.app_name {
701            return Err(Error::WrongApp {
702                name: wanted.name.clone(),
703                app_name: wanted.app_name.clone(),
704            });
705        }
706
707        let root = dir.canonicalize().map_err(Error::Io)?;
708        let marker = root.join(format!(".{}", self.ctx.app_name));
709        if !marker.is_dir() {
710            std::fs::create_dir(&marker).map_err(Error::Io)?;
711        }
712        let workspace = Workspace::from_root(self.ctx, &root)?;
713        if workspace.root != root {
714            return Err(Error::UnknownWorkspace(root, self.ctx.app_name));
715        }
716
717        // A directory already syncing under another identity is not remapped: the sync id
718        // is a device's word that it is the same workspace, and rewriting it silently
719        // would make two hosts disagree about what they share.
720        {
721            let synced = self.synced.lock().await;
722            if let Some(entry) = synced.get(&root)
723                && entry.workspace_id != wanted.workspace_id
724            {
725                return Err(Error::SyncId(format!(
726                    "{} already syncs as {}, not {}",
727                    root.display(),
728                    entry.workspace_id,
729                    wanted.workspace_id
730                )));
731            }
732        }
733
734        // The identity arrives from the workgroup, not from this host: two hosts that map
735        // the same workspace must sync as one, so the id is the one the workgroup lists,
736        // written before `enable` reads it. A different id already on disk is refused
737        // above; an absent one is written here.
738        let marker = workspace.marker_dir();
739        let id_path = marker.join(crate::sync::id::SYNC_ID_FILE);
740        match std::fs::read_to_string(&id_path) {
741            Ok(existing) if existing.trim() != wanted.workspace_id.to_string() => {
742                return Err(Error::SyncId(format!(
743                    "{} already holds identity {}, not {}",
744                    root.display(),
745                    existing.trim(),
746                    wanted.workspace_id
747                )));
748            }
749            Err(_) => {
750                std::fs::write(&id_path, format!("{}\n", wanted.workspace_id))
751                    .map_err(Error::Io)?;
752            }
753            Ok(_) => {}
754        }
755
756        // The map is what tells this application, on a later start, which workgroup
757        // workspace a directory is.
758        let map_path = root
759            .join(format!(".{}", self.ctx.app_name))
760            .join(WORKSPACE_MAP_FILE);
761        let workspace_id = wanted.workspace_id.to_string();
762        std::fs::write(&map_path, format!("{workspace_id}\n")).map_err(Error::Io)?;
763        Ok(root)
764    }
765
766    /// What `sync.status` answers with.
767    pub async fn status(&self, root: &Path) -> SyncStatus {
768        let bridge = self.bridge().ok();
769        let bridge_available = match &bridge {
770            Some(bridge) => bridge.status().await.is_ok(),
771            None => false,
772        };
773        let answer = match &bridge {
774            Some(_) => self.peers().await.ok(),
775            None => None,
776        };
777        let peers = answer
778            .as_ref()
779            .map_or(0, |p| p.peers.len().saturating_sub(1));
780        let Ok(key) = root.canonicalize() else {
781            return SyncStatus {
782                enabled: false,
783                workspace_id: None,
784                peers,
785                paused: None,
786                last_error: None,
787                bridge_available,
788                topology: proto::Topology::Mesh,
789                embedding: None,
790            };
791        };
792        let synced = self.synced.lock().await;
793        match synced.get(&key) {
794            Some(entry) => SyncStatus {
795                enabled: true,
796                workspace_id: Some(entry.workspace_id),
797                peers,
798                paused: entry.paused.map(|r| format!("{r:?}")),
799                last_error: entry.last_error.clone(),
800                bridge_available,
801                topology: topology::topology(
802                    answer
803                        .as_ref()
804                        .and_then(|p| p.roles_for(entry.workspace_id)),
805                ),
806                embedding: self.embedding_progress(&key).await,
807            },
808            None => {
809                drop(synced);
810                // Asked to sync while the bridge was away: it is enabled, waiting for the
811                // bridge, and `bridge_available` says why nothing moves.
812                let waiting = self.pending.lock().await.get(&key).copied();
813                SyncStatus {
814                    enabled: waiting.is_some(),
815                    workspace_id: waiting,
816                    peers,
817                    paused: None,
818                    last_error: None,
819                    bridge_available,
820                    topology: proto::Topology::Mesh,
821                    embedding: None,
822                }
823            }
824        }
825    }
826
827    /// How far `key`'s embedding has got, when this host embeds it. Reads the open
828    /// workspace's index; never opens one, so a status call stays cheap.
829    async fn embedding_progress(&self, key: &Path) -> Option<proto::EmbeddingProgress> {
830        let host = self.host.get()?;
831        let backend = host.open_backend(key)?;
832        let state = backend.state();
833        state.embedder()?;
834        let info = state.db_info().ok()?;
835        Some(proto::EmbeddingProgress {
836            vectors: info.vector_count,
837            pending: info.pending_count,
838            running: self.embed_running.lock().await.contains(key),
839        })
840    }
841
842    /// Bring the replica's view of the files up to date.
843    ///
844    /// Called straight after the app server's own writes, and by the watcher for everything
845    /// else. A scan that finds nothing is cheap; a scan that is skipped loses an edit until
846    /// the next one.
847    pub async fn scan(&self, root: &Path) -> Result<()> {
848        let Ok(key) = root.canonicalize() else {
849            return Ok(());
850        };
851        let replica = {
852            let synced = self.synced.lock().await;
853            match synced.get(&key) {
854                Some(entry) => Arc::clone(&entry.replica),
855                None => return Ok(()),
856            }
857        };
858        let (outcome, updates) = {
859            let mut replica = replica.lock().await;
860            let outcome = replica.scan().map_err(|e| Error::Sync(e.to_string()))?;
861            // What the scan committed, as path updates, so it can go out on the sessions
862            // that are already open. This is the whole point of the exact-path scan in
863            // `handlers.rs`: a write through the server reaches a peer live, not at the next
864            // dial.
865            let updates = match &outcome {
866                ScanOutcome::Scanned(report) => {
867                    // The unit of replication is the path state (design doc section 2.4):
868                    // the `seen` a peer prunes against is the *path's* version vector - the
869                    // context the recorded entry was built on plus its own dot - never the
870                    // whole replica vv. Sending the global vv here made receivers prune
871                    // causally-related versions they still held: a conflict copy on the
872                    // first remote edit, and tombstones pruned so deletes stopped
873                    // propagating (issue #157).
874                    report
875                        .recorded
876                        .iter()
877                        .map(|entry| PathUpdate {
878                            path: entry.path.clone(),
879                            versions: vec![entry.clone()],
880                            seen: {
881                                let mut seen = entry.context.clone();
882                                seen.add_dot(&entry.dot);
883                                seen
884                            },
885                        })
886                        .collect()
887                }
888                ScanOutcome::Paused(_) => Vec::new(),
889            };
890            (outcome, updates)
891        };
892        {
893            let mut synced = self.synced.lock().await;
894            if let Some(entry) = synced.get_mut(&key) {
895                entry.paused = match outcome {
896                    ScanOutcome::Paused(reason) => Some(reason),
897                    ScanOutcome::Scanned(_) => None,
898                };
899            }
900        }
901        // After the map is unlocked: a push awaits every peer, and the table must not be
902        // held behind a slow one.
903        self.after_commit(&key, updates).await;
904        Ok(())
905    }
906
907    /// Bring the search index up to date with files a session has written.
908    ///
909    /// A replication session moves files, not index rows: the index belongs to the app
910    /// server, and the sync core is deliberately free of it. So the side that *applied* a
911    /// change re-indexes, which is what makes a file that arrived from a peer searchable
912    /// without a manual reindex — the claim the whole architecture rests on.
913    ///
914    /// Incremental, not a full rebuild: the workspace's mtime/size stamps mean only what
915    /// actually changed is read back. A file the session wrote has a new stamp, so it is
916    /// picked up; one that was already indexed and untouched is skipped.
917    ///
918    /// Best-effort by contract: the caller's session has already succeeded, and a failure
919    /// here must not be reported as the session's. It is logged and the next session, local
920    /// edit or explicit `workspace.reindex` tries again.
921    async fn reindex(&self, root: &Path) {
922        // The marker's `config.toml` may be among what arrived.
923        self.refresh_settings();
924        let Some(host) = self.host.get() else {
925            return;
926        };
927        let host = Arc::clone(host);
928        let root = root.to_owned();
929        // One sweep at a time: two sessions that finish together would otherwise both walk
930        // the workspace through the same retrieve store.
931        let _serialised = self.reindexing.lock().await;
932        // Timed from here, so a run waiting for the one before it is not charged its wait.
933        let started = std::time::Instant::now();
934        // Opening the workspace is itself blocking work, and it is where the exclusive
935        // retrieve-store lock is taken, so it happens here rather than inside the sweep.
936        let backend = match host.backend_at(&root).await {
937            Ok(backend) => backend,
938            Err(err) => {
939                crate::instrument::reindexed(started, false);
940                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
941                return;
942            }
943        };
944        let state = Arc::clone(backend.state());
945        let result = tokio::task::spawn_blocking(move || state.sync_retrieve()).await;
946        crate::instrument::reindexed(started, matches!(result, Ok(Ok(_))));
947        match result {
948            Ok(Ok(report)) => {
949                tracing::debug!(root = %root.display(), ?report, "re-indexed after a session");
950            }
951            Ok(Err(err)) => {
952                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
953            }
954            Err(err) => {
955                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
956            }
957        }
958    }
959
960    /// Run an embedding pass over `root` in the background, after the pass already
961    /// waiting for it if there is one (#187).
962    ///
963    /// The pass reads the vector files that arrived, and computes the missing ones this
964    /// host is responsible for: the files whose winning version this replica wrote, and —
965    /// on the primary device — every other one. It never holds up the caller.
966    pub fn request_embed(self: &Arc<Self>, root: &Path) {
967        let Ok(key) = root.canonicalize() else {
968            return;
969        };
970        let runtime = Arc::clone(self);
971        self.spawn_background(async move {
972            if !runtime.embed_wanted.lock().await.insert(key.clone()) {
973                return;
974            }
975            let _one_at_a_time = runtime.embedding.lock().await;
976            // Taken off the list as the pass starts: a change during the pass asks again.
977            runtime.embed_wanted.lock().await.remove(&key);
978            runtime.embed_running.lock().await.insert(key.clone());
979            runtime.embed_pass(&key).await;
980            runtime.embed_running.lock().await.remove(&key);
981        });
982    }
983
984    /// One embedding pass over `key`. Best-effort, like the re-index: logged, never fatal.
985    async fn embed_pass(&self, key: &Path) {
986        let Some(host) = self.host.get() else {
987            return;
988        };
989        let (workspace_id, replica) = {
990            let synced = self.synced.lock().await;
991            match synced.get(key) {
992                Some(entry) => (entry.workspace_id, Arc::clone(&entry.replica)),
993                None => return,
994            }
995        };
996        let backend = match host.backend_at(key).await {
997            Ok(backend) => backend,
998            Err(err) => {
999                tracing::warn!(root = %key.display(), "embedding pass: {err}");
1000                return;
1001            }
1002        };
1003        let state = Arc::clone(backend.state());
1004        if let Err(err) = state
1005            .refresh_embedder_at(self.bridge_endpoint.clone())
1006            .await
1007        {
1008            tracing::warn!(root = %key.display(), "embedding pass: {err}");
1009            return;
1010        }
1011        if state.embedder().is_none() {
1012            return;
1013        }
1014        let primary = self.is_primary(workspace_id).await;
1015        // The replica's view, read once: which content each path has, and which of those
1016        // versions this replica wrote.
1017        let grace_ms = self.backfill_timing().grace.as_millis() as u64;
1018        let now_ms = std::time::SystemTime::now()
1019            .duration_since(std::time::UNIX_EPOCH)
1020            .map_or(0, |d| d.as_millis() as u64);
1021        let (mine, settled, live) = {
1022            let replica = replica.lock().await;
1023            let me = replica.replica_id();
1024            let states = match replica.states() {
1025                Ok(states) => states,
1026                Err(err) => {
1027                    tracing::warn!(root = %key.display(), "embedding pass: {err}");
1028                    return;
1029                }
1030            };
1031            let mut mine = HashMap::new();
1032            // Other devices' versions old enough for the primary to fill in.
1033            let mut settled = HashMap::new();
1034            let mut live = std::collections::HashSet::new();
1035            for (path, path_state) in states {
1036                let winner = path_state.winner();
1037                let Some(hash) = winner.content.hash() else {
1038                    continue;
1039                };
1040                let hash = hash.to_hex();
1041                if winner.dot.replica == me {
1042                    mine.insert(path, hash.clone());
1043                } else if now_ms.saturating_sub(winner.hlc.wall_ms) >= grace_ms {
1044                    settled.insert(path, hash.clone());
1045                }
1046                live.insert(hash);
1047            }
1048            (mine, settled, live)
1049        };
1050        let root = key.to_owned();
1051        let result = tokio::task::spawn_blocking(move || {
1052            // The author embeds its own versions at once; the primary device fills in other
1053            // devices' versions once they have settled, racing nobody.
1054            let policy = |rel: &str, hash: &str| {
1055                mine.get(rel).is_some_and(|h| h.as_str() == hash)
1056                    || (primary && settled.get(rel).is_some_and(|h| h.as_str() == hash))
1057            };
1058            let embedded = state.embed_pending_with(&policy, PASS_BATCH)?;
1059            let removed = if primary {
1060                state.remove_stale_vectors(&live, STALE_VECTOR_AGE)
1061            } else {
1062                0
1063            };
1064            Ok::<_, sapphire_workspace::Error>((embedded, removed))
1065        })
1066        .await;
1067        match result {
1068            Ok(Ok((embedded, removed))) if embedded > 0 || removed > 0 => tracing::info!(
1069                root = %root.display(),
1070                embedded,
1071                removed,
1072                "embedding pass"
1073            ),
1074            Ok(Ok(_)) => {}
1075            Ok(Err(err)) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
1076            Err(err) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
1077        }
1078    }
1079
1080    /// Open a session with every peer that will take one.
1081    pub async fn sync_now(self: &Arc<Self>, root: &Path) -> Result<()> {
1082        let Ok(key) = root.canonicalize() else {
1083            return Ok(());
1084        };
1085        let workspace_id = {
1086            let synced = self.synced.lock().await;
1087            match synced.get(&key) {
1088                Some(entry) => entry.workspace_id,
1089                None => return Ok(()),
1090            }
1091        };
1092        let peers = self.peers().await?;
1093        let me = self.device_id().await?;
1094
1095        // The same direction rule the dial loop follows: a host only dials peers with a
1096        // greater device id. Dialling the other way from here would reintroduce the
1097        // dial-both-ways deadlock the loop was cured of, just on a different trigger.
1098        // Inside a star, a pair with neither hub in it does not link at all.
1099        let roles = peers.roles_for(workspace_id).cloned();
1100        // Every peer at once, so one that is offline costs the others nothing (#228). On
1101        // this task, not spawned: cancelling the caller — the server's shutdown aborting the
1102        // watcher — must drop every dial with it, since each one pins the runtime.
1103        let mut dials: Vec<Dial<'_>> = Vec::new();
1104        for peer in peers
1105            .peers
1106            .iter()
1107            .filter(|p| topology::link(me, p.device_id, roles.as_ref()) == topology::Link::Dial)
1108            .cloned()
1109            .collect::<Vec<_>>()
1110        {
1111            // A peer that already has a live session is caught up by it: a second exchange
1112            // on the same workspace would be redundant, and a second stream between the same
1113            // pair is exactly what the tie-break cannot settle.
1114            if let Some(live) = self.live_peers(&key).await
1115                && live.has(&peer.device_id).await
1116            {
1117                continue;
1118            }
1119            // One dial per peer at a time. `sync_now` runs beside the dial loop, and two
1120            // overlapping outbound dials to one peer are the one case the tie-break rule
1121            // cannot resolve — each end would see its own stream as `outbound`. Whoever
1122            // arrives second simply leaves it to the next pass. The table is made here if
1123            // the dial loop has not yet: the first dial after enabling must not depend on
1124            // a tick that has not fired.
1125            let Some((_, table)) = self.live_peers_or_create(&key).await else {
1126                continue;
1127            };
1128            if !table.begin_dial(peer.device_id).await {
1129                continue;
1130            }
1131            let key = &key;
1132            dials.push(Box::pin(async move {
1133                // A peer that does not host this workspace refuses, which is normal and cheap.
1134                let result = self
1135                    .sync_one(key, &peer.device_id, peer.name.clone(), workspace_id)
1136                    .await;
1137                table.end_dial(peer.device_id).await;
1138                if let Err(err) = result {
1139                    tracing::debug!(peer = %peer.name, "no session: {err}");
1140                }
1141            }));
1142        }
1143        join_all(dials).await;
1144        Ok(())
1145    }
1146
1147    /// Dial one peer and adopt the stream as a live session.
1148    ///
1149    /// The one-shot path (`run_session`) would close the stream the moment the exchange
1150    /// finished — while the accepting peer keeps its half open and pushes into it, a session
1151    /// that dies with nothing reading it. So the dialled side is adopted live too, and both
1152    /// ends of a pair agree that a session stays open.
1153    async fn sync_one(
1154        self: &Arc<Self>,
1155        key: &Path,
1156        device: &GrainId,
1157        name: String,
1158        workspace_id: GrainId,
1159    ) -> Result<()> {
1160        let stream = self.open_stream(workspace_id, *device).await?;
1161        self.adopt_live_session(stream, key, *device, workspace_id)
1162            .await
1163            .map_err(|err| {
1164                tracing::warn!(peer = %name, "a live session could not be started: {err}");
1165                err
1166            })
1167    }
1168
1169    /// Ask the bridge for a stream to `device`, giving up after `open_timeout`.
1170    async fn open_stream(
1171        &self,
1172        workspace_id: GrainId,
1173        device: GrainId,
1174    ) -> Result<sapphire_ipc::RawStream> {
1175        match tokio::time::timeout(
1176            self.open_timeout,
1177            self.bridge()?.open_stream(workspace_id, device),
1178        )
1179        .await
1180        {
1181            Ok(opened) => Ok(opened?),
1182            Err(_) => Err(Error::Bridge(format!(
1183                "no stream to {device} within {} s; it may be offline",
1184                self.open_timeout.as_secs_f32()
1185            ))),
1186        }
1187    }
1188
1189    /// Answer the bridge's announcements, following the connection across reconnections
1190    /// (#244). Runs until the task is aborted.
1191    pub async fn run(self: Arc<Self>) -> Result<()> {
1192        let mut slot = self.bridge.subscribe();
1193        loop {
1194            let bridge = slot.borrow_and_update().clone();
1195            match bridge {
1196                Some(bridge) if !bridge.is_closed() => self.answer(&bridge, &mut slot).await,
1197                // Wait for a connection. The runtime holds the sender, so this never ends.
1198                _ => {
1199                    if slot.changed().await.is_err() {
1200                        return Ok(());
1201                    }
1202                }
1203            }
1204        }
1205    }
1206
1207    /// Answer `bridge`'s announcements until it closes or another connection replaces it.
1208    async fn answer(
1209        self: &Arc<Self>,
1210        bridge: &Arc<BridgeClient>,
1211        slot: &mut tokio::sync::watch::Receiver<Option<Arc<BridgeClient>>>,
1212    ) {
1213        let early = self
1214            .subscribed
1215            .lock()
1216            .expect("subscribed")
1217            .take_if(|(b, _)| Arc::ptr_eq(b, bridge));
1218        let mut incoming = match early {
1219            Some((_, incoming)) => incoming,
1220            None => bridge.incoming(),
1221        };
1222        loop {
1223            let received = tokio::select! {
1224                received = incoming.recv() => received,
1225                () = bridge.closed() => return,
1226                _ = slot.changed() => return,
1227            };
1228            let announcement = match received {
1229                Ok(a) => a,
1230                Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
1231                    tracing::warn!(missed = n, "fell behind on bridge announcements");
1232                    continue;
1233                }
1234                Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1235            };
1236
1237            let found = {
1238                let synced = self.synced.lock().await;
1239                synced
1240                    .iter()
1241                    .find(|(_, s)| s.workspace_id == announcement.workspace_id)
1242                    .map(|(root, s)| (root.clone(), Arc::clone(&s.replica)))
1243            };
1244            // `replica` is still what the accepted session shares, and `root` is where its
1245            // files and its index live.
1246            let Some((root, replica)) = found else {
1247                // The bridge routed to us for a workspace we no longer hold. Not fatal:
1248                // ignore it and let the ticket expire.
1249                tracing::debug!(
1250                    workspace = %announcement.workspace_id,
1251                    "an announcement for a workspace this server does not hold"
1252                );
1253                continue;
1254            };
1255
1256            let bridge = Arc::clone(bridge);
1257            let workspace_id = announcement.workspace_id;
1258            let peer = announcement.peer_device_id;
1259            let driver = Arc::clone(self);
1260            self.spawn_background(async move {
1261                let stream = match bridge.accept_stream(announcement.ticket).await {
1262                    Ok(stream) => stream,
1263                    Err(err) => {
1264                        tracing::warn!("could not claim an announced stream: {err}");
1265                        return;
1266                    }
1267                };
1268                if driver.should_skip(workspace_id, peer).await {
1269                    tracing::debug!(%peer, "dropped an inbound session: both ends are outside this workspace's star");
1270                    return;
1271                }
1272                // Live, not one-shot: the peer that dialled is keeping this session open and
1273                // pushing what it commits, so this side has to keep reading. A session that
1274                // closed after the exchange would leave the dialler pushing into a stream
1275                // nobody reads.
1276                let (_, session) = match sapphire_framework_session::open_live_session(
1277                    stream,
1278                    Arc::clone(&replica),
1279                    workspace_id,
1280                )
1281                .await
1282                {
1283                    Ok(started) => started,
1284                    Err(err) => {
1285                        crate::instrument::sync_session(false);
1286                        tracing::warn!("an inbound session failed: {err}");
1287                        return;
1288                    }
1289                };
1290                let Some((key, table)) = driver.live_peers_or_create(&root).await else {
1291                    return;
1292                };
1293                // The peer dialled: this stream survives only if the *peer* has the lower
1294                // device id. A stream that loses is dropped, which closes it, and the
1295                // connection ends tidily rather than being left half-read.
1296                let Some(updates) = table.insert(peer, session, false).await else {
1297                    return;
1298                };
1299                table.catch_up(peer, &replica).await;
1300                crate::instrument::sync_session(true);
1301                // This is the receiving side: the files are on disk now and nobody else will
1302                // index them.
1303                driver.reindex(&key).await;
1304                driver.request_embed(&key);
1305                driver.spawn_reader(table, updates, key, peer);
1306            });
1307        }
1308    }
1309
1310    /// Make the first attempt to reach the bridge, when the runtime has a
1311    /// [`BridgeConnector`] and is not connected yet.
1312    ///
1313    /// No bridge running is not an error: the server serves without sync, and
1314    /// [`keep_connected`](Self::keep_connected) picks the bridge up once it starts. Any
1315    /// other failure — a bridge of another API version, say — is, and stops the start.
1316    pub async fn connect(self: &Arc<Self>) -> Result<()> {
1317        let Some(connector) = &self.connector else {
1318            return Ok(());
1319        };
1320        if self.is_connected() {
1321            return Ok(());
1322        }
1323        match connector.connect().await? {
1324            Some(client) => self.attach(client).await,
1325            None => tracing::warn!(
1326                "no sapphire-bridge is running; serving without sync until it starts"
1327            ),
1328        }
1329        Ok(())
1330    }
1331
1332    /// Stay connected to the bridge: wait for the connection to close, then reach the
1333    /// bridge again with the runtime's [`BridgeConnector`], backing off from
1334    /// [`RECONNECT_BACKOFF`] to [`RECONNECT_BACKOFF_MAX`] while it cannot (#244).
1335    /// [`reconnect_now`](Self::reconnect_now) ends a wait early and starts the backoff over.
1336    ///
1337    /// Runs until the task is aborted, which is what the app server's shutdown does with
1338    /// it. Without a connector it only marks the runtime disconnected once the connection
1339    /// closes, and returns.
1340    pub async fn keep_connected(self: Arc<Self>) {
1341        let mut attempt = 0;
1342        // Whether this outage's failure has been logged at warn: once per outage, however
1343        // often `reconnect_now` starts the backoff over.
1344        let mut warned = false;
1345        loop {
1346            if let Ok(bridge) = self.bridge() {
1347                bridge.closed().await;
1348                tracing::warn!("lost the connection to sapphire-bridge; reconnecting");
1349                self.bridge.send_if_modified(|slot| {
1350                    let same = slot.as_ref().is_some_and(|b| Arc::ptr_eq(b, &bridge));
1351                    if same {
1352                        *slot = None;
1353                    }
1354                    same
1355                });
1356                attempt = 0;
1357                warned = false;
1358                continue;
1359            }
1360            let Some(connector) = &self.connector else {
1361                return;
1362            };
1363            match connector.connect().await {
1364                Ok(Some(client)) => {
1365                    tracing::info!("connected to sapphire-bridge");
1366                    self.attach(client).await;
1367                    attempt = 0;
1368                    warned = false;
1369                    continue;
1370                }
1371                Ok(None) => tracing::debug!("no sapphire-bridge is running yet"),
1372                // Logged once per outage, not once per attempt.
1373                Err(err) if !warned => {
1374                    tracing::warn!("could not connect to sapphire-bridge: {err}");
1375                    warned = true;
1376                }
1377                Err(err) => tracing::debug!("could not connect to sapphire-bridge: {err}"),
1378            }
1379            tokio::select! {
1380                () = tokio::time::sleep(reconnect_backoff(self.reconnect_first, attempt)) => {
1381                    attempt = attempt.saturating_add(1);
1382                }
1383                () = self.reconnect_now.notified() => attempt = 0,
1384            }
1385        }
1386    }
1387
1388    /// Start using `client`, and bring the bridge up to date: it knows nothing of this
1389    /// server yet, whether it has just started or this server has.
1390    async fn attach(self: &Arc<Self>, client: BridgeClient) {
1391        // Swapped under the pending lock, so an `enable` racing this either sees the new
1392        // connection or leaves its workspace for the drain below.
1393        let client = Arc::new(client);
1394        *self.subscribed.lock().expect("subscribed") =
1395            Some((Arc::clone(&client), client.incoming()));
1396        let waiting = {
1397            let mut pending = self.pending.lock().await;
1398            self.bridge.send_replace(Some(client));
1399            std::mem::take(&mut *pending)
1400        };
1401        // What was synced before the connection was lost.
1402        if !self.synced.lock().await.is_empty()
1403            && let Err(err) = self.reregister().await
1404        {
1405            tracing::warn!("could not register with sapphire-bridge: {err}");
1406        }
1407        // What was asked for while it was away. `enable` registers and dials each one.
1408        for root in waiting.into_keys() {
1409            if let Err(err) = self.enable(&root).await {
1410                tracing::warn!(root = %root.display(), "could not start sync: {err}");
1411            }
1412        }
1413    }
1414
1415    /// Every synced root, for the watcher.
1416    pub async fn roots(&self) -> Vec<PathBuf> {
1417        self.synced.lock().await.keys().cloned().collect()
1418    }
1419
1420    /// The live session table for `root`, if this runtime syncs it.
1421    ///
1422    /// `None` is not an error: a dial, a push or a drop for a workspace this server does not
1423    /// hold is a no-op, and every caller below treats it that way.
1424    async fn live_peers(&self, root: &Path) -> Option<Arc<LivePeers>> {
1425        let key = root.canonicalize().ok()?;
1426        self.live.lock().await.get(&key).cloned()
1427    }
1428
1429    /// The live session table for `root`, creating it if this runtime syncs it.
1430    ///
1431    /// The two locks are taken one after the other, never nested: `synced` to decide whether
1432    /// the workspace is ours at all, then `live` to get or make its table.
1433    async fn live_peers_or_create(&self, root: &Path) -> Option<(PathBuf, Arc<LivePeers>)> {
1434        let key = root.canonicalize().ok()?;
1435        if !self.synced.lock().await.contains_key(&key) {
1436            return None;
1437        }
1438        // A table is built by whoever walks the workspace first, and it needs this host's
1439        // device id — the tie-break between two simultaneous streams is decided from it.
1440        let me = self.device_id().await.ok()?;
1441        let mut live = self.live.lock().await;
1442        let peers = Arc::clone(
1443            live.entry(key.clone())
1444                .or_insert_with(|| Arc::new(LivePeers::new(me))),
1445        );
1446        Some((key, peers))
1447    }
1448
1449    /// Push `updates` to every open session for `root`.
1450    ///
1451    /// Called with what a scan recorded. `from` is always `None` here: a local commit came
1452    /// from this host, so no session is excluded. The forwarding path, which does name a
1453    /// source, is [`LivePeers::fan_out`](live::LivePeers::fan_out) called directly by the
1454    /// reader side of a session.
1455    ///
1456    /// Best-effort by contract: the commit already happened, and a peer that is gone is the
1457    /// dialer's problem, not the writer's.
1458    pub async fn after_commit(&self, root: &Path, updates: Vec<PathUpdate>) {
1459        if updates.is_empty() {
1460            return;
1461        }
1462        if let Some(peers) = self.live_peers(root).await {
1463            peers.fan_out(&updates, None).await;
1464        }
1465    }
1466
1467    /// The devices with an open live session for `root`.
1468    ///
1469    /// Used by tests to wait for the dialer; empty for a workspace this server does not hold.
1470    pub async fn live_session_devices(&self, root: &Path) -> Vec<GrainId> {
1471        match self.live_peers(root).await {
1472            Some(peers) => peers.devices().await,
1473            None => Vec::new(),
1474        }
1475    }
1476
1477    /// End the first dials [`enable`](SyncRuntime::enable) left running, waiting until
1478    /// they are gone.
1479    ///
1480    /// For the app server's shutdown: each one pins the runtime, and with it the replica
1481    /// stores, which are redb databases that stay locked — a dial outliving its server
1482    /// would lock the next one out.
1483    pub async fn stop_dials(&self) {
1484        let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1485        dials.abort_all();
1486        while dials.join_next().await.is_some() {}
1487    }
1488
1489    /// End the runtime's background work — inbound sessions, live-session readers and
1490    /// embedding passes — and wait until it has let go.
1491    ///
1492    /// The server's shutdown calls this, so a server restarted in the same process can open
1493    /// the replica stores again at once rather than whenever the last straggler finished
1494    /// (#235). An embedding pass already inside its blocking section runs that section to
1495    /// the end; it holds the workspace's index, never a replica.
1496    pub async fn stop_background(&self) {
1497        let mut background =
1498            std::mem::take(&mut *self.background.lock().expect("background tasks"));
1499        background.abort_all();
1500        while background.join_next().await.is_some() {}
1501    }
1502
1503    /// Wait for the first dials [`enable`](SyncRuntime::enable) left running to finish on
1504    /// their own: for a test that must not race them.
1505    #[cfg(test)]
1506    async fn settle_dials(&self) {
1507        let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1508        while dials.join_next().await.is_some() {}
1509    }
1510
1511    /// Close every open live session, as if the connections had been cut.
1512    ///
1513    /// Called for a workspace being disabled, and by tests that want to watch the dialer
1514    /// rebuild what was dropped.
1515    pub async fn drop_connections(&self) {
1516        let tables: Vec<Arc<LivePeers>> = self.live.lock().await.values().cloned().collect();
1517        for peers in tables {
1518            peers.drop_connections().await;
1519        }
1520    }
1521
1522    /// Keep a live session open to every peer, for every synced workspace.
1523    ///
1524    /// One walk per [`DIAL_INTERVAL`](live::DIAL_INTERVAL): for each synced workspace, ask
1525    /// the bridge who is in the workgroup, and open a session to every peer that is connected
1526    /// and does not already have one.
1527    ///
1528    /// A peer that cannot be reached is retried on a per-peer exponential backoff capped at
1529    /// [`DIAL_BACKOFF_MAX`] — one failed attempt per walk, not one per spin — and the counter
1530    /// resets when a session opens. An unreachable peer is therefore no obstacle to the
1531    /// others: its dial fails, the walk moves on, and the next workspace is dialled.
1532    ///
1533    /// Runs until the task is aborted, which is what the app server's shutdown does with it.
1534    /// It is allowed to fail like [`run`](SyncRuntime::run): only sync stops, and the app
1535    /// server keeps serving files (spec §10).
1536    pub async fn dial_loop(self: Arc<Self>) -> Result<()> {
1537        let mut tick = tokio::time::interval(live::DIAL_INTERVAL);
1538        // A missed tick is a late walk, not a burst of them: the interval is a pace, not a
1539        // quota to make up.
1540        tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1541        let mut waiting: HashMap<GrainId, std::time::Instant> = HashMap::new();
1542        let mut failures: HashMap<GrainId, u32> = HashMap::new();
1543        // The workspaces this host was the primary device of at the last walk.
1544        let mut primary_of: std::collections::HashSet<GrainId> = Default::default();
1545        // When the primary device last ran a backfill pass with nothing prompting it.
1546        let mut last_backfill = std::time::Instant::now();
1547        loop {
1548            tick.tick().await;
1549            let roots = self.roots().await;
1550            if roots.is_empty() {
1551                continue;
1552            }
1553            let peers = match self.peers().await {
1554                Ok(peers) => peers,
1555                // The bridge is down or has no workgroup: nothing to dial, and the next walk
1556                // asks again. Not an error for the caller — the app server is still serving.
1557                Err(err) => {
1558                    tracing::debug!("no peers to dial: {err}");
1559                    continue;
1560                }
1561            };
1562            let me = match self.device_id().await {
1563                Ok(me) => me,
1564                Err(err) => {
1565                    tracing::warn!("no device id, so nobody to dial: {err}");
1566                    continue;
1567                }
1568            };
1569            // A host that has just become a workspace's primary device owes it the vectors
1570            // nobody else computes (#187): files that arrived before the election settled
1571            // would otherwise wait for the next change.
1572            let ids: Vec<(PathBuf, GrainId)> = self
1573                .synced
1574                .lock()
1575                .await
1576                .iter()
1577                .map(|(root, entry)| (root.clone(), entry.workspace_id))
1578                .collect();
1579            // And every so often anyway: an author that never embedded leaves files only the
1580            // primary will fill in, and nothing else would prompt it (#188).
1581            let backfill_due = last_backfill.elapsed() >= self.backfill_timing().interval;
1582            if backfill_due {
1583                last_backfill = std::time::Instant::now();
1584            }
1585            for (root, workspace_id) in ids {
1586                let primary = peers
1587                    .roles_for(workspace_id)
1588                    .is_some_and(|r| r.primary == Some(me));
1589                if !primary {
1590                    primary_of.remove(&workspace_id);
1591                } else if primary_of.insert(workspace_id) || backfill_due {
1592                    self.request_embed(&root);
1593                }
1594            }
1595            let now = std::time::Instant::now();
1596            // Dial only peers whose device id is greater than ours. Every device dialling
1597            // every other can deadlock: two hosts that dial each other at once each hold
1598            // their replica's lock across their own exchange and wait for the other's
1599            // hello, which the other's dialler holds the lock for. With one direction per
1600            // pair the wait chain is a DAG — a higher-id host never dials, so it never
1601            // holds its lock out while a lower-id host's exchange needs it. The lower host
1602            // dials within `DIAL_INTERVAL`, and the exchange is symmetric, so nothing is
1603            // lost; the tie-break is left as the safety net it is.
1604            for root in roots {
1605                let Some((key, table)) = self.live_peers_or_create(&root).await else {
1606                    continue;
1607                };
1608                let workspace_id = {
1609                    let synced = self.synced.lock().await;
1610                    match synced.get(&key) {
1611                        Some(entry) => entry.workspace_id,
1612                        None => continue,
1613                    }
1614                };
1615                let roles = peers.roles_for(workspace_id);
1616                // Entering a star closes what it no longer needs. The hubs carry it all —
1617                // once one of them is reachable from here (#192).
1618                if topology::may_close_skipped(me, roles, &table.devices().await) {
1619                    table
1620                        .retain(|d| topology::link(me, *d, roles) != topology::Link::Skip)
1621                        .await;
1622                }
1623                for peer in peers.peers.iter().filter(|p| {
1624                    p.connected && topology::link(me, p.device_id, roles) == topology::Link::Dial
1625                }) {
1626                    let device = peer.device_id;
1627                    if table.has(&device).await {
1628                        // Already talking; a redial would drop a working session.
1629                        failures.remove(&device);
1630                        waiting.remove(&device);
1631                        continue;
1632                    }
1633                    if let Some(until) = waiting.get(&device)
1634                        && now < *until
1635                    {
1636                        // Backing off after a failure; the walk skips it until then.
1637                        continue;
1638                    }
1639                    // The same claim `sync_now` takes: this walk and it can overlap, and a
1640                    // double dial in one direction is the one race the tie-break cannot
1641                    // settle. Skipping is free — the next pass, or `sync_now` itself, dials.
1642                    if !table.begin_dial(device).await {
1643                        continue;
1644                    }
1645                    let opened = self.open_stream(workspace_id, device).await;
1646                    if opened.is_err() {
1647                        table.end_dial(device).await;
1648                    }
1649                    match opened {
1650                        Ok(stream) => {
1651                            let adopted = self
1652                                .adopt_live_session(stream, &root, device, workspace_id)
1653                                .await;
1654                            // The claim is out either way: the attempt is over, and a
1655                            // session that died mid-exchange is re-dialled next pass.
1656                            table.end_dial(device).await;
1657                            match adopted {
1658                                Ok(()) => {
1659                                    failures.remove(&device);
1660                                    waiting.remove(&device);
1661                                }
1662                                Err(err) => {
1663                                    tracing::debug!(
1664                                        peer = %peer.name,
1665                                        err = %err,
1666                                        "a live session could not be started"
1667                                    );
1668                                    let count = failures.entry(device).or_insert(0);
1669                                    *count = count.saturating_add(1);
1670                                    waiting.insert(device, now + dial_backoff(*count));
1671                                }
1672                            }
1673                        }
1674                        Err(err) => {
1675                            // Normal and cheap: a peer that does not host this workspace
1676                            // refuses, and one that is unreachable cannot be opened.
1677                            tracing::debug!("no stream to a peer: {err}");
1678                            let count = failures.entry(device).or_insert(0);
1679                            *count = count.saturating_add(1);
1680                            waiting.insert(device, now + dial_backoff(*count));
1681                        }
1682                    }
1683                }
1684            }
1685        }
1686    }
1687
1688    /// Take ownership of an opened stream: run the exchange, then keep it live.
1689    ///
1690    /// The session is stored under `device` and its reader pumped until the stream ends. A
1691    /// half-open or refused stream is not stored; the dialer retries it with backoff.
1692    async fn adopt_live_session(
1693        self: &Arc<Self>,
1694        stream: sapphire_ipc::RawStream,
1695        root: &Path,
1696        device: GrainId,
1697        workspace_id: GrainId,
1698    ) -> Result<()> {
1699        let replica = self.replica_of(root).await?;
1700        let (_, session) = sapphire_framework_session::open_live_session(
1701            stream,
1702            Arc::clone(&replica),
1703            workspace_id,
1704        )
1705        .await
1706        .map_err(|e| {
1707            crate::instrument::sync_session(false);
1708            Error::Sync(e.to_string())
1709        })?;
1710        let Some((key, table)) = self.live_peers_or_create(root).await else {
1711            return Ok(());
1712        };
1713        // This stream was dialled by us, so it is the one the pair keeps if this host's
1714        // device id is the lower of the two. If it lost the tie-break the session is
1715        // dropped here, which closes the stream, and the surviving one is already in the
1716        // table.
1717        let Some(updates) = table.insert(device, session, true).await else {
1718            return Ok(());
1719        };
1720        table.catch_up(device, &replica).await;
1721        crate::instrument::sync_session(true);
1722        // The session wrote files during the exchange; the index has to catch up before the
1723        // workspace is searchable again.
1724        self.reindex(&key).await;
1725        self.request_embed(&key);
1726        self.spawn_reader(Arc::clone(&table), updates, key, device);
1727        Ok(())
1728    }
1729
1730    /// Forward what one session receives, and re-index it.
1731    ///
1732    /// This is the loop that makes A → S → B work: a batch that arrived on S's session with A
1733    /// is applied (the session crate does that before publishing it), the workspace behind
1734    /// `key` is re-indexed, and the batch is handed to every *other* session whose peer lacks
1735    /// it — `from` is the device it came from, so it never goes back to A.
1736    fn spawn_reader(
1737        self: &Arc<Self>,
1738        table: Arc<LivePeers>,
1739        mut updates: broadcast::Receiver<Vec<PathUpdate>>,
1740        key: PathBuf,
1741        from: GrainId,
1742    ) {
1743        let driver = Arc::clone(self);
1744        self.spawn_background(async move {
1745            loop {
1746                match updates.recv().await {
1747                    Ok(batch) => {
1748                        // The files are on disk now (the session guarantees it before
1749                        // publishing), so the index is behind them until the sweep below.
1750                        driver.reindex(&key).await;
1751                        driver.request_embed(&key);
1752                        table.fan_out(&batch, Some(from)).await;
1753                    }
1754                    // A burst bigger than the channel: the newest is kept and the gap is
1755                    // covered by the next session's exchange. Losing a batch here cannot lose
1756                    // an entry — the peer's vector does not cover it, so the next dial or push
1757                    // carries it again.
1758                    Err(broadcast::error::RecvError::Lagged(missed)) => {
1759                        tracing::debug!(
1760                            device = %from,
1761                            missed,
1762                            "fell behind on a live session; the next exchange catches up"
1763                        );
1764                    }
1765                    // The session closed. Its table entry is reaped by the next fan-out or
1766                    // dial pass, and the dialer opens a fresh one.
1767                    Err(broadcast::error::RecvError::Closed) => {
1768                        return;
1769                    }
1770                }
1771            }
1772        });
1773    }
1774
1775    /// The replica [`disable`](Self::disable) let go of for `key`, if something still holds it
1776    /// open.
1777    fn reclaim(&self, key: &Path) -> Reclaimed {
1778        let stopped = self.stopped.lock().expect("stopped replicas").remove(key);
1779        match stopped {
1780            None => Reclaimed::Closed,
1781            Some(replica) => replica
1782                .upgrade()
1783                .map_or(Reclaimed::Closing, Reclaimed::Open),
1784        }
1785    }
1786
1787    /// The replica of a synced `root`.
1788    async fn replica_of(&self, root: &Path) -> Result<Arc<Mutex<Replica>>> {
1789        let key = root.canonicalize().map_err(Error::Io)?;
1790        let synced = self.synced.lock().await;
1791        match synced.get(&key) {
1792            Some(entry) => Ok(Arc::clone(&entry.replica)),
1793            None => Err(Error::UnknownWorkspace(key, self.ctx.app_name)),
1794        }
1795    }
1796
1797    /// Watch the synced roots and, on a debounced report, scan and dial.
1798    ///
1799    /// This is the safety net for everything the app server did not do itself — a file
1800    /// edited in an editor, a `git checkout`. The exact path is [`scan`] called from
1801    /// `handlers.rs` right after a write the server made. A scan that finds nothing is
1802    /// cheap; a scan that is skipped loses an edit until the next one.
1803    ///
1804    /// Returns when the reporting channel closes — that is, when the [`Watcher`] this
1805    /// runtime holds is dropped, which happens when the runtime itself is.
1806    ///
1807    /// [`scan`]: SyncRuntime::scan
1808    pub async fn watch_changes(self: Arc<Self>) -> Result<()> {
1809        let (tx, mut rx) = mpsc::channel(64);
1810        let watcher = Arc::new(Watcher::start(self.roots().await, tx)?);
1811        let _ = self.watcher.set(Arc::clone(&watcher));
1812
1813        // A root enabled between `roots()` above and the `set` just now was not in the
1814        // starting set, and `enable` could not reach a watcher that did not exist yet.
1815        // Watching the current set again is idempotent on `notify`'s side.
1816        for root in self.roots().await {
1817            if let Err(err) = watcher.watch(&root) {
1818                tracing::warn!(root = %root.display(), "could not watch: {err}");
1819            }
1820        }
1821
1822        while let Some(root) = rx.recv().await {
1823            // An edit of the marker's `config.toml` changes the workspace's settings.
1824            self.refresh_settings();
1825            // Logged, not fatal: a failed scan or dial leaves the replica behind, and the
1826            // next report tries again. Neither may take the app server down (spec §10).
1827            if let Err(err) = self.scan(&root).await {
1828                tracing::warn!(root = %root.display(), "scan after a local edit failed: {err}");
1829            }
1830            self.request_embed(&root);
1831            if let Err(err) = self.sync_now(&root).await {
1832                tracing::warn!(root = %root.display(), "dial after a local edit failed: {err}");
1833            }
1834        }
1835        Ok(())
1836    }
1837
1838    /// Tell the bridge the complete current set. A registration is not a delta.
1839    async fn reregister(&self) -> Result<()> {
1840        let workspaces: Vec<WorkspaceRegistration> = {
1841            let synced = self.synced.lock().await;
1842            synced
1843                .iter()
1844                .map(|(root, entry)| WorkspaceRegistration {
1845                    workspace_id: entry.workspace_id,
1846                    root: root.clone(),
1847                })
1848                .collect()
1849        };
1850        self.bridge()?
1851            .register(RegisterParams {
1852                app_name: self.ctx.app_name.to_owned(),
1853                exe_path: self.exe_path.clone(),
1854                managed_by: wire_managed_by(self.managed_by),
1855                workspaces,
1856            })
1857            .await
1858            .map_err(|e| Error::Bridge(e.to_string()))?;
1859        Ok(())
1860    }
1861
1862    /// Whether the last roles the bridge reported say this pair does not link.
1863    ///
1864    /// Reads the cached device id only: asking the bridge here would register with an empty
1865    /// workspace list, racing `enable`'s registration. `false` (accept) when it is unknown.
1866    async fn should_skip(&self, workspace_id: GrainId, peer: GrainId) -> bool {
1867        let Some(&me) = self.device_id.get() else {
1868            return false;
1869        };
1870        let cached = self.last_peers.lock().await;
1871        let roles = cached.as_ref().and_then(|p| p.roles_for(workspace_id));
1872        topology::link(me, peer, roles) == topology::Link::Skip
1873    }
1874
1875    /// Whether this host is `workspace_id`'s primary device, by the last roles the
1876    /// bridge reported. `false` when the bridge has not answered yet, or when this host's
1877    /// device id is not known yet (no workspace was enabled). Reads the cached id only:
1878    /// asking the bridge here would register with an empty workspace list, racing
1879    /// `enable`'s registration.
1880    pub async fn is_primary(&self, workspace_id: GrainId) -> bool {
1881        let Some(&me) = self.device_id.get() else {
1882            return false;
1883        };
1884        let cached = self.last_peers.lock().await;
1885        cached
1886            .as_ref()
1887            .and_then(|p| p.roles_for(workspace_id))
1888            .is_some_and(|r| r.primary == Some(me))
1889    }
1890
1891    /// This host's device id, once the bridge has told it (#250).
1892    ///
1893    /// `None` until this runtime first registered — no workspace was enabled yet, or the
1894    /// bridge was not connected when one was. Never asks the bridge: a registration made
1895    /// only to learn the id would carry an empty workspace list, and briefly clear this
1896    /// application's routes.
1897    pub fn this_device(&self) -> Option<GrainId> {
1898        self.device_id.get().copied()
1899    }
1900
1901    /// The workgroup's devices, this host's included, and the roles the bridge assigns per
1902    /// workspace (#250). Asked of the bridge each time; [`is_primary`](Self::is_primary)
1903    /// reads the answer afterwards.
1904    ///
1905    /// [`Error::BridgeDisconnected`] while there is no connection to the bridge.
1906    pub async fn peers(&self) -> Result<PeersResult> {
1907        let peers = self
1908            .bridge()?
1909            .peers()
1910            .await
1911            .map_err(|e| Error::Bridge(e.to_string()))?;
1912        *self.last_peers.lock().await = Some(peers.clone());
1913        Ok(peers)
1914    }
1915
1916    /// This host's device id, asked of the bridge once.
1917    async fn device_id(&self) -> Result<GrainId> {
1918        if let Some(id) = self.device_id.get() {
1919            return Ok(*id);
1920        }
1921        let result = self
1922            .bridge()?
1923            .register(RegisterParams {
1924                app_name: self.ctx.app_name.to_owned(),
1925                exe_path: self.exe_path.clone(),
1926                managed_by: wire_managed_by(self.managed_by),
1927                workspaces: Vec::new(),
1928            })
1929            .await
1930            .map_err(|e| Error::Bridge(e.to_string()))?;
1931        let _ = self.device_id.set(result.device_id);
1932        Ok(result.device_id)
1933    }
1934
1935    /// `<workspace cache dir>/sync/`.
1936    fn state_dir(&self, root: &Path) -> Result<PathBuf> {
1937        let workspace = Workspace::from_root(self.ctx, root)?;
1938        Ok(workspace.cache_dir().join("sync"))
1939    }
1940}
1941
1942/// One dial of [`SyncRuntime::sync_now`].
1943type Dial<'a> = std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>>;
1944
1945/// Run `dials` concurrently on the calling task, until every one has finished.
1946async fn join_all(mut dials: Vec<Dial<'_>>) {
1947    std::future::poll_fn(|cx| {
1948        dials.retain_mut(|dial| dial.as_mut().poll(cx).is_pending());
1949        if dials.is_empty() {
1950            std::task::Poll::Ready(())
1951        } else {
1952            std::task::Poll::Pending
1953        }
1954    })
1955    .await
1956}
1957
1958/// The bridge API's copy of how this server was started, for [`RegisterParams`].
1959fn wire_managed_by(managed_by: ManagedBy) -> sapphire_bridge_api::ManagedBy {
1960    match managed_by {
1961        ManagedBy::Service => sapphire_bridge_api::ManagedBy::Service,
1962        ManagedBy::Spawned => sapphire_bridge_api::ManagedBy::Spawned,
1963    }
1964}
1965
1966#[cfg(test)]
1967mod tests {
1968    use super::*;
1969    use crate::sync::testing::StubBridge;
1970    use crate::test_support;
1971    use sapphire_workspace::AppKind;
1972    use std::ffi::OsString;
1973
1974    static CTX: AppContext = AppContext::new("sapphire-synctest");
1975
1976    /// The env vars `CTX.init` reads.
1977    const DIR_VARS: [&str; 3] = [
1978        "SAPPHIRE_SYNCTEST_CACHE_DIR",
1979        "SAPPHIRE_SYNCTEST_DATA_DIR",
1980        "SAPPHIRE_SYNCTEST_CONFIG_DIR",
1981    ];
1982
1983    /// Points the context's directories at the test's scratch tree, and restores the
1984    /// previous values when dropped — including while unwinding from a panic.
1985    struct EnvGuard {
1986        previous: [Option<OsString>; 3],
1987        _lock: std::sync::MutexGuard<'static, ()>,
1988    }
1989
1990    impl Drop for EnvGuard {
1991        fn drop(&mut self) {
1992            // SAFETY: `self._lock` still serialises the environment; it is dropped only
1993            // after this method returns.
1994            for (name, previous) in DIR_VARS.iter().zip(self.previous.iter_mut()) {
1995                match previous.take() {
1996                    Some(value) => unsafe { std::env::set_var(name, value) },
1997                    None => test_support::remove(name),
1998                }
1999            }
2000        }
2001    }
2002
2003    /// Everything one test needs, held for the test's whole body.
2004    ///
2005    /// The context is a `static` shared by this binary, so its directories are resolved from
2006    /// the environment by whichever test initialises it first. The environment lock is
2007    /// therefore held for the whole test, as in the crate's other test modules, and `_tmp`
2008    /// is declared before `_env` so the scratch tree is gone only once the environment no
2009    /// longer points at it.
2010    struct Fixture {
2011        _tmp: tempfile::TempDir,
2012        _env: EnvGuard,
2013        root: PathBuf,
2014        stub: StubBridge,
2015        runtime: Arc<SyncRuntime>,
2016    }
2017
2018    impl Fixture {
2019        /// A second workspace root beside the first, marker directory and all.
2020        fn second_root(&self) -> PathBuf {
2021            let root = self._tmp.path().join("ws2");
2022            std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
2023            root.canonicalize().unwrap()
2024        }
2025    }
2026
2027    /// How long the fixture's runtime waits on a stream to a peer.
2028    const FIXTURE_OPEN_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
2029
2030    /// `n` peers this host dials (a greater device id), as `bridge.peers` lists them.
2031    fn dialled_peers(me: GrainId, n: usize) -> Vec<sapphire_bridge_api::PeerInfo> {
2032        std::iter::repeat_with(GrainId::random)
2033            .filter(|id| *id > me)
2034            .take(n)
2035            .enumerate()
2036            .map(|(i, device_id)| sapphire_bridge_api::PeerInfo {
2037                device_id,
2038                name: format!("offline-{i}"),
2039                node_id: format!("node-{i}"),
2040                connected: false,
2041                priority: 1,
2042                availability: None,
2043            })
2044            .collect()
2045    }
2046
2047    #[tokio::test(flavor = "multi_thread")]
2048    async fn enabling_answers_at_once_with_a_peer_that_never_answers() {
2049        // #228: the first dial waited on a peer that was offline, past the client's limit.
2050        let f = fixture().await;
2051        let _hung = f.stub.hang_streams().await;
2052        f.stub.set_peers(dialled_peers(f.stub.device_id, 1));
2053
2054        let started = std::time::Instant::now();
2055        f.runtime.enable(&f.root).await.unwrap();
2056        assert!(
2057            started.elapsed() < FIXTURE_OPEN_TIMEOUT,
2058            "enable waited {:?} on the dial",
2059            started.elapsed()
2060        );
2061    }
2062
2063    #[tokio::test(flavor = "multi_thread")]
2064    async fn peers_that_never_answer_are_dialled_together_and_given_up_on() {
2065        let f = fixture().await;
2066        let _hung = f.stub.hang_streams().await;
2067        f.runtime.enable(&f.root).await.unwrap();
2068        // The first dial, with nobody to dial yet, must be over before the peers appear:
2069        // one still running would claim them, and `sync_now` would skip them all.
2070        f.runtime.settle_dials().await;
2071        f.stub.set_peers(dialled_peers(f.stub.device_id, 3));
2072
2073        let started = std::time::Instant::now();
2074        f.runtime.sync_now(&f.root).await.unwrap();
2075        let took = started.elapsed();
2076        assert!(took >= FIXTURE_OPEN_TIMEOUT, "gave up early: {took:?}");
2077        assert!(
2078            took < FIXTURE_OPEN_TIMEOUT * 2,
2079            "three dials one after another: {took:?}"
2080        );
2081    }
2082
2083    async fn fixture() -> Fixture {
2084        let (tmp, env, root) = scratch();
2085        let (stub, client) = StubBridge::start_in(tmp.path().join("run")).await;
2086        let runtime = Arc::new(
2087            SyncRuntime::new(&CTX, client, "/bin/true".into(), ManagedBy::Service)
2088                .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
2089        );
2090        Fixture {
2091            _tmp: tmp,
2092            _env: env,
2093            root,
2094            stub,
2095            runtime,
2096        }
2097    }
2098
2099    /// The bridge connections a [`connecting`] fixture's connector hands out, in order;
2100    /// empty is "no bridge is running".
2101    type Bridges = Arc<std::sync::Mutex<std::collections::VecDeque<BridgeClient>>>;
2102
2103    /// A fixture whose runtime reaches the bridge through a connector, and has not yet.
2104    /// `f.stub`'s client is not handed out: push it to the returned queue for that.
2105    async fn connecting() -> (Fixture, Bridges, BridgeClient) {
2106        let (tmp, env, root) = scratch();
2107        let (stub, client) = StubBridge::start_client_in(tmp.path().join("run")).await;
2108        let bridges: Bridges = Arc::default();
2109        let connector = BridgeConnector::new({
2110            let bridges = Arc::clone(&bridges);
2111            move || {
2112                let next = bridges.lock().unwrap().pop_front();
2113                async move { Ok(next) }
2114            }
2115        });
2116        let runtime = Arc::new(
2117            SyncRuntime::connecting(&CTX, connector, "/bin/true".into(), ManagedBy::Service)
2118                .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
2119        );
2120        let f = Fixture {
2121            _tmp: tmp,
2122            _env: env,
2123            root,
2124            stub,
2125            runtime,
2126        };
2127        (f, bridges, client)
2128    }
2129
2130    /// Wait up to ten seconds for `done`.
2131    async fn eventually(what: &str, mut done: impl AsyncFnMut() -> bool) {
2132        let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
2133        while !done().await {
2134            assert!(tokio::time::Instant::now() < deadline, "never: {what}");
2135            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
2136        }
2137    }
2138
2139    #[tokio::test(flavor = "multi_thread")]
2140    async fn a_runtime_started_without_a_bridge_syncs_once_one_starts() {
2141        // #244: a server that started before its bridge never synced until restarted.
2142        let (f, bridges, client) = connecting().await;
2143        f.runtime.connect().await.unwrap();
2144        assert!(!f.runtime.is_connected());
2145
2146        let err = f.runtime.enable(&f.root).await.unwrap_err();
2147        assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2148        let status = f.runtime.status(&f.root).await;
2149        assert!(status.enabled, "enabled, waiting for the bridge");
2150        assert!(!status.bridge_available);
2151        let id = status.workspace_id.expect("its id");
2152
2153        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2154        bridges.lock().unwrap().push_back(client);
2155        eventually("the waiting workspace is registered", async || {
2156            f.stub.last_workspaces() == vec![id]
2157        })
2158        .await;
2159        assert_eq!(f.runtime.roots().await, vec![f.root.clone()]);
2160        assert!(f.runtime.status(&f.root).await.bridge_available);
2161        keeping.abort();
2162    }
2163
2164    #[tokio::test(flavor = "multi_thread")]
2165    async fn a_runtime_that_loses_its_bridge_registers_with_the_next_one() {
2166        let (f, bridges, client) = connecting().await;
2167        bridges.lock().unwrap().push_back(client);
2168        f.runtime.connect().await.unwrap();
2169        assert!(f.runtime.is_connected());
2170        let id = f.runtime.enable(&f.root).await.unwrap();
2171        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2172
2173        f.stub.disconnect();
2174        eventually("the runtime notices", async || !f.runtime.is_connected()).await;
2175        let err = f.runtime.workspaces().await.unwrap_err();
2176        assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2177
2178        let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2179        bridges.lock().unwrap().push_back(client);
2180        eventually("the synced workspace is registered again", async || {
2181            next.last_workspaces() == vec![id]
2182        })
2183        .await;
2184        assert!(f.runtime.is_connected());
2185        keeping.abort();
2186    }
2187
2188    #[tokio::test(flavor = "multi_thread")]
2189    async fn this_device_is_known_once_a_workspace_is_enabled_and_never_asked_for() {
2190        let f = fixture().await;
2191        assert_eq!(f.runtime.this_device(), None);
2192        assert!(
2193            f.stub.seen.lock().unwrap().registrations.is_empty(),
2194            "asking must not register"
2195        );
2196
2197        f.runtime.enable(&f.root).await.unwrap();
2198        assert_eq!(f.runtime.this_device(), Some(f.stub.device_id));
2199    }
2200
2201    #[tokio::test(flavor = "multi_thread")]
2202    async fn peers_answers_the_workgroup_from_the_bridge() {
2203        let f = fixture().await;
2204        let listed = dialled_peers(f.stub.device_id, 2);
2205        f.stub.set_peers(listed.clone());
2206
2207        let peers = f.runtime.peers().await.unwrap();
2208        let ids: Vec<_> = peers.peers.iter().map(|p| p.device_id).collect();
2209        assert_eq!(ids, listed.iter().map(|p| p.device_id).collect::<Vec<_>>());
2210    }
2211
2212    #[tokio::test(flavor = "multi_thread")]
2213    async fn peers_says_so_while_the_bridge_is_away() {
2214        let (f, _bridges, _client) = connecting().await;
2215        let err = f.runtime.peers().await.unwrap_err();
2216        assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2217    }
2218
2219    #[tokio::test(flavor = "multi_thread")]
2220    async fn disabling_while_the_bridge_is_away_forgets_the_waiting_workspace() {
2221        let (f, bridges, client) = connecting().await;
2222        f.runtime.enable(&f.root).await.unwrap_err();
2223        f.runtime.disable(&f.root).await.unwrap();
2224        assert!(!f.runtime.status(&f.root).await.enabled);
2225
2226        bridges.lock().unwrap().push_back(client);
2227        f.runtime.connect().await.unwrap();
2228        assert!(f.runtime.roots().await.is_empty());
2229        assert!(f.stub.last_workspaces().is_empty());
2230    }
2231
2232    #[tokio::test(flavor = "multi_thread")]
2233    async fn announcements_are_answered_from_a_reconnected_bridge() {
2234        let (f, bridges, client) = connecting().await;
2235        bridges.lock().unwrap().push_back(client);
2236        f.runtime.connect().await.unwrap();
2237        let id = f.runtime.enable(&f.root).await.unwrap();
2238        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2239        let answering = tokio::spawn(Arc::clone(&f.runtime).run());
2240
2241        f.stub.disconnect();
2242        let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2243        bridges.lock().unwrap().push_back(client);
2244        eventually("reconnected", async || next.last_workspaces() == vec![id]).await;
2245
2246        // The announcement loop follows the new connection: an announcement on it is
2247        // claimed, which with no data endpoint fails — and the loop lives on.
2248        next.announce(id, "ticket").await;
2249        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2250        assert!(!answering.is_finished(), "the announcement loop ended");
2251        answering.abort();
2252        keeping.abort();
2253    }
2254
2255    /// The scratch tree, the environment pointed at it, and a workspace root in it.
2256    fn scratch() -> (tempfile::TempDir, EnvGuard, PathBuf) {
2257        let lock = test_support::lock();
2258        let tmp = tempfile::tempdir().unwrap();
2259        let previous = DIR_VARS.map(std::env::var_os);
2260        // SAFETY (via `test_support::set`): `lock` serialises every read and write of the
2261        // process environment in this test binary, and it is held until `drop` has restored
2262        // the old values.
2263        for (name, dir) in DIR_VARS
2264            .iter()
2265            .zip(["cache", "data", "config"].map(|cat| tmp.path().join(cat)))
2266        {
2267            test_support::set(name, &dir);
2268        }
2269        let env = EnvGuard {
2270            previous,
2271            _lock: lock,
2272        };
2273        CTX.init(AppKind::Server);
2274
2275        let root = tmp.path().join("ws");
2276        std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
2277        let root = root.canonicalize().unwrap();
2278        (tmp, env, root)
2279    }
2280
2281    /// An application with one `Live` key in the workspace: `sapphire-synctest.note`.
2282    struct NoteApp;
2283
2284    impl crate::settings::AppSettings for NoteApp {
2285        fn defs(&self) -> Vec<sapphire_framework_settings::SettingDef> {
2286            vec![sapphire_framework_settings::SettingDef {
2287                key: "sapphire-synctest.note".into(),
2288                kind: sapphire_framework_settings::Kind::String { non_empty: false },
2289                scopes: vec![sapphire_framework_settings::Scope::Workspace],
2290                overridable: false,
2291                apply: sapphire_framework_settings::Apply::Live,
2292                default: None,
2293                section: "Test".into(),
2294                doc: "test".into(),
2295                process: sapphire_framework_settings::Process::App,
2296            }]
2297        }
2298    }
2299
2300    #[tokio::test(flavor = "multi_thread")]
2301    async fn a_workspace_config_changed_under_the_watcher_reaches_the_settings() {
2302        let f = fixture().await;
2303        // A context of its own, so its config directory is this test's: `CTX`'s is
2304        // whichever test resolved it first.
2305        let ctx: &'static AppContext = Box::leak(Box::new(AppContext::new("sapphire-synctest")));
2306        ctx.set_config_dir(f._tmp.path().join("settings"));
2307        let selection = crate::selection::SelectionFile::at(
2308            f._tmp.path().join("settings").join("workspace.toml"),
2309        );
2310        selection
2311            .save(&crate::selection::Selection {
2312                root: f.root.clone(),
2313                sync: true,
2314            })
2315            .unwrap();
2316        let settings =
2317            crate::settings::Settings::load_with(ctx, Arc::new(NoteApp), selection).unwrap();
2318        let changed = settings.changed();
2319        f.runtime.set_settings(Some(settings.clone()));
2320        f.runtime.enable(&f.root).await.unwrap();
2321        let watching = tokio::spawn(Arc::clone(&f.runtime).watch_changes());
2322        // Until the watcher is up, a write may go unseen: keep writing until it is noticed.
2323        let config = f.root.join(".sapphire-synctest").join("config.toml");
2324        eventually("the settings hear of the new value", async || {
2325            std::fs::write(&config, "[sapphire-synctest]\nnote = \"synced\"\n").unwrap();
2326            tokio::time::sleep(std::time::Duration::from_millis(200)).await;
2327            changed.has_changed().unwrap()
2328        })
2329        .await;
2330        assert_eq!(
2331            settings
2332                .get_as::<String>("sapphire-synctest.note")
2333                .as_deref(),
2334            Some("synced")
2335        );
2336        watching.abort();
2337    }
2338
2339    #[tokio::test(flavor = "multi_thread")]
2340    async fn enabling_registers_the_workspace_with_the_bridge() {
2341        let f = fixture().await;
2342
2343        let id = f.runtime.enable(&f.root).await.unwrap();
2344        assert_eq!(f.stub.last_workspaces(), vec![id]);
2345    }
2346
2347    #[tokio::test(flavor = "multi_thread")]
2348    async fn enabling_twice_is_idempotent_and_keeps_the_same_id() {
2349        let f = fixture().await;
2350
2351        let first = f.runtime.enable(&f.root).await.unwrap();
2352        let second = f.runtime.enable(&f.root).await.unwrap();
2353        assert_eq!(first, second);
2354        assert_eq!(
2355            f.stub.last_workspaces(),
2356            vec![first],
2357            "still exactly one workspace"
2358        );
2359    }
2360
2361    #[tokio::test(flavor = "multi_thread")]
2362    async fn a_registration_carries_the_complete_current_list() {
2363        let f = fixture().await;
2364        let second_root = f.second_root();
2365
2366        let a = f.runtime.enable(&f.root).await.unwrap();
2367        let b = f.runtime.enable(&second_root).await.unwrap();
2368
2369        let mut seen = f.stub.last_workspaces();
2370        seen.sort();
2371        let mut want = vec![a, b];
2372        want.sort();
2373        assert_eq!(seen, want, "registration is the whole set, not a delta");
2374    }
2375
2376    #[tokio::test(flavor = "multi_thread")]
2377    async fn no_later_registration_clears_this_apps_routes() {
2378        // The device id is asked of the bridge once, on the first `enable`, with an empty
2379        // workspace list — the one moment that is safe, because no route exists yet. Every
2380        // later registration must name the complete set: one with an empty list would
2381        // briefly clear this application's routes, and a peer asking for a workspace this
2382        // server owns would be told nobody has it.
2383        let f = fixture().await;
2384        let second_root = f.second_root();
2385
2386        f.runtime.enable(&f.root).await.unwrap();
2387        f.runtime.enable(&second_root).await.unwrap();
2388        f.runtime.sync_now(&f.root).await.unwrap();
2389        f.runtime.disable(&f.root).await.unwrap();
2390
2391        let seen = f.stub.seen.lock().expect("stub");
2392        let first_full = seen
2393            .registrations
2394            .iter()
2395            .position(|r| !r.workspaces.is_empty())
2396            .expect("enabling registers the workspace");
2397        let emptied_afterwards = seen.registrations[first_full + 1..]
2398            .iter()
2399            .filter(|r| r.workspaces.is_empty())
2400            .count();
2401        assert_eq!(
2402            emptied_afterwards, 0,
2403            "a registration after the first workspace must still name the whole set"
2404        );
2405    }
2406
2407    #[tokio::test(flavor = "multi_thread")]
2408    async fn disabling_unregisters_and_leaves_the_files_alone() {
2409        let f = fixture().await;
2410        std::fs::write(f.root.join("keep.md"), "content").unwrap();
2411
2412        let id = f.runtime.enable(&f.root).await.unwrap();
2413        f.runtime.disable(&f.root).await.unwrap();
2414
2415        assert!(f.stub.last_workspaces().is_empty());
2416        assert!(
2417            f.root.join("keep.md").exists(),
2418            "disabling sync must not touch files"
2419        );
2420        assert!(
2421            crate::sync::id::sync_id_path("sapphire-synctest", &f.root).exists(),
2422            "the sync id stays, so re-enabling rejoins the same workspace"
2423        );
2424        let _ = id;
2425    }
2426
2427    #[tokio::test(flavor = "multi_thread")]
2428    async fn re_enabling_after_disabling_reuses_the_identity() {
2429        let f = fixture().await;
2430
2431        let first = f.runtime.enable(&f.root).await.unwrap();
2432        f.runtime.disable(&f.root).await.unwrap();
2433        let again = f.runtime.enable(&f.root).await.unwrap();
2434
2435        assert_eq!(first, again);
2436    }
2437
2438    #[tokio::test(flavor = "multi_thread")]
2439    async fn status_reports_disabled_for_a_workspace_that_was_never_enabled() {
2440        let f = fixture().await;
2441
2442        let status = f.runtime.status(&f.root).await;
2443        assert!(!status.enabled);
2444        assert!(status.workspace_id.is_none());
2445    }
2446
2447    #[tokio::test(flavor = "multi_thread")]
2448    async fn disabling_something_that_was_never_enabled_is_not_an_error() {
2449        let f = fixture().await;
2450        f.runtime
2451            .disable(&f.root)
2452            .await
2453            .expect("disabling twice is fine");
2454    }
2455
2456    #[tokio::test(flavor = "multi_thread")]
2457    async fn a_file_already_on_disk_is_recorded_on_enable() {
2458        // No watcher fires for a file written before the workspace was enabled; without a
2459        // scan on enable it was never recorded, and every session sent nothing.
2460        let f = fixture().await;
2461        std::fs::write(f.root.join("a.md"), "content").unwrap();
2462        f.runtime.enable(&f.root).await.unwrap();
2463        f.runtime.settle_dials().await;
2464
2465        let replica = f.runtime.replica_of(&f.root).await.unwrap();
2466        let states = replica.lock().await.states().unwrap();
2467        assert!(
2468            states.iter().any(|(path, _)| path == "a.md"),
2469            "a file on disk before enable must be recorded: {states:?}"
2470        );
2471    }
2472
2473    #[tokio::test(flavor = "multi_thread")]
2474    async fn re_enabling_while_the_replica_is_still_held_takes_it_back() {
2475        // The first scan after enabling runs in the background; a disable and a re-enable
2476        // can both land while it still holds the store, which redb opens only once.
2477        let f = fixture().await;
2478        f.runtime.enable(&f.root).await.unwrap();
2479        f.runtime.settle_dials().await;
2480        let held = f.runtime.replica_of(&f.root).await.unwrap();
2481
2482        f.runtime.disable(&f.root).await.unwrap();
2483        f.runtime
2484            .enable(&f.root)
2485            .await
2486            .expect("a re-enable must not open the store a second time");
2487
2488        let again = f.runtime.replica_of(&f.root).await.unwrap();
2489        assert!(Arc::ptr_eq(&held, &again), "the open replica is taken back");
2490    }
2491
2492    #[tokio::test(flavor = "multi_thread")]
2493    async fn re_enabling_while_the_replica_is_still_closing_waits_for_it() {
2494        // The last holder let go just before the re-enable: the replica can no longer be
2495        // taken back, and its store closes a moment later, as it drops.
2496        let f = fixture().await;
2497        f.runtime.enable(&f.root).await.unwrap();
2498        f.runtime.settle_dials().await;
2499        let config = {
2500            let replica = f.runtime.replica_of(&f.root).await.unwrap();
2501            replica.lock().await.config().clone()
2502        };
2503        f.runtime.disable(&f.root).await.unwrap();
2504
2505        // The store, open still, with nothing left to take back.
2506        let closing = Replica::open(config, Arc::new(SystemClock)).unwrap();
2507        let key = f.root.canonicalize().unwrap();
2508        f.runtime
2509            .stopped
2510            .lock()
2511            .unwrap()
2512            .insert(key, std::sync::Weak::new());
2513        let closed = tokio::spawn(async move {
2514            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2515            drop(closing);
2516        });
2517
2518        f.runtime
2519            .enable(&f.root)
2520            .await
2521            .expect("a re-enable must wait for a store that is closing");
2522        closed.await.unwrap();
2523    }
2524
2525    #[tokio::test(flavor = "multi_thread")]
2526    async fn a_workspace_whose_root_vanished_is_reported_as_paused_not_deleted() {
2527        let f = fixture().await;
2528        std::fs::write(f.root.join("a.md"), "content").unwrap();
2529        f.runtime.enable(&f.root).await.unwrap();
2530        f.runtime.scan(&f.root).await.unwrap();
2531
2532        // The drive is unmounted: the marker directory goes with it.
2533        std::fs::remove_dir_all(f.root.join(".sapphire-synctest")).unwrap();
2534        let _ = f.runtime.scan(&f.root).await;
2535
2536        let status = f.runtime.status(&f.root).await;
2537        assert!(
2538            status.paused.is_some(),
2539            "a missing root must pause, not replicate as a mass deletion"
2540        );
2541    }
2542
2543    #[tokio::test(flavor = "multi_thread")]
2544    async fn a_directory_that_is_not_a_workspace_cannot_be_enabled() {
2545        let f = fixture().await;
2546        let plain = f._tmp.path().join("plain");
2547        std::fs::create_dir_all(&plain).unwrap();
2548
2549        assert!(f.runtime.enable(&plain).await.is_err());
2550    }
2551
2552    #[tokio::test(flavor = "multi_thread")]
2553    async fn a_write_through_the_workspace_router_is_committed_to_the_replica() {
2554        // The brief's "main path": a write the app server itself made is scanned straight
2555        // after, not left to the watcher's debounce. Asserted against the replica's own
2556        // state, which is the only thing that can tell the two paths apart — reading the
2557        // file back would pass even if the scan never ran.
2558        let f = fixture().await;
2559        f.runtime.enable(&f.root).await.unwrap();
2560
2561        let host = Arc::new(crate::WorkspaceHost::new(&CTX));
2562        host.select(&f.root, false).await.unwrap();
2563        let current = Arc::new(crate::Current::new(
2564            &CTX,
2565            host,
2566            Some(Arc::clone(&f.runtime)),
2567            crate::selection::SelectionFile::at(f._tmp.path().join("workspace.toml")),
2568        ));
2569        let router = Arc::new(crate::workspace_router(current));
2570        let (client_conn, server_conn) = sapphire_ipc::Connection::pair();
2571        tokio::spawn(async move {
2572            let info = sapphire_ipc::ServerInfo {
2573                version: "0.0.0".into(),
2574                api: sapphire_backend::protocol::API_VERSION,
2575                app_api: None,
2576                pid: std::process::id(),
2577                managed_by: ManagedBy::Spawned,
2578            };
2579            let _ = sapphire_ipc::serve(server_conn, router, "sapphire-synctest", info).await;
2580        });
2581        let info = sapphire_ipc::ClientInfo {
2582            kind: "test".into(),
2583            version: "0.0.0".into(),
2584            api: sapphire_backend::protocol::API_VERSION,
2585            app_api: None,
2586            pid: std::process::id(),
2587        };
2588        let (client, _) = sapphire_ipc::Client::handshake(client_conn, "sapphire-synctest", info)
2589            .await
2590            .unwrap();
2591
2592        let _: sapphire_backend::protocol::Ack = client
2593            .call(
2594                sapphire_backend::protocol::WRITE_FILE,
2595                sapphire_backend::protocol::ContentParams {
2596                    path: PathBuf::from("through-the-server.md"),
2597                    content: "written by the server".into(),
2598                },
2599            )
2600            .await
2601            .unwrap();
2602
2603        // `scan` runs before the reply is written, so by the time the call returns the
2604        // replica must already know the path.
2605        let synced = f.runtime.synced.lock().await;
2606        let entry = synced.get(&f.root).expect("enabled above");
2607        let replica = entry.replica.lock().await;
2608        let state = replica
2609            .state("through-the-server.md")
2610            .expect("a readable state");
2611        assert!(
2612            state.is_some(),
2613            "a write through the server must be scanned into the replica, not left to the watcher"
2614        );
2615    }
2616
2617    #[tokio::test(flavor = "multi_thread")]
2618    async fn an_announcement_for_an_unknown_workspace_is_ignored_not_fatal() {
2619        let f = fixture().await;
2620        f.runtime.enable(&f.root).await.unwrap();
2621
2622        let driver = Arc::clone(&f.runtime);
2623        let handle = tokio::spawn(async move { driver.run().await });
2624
2625        f.stub
2626            .announce(grain_id::GrainId::random(), "no-such-ticket")
2627            .await;
2628        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
2629
2630        assert!(
2631            !handle.is_finished(),
2632            "one bad announcement must not stop the runtime"
2633        );
2634        handle.abort();
2635    }
2636}