Skip to main content

sapphire_framework_server/sync/
mod.rs

1//! Replication, from the app server's side.
2//!
3//! One `Replica` per synced workspace, registered with the bridge, driven by announcements
4//! from it and by local edits.
5//!
6//! See `docs/superpowers/specs/2026-09-16-process-architecture-design.md` §4.2.
7
8use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use grain_id::GrainId;
13use sapphire_backend::BackendEvent;
14use sapphire_backend::protocol as proto;
15use sapphire_bridge_api::{
16    IncomingParams, RegisterParams, WorkspaceRegistration, WorkspacesResult,
17};
18use sapphire_bridge_client::BridgeClient;
19use sapphire_ipc::ManagedBy;
20use sapphire_sync::{PathUpdate, PauseReason, Replica, ReplicaConfig, ScanOutcome, SystemClock};
21use sapphire_workspace::{AppContext, Workspace};
22use tokio::sync::{Mutex, OnceCell, broadcast, mpsc};
23
24pub mod id;
25mod live;
26mod methods;
27#[cfg(any(test, feature = "test-util"))]
28pub mod testing;
29mod topology;
30mod watch;
31
32pub use id::{SYNC_ID_FILE, WORKSPACE_MAP_FILE, sync_id, sync_id_path};
33pub use live::{DIAL_BACKOFF_MAX, OPEN_TIMEOUT};
34pub use methods::sync_router;
35/// What [`SyncRuntime::peers`] answers with.
36pub use sapphire_bridge_api::{PeerInfo, PeersResult};
37pub use watch::{DEBOUNCE, Watcher};
38
39use live::{LivePeers, dial_backoff};
40
41use crate::error::{Error, Result};
42use crate::settings::Settings;
43
44use crate::host::WorkspaceHost;
45
46/// What `sync.status` reports.
47#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
48pub struct SyncStatus {
49    /// Whether this workspace is synced at all.
50    pub enabled: bool,
51    /// Its identity across devices, when it is.
52    pub workspace_id: Option<GrainId>,
53    /// How many devices the workgroup has, this host excluded.
54    pub peers: usize,
55    /// Why replication is paused, if it is — a missing root or marker directory.
56    pub paused: Option<String>,
57    /// The last failure, if any.
58    pub last_error: Option<String>,
59    /// Whether the bridge is reachable. `false` is not an outage: the app server works.
60    pub bridge_available: bool,
61    /// How the workspace is wired to its peers, by the roles the bridge last reported.
62    pub topology: proto::Topology,
63    /// How far embedding has got, when this host embeds (#188).
64    pub embedding: Option<proto::EmbeddingProgress>,
65    /// Some edit this device made since the workspace was enabled has not reached a peer.
66    pub pending_delivery: bool,
67    /// How many open live sessions have a peer that confirms what it is pushed.
68    pub acknowledging_peers: usize,
69}
70
71/// One workspace this runtime syncs.
72struct Synced {
73    /// Its identity across devices, shared with every peer.
74    workspace_id: GrainId,
75    /// Its replica store. Held open because the store is a redb database.
76    replica: Arc<Mutex<Replica>>,
77    /// Why replication is paused, from the last scan.
78    paused: Option<PauseReason>,
79    /// The last failure, if any.
80    last_error: Option<String>,
81    /// This device's own counter as far as a peer has confirmed having it: every local edit
82    /// up to here is known to have left the device. Starts at the counter the replica had
83    /// when it was enabled — what came before is not waited for — and moves only when a peer
84    /// covers *all* of this device's edits, so that `SyncDelivered` means "everything".
85    delivered_through: u64,
86}
87
88/// How old a vector file no live content needs must be before the primary device removes
89/// it: a vector can arrive before its file, and a day is far longer than that window.
90const STALE_VECTOR_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
91
92/// Texts per embedding request in a server pass: small, so a search query waiting on the
93/// bridge's one worker is never stuck behind a long backfill (#188).
94const PASS_BATCH: usize = 4;
95
96/// How the primary device paces its backfill (#188).
97#[derive(Clone, Copy, Debug)]
98pub struct BackfillTiming {
99    /// How old another device's version must be before the primary embeds it: its author
100    /// has embedded it by then, or will not.
101    pub grace: std::time::Duration,
102    /// How often the primary device runs a pass with nothing prompting it.
103    pub interval: std::time::Duration,
104}
105
106impl Default for BackfillTiming {
107    fn default() -> Self {
108        BackfillTiming {
109            grace: std::time::Duration::from_secs(10 * 60),
110            interval: std::time::Duration::from_secs(10 * 60),
111        }
112    }
113}
114
115/// How long [`SyncRuntime::keep_connected`] waits before its first retry after the bridge
116/// could not be reached. Each failure doubles it, up to [`RECONNECT_BACKOFF_MAX`].
117pub const RECONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_millis(500);
118
119/// The longest [`SyncRuntime::keep_connected`] waits between two attempts to reach the
120/// bridge: a bridge that comes up is found within this long.
121pub const RECONNECT_BACKOFF_MAX: std::time::Duration = std::time::Duration::from_secs(30);
122
123/// The wait before reconnection attempt `attempt` (0-based), backing off from `first`.
124fn reconnect_backoff(first: std::time::Duration, attempt: u32) -> std::time::Duration {
125    first
126        .saturating_mul(1u32 << attempt.min(16))
127        .min(RECONNECT_BACKOFF_MAX.max(first))
128}
129
130/// What a connection attempt to the bridge yields.
131type Connecting = std::pin::Pin<
132    Box<dyn std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>> + Send>,
133>;
134
135/// How a [`SyncRuntime`] reaches the bridge, on start and whenever it has lost it (#244).
136///
137/// An attempt answers `Ok(None)` when no bridge is running — the runtime then tries again
138/// later — and an error for anything else, such as a bridge of another API version.
139#[derive(Clone)]
140pub struct BridgeConnector(Arc<dyn Fn() -> Connecting + Send + Sync>);
141
142impl BridgeConnector {
143    /// Connect with `connect`.
144    pub fn new<F, Fut>(connect: F) -> BridgeConnector
145    where
146        F: Fn() -> Fut + Send + Sync + 'static,
147        Fut: std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>>
148            + Send
149            + 'static,
150    {
151        BridgeConnector(Arc::new(move || Box::pin(connect())))
152    }
153
154    /// The host's own bridge, at its standard endpoint, as a client of `kind` at
155    /// `version`.
156    pub fn standard(kind: &'static str, version: &'static str) -> BridgeConnector {
157        BridgeConnector::new(move || async move {
158            match BridgeClient::connect(kind, version).await {
159                Ok(client) => Ok(Some(client)),
160                Err(sapphire_ipc::Error::NotRunning(_)) => Ok(None),
161                Err(err) => Err(err),
162            }
163        })
164    }
165
166    /// The bridge listening at `endpoint`: for a fixture that runs its own.
167    pub fn at(
168        endpoint: sapphire_ipc::Endpoint,
169        kind: &'static str,
170        version: &'static str,
171    ) -> BridgeConnector {
172        BridgeConnector::new(move || {
173            let endpoint = endpoint.clone();
174            async move { BridgeClient::connect_at(&endpoint, kind, version).await }
175        })
176    }
177
178    /// One attempt: `None` when no bridge is running.
179    pub(crate) async fn connect(&self) -> sapphire_ipc::Result<Option<BridgeClient>> {
180        (self.0)().await
181    }
182}
183
184impl std::fmt::Debug for BridgeConnector {
185    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
186        f.write_str("BridgeConnector")
187    }
188}
189
190/// What [`SyncRuntime::reclaim`] found of a replica `disable` let go of.
191enum Reclaimed {
192    /// Still held: here it is.
193    Open(Arc<Mutex<Replica>>),
194    /// Let go of since, so its store may be closing still.
195    Closing,
196    /// Nothing to take back.
197    Closed,
198}
199
200/// How often, and how far apart, `enable` tries to open a store that may still be closing.
201const CLOSING_RETRIES: u32 = 50;
202const CLOSING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(20);
203
204/// Replication for one application's workspaces.
205pub struct SyncRuntime {
206    ctx: &'static AppContext,
207    /// The bridge connection, while there is one. Swapped when the runtime reconnects
208    /// (#244); [`run`](SyncRuntime::run) follows it.
209    bridge: tokio::sync::watch::Sender<Option<Arc<BridgeClient>>>,
210    /// How to reach the bridge again; `None` for a runtime given one connection to use.
211    connector: Option<BridgeConnector>,
212    /// Woken by [`reconnect_now`](SyncRuntime::reconnect_now): someone wants the bridge
213    /// now, so [`keep_connected`](SyncRuntime::keep_connected) cuts its backoff short.
214    reconnect_now: tokio::sync::Notify,
215    /// Workspaces asked to sync while the bridge was away, with their ids. Enabled once
216    /// it is back.
217    ///
218    /// Also the lock that orders an `enable` against a reconnection: whoever holds it sees
219    /// the bridge slot as the other left it, so a workspace is either enabled now or left
220    /// here for the reconnection to enable — never neither.
221    pending: Mutex<HashMap<PathBuf, GrainId>>,
222    /// The announcements of a connection [`attach`](SyncRuntime::attach) just made,
223    /// subscribed to before it registered: the bridge announces the streams that waited for
224    /// this server as soon as it registers (#244), and a subscription made afterwards by
225    /// [`run`](SyncRuntime::run) would miss them.
226    subscribed: std::sync::Mutex<Option<(Arc<BridgeClient>, broadcast::Receiver<IncomingParams>)>>,
227    exe_path: PathBuf,
228    managed_by: ManagedBy,
229    synced: Mutex<HashMap<PathBuf, Synced>>,
230    /// The replicas [`disable`](SyncRuntime::disable) let go of, by canonical root, for as
231    /// long as something else still holds one: a scan, a session or the first dial that
232    /// began before. The store is a redb database, open until its last holder drops it, so
233    /// a re-enable meanwhile could not open it again; [`enable`](SyncRuntime::enable)
234    /// takes the open one back instead. [`release`](SyncRuntime::release) puts nothing
235    /// here: its store is about to be moved or deleted.
236    stopped: std::sync::Mutex<HashMap<PathBuf, std::sync::Weak<Mutex<Replica>>>>,
237    /// This host's device id. Asked of the bridge once, on the first [`enable`], and never
238    /// again: a second registration with an empty workspace list would briefly clear this
239    /// application's routes.
240    ///
241    /// [`enable`]: SyncRuntime::enable
242    device_id: OnceCell<GrainId>,
243    /// The file watcher, once [`watch_changes`] has started it.
244    ///
245    /// The runtime owns it because the spec puts it here: the sync runtime is a watcher and
246    /// a replica per registered root, and the two must agree on which roots are synced.
247    /// [`enable`] and [`disable`] keep them in step.
248    ///
249    /// [`watch_changes`]: SyncRuntime::watch_changes
250    /// [`enable`]: SyncRuntime::enable
251    /// [`disable`]: SyncRuntime::disable
252    watcher: OnceCell<Arc<Watcher>>,
253    /// The app server that owns these workspaces' caches, once it has told us.
254    ///
255    /// A replication session writes files, and the search index is not part of a session: the
256    /// index belongs to the app server, which holds the workspace open. Reaching it through
257    /// the host is what keeps a received file and its index moving together, and it is the
258    /// *same* [`WorkspaceState`](sapphire_workspace::WorkspaceState) the server serves from —
259    /// opening a second one would collide on the retrieve store's exclusive lock.
260    ///
261    /// Absent for a runtime nobody has wired to an app server (a test stub driving the
262    /// replica directly). Nothing then indexes, which is correct: nothing owns an index.
263    host: OnceCell<Arc<WorkspaceHost>>,
264    /// The app server's settings, whose workspace scope is a file in the selected
265    /// workspace's marker: read again when a session or a local edit changes the files.
266    settings: std::sync::Mutex<Option<Settings>>,
267    /// Serialises re-indexing, so two sessions finishing at once do not both sweep the
268    /// workspace through the same store.
269    reindexing: Mutex<()>,
270    /// Roots with an embedding pass waiting to run (#187). A request for a root already
271    /// waiting is folded into that pass.
272    embed_wanted: Mutex<std::collections::HashSet<PathBuf>>,
273    /// Serialises embedding passes: one model, one queue.
274    embedding: Mutex<()>,
275    /// Roots whose embedding pass is running now, for `sync.status`.
276    embed_running: Mutex<std::collections::HashSet<PathBuf>>,
277    /// The primary device's backfill pacing (#188).
278    backfill: std::sync::Mutex<BackfillTiming>,
279    /// The bridge an embedding pass asks for its model; `None` is the host's standard one.
280    bridge_endpoint: Option<sapphire_ipc::Endpoint>,
281    /// The live session table of every synced workspace, keyed by canonical root.
282    ///
283    /// One table per workspace rather than one for the runtime: a session is about one
284    /// workspace, and the two storm rules are per-workspace rules. A root that is not
285    /// synced has none, so a dial or a push for it is a no-op rather than an error.
286    live: Mutex<HashMap<PathBuf, Arc<LivePeers>>>,
287    /// The bridge's last answer to `peers`, roles included. The inbound guard and
288    /// [`is_primary`](SyncRuntime::is_primary) read it rather than asking again.
289    last_peers: Mutex<Option<PeersResult>>,
290    /// How long a dial waits for the bridge to open a stream; [`OPEN_TIMEOUT`] by default.
291    open_timeout: std::time::Duration,
292    /// The first wait of [`keep_connected`](SyncRuntime::keep_connected)'s backoff;
293    /// [`RECONNECT_BACKOFF`] but for tests.
294    reconnect_first: std::time::Duration,
295    /// The first dials [`enable`](SyncRuntime::enable) leaves running. Each pins the
296    /// runtime, so [`stop_dials`](SyncRuntime::stop_dials) ends them with the server.
297    first_dials: std::sync::Mutex<tokio::task::JoinSet<()>>,
298    /// The rest of the runtime's background work: inbound sessions, the readers of live
299    /// sessions, and embedding passes. Each pins the runtime — and through it the replica
300    /// stores, which are redb databases locked until the last pin drops — so
301    /// [`stop_background`](SyncRuntime::stop_background) ends them with the server (#235).
302    background: std::sync::Mutex<tokio::task::JoinSet<()>>,
303}
304
305impl SyncRuntime {
306    /// A runtime for `ctx`'s application, syncing through `bridge`.
307    ///
308    /// It has no way to reach the bridge again: when the connection closes, sync stops until
309    /// the runtime is given a [`connector`](Self::with_connector).
310    pub fn new(
311        ctx: &'static AppContext,
312        bridge: Arc<BridgeClient>,
313        exe_path: PathBuf,
314        managed_by: ManagedBy,
315    ) -> SyncRuntime {
316        SyncRuntime::build(ctx, Some(bridge), None, exe_path, managed_by)
317    }
318
319    /// A runtime for `ctx`'s application that reaches the bridge with `connector`, and again
320    /// whenever it loses it (#244).
321    ///
322    /// Not connected yet: [`connect`](Self::connect) makes the first attempt, and
323    /// [`keep_connected`](Self::keep_connected) the rest. [`AppServer::run`] does both.
324    ///
325    /// [`AppServer::run`]: crate::AppServer::run
326    pub fn connecting(
327        ctx: &'static AppContext,
328        connector: BridgeConnector,
329        exe_path: PathBuf,
330        managed_by: ManagedBy,
331    ) -> SyncRuntime {
332        SyncRuntime::build(ctx, None, Some(connector), exe_path, managed_by)
333    }
334
335    /// Reach the bridge again with `connector` once the connection closes.
336    pub fn with_connector(mut self, connector: BridgeConnector) -> SyncRuntime {
337        self.connector = Some(connector);
338        self
339    }
340
341    fn build(
342        ctx: &'static AppContext,
343        bridge: Option<Arc<BridgeClient>>,
344        connector: Option<BridgeConnector>,
345        exe_path: PathBuf,
346        managed_by: ManagedBy,
347    ) -> SyncRuntime {
348        SyncRuntime {
349            ctx,
350            bridge: tokio::sync::watch::Sender::new(bridge),
351            connector,
352            reconnect_now: tokio::sync::Notify::new(),
353            pending: Mutex::new(HashMap::new()),
354            subscribed: std::sync::Mutex::new(None),
355            exe_path,
356            managed_by,
357            synced: Mutex::new(HashMap::new()),
358            stopped: std::sync::Mutex::new(HashMap::new()),
359            device_id: OnceCell::new(),
360            watcher: OnceCell::new(),
361            host: OnceCell::new(),
362            settings: std::sync::Mutex::new(None),
363            reindexing: Mutex::new(()),
364            embed_wanted: Mutex::new(std::collections::HashSet::new()),
365            embedding: Mutex::new(()),
366            embed_running: Mutex::new(std::collections::HashSet::new()),
367            backfill: std::sync::Mutex::new(BackfillTiming::default()),
368            bridge_endpoint: None,
369            live: Mutex::new(HashMap::new()),
370            last_peers: Mutex::new(None),
371            open_timeout: OPEN_TIMEOUT,
372            reconnect_first: RECONNECT_BACKOFF,
373            first_dials: std::sync::Mutex::new(tokio::task::JoinSet::new()),
374            background: std::sync::Mutex::new(tokio::task::JoinSet::new()),
375        }
376    }
377
378    /// The bridge, while connected.
379    fn bridge(&self) -> Result<Arc<BridgeClient>> {
380        match &*self.bridge.borrow() {
381            Some(bridge) if !bridge.is_closed() => Ok(Arc::clone(bridge)),
382            _ => Err(Error::BridgeDisconnected),
383        }
384    }
385
386    /// Whether the runtime has a live connection to the bridge.
387    pub fn is_connected(&self) -> bool {
388        self.bridge().is_ok()
389    }
390
391    /// The bridge connection, while there is an open one: what the relay forwards over.
392    pub(crate) fn connected_bridge(&self) -> Option<Arc<BridgeClient>> {
393        self.bridge().ok()
394    }
395
396    /// Have [`keep_connected`](Self::keep_connected) try the bridge now rather than when its
397    /// backoff next ends, and start its backoff over: for the relay, whose client may just
398    /// have started the bridge. A no-op while connected, bar one early attempt after the
399    /// next loss, which makes one at once anyway.
400    pub(crate) fn reconnect_now(&self) {
401        self.reconnect_now.notify_one();
402    }
403
404    /// The bridge slot, followed: changes each time the runtime connects or loses the
405    /// connection.
406    pub(crate) fn bridge_changes(&self) -> tokio::sync::watch::Receiver<Option<Arc<BridgeClient>>> {
407        self.bridge.subscribe()
408    }
409
410    /// Run `task` in the background, tracked so the server's shutdown ends it.
411    fn spawn_background(&self, task: impl std::future::Future<Output = ()> + Send + 'static) {
412        let mut background = self.background.lock().expect("background tasks");
413        // Reap the finished ones, so the set holds only what is running.
414        while background.try_join_next().is_some() {}
415        background.spawn(task);
416    }
417
418    /// Give up on opening a stream to a peer after `timeout` rather than [`OPEN_TIMEOUT`]:
419    /// for tests, which cannot wait that long for a peer that never answers.
420    pub fn with_open_timeout(mut self, timeout: std::time::Duration) -> SyncRuntime {
421        self.open_timeout = timeout;
422        self
423    }
424
425    /// Back off from `first` rather than [`RECONNECT_BACKOFF`] while the bridge cannot be
426    /// reached: for tests of a backoff that has grown long, which cannot wait for it to.
427    #[cfg(test)]
428    pub(crate) fn with_reconnect_backoff(mut self, first: std::time::Duration) -> SyncRuntime {
429        self.reconnect_first = first;
430        self
431    }
432
433    /// Tell the runtime which app server owns these workspaces' caches.
434    ///
435    /// Called by [`AppServer::sync`](crate::AppServer::sync). First writer wins: a runtime
436    /// belongs to the server that wired it.
437    /// Embed through the bridge at `endpoint` rather than the host's standard one: for a
438    /// fixture running several hosts in one process.
439    pub fn with_bridge_endpoint(mut self, endpoint: sapphire_ipc::Endpoint) -> SyncRuntime {
440        self.bridge_endpoint = Some(endpoint);
441        self
442    }
443
444    /// Pace the primary device's backfill differently: for tests, which cannot wait ten
445    /// minutes.
446    pub fn set_backfill_timing(&self, timing: BackfillTiming) {
447        *self.backfill.lock().expect("backfill timing") = timing;
448    }
449
450    fn backfill_timing(&self) -> BackfillTiming {
451        *self.backfill.lock().expect("backfill timing")
452    }
453
454    pub(crate) fn set_host(&self, host: Arc<WorkspaceHost>) {
455        let _ = self.host.set(host);
456    }
457
458    /// Read `settings` again whenever a workspace's files change under this runtime — a
459    /// session wrote them, or someone edited them — so workspace keys follow the marker's
460    /// `config.toml`. Replaced, not kept, by a later call (a restarted server's).
461    pub(crate) fn set_settings(&self, settings: Option<Settings>) {
462        *self.settings.lock().unwrap_or_else(|e| e.into_inner()) = settings;
463    }
464
465    /// Read the settings again after the files under a workspace changed. Cheap — two
466    /// small files — and holding no lock past the read, so it runs on every change.
467    fn refresh_settings(&self) {
468        let settings = self
469            .settings
470            .lock()
471            .unwrap_or_else(|e| e.into_inner())
472            .clone();
473        if let Some(settings) = settings
474            && settings.refresh().is_err()
475        {
476            // The error may quote the file; the change itself went through.
477            tracing::warn!("could not read the settings again after a change to the files");
478        }
479    }
480
481    /// Start syncing `root`, returning its identity across devices.
482    ///
483    /// Idempotent: enabling an already-synced workspace returns the same id and changes
484    /// nothing.
485    pub async fn enable(self: &Arc<Self>, root: &Path) -> Result<GrainId> {
486        let key = root.canonicalize().map_err(Error::Io)?;
487        let watch_key = key.clone();
488        let workspace_id = sync_id(self.ctx.app_name, &key)?;
489
490        // Without a bridge there is no device id to open the replica with, and nobody to
491        // register with: keep the workspace for the reconnection to enable (#244).
492        {
493            let mut pending = self.pending.lock().await;
494            if self.bridge().is_err() {
495                pending.insert(key, workspace_id);
496                return Err(Error::BridgeDisconnected);
497            }
498            pending.remove(&key);
499        }
500
501        // The map stays locked across opening the replica, so two simultaneous `enable`s for
502        // one root cannot both open it — the store is a redb database, and it is opened once.
503        {
504            let mut synced = self.synced.lock().await;
505            if let Some(existing) = synced.get(&key) {
506                return Ok(existing.workspace_id);
507            }
508            // A replica disabled a moment ago may still be open; it is this root's, with this
509            // runtime's one device id, so it is taken back rather than opened a second time.
510            let replica = match self.reclaim(&key) {
511                Reclaimed::Open(replica) => replica,
512                reclaimed => {
513                    let device_id = self.device_id().await?;
514                    let state_dir = self.state_dir(&key)?;
515                    std::fs::create_dir_all(&state_dir).map_err(Error::Io)?;
516                    let config =
517                        ReplicaConfig::new(self.ctx.app_name, key.clone(), device_id, &state_dir);
518                    // The last holder let go between the two: its store closes as the replica
519                    // drops, just after it can no longer be taken back. Give it that moment.
520                    let mut tries = match reclaimed {
521                        Reclaimed::Closing => CLOSING_RETRIES,
522                        _ => 0,
523                    };
524                    let replica = loop {
525                        match Replica::open(config.clone(), Arc::new(SystemClock)) {
526                            Ok(replica) => break replica,
527                            Err(_) if tries > 0 => {
528                                tries -= 1;
529                                tokio::time::sleep(CLOSING_RETRY_DELAY).await;
530                            }
531                            Err(err) => return Err(Error::Sync(err.to_string())),
532                        }
533                    };
534                    Arc::new(Mutex::new(replica))
535                }
536            };
537            // Before the first scan below, so an edit made while the server was down is
538            // waited for like any other.
539            let delivered_through = own_counter(&*replica.lock().await);
540            synced.insert(
541                key.clone(),
542                Synced {
543                    workspace_id,
544                    replica,
545                    paused: None,
546                    last_error: None,
547                    delivered_through,
548                },
549            );
550        }
551
552        // Watch the new root too, so an edit made outside the server from now on is seen.
553        // Not fatal if it fails: the app server's own writes take the exact path in
554        // `handlers.rs`, and the next `watch_changes` re-reads the root set anyway.
555        if let Some(watcher) = self.watcher.get()
556            && let Err(err) = watcher.watch(&watch_key)
557        {
558            tracing::warn!(root = %watch_key.display(), "could not watch: {err}");
559        }
560
561        // Register only once the replica is open, so a registration never names a workspace
562        // that cannot serve a session.
563        self.reregister().await?;
564
565        // Dial now, so enabling a workspace converges it rather than waiting for the next
566        // local edit to trigger a session. This is what catches a host up when it returns:
567        // the peer that has been running is the one the returning host has to ask, and no
568        // watcher on either side fires for a file that was never written locally.
569        //
570        // Best-effort: a workspace is enabled whether or not any peer is up, and a failure
571        // here is not a failure to enable. A session is symmetric, so dialing is enough —
572        // this side sends its own view, and the peer answers with what this side lacks.
573        //
574        // In the background, so the answer does not wait on it: a peer that is offline
575        // holds a dial up to `open_timeout`, longer than a client waits for `sync.enable`
576        // (#228).
577        //
578        // Scan first: a file already on disk when the workspace is enabled — a folder that
579        // existed before sync was turned on, or an edit made while the server was down —
580        // fires no watcher, so without this it is never recorded and the session has
581        // nothing to send. Logged, not fatal: the session still fetches what peers have.
582        let runtime = Arc::clone(self);
583        let mut dials = self.first_dials.lock().expect("first dials");
584        // Reap the finished ones, so the set holds only what is running.
585        while dials.try_join_next().is_some() {}
586        dials.spawn(async move {
587            if let Err(err) = runtime.scan(&key).await {
588                tracing::warn!(root = %key.display(), "the first scan after enabling failed: {err}");
589            }
590            if let Err(err) = runtime.sync_now(&key).await {
591                tracing::warn!(root = %key.display(), "the first session after enabling failed: {err}");
592            }
593        });
594        Ok(workspace_id)
595    }
596
597    /// Stop syncing `root`. Files and the sync id stay, so re-enabling rejoins the same
598    /// workspace rather than creating a second one.
599    ///
600    /// Something that began before — a scan, a session, the first dial — may still hold the
601    /// replica open; a re-enable meanwhile takes it back (see [`reclaim`](Self::reclaim)).
602    pub async fn disable(&self, root: &Path) -> Result<()> {
603        let (replica, unregistered) = self.stop(root).await;
604        if let Some(replica) = replica {
605            let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
606            let mut stopped = self.stopped.lock().expect("stopped replicas");
607            // Forget the ones nothing holds any more, so the map stays small.
608            stopped.retain(|_, replica| replica.strong_count() > 0);
609            stopped.insert(key, Arc::downgrade(&replica));
610        }
611        unregistered
612    }
613
614    /// Stop syncing `root`, like [`disable`](Self::disable), then wait up to `limit` until
615    /// nothing holds its replica store open any more — a session, a scan or an embedding
616    /// pass that started before may still have it. A store about to be moved or deleted
617    /// must be closed first.
618    ///
619    /// Only a store still in use at `limit` is an error. The bridge not hearing of it is
620    /// logged, not returned: the store is closed all the same, and the next registration
621    /// replaces the bridge's view.
622    pub async fn release(&self, root: &Path, limit: std::time::Duration) -> Result<()> {
623        let (replica, unregistered) = self.stop(root).await;
624        if let Err(err) = unregistered {
625            tracing::warn!(root = %root.display(), "stopping sync: {err}");
626        }
627        let Some(replica) = replica else {
628            return Ok(());
629        };
630        let deadline = tokio::time::Instant::now() + limit;
631        while Arc::strong_count(&replica) > 1 {
632            if tokio::time::Instant::now() >= deadline {
633                return Err(Error::Sync(format!(
634                    "the replica of {} is still in use; try again",
635                    root.display()
636                )));
637            }
638            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
639        }
640        Ok(())
641    }
642
643    /// Stop syncing `root`, handing back its replica — `None` when it was not synced — and
644    /// how telling the bridge went.
645    async fn stop(&self, root: &Path) -> (Option<Arc<Mutex<Replica>>>, Result<()>) {
646        // A root that no longer exists cannot be canonicalized, but the keys of `synced` are
647        // canonical already, so the path as given (a registry root) still names the entry.
648        let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
649        self.pending.lock().await.remove(&key);
650        let removed = self.synced.lock().await.remove(&key);
651        if removed.is_some()
652            && let Some(watcher) = self.watcher.get()
653        {
654            watcher.unwatch(&key);
655        }
656        // Any live session for a workspace this server no longer holds is closed: it would
657        // otherwise keep pushing files for a workspace nobody asked about any more.
658        if let Some(peers) = self.live.lock().await.remove(&key) {
659            peers.drop_connections().await;
660        }
661        let Some(removed) = removed else {
662            return (None, Ok(()));
663        };
664        let unregistered = async {
665            // A bridge that is gone has forgotten this server's registrations already, and
666            // the reconnection registers only what is still synced.
667            let Ok(bridge) = self.bridge() else {
668                return Ok(());
669            };
670            bridge
671                .unregister(removed.workspace_id)
672                .await
673                .map_err(|e| Error::Bridge(e.to_string()))?;
674            self.reregister().await
675        }
676        .await;
677        (Some(removed.replica), unregistered)
678    }
679
680    /// The workspaces the workgroup knows about.
681    ///
682    /// Read straight from the bridge, so it reflects every device's registrations. The
683    /// bridge being down is not an empty answer — that would read as "nothing to map" — so
684    /// it is an error, and `sync.map` reports it.
685    pub async fn workspaces(&self) -> Result<WorkspacesResult> {
686        self.bridge()?
687            .workspaces()
688            .await
689            .map_err(|e| Error::Bridge(e.to_string()))
690    }
691
692    /// Place a workgroup workspace at a directory of this host: write its identity and
693    /// map there, returning the canonical root. [`Current::map`](crate::Current::map)
694    /// then selects it and syncs it.
695    ///
696    /// `selector` is the workspace's name or id, as [`workspaces`] lists it; `dir` must
697    /// exist. Its marker is made when it has none, and nothing else: the marker's
698    /// `config.toml` is the application's, and arrives with the first sync — a local one
699    /// written first would meet it as a conflict (#253).
700    ///
701    /// [`workspaces`]: SyncRuntime::workspaces
702    pub async fn place(&self, selector: &str, dir: &Path) -> Result<PathBuf> {
703        let workgroup = self.workspaces().await?;
704        let wanted = workgroup
705            .workspaces
706            .iter()
707            // A name is the user-facing handle; the id is the exact one. Two workspaces
708            // may not share a name, so a name names at most one.
709            .find(|w| w.name == selector || w.workspace_id.to_string() == selector)
710            .ok_or_else(|| Error::UnknownWorkspaceName(selector.to_owned()))?;
711
712        // Refuse another application's workspace: this server syncs its own application's
713        // workspaces, and `Workspace::from_root` would later refuse the directory anyway.
714        if wanted.app_name != self.ctx.app_name {
715            return Err(Error::WrongApp {
716                name: wanted.name.clone(),
717                app_name: wanted.app_name.clone(),
718            });
719        }
720
721        let root = dir.canonicalize().map_err(Error::Io)?;
722        let marker = root.join(format!(".{}", self.ctx.app_name));
723        if !marker.is_dir() {
724            std::fs::create_dir(&marker).map_err(Error::Io)?;
725        }
726        let workspace = Workspace::from_root(self.ctx, &root)?;
727        if workspace.root != root {
728            return Err(Error::UnknownWorkspace(root, self.ctx.app_name));
729        }
730
731        // A directory already syncing under another identity is not remapped: the sync id
732        // is a device's word that it is the same workspace, and rewriting it silently
733        // would make two hosts disagree about what they share.
734        {
735            let synced = self.synced.lock().await;
736            if let Some(entry) = synced.get(&root)
737                && entry.workspace_id != wanted.workspace_id
738            {
739                return Err(Error::SyncId(format!(
740                    "{} already syncs as {}, not {}",
741                    root.display(),
742                    entry.workspace_id,
743                    wanted.workspace_id
744                )));
745            }
746        }
747
748        // The identity arrives from the workgroup, not from this host: two hosts that map
749        // the same workspace must sync as one, so the id is the one the workgroup lists,
750        // written before `enable` reads it. A different id already on disk is refused
751        // above; an absent one is written here.
752        let marker = workspace.marker_dir();
753        let id_path = marker.join(crate::sync::id::SYNC_ID_FILE);
754        match std::fs::read_to_string(&id_path) {
755            Ok(existing) if existing.trim() != wanted.workspace_id.to_string() => {
756                return Err(Error::SyncId(format!(
757                    "{} already holds identity {}, not {}",
758                    root.display(),
759                    existing.trim(),
760                    wanted.workspace_id
761                )));
762            }
763            Err(_) => {
764                std::fs::write(&id_path, format!("{}\n", wanted.workspace_id))
765                    .map_err(Error::Io)?;
766            }
767            Ok(_) => {}
768        }
769
770        // The map is what tells this application, on a later start, which workgroup
771        // workspace a directory is.
772        let map_path = root
773            .join(format!(".{}", self.ctx.app_name))
774            .join(WORKSPACE_MAP_FILE);
775        let workspace_id = wanted.workspace_id.to_string();
776        std::fs::write(&map_path, format!("{workspace_id}\n")).map_err(Error::Io)?;
777        Ok(root)
778    }
779
780    /// What `sync.status` answers with.
781    pub async fn status(&self, root: &Path) -> SyncStatus {
782        let bridge = self.bridge().ok();
783        let bridge_available = match &bridge {
784            Some(bridge) => bridge.status().await.is_ok(),
785            None => false,
786        };
787        let answer = match &bridge {
788            Some(_) => self.peers().await.ok(),
789            None => None,
790        };
791        let peers = answer
792            .as_ref()
793            .map_or(0, |p| p.peers.len().saturating_sub(1));
794        let Ok(key) = root.canonicalize() else {
795            return SyncStatus {
796                enabled: false,
797                workspace_id: None,
798                peers,
799                paused: None,
800                last_error: None,
801                bridge_available,
802                topology: proto::Topology::Mesh,
803                embedding: None,
804                pending_delivery: false,
805                acknowledging_peers: 0,
806            };
807        };
808        let synced = self.synced.lock().await;
809        match synced.get(&key) {
810            Some(entry) => {
811                let replica = Arc::clone(&entry.replica);
812                let delivered_through = entry.delivered_through;
813                let base = SyncStatus {
814                    enabled: true,
815                    workspace_id: Some(entry.workspace_id),
816                    peers,
817                    paused: entry.paused.map(|r| format!("{r:?}")),
818                    last_error: entry.last_error.clone(),
819                    bridge_available,
820                    topology: topology::topology(
821                        answer
822                            .as_ref()
823                            .and_then(|p| p.roles_for(entry.workspace_id)),
824                    ),
825                    embedding: None,
826                    pending_delivery: false,
827                    acknowledging_peers: 0,
828                };
829                drop(synced);
830                // The map is unlocked first: the replica's lock may be waiting on a session
831                // step that takes the map.
832                let own = own_counter(&*replica.lock().await);
833                let acknowledging_peers = match self.live_peers(&key).await {
834                    Some(table) => table.acknowledging().await,
835                    None => 0,
836                };
837                SyncStatus {
838                    embedding: self.embedding_progress(&key).await,
839                    pending_delivery: own > delivered_through,
840                    acknowledging_peers,
841                    ..base
842                }
843            }
844            None => {
845                drop(synced);
846                // Asked to sync while the bridge was away: it is enabled, waiting for the
847                // bridge, and `bridge_available` says why nothing moves.
848                let waiting = self.pending.lock().await.get(&key).copied();
849                SyncStatus {
850                    enabled: waiting.is_some(),
851                    workspace_id: waiting,
852                    peers,
853                    paused: None,
854                    last_error: None,
855                    bridge_available,
856                    topology: proto::Topology::Mesh,
857                    embedding: None,
858                    pending_delivery: false,
859                    acknowledging_peers: 0,
860                }
861            }
862        }
863    }
864
865    /// How far `key`'s embedding has got, when this host embeds it. Reads the open
866    /// workspace's index; never opens one, so a status call stays cheap.
867    async fn embedding_progress(&self, key: &Path) -> Option<proto::EmbeddingProgress> {
868        let host = self.host.get()?;
869        let backend = host.open_backend(key)?;
870        let state = backend.state();
871        state.embedder()?;
872        let info = state.db_info().ok()?;
873        Some(proto::EmbeddingProgress {
874            vectors: info.vector_count,
875            pending: info.pending_count,
876            running: self.embed_running.lock().await.contains(key),
877        })
878    }
879
880    /// Bring the replica's view of the files up to date.
881    ///
882    /// Called straight after the app server's own writes, and by the watcher for everything
883    /// else. A scan that finds nothing is cheap; a scan that is skipped loses an edit until
884    /// the next one.
885    pub async fn scan(&self, root: &Path) -> Result<()> {
886        let Ok(key) = root.canonicalize() else {
887            return Ok(());
888        };
889        let replica = {
890            let synced = self.synced.lock().await;
891            match synced.get(&key) {
892                Some(entry) => Arc::clone(&entry.replica),
893                None => return Ok(()),
894            }
895        };
896        let (outcome, updates) = {
897            let mut replica = replica.lock().await;
898            let outcome = replica.scan().map_err(|e| Error::Sync(e.to_string()))?;
899            // What the scan committed, as path updates, so it can go out on the sessions
900            // that are already open. This is the whole point of the exact-path scan in
901            // `handlers.rs`: a write through the server reaches a peer live, not at the next
902            // dial.
903            let updates = match &outcome {
904                ScanOutcome::Scanned(report) => {
905                    // The unit of replication is the path state (design doc section 2.4):
906                    // the `seen` a peer prunes against is the *path's* version vector - the
907                    // context the recorded entry was built on plus its own dot - never the
908                    // whole replica vv. Sending the global vv here made receivers prune
909                    // causally-related versions they still held: a conflict copy on the
910                    // first remote edit, and tombstones pruned so deletes stopped
911                    // propagating (issue #157).
912                    report
913                        .recorded
914                        .iter()
915                        .map(|entry| PathUpdate {
916                            path: entry.path.clone(),
917                            versions: vec![entry.clone()],
918                            seen: {
919                                let mut seen = entry.context.clone();
920                                seen.add_dot(&entry.dot);
921                                seen
922                            },
923                        })
924                        .collect()
925                }
926                ScanOutcome::Paused(_) => Vec::new(),
927            };
928            (outcome, updates)
929        };
930        {
931            let mut synced = self.synced.lock().await;
932            if let Some(entry) = synced.get_mut(&key) {
933                entry.paused = match outcome {
934                    ScanOutcome::Paused(reason) => Some(reason),
935                    ScanOutcome::Scanned(_) => None,
936                };
937            }
938        }
939        // After the map is unlocked: a push awaits every peer, and the table must not be
940        // held behind a slow one.
941        self.after_commit(&key, updates).await;
942        Ok(())
943    }
944
945    /// Bring the search index up to date with files a session has written.
946    ///
947    /// A replication session moves files, not index rows: the index belongs to the app
948    /// server, and the sync core is deliberately free of it. So the side that *applied* a
949    /// change re-indexes, which is what makes a file that arrived from a peer searchable
950    /// without a manual reindex — the claim the whole architecture rests on.
951    ///
952    /// Incremental, not a full rebuild: the workspace's mtime/size stamps mean only what
953    /// actually changed is read back. A file the session wrote has a new stamp, so it is
954    /// picked up; one that was already indexed and untouched is skipped.
955    ///
956    /// Best-effort by contract: the caller's session has already succeeded, and a failure
957    /// here must not be reported as the session's. It is logged and the next session, local
958    /// edit or explicit `workspace.reindex` tries again.
959    async fn reindex(&self, root: &Path) {
960        // The marker's `config.toml` may be among what arrived.
961        self.refresh_settings();
962        let Some(host) = self.host.get() else {
963            return;
964        };
965        let host = Arc::clone(host);
966        let root = root.to_owned();
967        // One sweep at a time: two sessions that finish together would otherwise both walk
968        // the workspace through the same retrieve store.
969        let _serialised = self.reindexing.lock().await;
970        // Timed from here, so a run waiting for the one before it is not charged its wait.
971        let started = std::time::Instant::now();
972        // Opening the workspace is itself blocking work, and it is where the exclusive
973        // retrieve-store lock is taken, so it happens here rather than inside the sweep.
974        let backend = match host.backend_at(&root).await {
975            Ok(backend) => backend,
976            Err(err) => {
977                crate::instrument::reindexed(started, false);
978                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
979                return;
980            }
981        };
982        let state = Arc::clone(backend.state());
983        let result = tokio::task::spawn_blocking(move || state.sync_retrieve()).await;
984        crate::instrument::reindexed(started, matches!(result, Ok(Ok(_))));
985        match result {
986            Ok(Ok(report)) => {
987                tracing::debug!(root = %root.display(), ?report, "re-indexed after a session");
988            }
989            Ok(Err(err)) => {
990                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
991            }
992            Err(err) => {
993                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
994            }
995        }
996    }
997
998    /// Run an embedding pass over `root` in the background, after the pass already
999    /// waiting for it if there is one (#187).
1000    ///
1001    /// The pass reads the vector files that arrived, and computes the missing ones this
1002    /// host is responsible for: the files whose winning version this replica wrote, and —
1003    /// on the primary device — every other one. It never holds up the caller.
1004    pub fn request_embed(self: &Arc<Self>, root: &Path) {
1005        let Ok(key) = root.canonicalize() else {
1006            return;
1007        };
1008        let runtime = Arc::clone(self);
1009        self.spawn_background(async move {
1010            if !runtime.embed_wanted.lock().await.insert(key.clone()) {
1011                return;
1012            }
1013            let _one_at_a_time = runtime.embedding.lock().await;
1014            // Taken off the list as the pass starts: a change during the pass asks again.
1015            runtime.embed_wanted.lock().await.remove(&key);
1016            runtime.embed_running.lock().await.insert(key.clone());
1017            runtime.embed_pass(&key).await;
1018            runtime.embed_running.lock().await.remove(&key);
1019        });
1020    }
1021
1022    /// One embedding pass over `key`. Best-effort, like the re-index: logged, never fatal.
1023    async fn embed_pass(&self, key: &Path) {
1024        let Some(host) = self.host.get() else {
1025            return;
1026        };
1027        let (workspace_id, replica) = {
1028            let synced = self.synced.lock().await;
1029            match synced.get(key) {
1030                Some(entry) => (entry.workspace_id, Arc::clone(&entry.replica)),
1031                None => return,
1032            }
1033        };
1034        let backend = match host.backend_at(key).await {
1035            Ok(backend) => backend,
1036            Err(err) => {
1037                tracing::warn!(root = %key.display(), "embedding pass: {err}");
1038                return;
1039            }
1040        };
1041        let state = Arc::clone(backend.state());
1042        if let Err(err) = state
1043            .refresh_embedder_at(self.bridge_endpoint.clone())
1044            .await
1045        {
1046            tracing::warn!(root = %key.display(), "embedding pass: {err}");
1047            return;
1048        }
1049        if state.embedder().is_none() {
1050            return;
1051        }
1052        let primary = self.is_primary(workspace_id).await;
1053        // The replica's view, read once: which content each path has, and which of those
1054        // versions this replica wrote.
1055        let grace_ms = self.backfill_timing().grace.as_millis() as u64;
1056        let now_ms = std::time::SystemTime::now()
1057            .duration_since(std::time::UNIX_EPOCH)
1058            .map_or(0, |d| d.as_millis() as u64);
1059        let (mine, settled, live) = {
1060            let replica = replica.lock().await;
1061            let me = replica.replica_id();
1062            let states = match replica.states() {
1063                Ok(states) => states,
1064                Err(err) => {
1065                    tracing::warn!(root = %key.display(), "embedding pass: {err}");
1066                    return;
1067                }
1068            };
1069            let mut mine = HashMap::new();
1070            // Other devices' versions old enough for the primary to fill in.
1071            let mut settled = HashMap::new();
1072            let mut live = std::collections::HashSet::new();
1073            for (path, path_state) in states {
1074                let winner = path_state.winner();
1075                let Some(hash) = winner.content.hash() else {
1076                    continue;
1077                };
1078                let hash = hash.to_hex();
1079                if winner.dot.replica == me {
1080                    mine.insert(path, hash.clone());
1081                } else if now_ms.saturating_sub(winner.hlc.wall_ms) >= grace_ms {
1082                    settled.insert(path, hash.clone());
1083                }
1084                live.insert(hash);
1085            }
1086            (mine, settled, live)
1087        };
1088        let root = key.to_owned();
1089        let result = tokio::task::spawn_blocking(move || {
1090            // The author embeds its own versions at once; the primary device fills in other
1091            // devices' versions once they have settled, racing nobody.
1092            let policy = |rel: &str, hash: &str| {
1093                mine.get(rel).is_some_and(|h| h.as_str() == hash)
1094                    || (primary && settled.get(rel).is_some_and(|h| h.as_str() == hash))
1095            };
1096            let embedded = state.embed_pending_with(&policy, PASS_BATCH)?;
1097            let removed = if primary {
1098                state.remove_stale_vectors(&live, STALE_VECTOR_AGE)
1099            } else {
1100                0
1101            };
1102            Ok::<_, sapphire_workspace::Error>((embedded, removed))
1103        })
1104        .await;
1105        match result {
1106            Ok(Ok((embedded, removed))) if embedded > 0 || removed > 0 => tracing::info!(
1107                root = %root.display(),
1108                embedded,
1109                removed,
1110                "embedding pass"
1111            ),
1112            Ok(Ok(_)) => {}
1113            Ok(Err(err)) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
1114            Err(err) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
1115        }
1116    }
1117
1118    /// Open a session with every peer that will take one.
1119    pub async fn sync_now(self: &Arc<Self>, root: &Path) -> Result<()> {
1120        let Ok(key) = root.canonicalize() else {
1121            return Ok(());
1122        };
1123        let workspace_id = {
1124            let synced = self.synced.lock().await;
1125            match synced.get(&key) {
1126                Some(entry) => entry.workspace_id,
1127                None => return Ok(()),
1128            }
1129        };
1130        let peers = self.peers().await?;
1131        let me = self.device_id().await?;
1132
1133        // The same direction rule the dial loop follows: a host only dials peers with a
1134        // greater device id. Dialling the other way from here would reintroduce the
1135        // dial-both-ways deadlock the loop was cured of, just on a different trigger.
1136        // Inside a star, a pair with neither hub in it does not link at all.
1137        let roles = peers.roles_for(workspace_id).cloned();
1138        // Every peer at once, so one that is offline costs the others nothing (#228). On
1139        // this task, not spawned: cancelling the caller — the server's shutdown aborting the
1140        // watcher — must drop every dial with it, since each one pins the runtime.
1141        let mut dials: Vec<Dial<'_>> = Vec::new();
1142        for peer in peers
1143            .peers
1144            .iter()
1145            .filter(|p| topology::link(me, p.device_id, roles.as_ref()) == topology::Link::Dial)
1146            .cloned()
1147            .collect::<Vec<_>>()
1148        {
1149            // A peer that already has a live session is caught up by it: a second exchange
1150            // on the same workspace would be redundant, and a second stream between the same
1151            // pair is exactly what the tie-break cannot settle.
1152            if let Some(live) = self.live_peers(&key).await
1153                && live.has(&peer.device_id).await
1154            {
1155                continue;
1156            }
1157            // One dial per peer at a time. `sync_now` runs beside the dial loop, and two
1158            // overlapping outbound dials to one peer are the one case the tie-break rule
1159            // cannot resolve — each end would see its own stream as `outbound`. Whoever
1160            // arrives second simply leaves it to the next pass. The table is made here if
1161            // the dial loop has not yet: the first dial after enabling must not depend on
1162            // a tick that has not fired.
1163            let Some((_, table)) = self.live_peers_or_create(&key).await else {
1164                continue;
1165            };
1166            if !table.begin_dial(peer.device_id).await {
1167                continue;
1168            }
1169            let key = &key;
1170            dials.push(Box::pin(async move {
1171                // A peer that does not host this workspace refuses, which is normal and cheap.
1172                let result = self
1173                    .sync_one(key, &peer.device_id, peer.name.clone(), workspace_id)
1174                    .await;
1175                table.end_dial(peer.device_id).await;
1176                if let Err(err) = result {
1177                    tracing::debug!(peer = %peer.name, "no session: {err}");
1178                }
1179            }));
1180        }
1181        join_all(dials).await;
1182        Ok(())
1183    }
1184
1185    /// Dial one peer and adopt the stream as a live session.
1186    ///
1187    /// The one-shot path (`run_session`) would close the stream the moment the exchange
1188    /// finished — while the accepting peer keeps its half open and pushes into it, a session
1189    /// that dies with nothing reading it. So the dialled side is adopted live too, and both
1190    /// ends of a pair agree that a session stays open.
1191    async fn sync_one(
1192        self: &Arc<Self>,
1193        key: &Path,
1194        device: &GrainId,
1195        name: String,
1196        workspace_id: GrainId,
1197    ) -> Result<()> {
1198        let stream = self.open_stream(workspace_id, *device).await?;
1199        self.adopt_live_session(stream, key, *device, workspace_id)
1200            .await
1201            .map_err(|err| {
1202                tracing::warn!(peer = %name, "a live session could not be started: {err}");
1203                err
1204            })
1205    }
1206
1207    /// Ask the bridge for a stream to `device`, giving up after `open_timeout`.
1208    async fn open_stream(
1209        &self,
1210        workspace_id: GrainId,
1211        device: GrainId,
1212    ) -> Result<sapphire_ipc::RawStream> {
1213        match tokio::time::timeout(
1214            self.open_timeout,
1215            self.bridge()?.open_stream(workspace_id, device),
1216        )
1217        .await
1218        {
1219            Ok(opened) => Ok(opened?),
1220            Err(_) => Err(Error::Bridge(format!(
1221                "no stream to {device} within {} s; it may be offline",
1222                self.open_timeout.as_secs_f32()
1223            ))),
1224        }
1225    }
1226
1227    /// Answer the bridge's announcements, following the connection across reconnections
1228    /// (#244). Runs until the task is aborted.
1229    pub async fn run(self: Arc<Self>) -> Result<()> {
1230        let mut slot = self.bridge.subscribe();
1231        loop {
1232            let bridge = slot.borrow_and_update().clone();
1233            match bridge {
1234                Some(bridge) if !bridge.is_closed() => self.answer(&bridge, &mut slot).await,
1235                // Wait for a connection. The runtime holds the sender, so this never ends.
1236                _ => {
1237                    if slot.changed().await.is_err() {
1238                        return Ok(());
1239                    }
1240                }
1241            }
1242        }
1243    }
1244
1245    /// Answer `bridge`'s announcements until it closes or another connection replaces it.
1246    async fn answer(
1247        self: &Arc<Self>,
1248        bridge: &Arc<BridgeClient>,
1249        slot: &mut tokio::sync::watch::Receiver<Option<Arc<BridgeClient>>>,
1250    ) {
1251        let early = self
1252            .subscribed
1253            .lock()
1254            .expect("subscribed")
1255            .take_if(|(b, _)| Arc::ptr_eq(b, bridge));
1256        let mut incoming = match early {
1257            Some((_, incoming)) => incoming,
1258            None => bridge.incoming(),
1259        };
1260        loop {
1261            let received = tokio::select! {
1262                received = incoming.recv() => received,
1263                () = bridge.closed() => return,
1264                _ = slot.changed() => return,
1265            };
1266            let announcement = match received {
1267                Ok(a) => a,
1268                Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
1269                    tracing::warn!(missed = n, "fell behind on bridge announcements");
1270                    continue;
1271                }
1272                Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1273            };
1274
1275            let found = {
1276                let synced = self.synced.lock().await;
1277                synced
1278                    .iter()
1279                    .find(|(_, s)| s.workspace_id == announcement.workspace_id)
1280                    .map(|(root, s)| (root.clone(), Arc::clone(&s.replica)))
1281            };
1282            // `replica` is still what the accepted session shares, and `root` is where its
1283            // files and its index live.
1284            let Some((root, replica)) = found else {
1285                // The bridge routed to us for a workspace we no longer hold. Not fatal:
1286                // ignore it and let the ticket expire.
1287                tracing::debug!(
1288                    workspace = %announcement.workspace_id,
1289                    "an announcement for a workspace this server does not hold"
1290                );
1291                continue;
1292            };
1293
1294            let bridge = Arc::clone(bridge);
1295            let workspace_id = announcement.workspace_id;
1296            let peer = announcement.peer_device_id;
1297            let driver = Arc::clone(self);
1298            self.spawn_background(async move {
1299                let stream = match bridge.accept_stream(announcement.ticket).await {
1300                    Ok(stream) => stream,
1301                    Err(err) => {
1302                        tracing::warn!("could not claim an announced stream: {err}");
1303                        return;
1304                    }
1305                };
1306                if driver.should_skip(workspace_id, peer).await {
1307                    tracing::debug!(%peer, "dropped an inbound session: both ends are outside this workspace's star");
1308                    return;
1309                }
1310                // Live, not one-shot: the peer that dialled is keeping this session open and
1311                // pushing what it commits, so this side has to keep reading. A session that
1312                // closed after the exchange would leave the dialler pushing into a stream
1313                // nobody reads.
1314                let (_, session) = match sapphire_framework_session::open_live_session(
1315                    stream,
1316                    Arc::clone(&replica),
1317                    workspace_id,
1318                )
1319                .await
1320                {
1321                    Ok(started) => started,
1322                    Err(err) => {
1323                        crate::instrument::sync_session(false);
1324                        tracing::warn!("an inbound session failed: {err}");
1325                        return;
1326                    }
1327                };
1328                let Some((key, table)) = driver.live_peers_or_create(&root).await else {
1329                    return;
1330                };
1331                let acknowledged = session.acknowledged();
1332                // The peer dialled: this stream survives only if the *peer* has the lower
1333                // device id. A stream that loses is dropped, which closes it, and the
1334                // connection ends tidily rather than being left half-read.
1335                let Some(updates) = table.insert(peer, session, false).await else {
1336                    return;
1337                };
1338                driver.watch_delivery(key.clone(), peer, acknowledged);
1339                table.catch_up(peer, &replica).await;
1340                crate::instrument::sync_session(true);
1341                // This is the receiving side: the files are on disk now and nobody else will
1342                // index them.
1343                driver.reindex(&key).await;
1344                driver.request_embed(&key);
1345                driver.spawn_reader(table, updates, key, peer);
1346            });
1347        }
1348    }
1349
1350    /// Make the first attempt to reach the bridge, when the runtime has a
1351    /// [`BridgeConnector`] and is not connected yet.
1352    ///
1353    /// No bridge running is not an error: the server serves without sync, and
1354    /// [`keep_connected`](Self::keep_connected) picks the bridge up once it starts. Any
1355    /// other failure — a bridge of another API version, say — is, and stops the start.
1356    pub async fn connect(self: &Arc<Self>) -> Result<()> {
1357        let Some(connector) = &self.connector else {
1358            return Ok(());
1359        };
1360        if self.is_connected() {
1361            return Ok(());
1362        }
1363        match connector.connect().await? {
1364            Some(client) => self.attach(client).await,
1365            None => tracing::warn!(
1366                "no sapphire-bridge is running; serving without sync until it starts"
1367            ),
1368        }
1369        Ok(())
1370    }
1371
1372    /// Stay connected to the bridge: wait for the connection to close, then reach the
1373    /// bridge again with the runtime's [`BridgeConnector`], backing off from
1374    /// [`RECONNECT_BACKOFF`] to [`RECONNECT_BACKOFF_MAX`] while it cannot (#244).
1375    /// [`reconnect_now`](Self::reconnect_now) ends a wait early and starts the backoff over.
1376    ///
1377    /// Runs until the task is aborted, which is what the app server's shutdown does with
1378    /// it. Without a connector it only marks the runtime disconnected once the connection
1379    /// closes, and returns.
1380    pub async fn keep_connected(self: Arc<Self>) {
1381        let mut attempt = 0;
1382        // Whether this outage's failure has been logged at warn: once per outage, however
1383        // often `reconnect_now` starts the backoff over.
1384        let mut warned = false;
1385        loop {
1386            if let Ok(bridge) = self.bridge() {
1387                bridge.closed().await;
1388                tracing::warn!("lost the connection to sapphire-bridge; reconnecting");
1389                self.bridge.send_if_modified(|slot| {
1390                    let same = slot.as_ref().is_some_and(|b| Arc::ptr_eq(b, &bridge));
1391                    if same {
1392                        *slot = None;
1393                    }
1394                    same
1395                });
1396                attempt = 0;
1397                warned = false;
1398                continue;
1399            }
1400            let Some(connector) = &self.connector else {
1401                return;
1402            };
1403            match connector.connect().await {
1404                Ok(Some(client)) => {
1405                    tracing::info!("connected to sapphire-bridge");
1406                    self.attach(client).await;
1407                    attempt = 0;
1408                    warned = false;
1409                    continue;
1410                }
1411                Ok(None) => tracing::debug!("no sapphire-bridge is running yet"),
1412                // Logged once per outage, not once per attempt.
1413                Err(err) if !warned => {
1414                    tracing::warn!("could not connect to sapphire-bridge: {err}");
1415                    warned = true;
1416                }
1417                Err(err) => tracing::debug!("could not connect to sapphire-bridge: {err}"),
1418            }
1419            tokio::select! {
1420                () = tokio::time::sleep(reconnect_backoff(self.reconnect_first, attempt)) => {
1421                    attempt = attempt.saturating_add(1);
1422                }
1423                () = self.reconnect_now.notified() => attempt = 0,
1424            }
1425        }
1426    }
1427
1428    /// Start using `client`, and bring the bridge up to date: it knows nothing of this
1429    /// server yet, whether it has just started or this server has.
1430    async fn attach(self: &Arc<Self>, client: BridgeClient) {
1431        // Swapped under the pending lock, so an `enable` racing this either sees the new
1432        // connection or leaves its workspace for the drain below.
1433        let client = Arc::new(client);
1434        *self.subscribed.lock().expect("subscribed") =
1435            Some((Arc::clone(&client), client.incoming()));
1436        let waiting = {
1437            let mut pending = self.pending.lock().await;
1438            self.bridge.send_replace(Some(client));
1439            std::mem::take(&mut *pending)
1440        };
1441        // What was synced before the connection was lost.
1442        if !self.synced.lock().await.is_empty()
1443            && let Err(err) = self.reregister().await
1444        {
1445            tracing::warn!("could not register with sapphire-bridge: {err}");
1446        }
1447        // What was asked for while it was away. `enable` registers and dials each one.
1448        for root in waiting.into_keys() {
1449            if let Err(err) = self.enable(&root).await {
1450                tracing::warn!(root = %root.display(), "could not start sync: {err}");
1451            }
1452        }
1453    }
1454
1455    /// Every synced root, for the watcher.
1456    pub async fn roots(&self) -> Vec<PathBuf> {
1457        self.synced.lock().await.keys().cloned().collect()
1458    }
1459
1460    /// The live session table for `root`, if this runtime syncs it.
1461    ///
1462    /// `None` is not an error: a dial, a push or a drop for a workspace this server does not
1463    /// hold is a no-op, and every caller below treats it that way.
1464    async fn live_peers(&self, root: &Path) -> Option<Arc<LivePeers>> {
1465        let key = root.canonicalize().ok()?;
1466        self.live.lock().await.get(&key).cloned()
1467    }
1468
1469    /// The live session table for `root`, creating it if this runtime syncs it.
1470    ///
1471    /// The two locks are taken one after the other, never nested: `synced` to decide whether
1472    /// the workspace is ours at all, then `live` to get or make its table.
1473    async fn live_peers_or_create(&self, root: &Path) -> Option<(PathBuf, Arc<LivePeers>)> {
1474        let key = root.canonicalize().ok()?;
1475        if !self.synced.lock().await.contains_key(&key) {
1476            return None;
1477        }
1478        // A table is built by whoever walks the workspace first, and it needs this host's
1479        // device id — the tie-break between two simultaneous streams is decided from it.
1480        let me = self.device_id().await.ok()?;
1481        let mut live = self.live.lock().await;
1482        let peers = Arc::clone(
1483            live.entry(key.clone())
1484                .or_insert_with(|| Arc::new(LivePeers::new(me))),
1485        );
1486        Some((key, peers))
1487    }
1488
1489    /// Push `updates` to every open session for `root`.
1490    ///
1491    /// Called with what a scan recorded. `from` is always `None` here: a local commit came
1492    /// from this host, so no session is excluded. The forwarding path, which does name a
1493    /// source, is [`LivePeers::fan_out`](live::LivePeers::fan_out) called directly by the
1494    /// reader side of a session.
1495    ///
1496    /// Best-effort by contract: the commit already happened, and a peer that is gone is the
1497    /// dialer's problem, not the writer's.
1498    pub async fn after_commit(&self, root: &Path, updates: Vec<PathUpdate>) {
1499        if updates.is_empty() {
1500            return;
1501        }
1502        if let Some(peers) = self.live_peers(root).await {
1503            peers.fan_out(&updates, None).await;
1504        }
1505    }
1506
1507    /// The devices with an open live session for `root`.
1508    ///
1509    /// Used by tests to wait for the dialer; empty for a workspace this server does not hold.
1510    pub async fn live_session_devices(&self, root: &Path) -> Vec<GrainId> {
1511        match self.live_peers(root).await {
1512            Some(peers) => peers.devices().await,
1513            None => Vec::new(),
1514        }
1515    }
1516
1517    /// End the first dials [`enable`](SyncRuntime::enable) left running, waiting until
1518    /// they are gone.
1519    ///
1520    /// For the app server's shutdown: each one pins the runtime, and with it the replica
1521    /// stores, which are redb databases that stay locked — a dial outliving its server
1522    /// would lock the next one out.
1523    pub async fn stop_dials(&self) {
1524        let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1525        dials.abort_all();
1526        while dials.join_next().await.is_some() {}
1527    }
1528
1529    /// End the runtime's background work — inbound sessions, live-session readers and
1530    /// embedding passes — and wait until it has let go.
1531    ///
1532    /// The server's shutdown calls this, so a server restarted in the same process can open
1533    /// the replica stores again at once rather than whenever the last straggler finished
1534    /// (#235). An embedding pass already inside its blocking section runs that section to
1535    /// the end; it holds the workspace's index, never a replica.
1536    pub async fn stop_background(&self) {
1537        let mut background =
1538            std::mem::take(&mut *self.background.lock().expect("background tasks"));
1539        background.abort_all();
1540        while background.join_next().await.is_some() {}
1541    }
1542
1543    /// Wait for the first dials [`enable`](SyncRuntime::enable) left running to finish on
1544    /// their own: for a test that must not race them.
1545    #[cfg(test)]
1546    async fn settle_dials(&self) {
1547        let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1548        while dials.join_next().await.is_some() {}
1549    }
1550
1551    /// Close every open live session, as if the connections had been cut.
1552    ///
1553    /// Called for a workspace being disabled, and by tests that want to watch the dialer
1554    /// rebuild what was dropped.
1555    pub async fn drop_connections(&self) {
1556        let tables: Vec<Arc<LivePeers>> = self.live.lock().await.values().cloned().collect();
1557        for peers in tables {
1558            peers.drop_connections().await;
1559        }
1560    }
1561
1562    /// Keep a live session open to every peer, for every synced workspace.
1563    ///
1564    /// One walk per [`DIAL_INTERVAL`](live::DIAL_INTERVAL): for each synced workspace, ask
1565    /// the bridge who is in the workgroup, and open a session to every peer that is connected
1566    /// and does not already have one.
1567    ///
1568    /// A peer that cannot be reached is retried on a per-peer exponential backoff capped at
1569    /// [`DIAL_BACKOFF_MAX`] — one failed attempt per walk, not one per spin — and the counter
1570    /// resets when a session opens. An unreachable peer is therefore no obstacle to the
1571    /// others: its dial fails, the walk moves on, and the next workspace is dialled.
1572    ///
1573    /// Runs until the task is aborted, which is what the app server's shutdown does with it.
1574    /// It is allowed to fail like [`run`](SyncRuntime::run): only sync stops, and the app
1575    /// server keeps serving files (spec §10).
1576    pub async fn dial_loop(self: Arc<Self>) -> Result<()> {
1577        let mut tick = tokio::time::interval(live::DIAL_INTERVAL);
1578        // A missed tick is a late walk, not a burst of them: the interval is a pace, not a
1579        // quota to make up.
1580        tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1581        let mut waiting: HashMap<GrainId, std::time::Instant> = HashMap::new();
1582        let mut failures: HashMap<GrainId, u32> = HashMap::new();
1583        // The workspaces this host was the primary device of at the last walk.
1584        let mut primary_of: std::collections::HashSet<GrainId> = Default::default();
1585        // When the primary device last ran a backfill pass with nothing prompting it.
1586        let mut last_backfill = std::time::Instant::now();
1587        loop {
1588            tick.tick().await;
1589            let roots = self.roots().await;
1590            if roots.is_empty() {
1591                continue;
1592            }
1593            let peers = match self.peers().await {
1594                Ok(peers) => peers,
1595                // The bridge is down or has no workgroup: nothing to dial, and the next walk
1596                // asks again. Not an error for the caller — the app server is still serving.
1597                Err(err) => {
1598                    tracing::debug!("no peers to dial: {err}");
1599                    continue;
1600                }
1601            };
1602            let me = match self.device_id().await {
1603                Ok(me) => me,
1604                Err(err) => {
1605                    tracing::warn!("no device id, so nobody to dial: {err}");
1606                    continue;
1607                }
1608            };
1609            // A host that has just become a workspace's primary device owes it the vectors
1610            // nobody else computes (#187): files that arrived before the election settled
1611            // would otherwise wait for the next change.
1612            let ids: Vec<(PathBuf, GrainId)> = self
1613                .synced
1614                .lock()
1615                .await
1616                .iter()
1617                .map(|(root, entry)| (root.clone(), entry.workspace_id))
1618                .collect();
1619            // And every so often anyway: an author that never embedded leaves files only the
1620            // primary will fill in, and nothing else would prompt it (#188).
1621            let backfill_due = last_backfill.elapsed() >= self.backfill_timing().interval;
1622            if backfill_due {
1623                last_backfill = std::time::Instant::now();
1624            }
1625            for (root, workspace_id) in ids {
1626                let primary = peers
1627                    .roles_for(workspace_id)
1628                    .is_some_and(|r| r.primary == Some(me));
1629                if !primary {
1630                    primary_of.remove(&workspace_id);
1631                } else if primary_of.insert(workspace_id) || backfill_due {
1632                    self.request_embed(&root);
1633                }
1634            }
1635            let now = std::time::Instant::now();
1636            // Dial only peers whose device id is greater than ours. Every device dialling
1637            // every other can deadlock: two hosts that dial each other at once each hold
1638            // their replica's lock across their own exchange and wait for the other's
1639            // hello, which the other's dialler holds the lock for. With one direction per
1640            // pair the wait chain is a DAG — a higher-id host never dials, so it never
1641            // holds its lock out while a lower-id host's exchange needs it. The lower host
1642            // dials within `DIAL_INTERVAL`, and the exchange is symmetric, so nothing is
1643            // lost; the tie-break is left as the safety net it is.
1644            for root in roots {
1645                let Some((key, table)) = self.live_peers_or_create(&root).await else {
1646                    continue;
1647                };
1648                let workspace_id = {
1649                    let synced = self.synced.lock().await;
1650                    match synced.get(&key) {
1651                        Some(entry) => entry.workspace_id,
1652                        None => continue,
1653                    }
1654                };
1655                let roles = peers.roles_for(workspace_id);
1656                // Entering a star closes what it no longer needs. The hubs carry it all —
1657                // once one of them is reachable from here (#192).
1658                if topology::may_close_skipped(me, roles, &table.devices().await) {
1659                    table
1660                        .retain(|d| topology::link(me, *d, roles) != topology::Link::Skip)
1661                        .await;
1662                }
1663                for peer in peers.peers.iter().filter(|p| {
1664                    p.connected && topology::link(me, p.device_id, roles) == topology::Link::Dial
1665                }) {
1666                    let device = peer.device_id;
1667                    if table.has(&device).await {
1668                        // Already talking; a redial would drop a working session.
1669                        failures.remove(&device);
1670                        waiting.remove(&device);
1671                        continue;
1672                    }
1673                    if let Some(until) = waiting.get(&device)
1674                        && now < *until
1675                    {
1676                        // Backing off after a failure; the walk skips it until then.
1677                        continue;
1678                    }
1679                    // The same claim `sync_now` takes: this walk and it can overlap, and a
1680                    // double dial in one direction is the one race the tie-break cannot
1681                    // settle. Skipping is free — the next pass, or `sync_now` itself, dials.
1682                    if !table.begin_dial(device).await {
1683                        continue;
1684                    }
1685                    let opened = self.open_stream(workspace_id, device).await;
1686                    if opened.is_err() {
1687                        table.end_dial(device).await;
1688                    }
1689                    match opened {
1690                        Ok(stream) => {
1691                            let adopted = self
1692                                .adopt_live_session(stream, &root, device, workspace_id)
1693                                .await;
1694                            // The claim is out either way: the attempt is over, and a
1695                            // session that died mid-exchange is re-dialled next pass.
1696                            table.end_dial(device).await;
1697                            match adopted {
1698                                Ok(()) => {
1699                                    failures.remove(&device);
1700                                    waiting.remove(&device);
1701                                }
1702                                Err(err) => {
1703                                    tracing::debug!(
1704                                        peer = %peer.name,
1705                                        err = %err,
1706                                        "a live session could not be started"
1707                                    );
1708                                    let count = failures.entry(device).or_insert(0);
1709                                    *count = count.saturating_add(1);
1710                                    waiting.insert(device, now + dial_backoff(*count));
1711                                }
1712                            }
1713                        }
1714                        Err(err) => {
1715                            // Normal and cheap: a peer that does not host this workspace
1716                            // refuses, and one that is unreachable cannot be opened.
1717                            tracing::debug!("no stream to a peer: {err}");
1718                            let count = failures.entry(device).or_insert(0);
1719                            *count = count.saturating_add(1);
1720                            waiting.insert(device, now + dial_backoff(*count));
1721                        }
1722                    }
1723                }
1724            }
1725        }
1726    }
1727
1728    /// Take ownership of an opened stream: run the exchange, then keep it live.
1729    ///
1730    /// The session is stored under `device` and its reader pumped until the stream ends. A
1731    /// half-open or refused stream is not stored; the dialer retries it with backoff.
1732    async fn adopt_live_session(
1733        self: &Arc<Self>,
1734        stream: sapphire_ipc::RawStream,
1735        root: &Path,
1736        device: GrainId,
1737        workspace_id: GrainId,
1738    ) -> Result<()> {
1739        let replica = self.replica_of(root).await?;
1740        let (_, session) = sapphire_framework_session::open_live_session(
1741            stream,
1742            Arc::clone(&replica),
1743            workspace_id,
1744        )
1745        .await
1746        .map_err(|e| {
1747            crate::instrument::sync_session(false);
1748            Error::Sync(e.to_string())
1749        })?;
1750        let Some((key, table)) = self.live_peers_or_create(root).await else {
1751            return Ok(());
1752        };
1753        let acknowledged = session.acknowledged();
1754        // This stream was dialled by us, so it is the one the pair keeps if this host's
1755        // device id is the lower of the two. If it lost the tie-break the session is
1756        // dropped here, which closes the stream, and the surviving one is already in the
1757        // table.
1758        let Some(updates) = table.insert(device, session, true).await else {
1759            return Ok(());
1760        };
1761        self.watch_delivery(key.clone(), device, acknowledged);
1762        table.catch_up(device, &replica).await;
1763        crate::instrument::sync_session(true);
1764        // The session wrote files during the exchange; the index has to catch up before the
1765        // workspace is searchable again.
1766        self.reindex(&key).await;
1767        self.request_embed(&key);
1768        self.spawn_reader(Arc::clone(&table), updates, key, device);
1769        Ok(())
1770    }
1771
1772    /// Forward what one session receives, and re-index it.
1773    ///
1774    /// This is the loop that makes A → S → B work: a batch that arrived on S's session with A
1775    /// is applied (the session crate does that before publishing it), the workspace behind
1776    /// `key` is re-indexed, and the batch is handed to every *other* session whose peer lacks
1777    /// it — `from` is the device it came from, so it never goes back to A.
1778    fn spawn_reader(
1779        self: &Arc<Self>,
1780        table: Arc<LivePeers>,
1781        mut updates: broadcast::Receiver<Vec<PathUpdate>>,
1782        key: PathBuf,
1783        from: GrainId,
1784    ) {
1785        let driver = Arc::clone(self);
1786        self.spawn_background(async move {
1787            loop {
1788                match updates.recv().await {
1789                    Ok(batch) => {
1790                        // The files are on disk now (the session guarantees it before
1791                        // publishing), so the index is behind them until the sweep below.
1792                        driver.reindex(&key).await;
1793                        driver.request_embed(&key);
1794                        table.fan_out(&batch, Some(from)).await;
1795                    }
1796                    // A burst bigger than the channel: the newest is kept and the gap is
1797                    // covered by the next session's exchange. Losing a batch here cannot lose
1798                    // an entry — the peer's vector does not cover it, so the next dial or push
1799                    // carries it again.
1800                    Err(broadcast::error::RecvError::Lagged(missed)) => {
1801                        tracing::debug!(
1802                            device = %from,
1803                            missed,
1804                            "fell behind on a live session; the next exchange catches up"
1805                        );
1806                    }
1807                    // The session closed. Its table entry is reaped by the next fan-out or
1808                    // dial pass, and the dialer opens a fresh one.
1809                    Err(broadcast::error::RecvError::Closed) => {
1810                        return;
1811                    }
1812                }
1813            }
1814        });
1815    }
1816
1817    /// Follow what `device`'s session for `key` is known to have, and announce a delivery
1818    /// when it comes to cover every edit this device made.
1819    ///
1820    /// Checked at once too: the initial exchange may already have carried everything. Ends
1821    /// with the session, which closes the watch.
1822    fn watch_delivery(
1823        self: &Arc<Self>,
1824        key: PathBuf,
1825        device: GrainId,
1826        mut acknowledged: tokio::sync::watch::Receiver<sapphire_sync::VersionVector>,
1827    ) {
1828        let driver = Arc::clone(self);
1829        self.spawn_background(async move {
1830            loop {
1831                let known = acknowledged.borrow_and_update().clone();
1832                driver.check_delivered(&key, device, &known).await;
1833                if acknowledged.changed().await.is_err() {
1834                    return;
1835                }
1836            }
1837        });
1838    }
1839
1840    /// Announce [`BackendEvent::SyncDelivered`] for `key` if `known` — what `device` is known to have —
1841    /// covers every edit this device has made and that was not announced yet.
1842    async fn check_delivered(
1843        &self,
1844        key: &Path,
1845        device: GrainId,
1846        known: &sapphire_sync::VersionVector,
1847    ) {
1848        let Some(replica) = self
1849            .synced
1850            .lock()
1851            .await
1852            .get(key)
1853            .map(|entry| Arc::clone(&entry.replica))
1854        else {
1855            return;
1856        };
1857        let (me, own) = {
1858            let replica = replica.lock().await;
1859            (replica.replica_id(), own_counter(&replica))
1860        };
1861        if known.get(&me) < own {
1862            return;
1863        }
1864        {
1865            let mut synced = self.synced.lock().await;
1866            let Some(entry) = synced.get_mut(key) else {
1867                return;
1868            };
1869            // Already announced, by this peer or another: a second peer catching up says
1870            // nothing new.
1871            if own <= entry.delivered_through {
1872                return;
1873            }
1874            entry.delivered_through = own;
1875        }
1876        let device_name = self
1877            .last_peers
1878            .lock()
1879            .await
1880            .as_ref()
1881            .and_then(|answer| answer.peers.iter().find(|p| p.device_id == device))
1882            .map_or_else(|| device.to_string(), |peer| peer.name.clone());
1883        tracing::info!(root = %key.display(), peer = %device_name, "local edits delivered");
1884        if let Some(host) = self.host.get() {
1885            host.emit(
1886                key,
1887                BackendEvent::SyncDelivered {
1888                    device_id: device,
1889                    device_name,
1890                },
1891            );
1892        }
1893    }
1894
1895    /// The replica [`disable`](Self::disable) let go of for `key`, if something still holds it
1896    /// open.
1897    fn reclaim(&self, key: &Path) -> Reclaimed {
1898        let stopped = self.stopped.lock().expect("stopped replicas").remove(key);
1899        match stopped {
1900            None => Reclaimed::Closed,
1901            Some(replica) => replica
1902                .upgrade()
1903                .map_or(Reclaimed::Closing, Reclaimed::Open),
1904        }
1905    }
1906
1907    /// The replica of a synced `root`.
1908    async fn replica_of(&self, root: &Path) -> Result<Arc<Mutex<Replica>>> {
1909        let key = root.canonicalize().map_err(Error::Io)?;
1910        let synced = self.synced.lock().await;
1911        match synced.get(&key) {
1912            Some(entry) => Ok(Arc::clone(&entry.replica)),
1913            None => Err(Error::UnknownWorkspace(key, self.ctx.app_name)),
1914        }
1915    }
1916
1917    /// Watch the synced roots and, on a debounced report, scan and dial.
1918    ///
1919    /// This is the safety net for everything the app server did not do itself — a file
1920    /// edited in an editor, a `git checkout`. The exact path is [`scan`] called from
1921    /// `handlers.rs` right after a write the server made. A scan that finds nothing is
1922    /// cheap; a scan that is skipped loses an edit until the next one.
1923    ///
1924    /// Returns when the reporting channel closes — that is, when the [`Watcher`] this
1925    /// runtime holds is dropped, which happens when the runtime itself is.
1926    ///
1927    /// [`scan`]: SyncRuntime::scan
1928    pub async fn watch_changes(self: Arc<Self>) -> Result<()> {
1929        let (tx, mut rx) = mpsc::channel(64);
1930        let watcher = Arc::new(Watcher::start(self.roots().await, tx)?);
1931        let _ = self.watcher.set(Arc::clone(&watcher));
1932
1933        // A root enabled between `roots()` above and the `set` just now was not in the
1934        // starting set, and `enable` could not reach a watcher that did not exist yet.
1935        // Watching the current set again is idempotent on `notify`'s side.
1936        for root in self.roots().await {
1937            if let Err(err) = watcher.watch(&root) {
1938                tracing::warn!(root = %root.display(), "could not watch: {err}");
1939            }
1940        }
1941
1942        while let Some(root) = rx.recv().await {
1943            // An edit of the marker's `config.toml` changes the workspace's settings.
1944            self.refresh_settings();
1945            // Logged, not fatal: a failed scan or dial leaves the replica behind, and the
1946            // next report tries again. Neither may take the app server down (spec §10).
1947            if let Err(err) = self.scan(&root).await {
1948                tracing::warn!(root = %root.display(), "scan after a local edit failed: {err}");
1949            }
1950            self.request_embed(&root);
1951            if let Err(err) = self.sync_now(&root).await {
1952                tracing::warn!(root = %root.display(), "dial after a local edit failed: {err}");
1953            }
1954        }
1955        Ok(())
1956    }
1957
1958    /// Tell the bridge the complete current set. A registration is not a delta.
1959    async fn reregister(&self) -> Result<()> {
1960        let workspaces: Vec<WorkspaceRegistration> = {
1961            let synced = self.synced.lock().await;
1962            synced
1963                .iter()
1964                .map(|(root, entry)| WorkspaceRegistration {
1965                    workspace_id: entry.workspace_id,
1966                    root: root.clone(),
1967                })
1968                .collect()
1969        };
1970        self.bridge()?
1971            .register(RegisterParams {
1972                app_name: self.ctx.app_name.to_owned(),
1973                exe_path: self.exe_path.clone(),
1974                managed_by: wire_managed_by(self.managed_by),
1975                workspaces,
1976            })
1977            .await
1978            .map_err(|e| Error::Bridge(e.to_string()))?;
1979        Ok(())
1980    }
1981
1982    /// Whether the last roles the bridge reported say this pair does not link.
1983    ///
1984    /// Reads the cached device id only: asking the bridge here would register with an empty
1985    /// workspace list, racing `enable`'s registration. `false` (accept) when it is unknown.
1986    async fn should_skip(&self, workspace_id: GrainId, peer: GrainId) -> bool {
1987        let Some(&me) = self.device_id.get() else {
1988            return false;
1989        };
1990        let cached = self.last_peers.lock().await;
1991        let roles = cached.as_ref().and_then(|p| p.roles_for(workspace_id));
1992        topology::link(me, peer, roles) == topology::Link::Skip
1993    }
1994
1995    /// Whether this host is `workspace_id`'s primary device, by the last roles the
1996    /// bridge reported. `false` when the bridge has not answered yet, or when this host's
1997    /// device id is not known yet (no workspace was enabled). Reads the cached id only:
1998    /// asking the bridge here would register with an empty workspace list, racing
1999    /// `enable`'s registration.
2000    pub async fn is_primary(&self, workspace_id: GrainId) -> bool {
2001        let Some(&me) = self.device_id.get() else {
2002            return false;
2003        };
2004        let cached = self.last_peers.lock().await;
2005        cached
2006            .as_ref()
2007            .and_then(|p| p.roles_for(workspace_id))
2008            .is_some_and(|r| r.primary == Some(me))
2009    }
2010
2011    /// This host's device id, once the bridge has told it (#250).
2012    ///
2013    /// `None` until this runtime first registered — no workspace was enabled yet, or the
2014    /// bridge was not connected when one was. Never asks the bridge: a registration made
2015    /// only to learn the id would carry an empty workspace list, and briefly clear this
2016    /// application's routes.
2017    pub fn this_device(&self) -> Option<GrainId> {
2018        self.device_id.get().copied()
2019    }
2020
2021    /// The workgroup's devices, this host's included, and the roles the bridge assigns per
2022    /// workspace (#250). Asked of the bridge each time; [`is_primary`](Self::is_primary)
2023    /// reads the answer afterwards.
2024    ///
2025    /// [`Error::BridgeDisconnected`] while there is no connection to the bridge.
2026    pub async fn peers(&self) -> Result<PeersResult> {
2027        let peers = self
2028            .bridge()?
2029            .peers()
2030            .await
2031            .map_err(|e| Error::Bridge(e.to_string()))?;
2032        *self.last_peers.lock().await = Some(peers.clone());
2033        Ok(peers)
2034    }
2035
2036    /// This host's device id, asked of the bridge once.
2037    async fn device_id(&self) -> Result<GrainId> {
2038        if let Some(id) = self.device_id.get() {
2039            return Ok(*id);
2040        }
2041        let result = self
2042            .bridge()?
2043            .register(RegisterParams {
2044                app_name: self.ctx.app_name.to_owned(),
2045                exe_path: self.exe_path.clone(),
2046                managed_by: wire_managed_by(self.managed_by),
2047                workspaces: Vec::new(),
2048            })
2049            .await
2050            .map_err(|e| Error::Bridge(e.to_string()))?;
2051        let _ = self.device_id.set(result.device_id);
2052        Ok(result.device_id)
2053    }
2054
2055    /// `<workspace cache dir>/sync/`.
2056    fn state_dir(&self, root: &Path) -> Result<PathBuf> {
2057        let workspace = Workspace::from_root(self.ctx, root)?;
2058        Ok(workspace.cache_dir().join("sync"))
2059    }
2060}
2061
2062/// One dial of [`SyncRuntime::sync_now`].
2063type Dial<'a> = std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>>;
2064
2065/// Run `dials` concurrently on the calling task, until every one has finished.
2066/// The counter of the last edit `replica` recorded itself.
2067fn own_counter(replica: &Replica) -> u64 {
2068    replica.vv().get(&replica.replica_id())
2069}
2070
2071async fn join_all(mut dials: Vec<Dial<'_>>) {
2072    std::future::poll_fn(|cx| {
2073        dials.retain_mut(|dial| dial.as_mut().poll(cx).is_pending());
2074        if dials.is_empty() {
2075            std::task::Poll::Ready(())
2076        } else {
2077            std::task::Poll::Pending
2078        }
2079    })
2080    .await
2081}
2082
2083/// The bridge API's copy of how this server was started, for [`RegisterParams`].
2084fn wire_managed_by(managed_by: ManagedBy) -> sapphire_bridge_api::ManagedBy {
2085    match managed_by {
2086        ManagedBy::Service => sapphire_bridge_api::ManagedBy::Service,
2087        ManagedBy::Spawned => sapphire_bridge_api::ManagedBy::Spawned,
2088    }
2089}
2090
2091#[cfg(test)]
2092mod tests {
2093    use super::*;
2094    use crate::sync::testing::StubBridge;
2095    use crate::test_support;
2096    use sapphire_workspace::AppKind;
2097    use std::ffi::OsString;
2098
2099    static CTX: AppContext = AppContext::new("sapphire-synctest");
2100
2101    /// The env vars `CTX.init` reads.
2102    const DIR_VARS: [&str; 3] = [
2103        "SAPPHIRE_SYNCTEST_CACHE_DIR",
2104        "SAPPHIRE_SYNCTEST_DATA_DIR",
2105        "SAPPHIRE_SYNCTEST_CONFIG_DIR",
2106    ];
2107
2108    /// Points the context's directories at the test's scratch tree, and restores the
2109    /// previous values when dropped — including while unwinding from a panic.
2110    struct EnvGuard {
2111        previous: [Option<OsString>; 3],
2112        _lock: std::sync::MutexGuard<'static, ()>,
2113    }
2114
2115    impl Drop for EnvGuard {
2116        fn drop(&mut self) {
2117            // SAFETY: `self._lock` still serialises the environment; it is dropped only
2118            // after this method returns.
2119            for (name, previous) in DIR_VARS.iter().zip(self.previous.iter_mut()) {
2120                match previous.take() {
2121                    Some(value) => unsafe { std::env::set_var(name, value) },
2122                    None => test_support::remove(name),
2123                }
2124            }
2125        }
2126    }
2127
2128    /// Everything one test needs, held for the test's whole body.
2129    ///
2130    /// The context is a `static` shared by this binary, so its directories are resolved from
2131    /// the environment by whichever test initialises it first. The environment lock is
2132    /// therefore held for the whole test, as in the crate's other test modules, and `_tmp`
2133    /// is declared before `_env` so the scratch tree is gone only once the environment no
2134    /// longer points at it.
2135    struct Fixture {
2136        _tmp: tempfile::TempDir,
2137        _env: EnvGuard,
2138        root: PathBuf,
2139        stub: StubBridge,
2140        runtime: Arc<SyncRuntime>,
2141    }
2142
2143    impl Fixture {
2144        /// A second workspace root beside the first, marker directory and all.
2145        fn second_root(&self) -> PathBuf {
2146            let root = self._tmp.path().join("ws2");
2147            std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
2148            root.canonicalize().unwrap()
2149        }
2150    }
2151
2152    /// How long the fixture's runtime waits on a stream to a peer.
2153    const FIXTURE_OPEN_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
2154
2155    /// `n` peers this host dials (a greater device id), as `bridge.peers` lists them.
2156    fn dialled_peers(me: GrainId, n: usize) -> Vec<sapphire_bridge_api::PeerInfo> {
2157        std::iter::repeat_with(GrainId::random)
2158            .filter(|id| *id > me)
2159            .take(n)
2160            .enumerate()
2161            .map(|(i, device_id)| sapphire_bridge_api::PeerInfo {
2162                device_id,
2163                name: format!("offline-{i}"),
2164                node_id: format!("node-{i}"),
2165                connected: false,
2166                priority: 1,
2167                availability: None,
2168            })
2169            .collect()
2170    }
2171
2172    #[tokio::test(flavor = "multi_thread")]
2173    async fn enabling_answers_at_once_with_a_peer_that_never_answers() {
2174        // #228: the first dial waited on a peer that was offline, past the client's limit.
2175        let f = fixture().await;
2176        let _hung = f.stub.hang_streams().await;
2177        f.stub.set_peers(dialled_peers(f.stub.device_id, 1));
2178
2179        let started = std::time::Instant::now();
2180        f.runtime.enable(&f.root).await.unwrap();
2181        assert!(
2182            started.elapsed() < FIXTURE_OPEN_TIMEOUT,
2183            "enable waited {:?} on the dial",
2184            started.elapsed()
2185        );
2186    }
2187
2188    #[tokio::test(flavor = "multi_thread")]
2189    async fn peers_that_never_answer_are_dialled_together_and_given_up_on() {
2190        let f = fixture().await;
2191        let _hung = f.stub.hang_streams().await;
2192        f.runtime.enable(&f.root).await.unwrap();
2193        // The first dial, with nobody to dial yet, must be over before the peers appear:
2194        // one still running would claim them, and `sync_now` would skip them all.
2195        f.runtime.settle_dials().await;
2196        f.stub.set_peers(dialled_peers(f.stub.device_id, 3));
2197
2198        let started = std::time::Instant::now();
2199        f.runtime.sync_now(&f.root).await.unwrap();
2200        let took = started.elapsed();
2201        assert!(took >= FIXTURE_OPEN_TIMEOUT, "gave up early: {took:?}");
2202        assert!(
2203            took < FIXTURE_OPEN_TIMEOUT * 2,
2204            "three dials one after another: {took:?}"
2205        );
2206    }
2207
2208    async fn fixture() -> Fixture {
2209        let (tmp, env, root) = scratch();
2210        let (stub, client) = StubBridge::start_in(tmp.path().join("run")).await;
2211        let runtime = Arc::new(
2212            SyncRuntime::new(&CTX, client, "/bin/true".into(), ManagedBy::Service)
2213                .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
2214        );
2215        Fixture {
2216            _tmp: tmp,
2217            _env: env,
2218            root,
2219            stub,
2220            runtime,
2221        }
2222    }
2223
2224    /// The bridge connections a [`connecting`] fixture's connector hands out, in order;
2225    /// empty is "no bridge is running".
2226    type Bridges = Arc<std::sync::Mutex<std::collections::VecDeque<BridgeClient>>>;
2227
2228    /// A fixture whose runtime reaches the bridge through a connector, and has not yet.
2229    /// `f.stub`'s client is not handed out: push it to the returned queue for that.
2230    async fn connecting() -> (Fixture, Bridges, BridgeClient) {
2231        let (tmp, env, root) = scratch();
2232        let (stub, client) = StubBridge::start_client_in(tmp.path().join("run")).await;
2233        let bridges: Bridges = Arc::default();
2234        let connector = BridgeConnector::new({
2235            let bridges = Arc::clone(&bridges);
2236            move || {
2237                let next = bridges.lock().unwrap().pop_front();
2238                async move { Ok(next) }
2239            }
2240        });
2241        let runtime = Arc::new(
2242            SyncRuntime::connecting(&CTX, connector, "/bin/true".into(), ManagedBy::Service)
2243                .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
2244        );
2245        let f = Fixture {
2246            _tmp: tmp,
2247            _env: env,
2248            root,
2249            stub,
2250            runtime,
2251        };
2252        (f, bridges, client)
2253    }
2254
2255    /// Wait up to ten seconds for `done`.
2256    async fn eventually(what: &str, mut done: impl AsyncFnMut() -> bool) {
2257        let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
2258        while !done().await {
2259            assert!(tokio::time::Instant::now() < deadline, "never: {what}");
2260            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
2261        }
2262    }
2263
2264    #[tokio::test(flavor = "multi_thread")]
2265    async fn a_runtime_started_without_a_bridge_syncs_once_one_starts() {
2266        // #244: a server that started before its bridge never synced until restarted.
2267        let (f, bridges, client) = connecting().await;
2268        f.runtime.connect().await.unwrap();
2269        assert!(!f.runtime.is_connected());
2270
2271        let err = f.runtime.enable(&f.root).await.unwrap_err();
2272        assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2273        let status = f.runtime.status(&f.root).await;
2274        assert!(status.enabled, "enabled, waiting for the bridge");
2275        assert!(!status.bridge_available);
2276        let id = status.workspace_id.expect("its id");
2277
2278        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2279        bridges.lock().unwrap().push_back(client);
2280        eventually("the waiting workspace is registered", async || {
2281            f.stub.last_workspaces() == vec![id]
2282        })
2283        .await;
2284        assert_eq!(f.runtime.roots().await, vec![f.root.clone()]);
2285        assert!(f.runtime.status(&f.root).await.bridge_available);
2286        keeping.abort();
2287    }
2288
2289    #[tokio::test(flavor = "multi_thread")]
2290    async fn a_runtime_that_loses_its_bridge_registers_with_the_next_one() {
2291        let (f, bridges, client) = connecting().await;
2292        bridges.lock().unwrap().push_back(client);
2293        f.runtime.connect().await.unwrap();
2294        assert!(f.runtime.is_connected());
2295        let id = f.runtime.enable(&f.root).await.unwrap();
2296        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2297
2298        f.stub.disconnect();
2299        eventually("the runtime notices", async || !f.runtime.is_connected()).await;
2300        let err = f.runtime.workspaces().await.unwrap_err();
2301        assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2302
2303        let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2304        bridges.lock().unwrap().push_back(client);
2305        eventually("the synced workspace is registered again", async || {
2306            next.last_workspaces() == vec![id]
2307        })
2308        .await;
2309        assert!(f.runtime.is_connected());
2310        keeping.abort();
2311    }
2312
2313    #[tokio::test(flavor = "multi_thread")]
2314    async fn this_device_is_known_once_a_workspace_is_enabled_and_never_asked_for() {
2315        let f = fixture().await;
2316        assert_eq!(f.runtime.this_device(), None);
2317        assert!(
2318            f.stub.seen.lock().unwrap().registrations.is_empty(),
2319            "asking must not register"
2320        );
2321
2322        f.runtime.enable(&f.root).await.unwrap();
2323        assert_eq!(f.runtime.this_device(), Some(f.stub.device_id));
2324    }
2325
2326    #[tokio::test(flavor = "multi_thread")]
2327    async fn peers_answers_the_workgroup_from_the_bridge() {
2328        let f = fixture().await;
2329        let listed = dialled_peers(f.stub.device_id, 2);
2330        f.stub.set_peers(listed.clone());
2331
2332        let peers = f.runtime.peers().await.unwrap();
2333        let ids: Vec<_> = peers.peers.iter().map(|p| p.device_id).collect();
2334        assert_eq!(ids, listed.iter().map(|p| p.device_id).collect::<Vec<_>>());
2335    }
2336
2337    #[tokio::test(flavor = "multi_thread")]
2338    async fn peers_says_so_while_the_bridge_is_away() {
2339        let (f, _bridges, _client) = connecting().await;
2340        let err = f.runtime.peers().await.unwrap_err();
2341        assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2342    }
2343
2344    #[tokio::test(flavor = "multi_thread")]
2345    async fn disabling_while_the_bridge_is_away_forgets_the_waiting_workspace() {
2346        let (f, bridges, client) = connecting().await;
2347        f.runtime.enable(&f.root).await.unwrap_err();
2348        f.runtime.disable(&f.root).await.unwrap();
2349        assert!(!f.runtime.status(&f.root).await.enabled);
2350
2351        bridges.lock().unwrap().push_back(client);
2352        f.runtime.connect().await.unwrap();
2353        assert!(f.runtime.roots().await.is_empty());
2354        assert!(f.stub.last_workspaces().is_empty());
2355    }
2356
2357    #[tokio::test(flavor = "multi_thread")]
2358    async fn announcements_are_answered_from_a_reconnected_bridge() {
2359        let (f, bridges, client) = connecting().await;
2360        bridges.lock().unwrap().push_back(client);
2361        f.runtime.connect().await.unwrap();
2362        let id = f.runtime.enable(&f.root).await.unwrap();
2363        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2364        let answering = tokio::spawn(Arc::clone(&f.runtime).run());
2365
2366        f.stub.disconnect();
2367        let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2368        bridges.lock().unwrap().push_back(client);
2369        eventually("reconnected", async || next.last_workspaces() == vec![id]).await;
2370
2371        // The announcement loop follows the new connection: an announcement on it is
2372        // claimed, which with no data endpoint fails — and the loop lives on.
2373        next.announce(id, "ticket").await;
2374        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2375        assert!(!answering.is_finished(), "the announcement loop ended");
2376        answering.abort();
2377        keeping.abort();
2378    }
2379
2380    /// The scratch tree, the environment pointed at it, and a workspace root in it.
2381    fn scratch() -> (tempfile::TempDir, EnvGuard, PathBuf) {
2382        let lock = test_support::lock();
2383        let tmp = tempfile::tempdir().unwrap();
2384        let previous = DIR_VARS.map(std::env::var_os);
2385        // SAFETY (via `test_support::set`): `lock` serialises every read and write of the
2386        // process environment in this test binary, and it is held until `drop` has restored
2387        // the old values.
2388        for (name, dir) in DIR_VARS
2389            .iter()
2390            .zip(["cache", "data", "config"].map(|cat| tmp.path().join(cat)))
2391        {
2392            test_support::set(name, &dir);
2393        }
2394        let env = EnvGuard {
2395            previous,
2396            _lock: lock,
2397        };
2398        CTX.init(AppKind::Server);
2399
2400        let root = tmp.path().join("ws");
2401        std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
2402        let root = root.canonicalize().unwrap();
2403        (tmp, env, root)
2404    }
2405
2406    /// An application with one `Live` key in the workspace: `sapphire-synctest.note`.
2407    struct NoteApp;
2408
2409    impl crate::settings::AppSettings for NoteApp {
2410        fn defs(&self) -> Vec<sapphire_framework_settings::SettingDef> {
2411            vec![sapphire_framework_settings::SettingDef {
2412                key: "sapphire-synctest.note".into(),
2413                kind: sapphire_framework_settings::Kind::String { non_empty: false },
2414                scopes: vec![sapphire_framework_settings::Scope::Workspace],
2415                overridable: false,
2416                apply: sapphire_framework_settings::Apply::Live,
2417                default: None,
2418                section: "Test".into(),
2419                doc: "test".into(),
2420                process: sapphire_framework_settings::Process::App,
2421            }]
2422        }
2423    }
2424
2425    #[tokio::test(flavor = "multi_thread")]
2426    async fn a_workspace_config_changed_under_the_watcher_reaches_the_settings() {
2427        let f = fixture().await;
2428        // A context of its own, so its config directory is this test's: `CTX`'s is
2429        // whichever test resolved it first.
2430        let ctx: &'static AppContext = Box::leak(Box::new(AppContext::new("sapphire-synctest")));
2431        ctx.set_config_dir(f._tmp.path().join("settings"));
2432        let selection = crate::selection::SelectionFile::at(
2433            f._tmp.path().join("settings").join("workspace.toml"),
2434        );
2435        selection
2436            .save(&crate::selection::Selection {
2437                root: f.root.clone(),
2438                sync: true,
2439            })
2440            .unwrap();
2441        let settings =
2442            crate::settings::Settings::load_with(ctx, Arc::new(NoteApp), selection).unwrap();
2443        let changed = settings.changed();
2444        f.runtime.set_settings(Some(settings.clone()));
2445        f.runtime.enable(&f.root).await.unwrap();
2446        let watching = tokio::spawn(Arc::clone(&f.runtime).watch_changes());
2447        // Until the watcher is up, a write may go unseen: keep writing until it is noticed.
2448        let config = f.root.join(".sapphire-synctest").join("config.toml");
2449        eventually("the settings hear of the new value", async || {
2450            std::fs::write(&config, "[sapphire-synctest]\nnote = \"synced\"\n").unwrap();
2451            tokio::time::sleep(std::time::Duration::from_millis(200)).await;
2452            changed.has_changed().unwrap()
2453        })
2454        .await;
2455        assert_eq!(
2456            settings
2457                .get_as::<String>("sapphire-synctest.note")
2458                .as_deref(),
2459            Some("synced")
2460        );
2461        watching.abort();
2462    }
2463
2464    #[tokio::test(flavor = "multi_thread")]
2465    async fn enabling_registers_the_workspace_with_the_bridge() {
2466        let f = fixture().await;
2467
2468        let id = f.runtime.enable(&f.root).await.unwrap();
2469        assert_eq!(f.stub.last_workspaces(), vec![id]);
2470    }
2471
2472    #[tokio::test(flavor = "multi_thread")]
2473    async fn enabling_twice_is_idempotent_and_keeps_the_same_id() {
2474        let f = fixture().await;
2475
2476        let first = f.runtime.enable(&f.root).await.unwrap();
2477        let second = f.runtime.enable(&f.root).await.unwrap();
2478        assert_eq!(first, second);
2479        assert_eq!(
2480            f.stub.last_workspaces(),
2481            vec![first],
2482            "still exactly one workspace"
2483        );
2484    }
2485
2486    #[tokio::test(flavor = "multi_thread")]
2487    async fn a_registration_carries_the_complete_current_list() {
2488        let f = fixture().await;
2489        let second_root = f.second_root();
2490
2491        let a = f.runtime.enable(&f.root).await.unwrap();
2492        let b = f.runtime.enable(&second_root).await.unwrap();
2493
2494        let mut seen = f.stub.last_workspaces();
2495        seen.sort();
2496        let mut want = vec![a, b];
2497        want.sort();
2498        assert_eq!(seen, want, "registration is the whole set, not a delta");
2499    }
2500
2501    #[tokio::test(flavor = "multi_thread")]
2502    async fn no_later_registration_clears_this_apps_routes() {
2503        // The device id is asked of the bridge once, on the first `enable`, with an empty
2504        // workspace list — the one moment that is safe, because no route exists yet. Every
2505        // later registration must name the complete set: one with an empty list would
2506        // briefly clear this application's routes, and a peer asking for a workspace this
2507        // server owns would be told nobody has it.
2508        let f = fixture().await;
2509        let second_root = f.second_root();
2510
2511        f.runtime.enable(&f.root).await.unwrap();
2512        f.runtime.enable(&second_root).await.unwrap();
2513        f.runtime.sync_now(&f.root).await.unwrap();
2514        f.runtime.disable(&f.root).await.unwrap();
2515
2516        let seen = f.stub.seen.lock().expect("stub");
2517        let first_full = seen
2518            .registrations
2519            .iter()
2520            .position(|r| !r.workspaces.is_empty())
2521            .expect("enabling registers the workspace");
2522        let emptied_afterwards = seen.registrations[first_full + 1..]
2523            .iter()
2524            .filter(|r| r.workspaces.is_empty())
2525            .count();
2526        assert_eq!(
2527            emptied_afterwards, 0,
2528            "a registration after the first workspace must still name the whole set"
2529        );
2530    }
2531
2532    #[tokio::test(flavor = "multi_thread")]
2533    async fn disabling_unregisters_and_leaves_the_files_alone() {
2534        let f = fixture().await;
2535        std::fs::write(f.root.join("keep.md"), "content").unwrap();
2536
2537        let id = f.runtime.enable(&f.root).await.unwrap();
2538        f.runtime.disable(&f.root).await.unwrap();
2539
2540        assert!(f.stub.last_workspaces().is_empty());
2541        assert!(
2542            f.root.join("keep.md").exists(),
2543            "disabling sync must not touch files"
2544        );
2545        assert!(
2546            crate::sync::id::sync_id_path("sapphire-synctest", &f.root).exists(),
2547            "the sync id stays, so re-enabling rejoins the same workspace"
2548        );
2549        let _ = id;
2550    }
2551
2552    #[tokio::test(flavor = "multi_thread")]
2553    async fn re_enabling_after_disabling_reuses_the_identity() {
2554        let f = fixture().await;
2555
2556        let first = f.runtime.enable(&f.root).await.unwrap();
2557        f.runtime.disable(&f.root).await.unwrap();
2558        let again = f.runtime.enable(&f.root).await.unwrap();
2559
2560        assert_eq!(first, again);
2561    }
2562
2563    #[tokio::test(flavor = "multi_thread")]
2564    async fn status_reports_disabled_for_a_workspace_that_was_never_enabled() {
2565        let f = fixture().await;
2566
2567        let status = f.runtime.status(&f.root).await;
2568        assert!(!status.enabled);
2569        assert!(status.workspace_id.is_none());
2570    }
2571
2572    #[tokio::test(flavor = "multi_thread")]
2573    async fn disabling_something_that_was_never_enabled_is_not_an_error() {
2574        let f = fixture().await;
2575        f.runtime
2576            .disable(&f.root)
2577            .await
2578            .expect("disabling twice is fine");
2579    }
2580
2581    #[tokio::test(flavor = "multi_thread")]
2582    async fn a_file_already_on_disk_is_recorded_on_enable() {
2583        // No watcher fires for a file written before the workspace was enabled; without a
2584        // scan on enable it was never recorded, and every session sent nothing.
2585        let f = fixture().await;
2586        std::fs::write(f.root.join("a.md"), "content").unwrap();
2587        f.runtime.enable(&f.root).await.unwrap();
2588        f.runtime.settle_dials().await;
2589
2590        let replica = f.runtime.replica_of(&f.root).await.unwrap();
2591        let states = replica.lock().await.states().unwrap();
2592        assert!(
2593            states.iter().any(|(path, _)| path == "a.md"),
2594            "a file on disk before enable must be recorded: {states:?}"
2595        );
2596    }
2597
2598    #[tokio::test(flavor = "multi_thread")]
2599    async fn re_enabling_while_the_replica_is_still_held_takes_it_back() {
2600        // The first scan after enabling runs in the background; a disable and a re-enable
2601        // can both land while it still holds the store, which redb opens only once.
2602        let f = fixture().await;
2603        f.runtime.enable(&f.root).await.unwrap();
2604        f.runtime.settle_dials().await;
2605        let held = f.runtime.replica_of(&f.root).await.unwrap();
2606
2607        f.runtime.disable(&f.root).await.unwrap();
2608        f.runtime
2609            .enable(&f.root)
2610            .await
2611            .expect("a re-enable must not open the store a second time");
2612
2613        let again = f.runtime.replica_of(&f.root).await.unwrap();
2614        assert!(Arc::ptr_eq(&held, &again), "the open replica is taken back");
2615    }
2616
2617    #[tokio::test(flavor = "multi_thread")]
2618    async fn re_enabling_while_the_replica_is_still_closing_waits_for_it() {
2619        // The last holder let go just before the re-enable: the replica can no longer be
2620        // taken back, and its store closes a moment later, as it drops.
2621        let f = fixture().await;
2622        f.runtime.enable(&f.root).await.unwrap();
2623        f.runtime.settle_dials().await;
2624        let config = {
2625            let replica = f.runtime.replica_of(&f.root).await.unwrap();
2626            replica.lock().await.config().clone()
2627        };
2628        f.runtime.disable(&f.root).await.unwrap();
2629
2630        // The store, open still, with nothing left to take back.
2631        let closing = Replica::open(config, Arc::new(SystemClock)).unwrap();
2632        let key = f.root.canonicalize().unwrap();
2633        f.runtime
2634            .stopped
2635            .lock()
2636            .unwrap()
2637            .insert(key, std::sync::Weak::new());
2638        let closed = tokio::spawn(async move {
2639            tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2640            drop(closing);
2641        });
2642
2643        f.runtime
2644            .enable(&f.root)
2645            .await
2646            .expect("a re-enable must wait for a store that is closing");
2647        closed.await.unwrap();
2648    }
2649
2650    #[tokio::test(flavor = "multi_thread")]
2651    async fn a_workspace_whose_root_vanished_is_reported_as_paused_not_deleted() {
2652        let f = fixture().await;
2653        std::fs::write(f.root.join("a.md"), "content").unwrap();
2654        f.runtime.enable(&f.root).await.unwrap();
2655        f.runtime.scan(&f.root).await.unwrap();
2656
2657        // The drive is unmounted: the marker directory goes with it.
2658        std::fs::remove_dir_all(f.root.join(".sapphire-synctest")).unwrap();
2659        let _ = f.runtime.scan(&f.root).await;
2660
2661        let status = f.runtime.status(&f.root).await;
2662        assert!(
2663            status.paused.is_some(),
2664            "a missing root must pause, not replicate as a mass deletion"
2665        );
2666    }
2667
2668    #[tokio::test(flavor = "multi_thread")]
2669    async fn a_directory_that_is_not_a_workspace_cannot_be_enabled() {
2670        let f = fixture().await;
2671        let plain = f._tmp.path().join("plain");
2672        std::fs::create_dir_all(&plain).unwrap();
2673
2674        assert!(f.runtime.enable(&plain).await.is_err());
2675    }
2676
2677    #[tokio::test(flavor = "multi_thread")]
2678    async fn a_write_through_the_workspace_router_is_committed_to_the_replica() {
2679        // The brief's "main path": a write the app server itself made is scanned straight
2680        // after, not left to the watcher's debounce. Asserted against the replica's own
2681        // state, which is the only thing that can tell the two paths apart — reading the
2682        // file back would pass even if the scan never ran.
2683        let f = fixture().await;
2684        f.runtime.enable(&f.root).await.unwrap();
2685
2686        let host = Arc::new(crate::WorkspaceHost::new(&CTX));
2687        host.select(&f.root, false).await.unwrap();
2688        let current = Arc::new(crate::Current::new(
2689            &CTX,
2690            host,
2691            Some(Arc::clone(&f.runtime)),
2692            crate::selection::SelectionFile::at(f._tmp.path().join("workspace.toml")),
2693        ));
2694        let router = Arc::new(crate::workspace_router(current));
2695        let (client_conn, server_conn) = sapphire_ipc::Connection::pair();
2696        tokio::spawn(async move {
2697            let info = sapphire_ipc::ServerInfo {
2698                version: "0.0.0".into(),
2699                api: sapphire_backend::protocol::API_VERSION,
2700                app_api: None,
2701                pid: std::process::id(),
2702                managed_by: ManagedBy::Spawned,
2703            };
2704            let _ = sapphire_ipc::serve(server_conn, router, "sapphire-synctest", info).await;
2705        });
2706        let info = sapphire_ipc::ClientInfo {
2707            kind: "test".into(),
2708            version: "0.0.0".into(),
2709            api: sapphire_backend::protocol::API_VERSION,
2710            app_api: None,
2711            pid: std::process::id(),
2712        };
2713        let (client, _) = sapphire_ipc::Client::handshake(client_conn, "sapphire-synctest", info)
2714            .await
2715            .unwrap();
2716
2717        let _: sapphire_backend::protocol::Ack = client
2718            .call(
2719                sapphire_backend::protocol::WRITE_FILE,
2720                sapphire_backend::protocol::ContentParams {
2721                    path: PathBuf::from("through-the-server.md"),
2722                    content: "written by the server".into(),
2723                },
2724            )
2725            .await
2726            .unwrap();
2727
2728        // `scan` runs before the reply is written, so by the time the call returns the
2729        // replica must already know the path.
2730        let synced = f.runtime.synced.lock().await;
2731        let entry = synced.get(&f.root).expect("enabled above");
2732        let replica = entry.replica.lock().await;
2733        let state = replica
2734            .state("through-the-server.md")
2735            .expect("a readable state");
2736        assert!(
2737            state.is_some(),
2738            "a write through the server must be scanned into the replica, not left to the watcher"
2739        );
2740    }
2741
2742    #[tokio::test(flavor = "multi_thread")]
2743    async fn an_announcement_for_an_unknown_workspace_is_ignored_not_fatal() {
2744        let f = fixture().await;
2745        f.runtime.enable(&f.root).await.unwrap();
2746
2747        let driver = Arc::clone(&f.runtime);
2748        let handle = tokio::spawn(async move { driver.run().await });
2749
2750        f.stub
2751            .announce(grain_id::GrainId::random(), "no-such-ticket")
2752            .await;
2753        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
2754
2755        assert!(
2756            !handle.is_finished(),
2757            "one bad announcement must not stop the runtime"
2758        );
2759        handle.abort();
2760    }
2761}