Skip to main content

sapphire_framework_server/sync/
mod.rs

1//! Replication, from the app server's side.
2//!
3//! One `Replica` per synced workspace, registered with the bridge, driven by announcements
4//! from it and by local edits.
5//!
6//! See `docs/superpowers/specs/2026-09-16-process-architecture-design.md` §4.2.
7
8use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use grain_id::GrainId;
13use sapphire_backend::protocol as proto;
14use sapphire_bridge_api::{
15    IncomingParams, RegisterParams, WorkspaceRegistration, WorkspacesResult,
16};
17use sapphire_bridge_client::BridgeClient;
18use sapphire_ipc::ManagedBy;
19use sapphire_sync::{PathUpdate, PauseReason, Replica, ReplicaConfig, ScanOutcome, SystemClock};
20use sapphire_workspace::{AppContext, Workspace};
21use tokio::sync::{Mutex, OnceCell, broadcast, mpsc};
22
23pub mod id;
24mod live;
25mod methods;
26#[cfg(any(test, feature = "test-util"))]
27pub mod testing;
28mod topology;
29mod watch;
30
31pub use id::{SYNC_ID_FILE, WORKSPACE_MAP_FILE, sync_id, sync_id_path};
32pub use live::{DIAL_BACKOFF_MAX, OPEN_TIMEOUT};
33pub use methods::sync_router;
34/// What [`SyncRuntime::peers`] answers with.
35pub use sapphire_bridge_api::{PeerInfo, PeersResult};
36pub use watch::{DEBOUNCE, Watcher};
37
38use live::{LivePeers, dial_backoff};
39
40use crate::error::{Error, Result};
41
42use crate::host::WorkspaceHost;
43
44/// What `sync.status` reports.
45#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
46pub struct SyncStatus {
47    /// Whether this workspace is synced at all.
48    pub enabled: bool,
49    /// Its identity across devices, when it is.
50    pub workspace_id: Option<GrainId>,
51    /// How many devices the workgroup has, this host excluded.
52    pub peers: usize,
53    /// Why replication is paused, if it is — a missing root or marker directory.
54    pub paused: Option<String>,
55    /// The last failure, if any.
56    pub last_error: Option<String>,
57    /// Whether the bridge is reachable. `false` is not an outage: the app server works.
58    pub bridge_available: bool,
59    /// How the workspace is wired to its peers, by the roles the bridge last reported.
60    pub topology: proto::Topology,
61    /// How far embedding has got, when this host embeds (#188).
62    pub embedding: Option<proto::EmbeddingProgress>,
63}
64
65/// One workspace this runtime syncs.
66struct Synced {
67    /// Its identity across devices, shared with every peer.
68    workspace_id: GrainId,
69    /// Its replica store. Held open because the store is a redb database.
70    replica: Arc<Mutex<Replica>>,
71    /// Why replication is paused, from the last scan.
72    paused: Option<PauseReason>,
73    /// The last failure, if any.
74    last_error: Option<String>,
75}
76
77/// How old a vector file no live content needs must be before the primary device removes
78/// it: a vector can arrive before its file, and a day is far longer than that window.
79const STALE_VECTOR_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
80
81/// Texts per embedding request in a server pass: small, so a search query waiting on the
82/// bridge's one worker is never stuck behind a long backfill (#188).
83const PASS_BATCH: usize = 4;
84
85/// How the primary device paces its backfill (#188).
86#[derive(Clone, Copy, Debug)]
87pub struct BackfillTiming {
88    /// How old another device's version must be before the primary embeds it: its author
89    /// has embedded it by then, or will not.
90    pub grace: std::time::Duration,
91    /// How often the primary device runs a pass with nothing prompting it.
92    pub interval: std::time::Duration,
93}
94
95impl Default for BackfillTiming {
96    fn default() -> Self {
97        BackfillTiming {
98            grace: std::time::Duration::from_secs(10 * 60),
99            interval: std::time::Duration::from_secs(10 * 60),
100        }
101    }
102}
103
104/// How long [`SyncRuntime::keep_connected`] waits before its first retry after the bridge
105/// could not be reached. Each failure doubles it, up to [`RECONNECT_BACKOFF_MAX`].
106pub const RECONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_millis(500);
107
108/// The longest [`SyncRuntime::keep_connected`] waits between two attempts to reach the
109/// bridge: a bridge that comes up is found within this long.
110pub const RECONNECT_BACKOFF_MAX: std::time::Duration = std::time::Duration::from_secs(30);
111
112/// The wait before reconnection attempt `attempt` (0-based).
113fn reconnect_backoff(attempt: u32) -> std::time::Duration {
114    RECONNECT_BACKOFF
115        .saturating_mul(1u32 << attempt.min(16))
116        .min(RECONNECT_BACKOFF_MAX)
117}
118
119/// What a connection attempt to the bridge yields.
120type Connecting = std::pin::Pin<
121    Box<dyn std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>> + Send>,
122>;
123
124/// How a [`SyncRuntime`] reaches the bridge, on start and whenever it has lost it (#244).
125///
126/// An attempt answers `Ok(None)` when no bridge is running — the runtime then tries again
127/// later — and an error for anything else, such as a bridge of another API version.
128#[derive(Clone)]
129pub struct BridgeConnector(Arc<dyn Fn() -> Connecting + Send + Sync>);
130
131impl BridgeConnector {
132    /// Connect with `connect`.
133    pub fn new<F, Fut>(connect: F) -> BridgeConnector
134    where
135        F: Fn() -> Fut + Send + Sync + 'static,
136        Fut: std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>>
137            + Send
138            + 'static,
139    {
140        BridgeConnector(Arc::new(move || Box::pin(connect())))
141    }
142
143    /// The host's own bridge, at its standard endpoint, as a client of `kind` at
144    /// `version`.
145    pub fn standard(kind: &'static str, version: &'static str) -> BridgeConnector {
146        BridgeConnector::new(move || async move {
147            match BridgeClient::connect(kind, version).await {
148                Ok(client) => Ok(Some(client)),
149                Err(sapphire_ipc::Error::NotRunning(_)) => Ok(None),
150                Err(err) => Err(err),
151            }
152        })
153    }
154
155    /// The bridge listening at `endpoint`: for a fixture that runs its own.
156    pub fn at(
157        endpoint: sapphire_ipc::Endpoint,
158        kind: &'static str,
159        version: &'static str,
160    ) -> BridgeConnector {
161        BridgeConnector::new(move || {
162            let endpoint = endpoint.clone();
163            async move { BridgeClient::connect_at(&endpoint, kind, version).await }
164        })
165    }
166
167    async fn connect(&self) -> sapphire_ipc::Result<Option<BridgeClient>> {
168        (self.0)().await
169    }
170}
171
172impl std::fmt::Debug for BridgeConnector {
173    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
174        f.write_str("BridgeConnector")
175    }
176}
177
178/// Replication for one application's workspaces.
179pub struct SyncRuntime {
180    ctx: &'static AppContext,
181    /// The bridge connection, while there is one. Swapped when the runtime reconnects
182    /// (#244); [`run`](SyncRuntime::run) follows it.
183    bridge: tokio::sync::watch::Sender<Option<Arc<BridgeClient>>>,
184    /// How to reach the bridge again; `None` for a runtime given one connection to use.
185    connector: Option<BridgeConnector>,
186    /// Workspaces asked to sync while the bridge was away, with their ids. Enabled once
187    /// it is back.
188    ///
189    /// Also the lock that orders an `enable` against a reconnection: whoever holds it sees
190    /// the bridge slot as the other left it, so a workspace is either enabled now or left
191    /// here for the reconnection to enable — never neither.
192    pending: Mutex<HashMap<PathBuf, GrainId>>,
193    /// The announcements of a connection [`attach`](SyncRuntime::attach) just made,
194    /// subscribed to before it registered: the bridge announces the streams that waited for
195    /// this server as soon as it registers (#244), and a subscription made afterwards by
196    /// [`run`](SyncRuntime::run) would miss them.
197    subscribed: std::sync::Mutex<Option<(Arc<BridgeClient>, broadcast::Receiver<IncomingParams>)>>,
198    exe_path: PathBuf,
199    managed_by: ManagedBy,
200    synced: Mutex<HashMap<PathBuf, Synced>>,
201    /// This host's device id. Asked of the bridge once, on the first [`enable`], and never
202    /// again: a second registration with an empty workspace list would briefly clear this
203    /// application's routes.
204    ///
205    /// [`enable`]: SyncRuntime::enable
206    device_id: OnceCell<GrainId>,
207    /// The file watcher, once [`watch_changes`] has started it.
208    ///
209    /// The runtime owns it because the spec puts it here: the sync runtime is a watcher and
210    /// a replica per registered root, and the two must agree on which roots are synced.
211    /// [`enable`] and [`disable`] keep them in step.
212    ///
213    /// [`watch_changes`]: SyncRuntime::watch_changes
214    /// [`enable`]: SyncRuntime::enable
215    /// [`disable`]: SyncRuntime::disable
216    watcher: OnceCell<Arc<Watcher>>,
217    /// The app server that owns these workspaces' caches, once it has told us.
218    ///
219    /// A replication session writes files, and the search index is not part of a session: the
220    /// index belongs to the app server, which holds the workspace open. Reaching it through
221    /// the host is what keeps a received file and its index moving together, and it is the
222    /// *same* [`WorkspaceState`](sapphire_workspace::WorkspaceState) the server serves from —
223    /// opening a second one would collide on the retrieve store's exclusive lock.
224    ///
225    /// Absent for a runtime nobody has wired to an app server (a test stub driving the
226    /// replica directly). Nothing then indexes, which is correct: nothing owns an index.
227    host: OnceCell<Arc<WorkspaceHost>>,
228    /// Serialises re-indexing, so two sessions finishing at once do not both sweep the
229    /// workspace through the same store.
230    reindexing: Mutex<()>,
231    /// Roots with an embedding pass waiting to run (#187). A request for a root already
232    /// waiting is folded into that pass.
233    embed_wanted: Mutex<std::collections::HashSet<PathBuf>>,
234    /// Serialises embedding passes: one model, one queue.
235    embedding: Mutex<()>,
236    /// Roots whose embedding pass is running now, for `sync.status`.
237    embed_running: Mutex<std::collections::HashSet<PathBuf>>,
238    /// The primary device's backfill pacing (#188).
239    backfill: std::sync::Mutex<BackfillTiming>,
240    /// The bridge an embedding pass asks for its model; `None` is the host's standard one.
241    bridge_endpoint: Option<sapphire_ipc::Endpoint>,
242    /// The live session table of every synced workspace, keyed by canonical root.
243    ///
244    /// One table per workspace rather than one for the runtime: a session is about one
245    /// workspace, and the two storm rules are per-workspace rules. A root that is not
246    /// synced has none, so a dial or a push for it is a no-op rather than an error.
247    live: Mutex<HashMap<PathBuf, Arc<LivePeers>>>,
248    /// The bridge's last answer to `peers`, roles included. The inbound guard and
249    /// [`is_primary`](SyncRuntime::is_primary) read it rather than asking again.
250    last_peers: Mutex<Option<PeersResult>>,
251    /// How long a dial waits for the bridge to open a stream; [`OPEN_TIMEOUT`] by default.
252    open_timeout: std::time::Duration,
253    /// The first dials [`enable`](SyncRuntime::enable) leaves running. Each pins the
254    /// runtime, so [`stop_dials`](SyncRuntime::stop_dials) ends them with the server.
255    first_dials: std::sync::Mutex<tokio::task::JoinSet<()>>,
256    /// The rest of the runtime's background work: inbound sessions, the readers of live
257    /// sessions, and embedding passes. Each pins the runtime — and through it the replica
258    /// stores, which are redb databases locked until the last pin drops — so
259    /// [`stop_background`](SyncRuntime::stop_background) ends them with the server (#235).
260    background: std::sync::Mutex<tokio::task::JoinSet<()>>,
261}
262
263impl SyncRuntime {
264    /// A runtime for `ctx`'s application, syncing through `bridge`.
265    ///
266    /// It has no way to reach the bridge again: when the connection closes, sync stops until
267    /// the runtime is given a [`connector`](Self::with_connector).
268    pub fn new(
269        ctx: &'static AppContext,
270        bridge: Arc<BridgeClient>,
271        exe_path: PathBuf,
272        managed_by: ManagedBy,
273    ) -> SyncRuntime {
274        SyncRuntime::build(ctx, Some(bridge), None, exe_path, managed_by)
275    }
276
277    /// A runtime for `ctx`'s application that reaches the bridge with `connector`, and again
278    /// whenever it loses it (#244).
279    ///
280    /// Not connected yet: [`connect`](Self::connect) makes the first attempt, and
281    /// [`keep_connected`](Self::keep_connected) the rest. [`AppServer::run`] does both.
282    ///
283    /// [`AppServer::run`]: crate::AppServer::run
284    pub fn connecting(
285        ctx: &'static AppContext,
286        connector: BridgeConnector,
287        exe_path: PathBuf,
288        managed_by: ManagedBy,
289    ) -> SyncRuntime {
290        SyncRuntime::build(ctx, None, Some(connector), exe_path, managed_by)
291    }
292
293    /// Reach the bridge again with `connector` once the connection closes.
294    pub fn with_connector(mut self, connector: BridgeConnector) -> SyncRuntime {
295        self.connector = Some(connector);
296        self
297    }
298
299    fn build(
300        ctx: &'static AppContext,
301        bridge: Option<Arc<BridgeClient>>,
302        connector: Option<BridgeConnector>,
303        exe_path: PathBuf,
304        managed_by: ManagedBy,
305    ) -> SyncRuntime {
306        SyncRuntime {
307            ctx,
308            bridge: tokio::sync::watch::Sender::new(bridge),
309            connector,
310            pending: Mutex::new(HashMap::new()),
311            subscribed: std::sync::Mutex::new(None),
312            exe_path,
313            managed_by,
314            synced: Mutex::new(HashMap::new()),
315            device_id: OnceCell::new(),
316            watcher: OnceCell::new(),
317            host: OnceCell::new(),
318            reindexing: Mutex::new(()),
319            embed_wanted: Mutex::new(std::collections::HashSet::new()),
320            embedding: Mutex::new(()),
321            embed_running: Mutex::new(std::collections::HashSet::new()),
322            backfill: std::sync::Mutex::new(BackfillTiming::default()),
323            bridge_endpoint: None,
324            live: Mutex::new(HashMap::new()),
325            last_peers: Mutex::new(None),
326            open_timeout: OPEN_TIMEOUT,
327            first_dials: std::sync::Mutex::new(tokio::task::JoinSet::new()),
328            background: std::sync::Mutex::new(tokio::task::JoinSet::new()),
329        }
330    }
331
332    /// The bridge, while connected.
333    fn bridge(&self) -> Result<Arc<BridgeClient>> {
334        match &*self.bridge.borrow() {
335            Some(bridge) if !bridge.is_closed() => Ok(Arc::clone(bridge)),
336            _ => Err(Error::BridgeDisconnected),
337        }
338    }
339
340    /// Whether the runtime has a live connection to the bridge.
341    pub fn is_connected(&self) -> bool {
342        self.bridge().is_ok()
343    }
344
345    /// Run `task` in the background, tracked so the server's shutdown ends it.
346    fn spawn_background(&self, task: impl std::future::Future<Output = ()> + Send + 'static) {
347        let mut background = self.background.lock().expect("background tasks");
348        // Reap the finished ones, so the set holds only what is running.
349        while background.try_join_next().is_some() {}
350        background.spawn(task);
351    }
352
353    /// Give up on opening a stream to a peer after `timeout` rather than [`OPEN_TIMEOUT`]:
354    /// for tests, which cannot wait that long for a peer that never answers.
355    pub fn with_open_timeout(mut self, timeout: std::time::Duration) -> SyncRuntime {
356        self.open_timeout = timeout;
357        self
358    }
359
360    /// Tell the runtime which app server owns these workspaces' caches.
361    ///
362    /// Called by [`AppServer::sync`](crate::AppServer::sync). First writer wins: a runtime
363    /// belongs to the server that wired it.
364    /// Embed through the bridge at `endpoint` rather than the host's standard one: for a
365    /// fixture running several hosts in one process.
366    pub fn with_bridge_endpoint(mut self, endpoint: sapphire_ipc::Endpoint) -> SyncRuntime {
367        self.bridge_endpoint = Some(endpoint);
368        self
369    }
370
371    /// Pace the primary device's backfill differently: for tests, which cannot wait ten
372    /// minutes.
373    pub fn set_backfill_timing(&self, timing: BackfillTiming) {
374        *self.backfill.lock().expect("backfill timing") = timing;
375    }
376
377    fn backfill_timing(&self) -> BackfillTiming {
378        *self.backfill.lock().expect("backfill timing")
379    }
380
381    pub(crate) fn set_host(&self, host: Arc<WorkspaceHost>) {
382        let _ = self.host.set(host);
383    }
384
385    /// Start syncing `root`, returning its identity across devices.
386    ///
387    /// Idempotent: enabling an already-synced workspace returns the same id and changes
388    /// nothing.
389    pub async fn enable(self: &Arc<Self>, root: &Path) -> Result<GrainId> {
390        let key = root.canonicalize().map_err(Error::Io)?;
391        let watch_key = key.clone();
392        let workspace_id = sync_id(self.ctx.app_name, &key)?;
393
394        // Without a bridge there is no device id to open the replica with, and nobody to
395        // register with: keep the workspace for the reconnection to enable (#244).
396        {
397            let mut pending = self.pending.lock().await;
398            if self.bridge().is_err() {
399                pending.insert(key, workspace_id);
400                return Err(Error::BridgeDisconnected);
401            }
402            pending.remove(&key);
403        }
404
405        // The map stays locked across opening the replica, so two simultaneous `enable`s for
406        // one root cannot both open it — the store is a redb database, and it is opened once.
407        {
408            let mut synced = self.synced.lock().await;
409            if let Some(existing) = synced.get(&key) {
410                return Ok(existing.workspace_id);
411            }
412            let device_id = self.device_id().await?;
413            let state_dir = self.state_dir(&key)?;
414            std::fs::create_dir_all(&state_dir).map_err(Error::Io)?;
415            let config = ReplicaConfig::new(self.ctx.app_name, key.clone(), device_id, &state_dir);
416            let replica = Replica::open(config, Arc::new(SystemClock))
417                .map_err(|e| Error::Sync(e.to_string()))?;
418            synced.insert(
419                key.clone(),
420                Synced {
421                    workspace_id,
422                    replica: Arc::new(Mutex::new(replica)),
423                    paused: None,
424                    last_error: None,
425                },
426            );
427        }
428
429        // Watch the new root too, so an edit made outside the server from now on is seen.
430        // Not fatal if it fails: the app server's own writes take the exact path in
431        // `handlers.rs`, and the next `watch_changes` re-reads the root set anyway.
432        if let Some(watcher) = self.watcher.get()
433            && let Err(err) = watcher.watch(&watch_key)
434        {
435            tracing::warn!(root = %watch_key.display(), "could not watch: {err}");
436        }
437
438        // Register only once the replica is open, so a registration never names a workspace
439        // that cannot serve a session.
440        self.reregister().await?;
441
442        // Dial now, so enabling a workspace converges it rather than waiting for the next
443        // local edit to trigger a session. This is what catches a host up when it returns:
444        // the peer that has been running is the one the returning host has to ask, and no
445        // watcher on either side fires for a file that was never written locally.
446        //
447        // Best-effort: a workspace is enabled whether or not any peer is up, and a failure
448        // here is not a failure to enable. A session is symmetric, so dialing is enough —
449        // this side sends its own view, and the peer answers with what this side lacks.
450        //
451        // In the background, so the answer does not wait on it: a peer that is offline
452        // holds a dial up to `open_timeout`, longer than a client waits for `sync.enable`
453        // (#228).
454        let runtime = Arc::clone(self);
455        let mut dials = self.first_dials.lock().expect("first dials");
456        // Reap the finished ones, so the set holds only what is running.
457        while dials.try_join_next().is_some() {}
458        dials.spawn(async move {
459            if let Err(err) = runtime.sync_now(&key).await {
460                tracing::warn!(root = %key.display(), "the first session after enabling failed: {err}");
461            }
462        });
463        Ok(workspace_id)
464    }
465
466    /// Stop syncing `root`. Files and the sync id stay, so re-enabling rejoins the same
467    /// workspace rather than creating a second one.
468    pub async fn disable(&self, root: &Path) -> Result<()> {
469        self.stop(root).await.1
470    }
471
472    /// Stop syncing `root`, like [`disable`](Self::disable), then wait up to `limit` until
473    /// nothing holds its replica store open any more — a session, a scan or an embedding
474    /// pass that started before may still have it. A store about to be moved or deleted
475    /// must be closed first.
476    ///
477    /// Only a store still in use at `limit` is an error. The bridge not hearing of it is
478    /// logged, not returned: the store is closed all the same, and the next registration
479    /// replaces the bridge's view.
480    pub async fn release(&self, root: &Path, limit: std::time::Duration) -> Result<()> {
481        let (replica, unregistered) = self.stop(root).await;
482        if let Err(err) = unregistered {
483            tracing::warn!(root = %root.display(), "stopping sync: {err}");
484        }
485        let Some(replica) = replica else {
486            return Ok(());
487        };
488        let deadline = tokio::time::Instant::now() + limit;
489        while Arc::strong_count(&replica) > 1 {
490            if tokio::time::Instant::now() >= deadline {
491                return Err(Error::Sync(format!(
492                    "the replica of {} is still in use; try again",
493                    root.display()
494                )));
495            }
496            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
497        }
498        Ok(())
499    }
500
501    /// Stop syncing `root`, handing back its replica — `None` when it was not synced — and
502    /// how telling the bridge went.
503    async fn stop(&self, root: &Path) -> (Option<Arc<Mutex<Replica>>>, Result<()>) {
504        // A root that no longer exists cannot be canonicalized, but the keys of `synced` are
505        // canonical already, so the path as given (a registry root) still names the entry.
506        let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
507        self.pending.lock().await.remove(&key);
508        let removed = self.synced.lock().await.remove(&key);
509        if removed.is_some()
510            && let Some(watcher) = self.watcher.get()
511        {
512            watcher.unwatch(&key);
513        }
514        // Any live session for a workspace this server no longer holds is closed: it would
515        // otherwise keep pushing files for a workspace nobody asked about any more.
516        if let Some(peers) = self.live.lock().await.remove(&key) {
517            peers.drop_connections().await;
518        }
519        let Some(removed) = removed else {
520            return (None, Ok(()));
521        };
522        let unregistered = async {
523            // A bridge that is gone has forgotten this server's registrations already, and
524            // the reconnection registers only what is still synced.
525            let Ok(bridge) = self.bridge() else {
526                return Ok(());
527            };
528            bridge
529                .unregister(removed.workspace_id)
530                .await
531                .map_err(|e| Error::Bridge(e.to_string()))?;
532            self.reregister().await
533        }
534        .await;
535        (Some(removed.replica), unregistered)
536    }
537
538    /// The workspaces the workgroup knows about.
539    ///
540    /// Read straight from the bridge, so it reflects every device's registrations. The
541    /// bridge being down is not an empty answer — that would read as "nothing to map" — so
542    /// it is an error, and `sync.map` reports it.
543    pub async fn workspaces(&self) -> Result<WorkspacesResult> {
544        self.bridge()?
545            .workspaces()
546            .await
547            .map_err(|e| Error::Bridge(e.to_string()))
548    }
549
550    /// Place a workgroup workspace at a directory of this host: write its identity and
551    /// map there, returning the canonical root. [`Current::map`](crate::Current::map)
552    /// then selects it and syncs it.
553    ///
554    /// `selector` is the workspace's name or id, as [`workspaces`] lists it; `dir` must
555    /// exist. Its marker is made when it has none, and nothing else: the marker's
556    /// `config.toml` is the application's, and arrives with the first sync — a local one
557    /// written first would meet it as a conflict (#253).
558    ///
559    /// [`workspaces`]: SyncRuntime::workspaces
560    pub async fn place(&self, selector: &str, dir: &Path) -> Result<PathBuf> {
561        let workgroup = self.workspaces().await?;
562        let wanted = workgroup
563            .workspaces
564            .iter()
565            // A name is the user-facing handle; the id is the exact one. Two workspaces
566            // may not share a name, so a name names at most one.
567            .find(|w| w.name == selector || w.workspace_id.to_string() == selector)
568            .ok_or_else(|| Error::UnknownWorkspaceName(selector.to_owned()))?;
569
570        // Refuse another application's workspace: this server syncs its own application's
571        // workspaces, and `Workspace::from_root` would later refuse the directory anyway.
572        if wanted.app_name != self.ctx.app_name {
573            return Err(Error::WrongApp {
574                name: wanted.name.clone(),
575                app_name: wanted.app_name.clone(),
576            });
577        }
578
579        let root = dir.canonicalize().map_err(Error::Io)?;
580        let marker = root.join(format!(".{}", self.ctx.app_name));
581        if !marker.is_dir() {
582            std::fs::create_dir(&marker).map_err(Error::Io)?;
583        }
584        let workspace = Workspace::from_root(self.ctx, &root)?;
585        if workspace.root != root {
586            return Err(Error::UnknownWorkspace(root, self.ctx.app_name));
587        }
588
589        // A directory already syncing under another identity is not remapped: the sync id
590        // is a device's word that it is the same workspace, and rewriting it silently
591        // would make two hosts disagree about what they share.
592        {
593            let synced = self.synced.lock().await;
594            if let Some(entry) = synced.get(&root)
595                && entry.workspace_id != wanted.workspace_id
596            {
597                return Err(Error::SyncId(format!(
598                    "{} already syncs as {}, not {}",
599                    root.display(),
600                    entry.workspace_id,
601                    wanted.workspace_id
602                )));
603            }
604        }
605
606        // The identity arrives from the workgroup, not from this host: two hosts that map
607        // the same workspace must sync as one, so the id is the one the workgroup lists,
608        // written before `enable` reads it. A different id already on disk is refused
609        // above; an absent one is written here.
610        let marker = workspace.marker_dir();
611        let id_path = marker.join(crate::sync::id::SYNC_ID_FILE);
612        match std::fs::read_to_string(&id_path) {
613            Ok(existing) if existing.trim() != wanted.workspace_id.to_string() => {
614                return Err(Error::SyncId(format!(
615                    "{} already holds identity {}, not {}",
616                    root.display(),
617                    existing.trim(),
618                    wanted.workspace_id
619                )));
620            }
621            Err(_) => {
622                std::fs::write(&id_path, format!("{}\n", wanted.workspace_id))
623                    .map_err(Error::Io)?;
624            }
625            Ok(_) => {}
626        }
627
628        // The map is what tells this application, on a later start, which workgroup
629        // workspace a directory is.
630        let map_path = root
631            .join(format!(".{}", self.ctx.app_name))
632            .join(WORKSPACE_MAP_FILE);
633        let workspace_id = wanted.workspace_id.to_string();
634        std::fs::write(&map_path, format!("{workspace_id}\n")).map_err(Error::Io)?;
635        Ok(root)
636    }
637
638    /// What `sync.status` answers with.
639    pub async fn status(&self, root: &Path) -> SyncStatus {
640        let bridge = self.bridge().ok();
641        let bridge_available = match &bridge {
642            Some(bridge) => bridge.status().await.is_ok(),
643            None => false,
644        };
645        let answer = match &bridge {
646            Some(_) => self.peers().await.ok(),
647            None => None,
648        };
649        let peers = answer
650            .as_ref()
651            .map_or(0, |p| p.peers.len().saturating_sub(1));
652        let Ok(key) = root.canonicalize() else {
653            return SyncStatus {
654                enabled: false,
655                workspace_id: None,
656                peers,
657                paused: None,
658                last_error: None,
659                bridge_available,
660                topology: proto::Topology::Mesh,
661                embedding: None,
662            };
663        };
664        let synced = self.synced.lock().await;
665        match synced.get(&key) {
666            Some(entry) => SyncStatus {
667                enabled: true,
668                workspace_id: Some(entry.workspace_id),
669                peers,
670                paused: entry.paused.map(|r| format!("{r:?}")),
671                last_error: entry.last_error.clone(),
672                bridge_available,
673                topology: topology::topology(
674                    answer
675                        .as_ref()
676                        .and_then(|p| p.roles_for(entry.workspace_id)),
677                ),
678                embedding: self.embedding_progress(&key).await,
679            },
680            None => {
681                drop(synced);
682                // Asked to sync while the bridge was away: it is enabled, waiting for the
683                // bridge, and `bridge_available` says why nothing moves.
684                let waiting = self.pending.lock().await.get(&key).copied();
685                SyncStatus {
686                    enabled: waiting.is_some(),
687                    workspace_id: waiting,
688                    peers,
689                    paused: None,
690                    last_error: None,
691                    bridge_available,
692                    topology: proto::Topology::Mesh,
693                    embedding: None,
694                }
695            }
696        }
697    }
698
699    /// How far `key`'s embedding has got, when this host embeds it. Reads the open
700    /// workspace's index; never opens one, so a status call stays cheap.
701    async fn embedding_progress(&self, key: &Path) -> Option<proto::EmbeddingProgress> {
702        let host = self.host.get()?;
703        let backend = host.open_backend(key)?;
704        let state = backend.state();
705        state.embedder()?;
706        let info = state.db_info().ok()?;
707        Some(proto::EmbeddingProgress {
708            vectors: info.vector_count,
709            pending: info.pending_count,
710            running: self.embed_running.lock().await.contains(key),
711        })
712    }
713
714    /// Bring the replica's view of the files up to date.
715    ///
716    /// Called straight after the app server's own writes, and by the watcher for everything
717    /// else. A scan that finds nothing is cheap; a scan that is skipped loses an edit until
718    /// the next one.
719    pub async fn scan(&self, root: &Path) -> Result<()> {
720        let Ok(key) = root.canonicalize() else {
721            return Ok(());
722        };
723        let replica = {
724            let synced = self.synced.lock().await;
725            match synced.get(&key) {
726                Some(entry) => Arc::clone(&entry.replica),
727                None => return Ok(()),
728            }
729        };
730        let (outcome, updates) = {
731            let mut replica = replica.lock().await;
732            let outcome = replica.scan().map_err(|e| Error::Sync(e.to_string()))?;
733            // What the scan committed, as path updates, so it can go out on the sessions
734            // that are already open. This is the whole point of the exact-path scan in
735            // `handlers.rs`: a write through the server reaches a peer live, not at the next
736            // dial.
737            let updates = match &outcome {
738                ScanOutcome::Scanned(report) => {
739                    // The unit of replication is the path state (design doc section 2.4):
740                    // the `seen` a peer prunes against is the *path's* version vector - the
741                    // context the recorded entry was built on plus its own dot - never the
742                    // whole replica vv. Sending the global vv here made receivers prune
743                    // causally-related versions they still held: a conflict copy on the
744                    // first remote edit, and tombstones pruned so deletes stopped
745                    // propagating (issue #157).
746                    report
747                        .recorded
748                        .iter()
749                        .map(|entry| PathUpdate {
750                            path: entry.path.clone(),
751                            versions: vec![entry.clone()],
752                            seen: {
753                                let mut seen = entry.context.clone();
754                                seen.add_dot(&entry.dot);
755                                seen
756                            },
757                        })
758                        .collect()
759                }
760                ScanOutcome::Paused(_) => Vec::new(),
761            };
762            (outcome, updates)
763        };
764        {
765            let mut synced = self.synced.lock().await;
766            if let Some(entry) = synced.get_mut(&key) {
767                entry.paused = match outcome {
768                    ScanOutcome::Paused(reason) => Some(reason),
769                    ScanOutcome::Scanned(_) => None,
770                };
771            }
772        }
773        // After the map is unlocked: a push awaits every peer, and the table must not be
774        // held behind a slow one.
775        self.after_commit(&key, updates).await;
776        Ok(())
777    }
778
779    /// Bring the search index up to date with files a session has written.
780    ///
781    /// A replication session moves files, not index rows: the index belongs to the app
782    /// server, and the sync core is deliberately free of it. So the side that *applied* a
783    /// change re-indexes, which is what makes a file that arrived from a peer searchable
784    /// without a manual reindex — the claim the whole architecture rests on.
785    ///
786    /// Incremental, not a full rebuild: the workspace's mtime/size stamps mean only what
787    /// actually changed is read back. A file the session wrote has a new stamp, so it is
788    /// picked up; one that was already indexed and untouched is skipped.
789    ///
790    /// Best-effort by contract: the caller's session has already succeeded, and a failure
791    /// here must not be reported as the session's. It is logged and the next session, local
792    /// edit or explicit `workspace.reindex` tries again.
793    async fn reindex(&self, root: &Path) {
794        let Some(host) = self.host.get() else {
795            return;
796        };
797        let host = Arc::clone(host);
798        let root = root.to_owned();
799        // One sweep at a time: two sessions that finish together would otherwise both walk
800        // the workspace through the same retrieve store.
801        let _serialised = self.reindexing.lock().await;
802        // Opening the workspace is itself blocking work, and it is where the exclusive
803        // retrieve-store lock is taken, so it happens here rather than inside the sweep.
804        let backend = match host.backend_at(&root).await {
805            Ok(backend) => backend,
806            Err(err) => {
807                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
808                return;
809            }
810        };
811        let state = Arc::clone(backend.state());
812        let result = tokio::task::spawn_blocking(move || state.sync_retrieve()).await;
813        match result {
814            Ok(Ok(report)) => {
815                tracing::debug!(root = %root.display(), ?report, "re-indexed after a session");
816            }
817            Ok(Err(err)) => {
818                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
819            }
820            Err(err) => {
821                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
822            }
823        }
824    }
825
826    /// Run an embedding pass over `root` in the background, after the pass already
827    /// waiting for it if there is one (#187).
828    ///
829    /// The pass reads the vector files that arrived, and computes the missing ones this
830    /// host is responsible for: the files whose winning version this replica wrote, and —
831    /// on the primary device — every other one. It never holds up the caller.
832    pub fn request_embed(self: &Arc<Self>, root: &Path) {
833        let Ok(key) = root.canonicalize() else {
834            return;
835        };
836        let runtime = Arc::clone(self);
837        self.spawn_background(async move {
838            if !runtime.embed_wanted.lock().await.insert(key.clone()) {
839                return;
840            }
841            let _one_at_a_time = runtime.embedding.lock().await;
842            // Taken off the list as the pass starts: a change during the pass asks again.
843            runtime.embed_wanted.lock().await.remove(&key);
844            runtime.embed_running.lock().await.insert(key.clone());
845            runtime.embed_pass(&key).await;
846            runtime.embed_running.lock().await.remove(&key);
847        });
848    }
849
850    /// One embedding pass over `key`. Best-effort, like the re-index: logged, never fatal.
851    async fn embed_pass(&self, key: &Path) {
852        let Some(host) = self.host.get() else {
853            return;
854        };
855        let (workspace_id, replica) = {
856            let synced = self.synced.lock().await;
857            match synced.get(key) {
858                Some(entry) => (entry.workspace_id, Arc::clone(&entry.replica)),
859                None => return,
860            }
861        };
862        let backend = match host.backend_at(key).await {
863            Ok(backend) => backend,
864            Err(err) => {
865                tracing::warn!(root = %key.display(), "embedding pass: {err}");
866                return;
867            }
868        };
869        let state = Arc::clone(backend.state());
870        if let Err(err) = state
871            .refresh_embedder_at(self.bridge_endpoint.clone())
872            .await
873        {
874            tracing::warn!(root = %key.display(), "embedding pass: {err}");
875            return;
876        }
877        if state.embedder().is_none() {
878            return;
879        }
880        let primary = self.is_primary(workspace_id).await;
881        // The replica's view, read once: which content each path has, and which of those
882        // versions this replica wrote.
883        let grace_ms = self.backfill_timing().grace.as_millis() as u64;
884        let now_ms = std::time::SystemTime::now()
885            .duration_since(std::time::UNIX_EPOCH)
886            .map_or(0, |d| d.as_millis() as u64);
887        let (mine, settled, live) = {
888            let replica = replica.lock().await;
889            let me = replica.replica_id();
890            let states = match replica.states() {
891                Ok(states) => states,
892                Err(err) => {
893                    tracing::warn!(root = %key.display(), "embedding pass: {err}");
894                    return;
895                }
896            };
897            let mut mine = HashMap::new();
898            // Other devices' versions old enough for the primary to fill in.
899            let mut settled = HashMap::new();
900            let mut live = std::collections::HashSet::new();
901            for (path, path_state) in states {
902                let winner = path_state.winner();
903                let Some(hash) = winner.content.hash() else {
904                    continue;
905                };
906                let hash = hash.to_hex();
907                if winner.dot.replica == me {
908                    mine.insert(path, hash.clone());
909                } else if now_ms.saturating_sub(winner.hlc.wall_ms) >= grace_ms {
910                    settled.insert(path, hash.clone());
911                }
912                live.insert(hash);
913            }
914            (mine, settled, live)
915        };
916        let root = key.to_owned();
917        let result = tokio::task::spawn_blocking(move || {
918            // The author embeds its own versions at once; the primary device fills in other
919            // devices' versions once they have settled, racing nobody.
920            let policy = |rel: &str, hash: &str| {
921                mine.get(rel).is_some_and(|h| h.as_str() == hash)
922                    || (primary && settled.get(rel).is_some_and(|h| h.as_str() == hash))
923            };
924            let embedded = state.embed_pending_with(&policy, PASS_BATCH)?;
925            let removed = if primary {
926                state.remove_stale_vectors(&live, STALE_VECTOR_AGE)
927            } else {
928                0
929            };
930            Ok::<_, sapphire_workspace::Error>((embedded, removed))
931        })
932        .await;
933        match result {
934            Ok(Ok((embedded, removed))) if embedded > 0 || removed > 0 => tracing::info!(
935                root = %root.display(),
936                embedded,
937                removed,
938                "embedding pass"
939            ),
940            Ok(Ok(_)) => {}
941            Ok(Err(err)) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
942            Err(err) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
943        }
944    }
945
946    /// Open a session with every peer that will take one.
947    pub async fn sync_now(self: &Arc<Self>, root: &Path) -> Result<()> {
948        let Ok(key) = root.canonicalize() else {
949            return Ok(());
950        };
951        let workspace_id = {
952            let synced = self.synced.lock().await;
953            match synced.get(&key) {
954                Some(entry) => entry.workspace_id,
955                None => return Ok(()),
956            }
957        };
958        let peers = self.peers().await?;
959        let me = self.device_id().await?;
960
961        // The same direction rule the dial loop follows: a host only dials peers with a
962        // greater device id. Dialling the other way from here would reintroduce the
963        // dial-both-ways deadlock the loop was cured of, just on a different trigger.
964        // Inside a star, a pair with neither hub in it does not link at all.
965        let roles = peers.roles_for(workspace_id).cloned();
966        // Every peer at once, so one that is offline costs the others nothing (#228). On
967        // this task, not spawned: cancelling the caller — the server's shutdown aborting the
968        // watcher — must drop every dial with it, since each one pins the runtime.
969        let mut dials: Vec<Dial<'_>> = Vec::new();
970        for peer in peers
971            .peers
972            .iter()
973            .filter(|p| topology::link(me, p.device_id, roles.as_ref()) == topology::Link::Dial)
974            .cloned()
975            .collect::<Vec<_>>()
976        {
977            // A peer that already has a live session is caught up by it: a second exchange
978            // on the same workspace would be redundant, and a second stream between the same
979            // pair is exactly what the tie-break cannot settle.
980            if let Some(live) = self.live_peers(&key).await
981                && live.has(&peer.device_id).await
982            {
983                continue;
984            }
985            // One dial per peer at a time. `sync_now` runs beside the dial loop, and two
986            // overlapping outbound dials to one peer are the one case the tie-break rule
987            // cannot resolve — each end would see its own stream as `outbound`. Whoever
988            // arrives second simply leaves it to the next pass. The table is made here if
989            // the dial loop has not yet: the first dial after enabling must not depend on
990            // a tick that has not fired.
991            let Some((_, table)) = self.live_peers_or_create(&key).await else {
992                continue;
993            };
994            if !table.begin_dial(peer.device_id).await {
995                continue;
996            }
997            let key = &key;
998            dials.push(Box::pin(async move {
999                // A peer that does not host this workspace refuses, which is normal and cheap.
1000                let result = self
1001                    .sync_one(key, &peer.device_id, peer.name.clone(), workspace_id)
1002                    .await;
1003                table.end_dial(peer.device_id).await;
1004                if let Err(err) = result {
1005                    tracing::debug!(peer = %peer.name, "no session: {err}");
1006                }
1007            }));
1008        }
1009        join_all(dials).await;
1010        Ok(())
1011    }
1012
1013    /// Dial one peer and adopt the stream as a live session.
1014    ///
1015    /// The one-shot path (`run_session`) would close the stream the moment the exchange
1016    /// finished — while the accepting peer keeps its half open and pushes into it, a session
1017    /// that dies with nothing reading it. So the dialled side is adopted live too, and both
1018    /// ends of a pair agree that a session stays open.
1019    async fn sync_one(
1020        self: &Arc<Self>,
1021        key: &Path,
1022        device: &GrainId,
1023        name: String,
1024        workspace_id: GrainId,
1025    ) -> Result<()> {
1026        let stream = self.open_stream(workspace_id, *device).await?;
1027        self.adopt_live_session(stream, key, *device, workspace_id)
1028            .await
1029            .map_err(|err| {
1030                tracing::warn!(peer = %name, "a live session could not be started: {err}");
1031                err
1032            })
1033    }
1034
1035    /// Ask the bridge for a stream to `device`, giving up after `open_timeout`.
1036    async fn open_stream(
1037        &self,
1038        workspace_id: GrainId,
1039        device: GrainId,
1040    ) -> Result<sapphire_ipc::RawStream> {
1041        match tokio::time::timeout(
1042            self.open_timeout,
1043            self.bridge()?.open_stream(workspace_id, device),
1044        )
1045        .await
1046        {
1047            Ok(opened) => Ok(opened?),
1048            Err(_) => Err(Error::Bridge(format!(
1049                "no stream to {device} within {} s; it may be offline",
1050                self.open_timeout.as_secs_f32()
1051            ))),
1052        }
1053    }
1054
1055    /// Answer the bridge's announcements, following the connection across reconnections
1056    /// (#244). Runs until the task is aborted.
1057    pub async fn run(self: Arc<Self>) -> Result<()> {
1058        let mut slot = self.bridge.subscribe();
1059        loop {
1060            let bridge = slot.borrow_and_update().clone();
1061            match bridge {
1062                Some(bridge) if !bridge.is_closed() => self.answer(&bridge, &mut slot).await,
1063                // Wait for a connection. The runtime holds the sender, so this never ends.
1064                _ => {
1065                    if slot.changed().await.is_err() {
1066                        return Ok(());
1067                    }
1068                }
1069            }
1070        }
1071    }
1072
1073    /// Answer `bridge`'s announcements until it closes or another connection replaces it.
1074    async fn answer(
1075        self: &Arc<Self>,
1076        bridge: &Arc<BridgeClient>,
1077        slot: &mut tokio::sync::watch::Receiver<Option<Arc<BridgeClient>>>,
1078    ) {
1079        let early = self
1080            .subscribed
1081            .lock()
1082            .expect("subscribed")
1083            .take_if(|(b, _)| Arc::ptr_eq(b, bridge));
1084        let mut incoming = match early {
1085            Some((_, incoming)) => incoming,
1086            None => bridge.incoming(),
1087        };
1088        loop {
1089            let received = tokio::select! {
1090                received = incoming.recv() => received,
1091                () = bridge.closed() => return,
1092                _ = slot.changed() => return,
1093            };
1094            let announcement = match received {
1095                Ok(a) => a,
1096                Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
1097                    tracing::warn!(missed = n, "fell behind on bridge announcements");
1098                    continue;
1099                }
1100                Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1101            };
1102
1103            let found = {
1104                let synced = self.synced.lock().await;
1105                synced
1106                    .iter()
1107                    .find(|(_, s)| s.workspace_id == announcement.workspace_id)
1108                    .map(|(root, s)| (root.clone(), Arc::clone(&s.replica)))
1109            };
1110            // `replica` is still what the accepted session shares, and `root` is where its
1111            // files and its index live.
1112            let Some((root, replica)) = found else {
1113                // The bridge routed to us for a workspace we no longer hold. Not fatal:
1114                // ignore it and let the ticket expire.
1115                tracing::debug!(
1116                    workspace = %announcement.workspace_id,
1117                    "an announcement for a workspace this server does not hold"
1118                );
1119                continue;
1120            };
1121
1122            let bridge = Arc::clone(bridge);
1123            let workspace_id = announcement.workspace_id;
1124            let peer = announcement.peer_device_id;
1125            let driver = Arc::clone(self);
1126            self.spawn_background(async move {
1127                let stream = match bridge.accept_stream(announcement.ticket).await {
1128                    Ok(stream) => stream,
1129                    Err(err) => {
1130                        tracing::warn!("could not claim an announced stream: {err}");
1131                        return;
1132                    }
1133                };
1134                if driver.should_skip(workspace_id, peer).await {
1135                    tracing::debug!(%peer, "dropped an inbound session: both ends are outside this workspace's star");
1136                    return;
1137                }
1138                // Live, not one-shot: the peer that dialled is keeping this session open and
1139                // pushing what it commits, so this side has to keep reading. A session that
1140                // closed after the exchange would leave the dialler pushing into a stream
1141                // nobody reads.
1142                let (_, session) = match sapphire_framework_session::open_live_session(
1143                    stream,
1144                    Arc::clone(&replica),
1145                    workspace_id,
1146                )
1147                .await
1148                {
1149                    Ok(started) => started,
1150                    Err(err) => {
1151                        tracing::warn!("an inbound session failed: {err}");
1152                        return;
1153                    }
1154                };
1155                let Some((key, table)) = driver.live_peers_or_create(&root).await else {
1156                    return;
1157                };
1158                // The peer dialled: this stream survives only if the *peer* has the lower
1159                // device id. A stream that loses is dropped, which closes it, and the
1160                // connection ends tidily rather than being left half-read.
1161                let Some(updates) = table.insert(peer, session, false).await else {
1162                    return;
1163                };
1164                table.catch_up(peer, &replica).await;
1165                // This is the receiving side: the files are on disk now and nobody else will
1166                // index them.
1167                driver.reindex(&key).await;
1168                driver.request_embed(&key);
1169                driver.spawn_reader(table, updates, key, peer);
1170            });
1171        }
1172    }
1173
1174    /// Make the first attempt to reach the bridge, when the runtime has a
1175    /// [`BridgeConnector`] and is not connected yet.
1176    ///
1177    /// No bridge running is not an error: the server serves without sync, and
1178    /// [`keep_connected`](Self::keep_connected) picks the bridge up once it starts. Any
1179    /// other failure — a bridge of another API version, say — is, and stops the start.
1180    pub async fn connect(self: &Arc<Self>) -> Result<()> {
1181        let Some(connector) = &self.connector else {
1182            return Ok(());
1183        };
1184        if self.is_connected() {
1185            return Ok(());
1186        }
1187        match connector.connect().await? {
1188            Some(client) => self.attach(client).await,
1189            None => tracing::warn!(
1190                "no sapphire-bridge is running; serving without sync until it starts"
1191            ),
1192        }
1193        Ok(())
1194    }
1195
1196    /// Stay connected to the bridge: wait for the connection to close, then reach the
1197    /// bridge again with the runtime's [`BridgeConnector`], backing off from
1198    /// [`RECONNECT_BACKOFF`] to [`RECONNECT_BACKOFF_MAX`] while it cannot (#244).
1199    ///
1200    /// Runs until the task is aborted, which is what the app server's shutdown does with
1201    /// it. Without a connector it only marks the runtime disconnected once the connection
1202    /// closes, and returns.
1203    pub async fn keep_connected(self: Arc<Self>) {
1204        let mut attempt = 0;
1205        loop {
1206            if let Ok(bridge) = self.bridge() {
1207                bridge.closed().await;
1208                tracing::warn!("lost the connection to sapphire-bridge; reconnecting");
1209                self.bridge.send_if_modified(|slot| {
1210                    let same = slot.as_ref().is_some_and(|b| Arc::ptr_eq(b, &bridge));
1211                    if same {
1212                        *slot = None;
1213                    }
1214                    same
1215                });
1216                attempt = 0;
1217                continue;
1218            }
1219            let Some(connector) = &self.connector else {
1220                return;
1221            };
1222            match connector.connect().await {
1223                Ok(Some(client)) => {
1224                    tracing::info!("connected to sapphire-bridge");
1225                    self.attach(client).await;
1226                    attempt = 0;
1227                    continue;
1228                }
1229                Ok(None) => tracing::debug!("no sapphire-bridge is running yet"),
1230                // Logged once per outage, not once per attempt.
1231                Err(err) if attempt == 0 => {
1232                    tracing::warn!("could not connect to sapphire-bridge: {err}")
1233                }
1234                Err(err) => tracing::debug!("could not connect to sapphire-bridge: {err}"),
1235            }
1236            tokio::time::sleep(reconnect_backoff(attempt)).await;
1237            attempt = attempt.saturating_add(1);
1238        }
1239    }
1240
1241    /// Start using `client`, and bring the bridge up to date: it knows nothing of this
1242    /// server yet, whether it has just started or this server has.
1243    async fn attach(self: &Arc<Self>, client: BridgeClient) {
1244        // Swapped under the pending lock, so an `enable` racing this either sees the new
1245        // connection or leaves its workspace for the drain below.
1246        let client = Arc::new(client);
1247        *self.subscribed.lock().expect("subscribed") =
1248            Some((Arc::clone(&client), client.incoming()));
1249        let waiting = {
1250            let mut pending = self.pending.lock().await;
1251            self.bridge.send_replace(Some(client));
1252            std::mem::take(&mut *pending)
1253        };
1254        // What was synced before the connection was lost.
1255        if !self.synced.lock().await.is_empty()
1256            && let Err(err) = self.reregister().await
1257        {
1258            tracing::warn!("could not register with sapphire-bridge: {err}");
1259        }
1260        // What was asked for while it was away. `enable` registers and dials each one.
1261        for root in waiting.into_keys() {
1262            if let Err(err) = self.enable(&root).await {
1263                tracing::warn!(root = %root.display(), "could not start sync: {err}");
1264            }
1265        }
1266    }
1267
1268    /// Every synced root, for the watcher.
1269    pub async fn roots(&self) -> Vec<PathBuf> {
1270        self.synced.lock().await.keys().cloned().collect()
1271    }
1272
1273    /// The live session table for `root`, if this runtime syncs it.
1274    ///
1275    /// `None` is not an error: a dial, a push or a drop for a workspace this server does not
1276    /// hold is a no-op, and every caller below treats it that way.
1277    async fn live_peers(&self, root: &Path) -> Option<Arc<LivePeers>> {
1278        let key = root.canonicalize().ok()?;
1279        self.live.lock().await.get(&key).cloned()
1280    }
1281
1282    /// The live session table for `root`, creating it if this runtime syncs it.
1283    ///
1284    /// The two locks are taken one after the other, never nested: `synced` to decide whether
1285    /// the workspace is ours at all, then `live` to get or make its table.
1286    async fn live_peers_or_create(&self, root: &Path) -> Option<(PathBuf, Arc<LivePeers>)> {
1287        let key = root.canonicalize().ok()?;
1288        if !self.synced.lock().await.contains_key(&key) {
1289            return None;
1290        }
1291        // A table is built by whoever walks the workspace first, and it needs this host's
1292        // device id — the tie-break between two simultaneous streams is decided from it.
1293        let me = self.device_id().await.ok()?;
1294        let mut live = self.live.lock().await;
1295        let peers = Arc::clone(
1296            live.entry(key.clone())
1297                .or_insert_with(|| Arc::new(LivePeers::new(me))),
1298        );
1299        Some((key, peers))
1300    }
1301
1302    /// Push `updates` to every open session for `root`.
1303    ///
1304    /// Called with what a scan recorded. `from` is always `None` here: a local commit came
1305    /// from this host, so no session is excluded. The forwarding path, which does name a
1306    /// source, is [`LivePeers::fan_out`](live::LivePeers::fan_out) called directly by the
1307    /// reader side of a session.
1308    ///
1309    /// Best-effort by contract: the commit already happened, and a peer that is gone is the
1310    /// dialer's problem, not the writer's.
1311    pub async fn after_commit(&self, root: &Path, updates: Vec<PathUpdate>) {
1312        if updates.is_empty() {
1313            return;
1314        }
1315        if let Some(peers) = self.live_peers(root).await {
1316            peers.fan_out(&updates, None).await;
1317        }
1318    }
1319
1320    /// The devices with an open live session for `root`.
1321    ///
1322    /// Used by tests to wait for the dialer; empty for a workspace this server does not hold.
1323    pub async fn live_session_devices(&self, root: &Path) -> Vec<GrainId> {
1324        match self.live_peers(root).await {
1325            Some(peers) => peers.devices().await,
1326            None => Vec::new(),
1327        }
1328    }
1329
1330    /// End the first dials [`enable`](SyncRuntime::enable) left running, waiting until
1331    /// they are gone.
1332    ///
1333    /// For the app server's shutdown: each one pins the runtime, and with it the replica
1334    /// stores, which are redb databases that stay locked — a dial outliving its server
1335    /// would lock the next one out.
1336    pub async fn stop_dials(&self) {
1337        let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1338        dials.abort_all();
1339        while dials.join_next().await.is_some() {}
1340    }
1341
1342    /// End the runtime's background work — inbound sessions, live-session readers and
1343    /// embedding passes — and wait until it has let go.
1344    ///
1345    /// The server's shutdown calls this, so a server restarted in the same process can open
1346    /// the replica stores again at once rather than whenever the last straggler finished
1347    /// (#235). An embedding pass already inside its blocking section runs that section to
1348    /// the end; it holds the workspace's index, never a replica.
1349    pub async fn stop_background(&self) {
1350        let mut background =
1351            std::mem::take(&mut *self.background.lock().expect("background tasks"));
1352        background.abort_all();
1353        while background.join_next().await.is_some() {}
1354    }
1355
1356    /// Wait for the first dials [`enable`](SyncRuntime::enable) left running to finish on
1357    /// their own: for a test that must not race them.
1358    #[cfg(test)]
1359    async fn settle_dials(&self) {
1360        let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1361        while dials.join_next().await.is_some() {}
1362    }
1363
1364    /// Close every open live session, as if the connections had been cut.
1365    ///
1366    /// Called for a workspace being disabled, and by tests that want to watch the dialer
1367    /// rebuild what was dropped.
1368    pub async fn drop_connections(&self) {
1369        let tables: Vec<Arc<LivePeers>> = self.live.lock().await.values().cloned().collect();
1370        for peers in tables {
1371            peers.drop_connections().await;
1372        }
1373    }
1374
1375    /// Keep a live session open to every peer, for every synced workspace.
1376    ///
1377    /// One walk per [`DIAL_INTERVAL`](live::DIAL_INTERVAL): for each synced workspace, ask
1378    /// the bridge who is in the workgroup, and open a session to every peer that is connected
1379    /// and does not already have one.
1380    ///
1381    /// A peer that cannot be reached is retried on a per-peer exponential backoff capped at
1382    /// [`DIAL_BACKOFF_MAX`] — one failed attempt per walk, not one per spin — and the counter
1383    /// resets when a session opens. An unreachable peer is therefore no obstacle to the
1384    /// others: its dial fails, the walk moves on, and the next workspace is dialled.
1385    ///
1386    /// Runs until the task is aborted, which is what the app server's shutdown does with it.
1387    /// It is allowed to fail like [`run`](SyncRuntime::run): only sync stops, and the app
1388    /// server keeps serving files (spec §10).
1389    pub async fn dial_loop(self: Arc<Self>) -> Result<()> {
1390        let mut tick = tokio::time::interval(live::DIAL_INTERVAL);
1391        // A missed tick is a late walk, not a burst of them: the interval is a pace, not a
1392        // quota to make up.
1393        tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1394        let mut waiting: HashMap<GrainId, std::time::Instant> = HashMap::new();
1395        let mut failures: HashMap<GrainId, u32> = HashMap::new();
1396        // The workspaces this host was the primary device of at the last walk.
1397        let mut primary_of: std::collections::HashSet<GrainId> = Default::default();
1398        // When the primary device last ran a backfill pass with nothing prompting it.
1399        let mut last_backfill = std::time::Instant::now();
1400        loop {
1401            tick.tick().await;
1402            let roots = self.roots().await;
1403            if roots.is_empty() {
1404                continue;
1405            }
1406            let peers = match self.peers().await {
1407                Ok(peers) => peers,
1408                // The bridge is down or has no workgroup: nothing to dial, and the next walk
1409                // asks again. Not an error for the caller — the app server is still serving.
1410                Err(err) => {
1411                    tracing::debug!("no peers to dial: {err}");
1412                    continue;
1413                }
1414            };
1415            let me = match self.device_id().await {
1416                Ok(me) => me,
1417                Err(err) => {
1418                    tracing::warn!("no device id, so nobody to dial: {err}");
1419                    continue;
1420                }
1421            };
1422            // A host that has just become a workspace's primary device owes it the vectors
1423            // nobody else computes (#187): files that arrived before the election settled
1424            // would otherwise wait for the next change.
1425            let ids: Vec<(PathBuf, GrainId)> = self
1426                .synced
1427                .lock()
1428                .await
1429                .iter()
1430                .map(|(root, entry)| (root.clone(), entry.workspace_id))
1431                .collect();
1432            // And every so often anyway: an author that never embedded leaves files only the
1433            // primary will fill in, and nothing else would prompt it (#188).
1434            let backfill_due = last_backfill.elapsed() >= self.backfill_timing().interval;
1435            if backfill_due {
1436                last_backfill = std::time::Instant::now();
1437            }
1438            for (root, workspace_id) in ids {
1439                let primary = peers
1440                    .roles_for(workspace_id)
1441                    .is_some_and(|r| r.primary == Some(me));
1442                if !primary {
1443                    primary_of.remove(&workspace_id);
1444                } else if primary_of.insert(workspace_id) || backfill_due {
1445                    self.request_embed(&root);
1446                }
1447            }
1448            let now = std::time::Instant::now();
1449            // Dial only peers whose device id is greater than ours. Every device dialling
1450            // every other can deadlock: two hosts that dial each other at once each hold
1451            // their replica's lock across their own exchange and wait for the other's
1452            // hello, which the other's dialler holds the lock for. With one direction per
1453            // pair the wait chain is a DAG — a higher-id host never dials, so it never
1454            // holds its lock out while a lower-id host's exchange needs it. The lower host
1455            // dials within `DIAL_INTERVAL`, and the exchange is symmetric, so nothing is
1456            // lost; the tie-break is left as the safety net it is.
1457            for root in roots {
1458                let Some((key, table)) = self.live_peers_or_create(&root).await else {
1459                    continue;
1460                };
1461                let workspace_id = {
1462                    let synced = self.synced.lock().await;
1463                    match synced.get(&key) {
1464                        Some(entry) => entry.workspace_id,
1465                        None => continue,
1466                    }
1467                };
1468                let roles = peers.roles_for(workspace_id);
1469                // Entering a star closes what it no longer needs. The hubs carry it all —
1470                // once one of them is reachable from here (#192).
1471                if topology::may_close_skipped(me, roles, &table.devices().await) {
1472                    table
1473                        .retain(|d| topology::link(me, *d, roles) != topology::Link::Skip)
1474                        .await;
1475                }
1476                for peer in peers.peers.iter().filter(|p| {
1477                    p.connected && topology::link(me, p.device_id, roles) == topology::Link::Dial
1478                }) {
1479                    let device = peer.device_id;
1480                    if table.has(&device).await {
1481                        // Already talking; a redial would drop a working session.
1482                        failures.remove(&device);
1483                        waiting.remove(&device);
1484                        continue;
1485                    }
1486                    if let Some(until) = waiting.get(&device)
1487                        && now < *until
1488                    {
1489                        // Backing off after a failure; the walk skips it until then.
1490                        continue;
1491                    }
1492                    // The same claim `sync_now` takes: this walk and it can overlap, and a
1493                    // double dial in one direction is the one race the tie-break cannot
1494                    // settle. Skipping is free — the next pass, or `sync_now` itself, dials.
1495                    if !table.begin_dial(device).await {
1496                        continue;
1497                    }
1498                    let opened = self.open_stream(workspace_id, device).await;
1499                    if opened.is_err() {
1500                        table.end_dial(device).await;
1501                    }
1502                    match opened {
1503                        Ok(stream) => {
1504                            let adopted = self
1505                                .adopt_live_session(stream, &root, device, workspace_id)
1506                                .await;
1507                            // The claim is out either way: the attempt is over, and a
1508                            // session that died mid-exchange is re-dialled next pass.
1509                            table.end_dial(device).await;
1510                            match adopted {
1511                                Ok(()) => {
1512                                    failures.remove(&device);
1513                                    waiting.remove(&device);
1514                                }
1515                                Err(err) => {
1516                                    tracing::debug!(
1517                                        peer = %peer.name,
1518                                        err = %err,
1519                                        "a live session could not be started"
1520                                    );
1521                                    let count = failures.entry(device).or_insert(0);
1522                                    *count = count.saturating_add(1);
1523                                    waiting.insert(device, now + dial_backoff(*count));
1524                                }
1525                            }
1526                        }
1527                        Err(err) => {
1528                            // Normal and cheap: a peer that does not host this workspace
1529                            // refuses, and one that is unreachable cannot be opened.
1530                            tracing::debug!("no stream to a peer: {err}");
1531                            let count = failures.entry(device).or_insert(0);
1532                            *count = count.saturating_add(1);
1533                            waiting.insert(device, now + dial_backoff(*count));
1534                        }
1535                    }
1536                }
1537            }
1538        }
1539    }
1540
1541    /// Take ownership of an opened stream: run the exchange, then keep it live.
1542    ///
1543    /// The session is stored under `device` and its reader pumped until the stream ends. A
1544    /// half-open or refused stream is not stored; the dialer retries it with backoff.
1545    async fn adopt_live_session(
1546        self: &Arc<Self>,
1547        stream: sapphire_ipc::RawStream,
1548        root: &Path,
1549        device: GrainId,
1550        workspace_id: GrainId,
1551    ) -> Result<()> {
1552        let replica = self.replica_of(root).await?;
1553        let (_, session) = sapphire_framework_session::open_live_session(
1554            stream,
1555            Arc::clone(&replica),
1556            workspace_id,
1557        )
1558        .await
1559        .map_err(|e| Error::Sync(e.to_string()))?;
1560        let Some((key, table)) = self.live_peers_or_create(root).await else {
1561            return Ok(());
1562        };
1563        // This stream was dialled by us, so it is the one the pair keeps if this host's
1564        // device id is the lower of the two. If it lost the tie-break the session is
1565        // dropped here, which closes the stream, and the surviving one is already in the
1566        // table.
1567        let Some(updates) = table.insert(device, session, true).await else {
1568            return Ok(());
1569        };
1570        table.catch_up(device, &replica).await;
1571        // The session wrote files during the exchange; the index has to catch up before the
1572        // workspace is searchable again.
1573        self.reindex(&key).await;
1574        self.request_embed(&key);
1575        self.spawn_reader(Arc::clone(&table), updates, key, device);
1576        Ok(())
1577    }
1578
1579    /// Forward what one session receives, and re-index it.
1580    ///
1581    /// This is the loop that makes A → S → B work: a batch that arrived on S's session with A
1582    /// is applied (the session crate does that before publishing it), the workspace behind
1583    /// `key` is re-indexed, and the batch is handed to every *other* session whose peer lacks
1584    /// it — `from` is the device it came from, so it never goes back to A.
1585    fn spawn_reader(
1586        self: &Arc<Self>,
1587        table: Arc<LivePeers>,
1588        mut updates: broadcast::Receiver<Vec<PathUpdate>>,
1589        key: PathBuf,
1590        from: GrainId,
1591    ) {
1592        let driver = Arc::clone(self);
1593        self.spawn_background(async move {
1594            loop {
1595                match updates.recv().await {
1596                    Ok(batch) => {
1597                        // The files are on disk now (the session guarantees it before
1598                        // publishing), so the index is behind them until the sweep below.
1599                        driver.reindex(&key).await;
1600                        driver.request_embed(&key);
1601                        table.fan_out(&batch, Some(from)).await;
1602                    }
1603                    // A burst bigger than the channel: the newest is kept and the gap is
1604                    // covered by the next session's exchange. Losing a batch here cannot lose
1605                    // an entry — the peer's vector does not cover it, so the next dial or push
1606                    // carries it again.
1607                    Err(broadcast::error::RecvError::Lagged(missed)) => {
1608                        tracing::debug!(
1609                            device = %from,
1610                            missed,
1611                            "fell behind on a live session; the next exchange catches up"
1612                        );
1613                    }
1614                    // The session closed. Its table entry is reaped by the next fan-out or
1615                    // dial pass, and the dialer opens a fresh one.
1616                    Err(broadcast::error::RecvError::Closed) => {
1617                        return;
1618                    }
1619                }
1620            }
1621        });
1622    }
1623
1624    /// The replica of a synced `root`.
1625    async fn replica_of(&self, root: &Path) -> Result<Arc<Mutex<Replica>>> {
1626        let key = root.canonicalize().map_err(Error::Io)?;
1627        let synced = self.synced.lock().await;
1628        match synced.get(&key) {
1629            Some(entry) => Ok(Arc::clone(&entry.replica)),
1630            None => Err(Error::UnknownWorkspace(key, self.ctx.app_name)),
1631        }
1632    }
1633
1634    /// Watch the synced roots and, on a debounced report, scan and dial.
1635    ///
1636    /// This is the safety net for everything the app server did not do itself — a file
1637    /// edited in an editor, a `git checkout`. The exact path is [`scan`] called from
1638    /// `handlers.rs` right after a write the server made. A scan that finds nothing is
1639    /// cheap; a scan that is skipped loses an edit until the next one.
1640    ///
1641    /// Returns when the reporting channel closes — that is, when the [`Watcher`] this
1642    /// runtime holds is dropped, which happens when the runtime itself is.
1643    ///
1644    /// [`scan`]: SyncRuntime::scan
1645    pub async fn watch_changes(self: Arc<Self>) -> Result<()> {
1646        let (tx, mut rx) = mpsc::channel(64);
1647        let watcher = Arc::new(Watcher::start(self.roots().await, tx)?);
1648        let _ = self.watcher.set(Arc::clone(&watcher));
1649
1650        // A root enabled between `roots()` above and the `set` just now was not in the
1651        // starting set, and `enable` could not reach a watcher that did not exist yet.
1652        // Watching the current set again is idempotent on `notify`'s side.
1653        for root in self.roots().await {
1654            if let Err(err) = watcher.watch(&root) {
1655                tracing::warn!(root = %root.display(), "could not watch: {err}");
1656            }
1657        }
1658
1659        while let Some(root) = rx.recv().await {
1660            // Logged, not fatal: a failed scan or dial leaves the replica behind, and the
1661            // next report tries again. Neither may take the app server down (spec §10).
1662            if let Err(err) = self.scan(&root).await {
1663                tracing::warn!(root = %root.display(), "scan after a local edit failed: {err}");
1664            }
1665            self.request_embed(&root);
1666            if let Err(err) = self.sync_now(&root).await {
1667                tracing::warn!(root = %root.display(), "dial after a local edit failed: {err}");
1668            }
1669        }
1670        Ok(())
1671    }
1672
1673    /// Tell the bridge the complete current set. A registration is not a delta.
1674    async fn reregister(&self) -> Result<()> {
1675        let workspaces: Vec<WorkspaceRegistration> = {
1676            let synced = self.synced.lock().await;
1677            synced
1678                .iter()
1679                .map(|(root, entry)| WorkspaceRegistration {
1680                    workspace_id: entry.workspace_id,
1681                    root: root.clone(),
1682                })
1683                .collect()
1684        };
1685        self.bridge()?
1686            .register(RegisterParams {
1687                app_name: self.ctx.app_name.to_owned(),
1688                exe_path: self.exe_path.clone(),
1689                managed_by: wire_managed_by(self.managed_by),
1690                workspaces,
1691            })
1692            .await
1693            .map_err(|e| Error::Bridge(e.to_string()))?;
1694        Ok(())
1695    }
1696
1697    /// Whether the last roles the bridge reported say this pair does not link.
1698    ///
1699    /// Reads the cached device id only: asking the bridge here would register with an empty
1700    /// workspace list, racing `enable`'s registration. `false` (accept) when it is unknown.
1701    async fn should_skip(&self, workspace_id: GrainId, peer: GrainId) -> bool {
1702        let Some(&me) = self.device_id.get() else {
1703            return false;
1704        };
1705        let cached = self.last_peers.lock().await;
1706        let roles = cached.as_ref().and_then(|p| p.roles_for(workspace_id));
1707        topology::link(me, peer, roles) == topology::Link::Skip
1708    }
1709
1710    /// Whether this host is `workspace_id`'s primary device, by the last roles the
1711    /// bridge reported. `false` when the bridge has not answered yet, or when this host's
1712    /// device id is not known yet (no workspace was enabled). Reads the cached id only:
1713    /// asking the bridge here would register with an empty workspace list, racing
1714    /// `enable`'s registration.
1715    pub async fn is_primary(&self, workspace_id: GrainId) -> bool {
1716        let Some(&me) = self.device_id.get() else {
1717            return false;
1718        };
1719        let cached = self.last_peers.lock().await;
1720        cached
1721            .as_ref()
1722            .and_then(|p| p.roles_for(workspace_id))
1723            .is_some_and(|r| r.primary == Some(me))
1724    }
1725
1726    /// This host's device id, once the bridge has told it (#250).
1727    ///
1728    /// `None` until this runtime first registered — no workspace was enabled yet, or the
1729    /// bridge was not connected when one was. Never asks the bridge: a registration made
1730    /// only to learn the id would carry an empty workspace list, and briefly clear this
1731    /// application's routes.
1732    pub fn this_device(&self) -> Option<GrainId> {
1733        self.device_id.get().copied()
1734    }
1735
1736    /// The workgroup's devices, this host's included, and the roles the bridge assigns per
1737    /// workspace (#250). Asked of the bridge each time; [`is_primary`](Self::is_primary)
1738    /// reads the answer afterwards.
1739    ///
1740    /// [`Error::BridgeDisconnected`] while there is no connection to the bridge.
1741    pub async fn peers(&self) -> Result<PeersResult> {
1742        let peers = self
1743            .bridge()?
1744            .peers()
1745            .await
1746            .map_err(|e| Error::Bridge(e.to_string()))?;
1747        *self.last_peers.lock().await = Some(peers.clone());
1748        Ok(peers)
1749    }
1750
1751    /// This host's device id, asked of the bridge once.
1752    async fn device_id(&self) -> Result<GrainId> {
1753        if let Some(id) = self.device_id.get() {
1754            return Ok(*id);
1755        }
1756        let result = self
1757            .bridge()?
1758            .register(RegisterParams {
1759                app_name: self.ctx.app_name.to_owned(),
1760                exe_path: self.exe_path.clone(),
1761                managed_by: wire_managed_by(self.managed_by),
1762                workspaces: Vec::new(),
1763            })
1764            .await
1765            .map_err(|e| Error::Bridge(e.to_string()))?;
1766        let _ = self.device_id.set(result.device_id);
1767        Ok(result.device_id)
1768    }
1769
1770    /// `<workspace cache dir>/sync/`.
1771    fn state_dir(&self, root: &Path) -> Result<PathBuf> {
1772        let workspace = Workspace::from_root(self.ctx, root)?;
1773        Ok(workspace.cache_dir().join("sync"))
1774    }
1775}
1776
1777/// One dial of [`SyncRuntime::sync_now`].
1778type Dial<'a> = std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>>;
1779
1780/// Run `dials` concurrently on the calling task, until every one has finished.
1781async fn join_all(mut dials: Vec<Dial<'_>>) {
1782    std::future::poll_fn(|cx| {
1783        dials.retain_mut(|dial| dial.as_mut().poll(cx).is_pending());
1784        if dials.is_empty() {
1785            std::task::Poll::Ready(())
1786        } else {
1787            std::task::Poll::Pending
1788        }
1789    })
1790    .await
1791}
1792
1793/// The bridge API's copy of how this server was started, for [`RegisterParams`].
1794fn wire_managed_by(managed_by: ManagedBy) -> sapphire_bridge_api::ManagedBy {
1795    match managed_by {
1796        ManagedBy::Service => sapphire_bridge_api::ManagedBy::Service,
1797        ManagedBy::Spawned => sapphire_bridge_api::ManagedBy::Spawned,
1798    }
1799}
1800
1801#[cfg(test)]
1802mod tests {
1803    use super::*;
1804    use crate::sync::testing::StubBridge;
1805    use crate::test_support;
1806    use sapphire_workspace::AppKind;
1807    use std::ffi::OsString;
1808
1809    static CTX: AppContext = AppContext::new("sapphire-synctest");
1810
1811    /// The env vars `CTX.init` reads.
1812    const DIR_VARS: [&str; 3] = [
1813        "SAPPHIRE_SYNCTEST_CACHE_DIR",
1814        "SAPPHIRE_SYNCTEST_DATA_DIR",
1815        "SAPPHIRE_SYNCTEST_CONFIG_DIR",
1816    ];
1817
1818    /// Points the context's directories at the test's scratch tree, and restores the
1819    /// previous values when dropped — including while unwinding from a panic.
1820    struct EnvGuard {
1821        previous: [Option<OsString>; 3],
1822        _lock: std::sync::MutexGuard<'static, ()>,
1823    }
1824
1825    impl Drop for EnvGuard {
1826        fn drop(&mut self) {
1827            // SAFETY: `self._lock` still serialises the environment; it is dropped only
1828            // after this method returns.
1829            for (name, previous) in DIR_VARS.iter().zip(self.previous.iter_mut()) {
1830                match previous.take() {
1831                    Some(value) => unsafe { std::env::set_var(name, value) },
1832                    None => test_support::remove(name),
1833                }
1834            }
1835        }
1836    }
1837
1838    /// Everything one test needs, held for the test's whole body.
1839    ///
1840    /// The context is a `static` shared by this binary, so its directories are resolved from
1841    /// the environment by whichever test initialises it first. The environment lock is
1842    /// therefore held for the whole test, as in the crate's other test modules, and `_tmp`
1843    /// is declared before `_env` so the scratch tree is gone only once the environment no
1844    /// longer points at it.
1845    struct Fixture {
1846        _tmp: tempfile::TempDir,
1847        _env: EnvGuard,
1848        root: PathBuf,
1849        stub: StubBridge,
1850        runtime: Arc<SyncRuntime>,
1851    }
1852
1853    impl Fixture {
1854        /// A second workspace root beside the first, marker directory and all.
1855        fn second_root(&self) -> PathBuf {
1856            let root = self._tmp.path().join("ws2");
1857            std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
1858            root.canonicalize().unwrap()
1859        }
1860    }
1861
1862    /// How long the fixture's runtime waits on a stream to a peer.
1863    const FIXTURE_OPEN_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
1864
1865    /// `n` peers this host dials (a greater device id), as `bridge.peers` lists them.
1866    fn dialled_peers(me: GrainId, n: usize) -> Vec<sapphire_bridge_api::PeerInfo> {
1867        std::iter::repeat_with(GrainId::random)
1868            .filter(|id| *id > me)
1869            .take(n)
1870            .enumerate()
1871            .map(|(i, device_id)| sapphire_bridge_api::PeerInfo {
1872                device_id,
1873                name: format!("offline-{i}"),
1874                node_id: format!("node-{i}"),
1875                connected: false,
1876                priority: 1,
1877                availability: None,
1878            })
1879            .collect()
1880    }
1881
1882    #[tokio::test(flavor = "multi_thread")]
1883    async fn enabling_answers_at_once_with_a_peer_that_never_answers() {
1884        // #228: the first dial waited on a peer that was offline, past the client's limit.
1885        let f = fixture().await;
1886        let _hung = f.stub.hang_streams().await;
1887        f.stub.set_peers(dialled_peers(f.stub.device_id, 1));
1888
1889        let started = std::time::Instant::now();
1890        f.runtime.enable(&f.root).await.unwrap();
1891        assert!(
1892            started.elapsed() < FIXTURE_OPEN_TIMEOUT,
1893            "enable waited {:?} on the dial",
1894            started.elapsed()
1895        );
1896    }
1897
1898    #[tokio::test(flavor = "multi_thread")]
1899    async fn peers_that_never_answer_are_dialled_together_and_given_up_on() {
1900        let f = fixture().await;
1901        let _hung = f.stub.hang_streams().await;
1902        f.runtime.enable(&f.root).await.unwrap();
1903        // The first dial, with nobody to dial yet, must be over before the peers appear:
1904        // one still running would claim them, and `sync_now` would skip them all.
1905        f.runtime.settle_dials().await;
1906        f.stub.set_peers(dialled_peers(f.stub.device_id, 3));
1907
1908        let started = std::time::Instant::now();
1909        f.runtime.sync_now(&f.root).await.unwrap();
1910        let took = started.elapsed();
1911        assert!(took >= FIXTURE_OPEN_TIMEOUT, "gave up early: {took:?}");
1912        assert!(
1913            took < FIXTURE_OPEN_TIMEOUT * 2,
1914            "three dials one after another: {took:?}"
1915        );
1916    }
1917
1918    async fn fixture() -> Fixture {
1919        let (tmp, env, root) = scratch();
1920        let (stub, client) = StubBridge::start_in(tmp.path().join("run")).await;
1921        let runtime = Arc::new(
1922            SyncRuntime::new(&CTX, client, "/bin/true".into(), ManagedBy::Service)
1923                .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
1924        );
1925        Fixture {
1926            _tmp: tmp,
1927            _env: env,
1928            root,
1929            stub,
1930            runtime,
1931        }
1932    }
1933
1934    /// The bridge connections a [`connecting`] fixture's connector hands out, in order;
1935    /// empty is "no bridge is running".
1936    type Bridges = Arc<std::sync::Mutex<std::collections::VecDeque<BridgeClient>>>;
1937
1938    /// A fixture whose runtime reaches the bridge through a connector, and has not yet.
1939    /// `f.stub`'s client is not handed out: push it to the returned queue for that.
1940    async fn connecting() -> (Fixture, Bridges, BridgeClient) {
1941        let (tmp, env, root) = scratch();
1942        let (stub, client) = StubBridge::start_client_in(tmp.path().join("run")).await;
1943        let bridges: Bridges = Arc::default();
1944        let connector = BridgeConnector::new({
1945            let bridges = Arc::clone(&bridges);
1946            move || {
1947                let next = bridges.lock().unwrap().pop_front();
1948                async move { Ok(next) }
1949            }
1950        });
1951        let runtime = Arc::new(
1952            SyncRuntime::connecting(&CTX, connector, "/bin/true".into(), ManagedBy::Service)
1953                .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
1954        );
1955        let f = Fixture {
1956            _tmp: tmp,
1957            _env: env,
1958            root,
1959            stub,
1960            runtime,
1961        };
1962        (f, bridges, client)
1963    }
1964
1965    /// Wait up to ten seconds for `done`.
1966    async fn eventually(what: &str, mut done: impl AsyncFnMut() -> bool) {
1967        let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
1968        while !done().await {
1969            assert!(tokio::time::Instant::now() < deadline, "never: {what}");
1970            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
1971        }
1972    }
1973
1974    #[tokio::test(flavor = "multi_thread")]
1975    async fn a_runtime_started_without_a_bridge_syncs_once_one_starts() {
1976        // #244: a server that started before its bridge never synced until restarted.
1977        let (f, bridges, client) = connecting().await;
1978        f.runtime.connect().await.unwrap();
1979        assert!(!f.runtime.is_connected());
1980
1981        let err = f.runtime.enable(&f.root).await.unwrap_err();
1982        assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
1983        let status = f.runtime.status(&f.root).await;
1984        assert!(status.enabled, "enabled, waiting for the bridge");
1985        assert!(!status.bridge_available);
1986        let id = status.workspace_id.expect("its id");
1987
1988        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
1989        bridges.lock().unwrap().push_back(client);
1990        eventually("the waiting workspace is registered", async || {
1991            f.stub.last_workspaces() == vec![id]
1992        })
1993        .await;
1994        assert_eq!(f.runtime.roots().await, vec![f.root.clone()]);
1995        assert!(f.runtime.status(&f.root).await.bridge_available);
1996        keeping.abort();
1997    }
1998
1999    #[tokio::test(flavor = "multi_thread")]
2000    async fn a_runtime_that_loses_its_bridge_registers_with_the_next_one() {
2001        let (f, bridges, client) = connecting().await;
2002        bridges.lock().unwrap().push_back(client);
2003        f.runtime.connect().await.unwrap();
2004        assert!(f.runtime.is_connected());
2005        let id = f.runtime.enable(&f.root).await.unwrap();
2006        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2007
2008        f.stub.disconnect();
2009        eventually("the runtime notices", async || !f.runtime.is_connected()).await;
2010        let err = f.runtime.workspaces().await.unwrap_err();
2011        assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2012
2013        let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2014        bridges.lock().unwrap().push_back(client);
2015        eventually("the synced workspace is registered again", async || {
2016            next.last_workspaces() == vec![id]
2017        })
2018        .await;
2019        assert!(f.runtime.is_connected());
2020        keeping.abort();
2021    }
2022
2023    #[tokio::test(flavor = "multi_thread")]
2024    async fn this_device_is_known_once_a_workspace_is_enabled_and_never_asked_for() {
2025        let f = fixture().await;
2026        assert_eq!(f.runtime.this_device(), None);
2027        assert!(
2028            f.stub.seen.lock().unwrap().registrations.is_empty(),
2029            "asking must not register"
2030        );
2031
2032        f.runtime.enable(&f.root).await.unwrap();
2033        assert_eq!(f.runtime.this_device(), Some(f.stub.device_id));
2034    }
2035
2036    #[tokio::test(flavor = "multi_thread")]
2037    async fn peers_answers_the_workgroup_from_the_bridge() {
2038        let f = fixture().await;
2039        let listed = dialled_peers(f.stub.device_id, 2);
2040        f.stub.set_peers(listed.clone());
2041
2042        let peers = f.runtime.peers().await.unwrap();
2043        let ids: Vec<_> = peers.peers.iter().map(|p| p.device_id).collect();
2044        assert_eq!(ids, listed.iter().map(|p| p.device_id).collect::<Vec<_>>());
2045    }
2046
2047    #[tokio::test(flavor = "multi_thread")]
2048    async fn peers_says_so_while_the_bridge_is_away() {
2049        let (f, _bridges, _client) = connecting().await;
2050        let err = f.runtime.peers().await.unwrap_err();
2051        assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2052    }
2053
2054    #[tokio::test(flavor = "multi_thread")]
2055    async fn disabling_while_the_bridge_is_away_forgets_the_waiting_workspace() {
2056        let (f, bridges, client) = connecting().await;
2057        f.runtime.enable(&f.root).await.unwrap_err();
2058        f.runtime.disable(&f.root).await.unwrap();
2059        assert!(!f.runtime.status(&f.root).await.enabled);
2060
2061        bridges.lock().unwrap().push_back(client);
2062        f.runtime.connect().await.unwrap();
2063        assert!(f.runtime.roots().await.is_empty());
2064        assert!(f.stub.last_workspaces().is_empty());
2065    }
2066
2067    #[tokio::test(flavor = "multi_thread")]
2068    async fn announcements_are_answered_from_a_reconnected_bridge() {
2069        let (f, bridges, client) = connecting().await;
2070        bridges.lock().unwrap().push_back(client);
2071        f.runtime.connect().await.unwrap();
2072        let id = f.runtime.enable(&f.root).await.unwrap();
2073        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2074        let answering = tokio::spawn(Arc::clone(&f.runtime).run());
2075
2076        f.stub.disconnect();
2077        let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2078        bridges.lock().unwrap().push_back(client);
2079        eventually("reconnected", async || next.last_workspaces() == vec![id]).await;
2080
2081        // The announcement loop follows the new connection: an announcement on it is
2082        // claimed, which with no data endpoint fails — and the loop lives on.
2083        next.announce(id, "ticket").await;
2084        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2085        assert!(!answering.is_finished(), "the announcement loop ended");
2086        answering.abort();
2087        keeping.abort();
2088    }
2089
2090    /// The scratch tree, the environment pointed at it, and a workspace root in it.
2091    fn scratch() -> (tempfile::TempDir, EnvGuard, PathBuf) {
2092        let lock = test_support::lock();
2093        let tmp = tempfile::tempdir().unwrap();
2094        let previous = DIR_VARS.map(std::env::var_os);
2095        // SAFETY (via `test_support::set`): `lock` serialises every read and write of the
2096        // process environment in this test binary, and it is held until `drop` has restored
2097        // the old values.
2098        for (name, dir) in DIR_VARS
2099            .iter()
2100            .zip(["cache", "data", "config"].map(|cat| tmp.path().join(cat)))
2101        {
2102            test_support::set(name, &dir);
2103        }
2104        let env = EnvGuard {
2105            previous,
2106            _lock: lock,
2107        };
2108        CTX.init(AppKind::Server);
2109
2110        let root = tmp.path().join("ws");
2111        std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
2112        let root = root.canonicalize().unwrap();
2113        (tmp, env, root)
2114    }
2115
2116    #[tokio::test(flavor = "multi_thread")]
2117    async fn enabling_registers_the_workspace_with_the_bridge() {
2118        let f = fixture().await;
2119
2120        let id = f.runtime.enable(&f.root).await.unwrap();
2121        assert_eq!(f.stub.last_workspaces(), vec![id]);
2122    }
2123
2124    #[tokio::test(flavor = "multi_thread")]
2125    async fn enabling_twice_is_idempotent_and_keeps_the_same_id() {
2126        let f = fixture().await;
2127
2128        let first = f.runtime.enable(&f.root).await.unwrap();
2129        let second = f.runtime.enable(&f.root).await.unwrap();
2130        assert_eq!(first, second);
2131        assert_eq!(
2132            f.stub.last_workspaces(),
2133            vec![first],
2134            "still exactly one workspace"
2135        );
2136    }
2137
2138    #[tokio::test(flavor = "multi_thread")]
2139    async fn a_registration_carries_the_complete_current_list() {
2140        let f = fixture().await;
2141        let second_root = f.second_root();
2142
2143        let a = f.runtime.enable(&f.root).await.unwrap();
2144        let b = f.runtime.enable(&second_root).await.unwrap();
2145
2146        let mut seen = f.stub.last_workspaces();
2147        seen.sort();
2148        let mut want = vec![a, b];
2149        want.sort();
2150        assert_eq!(seen, want, "registration is the whole set, not a delta");
2151    }
2152
2153    #[tokio::test(flavor = "multi_thread")]
2154    async fn no_later_registration_clears_this_apps_routes() {
2155        // The device id is asked of the bridge once, on the first `enable`, with an empty
2156        // workspace list — the one moment that is safe, because no route exists yet. Every
2157        // later registration must name the complete set: one with an empty list would
2158        // briefly clear this application's routes, and a peer asking for a workspace this
2159        // server owns would be told nobody has it.
2160        let f = fixture().await;
2161        let second_root = f.second_root();
2162
2163        f.runtime.enable(&f.root).await.unwrap();
2164        f.runtime.enable(&second_root).await.unwrap();
2165        f.runtime.sync_now(&f.root).await.unwrap();
2166        f.runtime.disable(&f.root).await.unwrap();
2167
2168        let seen = f.stub.seen.lock().expect("stub");
2169        let first_full = seen
2170            .registrations
2171            .iter()
2172            .position(|r| !r.workspaces.is_empty())
2173            .expect("enabling registers the workspace");
2174        let emptied_afterwards = seen.registrations[first_full + 1..]
2175            .iter()
2176            .filter(|r| r.workspaces.is_empty())
2177            .count();
2178        assert_eq!(
2179            emptied_afterwards, 0,
2180            "a registration after the first workspace must still name the whole set"
2181        );
2182    }
2183
2184    #[tokio::test(flavor = "multi_thread")]
2185    async fn disabling_unregisters_and_leaves_the_files_alone() {
2186        let f = fixture().await;
2187        std::fs::write(f.root.join("keep.md"), "content").unwrap();
2188
2189        let id = f.runtime.enable(&f.root).await.unwrap();
2190        f.runtime.disable(&f.root).await.unwrap();
2191
2192        assert!(f.stub.last_workspaces().is_empty());
2193        assert!(
2194            f.root.join("keep.md").exists(),
2195            "disabling sync must not touch files"
2196        );
2197        assert!(
2198            crate::sync::id::sync_id_path("sapphire-synctest", &f.root).exists(),
2199            "the sync id stays, so re-enabling rejoins the same workspace"
2200        );
2201        let _ = id;
2202    }
2203
2204    #[tokio::test(flavor = "multi_thread")]
2205    async fn re_enabling_after_disabling_reuses_the_identity() {
2206        let f = fixture().await;
2207
2208        let first = f.runtime.enable(&f.root).await.unwrap();
2209        f.runtime.disable(&f.root).await.unwrap();
2210        let again = f.runtime.enable(&f.root).await.unwrap();
2211
2212        assert_eq!(first, again);
2213    }
2214
2215    #[tokio::test(flavor = "multi_thread")]
2216    async fn status_reports_disabled_for_a_workspace_that_was_never_enabled() {
2217        let f = fixture().await;
2218
2219        let status = f.runtime.status(&f.root).await;
2220        assert!(!status.enabled);
2221        assert!(status.workspace_id.is_none());
2222    }
2223
2224    #[tokio::test(flavor = "multi_thread")]
2225    async fn disabling_something_that_was_never_enabled_is_not_an_error() {
2226        let f = fixture().await;
2227        f.runtime
2228            .disable(&f.root)
2229            .await
2230            .expect("disabling twice is fine");
2231    }
2232
2233    #[tokio::test(flavor = "multi_thread")]
2234    async fn a_workspace_whose_root_vanished_is_reported_as_paused_not_deleted() {
2235        let f = fixture().await;
2236        std::fs::write(f.root.join("a.md"), "content").unwrap();
2237        f.runtime.enable(&f.root).await.unwrap();
2238        f.runtime.scan(&f.root).await.unwrap();
2239
2240        // The drive is unmounted: the marker directory goes with it.
2241        std::fs::remove_dir_all(f.root.join(".sapphire-synctest")).unwrap();
2242        let _ = f.runtime.scan(&f.root).await;
2243
2244        let status = f.runtime.status(&f.root).await;
2245        assert!(
2246            status.paused.is_some(),
2247            "a missing root must pause, not replicate as a mass deletion"
2248        );
2249    }
2250
2251    #[tokio::test(flavor = "multi_thread")]
2252    async fn a_directory_that_is_not_a_workspace_cannot_be_enabled() {
2253        let f = fixture().await;
2254        let plain = f._tmp.path().join("plain");
2255        std::fs::create_dir_all(&plain).unwrap();
2256
2257        assert!(f.runtime.enable(&plain).await.is_err());
2258    }
2259
2260    #[tokio::test(flavor = "multi_thread")]
2261    async fn a_write_through_the_workspace_router_is_committed_to_the_replica() {
2262        // The brief's "main path": a write the app server itself made is scanned straight
2263        // after, not left to the watcher's debounce. Asserted against the replica's own
2264        // state, which is the only thing that can tell the two paths apart — reading the
2265        // file back would pass even if the scan never ran.
2266        let f = fixture().await;
2267        f.runtime.enable(&f.root).await.unwrap();
2268
2269        let host = Arc::new(crate::WorkspaceHost::new(&CTX));
2270        host.select(&f.root, false).await.unwrap();
2271        let current = Arc::new(crate::Current::new(
2272            &CTX,
2273            host,
2274            Some(Arc::clone(&f.runtime)),
2275            crate::selection::SelectionFile::at(f._tmp.path().join("workspace.toml")),
2276        ));
2277        let router = Arc::new(crate::workspace_router(current));
2278        let (client_conn, server_conn) = sapphire_ipc::Connection::pair();
2279        tokio::spawn(async move {
2280            let info = sapphire_ipc::ServerInfo {
2281                version: "0.0.0".into(),
2282                api: sapphire_backend::protocol::API_VERSION,
2283                app_api: None,
2284                pid: std::process::id(),
2285                managed_by: ManagedBy::Spawned,
2286            };
2287            let _ = sapphire_ipc::serve(server_conn, router, "sapphire-synctest", info).await;
2288        });
2289        let info = sapphire_ipc::ClientInfo {
2290            kind: "test".into(),
2291            version: "0.0.0".into(),
2292            api: sapphire_backend::protocol::API_VERSION,
2293            app_api: None,
2294            pid: std::process::id(),
2295        };
2296        let (client, _) = sapphire_ipc::Client::handshake(client_conn, "sapphire-synctest", info)
2297            .await
2298            .unwrap();
2299
2300        let _: sapphire_backend::protocol::Ack = client
2301            .call(
2302                sapphire_backend::protocol::WRITE_FILE,
2303                sapphire_backend::protocol::ContentParams {
2304                    path: PathBuf::from("through-the-server.md"),
2305                    content: "written by the server".into(),
2306                },
2307            )
2308            .await
2309            .unwrap();
2310
2311        // `scan` runs before the reply is written, so by the time the call returns the
2312        // replica must already know the path.
2313        let synced = f.runtime.synced.lock().await;
2314        let entry = synced.get(&f.root).expect("enabled above");
2315        let replica = entry.replica.lock().await;
2316        let state = replica
2317            .state("through-the-server.md")
2318            .expect("a readable state");
2319        assert!(
2320            state.is_some(),
2321            "a write through the server must be scanned into the replica, not left to the watcher"
2322        );
2323    }
2324
2325    #[tokio::test(flavor = "multi_thread")]
2326    async fn an_announcement_for_an_unknown_workspace_is_ignored_not_fatal() {
2327        let f = fixture().await;
2328        f.runtime.enable(&f.root).await.unwrap();
2329
2330        let driver = Arc::clone(&f.runtime);
2331        let handle = tokio::spawn(async move { driver.run().await });
2332
2333        f.stub
2334            .announce(grain_id::GrainId::random(), "no-such-ticket")
2335            .await;
2336        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
2337
2338        assert!(
2339            !handle.is_finished(),
2340            "one bad announcement must not stop the runtime"
2341        );
2342        handle.abort();
2343    }
2344}