Skip to main content

sapphire_framework_server/sync/
mod.rs

1//! Replication, from the app server's side.
2//!
3//! One `Replica` per synced workspace, registered with the bridge, driven by announcements
4//! from it and by local edits.
5//!
6//! See `docs/superpowers/specs/2026-09-16-process-architecture-design.md` §4.2.
7
8use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use grain_id::GrainId;
13use sapphire_backend::protocol as proto;
14use sapphire_bridge_api::{
15    IncomingParams, PeersResult, RegisterParams, WorkspaceRegistration, WorkspacesResult,
16};
17use sapphire_bridge_client::BridgeClient;
18use sapphire_ipc::ManagedBy;
19use sapphire_sync::{PathUpdate, PauseReason, Replica, ReplicaConfig, ScanOutcome, SystemClock};
20use sapphire_workspace::{AppContext, Workspace};
21use tokio::sync::{Mutex, OnceCell, broadcast, mpsc};
22
23pub mod id;
24mod live;
25mod methods;
26#[cfg(any(test, feature = "test-util"))]
27pub mod testing;
28mod topology;
29mod watch;
30
31pub use id::{SYNC_ID_FILE, WORKSPACE_MAP_FILE, sync_id, sync_id_path};
32pub use live::{DIAL_BACKOFF_MAX, OPEN_TIMEOUT};
33pub use methods::sync_router;
34pub use watch::{DEBOUNCE, Watcher};
35
36use live::{LivePeers, dial_backoff};
37
38use crate::error::{Error, Result};
39
40use crate::host::WorkspaceHost;
41
42/// What `sync.status` reports.
43#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
44pub struct SyncStatus {
45    /// Whether this workspace is synced at all.
46    pub enabled: bool,
47    /// Its identity across devices, when it is.
48    pub workspace_id: Option<GrainId>,
49    /// How many devices the workgroup has, this host excluded.
50    pub peers: usize,
51    /// Why replication is paused, if it is — a missing root or marker directory.
52    pub paused: Option<String>,
53    /// The last failure, if any.
54    pub last_error: Option<String>,
55    /// Whether the bridge is reachable. `false` is not an outage: the app server works.
56    pub bridge_available: bool,
57    /// How the workspace is wired to its peers, by the roles the bridge last reported.
58    pub topology: proto::Topology,
59    /// How far embedding has got, when this host embeds (#188).
60    pub embedding: Option<proto::EmbeddingProgress>,
61}
62
63/// One workspace this runtime syncs.
64struct Synced {
65    /// Its identity across devices, shared with every peer.
66    workspace_id: GrainId,
67    /// Its replica store. Held open because the store is a redb database.
68    replica: Arc<Mutex<Replica>>,
69    /// Why replication is paused, from the last scan.
70    paused: Option<PauseReason>,
71    /// The last failure, if any.
72    last_error: Option<String>,
73}
74
75/// How old a vector file no live content needs must be before the primary device removes
76/// it: a vector can arrive before its file, and a day is far longer than that window.
77const STALE_VECTOR_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
78
79/// Texts per embedding request in a server pass: small, so a search query waiting on the
80/// bridge's one worker is never stuck behind a long backfill (#188).
81const PASS_BATCH: usize = 4;
82
83/// How the primary device paces its backfill (#188).
84#[derive(Clone, Copy, Debug)]
85pub struct BackfillTiming {
86    /// How old another device's version must be before the primary embeds it: its author
87    /// has embedded it by then, or will not.
88    pub grace: std::time::Duration,
89    /// How often the primary device runs a pass with nothing prompting it.
90    pub interval: std::time::Duration,
91}
92
93impl Default for BackfillTiming {
94    fn default() -> Self {
95        BackfillTiming {
96            grace: std::time::Duration::from_secs(10 * 60),
97            interval: std::time::Duration::from_secs(10 * 60),
98        }
99    }
100}
101
102/// How long [`SyncRuntime::keep_connected`] waits before its first retry after the bridge
103/// could not be reached. Each failure doubles it, up to [`RECONNECT_BACKOFF_MAX`].
104pub const RECONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_millis(500);
105
106/// The longest [`SyncRuntime::keep_connected`] waits between two attempts to reach the
107/// bridge: a bridge that comes up is found within this long.
108pub const RECONNECT_BACKOFF_MAX: std::time::Duration = std::time::Duration::from_secs(30);
109
110/// The wait before reconnection attempt `attempt` (0-based).
111fn reconnect_backoff(attempt: u32) -> std::time::Duration {
112    RECONNECT_BACKOFF
113        .saturating_mul(1u32 << attempt.min(16))
114        .min(RECONNECT_BACKOFF_MAX)
115}
116
117/// What a connection attempt to the bridge yields.
118type Connecting = std::pin::Pin<
119    Box<dyn std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>> + Send>,
120>;
121
122/// How a [`SyncRuntime`] reaches the bridge, on start and whenever it has lost it (#244).
123///
124/// An attempt answers `Ok(None)` when no bridge is running — the runtime then tries again
125/// later — and an error for anything else, such as a bridge of another API version.
126#[derive(Clone)]
127pub struct BridgeConnector(Arc<dyn Fn() -> Connecting + Send + Sync>);
128
129impl BridgeConnector {
130    /// Connect with `connect`.
131    pub fn new<F, Fut>(connect: F) -> BridgeConnector
132    where
133        F: Fn() -> Fut + Send + Sync + 'static,
134        Fut: std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>>
135            + Send
136            + 'static,
137    {
138        BridgeConnector(Arc::new(move || Box::pin(connect())))
139    }
140
141    /// The host's own bridge, at its standard endpoint, as a client of `kind` at
142    /// `version`.
143    pub fn standard(kind: &'static str, version: &'static str) -> BridgeConnector {
144        BridgeConnector::new(move || async move {
145            match BridgeClient::connect(kind, version).await {
146                Ok(client) => Ok(Some(client)),
147                Err(sapphire_ipc::Error::NotRunning(_)) => Ok(None),
148                Err(err) => Err(err),
149            }
150        })
151    }
152
153    /// The bridge listening at `endpoint`: for a fixture that runs its own.
154    pub fn at(
155        endpoint: sapphire_ipc::Endpoint,
156        kind: &'static str,
157        version: &'static str,
158    ) -> BridgeConnector {
159        BridgeConnector::new(move || {
160            let endpoint = endpoint.clone();
161            async move { BridgeClient::connect_at(&endpoint, kind, version).await }
162        })
163    }
164
165    async fn connect(&self) -> sapphire_ipc::Result<Option<BridgeClient>> {
166        (self.0)().await
167    }
168}
169
170impl std::fmt::Debug for BridgeConnector {
171    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
172        f.write_str("BridgeConnector")
173    }
174}
175
176/// Replication for one application's workspaces.
177pub struct SyncRuntime {
178    ctx: &'static AppContext,
179    /// The bridge connection, while there is one. Swapped when the runtime reconnects
180    /// (#244); [`run`](SyncRuntime::run) follows it.
181    bridge: tokio::sync::watch::Sender<Option<Arc<BridgeClient>>>,
182    /// How to reach the bridge again; `None` for a runtime given one connection to use.
183    connector: Option<BridgeConnector>,
184    /// Workspaces asked to sync while the bridge was away, with their ids. Enabled once
185    /// it is back.
186    ///
187    /// Also the lock that orders an `enable` against a reconnection: whoever holds it sees
188    /// the bridge slot as the other left it, so a workspace is either enabled now or left
189    /// here for the reconnection to enable — never neither.
190    pending: Mutex<HashMap<PathBuf, GrainId>>,
191    /// The announcements of a connection [`attach`](SyncRuntime::attach) just made,
192    /// subscribed to before it registered: the bridge announces the streams that waited for
193    /// this server as soon as it registers (#244), and a subscription made afterwards by
194    /// [`run`](SyncRuntime::run) would miss them.
195    subscribed: std::sync::Mutex<Option<(Arc<BridgeClient>, broadcast::Receiver<IncomingParams>)>>,
196    exe_path: PathBuf,
197    managed_by: ManagedBy,
198    synced: Mutex<HashMap<PathBuf, Synced>>,
199    /// This host's device id. Asked of the bridge once, on the first [`enable`], and never
200    /// again: a second registration with an empty workspace list would briefly clear this
201    /// application's routes.
202    ///
203    /// [`enable`]: SyncRuntime::enable
204    device_id: OnceCell<GrainId>,
205    /// The file watcher, once [`watch_changes`] has started it.
206    ///
207    /// The runtime owns it because the spec puts it here: the sync runtime is a watcher and
208    /// a replica per registered root, and the two must agree on which roots are synced.
209    /// [`enable`] and [`disable`] keep them in step.
210    ///
211    /// [`watch_changes`]: SyncRuntime::watch_changes
212    /// [`enable`]: SyncRuntime::enable
213    /// [`disable`]: SyncRuntime::disable
214    watcher: OnceCell<Arc<Watcher>>,
215    /// The app server that owns these workspaces' caches, once it has told us.
216    ///
217    /// A replication session writes files, and the search index is not part of a session: the
218    /// index belongs to the app server, which holds the workspace open. Reaching it through
219    /// the host is what keeps a received file and its index moving together, and it is the
220    /// *same* [`WorkspaceState`](sapphire_workspace::WorkspaceState) the server serves from —
221    /// opening a second one would collide on the retrieve store's exclusive lock.
222    ///
223    /// Absent for a runtime nobody has wired to an app server (a test stub driving the
224    /// replica directly). Nothing then indexes, which is correct: nothing owns an index.
225    host: OnceCell<Arc<WorkspaceHost>>,
226    /// Serialises re-indexing, so two sessions finishing at once do not both sweep the
227    /// workspace through the same store.
228    reindexing: Mutex<()>,
229    /// Roots with an embedding pass waiting to run (#187). A request for a root already
230    /// waiting is folded into that pass.
231    embed_wanted: Mutex<std::collections::HashSet<PathBuf>>,
232    /// Serialises embedding passes: one model, one queue.
233    embedding: Mutex<()>,
234    /// Roots whose embedding pass is running now, for `sync.status`.
235    embed_running: Mutex<std::collections::HashSet<PathBuf>>,
236    /// The primary device's backfill pacing (#188).
237    backfill: std::sync::Mutex<BackfillTiming>,
238    /// The bridge an embedding pass asks for its model; `None` is the host's standard one.
239    bridge_endpoint: Option<sapphire_ipc::Endpoint>,
240    /// The live session table of every synced workspace, keyed by canonical root.
241    ///
242    /// One table per workspace rather than one for the runtime: a session is about one
243    /// workspace, and the two storm rules are per-workspace rules. A root that is not
244    /// synced has none, so a dial or a push for it is a no-op rather than an error.
245    live: Mutex<HashMap<PathBuf, Arc<LivePeers>>>,
246    /// The bridge's last answer to `peers`, roles included. The inbound guard and
247    /// [`is_primary`](SyncRuntime::is_primary) read it rather than asking again.
248    last_peers: Mutex<Option<PeersResult>>,
249    /// How long a dial waits for the bridge to open a stream; [`OPEN_TIMEOUT`] by default.
250    open_timeout: std::time::Duration,
251    /// The first dials [`enable`](SyncRuntime::enable) leaves running. Each pins the
252    /// runtime, so [`stop_dials`](SyncRuntime::stop_dials) ends them with the server.
253    first_dials: std::sync::Mutex<tokio::task::JoinSet<()>>,
254    /// The rest of the runtime's background work: inbound sessions, the readers of live
255    /// sessions, and embedding passes. Each pins the runtime — and through it the replica
256    /// stores, which are redb databases locked until the last pin drops — so
257    /// [`stop_background`](SyncRuntime::stop_background) ends them with the server (#235).
258    background: std::sync::Mutex<tokio::task::JoinSet<()>>,
259}
260
261impl SyncRuntime {
262    /// A runtime for `ctx`'s application, syncing through `bridge`.
263    ///
264    /// It has no way to reach the bridge again: when the connection closes, sync stops until
265    /// the runtime is given a [`connector`](Self::with_connector).
266    pub fn new(
267        ctx: &'static AppContext,
268        bridge: Arc<BridgeClient>,
269        exe_path: PathBuf,
270        managed_by: ManagedBy,
271    ) -> SyncRuntime {
272        SyncRuntime::build(ctx, Some(bridge), None, exe_path, managed_by)
273    }
274
275    /// A runtime for `ctx`'s application that reaches the bridge with `connector`, and again
276    /// whenever it loses it (#244).
277    ///
278    /// Not connected yet: [`connect`](Self::connect) makes the first attempt, and
279    /// [`keep_connected`](Self::keep_connected) the rest. [`AppServer::run`] does both.
280    ///
281    /// [`AppServer::run`]: crate::AppServer::run
282    pub fn connecting(
283        ctx: &'static AppContext,
284        connector: BridgeConnector,
285        exe_path: PathBuf,
286        managed_by: ManagedBy,
287    ) -> SyncRuntime {
288        SyncRuntime::build(ctx, None, Some(connector), exe_path, managed_by)
289    }
290
291    /// Reach the bridge again with `connector` once the connection closes.
292    pub fn with_connector(mut self, connector: BridgeConnector) -> SyncRuntime {
293        self.connector = Some(connector);
294        self
295    }
296
297    fn build(
298        ctx: &'static AppContext,
299        bridge: Option<Arc<BridgeClient>>,
300        connector: Option<BridgeConnector>,
301        exe_path: PathBuf,
302        managed_by: ManagedBy,
303    ) -> SyncRuntime {
304        SyncRuntime {
305            ctx,
306            bridge: tokio::sync::watch::Sender::new(bridge),
307            connector,
308            pending: Mutex::new(HashMap::new()),
309            subscribed: std::sync::Mutex::new(None),
310            exe_path,
311            managed_by,
312            synced: Mutex::new(HashMap::new()),
313            device_id: OnceCell::new(),
314            watcher: OnceCell::new(),
315            host: OnceCell::new(),
316            reindexing: Mutex::new(()),
317            embed_wanted: Mutex::new(std::collections::HashSet::new()),
318            embedding: Mutex::new(()),
319            embed_running: Mutex::new(std::collections::HashSet::new()),
320            backfill: std::sync::Mutex::new(BackfillTiming::default()),
321            bridge_endpoint: None,
322            live: Mutex::new(HashMap::new()),
323            last_peers: Mutex::new(None),
324            open_timeout: OPEN_TIMEOUT,
325            first_dials: std::sync::Mutex::new(tokio::task::JoinSet::new()),
326            background: std::sync::Mutex::new(tokio::task::JoinSet::new()),
327        }
328    }
329
330    /// The bridge, while connected.
331    fn bridge(&self) -> Result<Arc<BridgeClient>> {
332        match &*self.bridge.borrow() {
333            Some(bridge) if !bridge.is_closed() => Ok(Arc::clone(bridge)),
334            _ => Err(Error::BridgeDisconnected),
335        }
336    }
337
338    /// Whether the runtime has a live connection to the bridge.
339    pub fn is_connected(&self) -> bool {
340        self.bridge().is_ok()
341    }
342
343    /// Run `task` in the background, tracked so the server's shutdown ends it.
344    fn spawn_background(&self, task: impl std::future::Future<Output = ()> + Send + 'static) {
345        let mut background = self.background.lock().expect("background tasks");
346        // Reap the finished ones, so the set holds only what is running.
347        while background.try_join_next().is_some() {}
348        background.spawn(task);
349    }
350
351    /// Give up on opening a stream to a peer after `timeout` rather than [`OPEN_TIMEOUT`]:
352    /// for tests, which cannot wait that long for a peer that never answers.
353    pub fn with_open_timeout(mut self, timeout: std::time::Duration) -> SyncRuntime {
354        self.open_timeout = timeout;
355        self
356    }
357
358    /// Tell the runtime which app server owns these workspaces' caches.
359    ///
360    /// Called by [`AppServer::sync`](crate::AppServer::sync). First writer wins: a runtime
361    /// belongs to the server that wired it.
362    /// Embed through the bridge at `endpoint` rather than the host's standard one: for a
363    /// fixture running several hosts in one process.
364    pub fn with_bridge_endpoint(mut self, endpoint: sapphire_ipc::Endpoint) -> SyncRuntime {
365        self.bridge_endpoint = Some(endpoint);
366        self
367    }
368
369    /// Pace the primary device's backfill differently: for tests, which cannot wait ten
370    /// minutes.
371    pub fn set_backfill_timing(&self, timing: BackfillTiming) {
372        *self.backfill.lock().expect("backfill timing") = timing;
373    }
374
375    fn backfill_timing(&self) -> BackfillTiming {
376        *self.backfill.lock().expect("backfill timing")
377    }
378
379    pub(crate) fn set_host(&self, host: Arc<WorkspaceHost>) {
380        let _ = self.host.set(host);
381    }
382
383    /// Start syncing `root`, returning its identity across devices.
384    ///
385    /// Idempotent: enabling an already-synced workspace returns the same id and changes
386    /// nothing.
387    pub async fn enable(self: &Arc<Self>, root: &Path) -> Result<GrainId> {
388        let key = root.canonicalize().map_err(Error::Io)?;
389        let watch_key = key.clone();
390        let workspace_id = sync_id(self.ctx.app_name, &key)?;
391
392        // Without a bridge there is no device id to open the replica with, and nobody to
393        // register with: keep the workspace for the reconnection to enable (#244).
394        {
395            let mut pending = self.pending.lock().await;
396            if self.bridge().is_err() {
397                pending.insert(key, workspace_id);
398                return Err(Error::BridgeDisconnected);
399            }
400            pending.remove(&key);
401        }
402
403        // The map stays locked across opening the replica, so two simultaneous `enable`s for
404        // one root cannot both open it — the store is a redb database, and it is opened once.
405        {
406            let mut synced = self.synced.lock().await;
407            if let Some(existing) = synced.get(&key) {
408                return Ok(existing.workspace_id);
409            }
410            let device_id = self.device_id().await?;
411            let state_dir = self.state_dir(&key)?;
412            std::fs::create_dir_all(&state_dir).map_err(Error::Io)?;
413            let config = ReplicaConfig::new(self.ctx.app_name, key.clone(), device_id, &state_dir);
414            let replica = Replica::open(config, Arc::new(SystemClock))
415                .map_err(|e| Error::Sync(e.to_string()))?;
416            synced.insert(
417                key.clone(),
418                Synced {
419                    workspace_id,
420                    replica: Arc::new(Mutex::new(replica)),
421                    paused: None,
422                    last_error: None,
423                },
424            );
425        }
426
427        // Watch the new root too, so an edit made outside the server from now on is seen.
428        // Not fatal if it fails: the app server's own writes take the exact path in
429        // `handlers.rs`, and the next `watch_changes` re-reads the root set anyway.
430        if let Some(watcher) = self.watcher.get()
431            && let Err(err) = watcher.watch(&watch_key)
432        {
433            tracing::warn!(root = %watch_key.display(), "could not watch: {err}");
434        }
435
436        // Register only once the replica is open, so a registration never names a workspace
437        // that cannot serve a session.
438        self.reregister().await?;
439
440        // Dial now, so enabling a workspace converges it rather than waiting for the next
441        // local edit to trigger a session. This is what catches a host up when it returns:
442        // the peer that has been running is the one the returning host has to ask, and no
443        // watcher on either side fires for a file that was never written locally.
444        //
445        // Best-effort: a workspace is enabled whether or not any peer is up, and a failure
446        // here is not a failure to enable. A session is symmetric, so dialing is enough —
447        // this side sends its own view, and the peer answers with what this side lacks.
448        //
449        // In the background, so the answer does not wait on it: a peer that is offline
450        // holds a dial up to `open_timeout`, longer than a client waits for `sync.enable`
451        // (#228).
452        let runtime = Arc::clone(self);
453        let mut dials = self.first_dials.lock().expect("first dials");
454        // Reap the finished ones, so the set holds only what is running.
455        while dials.try_join_next().is_some() {}
456        dials.spawn(async move {
457            if let Err(err) = runtime.sync_now(&key).await {
458                tracing::warn!(root = %key.display(), "the first session after enabling failed: {err}");
459            }
460        });
461        Ok(workspace_id)
462    }
463
464    /// Stop syncing `root`. Files and the sync id stay, so re-enabling rejoins the same
465    /// workspace rather than creating a second one.
466    pub async fn disable(&self, root: &Path) -> Result<()> {
467        self.stop(root).await.1
468    }
469
470    /// Stop syncing `root`, like [`disable`](Self::disable), then wait up to `limit` until
471    /// nothing holds its replica store open any more — a session, a scan or an embedding
472    /// pass that started before may still have it. A store about to be moved or deleted
473    /// must be closed first.
474    ///
475    /// Only a store still in use at `limit` is an error. The bridge not hearing of it is
476    /// logged, not returned: the store is closed all the same, and the next registration
477    /// replaces the bridge's view.
478    pub async fn release(&self, root: &Path, limit: std::time::Duration) -> Result<()> {
479        let (replica, unregistered) = self.stop(root).await;
480        if let Err(err) = unregistered {
481            tracing::warn!(root = %root.display(), "stopping sync: {err}");
482        }
483        let Some(replica) = replica else {
484            return Ok(());
485        };
486        let deadline = tokio::time::Instant::now() + limit;
487        while Arc::strong_count(&replica) > 1 {
488            if tokio::time::Instant::now() >= deadline {
489                return Err(Error::Sync(format!(
490                    "the replica of {} is still in use; try again",
491                    root.display()
492                )));
493            }
494            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
495        }
496        Ok(())
497    }
498
499    /// Stop syncing `root`, handing back its replica — `None` when it was not synced — and
500    /// how telling the bridge went.
501    async fn stop(&self, root: &Path) -> (Option<Arc<Mutex<Replica>>>, Result<()>) {
502        // A root that no longer exists cannot be canonicalized, but the keys of `synced` are
503        // canonical already, so the path as given (a registry root) still names the entry.
504        let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
505        self.pending.lock().await.remove(&key);
506        let removed = self.synced.lock().await.remove(&key);
507        if removed.is_some()
508            && let Some(watcher) = self.watcher.get()
509        {
510            watcher.unwatch(&key);
511        }
512        // Any live session for a workspace this server no longer holds is closed: it would
513        // otherwise keep pushing files for a workspace nobody asked about any more.
514        if let Some(peers) = self.live.lock().await.remove(&key) {
515            peers.drop_connections().await;
516        }
517        let Some(removed) = removed else {
518            return (None, Ok(()));
519        };
520        let unregistered = async {
521            // A bridge that is gone has forgotten this server's registrations already, and
522            // the reconnection registers only what is still synced.
523            let Ok(bridge) = self.bridge() else {
524                return Ok(());
525            };
526            bridge
527                .unregister(removed.workspace_id)
528                .await
529                .map_err(|e| Error::Bridge(e.to_string()))?;
530            self.reregister().await
531        }
532        .await;
533        (Some(removed.replica), unregistered)
534    }
535
536    /// The workspaces the workgroup knows about.
537    ///
538    /// Read straight from the bridge, so it reflects every device's registrations. The
539    /// bridge being down is not an empty answer — that would read as "nothing to map" — so
540    /// it is an error, and `sync.map` reports it.
541    pub async fn workspaces(&self) -> Result<WorkspacesResult> {
542        self.bridge()?
543            .workspaces()
544            .await
545            .map_err(|e| Error::Bridge(e.to_string()))
546    }
547
548    /// Place a workgroup workspace at a directory of this host: write its identity and
549    /// map there, returning the canonical root. [`Current::map`](crate::Current::map)
550    /// then selects it and syncs it.
551    ///
552    /// `selector` is the workspace's name or id, as [`workspaces`] lists it; `dir` must
553    /// already be a workspace of this application — the directory is placed, not made.
554    ///
555    /// [`workspaces`]: SyncRuntime::workspaces
556    pub async fn place(&self, selector: &str, dir: &Path) -> Result<PathBuf> {
557        let workgroup = self.workspaces().await?;
558        let wanted = workgroup
559            .workspaces
560            .iter()
561            // A name is the user-facing handle; the id is the exact one. Two workspaces
562            // may not share a name, so a name names at most one.
563            .find(|w| w.name == selector || w.workspace_id.to_string() == selector)
564            .ok_or_else(|| Error::UnknownWorkspaceName(selector.to_owned()))?;
565
566        // Refuse another application's workspace: this server syncs its own application's
567        // workspaces, and `Workspace::from_root` would later refuse the directory anyway.
568        if wanted.app_name != self.ctx.app_name {
569            return Err(Error::WrongApp {
570                name: wanted.name.clone(),
571                app_name: wanted.app_name.clone(),
572            });
573        }
574
575        let root = dir.canonicalize().map_err(Error::Io)?;
576        let workspace = Workspace::from_root(self.ctx, &root)?;
577        if workspace.root != root {
578            return Err(Error::UnknownWorkspace(root, self.ctx.app_name));
579        }
580
581        // A directory already syncing under another identity is not remapped: the sync id
582        // is a device's word that it is the same workspace, and rewriting it silently
583        // would make two hosts disagree about what they share.
584        {
585            let synced = self.synced.lock().await;
586            if let Some(entry) = synced.get(&root)
587                && entry.workspace_id != wanted.workspace_id
588            {
589                return Err(Error::SyncId(format!(
590                    "{} already syncs as {}, not {}",
591                    root.display(),
592                    entry.workspace_id,
593                    wanted.workspace_id
594                )));
595            }
596        }
597
598        // The identity arrives from the workgroup, not from this host: two hosts that map
599        // the same workspace must sync as one, so the id is the one the workgroup lists,
600        // written before `enable` reads it. A different id already on disk is refused
601        // above; an absent one is written here.
602        let marker = workspace.marker_dir();
603        let id_path = marker.join(crate::sync::id::SYNC_ID_FILE);
604        match std::fs::read_to_string(&id_path) {
605            Ok(existing) if existing.trim() != wanted.workspace_id.to_string() => {
606                return Err(Error::SyncId(format!(
607                    "{} already holds identity {}, not {}",
608                    root.display(),
609                    existing.trim(),
610                    wanted.workspace_id
611                )));
612            }
613            Err(_) => {
614                std::fs::write(&id_path, format!("{}\n", wanted.workspace_id))
615                    .map_err(Error::Io)?;
616            }
617            Ok(_) => {}
618        }
619
620        // The map is what tells this application, on a later start, which workgroup
621        // workspace a directory is.
622        let map_path = root
623            .join(format!(".{}", self.ctx.app_name))
624            .join(WORKSPACE_MAP_FILE);
625        let workspace_id = wanted.workspace_id.to_string();
626        std::fs::write(&map_path, format!("{workspace_id}\n")).map_err(Error::Io)?;
627        Ok(root)
628    }
629
630    /// What `sync.status` answers with.
631    pub async fn status(&self, root: &Path) -> SyncStatus {
632        let bridge = self.bridge().ok();
633        let bridge_available = match &bridge {
634            Some(bridge) => bridge.status().await.is_ok(),
635            None => false,
636        };
637        let answer = match &bridge {
638            Some(bridge) => bridge.peers().await.ok(),
639            None => None,
640        };
641        if let Some(answer) = &answer {
642            *self.last_peers.lock().await = Some(answer.clone());
643        }
644        let peers = answer
645            .as_ref()
646            .map_or(0, |p| p.peers.len().saturating_sub(1));
647        let Ok(key) = root.canonicalize() else {
648            return SyncStatus {
649                enabled: false,
650                workspace_id: None,
651                peers,
652                paused: None,
653                last_error: None,
654                bridge_available,
655                topology: proto::Topology::Mesh,
656                embedding: None,
657            };
658        };
659        let synced = self.synced.lock().await;
660        match synced.get(&key) {
661            Some(entry) => SyncStatus {
662                enabled: true,
663                workspace_id: Some(entry.workspace_id),
664                peers,
665                paused: entry.paused.map(|r| format!("{r:?}")),
666                last_error: entry.last_error.clone(),
667                bridge_available,
668                topology: topology::topology(
669                    answer
670                        .as_ref()
671                        .and_then(|p| p.roles_for(entry.workspace_id)),
672                ),
673                embedding: self.embedding_progress(&key).await,
674            },
675            None => {
676                drop(synced);
677                // Asked to sync while the bridge was away: it is enabled, waiting for the
678                // bridge, and `bridge_available` says why nothing moves.
679                let waiting = self.pending.lock().await.get(&key).copied();
680                SyncStatus {
681                    enabled: waiting.is_some(),
682                    workspace_id: waiting,
683                    peers,
684                    paused: None,
685                    last_error: None,
686                    bridge_available,
687                    topology: proto::Topology::Mesh,
688                    embedding: None,
689                }
690            }
691        }
692    }
693
694    /// How far `key`'s embedding has got, when this host embeds it. Reads the open
695    /// workspace's index; never opens one, so a status call stays cheap.
696    async fn embedding_progress(&self, key: &Path) -> Option<proto::EmbeddingProgress> {
697        let host = self.host.get()?;
698        let backend = host.open_backend(key)?;
699        let state = backend.state();
700        state.embedder()?;
701        let info = state.db_info().ok()?;
702        Some(proto::EmbeddingProgress {
703            vectors: info.vector_count,
704            pending: info.pending_count,
705            running: self.embed_running.lock().await.contains(key),
706        })
707    }
708
709    /// Bring the replica's view of the files up to date.
710    ///
711    /// Called straight after the app server's own writes, and by the watcher for everything
712    /// else. A scan that finds nothing is cheap; a scan that is skipped loses an edit until
713    /// the next one.
714    pub async fn scan(&self, root: &Path) -> Result<()> {
715        let Ok(key) = root.canonicalize() else {
716            return Ok(());
717        };
718        let replica = {
719            let synced = self.synced.lock().await;
720            match synced.get(&key) {
721                Some(entry) => Arc::clone(&entry.replica),
722                None => return Ok(()),
723            }
724        };
725        let (outcome, updates) = {
726            let mut replica = replica.lock().await;
727            let outcome = replica.scan().map_err(|e| Error::Sync(e.to_string()))?;
728            // What the scan committed, as path updates, so it can go out on the sessions
729            // that are already open. This is the whole point of the exact-path scan in
730            // `handlers.rs`: a write through the server reaches a peer live, not at the next
731            // dial.
732            let updates = match &outcome {
733                ScanOutcome::Scanned(report) => {
734                    // The unit of replication is the path state (design doc section 2.4):
735                    // the `seen` a peer prunes against is the *path's* version vector - the
736                    // context the recorded entry was built on plus its own dot - never the
737                    // whole replica vv. Sending the global vv here made receivers prune
738                    // causally-related versions they still held: a conflict copy on the
739                    // first remote edit, and tombstones pruned so deletes stopped
740                    // propagating (issue #157).
741                    report
742                        .recorded
743                        .iter()
744                        .map(|entry| PathUpdate {
745                            path: entry.path.clone(),
746                            versions: vec![entry.clone()],
747                            seen: {
748                                let mut seen = entry.context.clone();
749                                seen.add_dot(&entry.dot);
750                                seen
751                            },
752                        })
753                        .collect()
754                }
755                ScanOutcome::Paused(_) => Vec::new(),
756            };
757            (outcome, updates)
758        };
759        {
760            let mut synced = self.synced.lock().await;
761            if let Some(entry) = synced.get_mut(&key) {
762                entry.paused = match outcome {
763                    ScanOutcome::Paused(reason) => Some(reason),
764                    ScanOutcome::Scanned(_) => None,
765                };
766            }
767        }
768        // After the map is unlocked: a push awaits every peer, and the table must not be
769        // held behind a slow one.
770        self.after_commit(&key, updates).await;
771        Ok(())
772    }
773
774    /// Bring the search index up to date with files a session has written.
775    ///
776    /// A replication session moves files, not index rows: the index belongs to the app
777    /// server, and the sync core is deliberately free of it. So the side that *applied* a
778    /// change re-indexes, which is what makes a file that arrived from a peer searchable
779    /// without a manual reindex — the claim the whole architecture rests on.
780    ///
781    /// Incremental, not a full rebuild: the workspace's mtime/size stamps mean only what
782    /// actually changed is read back. A file the session wrote has a new stamp, so it is
783    /// picked up; one that was already indexed and untouched is skipped.
784    ///
785    /// Best-effort by contract: the caller's session has already succeeded, and a failure
786    /// here must not be reported as the session's. It is logged and the next session, local
787    /// edit or explicit `workspace.reindex` tries again.
788    async fn reindex(&self, root: &Path) {
789        let Some(host) = self.host.get() else {
790            return;
791        };
792        let host = Arc::clone(host);
793        let root = root.to_owned();
794        // One sweep at a time: two sessions that finish together would otherwise both walk
795        // the workspace through the same retrieve store.
796        let _serialised = self.reindexing.lock().await;
797        // Opening the workspace is itself blocking work, and it is where the exclusive
798        // retrieve-store lock is taken, so it happens here rather than inside the sweep.
799        let backend = match host.backend_at(&root).await {
800            Ok(backend) => backend,
801            Err(err) => {
802                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
803                return;
804            }
805        };
806        let state = Arc::clone(backend.state());
807        let result = tokio::task::spawn_blocking(move || state.sync_retrieve()).await;
808        match result {
809            Ok(Ok(report)) => {
810                tracing::debug!(root = %root.display(), ?report, "re-indexed after a session");
811            }
812            Ok(Err(err)) => {
813                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
814            }
815            Err(err) => {
816                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
817            }
818        }
819    }
820
821    /// Run an embedding pass over `root` in the background, after the pass already
822    /// waiting for it if there is one (#187).
823    ///
824    /// The pass reads the vector files that arrived, and computes the missing ones this
825    /// host is responsible for: the files whose winning version this replica wrote, and —
826    /// on the primary device — every other one. It never holds up the caller.
827    pub fn request_embed(self: &Arc<Self>, root: &Path) {
828        let Ok(key) = root.canonicalize() else {
829            return;
830        };
831        let runtime = Arc::clone(self);
832        self.spawn_background(async move {
833            if !runtime.embed_wanted.lock().await.insert(key.clone()) {
834                return;
835            }
836            let _one_at_a_time = runtime.embedding.lock().await;
837            // Taken off the list as the pass starts: a change during the pass asks again.
838            runtime.embed_wanted.lock().await.remove(&key);
839            runtime.embed_running.lock().await.insert(key.clone());
840            runtime.embed_pass(&key).await;
841            runtime.embed_running.lock().await.remove(&key);
842        });
843    }
844
845    /// One embedding pass over `key`. Best-effort, like the re-index: logged, never fatal.
846    async fn embed_pass(&self, key: &Path) {
847        let Some(host) = self.host.get() else {
848            return;
849        };
850        let (workspace_id, replica) = {
851            let synced = self.synced.lock().await;
852            match synced.get(key) {
853                Some(entry) => (entry.workspace_id, Arc::clone(&entry.replica)),
854                None => return,
855            }
856        };
857        let backend = match host.backend_at(key).await {
858            Ok(backend) => backend,
859            Err(err) => {
860                tracing::warn!(root = %key.display(), "embedding pass: {err}");
861                return;
862            }
863        };
864        let state = Arc::clone(backend.state());
865        if let Err(err) = state
866            .refresh_embedder_at(self.bridge_endpoint.clone())
867            .await
868        {
869            tracing::warn!(root = %key.display(), "embedding pass: {err}");
870            return;
871        }
872        if state.embedder().is_none() {
873            return;
874        }
875        let primary = self.is_primary(workspace_id).await;
876        // The replica's view, read once: which content each path has, and which of those
877        // versions this replica wrote.
878        let grace_ms = self.backfill_timing().grace.as_millis() as u64;
879        let now_ms = std::time::SystemTime::now()
880            .duration_since(std::time::UNIX_EPOCH)
881            .map_or(0, |d| d.as_millis() as u64);
882        let (mine, settled, live) = {
883            let replica = replica.lock().await;
884            let me = replica.replica_id();
885            let states = match replica.states() {
886                Ok(states) => states,
887                Err(err) => {
888                    tracing::warn!(root = %key.display(), "embedding pass: {err}");
889                    return;
890                }
891            };
892            let mut mine = HashMap::new();
893            // Other devices' versions old enough for the primary to fill in.
894            let mut settled = HashMap::new();
895            let mut live = std::collections::HashSet::new();
896            for (path, path_state) in states {
897                let winner = path_state.winner();
898                let Some(hash) = winner.content.hash() else {
899                    continue;
900                };
901                let hash = hash.to_hex();
902                if winner.dot.replica == me {
903                    mine.insert(path, hash.clone());
904                } else if now_ms.saturating_sub(winner.hlc.wall_ms) >= grace_ms {
905                    settled.insert(path, hash.clone());
906                }
907                live.insert(hash);
908            }
909            (mine, settled, live)
910        };
911        let root = key.to_owned();
912        let result = tokio::task::spawn_blocking(move || {
913            // The author embeds its own versions at once; the primary device fills in other
914            // devices' versions once they have settled, racing nobody.
915            let policy = |rel: &str, hash: &str| {
916                mine.get(rel).is_some_and(|h| h.as_str() == hash)
917                    || (primary && settled.get(rel).is_some_and(|h| h.as_str() == hash))
918            };
919            let embedded = state.embed_pending_with(&policy, PASS_BATCH)?;
920            let removed = if primary {
921                state.remove_stale_vectors(&live, STALE_VECTOR_AGE)
922            } else {
923                0
924            };
925            Ok::<_, sapphire_workspace::Error>((embedded, removed))
926        })
927        .await;
928        match result {
929            Ok(Ok((embedded, removed))) if embedded > 0 || removed > 0 => tracing::info!(
930                root = %root.display(),
931                embedded,
932                removed,
933                "embedding pass"
934            ),
935            Ok(Ok(_)) => {}
936            Ok(Err(err)) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
937            Err(err) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
938        }
939    }
940
941    /// Open a session with every peer that will take one.
942    pub async fn sync_now(self: &Arc<Self>, root: &Path) -> Result<()> {
943        let Ok(key) = root.canonicalize() else {
944            return Ok(());
945        };
946        let workspace_id = {
947            let synced = self.synced.lock().await;
948            match synced.get(&key) {
949                Some(entry) => entry.workspace_id,
950                None => return Ok(()),
951            }
952        };
953        let peers = self
954            .bridge()?
955            .peers()
956            .await
957            .map_err(|e| Error::Bridge(e.to_string()))?;
958        *self.last_peers.lock().await = Some(peers.clone());
959        let me = self.device_id().await?;
960
961        // The same direction rule the dial loop follows: a host only dials peers with a
962        // greater device id. Dialling the other way from here would reintroduce the
963        // dial-both-ways deadlock the loop was cured of, just on a different trigger.
964        // Inside a star, a pair with neither hub in it does not link at all.
965        let roles = peers.roles_for(workspace_id).cloned();
966        // Every peer at once, so one that is offline costs the others nothing (#228). On
967        // this task, not spawned: cancelling the caller — the server's shutdown aborting the
968        // watcher — must drop every dial with it, since each one pins the runtime.
969        let mut dials: Vec<Dial<'_>> = Vec::new();
970        for peer in peers
971            .peers
972            .iter()
973            .filter(|p| topology::link(me, p.device_id, roles.as_ref()) == topology::Link::Dial)
974            .cloned()
975            .collect::<Vec<_>>()
976        {
977            // A peer that already has a live session is caught up by it: a second exchange
978            // on the same workspace would be redundant, and a second stream between the same
979            // pair is exactly what the tie-break cannot settle.
980            if let Some(live) = self.live_peers(&key).await
981                && live.has(&peer.device_id).await
982            {
983                continue;
984            }
985            // One dial per peer at a time. `sync_now` runs beside the dial loop, and two
986            // overlapping outbound dials to one peer are the one case the tie-break rule
987            // cannot resolve — each end would see its own stream as `outbound`. Whoever
988            // arrives second simply leaves it to the next pass. The table is made here if
989            // the dial loop has not yet: the first dial after enabling must not depend on
990            // a tick that has not fired.
991            let Some((_, table)) = self.live_peers_or_create(&key).await else {
992                continue;
993            };
994            if !table.begin_dial(peer.device_id).await {
995                continue;
996            }
997            let key = &key;
998            dials.push(Box::pin(async move {
999                // A peer that does not host this workspace refuses, which is normal and cheap.
1000                let result = self
1001                    .sync_one(key, &peer.device_id, peer.name.clone(), workspace_id)
1002                    .await;
1003                table.end_dial(peer.device_id).await;
1004                if let Err(err) = result {
1005                    tracing::debug!(peer = %peer.name, "no session: {err}");
1006                }
1007            }));
1008        }
1009        join_all(dials).await;
1010        Ok(())
1011    }
1012
1013    /// Dial one peer and adopt the stream as a live session.
1014    ///
1015    /// The one-shot path (`run_session`) would close the stream the moment the exchange
1016    /// finished — while the accepting peer keeps its half open and pushes into it, a session
1017    /// that dies with nothing reading it. So the dialled side is adopted live too, and both
1018    /// ends of a pair agree that a session stays open.
1019    async fn sync_one(
1020        self: &Arc<Self>,
1021        key: &Path,
1022        device: &GrainId,
1023        name: String,
1024        workspace_id: GrainId,
1025    ) -> Result<()> {
1026        let stream = self.open_stream(workspace_id, *device).await?;
1027        self.adopt_live_session(stream, key, *device, workspace_id)
1028            .await
1029            .map_err(|err| {
1030                tracing::warn!(peer = %name, "a live session could not be started: {err}");
1031                err
1032            })
1033    }
1034
1035    /// Ask the bridge for a stream to `device`, giving up after `open_timeout`.
1036    async fn open_stream(
1037        &self,
1038        workspace_id: GrainId,
1039        device: GrainId,
1040    ) -> Result<sapphire_ipc::RawStream> {
1041        match tokio::time::timeout(
1042            self.open_timeout,
1043            self.bridge()?.open_stream(workspace_id, device),
1044        )
1045        .await
1046        {
1047            Ok(opened) => Ok(opened?),
1048            Err(_) => Err(Error::Bridge(format!(
1049                "no stream to {device} within {} s; it may be offline",
1050                self.open_timeout.as_secs_f32()
1051            ))),
1052        }
1053    }
1054
1055    /// Answer the bridge's announcements, following the connection across reconnections
1056    /// (#244). Runs until the task is aborted.
1057    pub async fn run(self: Arc<Self>) -> Result<()> {
1058        let mut slot = self.bridge.subscribe();
1059        loop {
1060            let bridge = slot.borrow_and_update().clone();
1061            match bridge {
1062                Some(bridge) if !bridge.is_closed() => self.answer(&bridge, &mut slot).await,
1063                // Wait for a connection. The runtime holds the sender, so this never ends.
1064                _ => {
1065                    if slot.changed().await.is_err() {
1066                        return Ok(());
1067                    }
1068                }
1069            }
1070        }
1071    }
1072
1073    /// Answer `bridge`'s announcements until it closes or another connection replaces it.
1074    async fn answer(
1075        self: &Arc<Self>,
1076        bridge: &Arc<BridgeClient>,
1077        slot: &mut tokio::sync::watch::Receiver<Option<Arc<BridgeClient>>>,
1078    ) {
1079        let early = self
1080            .subscribed
1081            .lock()
1082            .expect("subscribed")
1083            .take_if(|(b, _)| Arc::ptr_eq(b, bridge));
1084        let mut incoming = match early {
1085            Some((_, incoming)) => incoming,
1086            None => bridge.incoming(),
1087        };
1088        loop {
1089            let received = tokio::select! {
1090                received = incoming.recv() => received,
1091                () = bridge.closed() => return,
1092                _ = slot.changed() => return,
1093            };
1094            let announcement = match received {
1095                Ok(a) => a,
1096                Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
1097                    tracing::warn!(missed = n, "fell behind on bridge announcements");
1098                    continue;
1099                }
1100                Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1101            };
1102
1103            let found = {
1104                let synced = self.synced.lock().await;
1105                synced
1106                    .iter()
1107                    .find(|(_, s)| s.workspace_id == announcement.workspace_id)
1108                    .map(|(root, s)| (root.clone(), Arc::clone(&s.replica)))
1109            };
1110            // `replica` is still what the accepted session shares, and `root` is where its
1111            // files and its index live.
1112            let Some((root, replica)) = found else {
1113                // The bridge routed to us for a workspace we no longer hold. Not fatal:
1114                // ignore it and let the ticket expire.
1115                tracing::debug!(
1116                    workspace = %announcement.workspace_id,
1117                    "an announcement for a workspace this server does not hold"
1118                );
1119                continue;
1120            };
1121
1122            let bridge = Arc::clone(bridge);
1123            let workspace_id = announcement.workspace_id;
1124            let peer = announcement.peer_device_id;
1125            let driver = Arc::clone(self);
1126            self.spawn_background(async move {
1127                let stream = match bridge.accept_stream(announcement.ticket).await {
1128                    Ok(stream) => stream,
1129                    Err(err) => {
1130                        tracing::warn!("could not claim an announced stream: {err}");
1131                        return;
1132                    }
1133                };
1134                if driver.should_skip(workspace_id, peer).await {
1135                    tracing::debug!(%peer, "dropped an inbound session: both ends are outside this workspace's star");
1136                    return;
1137                }
1138                // Live, not one-shot: the peer that dialled is keeping this session open and
1139                // pushing what it commits, so this side has to keep reading. A session that
1140                // closed after the exchange would leave the dialler pushing into a stream
1141                // nobody reads.
1142                let (_, session) = match sapphire_framework_session::open_live_session(
1143                    stream,
1144                    Arc::clone(&replica),
1145                    workspace_id,
1146                )
1147                .await
1148                {
1149                    Ok(started) => started,
1150                    Err(err) => {
1151                        tracing::warn!("an inbound session failed: {err}");
1152                        return;
1153                    }
1154                };
1155                let Some((key, table)) = driver.live_peers_or_create(&root).await else {
1156                    return;
1157                };
1158                // The peer dialled: this stream survives only if the *peer* has the lower
1159                // device id. A stream that loses is dropped, which closes it, and the
1160                // connection ends tidily rather than being left half-read.
1161                let Some(updates) = table.insert(peer, session, false).await else {
1162                    return;
1163                };
1164                table.catch_up(peer, &replica).await;
1165                // This is the receiving side: the files are on disk now and nobody else will
1166                // index them.
1167                driver.reindex(&key).await;
1168                driver.request_embed(&key);
1169                driver.spawn_reader(table, updates, key, peer);
1170            });
1171        }
1172    }
1173
1174    /// Make the first attempt to reach the bridge, when the runtime has a
1175    /// [`BridgeConnector`] and is not connected yet.
1176    ///
1177    /// No bridge running is not an error: the server serves without sync, and
1178    /// [`keep_connected`](Self::keep_connected) picks the bridge up once it starts. Any
1179    /// other failure — a bridge of another API version, say — is, and stops the start.
1180    pub async fn connect(self: &Arc<Self>) -> Result<()> {
1181        let Some(connector) = &self.connector else {
1182            return Ok(());
1183        };
1184        if self.is_connected() {
1185            return Ok(());
1186        }
1187        match connector.connect().await? {
1188            Some(client) => self.attach(client).await,
1189            None => tracing::warn!(
1190                "no sapphire-bridge is running; serving without sync until it starts"
1191            ),
1192        }
1193        Ok(())
1194    }
1195
1196    /// Stay connected to the bridge: wait for the connection to close, then reach the
1197    /// bridge again with the runtime's [`BridgeConnector`], backing off from
1198    /// [`RECONNECT_BACKOFF`] to [`RECONNECT_BACKOFF_MAX`] while it cannot (#244).
1199    ///
1200    /// Runs until the task is aborted, which is what the app server's shutdown does with
1201    /// it. Without a connector it only marks the runtime disconnected once the connection
1202    /// closes, and returns.
1203    pub async fn keep_connected(self: Arc<Self>) {
1204        let mut attempt = 0;
1205        loop {
1206            if let Ok(bridge) = self.bridge() {
1207                bridge.closed().await;
1208                tracing::warn!("lost the connection to sapphire-bridge; reconnecting");
1209                self.bridge.send_if_modified(|slot| {
1210                    let same = slot.as_ref().is_some_and(|b| Arc::ptr_eq(b, &bridge));
1211                    if same {
1212                        *slot = None;
1213                    }
1214                    same
1215                });
1216                attempt = 0;
1217                continue;
1218            }
1219            let Some(connector) = &self.connector else {
1220                return;
1221            };
1222            match connector.connect().await {
1223                Ok(Some(client)) => {
1224                    tracing::info!("connected to sapphire-bridge");
1225                    self.attach(client).await;
1226                    attempt = 0;
1227                    continue;
1228                }
1229                Ok(None) => tracing::debug!("no sapphire-bridge is running yet"),
1230                // Logged once per outage, not once per attempt.
1231                Err(err) if attempt == 0 => {
1232                    tracing::warn!("could not connect to sapphire-bridge: {err}")
1233                }
1234                Err(err) => tracing::debug!("could not connect to sapphire-bridge: {err}"),
1235            }
1236            tokio::time::sleep(reconnect_backoff(attempt)).await;
1237            attempt = attempt.saturating_add(1);
1238        }
1239    }
1240
1241    /// Start using `client`, and bring the bridge up to date: it knows nothing of this
1242    /// server yet, whether it has just started or this server has.
1243    async fn attach(self: &Arc<Self>, client: BridgeClient) {
1244        // Swapped under the pending lock, so an `enable` racing this either sees the new
1245        // connection or leaves its workspace for the drain below.
1246        let client = Arc::new(client);
1247        *self.subscribed.lock().expect("subscribed") =
1248            Some((Arc::clone(&client), client.incoming()));
1249        let waiting = {
1250            let mut pending = self.pending.lock().await;
1251            self.bridge.send_replace(Some(client));
1252            std::mem::take(&mut *pending)
1253        };
1254        // What was synced before the connection was lost.
1255        if !self.synced.lock().await.is_empty()
1256            && let Err(err) = self.reregister().await
1257        {
1258            tracing::warn!("could not register with sapphire-bridge: {err}");
1259        }
1260        // What was asked for while it was away. `enable` registers and dials each one.
1261        for root in waiting.into_keys() {
1262            if let Err(err) = self.enable(&root).await {
1263                tracing::warn!(root = %root.display(), "could not start sync: {err}");
1264            }
1265        }
1266    }
1267
1268    /// Every synced root, for the watcher.
1269    pub async fn roots(&self) -> Vec<PathBuf> {
1270        self.synced.lock().await.keys().cloned().collect()
1271    }
1272
1273    /// The live session table for `root`, if this runtime syncs it.
1274    ///
1275    /// `None` is not an error: a dial, a push or a drop for a workspace this server does not
1276    /// hold is a no-op, and every caller below treats it that way.
1277    async fn live_peers(&self, root: &Path) -> Option<Arc<LivePeers>> {
1278        let key = root.canonicalize().ok()?;
1279        self.live.lock().await.get(&key).cloned()
1280    }
1281
1282    /// The live session table for `root`, creating it if this runtime syncs it.
1283    ///
1284    /// The two locks are taken one after the other, never nested: `synced` to decide whether
1285    /// the workspace is ours at all, then `live` to get or make its table.
1286    async fn live_peers_or_create(&self, root: &Path) -> Option<(PathBuf, Arc<LivePeers>)> {
1287        let key = root.canonicalize().ok()?;
1288        if !self.synced.lock().await.contains_key(&key) {
1289            return None;
1290        }
1291        // A table is built by whoever walks the workspace first, and it needs this host's
1292        // device id — the tie-break between two simultaneous streams is decided from it.
1293        let me = self.device_id().await.ok()?;
1294        let mut live = self.live.lock().await;
1295        let peers = Arc::clone(
1296            live.entry(key.clone())
1297                .or_insert_with(|| Arc::new(LivePeers::new(me))),
1298        );
1299        Some((key, peers))
1300    }
1301
1302    /// Push `updates` to every open session for `root`.
1303    ///
1304    /// Called with what a scan recorded. `from` is always `None` here: a local commit came
1305    /// from this host, so no session is excluded. The forwarding path, which does name a
1306    /// source, is [`LivePeers::fan_out`](live::LivePeers::fan_out) called directly by the
1307    /// reader side of a session.
1308    ///
1309    /// Best-effort by contract: the commit already happened, and a peer that is gone is the
1310    /// dialer's problem, not the writer's.
1311    pub async fn after_commit(&self, root: &Path, updates: Vec<PathUpdate>) {
1312        if updates.is_empty() {
1313            return;
1314        }
1315        if let Some(peers) = self.live_peers(root).await {
1316            peers.fan_out(&updates, None).await;
1317        }
1318    }
1319
1320    /// The devices with an open live session for `root`.
1321    ///
1322    /// Used by tests to wait for the dialer; empty for a workspace this server does not hold.
1323    pub async fn live_session_devices(&self, root: &Path) -> Vec<GrainId> {
1324        match self.live_peers(root).await {
1325            Some(peers) => peers.devices().await,
1326            None => Vec::new(),
1327        }
1328    }
1329
1330    /// End the first dials [`enable`](SyncRuntime::enable) left running, waiting until
1331    /// they are gone.
1332    ///
1333    /// For the app server's shutdown: each one pins the runtime, and with it the replica
1334    /// stores, which are redb databases that stay locked — a dial outliving its server
1335    /// would lock the next one out.
1336    pub async fn stop_dials(&self) {
1337        let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1338        dials.abort_all();
1339        while dials.join_next().await.is_some() {}
1340    }
1341
1342    /// End the runtime's background work — inbound sessions, live-session readers and
1343    /// embedding passes — and wait until it has let go.
1344    ///
1345    /// The server's shutdown calls this, so a server restarted in the same process can open
1346    /// the replica stores again at once rather than whenever the last straggler finished
1347    /// (#235). An embedding pass already inside its blocking section runs that section to
1348    /// the end; it holds the workspace's index, never a replica.
1349    pub async fn stop_background(&self) {
1350        let mut background =
1351            std::mem::take(&mut *self.background.lock().expect("background tasks"));
1352        background.abort_all();
1353        while background.join_next().await.is_some() {}
1354    }
1355
1356    /// Wait for the first dials [`enable`](SyncRuntime::enable) left running to finish on
1357    /// their own: for a test that must not race them.
1358    #[cfg(test)]
1359    async fn settle_dials(&self) {
1360        let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1361        while dials.join_next().await.is_some() {}
1362    }
1363
1364    /// Close every open live session, as if the connections had been cut.
1365    ///
1366    /// Called for a workspace being disabled, and by tests that want to watch the dialer
1367    /// rebuild what was dropped.
1368    pub async fn drop_connections(&self) {
1369        let tables: Vec<Arc<LivePeers>> = self.live.lock().await.values().cloned().collect();
1370        for peers in tables {
1371            peers.drop_connections().await;
1372        }
1373    }
1374
1375    /// Keep a live session open to every peer, for every synced workspace.
1376    ///
1377    /// One walk per [`DIAL_INTERVAL`](live::DIAL_INTERVAL): for each synced workspace, ask
1378    /// the bridge who is in the workgroup, and open a session to every peer that is connected
1379    /// and does not already have one.
1380    ///
1381    /// A peer that cannot be reached is retried on a per-peer exponential backoff capped at
1382    /// [`DIAL_BACKOFF_MAX`] — one failed attempt per walk, not one per spin — and the counter
1383    /// resets when a session opens. An unreachable peer is therefore no obstacle to the
1384    /// others: its dial fails, the walk moves on, and the next workspace is dialled.
1385    ///
1386    /// Runs until the task is aborted, which is what the app server's shutdown does with it.
1387    /// It is allowed to fail like [`run`](SyncRuntime::run): only sync stops, and the app
1388    /// server keeps serving files (spec §10).
1389    pub async fn dial_loop(self: Arc<Self>) -> Result<()> {
1390        let mut tick = tokio::time::interval(live::DIAL_INTERVAL);
1391        // A missed tick is a late walk, not a burst of them: the interval is a pace, not a
1392        // quota to make up.
1393        tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1394        let mut waiting: HashMap<GrainId, std::time::Instant> = HashMap::new();
1395        let mut failures: HashMap<GrainId, u32> = HashMap::new();
1396        // The workspaces this host was the primary device of at the last walk.
1397        let mut primary_of: std::collections::HashSet<GrainId> = Default::default();
1398        // When the primary device last ran a backfill pass with nothing prompting it.
1399        let mut last_backfill = std::time::Instant::now();
1400        loop {
1401            tick.tick().await;
1402            let roots = self.roots().await;
1403            if roots.is_empty() {
1404                continue;
1405            }
1406            let peers = match async { self.bridge()?.peers().await.map_err(Error::from) }.await {
1407                Ok(peers) => {
1408                    *self.last_peers.lock().await = Some(peers.clone());
1409                    peers
1410                }
1411                // The bridge is down or has no workgroup: nothing to dial, and the next walk
1412                // asks again. Not an error for the caller — the app server is still serving.
1413                Err(err) => {
1414                    tracing::debug!("no peers to dial: {err}");
1415                    continue;
1416                }
1417            };
1418            let me = match self.device_id().await {
1419                Ok(me) => me,
1420                Err(err) => {
1421                    tracing::warn!("no device id, so nobody to dial: {err}");
1422                    continue;
1423                }
1424            };
1425            // A host that has just become a workspace's primary device owes it the vectors
1426            // nobody else computes (#187): files that arrived before the election settled
1427            // would otherwise wait for the next change.
1428            let ids: Vec<(PathBuf, GrainId)> = self
1429                .synced
1430                .lock()
1431                .await
1432                .iter()
1433                .map(|(root, entry)| (root.clone(), entry.workspace_id))
1434                .collect();
1435            // And every so often anyway: an author that never embedded leaves files only the
1436            // primary will fill in, and nothing else would prompt it (#188).
1437            let backfill_due = last_backfill.elapsed() >= self.backfill_timing().interval;
1438            if backfill_due {
1439                last_backfill = std::time::Instant::now();
1440            }
1441            for (root, workspace_id) in ids {
1442                let primary = peers
1443                    .roles_for(workspace_id)
1444                    .is_some_and(|r| r.primary == Some(me));
1445                if !primary {
1446                    primary_of.remove(&workspace_id);
1447                } else if primary_of.insert(workspace_id) || backfill_due {
1448                    self.request_embed(&root);
1449                }
1450            }
1451            let now = std::time::Instant::now();
1452            // Dial only peers whose device id is greater than ours. Every device dialling
1453            // every other can deadlock: two hosts that dial each other at once each hold
1454            // their replica's lock across their own exchange and wait for the other's
1455            // hello, which the other's dialler holds the lock for. With one direction per
1456            // pair the wait chain is a DAG — a higher-id host never dials, so it never
1457            // holds its lock out while a lower-id host's exchange needs it. The lower host
1458            // dials within `DIAL_INTERVAL`, and the exchange is symmetric, so nothing is
1459            // lost; the tie-break is left as the safety net it is.
1460            for root in roots {
1461                let Some((key, table)) = self.live_peers_or_create(&root).await else {
1462                    continue;
1463                };
1464                let workspace_id = {
1465                    let synced = self.synced.lock().await;
1466                    match synced.get(&key) {
1467                        Some(entry) => entry.workspace_id,
1468                        None => continue,
1469                    }
1470                };
1471                let roles = peers.roles_for(workspace_id);
1472                // Entering a star closes what it no longer needs. The hubs carry it all —
1473                // once one of them is reachable from here (#192).
1474                if topology::may_close_skipped(me, roles, &table.devices().await) {
1475                    table
1476                        .retain(|d| topology::link(me, *d, roles) != topology::Link::Skip)
1477                        .await;
1478                }
1479                for peer in peers.peers.iter().filter(|p| {
1480                    p.connected && topology::link(me, p.device_id, roles) == topology::Link::Dial
1481                }) {
1482                    let device = peer.device_id;
1483                    if table.has(&device).await {
1484                        // Already talking; a redial would drop a working session.
1485                        failures.remove(&device);
1486                        waiting.remove(&device);
1487                        continue;
1488                    }
1489                    if let Some(until) = waiting.get(&device)
1490                        && now < *until
1491                    {
1492                        // Backing off after a failure; the walk skips it until then.
1493                        continue;
1494                    }
1495                    // The same claim `sync_now` takes: this walk and it can overlap, and a
1496                    // double dial in one direction is the one race the tie-break cannot
1497                    // settle. Skipping is free — the next pass, or `sync_now` itself, dials.
1498                    if !table.begin_dial(device).await {
1499                        continue;
1500                    }
1501                    let opened = self.open_stream(workspace_id, device).await;
1502                    if opened.is_err() {
1503                        table.end_dial(device).await;
1504                    }
1505                    match opened {
1506                        Ok(stream) => {
1507                            let adopted = self
1508                                .adopt_live_session(stream, &root, device, workspace_id)
1509                                .await;
1510                            // The claim is out either way: the attempt is over, and a
1511                            // session that died mid-exchange is re-dialled next pass.
1512                            table.end_dial(device).await;
1513                            match adopted {
1514                                Ok(()) => {
1515                                    failures.remove(&device);
1516                                    waiting.remove(&device);
1517                                }
1518                                Err(err) => {
1519                                    tracing::debug!(
1520                                        peer = %peer.name,
1521                                        err = %err,
1522                                        "a live session could not be started"
1523                                    );
1524                                    let count = failures.entry(device).or_insert(0);
1525                                    *count = count.saturating_add(1);
1526                                    waiting.insert(device, now + dial_backoff(*count));
1527                                }
1528                            }
1529                        }
1530                        Err(err) => {
1531                            // Normal and cheap: a peer that does not host this workspace
1532                            // refuses, and one that is unreachable cannot be opened.
1533                            tracing::debug!("no stream to a peer: {err}");
1534                            let count = failures.entry(device).or_insert(0);
1535                            *count = count.saturating_add(1);
1536                            waiting.insert(device, now + dial_backoff(*count));
1537                        }
1538                    }
1539                }
1540            }
1541        }
1542    }
1543
1544    /// Take ownership of an opened stream: run the exchange, then keep it live.
1545    ///
1546    /// The session is stored under `device` and its reader pumped until the stream ends. A
1547    /// half-open or refused stream is not stored; the dialer retries it with backoff.
1548    async fn adopt_live_session(
1549        self: &Arc<Self>,
1550        stream: sapphire_ipc::RawStream,
1551        root: &Path,
1552        device: GrainId,
1553        workspace_id: GrainId,
1554    ) -> Result<()> {
1555        let replica = self.replica_of(root).await?;
1556        let (_, session) = sapphire_framework_session::open_live_session(
1557            stream,
1558            Arc::clone(&replica),
1559            workspace_id,
1560        )
1561        .await
1562        .map_err(|e| Error::Sync(e.to_string()))?;
1563        let Some((key, table)) = self.live_peers_or_create(root).await else {
1564            return Ok(());
1565        };
1566        // This stream was dialled by us, so it is the one the pair keeps if this host's
1567        // device id is the lower of the two. If it lost the tie-break the session is
1568        // dropped here, which closes the stream, and the surviving one is already in the
1569        // table.
1570        let Some(updates) = table.insert(device, session, true).await else {
1571            return Ok(());
1572        };
1573        table.catch_up(device, &replica).await;
1574        // The session wrote files during the exchange; the index has to catch up before the
1575        // workspace is searchable again.
1576        self.reindex(&key).await;
1577        self.request_embed(&key);
1578        self.spawn_reader(Arc::clone(&table), updates, key, device);
1579        Ok(())
1580    }
1581
1582    /// Forward what one session receives, and re-index it.
1583    ///
1584    /// This is the loop that makes A → S → B work: a batch that arrived on S's session with A
1585    /// is applied (the session crate does that before publishing it), the workspace behind
1586    /// `key` is re-indexed, and the batch is handed to every *other* session whose peer lacks
1587    /// it — `from` is the device it came from, so it never goes back to A.
1588    fn spawn_reader(
1589        self: &Arc<Self>,
1590        table: Arc<LivePeers>,
1591        mut updates: broadcast::Receiver<Vec<PathUpdate>>,
1592        key: PathBuf,
1593        from: GrainId,
1594    ) {
1595        let driver = Arc::clone(self);
1596        self.spawn_background(async move {
1597            loop {
1598                match updates.recv().await {
1599                    Ok(batch) => {
1600                        // The files are on disk now (the session guarantees it before
1601                        // publishing), so the index is behind them until the sweep below.
1602                        driver.reindex(&key).await;
1603                        driver.request_embed(&key);
1604                        table.fan_out(&batch, Some(from)).await;
1605                    }
1606                    // A burst bigger than the channel: the newest is kept and the gap is
1607                    // covered by the next session's exchange. Losing a batch here cannot lose
1608                    // an entry — the peer's vector does not cover it, so the next dial or push
1609                    // carries it again.
1610                    Err(broadcast::error::RecvError::Lagged(missed)) => {
1611                        tracing::debug!(
1612                            device = %from,
1613                            missed,
1614                            "fell behind on a live session; the next exchange catches up"
1615                        );
1616                    }
1617                    // The session closed. Its table entry is reaped by the next fan-out or
1618                    // dial pass, and the dialer opens a fresh one.
1619                    Err(broadcast::error::RecvError::Closed) => {
1620                        return;
1621                    }
1622                }
1623            }
1624        });
1625    }
1626
1627    /// The replica of a synced `root`.
1628    async fn replica_of(&self, root: &Path) -> Result<Arc<Mutex<Replica>>> {
1629        let key = root.canonicalize().map_err(Error::Io)?;
1630        let synced = self.synced.lock().await;
1631        match synced.get(&key) {
1632            Some(entry) => Ok(Arc::clone(&entry.replica)),
1633            None => Err(Error::UnknownWorkspace(key, self.ctx.app_name)),
1634        }
1635    }
1636
1637    /// Watch the synced roots and, on a debounced report, scan and dial.
1638    ///
1639    /// This is the safety net for everything the app server did not do itself — a file
1640    /// edited in an editor, a `git checkout`. The exact path is [`scan`] called from
1641    /// `handlers.rs` right after a write the server made. A scan that finds nothing is
1642    /// cheap; a scan that is skipped loses an edit until the next one.
1643    ///
1644    /// Returns when the reporting channel closes — that is, when the [`Watcher`] this
1645    /// runtime holds is dropped, which happens when the runtime itself is.
1646    ///
1647    /// [`scan`]: SyncRuntime::scan
1648    pub async fn watch_changes(self: Arc<Self>) -> Result<()> {
1649        let (tx, mut rx) = mpsc::channel(64);
1650        let watcher = Arc::new(Watcher::start(self.roots().await, tx)?);
1651        let _ = self.watcher.set(Arc::clone(&watcher));
1652
1653        // A root enabled between `roots()` above and the `set` just now was not in the
1654        // starting set, and `enable` could not reach a watcher that did not exist yet.
1655        // Watching the current set again is idempotent on `notify`'s side.
1656        for root in self.roots().await {
1657            if let Err(err) = watcher.watch(&root) {
1658                tracing::warn!(root = %root.display(), "could not watch: {err}");
1659            }
1660        }
1661
1662        while let Some(root) = rx.recv().await {
1663            // Logged, not fatal: a failed scan or dial leaves the replica behind, and the
1664            // next report tries again. Neither may take the app server down (spec §10).
1665            if let Err(err) = self.scan(&root).await {
1666                tracing::warn!(root = %root.display(), "scan after a local edit failed: {err}");
1667            }
1668            self.request_embed(&root);
1669            if let Err(err) = self.sync_now(&root).await {
1670                tracing::warn!(root = %root.display(), "dial after a local edit failed: {err}");
1671            }
1672        }
1673        Ok(())
1674    }
1675
1676    /// Tell the bridge the complete current set. A registration is not a delta.
1677    async fn reregister(&self) -> Result<()> {
1678        let workspaces: Vec<WorkspaceRegistration> = {
1679            let synced = self.synced.lock().await;
1680            synced
1681                .iter()
1682                .map(|(root, entry)| WorkspaceRegistration {
1683                    workspace_id: entry.workspace_id,
1684                    root: root.clone(),
1685                })
1686                .collect()
1687        };
1688        self.bridge()?
1689            .register(RegisterParams {
1690                app_name: self.ctx.app_name.to_owned(),
1691                exe_path: self.exe_path.clone(),
1692                managed_by: wire_managed_by(self.managed_by),
1693                workspaces,
1694            })
1695            .await
1696            .map_err(|e| Error::Bridge(e.to_string()))?;
1697        Ok(())
1698    }
1699
1700    /// Whether the last roles the bridge reported say this pair does not link.
1701    ///
1702    /// Reads the cached device id only: asking the bridge here would register with an empty
1703    /// workspace list, racing `enable`'s registration. `false` (accept) when it is unknown.
1704    async fn should_skip(&self, workspace_id: GrainId, peer: GrainId) -> bool {
1705        let Some(&me) = self.device_id.get() else {
1706            return false;
1707        };
1708        let cached = self.last_peers.lock().await;
1709        let roles = cached.as_ref().and_then(|p| p.roles_for(workspace_id));
1710        topology::link(me, peer, roles) == topology::Link::Skip
1711    }
1712
1713    /// Whether this host is `workspace_id`'s primary device, by the last roles the
1714    /// bridge reported. `false` when the bridge has not answered yet, or when this host's
1715    /// device id is not known yet (no workspace was enabled). Reads the cached id only:
1716    /// asking the bridge here would register with an empty workspace list, racing
1717    /// `enable`'s registration.
1718    pub async fn is_primary(&self, workspace_id: GrainId) -> bool {
1719        let Some(&me) = self.device_id.get() else {
1720            return false;
1721        };
1722        let cached = self.last_peers.lock().await;
1723        cached
1724            .as_ref()
1725            .and_then(|p| p.roles_for(workspace_id))
1726            .is_some_and(|r| r.primary == Some(me))
1727    }
1728
1729    /// This host's device id, asked of the bridge once.
1730    async fn device_id(&self) -> Result<GrainId> {
1731        if let Some(id) = self.device_id.get() {
1732            return Ok(*id);
1733        }
1734        let result = self
1735            .bridge()?
1736            .register(RegisterParams {
1737                app_name: self.ctx.app_name.to_owned(),
1738                exe_path: self.exe_path.clone(),
1739                managed_by: wire_managed_by(self.managed_by),
1740                workspaces: Vec::new(),
1741            })
1742            .await
1743            .map_err(|e| Error::Bridge(e.to_string()))?;
1744        let _ = self.device_id.set(result.device_id);
1745        Ok(result.device_id)
1746    }
1747
1748    /// `<workspace cache dir>/sync/`.
1749    fn state_dir(&self, root: &Path) -> Result<PathBuf> {
1750        let workspace = Workspace::from_root(self.ctx, root)?;
1751        Ok(workspace.cache_dir().join("sync"))
1752    }
1753}
1754
1755/// One dial of [`SyncRuntime::sync_now`].
1756type Dial<'a> = std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>>;
1757
1758/// Run `dials` concurrently on the calling task, until every one has finished.
1759async fn join_all(mut dials: Vec<Dial<'_>>) {
1760    std::future::poll_fn(|cx| {
1761        dials.retain_mut(|dial| dial.as_mut().poll(cx).is_pending());
1762        if dials.is_empty() {
1763            std::task::Poll::Ready(())
1764        } else {
1765            std::task::Poll::Pending
1766        }
1767    })
1768    .await
1769}
1770
1771/// The bridge API's copy of how this server was started, for [`RegisterParams`].
1772fn wire_managed_by(managed_by: ManagedBy) -> sapphire_bridge_api::ManagedBy {
1773    match managed_by {
1774        ManagedBy::Service => sapphire_bridge_api::ManagedBy::Service,
1775        ManagedBy::Spawned => sapphire_bridge_api::ManagedBy::Spawned,
1776    }
1777}
1778
1779#[cfg(test)]
1780mod tests {
1781    use super::*;
1782    use crate::sync::testing::StubBridge;
1783    use crate::test_support;
1784    use sapphire_workspace::AppKind;
1785    use std::ffi::OsString;
1786
1787    static CTX: AppContext = AppContext::new("sapphire-synctest");
1788
1789    /// The env vars `CTX.init` reads.
1790    const DIR_VARS: [&str; 3] = [
1791        "SAPPHIRE_SYNCTEST_CACHE_DIR",
1792        "SAPPHIRE_SYNCTEST_DATA_DIR",
1793        "SAPPHIRE_SYNCTEST_CONFIG_DIR",
1794    ];
1795
1796    /// Points the context's directories at the test's scratch tree, and restores the
1797    /// previous values when dropped — including while unwinding from a panic.
1798    struct EnvGuard {
1799        previous: [Option<OsString>; 3],
1800        _lock: std::sync::MutexGuard<'static, ()>,
1801    }
1802
1803    impl Drop for EnvGuard {
1804        fn drop(&mut self) {
1805            // SAFETY: `self._lock` still serialises the environment; it is dropped only
1806            // after this method returns.
1807            for (name, previous) in DIR_VARS.iter().zip(self.previous.iter_mut()) {
1808                match previous.take() {
1809                    Some(value) => unsafe { std::env::set_var(name, value) },
1810                    None => test_support::remove(name),
1811                }
1812            }
1813        }
1814    }
1815
1816    /// Everything one test needs, held for the test's whole body.
1817    ///
1818    /// The context is a `static` shared by this binary, so its directories are resolved from
1819    /// the environment by whichever test initialises it first. The environment lock is
1820    /// therefore held for the whole test, as in the crate's other test modules, and `_tmp`
1821    /// is declared before `_env` so the scratch tree is gone only once the environment no
1822    /// longer points at it.
1823    struct Fixture {
1824        _tmp: tempfile::TempDir,
1825        _env: EnvGuard,
1826        root: PathBuf,
1827        stub: StubBridge,
1828        runtime: Arc<SyncRuntime>,
1829    }
1830
1831    impl Fixture {
1832        /// A second workspace root beside the first, marker directory and all.
1833        fn second_root(&self) -> PathBuf {
1834            let root = self._tmp.path().join("ws2");
1835            std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
1836            root.canonicalize().unwrap()
1837        }
1838    }
1839
1840    /// How long the fixture's runtime waits on a stream to a peer.
1841    const FIXTURE_OPEN_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
1842
1843    /// `n` peers this host dials (a greater device id), as `bridge.peers` lists them.
1844    fn dialled_peers(me: GrainId, n: usize) -> Vec<sapphire_bridge_api::PeerInfo> {
1845        std::iter::repeat_with(GrainId::random)
1846            .filter(|id| *id > me)
1847            .take(n)
1848            .enumerate()
1849            .map(|(i, device_id)| sapphire_bridge_api::PeerInfo {
1850                device_id,
1851                name: format!("offline-{i}"),
1852                node_id: format!("node-{i}"),
1853                connected: false,
1854                priority: 1,
1855                availability: None,
1856            })
1857            .collect()
1858    }
1859
1860    #[tokio::test(flavor = "multi_thread")]
1861    async fn enabling_answers_at_once_with_a_peer_that_never_answers() {
1862        // #228: the first dial waited on a peer that was offline, past the client's limit.
1863        let f = fixture().await;
1864        let _hung = f.stub.hang_streams().await;
1865        f.stub.set_peers(dialled_peers(f.stub.device_id, 1));
1866
1867        let started = std::time::Instant::now();
1868        f.runtime.enable(&f.root).await.unwrap();
1869        assert!(
1870            started.elapsed() < FIXTURE_OPEN_TIMEOUT,
1871            "enable waited {:?} on the dial",
1872            started.elapsed()
1873        );
1874    }
1875
1876    #[tokio::test(flavor = "multi_thread")]
1877    async fn peers_that_never_answer_are_dialled_together_and_given_up_on() {
1878        let f = fixture().await;
1879        let _hung = f.stub.hang_streams().await;
1880        f.runtime.enable(&f.root).await.unwrap();
1881        // The first dial, with nobody to dial yet, must be over before the peers appear:
1882        // one still running would claim them, and `sync_now` would skip them all.
1883        f.runtime.settle_dials().await;
1884        f.stub.set_peers(dialled_peers(f.stub.device_id, 3));
1885
1886        let started = std::time::Instant::now();
1887        f.runtime.sync_now(&f.root).await.unwrap();
1888        let took = started.elapsed();
1889        assert!(took >= FIXTURE_OPEN_TIMEOUT, "gave up early: {took:?}");
1890        assert!(
1891            took < FIXTURE_OPEN_TIMEOUT * 2,
1892            "three dials one after another: {took:?}"
1893        );
1894    }
1895
1896    async fn fixture() -> Fixture {
1897        let (tmp, env, root) = scratch();
1898        let (stub, client) = StubBridge::start_in(tmp.path().join("run")).await;
1899        let runtime = Arc::new(
1900            SyncRuntime::new(&CTX, client, "/bin/true".into(), ManagedBy::Service)
1901                .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
1902        );
1903        Fixture {
1904            _tmp: tmp,
1905            _env: env,
1906            root,
1907            stub,
1908            runtime,
1909        }
1910    }
1911
1912    /// The bridge connections a [`connecting`] fixture's connector hands out, in order;
1913    /// empty is "no bridge is running".
1914    type Bridges = Arc<std::sync::Mutex<std::collections::VecDeque<BridgeClient>>>;
1915
1916    /// A fixture whose runtime reaches the bridge through a connector, and has not yet.
1917    /// `f.stub`'s client is not handed out: push it to the returned queue for that.
1918    async fn connecting() -> (Fixture, Bridges, BridgeClient) {
1919        let (tmp, env, root) = scratch();
1920        let (stub, client) = StubBridge::start_client_in(tmp.path().join("run")).await;
1921        let bridges: Bridges = Arc::default();
1922        let connector = BridgeConnector::new({
1923            let bridges = Arc::clone(&bridges);
1924            move || {
1925                let next = bridges.lock().unwrap().pop_front();
1926                async move { Ok(next) }
1927            }
1928        });
1929        let runtime = Arc::new(
1930            SyncRuntime::connecting(&CTX, connector, "/bin/true".into(), ManagedBy::Service)
1931                .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
1932        );
1933        let f = Fixture {
1934            _tmp: tmp,
1935            _env: env,
1936            root,
1937            stub,
1938            runtime,
1939        };
1940        (f, bridges, client)
1941    }
1942
1943    /// Wait up to ten seconds for `done`.
1944    async fn eventually(what: &str, mut done: impl AsyncFnMut() -> bool) {
1945        let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
1946        while !done().await {
1947            assert!(tokio::time::Instant::now() < deadline, "never: {what}");
1948            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
1949        }
1950    }
1951
1952    #[tokio::test(flavor = "multi_thread")]
1953    async fn a_runtime_started_without_a_bridge_syncs_once_one_starts() {
1954        // #244: a server that started before its bridge never synced until restarted.
1955        let (f, bridges, client) = connecting().await;
1956        f.runtime.connect().await.unwrap();
1957        assert!(!f.runtime.is_connected());
1958
1959        let err = f.runtime.enable(&f.root).await.unwrap_err();
1960        assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
1961        let status = f.runtime.status(&f.root).await;
1962        assert!(status.enabled, "enabled, waiting for the bridge");
1963        assert!(!status.bridge_available);
1964        let id = status.workspace_id.expect("its id");
1965
1966        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
1967        bridges.lock().unwrap().push_back(client);
1968        eventually("the waiting workspace is registered", async || {
1969            f.stub.last_workspaces() == vec![id]
1970        })
1971        .await;
1972        assert_eq!(f.runtime.roots().await, vec![f.root.clone()]);
1973        assert!(f.runtime.status(&f.root).await.bridge_available);
1974        keeping.abort();
1975    }
1976
1977    #[tokio::test(flavor = "multi_thread")]
1978    async fn a_runtime_that_loses_its_bridge_registers_with_the_next_one() {
1979        let (f, bridges, client) = connecting().await;
1980        bridges.lock().unwrap().push_back(client);
1981        f.runtime.connect().await.unwrap();
1982        assert!(f.runtime.is_connected());
1983        let id = f.runtime.enable(&f.root).await.unwrap();
1984        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
1985
1986        f.stub.disconnect();
1987        eventually("the runtime notices", async || !f.runtime.is_connected()).await;
1988        let err = f.runtime.workspaces().await.unwrap_err();
1989        assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
1990
1991        let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
1992        bridges.lock().unwrap().push_back(client);
1993        eventually("the synced workspace is registered again", async || {
1994            next.last_workspaces() == vec![id]
1995        })
1996        .await;
1997        assert!(f.runtime.is_connected());
1998        keeping.abort();
1999    }
2000
2001    #[tokio::test(flavor = "multi_thread")]
2002    async fn disabling_while_the_bridge_is_away_forgets_the_waiting_workspace() {
2003        let (f, bridges, client) = connecting().await;
2004        f.runtime.enable(&f.root).await.unwrap_err();
2005        f.runtime.disable(&f.root).await.unwrap();
2006        assert!(!f.runtime.status(&f.root).await.enabled);
2007
2008        bridges.lock().unwrap().push_back(client);
2009        f.runtime.connect().await.unwrap();
2010        assert!(f.runtime.roots().await.is_empty());
2011        assert!(f.stub.last_workspaces().is_empty());
2012    }
2013
2014    #[tokio::test(flavor = "multi_thread")]
2015    async fn announcements_are_answered_from_a_reconnected_bridge() {
2016        let (f, bridges, client) = connecting().await;
2017        bridges.lock().unwrap().push_back(client);
2018        f.runtime.connect().await.unwrap();
2019        let id = f.runtime.enable(&f.root).await.unwrap();
2020        let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2021        let answering = tokio::spawn(Arc::clone(&f.runtime).run());
2022
2023        f.stub.disconnect();
2024        let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2025        bridges.lock().unwrap().push_back(client);
2026        eventually("reconnected", async || next.last_workspaces() == vec![id]).await;
2027
2028        // The announcement loop follows the new connection: an announcement on it is
2029        // claimed, which with no data endpoint fails — and the loop lives on.
2030        next.announce(id, "ticket").await;
2031        tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2032        assert!(!answering.is_finished(), "the announcement loop ended");
2033        answering.abort();
2034        keeping.abort();
2035    }
2036
2037    /// The scratch tree, the environment pointed at it, and a workspace root in it.
2038    fn scratch() -> (tempfile::TempDir, EnvGuard, PathBuf) {
2039        let lock = test_support::lock();
2040        let tmp = tempfile::tempdir().unwrap();
2041        let previous = DIR_VARS.map(std::env::var_os);
2042        // SAFETY (via `test_support::set`): `lock` serialises every read and write of the
2043        // process environment in this test binary, and it is held until `drop` has restored
2044        // the old values.
2045        for (name, dir) in DIR_VARS
2046            .iter()
2047            .zip(["cache", "data", "config"].map(|cat| tmp.path().join(cat)))
2048        {
2049            test_support::set(name, &dir);
2050        }
2051        let env = EnvGuard {
2052            previous,
2053            _lock: lock,
2054        };
2055        CTX.init(AppKind::Server);
2056
2057        let root = tmp.path().join("ws");
2058        std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
2059        let root = root.canonicalize().unwrap();
2060        (tmp, env, root)
2061    }
2062
2063    #[tokio::test(flavor = "multi_thread")]
2064    async fn enabling_registers_the_workspace_with_the_bridge() {
2065        let f = fixture().await;
2066
2067        let id = f.runtime.enable(&f.root).await.unwrap();
2068        assert_eq!(f.stub.last_workspaces(), vec![id]);
2069    }
2070
2071    #[tokio::test(flavor = "multi_thread")]
2072    async fn enabling_twice_is_idempotent_and_keeps_the_same_id() {
2073        let f = fixture().await;
2074
2075        let first = f.runtime.enable(&f.root).await.unwrap();
2076        let second = f.runtime.enable(&f.root).await.unwrap();
2077        assert_eq!(first, second);
2078        assert_eq!(
2079            f.stub.last_workspaces(),
2080            vec![first],
2081            "still exactly one workspace"
2082        );
2083    }
2084
2085    #[tokio::test(flavor = "multi_thread")]
2086    async fn a_registration_carries_the_complete_current_list() {
2087        let f = fixture().await;
2088        let second_root = f.second_root();
2089
2090        let a = f.runtime.enable(&f.root).await.unwrap();
2091        let b = f.runtime.enable(&second_root).await.unwrap();
2092
2093        let mut seen = f.stub.last_workspaces();
2094        seen.sort();
2095        let mut want = vec![a, b];
2096        want.sort();
2097        assert_eq!(seen, want, "registration is the whole set, not a delta");
2098    }
2099
2100    #[tokio::test(flavor = "multi_thread")]
2101    async fn no_later_registration_clears_this_apps_routes() {
2102        // The device id is asked of the bridge once, on the first `enable`, with an empty
2103        // workspace list — the one moment that is safe, because no route exists yet. Every
2104        // later registration must name the complete set: one with an empty list would
2105        // briefly clear this application's routes, and a peer asking for a workspace this
2106        // server owns would be told nobody has it.
2107        let f = fixture().await;
2108        let second_root = f.second_root();
2109
2110        f.runtime.enable(&f.root).await.unwrap();
2111        f.runtime.enable(&second_root).await.unwrap();
2112        f.runtime.sync_now(&f.root).await.unwrap();
2113        f.runtime.disable(&f.root).await.unwrap();
2114
2115        let seen = f.stub.seen.lock().expect("stub");
2116        let first_full = seen
2117            .registrations
2118            .iter()
2119            .position(|r| !r.workspaces.is_empty())
2120            .expect("enabling registers the workspace");
2121        let emptied_afterwards = seen.registrations[first_full + 1..]
2122            .iter()
2123            .filter(|r| r.workspaces.is_empty())
2124            .count();
2125        assert_eq!(
2126            emptied_afterwards, 0,
2127            "a registration after the first workspace must still name the whole set"
2128        );
2129    }
2130
2131    #[tokio::test(flavor = "multi_thread")]
2132    async fn disabling_unregisters_and_leaves_the_files_alone() {
2133        let f = fixture().await;
2134        std::fs::write(f.root.join("keep.md"), "content").unwrap();
2135
2136        let id = f.runtime.enable(&f.root).await.unwrap();
2137        f.runtime.disable(&f.root).await.unwrap();
2138
2139        assert!(f.stub.last_workspaces().is_empty());
2140        assert!(
2141            f.root.join("keep.md").exists(),
2142            "disabling sync must not touch files"
2143        );
2144        assert!(
2145            crate::sync::id::sync_id_path("sapphire-synctest", &f.root).exists(),
2146            "the sync id stays, so re-enabling rejoins the same workspace"
2147        );
2148        let _ = id;
2149    }
2150
2151    #[tokio::test(flavor = "multi_thread")]
2152    async fn re_enabling_after_disabling_reuses_the_identity() {
2153        let f = fixture().await;
2154
2155        let first = f.runtime.enable(&f.root).await.unwrap();
2156        f.runtime.disable(&f.root).await.unwrap();
2157        let again = f.runtime.enable(&f.root).await.unwrap();
2158
2159        assert_eq!(first, again);
2160    }
2161
2162    #[tokio::test(flavor = "multi_thread")]
2163    async fn status_reports_disabled_for_a_workspace_that_was_never_enabled() {
2164        let f = fixture().await;
2165
2166        let status = f.runtime.status(&f.root).await;
2167        assert!(!status.enabled);
2168        assert!(status.workspace_id.is_none());
2169    }
2170
2171    #[tokio::test(flavor = "multi_thread")]
2172    async fn disabling_something_that_was_never_enabled_is_not_an_error() {
2173        let f = fixture().await;
2174        f.runtime
2175            .disable(&f.root)
2176            .await
2177            .expect("disabling twice is fine");
2178    }
2179
2180    #[tokio::test(flavor = "multi_thread")]
2181    async fn a_workspace_whose_root_vanished_is_reported_as_paused_not_deleted() {
2182        let f = fixture().await;
2183        std::fs::write(f.root.join("a.md"), "content").unwrap();
2184        f.runtime.enable(&f.root).await.unwrap();
2185        f.runtime.scan(&f.root).await.unwrap();
2186
2187        // The drive is unmounted: the marker directory goes with it.
2188        std::fs::remove_dir_all(f.root.join(".sapphire-synctest")).unwrap();
2189        let _ = f.runtime.scan(&f.root).await;
2190
2191        let status = f.runtime.status(&f.root).await;
2192        assert!(
2193            status.paused.is_some(),
2194            "a missing root must pause, not replicate as a mass deletion"
2195        );
2196    }
2197
2198    #[tokio::test(flavor = "multi_thread")]
2199    async fn a_directory_that_is_not_a_workspace_cannot_be_enabled() {
2200        let f = fixture().await;
2201        let plain = f._tmp.path().join("plain");
2202        std::fs::create_dir_all(&plain).unwrap();
2203
2204        assert!(f.runtime.enable(&plain).await.is_err());
2205    }
2206
2207    #[tokio::test(flavor = "multi_thread")]
2208    async fn a_write_through_the_workspace_router_is_committed_to_the_replica() {
2209        // The brief's "main path": a write the app server itself made is scanned straight
2210        // after, not left to the watcher's debounce. Asserted against the replica's own
2211        // state, which is the only thing that can tell the two paths apart — reading the
2212        // file back would pass even if the scan never ran.
2213        let f = fixture().await;
2214        f.runtime.enable(&f.root).await.unwrap();
2215
2216        let host = Arc::new(crate::WorkspaceHost::new(&CTX));
2217        host.select(&f.root, false).await.unwrap();
2218        let current = Arc::new(crate::Current::new(
2219            &CTX,
2220            host,
2221            Some(Arc::clone(&f.runtime)),
2222            crate::selection::SelectionFile::at(f._tmp.path().join("workspace.toml")),
2223        ));
2224        let router = Arc::new(crate::workspace_router(current));
2225        let (client_conn, server_conn) = sapphire_ipc::Connection::pair();
2226        tokio::spawn(async move {
2227            let info = sapphire_ipc::ServerInfo {
2228                version: "0.0.0".into(),
2229                api: sapphire_backend::protocol::API_VERSION,
2230                app_api: None,
2231                pid: std::process::id(),
2232                managed_by: ManagedBy::Spawned,
2233            };
2234            let _ = sapphire_ipc::serve(server_conn, router, "sapphire-synctest", info).await;
2235        });
2236        let info = sapphire_ipc::ClientInfo {
2237            kind: "test".into(),
2238            version: "0.0.0".into(),
2239            api: sapphire_backend::protocol::API_VERSION,
2240            app_api: None,
2241            pid: std::process::id(),
2242        };
2243        let (client, _) = sapphire_ipc::Client::handshake(client_conn, "sapphire-synctest", info)
2244            .await
2245            .unwrap();
2246
2247        let _: sapphire_backend::protocol::Ack = client
2248            .call(
2249                sapphire_backend::protocol::WRITE_FILE,
2250                sapphire_backend::protocol::ContentParams {
2251                    path: PathBuf::from("through-the-server.md"),
2252                    content: "written by the server".into(),
2253                },
2254            )
2255            .await
2256            .unwrap();
2257
2258        // `scan` runs before the reply is written, so by the time the call returns the
2259        // replica must already know the path.
2260        let synced = f.runtime.synced.lock().await;
2261        let entry = synced.get(&f.root).expect("enabled above");
2262        let replica = entry.replica.lock().await;
2263        let state = replica
2264            .state("through-the-server.md")
2265            .expect("a readable state");
2266        assert!(
2267            state.is_some(),
2268            "a write through the server must be scanned into the replica, not left to the watcher"
2269        );
2270    }
2271
2272    #[tokio::test(flavor = "multi_thread")]
2273    async fn an_announcement_for_an_unknown_workspace_is_ignored_not_fatal() {
2274        let f = fixture().await;
2275        f.runtime.enable(&f.root).await.unwrap();
2276
2277        let driver = Arc::clone(&f.runtime);
2278        let handle = tokio::spawn(async move { driver.run().await });
2279
2280        f.stub
2281            .announce(grain_id::GrainId::random(), "no-such-ticket")
2282            .await;
2283        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
2284
2285        assert!(
2286            !handle.is_finished(),
2287            "one bad announcement must not stop the runtime"
2288        );
2289        handle.abort();
2290    }
2291}