Skip to main content

sapphire_framework_server/sync/
mod.rs

1//! Replication, from the app server's side.
2//!
3//! One `Replica` per synced workspace, registered with the bridge, driven by announcements
4//! from it and by local edits.
5//!
6//! See `docs/superpowers/specs/2026-09-16-process-architecture-design.md` §4.2.
7
8use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use grain_id::GrainId;
13use sapphire_backend::protocol as proto;
14use sapphire_bridge_api::{PeersResult, RegisterParams, WorkspaceRegistration, WorkspacesResult};
15use sapphire_bridge_client::BridgeClient;
16use sapphire_ipc::ManagedBy;
17use sapphire_sync::{PathUpdate, PauseReason, Replica, ReplicaConfig, ScanOutcome, SystemClock};
18use sapphire_workspace::{AppContext, Workspace};
19use tokio::sync::{Mutex, OnceCell, broadcast, mpsc};
20
21pub mod id;
22mod live;
23mod methods;
24#[cfg(any(test, feature = "test-util"))]
25pub mod testing;
26mod topology;
27mod watch;
28
29pub use id::{SYNC_ID_FILE, WORKSPACE_MAP_FILE, sync_id, sync_id_path};
30pub use live::{DIAL_BACKOFF_MAX, OPEN_TIMEOUT};
31pub use methods::sync_router;
32pub use watch::{DEBOUNCE, Watcher};
33
34use live::{LivePeers, dial_backoff};
35
36use crate::error::{Error, Result};
37
38use crate::host::WorkspaceHost;
39
40/// What `sync.status` reports.
41#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
42pub struct SyncStatus {
43    /// Whether this workspace is synced at all.
44    pub enabled: bool,
45    /// Its identity across devices, when it is.
46    pub workspace_id: Option<GrainId>,
47    /// How many devices the workgroup has, this host excluded.
48    pub peers: usize,
49    /// Why replication is paused, if it is — a missing root or marker directory.
50    pub paused: Option<String>,
51    /// The last failure, if any.
52    pub last_error: Option<String>,
53    /// Whether the bridge is reachable. `false` is not an outage: the app server works.
54    pub bridge_available: bool,
55    /// How the workspace is wired to its peers, by the roles the bridge last reported.
56    pub topology: proto::Topology,
57    /// How far embedding has got, when this host embeds (#188).
58    pub embedding: Option<proto::EmbeddingProgress>,
59}
60
61/// One workspace this runtime syncs.
62struct Synced {
63    /// Its identity across devices, shared with every peer.
64    workspace_id: GrainId,
65    /// Its replica store. Held open because the store is a redb database.
66    replica: Arc<Mutex<Replica>>,
67    /// Why replication is paused, from the last scan.
68    paused: Option<PauseReason>,
69    /// The last failure, if any.
70    last_error: Option<String>,
71}
72
73/// How old a vector file no live content needs must be before the primary device removes
74/// it: a vector can arrive before its file, and a day is far longer than that window.
75const STALE_VECTOR_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
76
77/// Texts per embedding request in a server pass: small, so a search query waiting on the
78/// bridge's one worker is never stuck behind a long backfill (#188).
79const PASS_BATCH: usize = 4;
80
81/// How the primary device paces its backfill (#188).
82#[derive(Clone, Copy, Debug)]
83pub struct BackfillTiming {
84    /// How old another device's version must be before the primary embeds it: its author
85    /// has embedded it by then, or will not.
86    pub grace: std::time::Duration,
87    /// How often the primary device runs a pass with nothing prompting it.
88    pub interval: std::time::Duration,
89}
90
91impl Default for BackfillTiming {
92    fn default() -> Self {
93        BackfillTiming {
94            grace: std::time::Duration::from_secs(10 * 60),
95            interval: std::time::Duration::from_secs(10 * 60),
96        }
97    }
98}
99
100/// Replication for one application's workspaces.
101pub struct SyncRuntime {
102    ctx: &'static AppContext,
103    bridge: Arc<BridgeClient>,
104    exe_path: PathBuf,
105    managed_by: ManagedBy,
106    synced: Mutex<HashMap<PathBuf, Synced>>,
107    /// This host's device id. Asked of the bridge once, on the first [`enable`], and never
108    /// again: a second registration with an empty workspace list would briefly clear this
109    /// application's routes.
110    ///
111    /// [`enable`]: SyncRuntime::enable
112    device_id: OnceCell<GrainId>,
113    /// The file watcher, once [`watch_changes`] has started it.
114    ///
115    /// The runtime owns it because the spec puts it here: the sync runtime is a watcher and
116    /// a replica per registered root, and the two must agree on which roots are synced.
117    /// [`enable`] and [`disable`] keep them in step.
118    ///
119    /// [`watch_changes`]: SyncRuntime::watch_changes
120    /// [`enable`]: SyncRuntime::enable
121    /// [`disable`]: SyncRuntime::disable
122    watcher: OnceCell<Arc<Watcher>>,
123    /// The app server that owns these workspaces' caches, once it has told us.
124    ///
125    /// A replication session writes files, and the search index is not part of a session: the
126    /// index belongs to the app server, which holds the workspace open. Reaching it through
127    /// the host is what keeps a received file and its index moving together, and it is the
128    /// *same* [`WorkspaceState`](sapphire_workspace::WorkspaceState) the server serves from —
129    /// opening a second one would collide on the retrieve store's exclusive lock.
130    ///
131    /// Absent for a runtime nobody has wired to an app server (a test stub driving the
132    /// replica directly). Nothing then indexes, which is correct: nothing owns an index.
133    host: OnceCell<Arc<WorkspaceHost>>,
134    /// Serialises re-indexing, so two sessions finishing at once do not both sweep the
135    /// workspace through the same store.
136    reindexing: Mutex<()>,
137    /// Roots with an embedding pass waiting to run (#187). A request for a root already
138    /// waiting is folded into that pass.
139    embed_wanted: Mutex<std::collections::HashSet<PathBuf>>,
140    /// Serialises embedding passes: one model, one queue.
141    embedding: Mutex<()>,
142    /// Roots whose embedding pass is running now, for `sync.status`.
143    embed_running: Mutex<std::collections::HashSet<PathBuf>>,
144    /// The primary device's backfill pacing (#188).
145    backfill: std::sync::Mutex<BackfillTiming>,
146    /// The bridge an embedding pass asks for its model; `None` is the host's standard one.
147    bridge_endpoint: Option<sapphire_ipc::Endpoint>,
148    /// The live session table of every synced workspace, keyed by canonical root.
149    ///
150    /// One table per workspace rather than one for the runtime: a session is about one
151    /// workspace, and the two storm rules are per-workspace rules. A root that is not
152    /// synced has none, so a dial or a push for it is a no-op rather than an error.
153    live: Mutex<HashMap<PathBuf, Arc<LivePeers>>>,
154    /// The bridge's last answer to `peers`, roles included. The inbound guard and
155    /// [`is_primary`](SyncRuntime::is_primary) read it rather than asking again.
156    last_peers: Mutex<Option<PeersResult>>,
157    /// How long a dial waits for the bridge to open a stream; [`OPEN_TIMEOUT`] by default.
158    open_timeout: std::time::Duration,
159    /// The first dials [`enable`](SyncRuntime::enable) leaves running. Each pins the
160    /// runtime, so [`stop_dials`](SyncRuntime::stop_dials) ends them with the server.
161    first_dials: std::sync::Mutex<tokio::task::JoinSet<()>>,
162}
163
164impl SyncRuntime {
165    /// A runtime for `ctx`'s application.
166    pub fn new(
167        ctx: &'static AppContext,
168        bridge: Arc<BridgeClient>,
169        exe_path: PathBuf,
170        managed_by: ManagedBy,
171    ) -> SyncRuntime {
172        SyncRuntime {
173            ctx,
174            bridge,
175            exe_path,
176            managed_by,
177            synced: Mutex::new(HashMap::new()),
178            device_id: OnceCell::new(),
179            watcher: OnceCell::new(),
180            host: OnceCell::new(),
181            reindexing: Mutex::new(()),
182            embed_wanted: Mutex::new(std::collections::HashSet::new()),
183            embedding: Mutex::new(()),
184            embed_running: Mutex::new(std::collections::HashSet::new()),
185            backfill: std::sync::Mutex::new(BackfillTiming::default()),
186            bridge_endpoint: None,
187            live: Mutex::new(HashMap::new()),
188            last_peers: Mutex::new(None),
189            open_timeout: OPEN_TIMEOUT,
190            first_dials: std::sync::Mutex::new(tokio::task::JoinSet::new()),
191        }
192    }
193
194    /// Give up on opening a stream to a peer after `timeout` rather than [`OPEN_TIMEOUT`]:
195    /// for tests, which cannot wait that long for a peer that never answers.
196    pub fn with_open_timeout(mut self, timeout: std::time::Duration) -> SyncRuntime {
197        self.open_timeout = timeout;
198        self
199    }
200
201    /// Tell the runtime which app server owns these workspaces' caches.
202    ///
203    /// Called by [`AppServer::sync`](crate::AppServer::sync). First writer wins: a runtime
204    /// belongs to the server that wired it.
205    /// Embed through the bridge at `endpoint` rather than the host's standard one: for a
206    /// fixture running several hosts in one process.
207    pub fn with_bridge_endpoint(mut self, endpoint: sapphire_ipc::Endpoint) -> SyncRuntime {
208        self.bridge_endpoint = Some(endpoint);
209        self
210    }
211
212    /// Pace the primary device's backfill differently: for tests, which cannot wait ten
213    /// minutes.
214    pub fn set_backfill_timing(&self, timing: BackfillTiming) {
215        *self.backfill.lock().expect("backfill timing") = timing;
216    }
217
218    fn backfill_timing(&self) -> BackfillTiming {
219        *self.backfill.lock().expect("backfill timing")
220    }
221
222    pub(crate) fn set_host(&self, host: Arc<WorkspaceHost>) {
223        let _ = self.host.set(host);
224    }
225
226    /// Start syncing `root`, returning its identity across devices.
227    ///
228    /// Idempotent: enabling an already-synced workspace returns the same id and changes
229    /// nothing.
230    pub async fn enable(self: &Arc<Self>, root: &Path) -> Result<GrainId> {
231        let key = root.canonicalize().map_err(Error::Io)?;
232        let watch_key = key.clone();
233        let workspace_id = sync_id(self.ctx.app_name, &key)?;
234
235        // The map stays locked across opening the replica, so two simultaneous `enable`s for
236        // one root cannot both open it — the store is a redb database, and it is opened once.
237        {
238            let mut synced = self.synced.lock().await;
239            if let Some(existing) = synced.get(&key) {
240                return Ok(existing.workspace_id);
241            }
242            let device_id = self.device_id().await?;
243            let state_dir = self.state_dir(&key)?;
244            std::fs::create_dir_all(&state_dir).map_err(Error::Io)?;
245            let config = ReplicaConfig::new(self.ctx.app_name, key.clone(), device_id, &state_dir);
246            let replica = Replica::open(config, Arc::new(SystemClock))
247                .map_err(|e| Error::Sync(e.to_string()))?;
248            synced.insert(
249                key.clone(),
250                Synced {
251                    workspace_id,
252                    replica: Arc::new(Mutex::new(replica)),
253                    paused: None,
254                    last_error: None,
255                },
256            );
257        }
258
259        // Watch the new root too, so an edit made outside the server from now on is seen.
260        // Not fatal if it fails: the app server's own writes take the exact path in
261        // `handlers.rs`, and the next `watch_changes` re-reads the root set anyway.
262        if let Some(watcher) = self.watcher.get()
263            && let Err(err) = watcher.watch(&watch_key)
264        {
265            tracing::warn!(root = %watch_key.display(), "could not watch: {err}");
266        }
267
268        // Register only once the replica is open, so a registration never names a workspace
269        // that cannot serve a session.
270        self.reregister().await?;
271
272        // Dial now, so enabling a workspace converges it rather than waiting for the next
273        // local edit to trigger a session. This is what catches a host up when it returns:
274        // the peer that has been running is the one the returning host has to ask, and no
275        // watcher on either side fires for a file that was never written locally.
276        //
277        // Best-effort: a workspace is enabled whether or not any peer is up, and a failure
278        // here is not a failure to enable. A session is symmetric, so dialing is enough —
279        // this side sends its own view, and the peer answers with what this side lacks.
280        //
281        // In the background, so the answer does not wait on it: a peer that is offline
282        // holds a dial up to `open_timeout`, longer than a client waits for `sync.enable`
283        // (#228).
284        let runtime = Arc::clone(self);
285        let mut dials = self.first_dials.lock().expect("first dials");
286        // Reap the finished ones, so the set holds only what is running.
287        while dials.try_join_next().is_some() {}
288        dials.spawn(async move {
289            if let Err(err) = runtime.sync_now(&key).await {
290                tracing::warn!(root = %key.display(), "the first session after enabling failed: {err}");
291            }
292        });
293        Ok(workspace_id)
294    }
295
296    /// Stop syncing `root`. Files and the sync id stay, so re-enabling rejoins the same
297    /// workspace rather than creating a second one.
298    pub async fn disable(&self, root: &Path) -> Result<()> {
299        self.stop(root).await.1
300    }
301
302    /// Stop syncing `root`, like [`disable`](Self::disable), then wait up to `limit` until
303    /// nothing holds its replica store open any more — a session, a scan or an embedding
304    /// pass that started before may still have it. A store about to be moved or deleted
305    /// must be closed first.
306    ///
307    /// Only a store still in use at `limit` is an error. The bridge not hearing of it is
308    /// logged, not returned: the store is closed all the same, and the next registration
309    /// replaces the bridge's view.
310    pub async fn release(&self, root: &Path, limit: std::time::Duration) -> Result<()> {
311        let (replica, unregistered) = self.stop(root).await;
312        if let Err(err) = unregistered {
313            tracing::warn!(root = %root.display(), "stopping sync: {err}");
314        }
315        let Some(replica) = replica else {
316            return Ok(());
317        };
318        let deadline = tokio::time::Instant::now() + limit;
319        while Arc::strong_count(&replica) > 1 {
320            if tokio::time::Instant::now() >= deadline {
321                return Err(Error::Sync(format!(
322                    "the replica of {} is still in use; try again",
323                    root.display()
324                )));
325            }
326            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
327        }
328        Ok(())
329    }
330
331    /// Stop syncing `root`, handing back its replica — `None` when it was not synced — and
332    /// how telling the bridge went.
333    async fn stop(&self, root: &Path) -> (Option<Arc<Mutex<Replica>>>, Result<()>) {
334        // A root that no longer exists cannot be canonicalized, but the keys of `synced` are
335        // canonical already, so the path as given (a registry root) still names the entry.
336        let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
337        let removed = self.synced.lock().await.remove(&key);
338        if removed.is_some()
339            && let Some(watcher) = self.watcher.get()
340        {
341            watcher.unwatch(&key);
342        }
343        // Any live session for a workspace this server no longer holds is closed: it would
344        // otherwise keep pushing files for a workspace nobody asked about any more.
345        if let Some(peers) = self.live.lock().await.remove(&key) {
346            peers.drop_connections().await;
347        }
348        let Some(removed) = removed else {
349            return (None, Ok(()));
350        };
351        let unregistered = async {
352            self.bridge
353                .unregister(removed.workspace_id)
354                .await
355                .map_err(|e| Error::Bridge(e.to_string()))?;
356            self.reregister().await
357        }
358        .await;
359        (Some(removed.replica), unregistered)
360    }
361
362    /// The workspaces the workgroup knows about.
363    ///
364    /// Read straight from the bridge, so it reflects every device's registrations. The
365    /// bridge being down is not an empty answer — that would read as "nothing to map" — so
366    /// it is an error, and `sync.map` reports it.
367    pub async fn workspaces(&self) -> Result<WorkspacesResult> {
368        self.bridge
369            .workspaces()
370            .await
371            .map_err(|e| Error::Bridge(e.to_string()))
372    }
373
374    /// Place a workgroup workspace at a directory of this host: write its identity and
375    /// map there, returning the canonical root. [`Current::map`](crate::Current::map)
376    /// then selects it and syncs it.
377    ///
378    /// `selector` is the workspace's name or id, as [`workspaces`] lists it; `dir` must
379    /// already be a workspace of this application — the directory is placed, not made.
380    ///
381    /// [`workspaces`]: SyncRuntime::workspaces
382    pub async fn place(&self, selector: &str, dir: &Path) -> Result<PathBuf> {
383        let workgroup = self.workspaces().await?;
384        let wanted = workgroup
385            .workspaces
386            .iter()
387            // A name is the user-facing handle; the id is the exact one. Two workspaces
388            // may not share a name, so a name names at most one.
389            .find(|w| w.name == selector || w.workspace_id.to_string() == selector)
390            .ok_or_else(|| Error::UnknownWorkspaceName(selector.to_owned()))?;
391
392        // Refuse another application's workspace: this server syncs its own application's
393        // workspaces, and `Workspace::from_root` would later refuse the directory anyway.
394        if wanted.app_name != self.ctx.app_name {
395            return Err(Error::WrongApp {
396                name: wanted.name.clone(),
397                app_name: wanted.app_name.clone(),
398            });
399        }
400
401        let root = dir.canonicalize().map_err(Error::Io)?;
402        let workspace = Workspace::from_root(self.ctx, &root)?;
403        if workspace.root != root {
404            return Err(Error::UnknownWorkspace(root, self.ctx.app_name));
405        }
406
407        // A directory already syncing under another identity is not remapped: the sync id
408        // is a device's word that it is the same workspace, and rewriting it silently
409        // would make two hosts disagree about what they share.
410        {
411            let synced = self.synced.lock().await;
412            if let Some(entry) = synced.get(&root)
413                && entry.workspace_id != wanted.workspace_id
414            {
415                return Err(Error::SyncId(format!(
416                    "{} already syncs as {}, not {}",
417                    root.display(),
418                    entry.workspace_id,
419                    wanted.workspace_id
420                )));
421            }
422        }
423
424        // The identity arrives from the workgroup, not from this host: two hosts that map
425        // the same workspace must sync as one, so the id is the one the workgroup lists,
426        // written before `enable` reads it. A different id already on disk is refused
427        // above; an absent one is written here.
428        let marker = workspace.marker_dir();
429        let id_path = marker.join(crate::sync::id::SYNC_ID_FILE);
430        match std::fs::read_to_string(&id_path) {
431            Ok(existing) if existing.trim() != wanted.workspace_id.to_string() => {
432                return Err(Error::SyncId(format!(
433                    "{} already holds identity {}, not {}",
434                    root.display(),
435                    existing.trim(),
436                    wanted.workspace_id
437                )));
438            }
439            Err(_) => {
440                std::fs::write(&id_path, format!("{}\n", wanted.workspace_id))
441                    .map_err(Error::Io)?;
442            }
443            Ok(_) => {}
444        }
445
446        // The map is what tells this application, on a later start, which workgroup
447        // workspace a directory is.
448        let map_path = root
449            .join(format!(".{}", self.ctx.app_name))
450            .join(WORKSPACE_MAP_FILE);
451        let workspace_id = wanted.workspace_id.to_string();
452        std::fs::write(&map_path, format!("{workspace_id}\n")).map_err(Error::Io)?;
453        Ok(root)
454    }
455
456    /// What `sync.status` answers with.
457    pub async fn status(&self, root: &Path) -> SyncStatus {
458        let bridge_available = self.bridge.status().await.is_ok();
459        let answer = self.bridge.peers().await.ok();
460        if let Some(answer) = &answer {
461            *self.last_peers.lock().await = Some(answer.clone());
462        }
463        let peers = answer
464            .as_ref()
465            .map_or(0, |p| p.peers.len().saturating_sub(1));
466        let Ok(key) = root.canonicalize() else {
467            return SyncStatus {
468                enabled: false,
469                workspace_id: None,
470                peers,
471                paused: None,
472                last_error: None,
473                bridge_available,
474                topology: proto::Topology::Mesh,
475                embedding: None,
476            };
477        };
478        let synced = self.synced.lock().await;
479        match synced.get(&key) {
480            Some(entry) => SyncStatus {
481                enabled: true,
482                workspace_id: Some(entry.workspace_id),
483                peers,
484                paused: entry.paused.map(|r| format!("{r:?}")),
485                last_error: entry.last_error.clone(),
486                bridge_available,
487                topology: topology::topology(
488                    answer
489                        .as_ref()
490                        .and_then(|p| p.roles_for(entry.workspace_id)),
491                ),
492                embedding: self.embedding_progress(&key).await,
493            },
494            None => SyncStatus {
495                enabled: false,
496                workspace_id: None,
497                peers,
498                paused: None,
499                last_error: None,
500                bridge_available,
501                topology: proto::Topology::Mesh,
502                embedding: None,
503            },
504        }
505    }
506
507    /// How far `key`'s embedding has got, when this host embeds it. Reads the open
508    /// workspace's index; never opens one, so a status call stays cheap.
509    async fn embedding_progress(&self, key: &Path) -> Option<proto::EmbeddingProgress> {
510        let host = self.host.get()?;
511        let backend = host.open_backend(key)?;
512        let state = backend.state();
513        state.embedder()?;
514        let info = state.db_info().ok()?;
515        Some(proto::EmbeddingProgress {
516            vectors: info.vector_count,
517            pending: info.pending_count,
518            running: self.embed_running.lock().await.contains(key),
519        })
520    }
521
522    /// Bring the replica's view of the files up to date.
523    ///
524    /// Called straight after the app server's own writes, and by the watcher for everything
525    /// else. A scan that finds nothing is cheap; a scan that is skipped loses an edit until
526    /// the next one.
527    pub async fn scan(&self, root: &Path) -> Result<()> {
528        let Ok(key) = root.canonicalize() else {
529            return Ok(());
530        };
531        let replica = {
532            let synced = self.synced.lock().await;
533            match synced.get(&key) {
534                Some(entry) => Arc::clone(&entry.replica),
535                None => return Ok(()),
536            }
537        };
538        let (outcome, updates) = {
539            let mut replica = replica.lock().await;
540            let outcome = replica.scan().map_err(|e| Error::Sync(e.to_string()))?;
541            // What the scan committed, as path updates, so it can go out on the sessions
542            // that are already open. This is the whole point of the exact-path scan in
543            // `handlers.rs`: a write through the server reaches a peer live, not at the next
544            // dial.
545            let updates = match &outcome {
546                ScanOutcome::Scanned(report) => {
547                    // The unit of replication is the path state (design doc section 2.4):
548                    // the `seen` a peer prunes against is the *path's* version vector - the
549                    // context the recorded entry was built on plus its own dot - never the
550                    // whole replica vv. Sending the global vv here made receivers prune
551                    // causally-related versions they still held: a conflict copy on the
552                    // first remote edit, and tombstones pruned so deletes stopped
553                    // propagating (issue #157).
554                    report
555                        .recorded
556                        .iter()
557                        .map(|entry| PathUpdate {
558                            path: entry.path.clone(),
559                            versions: vec![entry.clone()],
560                            seen: {
561                                let mut seen = entry.context.clone();
562                                seen.add_dot(&entry.dot);
563                                seen
564                            },
565                        })
566                        .collect()
567                }
568                ScanOutcome::Paused(_) => Vec::new(),
569            };
570            (outcome, updates)
571        };
572        {
573            let mut synced = self.synced.lock().await;
574            if let Some(entry) = synced.get_mut(&key) {
575                entry.paused = match outcome {
576                    ScanOutcome::Paused(reason) => Some(reason),
577                    ScanOutcome::Scanned(_) => None,
578                };
579            }
580        }
581        // After the map is unlocked: a push awaits every peer, and the table must not be
582        // held behind a slow one.
583        self.after_commit(&key, updates).await;
584        Ok(())
585    }
586
587    /// Bring the search index up to date with files a session has written.
588    ///
589    /// A replication session moves files, not index rows: the index belongs to the app
590    /// server, and the sync core is deliberately free of it. So the side that *applied* a
591    /// change re-indexes, which is what makes a file that arrived from a peer searchable
592    /// without a manual reindex — the claim the whole architecture rests on.
593    ///
594    /// Incremental, not a full rebuild: the workspace's mtime/size stamps mean only what
595    /// actually changed is read back. A file the session wrote has a new stamp, so it is
596    /// picked up; one that was already indexed and untouched is skipped.
597    ///
598    /// Best-effort by contract: the caller's session has already succeeded, and a failure
599    /// here must not be reported as the session's. It is logged and the next session, local
600    /// edit or explicit `workspace.reindex` tries again.
601    async fn reindex(&self, root: &Path) {
602        let Some(host) = self.host.get() else {
603            return;
604        };
605        let host = Arc::clone(host);
606        let root = root.to_owned();
607        // One sweep at a time: two sessions that finish together would otherwise both walk
608        // the workspace through the same retrieve store.
609        let _serialised = self.reindexing.lock().await;
610        // Opening the workspace is itself blocking work, and it is where the exclusive
611        // retrieve-store lock is taken, so it happens here rather than inside the sweep.
612        let backend = match host.backend_at(&root).await {
613            Ok(backend) => backend,
614            Err(err) => {
615                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
616                return;
617            }
618        };
619        let state = Arc::clone(backend.state());
620        let result = tokio::task::spawn_blocking(move || state.sync_retrieve()).await;
621        match result {
622            Ok(Ok(report)) => {
623                tracing::debug!(root = %root.display(), ?report, "re-indexed after a session");
624            }
625            Ok(Err(err)) => {
626                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
627            }
628            Err(err) => {
629                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
630            }
631        }
632    }
633
634    /// Run an embedding pass over `root` in the background, after the pass already
635    /// waiting for it if there is one (#187).
636    ///
637    /// The pass reads the vector files that arrived, and computes the missing ones this
638    /// host is responsible for: the files whose winning version this replica wrote, and —
639    /// on the primary device — every other one. It never holds up the caller.
640    pub fn request_embed(self: &Arc<Self>, root: &Path) {
641        let Ok(key) = root.canonicalize() else {
642            return;
643        };
644        let runtime = Arc::clone(self);
645        tokio::spawn(async move {
646            if !runtime.embed_wanted.lock().await.insert(key.clone()) {
647                return;
648            }
649            let _one_at_a_time = runtime.embedding.lock().await;
650            // Taken off the list as the pass starts: a change during the pass asks again.
651            runtime.embed_wanted.lock().await.remove(&key);
652            runtime.embed_running.lock().await.insert(key.clone());
653            runtime.embed_pass(&key).await;
654            runtime.embed_running.lock().await.remove(&key);
655        });
656    }
657
658    /// One embedding pass over `key`. Best-effort, like the re-index: logged, never fatal.
659    async fn embed_pass(&self, key: &Path) {
660        let Some(host) = self.host.get() else {
661            return;
662        };
663        let (workspace_id, replica) = {
664            let synced = self.synced.lock().await;
665            match synced.get(key) {
666                Some(entry) => (entry.workspace_id, Arc::clone(&entry.replica)),
667                None => return,
668            }
669        };
670        let backend = match host.backend_at(key).await {
671            Ok(backend) => backend,
672            Err(err) => {
673                tracing::warn!(root = %key.display(), "embedding pass: {err}");
674                return;
675            }
676        };
677        let state = Arc::clone(backend.state());
678        if let Err(err) = state
679            .refresh_embedder_at(self.bridge_endpoint.clone())
680            .await
681        {
682            tracing::warn!(root = %key.display(), "embedding pass: {err}");
683            return;
684        }
685        if state.embedder().is_none() {
686            return;
687        }
688        let primary = self.is_primary(workspace_id).await;
689        // The replica's view, read once: which content each path has, and which of those
690        // versions this replica wrote.
691        let grace_ms = self.backfill_timing().grace.as_millis() as u64;
692        let now_ms = std::time::SystemTime::now()
693            .duration_since(std::time::UNIX_EPOCH)
694            .map_or(0, |d| d.as_millis() as u64);
695        let (mine, settled, live) = {
696            let replica = replica.lock().await;
697            let me = replica.replica_id();
698            let states = match replica.states() {
699                Ok(states) => states,
700                Err(err) => {
701                    tracing::warn!(root = %key.display(), "embedding pass: {err}");
702                    return;
703                }
704            };
705            let mut mine = HashMap::new();
706            // Other devices' versions old enough for the primary to fill in.
707            let mut settled = HashMap::new();
708            let mut live = std::collections::HashSet::new();
709            for (path, path_state) in states {
710                let winner = path_state.winner();
711                let Some(hash) = winner.content.hash() else {
712                    continue;
713                };
714                let hash = hash.to_hex();
715                if winner.dot.replica == me {
716                    mine.insert(path, hash.clone());
717                } else if now_ms.saturating_sub(winner.hlc.wall_ms) >= grace_ms {
718                    settled.insert(path, hash.clone());
719                }
720                live.insert(hash);
721            }
722            (mine, settled, live)
723        };
724        let root = key.to_owned();
725        let result = tokio::task::spawn_blocking(move || {
726            // The author embeds its own versions at once; the primary device fills in other
727            // devices' versions once they have settled, racing nobody.
728            let policy = |rel: &str, hash: &str| {
729                mine.get(rel).is_some_and(|h| h.as_str() == hash)
730                    || (primary && settled.get(rel).is_some_and(|h| h.as_str() == hash))
731            };
732            let embedded = state.embed_pending_with(&policy, PASS_BATCH)?;
733            let removed = if primary {
734                state.remove_stale_vectors(&live, STALE_VECTOR_AGE)
735            } else {
736                0
737            };
738            Ok::<_, sapphire_workspace::Error>((embedded, removed))
739        })
740        .await;
741        match result {
742            Ok(Ok((embedded, removed))) if embedded > 0 || removed > 0 => tracing::info!(
743                root = %root.display(),
744                embedded,
745                removed,
746                "embedding pass"
747            ),
748            Ok(Ok(_)) => {}
749            Ok(Err(err)) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
750            Err(err) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
751        }
752    }
753
754    /// Open a session with every peer that will take one.
755    pub async fn sync_now(self: &Arc<Self>, root: &Path) -> Result<()> {
756        let Ok(key) = root.canonicalize() else {
757            return Ok(());
758        };
759        let workspace_id = {
760            let synced = self.synced.lock().await;
761            match synced.get(&key) {
762                Some(entry) => entry.workspace_id,
763                None => return Ok(()),
764            }
765        };
766        let peers = self
767            .bridge
768            .peers()
769            .await
770            .map_err(|e| Error::Bridge(e.to_string()))?;
771        *self.last_peers.lock().await = Some(peers.clone());
772        let me = self.device_id().await?;
773
774        // The same direction rule the dial loop follows: a host only dials peers with a
775        // greater device id. Dialling the other way from here would reintroduce the
776        // dial-both-ways deadlock the loop was cured of, just on a different trigger.
777        // Inside a star, a pair with neither hub in it does not link at all.
778        let roles = peers.roles_for(workspace_id).cloned();
779        // Every peer at once, so one that is offline costs the others nothing (#228). On
780        // this task, not spawned: cancelling the caller — the server's shutdown aborting the
781        // watcher — must drop every dial with it, since each one pins the runtime.
782        let mut dials: Vec<Dial<'_>> = Vec::new();
783        for peer in peers
784            .peers
785            .iter()
786            .filter(|p| topology::link(me, p.device_id, roles.as_ref()) == topology::Link::Dial)
787            .cloned()
788            .collect::<Vec<_>>()
789        {
790            // A peer that already has a live session is caught up by it: a second exchange
791            // on the same workspace would be redundant, and a second stream between the same
792            // pair is exactly what the tie-break cannot settle.
793            if let Some(live) = self.live_peers(&key).await
794                && live.has(&peer.device_id).await
795            {
796                continue;
797            }
798            // One dial per peer at a time. `sync_now` runs beside the dial loop, and two
799            // overlapping outbound dials to one peer are the one case the tie-break rule
800            // cannot resolve — each end would see its own stream as `outbound`. Whoever
801            // arrives second simply leaves it to the next pass. The table is made here if
802            // the dial loop has not yet: the first dial after enabling must not depend on
803            // a tick that has not fired.
804            let Some((_, table)) = self.live_peers_or_create(&key).await else {
805                continue;
806            };
807            if !table.begin_dial(peer.device_id).await {
808                continue;
809            }
810            let key = &key;
811            dials.push(Box::pin(async move {
812                // A peer that does not host this workspace refuses, which is normal and cheap.
813                let result = self
814                    .sync_one(key, &peer.device_id, peer.name.clone(), workspace_id)
815                    .await;
816                table.end_dial(peer.device_id).await;
817                if let Err(err) = result {
818                    tracing::debug!(peer = %peer.name, "no session: {err}");
819                }
820            }));
821        }
822        join_all(dials).await;
823        Ok(())
824    }
825
826    /// Dial one peer and adopt the stream as a live session.
827    ///
828    /// The one-shot path (`run_session`) would close the stream the moment the exchange
829    /// finished — while the accepting peer keeps its half open and pushes into it, a session
830    /// that dies with nothing reading it. So the dialled side is adopted live too, and both
831    /// ends of a pair agree that a session stays open.
832    async fn sync_one(
833        self: &Arc<Self>,
834        key: &Path,
835        device: &GrainId,
836        name: String,
837        workspace_id: GrainId,
838    ) -> Result<()> {
839        let stream = self.open_stream(workspace_id, *device).await?;
840        self.adopt_live_session(stream, key, *device, workspace_id)
841            .await
842            .map_err(|err| {
843                tracing::warn!(peer = %name, "a live session could not be started: {err}");
844                err
845            })
846    }
847
848    /// Ask the bridge for a stream to `device`, giving up after `open_timeout`.
849    async fn open_stream(
850        &self,
851        workspace_id: GrainId,
852        device: GrainId,
853    ) -> Result<sapphire_ipc::RawStream> {
854        match tokio::time::timeout(
855            self.open_timeout,
856            self.bridge.open_stream(workspace_id, device),
857        )
858        .await
859        {
860            Ok(opened) => Ok(opened?),
861            Err(_) => Err(Error::Bridge(format!(
862                "no stream to {device} within {} s; it may be offline",
863                self.open_timeout.as_secs_f32()
864            ))),
865        }
866    }
867
868    /// Answer the bridge's announcements until the connection closes.
869    pub async fn run(self: Arc<Self>) -> Result<()> {
870        let mut incoming = self.bridge.incoming();
871        loop {
872            let announcement = match incoming.recv().await {
873                Ok(a) => a,
874                Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
875                    tracing::warn!(missed = n, "fell behind on bridge announcements");
876                    continue;
877                }
878                Err(tokio::sync::broadcast::error::RecvError::Closed) => return Ok(()),
879            };
880
881            let found = {
882                let synced = self.synced.lock().await;
883                synced
884                    .iter()
885                    .find(|(_, s)| s.workspace_id == announcement.workspace_id)
886                    .map(|(root, s)| (root.clone(), Arc::clone(&s.replica)))
887            };
888            // `replica` is still what the accepted session shares, and `root` is where its
889            // files and its index live.
890            let Some((root, replica)) = found else {
891                // The bridge routed to us for a workspace we no longer hold. Not fatal:
892                // ignore it and let the ticket expire.
893                tracing::debug!(
894                    workspace = %announcement.workspace_id,
895                    "an announcement for a workspace this server does not hold"
896                );
897                continue;
898            };
899
900            let bridge = Arc::clone(&self.bridge);
901            let workspace_id = announcement.workspace_id;
902            let peer = announcement.peer_device_id;
903            let driver = Arc::clone(&self);
904            tokio::spawn(async move {
905                let stream = match bridge.accept_stream(announcement.ticket).await {
906                    Ok(stream) => stream,
907                    Err(err) => {
908                        tracing::warn!("could not claim an announced stream: {err}");
909                        return;
910                    }
911                };
912                if driver.should_skip(workspace_id, peer).await {
913                    tracing::debug!(%peer, "dropped an inbound session: both ends are outside this workspace's star");
914                    return;
915                }
916                // Live, not one-shot: the peer that dialled is keeping this session open and
917                // pushing what it commits, so this side has to keep reading. A session that
918                // closed after the exchange would leave the dialler pushing into a stream
919                // nobody reads.
920                let (_, session) = match sapphire_framework_session::open_live_session(
921                    stream,
922                    Arc::clone(&replica),
923                    workspace_id,
924                )
925                .await
926                {
927                    Ok(started) => started,
928                    Err(err) => {
929                        tracing::warn!("an inbound session failed: {err}");
930                        return;
931                    }
932                };
933                let Some((key, table)) = driver.live_peers_or_create(&root).await else {
934                    return;
935                };
936                // The peer dialled: this stream survives only if the *peer* has the lower
937                // device id. A stream that loses is dropped, which closes it, and the
938                // connection ends tidily rather than being left half-read.
939                let Some(updates) = table.insert(peer, session, false).await else {
940                    return;
941                };
942                // This is the receiving side: the files are on disk now and nobody else will
943                // index them.
944                driver.reindex(&key).await;
945                driver.request_embed(&key);
946                driver.spawn_reader(table, updates, key, peer);
947            });
948        }
949    }
950
951    /// Every synced root, for the watcher.
952    pub async fn roots(&self) -> Vec<PathBuf> {
953        self.synced.lock().await.keys().cloned().collect()
954    }
955
956    /// The live session table for `root`, if this runtime syncs it.
957    ///
958    /// `None` is not an error: a dial, a push or a drop for a workspace this server does not
959    /// hold is a no-op, and every caller below treats it that way.
960    async fn live_peers(&self, root: &Path) -> Option<Arc<LivePeers>> {
961        let key = root.canonicalize().ok()?;
962        self.live.lock().await.get(&key).cloned()
963    }
964
965    /// The live session table for `root`, creating it if this runtime syncs it.
966    ///
967    /// The two locks are taken one after the other, never nested: `synced` to decide whether
968    /// the workspace is ours at all, then `live` to get or make its table.
969    async fn live_peers_or_create(&self, root: &Path) -> Option<(PathBuf, Arc<LivePeers>)> {
970        let key = root.canonicalize().ok()?;
971        if !self.synced.lock().await.contains_key(&key) {
972            return None;
973        }
974        // A table is built by whoever walks the workspace first, and it needs this host's
975        // device id — the tie-break between two simultaneous streams is decided from it.
976        let me = self.device_id().await.ok()?;
977        let mut live = self.live.lock().await;
978        let peers = Arc::clone(
979            live.entry(key.clone())
980                .or_insert_with(|| Arc::new(LivePeers::new(me))),
981        );
982        Some((key, peers))
983    }
984
985    /// Push `updates` to every open session for `root`.
986    ///
987    /// Called with what a scan recorded. `from` is always `None` here: a local commit came
988    /// from this host, so no session is excluded. The forwarding path, which does name a
989    /// source, is [`LivePeers::fan_out`](live::LivePeers::fan_out) called directly by the
990    /// reader side of a session.
991    ///
992    /// Best-effort by contract: the commit already happened, and a peer that is gone is the
993    /// dialer's problem, not the writer's.
994    pub async fn after_commit(&self, root: &Path, updates: Vec<PathUpdate>) {
995        if updates.is_empty() {
996            return;
997        }
998        if let Some(peers) = self.live_peers(root).await {
999            peers.fan_out(&updates, None).await;
1000        }
1001    }
1002
1003    /// The devices with an open live session for `root`.
1004    ///
1005    /// Used by tests to wait for the dialer; empty for a workspace this server does not hold.
1006    pub async fn live_session_devices(&self, root: &Path) -> Vec<GrainId> {
1007        match self.live_peers(root).await {
1008            Some(peers) => peers.devices().await,
1009            None => Vec::new(),
1010        }
1011    }
1012
1013    /// End the first dials [`enable`](SyncRuntime::enable) left running, waiting until
1014    /// they are gone.
1015    ///
1016    /// For the app server's shutdown: each one pins the runtime, and with it the replica
1017    /// stores, which are redb databases that stay locked — a dial outliving its server
1018    /// would lock the next one out.
1019    pub async fn stop_dials(&self) {
1020        let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1021        dials.abort_all();
1022        while dials.join_next().await.is_some() {}
1023    }
1024
1025    /// Wait for the first dials [`enable`](SyncRuntime::enable) left running to finish on
1026    /// their own: for a test that must not race them.
1027    #[cfg(test)]
1028    async fn settle_dials(&self) {
1029        let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1030        while dials.join_next().await.is_some() {}
1031    }
1032
1033    /// Close every open live session, as if the connections had been cut.
1034    ///
1035    /// Called for a workspace being disabled, and by tests that want to watch the dialer
1036    /// rebuild what was dropped.
1037    pub async fn drop_connections(&self) {
1038        let tables: Vec<Arc<LivePeers>> = self.live.lock().await.values().cloned().collect();
1039        for peers in tables {
1040            peers.drop_connections().await;
1041        }
1042    }
1043
1044    /// Keep a live session open to every peer, for every synced workspace.
1045    ///
1046    /// One walk per [`DIAL_INTERVAL`](live::DIAL_INTERVAL): for each synced workspace, ask
1047    /// the bridge who is in the workgroup, and open a session to every peer that is connected
1048    /// and does not already have one.
1049    ///
1050    /// A peer that cannot be reached is retried on a per-peer exponential backoff capped at
1051    /// [`DIAL_BACKOFF_MAX`] — one failed attempt per walk, not one per spin — and the counter
1052    /// resets when a session opens. An unreachable peer is therefore no obstacle to the
1053    /// others: its dial fails, the walk moves on, and the next workspace is dialled.
1054    ///
1055    /// Runs until the task is aborted, which is what the app server's shutdown does with it.
1056    /// It is allowed to fail like [`run`](SyncRuntime::run): only sync stops, and the app
1057    /// server keeps serving files (spec §10).
1058    pub async fn dial_loop(self: Arc<Self>) -> Result<()> {
1059        let mut tick = tokio::time::interval(live::DIAL_INTERVAL);
1060        // A missed tick is a late walk, not a burst of them: the interval is a pace, not a
1061        // quota to make up.
1062        tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1063        let mut waiting: HashMap<GrainId, std::time::Instant> = HashMap::new();
1064        let mut failures: HashMap<GrainId, u32> = HashMap::new();
1065        // The workspaces this host was the primary device of at the last walk.
1066        let mut primary_of: std::collections::HashSet<GrainId> = Default::default();
1067        // When the primary device last ran a backfill pass with nothing prompting it.
1068        let mut last_backfill = std::time::Instant::now();
1069        loop {
1070            tick.tick().await;
1071            let roots = self.roots().await;
1072            if roots.is_empty() {
1073                continue;
1074            }
1075            let peers = match self.bridge.peers().await {
1076                Ok(peers) => {
1077                    *self.last_peers.lock().await = Some(peers.clone());
1078                    peers
1079                }
1080                // The bridge is down or has no workgroup: nothing to dial, and the next walk
1081                // asks again. Not an error for the caller — the app server is still serving.
1082                Err(err) => {
1083                    tracing::debug!("no peers to dial: {err}");
1084                    continue;
1085                }
1086            };
1087            let me = match self.device_id().await {
1088                Ok(me) => me,
1089                Err(err) => {
1090                    tracing::warn!("no device id, so nobody to dial: {err}");
1091                    continue;
1092                }
1093            };
1094            // A host that has just become a workspace's primary device owes it the vectors
1095            // nobody else computes (#187): files that arrived before the election settled
1096            // would otherwise wait for the next change.
1097            let ids: Vec<(PathBuf, GrainId)> = self
1098                .synced
1099                .lock()
1100                .await
1101                .iter()
1102                .map(|(root, entry)| (root.clone(), entry.workspace_id))
1103                .collect();
1104            // And every so often anyway: an author that never embedded leaves files only the
1105            // primary will fill in, and nothing else would prompt it (#188).
1106            let backfill_due = last_backfill.elapsed() >= self.backfill_timing().interval;
1107            if backfill_due {
1108                last_backfill = std::time::Instant::now();
1109            }
1110            for (root, workspace_id) in ids {
1111                let primary = peers
1112                    .roles_for(workspace_id)
1113                    .is_some_and(|r| r.primary == Some(me));
1114                if !primary {
1115                    primary_of.remove(&workspace_id);
1116                } else if primary_of.insert(workspace_id) || backfill_due {
1117                    self.request_embed(&root);
1118                }
1119            }
1120            let now = std::time::Instant::now();
1121            // Dial only peers whose device id is greater than ours. Every device dialling
1122            // every other can deadlock: two hosts that dial each other at once each hold
1123            // their replica's lock across their own exchange and wait for the other's
1124            // hello, which the other's dialler holds the lock for. With one direction per
1125            // pair the wait chain is a DAG — a higher-id host never dials, so it never
1126            // holds its lock out while a lower-id host's exchange needs it. The lower host
1127            // dials within `DIAL_INTERVAL`, and the exchange is symmetric, so nothing is
1128            // lost; the tie-break is left as the safety net it is.
1129            for root in roots {
1130                let Some((key, table)) = self.live_peers_or_create(&root).await else {
1131                    continue;
1132                };
1133                let workspace_id = {
1134                    let synced = self.synced.lock().await;
1135                    match synced.get(&key) {
1136                        Some(entry) => entry.workspace_id,
1137                        None => continue,
1138                    }
1139                };
1140                let roles = peers.roles_for(workspace_id);
1141                // Entering a star closes what it no longer needs. The hubs carry it all —
1142                // once one of them is reachable from here (#192).
1143                if topology::may_close_skipped(me, roles, &table.devices().await) {
1144                    table
1145                        .retain(|d| topology::link(me, *d, roles) != topology::Link::Skip)
1146                        .await;
1147                }
1148                for peer in peers.peers.iter().filter(|p| {
1149                    p.connected && topology::link(me, p.device_id, roles) == topology::Link::Dial
1150                }) {
1151                    let device = peer.device_id;
1152                    if table.has(&device).await {
1153                        // Already talking; a redial would drop a working session.
1154                        failures.remove(&device);
1155                        waiting.remove(&device);
1156                        continue;
1157                    }
1158                    if let Some(until) = waiting.get(&device)
1159                        && now < *until
1160                    {
1161                        // Backing off after a failure; the walk skips it until then.
1162                        continue;
1163                    }
1164                    // The same claim `sync_now` takes: this walk and it can overlap, and a
1165                    // double dial in one direction is the one race the tie-break cannot
1166                    // settle. Skipping is free — the next pass, or `sync_now` itself, dials.
1167                    if !table.begin_dial(device).await {
1168                        continue;
1169                    }
1170                    let opened = self.open_stream(workspace_id, device).await;
1171                    if opened.is_err() {
1172                        table.end_dial(device).await;
1173                    }
1174                    match opened {
1175                        Ok(stream) => {
1176                            let adopted = self
1177                                .adopt_live_session(stream, &root, device, workspace_id)
1178                                .await;
1179                            // The claim is out either way: the attempt is over, and a
1180                            // session that died mid-exchange is re-dialled next pass.
1181                            table.end_dial(device).await;
1182                            match adopted {
1183                                Ok(()) => {
1184                                    failures.remove(&device);
1185                                    waiting.remove(&device);
1186                                }
1187                                Err(err) => {
1188                                    tracing::debug!(
1189                                        peer = %peer.name,
1190                                        err = %err,
1191                                        "a live session could not be started"
1192                                    );
1193                                    let count = failures.entry(device).or_insert(0);
1194                                    *count = count.saturating_add(1);
1195                                    waiting.insert(device, now + dial_backoff(*count));
1196                                }
1197                            }
1198                        }
1199                        Err(err) => {
1200                            // Normal and cheap: a peer that does not host this workspace
1201                            // refuses, and one that is unreachable cannot be opened.
1202                            tracing::debug!("no stream to a peer: {err}");
1203                            let count = failures.entry(device).or_insert(0);
1204                            *count = count.saturating_add(1);
1205                            waiting.insert(device, now + dial_backoff(*count));
1206                        }
1207                    }
1208                }
1209            }
1210        }
1211    }
1212
1213    /// Take ownership of an opened stream: run the exchange, then keep it live.
1214    ///
1215    /// The session is stored under `device` and its reader pumped until the stream ends. A
1216    /// half-open or refused stream is not stored; the dialer retries it with backoff.
1217    async fn adopt_live_session(
1218        self: &Arc<Self>,
1219        stream: sapphire_ipc::RawStream,
1220        root: &Path,
1221        device: GrainId,
1222        workspace_id: GrainId,
1223    ) -> Result<()> {
1224        let replica = self.replica_of(root).await?;
1225        let (_, session) = sapphire_framework_session::open_live_session(
1226            stream,
1227            Arc::clone(&replica),
1228            workspace_id,
1229        )
1230        .await
1231        .map_err(|e| Error::Sync(e.to_string()))?;
1232        let Some((key, table)) = self.live_peers_or_create(root).await else {
1233            return Ok(());
1234        };
1235        // This stream was dialled by us, so it is the one the pair keeps if this host's
1236        // device id is the lower of the two. If it lost the tie-break the session is
1237        // dropped here, which closes the stream, and the surviving one is already in the
1238        // table.
1239        let Some(updates) = table.insert(device, session, true).await else {
1240            return Ok(());
1241        };
1242        // The session wrote files during the exchange; the index has to catch up before the
1243        // workspace is searchable again.
1244        self.reindex(&key).await;
1245        self.request_embed(&key);
1246        self.spawn_reader(Arc::clone(&table), updates, key, device);
1247        Ok(())
1248    }
1249
1250    /// Forward what one session receives, and re-index it.
1251    ///
1252    /// This is the loop that makes A → S → B work: a batch that arrived on S's session with A
1253    /// is applied (the session crate does that before publishing it), the workspace behind
1254    /// `key` is re-indexed, and the batch is handed to every *other* session whose peer lacks
1255    /// it — `from` is the device it came from, so it never goes back to A.
1256    fn spawn_reader(
1257        self: &Arc<Self>,
1258        table: Arc<LivePeers>,
1259        mut updates: broadcast::Receiver<Vec<PathUpdate>>,
1260        key: PathBuf,
1261        from: GrainId,
1262    ) {
1263        let driver = Arc::clone(self);
1264        tokio::spawn(async move {
1265            loop {
1266                match updates.recv().await {
1267                    Ok(batch) => {
1268                        // The files are on disk now (the session guarantees it before
1269                        // publishing), so the index is behind them until the sweep below.
1270                        driver.reindex(&key).await;
1271                        driver.request_embed(&key);
1272                        table.fan_out(&batch, Some(from)).await;
1273                    }
1274                    // A burst bigger than the channel: the newest is kept and the gap is
1275                    // covered by the next session's exchange. Losing a batch here cannot lose
1276                    // an entry — the peer's vector does not cover it, so the next dial or push
1277                    // carries it again.
1278                    Err(broadcast::error::RecvError::Lagged(missed)) => {
1279                        tracing::debug!(
1280                            device = %from,
1281                            missed,
1282                            "fell behind on a live session; the next exchange catches up"
1283                        );
1284                    }
1285                    // The session closed. Its table entry is reaped by the next fan-out or
1286                    // dial pass, and the dialer opens a fresh one.
1287                    Err(broadcast::error::RecvError::Closed) => {
1288                        return;
1289                    }
1290                }
1291            }
1292        });
1293    }
1294
1295    /// The replica of a synced `root`.
1296    async fn replica_of(&self, root: &Path) -> Result<Arc<Mutex<Replica>>> {
1297        let key = root.canonicalize().map_err(Error::Io)?;
1298        let synced = self.synced.lock().await;
1299        match synced.get(&key) {
1300            Some(entry) => Ok(Arc::clone(&entry.replica)),
1301            None => Err(Error::UnknownWorkspace(key, self.ctx.app_name)),
1302        }
1303    }
1304
1305    /// Watch the synced roots and, on a debounced report, scan and dial.
1306    ///
1307    /// This is the safety net for everything the app server did not do itself — a file
1308    /// edited in an editor, a `git checkout`. The exact path is [`scan`] called from
1309    /// `handlers.rs` right after a write the server made. A scan that finds nothing is
1310    /// cheap; a scan that is skipped loses an edit until the next one.
1311    ///
1312    /// Returns when the reporting channel closes — that is, when the [`Watcher`] this
1313    /// runtime holds is dropped, which happens when the runtime itself is.
1314    ///
1315    /// [`scan`]: SyncRuntime::scan
1316    pub async fn watch_changes(self: Arc<Self>) -> Result<()> {
1317        let (tx, mut rx) = mpsc::channel(64);
1318        let watcher = Arc::new(Watcher::start(self.roots().await, tx)?);
1319        let _ = self.watcher.set(Arc::clone(&watcher));
1320
1321        // A root enabled between `roots()` above and the `set` just now was not in the
1322        // starting set, and `enable` could not reach a watcher that did not exist yet.
1323        // Watching the current set again is idempotent on `notify`'s side.
1324        for root in self.roots().await {
1325            if let Err(err) = watcher.watch(&root) {
1326                tracing::warn!(root = %root.display(), "could not watch: {err}");
1327            }
1328        }
1329
1330        while let Some(root) = rx.recv().await {
1331            // Logged, not fatal: a failed scan or dial leaves the replica behind, and the
1332            // next report tries again. Neither may take the app server down (spec §10).
1333            if let Err(err) = self.scan(&root).await {
1334                tracing::warn!(root = %root.display(), "scan after a local edit failed: {err}");
1335            }
1336            self.request_embed(&root);
1337            if let Err(err) = self.sync_now(&root).await {
1338                tracing::warn!(root = %root.display(), "dial after a local edit failed: {err}");
1339            }
1340        }
1341        Ok(())
1342    }
1343
1344    /// Tell the bridge the complete current set. A registration is not a delta.
1345    async fn reregister(&self) -> Result<()> {
1346        let workspaces: Vec<WorkspaceRegistration> = {
1347            let synced = self.synced.lock().await;
1348            synced
1349                .iter()
1350                .map(|(root, entry)| WorkspaceRegistration {
1351                    workspace_id: entry.workspace_id,
1352                    root: root.clone(),
1353                })
1354                .collect()
1355        };
1356        self.bridge
1357            .register(RegisterParams {
1358                app_name: self.ctx.app_name.to_owned(),
1359                exe_path: self.exe_path.clone(),
1360                managed_by: wire_managed_by(self.managed_by),
1361                workspaces,
1362            })
1363            .await
1364            .map_err(|e| Error::Bridge(e.to_string()))?;
1365        Ok(())
1366    }
1367
1368    /// Whether the last roles the bridge reported say this pair does not link.
1369    ///
1370    /// Reads the cached device id only: asking the bridge here would register with an empty
1371    /// workspace list, racing `enable`'s registration. `false` (accept) when it is unknown.
1372    async fn should_skip(&self, workspace_id: GrainId, peer: GrainId) -> bool {
1373        let Some(&me) = self.device_id.get() else {
1374            return false;
1375        };
1376        let cached = self.last_peers.lock().await;
1377        let roles = cached.as_ref().and_then(|p| p.roles_for(workspace_id));
1378        topology::link(me, peer, roles) == topology::Link::Skip
1379    }
1380
1381    /// Whether this host is `workspace_id`'s primary device, by the last roles the
1382    /// bridge reported. `false` when the bridge has not answered yet, or when this host's
1383    /// device id is not known yet (no workspace was enabled). Reads the cached id only:
1384    /// asking the bridge here would register with an empty workspace list, racing
1385    /// `enable`'s registration.
1386    pub async fn is_primary(&self, workspace_id: GrainId) -> bool {
1387        let Some(&me) = self.device_id.get() else {
1388            return false;
1389        };
1390        let cached = self.last_peers.lock().await;
1391        cached
1392            .as_ref()
1393            .and_then(|p| p.roles_for(workspace_id))
1394            .is_some_and(|r| r.primary == Some(me))
1395    }
1396
1397    /// This host's device id, asked of the bridge once.
1398    async fn device_id(&self) -> Result<GrainId> {
1399        if let Some(id) = self.device_id.get() {
1400            return Ok(*id);
1401        }
1402        let result = self
1403            .bridge
1404            .register(RegisterParams {
1405                app_name: self.ctx.app_name.to_owned(),
1406                exe_path: self.exe_path.clone(),
1407                managed_by: wire_managed_by(self.managed_by),
1408                workspaces: Vec::new(),
1409            })
1410            .await
1411            .map_err(|e| Error::Bridge(e.to_string()))?;
1412        let _ = self.device_id.set(result.device_id);
1413        Ok(result.device_id)
1414    }
1415
1416    /// `<workspace cache dir>/sync/`.
1417    fn state_dir(&self, root: &Path) -> Result<PathBuf> {
1418        let workspace = Workspace::from_root(self.ctx, root)?;
1419        Ok(workspace.cache_dir().join("sync"))
1420    }
1421}
1422
1423/// One dial of [`SyncRuntime::sync_now`].
1424type Dial<'a> = std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>>;
1425
1426/// Run `dials` concurrently on the calling task, until every one has finished.
1427async fn join_all(mut dials: Vec<Dial<'_>>) {
1428    std::future::poll_fn(|cx| {
1429        dials.retain_mut(|dial| dial.as_mut().poll(cx).is_pending());
1430        if dials.is_empty() {
1431            std::task::Poll::Ready(())
1432        } else {
1433            std::task::Poll::Pending
1434        }
1435    })
1436    .await
1437}
1438
1439/// The bridge API's copy of how this server was started, for [`RegisterParams`].
1440fn wire_managed_by(managed_by: ManagedBy) -> sapphire_bridge_api::ManagedBy {
1441    match managed_by {
1442        ManagedBy::Service => sapphire_bridge_api::ManagedBy::Service,
1443        ManagedBy::Spawned => sapphire_bridge_api::ManagedBy::Spawned,
1444    }
1445}
1446
1447#[cfg(test)]
1448mod tests {
1449    use super::*;
1450    use crate::sync::testing::StubBridge;
1451    use crate::test_support;
1452    use sapphire_workspace::AppKind;
1453    use std::ffi::OsString;
1454
1455    static CTX: AppContext = AppContext::new("sapphire-synctest");
1456
1457    /// The env vars `CTX.init` reads.
1458    const DIR_VARS: [&str; 3] = [
1459        "SAPPHIRE_SYNCTEST_CACHE_DIR",
1460        "SAPPHIRE_SYNCTEST_DATA_DIR",
1461        "SAPPHIRE_SYNCTEST_CONFIG_DIR",
1462    ];
1463
1464    /// Points the context's directories at the test's scratch tree, and restores the
1465    /// previous values when dropped — including while unwinding from a panic.
1466    struct EnvGuard {
1467        previous: [Option<OsString>; 3],
1468        _lock: std::sync::MutexGuard<'static, ()>,
1469    }
1470
1471    impl Drop for EnvGuard {
1472        fn drop(&mut self) {
1473            // SAFETY: `self._lock` still serialises the environment; it is dropped only
1474            // after this method returns.
1475            for (name, previous) in DIR_VARS.iter().zip(self.previous.iter_mut()) {
1476                match previous.take() {
1477                    Some(value) => unsafe { std::env::set_var(name, value) },
1478                    None => test_support::remove(name),
1479                }
1480            }
1481        }
1482    }
1483
1484    /// Everything one test needs, held for the test's whole body.
1485    ///
1486    /// The context is a `static` shared by this binary, so its directories are resolved from
1487    /// the environment by whichever test initialises it first. The environment lock is
1488    /// therefore held for the whole test, as in the crate's other test modules, and `_tmp`
1489    /// is declared before `_env` so the scratch tree is gone only once the environment no
1490    /// longer points at it.
1491    struct Fixture {
1492        _tmp: tempfile::TempDir,
1493        _env: EnvGuard,
1494        root: PathBuf,
1495        stub: StubBridge,
1496        runtime: Arc<SyncRuntime>,
1497    }
1498
1499    impl Fixture {
1500        /// A second workspace root beside the first, marker directory and all.
1501        fn second_root(&self) -> PathBuf {
1502            let root = self._tmp.path().join("ws2");
1503            std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
1504            root.canonicalize().unwrap()
1505        }
1506    }
1507
1508    /// How long the fixture's runtime waits on a stream to a peer.
1509    const FIXTURE_OPEN_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
1510
1511    /// `n` peers this host dials (a greater device id), as `bridge.peers` lists them.
1512    fn dialled_peers(me: GrainId, n: usize) -> Vec<sapphire_bridge_api::PeerInfo> {
1513        std::iter::repeat_with(GrainId::random)
1514            .filter(|id| *id > me)
1515            .take(n)
1516            .enumerate()
1517            .map(|(i, device_id)| sapphire_bridge_api::PeerInfo {
1518                device_id,
1519                name: format!("offline-{i}"),
1520                node_id: format!("node-{i}"),
1521                connected: false,
1522                priority: 1,
1523                availability: None,
1524            })
1525            .collect()
1526    }
1527
1528    #[tokio::test(flavor = "multi_thread")]
1529    async fn enabling_answers_at_once_with_a_peer_that_never_answers() {
1530        // #228: the first dial waited on a peer that was offline, past the client's limit.
1531        let f = fixture().await;
1532        let _hung = f.stub.hang_streams().await;
1533        f.stub.set_peers(dialled_peers(f.stub.device_id, 1));
1534
1535        let started = std::time::Instant::now();
1536        f.runtime.enable(&f.root).await.unwrap();
1537        assert!(
1538            started.elapsed() < FIXTURE_OPEN_TIMEOUT,
1539            "enable waited {:?} on the dial",
1540            started.elapsed()
1541        );
1542    }
1543
1544    #[tokio::test(flavor = "multi_thread")]
1545    async fn peers_that_never_answer_are_dialled_together_and_given_up_on() {
1546        let f = fixture().await;
1547        let _hung = f.stub.hang_streams().await;
1548        f.runtime.enable(&f.root).await.unwrap();
1549        // The first dial, with nobody to dial yet, must be over before the peers appear:
1550        // one still running would claim them, and `sync_now` would skip them all.
1551        f.runtime.settle_dials().await;
1552        f.stub.set_peers(dialled_peers(f.stub.device_id, 3));
1553
1554        let started = std::time::Instant::now();
1555        f.runtime.sync_now(&f.root).await.unwrap();
1556        let took = started.elapsed();
1557        assert!(took >= FIXTURE_OPEN_TIMEOUT, "gave up early: {took:?}");
1558        assert!(
1559            took < FIXTURE_OPEN_TIMEOUT * 2,
1560            "three dials one after another: {took:?}"
1561        );
1562    }
1563
1564    async fn fixture() -> Fixture {
1565        let lock = test_support::lock();
1566        let tmp = tempfile::tempdir().unwrap();
1567        let previous = DIR_VARS.map(std::env::var_os);
1568        // SAFETY (via `test_support::set`): `lock` serialises every read and write of the
1569        // process environment in this test binary, and it is held until `drop` has restored
1570        // the old values.
1571        for (name, dir) in DIR_VARS
1572            .iter()
1573            .zip(["cache", "data", "config"].map(|cat| tmp.path().join(cat)))
1574        {
1575            test_support::set(name, &dir);
1576        }
1577        let env = EnvGuard {
1578            previous,
1579            _lock: lock,
1580        };
1581        CTX.init(AppKind::Server);
1582
1583        let root = tmp.path().join("ws");
1584        std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
1585        let root = root.canonicalize().unwrap();
1586
1587        let (stub, client) = StubBridge::start_in(tmp.path().join("run")).await;
1588        let runtime = Arc::new(
1589            SyncRuntime::new(&CTX, client, "/bin/true".into(), ManagedBy::Service)
1590                .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
1591        );
1592        Fixture {
1593            _tmp: tmp,
1594            _env: env,
1595            root,
1596            stub,
1597            runtime,
1598        }
1599    }
1600
1601    #[tokio::test(flavor = "multi_thread")]
1602    async fn enabling_registers_the_workspace_with_the_bridge() {
1603        let f = fixture().await;
1604
1605        let id = f.runtime.enable(&f.root).await.unwrap();
1606        assert_eq!(f.stub.last_workspaces(), vec![id]);
1607    }
1608
1609    #[tokio::test(flavor = "multi_thread")]
1610    async fn enabling_twice_is_idempotent_and_keeps_the_same_id() {
1611        let f = fixture().await;
1612
1613        let first = f.runtime.enable(&f.root).await.unwrap();
1614        let second = f.runtime.enable(&f.root).await.unwrap();
1615        assert_eq!(first, second);
1616        assert_eq!(
1617            f.stub.last_workspaces(),
1618            vec![first],
1619            "still exactly one workspace"
1620        );
1621    }
1622
1623    #[tokio::test(flavor = "multi_thread")]
1624    async fn a_registration_carries_the_complete_current_list() {
1625        let f = fixture().await;
1626        let second_root = f.second_root();
1627
1628        let a = f.runtime.enable(&f.root).await.unwrap();
1629        let b = f.runtime.enable(&second_root).await.unwrap();
1630
1631        let mut seen = f.stub.last_workspaces();
1632        seen.sort();
1633        let mut want = vec![a, b];
1634        want.sort();
1635        assert_eq!(seen, want, "registration is the whole set, not a delta");
1636    }
1637
1638    #[tokio::test(flavor = "multi_thread")]
1639    async fn no_later_registration_clears_this_apps_routes() {
1640        // The device id is asked of the bridge once, on the first `enable`, with an empty
1641        // workspace list — the one moment that is safe, because no route exists yet. Every
1642        // later registration must name the complete set: one with an empty list would
1643        // briefly clear this application's routes, and a peer asking for a workspace this
1644        // server owns would be told nobody has it.
1645        let f = fixture().await;
1646        let second_root = f.second_root();
1647
1648        f.runtime.enable(&f.root).await.unwrap();
1649        f.runtime.enable(&second_root).await.unwrap();
1650        f.runtime.sync_now(&f.root).await.unwrap();
1651        f.runtime.disable(&f.root).await.unwrap();
1652
1653        let seen = f.stub.seen.lock().expect("stub");
1654        let first_full = seen
1655            .registrations
1656            .iter()
1657            .position(|r| !r.workspaces.is_empty())
1658            .expect("enabling registers the workspace");
1659        let emptied_afterwards = seen.registrations[first_full + 1..]
1660            .iter()
1661            .filter(|r| r.workspaces.is_empty())
1662            .count();
1663        assert_eq!(
1664            emptied_afterwards, 0,
1665            "a registration after the first workspace must still name the whole set"
1666        );
1667    }
1668
1669    #[tokio::test(flavor = "multi_thread")]
1670    async fn disabling_unregisters_and_leaves_the_files_alone() {
1671        let f = fixture().await;
1672        std::fs::write(f.root.join("keep.md"), "content").unwrap();
1673
1674        let id = f.runtime.enable(&f.root).await.unwrap();
1675        f.runtime.disable(&f.root).await.unwrap();
1676
1677        assert!(f.stub.last_workspaces().is_empty());
1678        assert!(
1679            f.root.join("keep.md").exists(),
1680            "disabling sync must not touch files"
1681        );
1682        assert!(
1683            crate::sync::id::sync_id_path("sapphire-synctest", &f.root).exists(),
1684            "the sync id stays, so re-enabling rejoins the same workspace"
1685        );
1686        let _ = id;
1687    }
1688
1689    #[tokio::test(flavor = "multi_thread")]
1690    async fn re_enabling_after_disabling_reuses_the_identity() {
1691        let f = fixture().await;
1692
1693        let first = f.runtime.enable(&f.root).await.unwrap();
1694        f.runtime.disable(&f.root).await.unwrap();
1695        let again = f.runtime.enable(&f.root).await.unwrap();
1696
1697        assert_eq!(first, again);
1698    }
1699
1700    #[tokio::test(flavor = "multi_thread")]
1701    async fn status_reports_disabled_for_a_workspace_that_was_never_enabled() {
1702        let f = fixture().await;
1703
1704        let status = f.runtime.status(&f.root).await;
1705        assert!(!status.enabled);
1706        assert!(status.workspace_id.is_none());
1707    }
1708
1709    #[tokio::test(flavor = "multi_thread")]
1710    async fn disabling_something_that_was_never_enabled_is_not_an_error() {
1711        let f = fixture().await;
1712        f.runtime
1713            .disable(&f.root)
1714            .await
1715            .expect("disabling twice is fine");
1716    }
1717
1718    #[tokio::test(flavor = "multi_thread")]
1719    async fn a_workspace_whose_root_vanished_is_reported_as_paused_not_deleted() {
1720        let f = fixture().await;
1721        std::fs::write(f.root.join("a.md"), "content").unwrap();
1722        f.runtime.enable(&f.root).await.unwrap();
1723        f.runtime.scan(&f.root).await.unwrap();
1724
1725        // The drive is unmounted: the marker directory goes with it.
1726        std::fs::remove_dir_all(f.root.join(".sapphire-synctest")).unwrap();
1727        let _ = f.runtime.scan(&f.root).await;
1728
1729        let status = f.runtime.status(&f.root).await;
1730        assert!(
1731            status.paused.is_some(),
1732            "a missing root must pause, not replicate as a mass deletion"
1733        );
1734    }
1735
1736    #[tokio::test(flavor = "multi_thread")]
1737    async fn a_directory_that_is_not_a_workspace_cannot_be_enabled() {
1738        let f = fixture().await;
1739        let plain = f._tmp.path().join("plain");
1740        std::fs::create_dir_all(&plain).unwrap();
1741
1742        assert!(f.runtime.enable(&plain).await.is_err());
1743    }
1744
1745    #[tokio::test(flavor = "multi_thread")]
1746    async fn a_write_through_the_workspace_router_is_committed_to_the_replica() {
1747        // The brief's "main path": a write the app server itself made is scanned straight
1748        // after, not left to the watcher's debounce. Asserted against the replica's own
1749        // state, which is the only thing that can tell the two paths apart — reading the
1750        // file back would pass even if the scan never ran.
1751        let f = fixture().await;
1752        f.runtime.enable(&f.root).await.unwrap();
1753
1754        let host = Arc::new(crate::WorkspaceHost::new(&CTX));
1755        host.select(&f.root, false).await.unwrap();
1756        let current = Arc::new(crate::Current::new(
1757            &CTX,
1758            host,
1759            Some(Arc::clone(&f.runtime)),
1760            crate::selection::SelectionFile::at(f._tmp.path().join("workspace.toml")),
1761        ));
1762        let router = Arc::new(crate::workspace_router(current));
1763        let (client_conn, server_conn) = sapphire_ipc::Connection::pair();
1764        tokio::spawn(async move {
1765            let info = sapphire_ipc::ServerInfo {
1766                version: "0.0.0".into(),
1767                api: sapphire_backend::protocol::API_VERSION,
1768                app_api: None,
1769                pid: std::process::id(),
1770                managed_by: ManagedBy::Spawned,
1771            };
1772            let _ = sapphire_ipc::serve(server_conn, router, "sapphire-synctest", info).await;
1773        });
1774        let info = sapphire_ipc::ClientInfo {
1775            kind: "test".into(),
1776            version: "0.0.0".into(),
1777            api: sapphire_backend::protocol::API_VERSION,
1778            app_api: None,
1779            pid: std::process::id(),
1780        };
1781        let (client, _) = sapphire_ipc::Client::handshake(client_conn, "sapphire-synctest", info)
1782            .await
1783            .unwrap();
1784
1785        let _: sapphire_backend::protocol::Ack = client
1786            .call(
1787                sapphire_backend::protocol::WRITE_FILE,
1788                sapphire_backend::protocol::ContentParams {
1789                    path: PathBuf::from("through-the-server.md"),
1790                    content: "written by the server".into(),
1791                },
1792            )
1793            .await
1794            .unwrap();
1795
1796        // `scan` runs before the reply is written, so by the time the call returns the
1797        // replica must already know the path.
1798        let synced = f.runtime.synced.lock().await;
1799        let entry = synced.get(&f.root).expect("enabled above");
1800        let replica = entry.replica.lock().await;
1801        let state = replica
1802            .state("through-the-server.md")
1803            .expect("a readable state");
1804        assert!(
1805            state.is_some(),
1806            "a write through the server must be scanned into the replica, not left to the watcher"
1807        );
1808    }
1809
1810    #[tokio::test(flavor = "multi_thread")]
1811    async fn an_announcement_for_an_unknown_workspace_is_ignored_not_fatal() {
1812        let f = fixture().await;
1813        f.runtime.enable(&f.root).await.unwrap();
1814
1815        let driver = Arc::clone(&f.runtime);
1816        let handle = tokio::spawn(async move { driver.run().await });
1817
1818        f.stub
1819            .announce(grain_id::GrainId::random(), "no-such-ticket")
1820            .await;
1821        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
1822
1823        assert!(
1824            !handle.is_finished(),
1825            "one bad announcement must not stop the runtime"
1826        );
1827        handle.abort();
1828    }
1829}