Skip to main content

sapphire_framework_server/sync/
mod.rs

1//! Replication, from the app server's side.
2//!
3//! One `Replica` per synced workspace, registered with the bridge, driven by announcements
4//! from it and by local edits.
5//!
6//! See `docs/superpowers/specs/2026-09-16-process-architecture-design.md` §4.2.
7
8use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use grain_id::GrainId;
13use sapphire_backend::protocol as proto;
14use sapphire_bridge_api::{PeersResult, RegisterParams, WorkspaceRegistration, WorkspacesResult};
15use sapphire_bridge_client::BridgeClient;
16use sapphire_ipc::ManagedBy;
17use sapphire_sync::{PathUpdate, PauseReason, Replica, ReplicaConfig, ScanOutcome, SystemClock};
18use sapphire_workspace::{AppContext, Workspace};
19use tokio::sync::{Mutex, OnceCell, broadcast, mpsc};
20
21pub mod id;
22mod live;
23mod methods;
24#[cfg(any(test, feature = "test-util"))]
25pub mod testing;
26mod topology;
27mod watch;
28
29pub use id::{SYNC_ID_FILE, WORKSPACE_MAP_FILE, sync_id, sync_id_path};
30pub use live::DIAL_BACKOFF_MAX;
31pub use methods::sync_router;
32pub use watch::{DEBOUNCE, Watcher};
33
34use live::{LivePeers, dial_backoff};
35
36use crate::error::{Error, Result};
37
38use crate::host::WorkspaceHost;
39
40/// What `sync.status` reports.
41#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
42pub struct SyncStatus {
43    /// Whether this workspace is synced at all.
44    pub enabled: bool,
45    /// Its identity across devices, when it is.
46    pub workspace_id: Option<GrainId>,
47    /// How many devices the workgroup has, this host excluded.
48    pub peers: usize,
49    /// Why replication is paused, if it is — a missing root or marker directory.
50    pub paused: Option<String>,
51    /// The last failure, if any.
52    pub last_error: Option<String>,
53    /// Whether the bridge is reachable. `false` is not an outage: the app server works.
54    pub bridge_available: bool,
55    /// How the workspace is wired to its peers, by the roles the bridge last reported.
56    pub topology: proto::Topology,
57    /// How far embedding has got, when this host embeds (#188).
58    pub embedding: Option<proto::EmbeddingProgress>,
59}
60
61/// One workspace this runtime syncs.
62struct Synced {
63    /// Its identity across devices, shared with every peer.
64    workspace_id: GrainId,
65    /// Its replica store. Held open because the store is a redb database.
66    replica: Arc<Mutex<Replica>>,
67    /// Why replication is paused, from the last scan.
68    paused: Option<PauseReason>,
69    /// The last failure, if any.
70    last_error: Option<String>,
71}
72
73/// How old a vector file no live content needs must be before the primary device removes
74/// it: a vector can arrive before its file, and a day is far longer than that window.
75const STALE_VECTOR_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
76
77/// Texts per embedding request in a server pass: small, so a search query waiting on the
78/// bridge's one worker is never stuck behind a long backfill (#188).
79const PASS_BATCH: usize = 4;
80
81/// How the primary device paces its backfill (#188).
82#[derive(Clone, Copy, Debug)]
83pub struct BackfillTiming {
84    /// How old another device's version must be before the primary embeds it: its author
85    /// has embedded it by then, or will not.
86    pub grace: std::time::Duration,
87    /// How often the primary device runs a pass with nothing prompting it.
88    pub interval: std::time::Duration,
89}
90
91impl Default for BackfillTiming {
92    fn default() -> Self {
93        BackfillTiming {
94            grace: std::time::Duration::from_secs(10 * 60),
95            interval: std::time::Duration::from_secs(10 * 60),
96        }
97    }
98}
99
100/// Replication for one application's workspaces.
101pub struct SyncRuntime {
102    ctx: &'static AppContext,
103    bridge: Arc<BridgeClient>,
104    exe_path: PathBuf,
105    managed_by: ManagedBy,
106    synced: Mutex<HashMap<PathBuf, Synced>>,
107    /// This host's device id. Asked of the bridge once, on the first [`enable`], and never
108    /// again: a second registration with an empty workspace list would briefly clear this
109    /// application's routes.
110    ///
111    /// [`enable`]: SyncRuntime::enable
112    device_id: OnceCell<GrainId>,
113    /// The file watcher, once [`watch_changes`] has started it.
114    ///
115    /// The runtime owns it because the spec puts it here: the sync runtime is a watcher and
116    /// a replica per registered root, and the two must agree on which roots are synced.
117    /// [`enable`] and [`disable`] keep them in step.
118    ///
119    /// [`watch_changes`]: SyncRuntime::watch_changes
120    /// [`enable`]: SyncRuntime::enable
121    /// [`disable`]: SyncRuntime::disable
122    watcher: OnceCell<Arc<Watcher>>,
123    /// The app server that owns these workspaces' caches, once it has told us.
124    ///
125    /// A replication session writes files, and the search index is not part of a session: the
126    /// index belongs to the app server, which holds the workspace open. Reaching it through
127    /// the host is what keeps a received file and its index moving together, and it is the
128    /// *same* [`WorkspaceState`](sapphire_workspace::WorkspaceState) the server serves from —
129    /// opening a second one would collide on the retrieve store's exclusive lock.
130    ///
131    /// Absent for a runtime nobody has wired to an app server (a test stub driving the
132    /// replica directly). Nothing then indexes, which is correct: nothing owns an index.
133    host: OnceCell<Arc<WorkspaceHost>>,
134    /// Serialises re-indexing, so two sessions finishing at once do not both sweep the
135    /// workspace through the same store.
136    reindexing: Mutex<()>,
137    /// Roots with an embedding pass waiting to run (#187). A request for a root already
138    /// waiting is folded into that pass.
139    embed_wanted: Mutex<std::collections::HashSet<PathBuf>>,
140    /// Serialises embedding passes: one model, one queue.
141    embedding: Mutex<()>,
142    /// Roots whose embedding pass is running now, for `sync.status`.
143    embed_running: Mutex<std::collections::HashSet<PathBuf>>,
144    /// The primary device's backfill pacing (#188).
145    backfill: std::sync::Mutex<BackfillTiming>,
146    /// The bridge an embedding pass asks for its model; `None` is the host's standard one.
147    bridge_endpoint: Option<sapphire_ipc::Endpoint>,
148    /// The live session table of every synced workspace, keyed by canonical root.
149    ///
150    /// One table per workspace rather than one for the runtime: a session is about one
151    /// workspace, and the two storm rules are per-workspace rules. A root that is not
152    /// synced has none, so a dial or a push for it is a no-op rather than an error.
153    live: Mutex<HashMap<PathBuf, Arc<LivePeers>>>,
154    /// The bridge's last answer to `peers`, roles included. The inbound guard and
155    /// [`is_primary`](SyncRuntime::is_primary) read it rather than asking again.
156    last_peers: Mutex<Option<PeersResult>>,
157}
158
159impl SyncRuntime {
160    /// A runtime for `ctx`'s application.
161    pub fn new(
162        ctx: &'static AppContext,
163        bridge: Arc<BridgeClient>,
164        exe_path: PathBuf,
165        managed_by: ManagedBy,
166    ) -> SyncRuntime {
167        SyncRuntime {
168            ctx,
169            bridge,
170            exe_path,
171            managed_by,
172            synced: Mutex::new(HashMap::new()),
173            device_id: OnceCell::new(),
174            watcher: OnceCell::new(),
175            host: OnceCell::new(),
176            reindexing: Mutex::new(()),
177            embed_wanted: Mutex::new(std::collections::HashSet::new()),
178            embedding: Mutex::new(()),
179            embed_running: Mutex::new(std::collections::HashSet::new()),
180            backfill: std::sync::Mutex::new(BackfillTiming::default()),
181            bridge_endpoint: None,
182            live: Mutex::new(HashMap::new()),
183            last_peers: Mutex::new(None),
184        }
185    }
186
187    /// Tell the runtime which app server owns these workspaces' caches.
188    ///
189    /// Called by [`AppServer::sync`](crate::AppServer::sync). First writer wins: a runtime
190    /// belongs to the server that wired it.
191    /// Embed through the bridge at `endpoint` rather than the host's standard one: for a
192    /// fixture running several hosts in one process.
193    pub fn with_bridge_endpoint(mut self, endpoint: sapphire_ipc::Endpoint) -> SyncRuntime {
194        self.bridge_endpoint = Some(endpoint);
195        self
196    }
197
198    /// Pace the primary device's backfill differently: for tests, which cannot wait ten
199    /// minutes.
200    pub fn set_backfill_timing(&self, timing: BackfillTiming) {
201        *self.backfill.lock().expect("backfill timing") = timing;
202    }
203
204    fn backfill_timing(&self) -> BackfillTiming {
205        *self.backfill.lock().expect("backfill timing")
206    }
207
208    pub(crate) fn set_host(&self, host: Arc<WorkspaceHost>) {
209        let _ = self.host.set(host);
210    }
211
212    /// Start syncing `root`, returning its identity across devices.
213    ///
214    /// Idempotent: enabling an already-synced workspace returns the same id and changes
215    /// nothing.
216    pub async fn enable(self: &Arc<Self>, root: &Path) -> Result<GrainId> {
217        let key = root.canonicalize().map_err(Error::Io)?;
218        let watch_key = key.clone();
219        let workspace_id = sync_id(self.ctx.app_name, &key)?;
220
221        // The map stays locked across opening the replica, so two simultaneous `enable`s for
222        // one root cannot both open it — the store is a redb database, and it is opened once.
223        {
224            let mut synced = self.synced.lock().await;
225            if let Some(existing) = synced.get(&key) {
226                return Ok(existing.workspace_id);
227            }
228            let device_id = self.device_id().await?;
229            let state_dir = self.state_dir(&key)?;
230            std::fs::create_dir_all(&state_dir).map_err(Error::Io)?;
231            let config = ReplicaConfig::new(self.ctx.app_name, key.clone(), device_id, &state_dir);
232            let replica = Replica::open(config, Arc::new(SystemClock))
233                .map_err(|e| Error::Sync(e.to_string()))?;
234            synced.insert(
235                key.clone(),
236                Synced {
237                    workspace_id,
238                    replica: Arc::new(Mutex::new(replica)),
239                    paused: None,
240                    last_error: None,
241                },
242            );
243        }
244
245        // Watch the new root too, so an edit made outside the server from now on is seen.
246        // Not fatal if it fails: the app server's own writes take the exact path in
247        // `handlers.rs`, and the next `watch_changes` re-reads the root set anyway.
248        if let Some(watcher) = self.watcher.get()
249            && let Err(err) = watcher.watch(&watch_key)
250        {
251            tracing::warn!(root = %watch_key.display(), "could not watch: {err}");
252        }
253
254        // Register only once the replica is open, so a registration never names a workspace
255        // that cannot serve a session.
256        self.reregister().await?;
257
258        // Dial now, so enabling a workspace converges it rather than waiting for the next
259        // local edit to trigger a session. This is what catches a host up when it returns:
260        // the peer that has been running is the one the returning host has to ask, and no
261        // watcher on either side fires for a file that was never written locally.
262        //
263        // Best-effort: a workspace is enabled whether or not any peer is up, and a failure
264        // here is not a failure to enable. A session is symmetric, so dialing is enough —
265        // this side sends its own view, and the peer answers with what this side lacks.
266        if let Err(err) = self.sync_now(&key).await {
267            tracing::warn!(root = %key.display(), "the first session after enabling failed: {err}");
268        }
269        Ok(workspace_id)
270    }
271
272    /// Stop syncing `root`. Files and the sync id stay, so re-enabling rejoins the same
273    /// workspace rather than creating a second one.
274    pub async fn disable(&self, root: &Path) -> Result<()> {
275        // A root that no longer exists cannot be canonicalized, but the keys of `synced` are
276        // canonical already, so the path as given (a registry root) still names the entry.
277        let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
278        let removed = self.synced.lock().await.remove(&key);
279        if removed.is_some()
280            && let Some(watcher) = self.watcher.get()
281        {
282            watcher.unwatch(&key);
283        }
284        // Any live session for a workspace this server no longer holds is closed: it would
285        // otherwise keep pushing files for a workspace nobody asked about any more.
286        if let Some(peers) = self.live.lock().await.remove(&key) {
287            peers.drop_connections().await;
288        }
289        if let Some(removed) = removed {
290            self.bridge
291                .unregister(removed.workspace_id)
292                .await
293                .map_err(|e| Error::Bridge(e.to_string()))?;
294            self.reregister().await?;
295        }
296        Ok(())
297    }
298
299    /// The workspaces the workgroup knows about.
300    ///
301    /// Read straight from the bridge, so it reflects every device's registrations. The
302    /// bridge being down is not an empty answer — that would read as "nothing to map" — so
303    /// it is an error, and `sync.map` reports it.
304    pub async fn workspaces(&self) -> Result<WorkspacesResult> {
305        self.bridge
306            .workspaces()
307            .await
308            .map_err(|e| Error::Bridge(e.to_string()))
309    }
310
311    /// Place a workgroup workspace at a directory of this host: write its identity and
312    /// map there, returning the canonical root. [`Current::map`](crate::Current::map)
313    /// then selects it and syncs it.
314    ///
315    /// `selector` is the workspace's name or id, as [`workspaces`] lists it; `dir` must
316    /// already be a workspace of this application — the directory is placed, not made.
317    ///
318    /// [`workspaces`]: SyncRuntime::workspaces
319    pub async fn place(&self, selector: &str, dir: &Path) -> Result<PathBuf> {
320        let workgroup = self.workspaces().await?;
321        let wanted = workgroup
322            .workspaces
323            .iter()
324            // A name is the user-facing handle; the id is the exact one. Two workspaces
325            // may not share a name, so a name names at most one.
326            .find(|w| w.name == selector || w.workspace_id.to_string() == selector)
327            .ok_or_else(|| Error::UnknownWorkspaceName(selector.to_owned()))?;
328
329        // Refuse another application's workspace: this server syncs its own application's
330        // workspaces, and `Workspace::from_root` would later refuse the directory anyway.
331        if wanted.app_name != self.ctx.app_name {
332            return Err(Error::WrongApp {
333                name: wanted.name.clone(),
334                app_name: wanted.app_name.clone(),
335            });
336        }
337
338        let root = dir.canonicalize().map_err(Error::Io)?;
339        let workspace = Workspace::from_root(self.ctx, &root)?;
340        if workspace.root != root {
341            return Err(Error::UnknownWorkspace(root, self.ctx.app_name));
342        }
343
344        // A directory already syncing under another identity is not remapped: the sync id
345        // is a device's word that it is the same workspace, and rewriting it silently
346        // would make two hosts disagree about what they share.
347        {
348            let synced = self.synced.lock().await;
349            if let Some(entry) = synced.get(&root)
350                && entry.workspace_id != wanted.workspace_id
351            {
352                return Err(Error::SyncId(format!(
353                    "{} already syncs as {}, not {}",
354                    root.display(),
355                    entry.workspace_id,
356                    wanted.workspace_id
357                )));
358            }
359        }
360
361        // The identity arrives from the workgroup, not from this host: two hosts that map
362        // the same workspace must sync as one, so the id is the one the workgroup lists,
363        // written before `enable` reads it. A different id already on disk is refused
364        // above; an absent one is written here.
365        let marker = workspace.marker_dir();
366        let id_path = marker.join(crate::sync::id::SYNC_ID_FILE);
367        match std::fs::read_to_string(&id_path) {
368            Ok(existing) if existing.trim() != wanted.workspace_id.to_string() => {
369                return Err(Error::SyncId(format!(
370                    "{} already holds identity {}, not {}",
371                    root.display(),
372                    existing.trim(),
373                    wanted.workspace_id
374                )));
375            }
376            Err(_) => {
377                std::fs::write(&id_path, format!("{}\n", wanted.workspace_id))
378                    .map_err(Error::Io)?;
379            }
380            Ok(_) => {}
381        }
382
383        // The map is what tells this application, on a later start, which workgroup
384        // workspace a directory is.
385        let map_path = root
386            .join(format!(".{}", self.ctx.app_name))
387            .join(WORKSPACE_MAP_FILE);
388        let workspace_id = wanted.workspace_id.to_string();
389        std::fs::write(&map_path, format!("{workspace_id}\n")).map_err(Error::Io)?;
390        Ok(root)
391    }
392
393    /// What `sync.status` answers with.
394    pub async fn status(&self, root: &Path) -> SyncStatus {
395        let bridge_available = self.bridge.status().await.is_ok();
396        let answer = self.bridge.peers().await.ok();
397        if let Some(answer) = &answer {
398            *self.last_peers.lock().await = Some(answer.clone());
399        }
400        let peers = answer
401            .as_ref()
402            .map_or(0, |p| p.peers.len().saturating_sub(1));
403        let Ok(key) = root.canonicalize() else {
404            return SyncStatus {
405                enabled: false,
406                workspace_id: None,
407                peers,
408                paused: None,
409                last_error: None,
410                bridge_available,
411                topology: proto::Topology::Mesh,
412                embedding: None,
413            };
414        };
415        let synced = self.synced.lock().await;
416        match synced.get(&key) {
417            Some(entry) => SyncStatus {
418                enabled: true,
419                workspace_id: Some(entry.workspace_id),
420                peers,
421                paused: entry.paused.map(|r| format!("{r:?}")),
422                last_error: entry.last_error.clone(),
423                bridge_available,
424                topology: topology::topology(
425                    answer
426                        .as_ref()
427                        .and_then(|p| p.roles_for(entry.workspace_id)),
428                ),
429                embedding: self.embedding_progress(&key).await,
430            },
431            None => SyncStatus {
432                enabled: false,
433                workspace_id: None,
434                peers,
435                paused: None,
436                last_error: None,
437                bridge_available,
438                topology: proto::Topology::Mesh,
439                embedding: None,
440            },
441        }
442    }
443
444    /// How far `key`'s embedding has got, when this host embeds it. Reads the open
445    /// workspace's index; never opens one, so a status call stays cheap.
446    async fn embedding_progress(&self, key: &Path) -> Option<proto::EmbeddingProgress> {
447        let host = self.host.get()?;
448        let backend = host.open_backend(key)?;
449        let state = backend.state();
450        state.embedder()?;
451        let info = state.db_info().ok()?;
452        Some(proto::EmbeddingProgress {
453            vectors: info.vector_count,
454            pending: info.pending_count,
455            running: self.embed_running.lock().await.contains(key),
456        })
457    }
458
459    /// Bring the replica's view of the files up to date.
460    ///
461    /// Called straight after the app server's own writes, and by the watcher for everything
462    /// else. A scan that finds nothing is cheap; a scan that is skipped loses an edit until
463    /// the next one.
464    pub async fn scan(&self, root: &Path) -> Result<()> {
465        let Ok(key) = root.canonicalize() else {
466            return Ok(());
467        };
468        let replica = {
469            let synced = self.synced.lock().await;
470            match synced.get(&key) {
471                Some(entry) => Arc::clone(&entry.replica),
472                None => return Ok(()),
473            }
474        };
475        let (outcome, updates) = {
476            let mut replica = replica.lock().await;
477            let outcome = replica.scan().map_err(|e| Error::Sync(e.to_string()))?;
478            // What the scan committed, as path updates, so it can go out on the sessions
479            // that are already open. This is the whole point of the exact-path scan in
480            // `handlers.rs`: a write through the server reaches a peer live, not at the next
481            // dial.
482            let updates = match &outcome {
483                ScanOutcome::Scanned(report) => {
484                    // The unit of replication is the path state (design doc section 2.4):
485                    // the `seen` a peer prunes against is the *path's* version vector - the
486                    // context the recorded entry was built on plus its own dot - never the
487                    // whole replica vv. Sending the global vv here made receivers prune
488                    // causally-related versions they still held: a conflict copy on the
489                    // first remote edit, and tombstones pruned so deletes stopped
490                    // propagating (issue #157).
491                    report
492                        .recorded
493                        .iter()
494                        .map(|entry| PathUpdate {
495                            path: entry.path.clone(),
496                            versions: vec![entry.clone()],
497                            seen: {
498                                let mut seen = entry.context.clone();
499                                seen.add_dot(&entry.dot);
500                                seen
501                            },
502                        })
503                        .collect()
504                }
505                ScanOutcome::Paused(_) => Vec::new(),
506            };
507            (outcome, updates)
508        };
509        {
510            let mut synced = self.synced.lock().await;
511            if let Some(entry) = synced.get_mut(&key) {
512                entry.paused = match outcome {
513                    ScanOutcome::Paused(reason) => Some(reason),
514                    ScanOutcome::Scanned(_) => None,
515                };
516            }
517        }
518        // After the map is unlocked: a push awaits every peer, and the table must not be
519        // held behind a slow one.
520        self.after_commit(&key, updates).await;
521        Ok(())
522    }
523
524    /// Bring the search index up to date with files a session has written.
525    ///
526    /// A replication session moves files, not index rows: the index belongs to the app
527    /// server, and the sync core is deliberately free of it. So the side that *applied* a
528    /// change re-indexes, which is what makes a file that arrived from a peer searchable
529    /// without a manual reindex — the claim the whole architecture rests on.
530    ///
531    /// Incremental, not a full rebuild: the workspace's mtime/size stamps mean only what
532    /// actually changed is read back. A file the session wrote has a new stamp, so it is
533    /// picked up; one that was already indexed and untouched is skipped.
534    ///
535    /// Best-effort by contract: the caller's session has already succeeded, and a failure
536    /// here must not be reported as the session's. It is logged and the next session, local
537    /// edit or explicit `workspace.reindex` tries again.
538    async fn reindex(&self, root: &Path) {
539        let Some(host) = self.host.get() else {
540            return;
541        };
542        let host = Arc::clone(host);
543        let root = root.to_owned();
544        // One sweep at a time: two sessions that finish together would otherwise both walk
545        // the workspace through the same retrieve store.
546        let _serialised = self.reindexing.lock().await;
547        // Opening the workspace is itself blocking work, and it is where the exclusive
548        // retrieve-store lock is taken, so it happens here rather than inside the sweep.
549        let backend = match host.backend_at(&root).await {
550            Ok(backend) => backend,
551            Err(err) => {
552                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
553                return;
554            }
555        };
556        let state = Arc::clone(backend.state());
557        let result = tokio::task::spawn_blocking(move || state.sync_retrieve()).await;
558        match result {
559            Ok(Ok(report)) => {
560                tracing::debug!(root = %root.display(), ?report, "re-indexed after a session");
561            }
562            Ok(Err(err)) => {
563                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
564            }
565            Err(err) => {
566                tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
567            }
568        }
569    }
570
571    /// Run an embedding pass over `root` in the background, after the pass already
572    /// waiting for it if there is one (#187).
573    ///
574    /// The pass reads the vector files that arrived, and computes the missing ones this
575    /// host is responsible for: the files whose winning version this replica wrote, and —
576    /// on the primary device — every other one. It never holds up the caller.
577    pub fn request_embed(self: &Arc<Self>, root: &Path) {
578        let Ok(key) = root.canonicalize() else {
579            return;
580        };
581        let runtime = Arc::clone(self);
582        tokio::spawn(async move {
583            if !runtime.embed_wanted.lock().await.insert(key.clone()) {
584                return;
585            }
586            let _one_at_a_time = runtime.embedding.lock().await;
587            // Taken off the list as the pass starts: a change during the pass asks again.
588            runtime.embed_wanted.lock().await.remove(&key);
589            runtime.embed_running.lock().await.insert(key.clone());
590            runtime.embed_pass(&key).await;
591            runtime.embed_running.lock().await.remove(&key);
592        });
593    }
594
595    /// One embedding pass over `key`. Best-effort, like the re-index: logged, never fatal.
596    async fn embed_pass(&self, key: &Path) {
597        let Some(host) = self.host.get() else {
598            return;
599        };
600        let (workspace_id, replica) = {
601            let synced = self.synced.lock().await;
602            match synced.get(key) {
603                Some(entry) => (entry.workspace_id, Arc::clone(&entry.replica)),
604                None => return,
605            }
606        };
607        let backend = match host.backend_at(key).await {
608            Ok(backend) => backend,
609            Err(err) => {
610                tracing::warn!(root = %key.display(), "embedding pass: {err}");
611                return;
612            }
613        };
614        let state = Arc::clone(backend.state());
615        if let Err(err) = state
616            .refresh_embedder_at(self.bridge_endpoint.clone())
617            .await
618        {
619            tracing::warn!(root = %key.display(), "embedding pass: {err}");
620            return;
621        }
622        if state.embedder().is_none() {
623            return;
624        }
625        let primary = self.is_primary(workspace_id).await;
626        // The replica's view, read once: which content each path has, and which of those
627        // versions this replica wrote.
628        let grace_ms = self.backfill_timing().grace.as_millis() as u64;
629        let now_ms = std::time::SystemTime::now()
630            .duration_since(std::time::UNIX_EPOCH)
631            .map_or(0, |d| d.as_millis() as u64);
632        let (mine, settled, live) = {
633            let replica = replica.lock().await;
634            let me = replica.replica_id();
635            let states = match replica.states() {
636                Ok(states) => states,
637                Err(err) => {
638                    tracing::warn!(root = %key.display(), "embedding pass: {err}");
639                    return;
640                }
641            };
642            let mut mine = HashMap::new();
643            // Other devices' versions old enough for the primary to fill in.
644            let mut settled = HashMap::new();
645            let mut live = std::collections::HashSet::new();
646            for (path, path_state) in states {
647                let winner = path_state.winner();
648                let Some(hash) = winner.content.hash() else {
649                    continue;
650                };
651                let hash = hash.to_hex();
652                if winner.dot.replica == me {
653                    mine.insert(path, hash.clone());
654                } else if now_ms.saturating_sub(winner.hlc.wall_ms) >= grace_ms {
655                    settled.insert(path, hash.clone());
656                }
657                live.insert(hash);
658            }
659            (mine, settled, live)
660        };
661        let root = key.to_owned();
662        let result = tokio::task::spawn_blocking(move || {
663            // The author embeds its own versions at once; the primary device fills in other
664            // devices' versions once they have settled, racing nobody.
665            let policy = |rel: &str, hash: &str| {
666                mine.get(rel).is_some_and(|h| h.as_str() == hash)
667                    || (primary && settled.get(rel).is_some_and(|h| h.as_str() == hash))
668            };
669            let embedded = state.embed_pending_with(&policy, PASS_BATCH)?;
670            let removed = if primary {
671                state.remove_stale_vectors(&live, STALE_VECTOR_AGE)
672            } else {
673                0
674            };
675            Ok::<_, sapphire_workspace::Error>((embedded, removed))
676        })
677        .await;
678        match result {
679            Ok(Ok((embedded, removed))) if embedded > 0 || removed > 0 => tracing::info!(
680                root = %root.display(),
681                embedded,
682                removed,
683                "embedding pass"
684            ),
685            Ok(Ok(_)) => {}
686            Ok(Err(err)) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
687            Err(err) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
688        }
689    }
690
691    /// Open a session with every peer that will take one.
692    pub async fn sync_now(self: &Arc<Self>, root: &Path) -> Result<()> {
693        let Ok(key) = root.canonicalize() else {
694            return Ok(());
695        };
696        let workspace_id = {
697            let synced = self.synced.lock().await;
698            match synced.get(&key) {
699                Some(entry) => entry.workspace_id,
700                None => return Ok(()),
701            }
702        };
703        let peers = self
704            .bridge
705            .peers()
706            .await
707            .map_err(|e| Error::Bridge(e.to_string()))?;
708        *self.last_peers.lock().await = Some(peers.clone());
709        let me = self.device_id().await?;
710
711        // The same direction rule the dial loop follows: a host only dials peers with a
712        // greater device id. Dialling the other way from here would reintroduce the
713        // dial-both-ways deadlock the loop was cured of, just on a different trigger.
714        // Inside a star, a pair with neither hub in it does not link at all.
715        let roles = peers.roles_for(workspace_id).cloned();
716        for peer in peers
717            .peers
718            .iter()
719            .filter(|p| topology::link(me, p.device_id, roles.as_ref()) == topology::Link::Dial)
720            .cloned()
721            .collect::<Vec<_>>()
722        {
723            // A peer that already has a live session is caught up by it: a second exchange
724            // on the same workspace would be redundant, and a second stream between the same
725            // pair is exactly what the tie-break cannot settle.
726            if let Some(live) = self.live_peers(&key).await
727                && live.has(&peer.device_id).await
728            {
729                continue;
730            }
731            // One dial per peer at a time. `sync_now` runs beside the dial loop, and two
732            // overlapping outbound dials to one peer are the one case the tie-break rule
733            // cannot resolve — each end would see its own stream as `outbound`. Whoever
734            // arrives second simply leaves it to the next pass. The table is made here if
735            // the dial loop has not yet: the first dial after enabling must not depend on
736            // a tick that has not fired.
737            let Some((_, table)) = self.live_peers_or_create(&key).await else {
738                continue;
739            };
740            if !table.begin_dial(peer.device_id).await {
741                continue;
742            }
743            // A peer that does not host this workspace refuses, which is normal and cheap.
744            let result = self
745                .sync_one(&key, &peer.device_id, peer.name.clone(), workspace_id)
746                .await;
747            table.end_dial(peer.device_id).await;
748            if let Err(err) = result {
749                tracing::debug!(peer = %peer.name, "no session: {err}");
750            }
751        }
752        Ok(())
753    }
754
755    /// Dial one peer and adopt the stream as a live session.
756    ///
757    /// The one-shot path (`run_session`) would close the stream the moment the exchange
758    /// finished — while the accepting peer keeps its half open and pushes into it, a session
759    /// that dies with nothing reading it. So the dialled side is adopted live too, and both
760    /// ends of a pair agree that a session stays open.
761    async fn sync_one(
762        self: &Arc<Self>,
763        key: &Path,
764        device: &GrainId,
765        name: String,
766        workspace_id: GrainId,
767    ) -> Result<()> {
768        let stream = self.bridge.open_stream(workspace_id, *device).await?;
769        self.adopt_live_session(stream, key, *device, workspace_id)
770            .await
771            .map_err(|err| {
772                tracing::warn!(peer = %name, "a live session could not be started: {err}");
773                err
774            })
775    }
776
777    /// Answer the bridge's announcements until the connection closes.
778    pub async fn run(self: Arc<Self>) -> Result<()> {
779        let mut incoming = self.bridge.incoming();
780        loop {
781            let announcement = match incoming.recv().await {
782                Ok(a) => a,
783                Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
784                    tracing::warn!(missed = n, "fell behind on bridge announcements");
785                    continue;
786                }
787                Err(tokio::sync::broadcast::error::RecvError::Closed) => return Ok(()),
788            };
789
790            let found = {
791                let synced = self.synced.lock().await;
792                synced
793                    .iter()
794                    .find(|(_, s)| s.workspace_id == announcement.workspace_id)
795                    .map(|(root, s)| (root.clone(), Arc::clone(&s.replica)))
796            };
797            // `replica` is still what the accepted session shares, and `root` is where its
798            // files and its index live.
799            let Some((root, replica)) = found else {
800                // The bridge routed to us for a workspace we no longer hold. Not fatal:
801                // ignore it and let the ticket expire.
802                tracing::debug!(
803                    workspace = %announcement.workspace_id,
804                    "an announcement for a workspace this server does not hold"
805                );
806                continue;
807            };
808
809            let bridge = Arc::clone(&self.bridge);
810            let workspace_id = announcement.workspace_id;
811            let peer = announcement.peer_device_id;
812            let driver = Arc::clone(&self);
813            tokio::spawn(async move {
814                let stream = match bridge.accept_stream(announcement.ticket).await {
815                    Ok(stream) => stream,
816                    Err(err) => {
817                        tracing::warn!("could not claim an announced stream: {err}");
818                        return;
819                    }
820                };
821                if driver.should_skip(workspace_id, peer).await {
822                    tracing::debug!(%peer, "dropped an inbound session: both ends are outside this workspace's star");
823                    return;
824                }
825                // Live, not one-shot: the peer that dialled is keeping this session open and
826                // pushing what it commits, so this side has to keep reading. A session that
827                // closed after the exchange would leave the dialler pushing into a stream
828                // nobody reads.
829                let (_, session) = match sapphire_framework_session::open_live_session(
830                    stream,
831                    Arc::clone(&replica),
832                    workspace_id,
833                )
834                .await
835                {
836                    Ok(started) => started,
837                    Err(err) => {
838                        tracing::warn!("an inbound session failed: {err}");
839                        return;
840                    }
841                };
842                let Some((key, table)) = driver.live_peers_or_create(&root).await else {
843                    return;
844                };
845                // The peer dialled: this stream survives only if the *peer* has the lower
846                // device id. A stream that loses is dropped, which closes it, and the
847                // connection ends tidily rather than being left half-read.
848                let Some(updates) = table.insert(peer, session, false).await else {
849                    return;
850                };
851                // This is the receiving side: the files are on disk now and nobody else will
852                // index them.
853                driver.reindex(&key).await;
854                driver.request_embed(&key);
855                driver.spawn_reader(table, updates, key, peer);
856            });
857        }
858    }
859
860    /// Every synced root, for the watcher.
861    pub async fn roots(&self) -> Vec<PathBuf> {
862        self.synced.lock().await.keys().cloned().collect()
863    }
864
865    /// The live session table for `root`, if this runtime syncs it.
866    ///
867    /// `None` is not an error: a dial, a push or a drop for a workspace this server does not
868    /// hold is a no-op, and every caller below treats it that way.
869    async fn live_peers(&self, root: &Path) -> Option<Arc<LivePeers>> {
870        let key = root.canonicalize().ok()?;
871        self.live.lock().await.get(&key).cloned()
872    }
873
874    /// The live session table for `root`, creating it if this runtime syncs it.
875    ///
876    /// The two locks are taken one after the other, never nested: `synced` to decide whether
877    /// the workspace is ours at all, then `live` to get or make its table.
878    async fn live_peers_or_create(&self, root: &Path) -> Option<(PathBuf, Arc<LivePeers>)> {
879        let key = root.canonicalize().ok()?;
880        if !self.synced.lock().await.contains_key(&key) {
881            return None;
882        }
883        // A table is built by whoever walks the workspace first, and it needs this host's
884        // device id — the tie-break between two simultaneous streams is decided from it.
885        let me = self.device_id().await.ok()?;
886        let mut live = self.live.lock().await;
887        let peers = Arc::clone(
888            live.entry(key.clone())
889                .or_insert_with(|| Arc::new(LivePeers::new(me))),
890        );
891        Some((key, peers))
892    }
893
894    /// Push `updates` to every open session for `root`.
895    ///
896    /// Called with what a scan recorded. `from` is always `None` here: a local commit came
897    /// from this host, so no session is excluded. The forwarding path, which does name a
898    /// source, is [`LivePeers::fan_out`](live::LivePeers::fan_out) called directly by the
899    /// reader side of a session.
900    ///
901    /// Best-effort by contract: the commit already happened, and a peer that is gone is the
902    /// dialer's problem, not the writer's.
903    pub async fn after_commit(&self, root: &Path, updates: Vec<PathUpdate>) {
904        if updates.is_empty() {
905            return;
906        }
907        if let Some(peers) = self.live_peers(root).await {
908            peers.fan_out(&updates, None).await;
909        }
910    }
911
912    /// The devices with an open live session for `root`.
913    ///
914    /// Used by tests to wait for the dialer; empty for a workspace this server does not hold.
915    pub async fn live_session_devices(&self, root: &Path) -> Vec<GrainId> {
916        match self.live_peers(root).await {
917            Some(peers) => peers.devices().await,
918            None => Vec::new(),
919        }
920    }
921
922    /// Close every open live session, as if the connections had been cut.
923    ///
924    /// Called for a workspace being disabled, and by tests that want to watch the dialer
925    /// rebuild what was dropped.
926    pub async fn drop_connections(&self) {
927        let tables: Vec<Arc<LivePeers>> = self.live.lock().await.values().cloned().collect();
928        for peers in tables {
929            peers.drop_connections().await;
930        }
931    }
932
933    /// Keep a live session open to every peer, for every synced workspace.
934    ///
935    /// One walk per [`DIAL_INTERVAL`](live::DIAL_INTERVAL): for each synced workspace, ask
936    /// the bridge who is in the workgroup, and open a session to every peer that is connected
937    /// and does not already have one.
938    ///
939    /// A peer that cannot be reached is retried on a per-peer exponential backoff capped at
940    /// [`DIAL_BACKOFF_MAX`] — one failed attempt per walk, not one per spin — and the counter
941    /// resets when a session opens. An unreachable peer is therefore no obstacle to the
942    /// others: its dial fails, the walk moves on, and the next workspace is dialled.
943    ///
944    /// Runs until the task is aborted, which is what the app server's shutdown does with it.
945    /// It is allowed to fail like [`run`](SyncRuntime::run): only sync stops, and the app
946    /// server keeps serving files (spec §10).
947    pub async fn dial_loop(self: Arc<Self>) -> Result<()> {
948        let mut tick = tokio::time::interval(live::DIAL_INTERVAL);
949        // A missed tick is a late walk, not a burst of them: the interval is a pace, not a
950        // quota to make up.
951        tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
952        let mut waiting: HashMap<GrainId, std::time::Instant> = HashMap::new();
953        let mut failures: HashMap<GrainId, u32> = HashMap::new();
954        // The workspaces this host was the primary device of at the last walk.
955        let mut primary_of: std::collections::HashSet<GrainId> = Default::default();
956        // When the primary device last ran a backfill pass with nothing prompting it.
957        let mut last_backfill = std::time::Instant::now();
958        loop {
959            tick.tick().await;
960            let roots = self.roots().await;
961            if roots.is_empty() {
962                continue;
963            }
964            let peers = match self.bridge.peers().await {
965                Ok(peers) => {
966                    *self.last_peers.lock().await = Some(peers.clone());
967                    peers
968                }
969                // The bridge is down or has no workgroup: nothing to dial, and the next walk
970                // asks again. Not an error for the caller — the app server is still serving.
971                Err(err) => {
972                    tracing::debug!("no peers to dial: {err}");
973                    continue;
974                }
975            };
976            let me = match self.device_id().await {
977                Ok(me) => me,
978                Err(err) => {
979                    tracing::warn!("no device id, so nobody to dial: {err}");
980                    continue;
981                }
982            };
983            // A host that has just become a workspace's primary device owes it the vectors
984            // nobody else computes (#187): files that arrived before the election settled
985            // would otherwise wait for the next change.
986            let ids: Vec<(PathBuf, GrainId)> = self
987                .synced
988                .lock()
989                .await
990                .iter()
991                .map(|(root, entry)| (root.clone(), entry.workspace_id))
992                .collect();
993            // And every so often anyway: an author that never embedded leaves files only the
994            // primary will fill in, and nothing else would prompt it (#188).
995            let backfill_due = last_backfill.elapsed() >= self.backfill_timing().interval;
996            if backfill_due {
997                last_backfill = std::time::Instant::now();
998            }
999            for (root, workspace_id) in ids {
1000                let primary = peers
1001                    .roles_for(workspace_id)
1002                    .is_some_and(|r| r.primary == Some(me));
1003                if !primary {
1004                    primary_of.remove(&workspace_id);
1005                } else if primary_of.insert(workspace_id) || backfill_due {
1006                    self.request_embed(&root);
1007                }
1008            }
1009            let now = std::time::Instant::now();
1010            // Dial only peers whose device id is greater than ours. Every device dialling
1011            // every other can deadlock: two hosts that dial each other at once each hold
1012            // their replica's lock across their own exchange and wait for the other's
1013            // hello, which the other's dialler holds the lock for. With one direction per
1014            // pair the wait chain is a DAG — a higher-id host never dials, so it never
1015            // holds its lock out while a lower-id host's exchange needs it. The lower host
1016            // dials within `DIAL_INTERVAL`, and the exchange is symmetric, so nothing is
1017            // lost; the tie-break is left as the safety net it is.
1018            for root in roots {
1019                let Some((key, table)) = self.live_peers_or_create(&root).await else {
1020                    continue;
1021                };
1022                let workspace_id = {
1023                    let synced = self.synced.lock().await;
1024                    match synced.get(&key) {
1025                        Some(entry) => entry.workspace_id,
1026                        None => continue,
1027                    }
1028                };
1029                let roles = peers.roles_for(workspace_id);
1030                // Entering a star closes what it no longer needs. The hubs carry it all —
1031                // once one of them is reachable from here (#192).
1032                if topology::may_close_skipped(me, roles, &table.devices().await) {
1033                    table
1034                        .retain(|d| topology::link(me, *d, roles) != topology::Link::Skip)
1035                        .await;
1036                }
1037                for peer in peers.peers.iter().filter(|p| {
1038                    p.connected && topology::link(me, p.device_id, roles) == topology::Link::Dial
1039                }) {
1040                    let device = peer.device_id;
1041                    if table.has(&device).await {
1042                        // Already talking; a redial would drop a working session.
1043                        failures.remove(&device);
1044                        waiting.remove(&device);
1045                        continue;
1046                    }
1047                    if let Some(until) = waiting.get(&device)
1048                        && now < *until
1049                    {
1050                        // Backing off after a failure; the walk skips it until then.
1051                        continue;
1052                    }
1053                    // The same claim `sync_now` takes: this walk and it can overlap, and a
1054                    // double dial in one direction is the one race the tie-break cannot
1055                    // settle. Skipping is free — the next pass, or `sync_now` itself, dials.
1056                    if !table.begin_dial(device).await {
1057                        continue;
1058                    }
1059                    let opened = self.bridge.open_stream(workspace_id, device).await;
1060                    if opened.is_err() {
1061                        table.end_dial(device).await;
1062                    }
1063                    match opened {
1064                        Ok(stream) => {
1065                            let adopted = self
1066                                .adopt_live_session(stream, &root, device, workspace_id)
1067                                .await;
1068                            // The claim is out either way: the attempt is over, and a
1069                            // session that died mid-exchange is re-dialled next pass.
1070                            table.end_dial(device).await;
1071                            match adopted {
1072                                Ok(()) => {
1073                                    failures.remove(&device);
1074                                    waiting.remove(&device);
1075                                }
1076                                Err(err) => {
1077                                    tracing::debug!(
1078                                        peer = %peer.name,
1079                                        err = %err,
1080                                        "a live session could not be started"
1081                                    );
1082                                    let count = failures.entry(device).or_insert(0);
1083                                    *count = count.saturating_add(1);
1084                                    waiting.insert(device, now + dial_backoff(*count));
1085                                }
1086                            }
1087                        }
1088                        Err(err) => {
1089                            // Normal and cheap: a peer that does not host this workspace
1090                            // refuses, and one that is unreachable cannot be opened.
1091                            tracing::debug!("no stream to a peer: {err}");
1092                            let count = failures.entry(device).or_insert(0);
1093                            *count = count.saturating_add(1);
1094                            waiting.insert(device, now + dial_backoff(*count));
1095                        }
1096                    }
1097                }
1098            }
1099        }
1100    }
1101
1102    /// Take ownership of an opened stream: run the exchange, then keep it live.
1103    ///
1104    /// The session is stored under `device` and its reader pumped until the stream ends. A
1105    /// half-open or refused stream is not stored; the dialer retries it with backoff.
1106    async fn adopt_live_session(
1107        self: &Arc<Self>,
1108        stream: sapphire_ipc::RawStream,
1109        root: &Path,
1110        device: GrainId,
1111        workspace_id: GrainId,
1112    ) -> Result<()> {
1113        let replica = self.replica_of(root).await?;
1114        let (_, session) = sapphire_framework_session::open_live_session(
1115            stream,
1116            Arc::clone(&replica),
1117            workspace_id,
1118        )
1119        .await
1120        .map_err(|e| Error::Sync(e.to_string()))?;
1121        let Some((key, table)) = self.live_peers_or_create(root).await else {
1122            return Ok(());
1123        };
1124        // This stream was dialled by us, so it is the one the pair keeps if this host's
1125        // device id is the lower of the two. If it lost the tie-break the session is
1126        // dropped here, which closes the stream, and the surviving one is already in the
1127        // table.
1128        let Some(updates) = table.insert(device, session, true).await else {
1129            return Ok(());
1130        };
1131        // The session wrote files during the exchange; the index has to catch up before the
1132        // workspace is searchable again.
1133        self.reindex(&key).await;
1134        self.request_embed(&key);
1135        self.spawn_reader(Arc::clone(&table), updates, key, device);
1136        Ok(())
1137    }
1138
1139    /// Forward what one session receives, and re-index it.
1140    ///
1141    /// This is the loop that makes A → S → B work: a batch that arrived on S's session with A
1142    /// is applied (the session crate does that before publishing it), the workspace behind
1143    /// `key` is re-indexed, and the batch is handed to every *other* session whose peer lacks
1144    /// it — `from` is the device it came from, so it never goes back to A.
1145    fn spawn_reader(
1146        self: &Arc<Self>,
1147        table: Arc<LivePeers>,
1148        mut updates: broadcast::Receiver<Vec<PathUpdate>>,
1149        key: PathBuf,
1150        from: GrainId,
1151    ) {
1152        let driver = Arc::clone(self);
1153        tokio::spawn(async move {
1154            loop {
1155                match updates.recv().await {
1156                    Ok(batch) => {
1157                        // The files are on disk now (the session guarantees it before
1158                        // publishing), so the index is behind them until the sweep below.
1159                        driver.reindex(&key).await;
1160                        driver.request_embed(&key);
1161                        table.fan_out(&batch, Some(from)).await;
1162                    }
1163                    // A burst bigger than the channel: the newest is kept and the gap is
1164                    // covered by the next session's exchange. Losing a batch here cannot lose
1165                    // an entry — the peer's vector does not cover it, so the next dial or push
1166                    // carries it again.
1167                    Err(broadcast::error::RecvError::Lagged(missed)) => {
1168                        tracing::debug!(
1169                            device = %from,
1170                            missed,
1171                            "fell behind on a live session; the next exchange catches up"
1172                        );
1173                    }
1174                    // The session closed. Its table entry is reaped by the next fan-out or
1175                    // dial pass, and the dialer opens a fresh one.
1176                    Err(broadcast::error::RecvError::Closed) => {
1177                        return;
1178                    }
1179                }
1180            }
1181        });
1182    }
1183
1184    /// The replica of a synced `root`.
1185    async fn replica_of(&self, root: &Path) -> Result<Arc<Mutex<Replica>>> {
1186        let key = root.canonicalize().map_err(Error::Io)?;
1187        let synced = self.synced.lock().await;
1188        match synced.get(&key) {
1189            Some(entry) => Ok(Arc::clone(&entry.replica)),
1190            None => Err(Error::UnknownWorkspace(key, self.ctx.app_name)),
1191        }
1192    }
1193
1194    /// Watch the synced roots and, on a debounced report, scan and dial.
1195    ///
1196    /// This is the safety net for everything the app server did not do itself — a file
1197    /// edited in an editor, a `git checkout`. The exact path is [`scan`] called from
1198    /// `handlers.rs` right after a write the server made. A scan that finds nothing is
1199    /// cheap; a scan that is skipped loses an edit until the next one.
1200    ///
1201    /// Returns when the reporting channel closes — that is, when the [`Watcher`] this
1202    /// runtime holds is dropped, which happens when the runtime itself is.
1203    ///
1204    /// [`scan`]: SyncRuntime::scan
1205    pub async fn watch_changes(self: Arc<Self>) -> Result<()> {
1206        let (tx, mut rx) = mpsc::channel(64);
1207        let watcher = Arc::new(Watcher::start(self.roots().await, tx)?);
1208        let _ = self.watcher.set(Arc::clone(&watcher));
1209
1210        // A root enabled between `roots()` above and the `set` just now was not in the
1211        // starting set, and `enable` could not reach a watcher that did not exist yet.
1212        // Watching the current set again is idempotent on `notify`'s side.
1213        for root in self.roots().await {
1214            if let Err(err) = watcher.watch(&root) {
1215                tracing::warn!(root = %root.display(), "could not watch: {err}");
1216            }
1217        }
1218
1219        while let Some(root) = rx.recv().await {
1220            // Logged, not fatal: a failed scan or dial leaves the replica behind, and the
1221            // next report tries again. Neither may take the app server down (spec §10).
1222            if let Err(err) = self.scan(&root).await {
1223                tracing::warn!(root = %root.display(), "scan after a local edit failed: {err}");
1224            }
1225            self.request_embed(&root);
1226            if let Err(err) = self.sync_now(&root).await {
1227                tracing::warn!(root = %root.display(), "dial after a local edit failed: {err}");
1228            }
1229        }
1230        Ok(())
1231    }
1232
1233    /// Tell the bridge the complete current set. A registration is not a delta.
1234    async fn reregister(&self) -> Result<()> {
1235        let workspaces: Vec<WorkspaceRegistration> = {
1236            let synced = self.synced.lock().await;
1237            synced
1238                .iter()
1239                .map(|(root, entry)| WorkspaceRegistration {
1240                    workspace_id: entry.workspace_id,
1241                    root: root.clone(),
1242                })
1243                .collect()
1244        };
1245        self.bridge
1246            .register(RegisterParams {
1247                app_name: self.ctx.app_name.to_owned(),
1248                exe_path: self.exe_path.clone(),
1249                managed_by: wire_managed_by(self.managed_by),
1250                workspaces,
1251            })
1252            .await
1253            .map_err(|e| Error::Bridge(e.to_string()))?;
1254        Ok(())
1255    }
1256
1257    /// Whether the last roles the bridge reported say this pair does not link.
1258    ///
1259    /// Reads the cached device id only: asking the bridge here would register with an empty
1260    /// workspace list, racing `enable`'s registration. `false` (accept) when it is unknown.
1261    async fn should_skip(&self, workspace_id: GrainId, peer: GrainId) -> bool {
1262        let Some(&me) = self.device_id.get() else {
1263            return false;
1264        };
1265        let cached = self.last_peers.lock().await;
1266        let roles = cached.as_ref().and_then(|p| p.roles_for(workspace_id));
1267        topology::link(me, peer, roles) == topology::Link::Skip
1268    }
1269
1270    /// Whether this host is `workspace_id`'s primary device, by the last roles the
1271    /// bridge reported. `false` when the bridge has not answered yet, or when this host's
1272    /// device id is not known yet (no workspace was enabled). Reads the cached id only:
1273    /// asking the bridge here would register with an empty workspace list, racing
1274    /// `enable`'s registration.
1275    pub async fn is_primary(&self, workspace_id: GrainId) -> bool {
1276        let Some(&me) = self.device_id.get() else {
1277            return false;
1278        };
1279        let cached = self.last_peers.lock().await;
1280        cached
1281            .as_ref()
1282            .and_then(|p| p.roles_for(workspace_id))
1283            .is_some_and(|r| r.primary == Some(me))
1284    }
1285
1286    /// This host's device id, asked of the bridge once.
1287    async fn device_id(&self) -> Result<GrainId> {
1288        if let Some(id) = self.device_id.get() {
1289            return Ok(*id);
1290        }
1291        let result = self
1292            .bridge
1293            .register(RegisterParams {
1294                app_name: self.ctx.app_name.to_owned(),
1295                exe_path: self.exe_path.clone(),
1296                managed_by: wire_managed_by(self.managed_by),
1297                workspaces: Vec::new(),
1298            })
1299            .await
1300            .map_err(|e| Error::Bridge(e.to_string()))?;
1301        let _ = self.device_id.set(result.device_id);
1302        Ok(result.device_id)
1303    }
1304
1305    /// `<workspace cache dir>/sync/`.
1306    fn state_dir(&self, root: &Path) -> Result<PathBuf> {
1307        let workspace = Workspace::from_root(self.ctx, root)?;
1308        Ok(workspace.cache_dir().join("sync"))
1309    }
1310}
1311
1312/// The bridge API's copy of how this server was started, for [`RegisterParams`].
1313fn wire_managed_by(managed_by: ManagedBy) -> sapphire_bridge_api::ManagedBy {
1314    match managed_by {
1315        ManagedBy::Service => sapphire_bridge_api::ManagedBy::Service,
1316        ManagedBy::Spawned => sapphire_bridge_api::ManagedBy::Spawned,
1317    }
1318}
1319
1320#[cfg(test)]
1321mod tests {
1322    use super::*;
1323    use crate::sync::testing::StubBridge;
1324    use crate::test_support;
1325    use sapphire_workspace::AppKind;
1326    use std::ffi::OsString;
1327
1328    static CTX: AppContext = AppContext::new("sapphire-synctest");
1329
1330    /// The env vars `CTX.init` reads.
1331    const DIR_VARS: [&str; 3] = [
1332        "SAPPHIRE_SYNCTEST_CACHE_DIR",
1333        "SAPPHIRE_SYNCTEST_DATA_DIR",
1334        "SAPPHIRE_SYNCTEST_CONFIG_DIR",
1335    ];
1336
1337    /// Points the context's directories at the test's scratch tree, and restores the
1338    /// previous values when dropped — including while unwinding from a panic.
1339    struct EnvGuard {
1340        previous: [Option<OsString>; 3],
1341        _lock: std::sync::MutexGuard<'static, ()>,
1342    }
1343
1344    impl Drop for EnvGuard {
1345        fn drop(&mut self) {
1346            // SAFETY: `self._lock` still serialises the environment; it is dropped only
1347            // after this method returns.
1348            for (name, previous) in DIR_VARS.iter().zip(self.previous.iter_mut()) {
1349                match previous.take() {
1350                    Some(value) => unsafe { std::env::set_var(name, value) },
1351                    None => test_support::remove(name),
1352                }
1353            }
1354        }
1355    }
1356
1357    /// Everything one test needs, held for the test's whole body.
1358    ///
1359    /// The context is a `static` shared by this binary, so its directories are resolved from
1360    /// the environment by whichever test initialises it first. The environment lock is
1361    /// therefore held for the whole test, as in the crate's other test modules, and `_tmp`
1362    /// is declared before `_env` so the scratch tree is gone only once the environment no
1363    /// longer points at it.
1364    struct Fixture {
1365        _tmp: tempfile::TempDir,
1366        _env: EnvGuard,
1367        root: PathBuf,
1368        stub: StubBridge,
1369        runtime: Arc<SyncRuntime>,
1370    }
1371
1372    impl Fixture {
1373        /// A second workspace root beside the first, marker directory and all.
1374        fn second_root(&self) -> PathBuf {
1375            let root = self._tmp.path().join("ws2");
1376            std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
1377            root.canonicalize().unwrap()
1378        }
1379    }
1380
1381    async fn fixture() -> Fixture {
1382        let lock = test_support::lock();
1383        let tmp = tempfile::tempdir().unwrap();
1384        let previous = DIR_VARS.map(std::env::var_os);
1385        // SAFETY (via `test_support::set`): `lock` serialises every read and write of the
1386        // process environment in this test binary, and it is held until `drop` has restored
1387        // the old values.
1388        for (name, dir) in DIR_VARS
1389            .iter()
1390            .zip(["cache", "data", "config"].map(|cat| tmp.path().join(cat)))
1391        {
1392            test_support::set(name, &dir);
1393        }
1394        let env = EnvGuard {
1395            previous,
1396            _lock: lock,
1397        };
1398        CTX.init(AppKind::Server);
1399
1400        let root = tmp.path().join("ws");
1401        std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
1402        let root = root.canonicalize().unwrap();
1403
1404        let (stub, client) = StubBridge::start().await;
1405        let runtime = Arc::new(SyncRuntime::new(
1406            &CTX,
1407            client,
1408            "/bin/true".into(),
1409            ManagedBy::Service,
1410        ));
1411        Fixture {
1412            _tmp: tmp,
1413            _env: env,
1414            root,
1415            stub,
1416            runtime,
1417        }
1418    }
1419
1420    #[tokio::test(flavor = "multi_thread")]
1421    async fn enabling_registers_the_workspace_with_the_bridge() {
1422        let f = fixture().await;
1423
1424        let id = f.runtime.enable(&f.root).await.unwrap();
1425        assert_eq!(f.stub.last_workspaces(), vec![id]);
1426    }
1427
1428    #[tokio::test(flavor = "multi_thread")]
1429    async fn enabling_twice_is_idempotent_and_keeps_the_same_id() {
1430        let f = fixture().await;
1431
1432        let first = f.runtime.enable(&f.root).await.unwrap();
1433        let second = f.runtime.enable(&f.root).await.unwrap();
1434        assert_eq!(first, second);
1435        assert_eq!(
1436            f.stub.last_workspaces(),
1437            vec![first],
1438            "still exactly one workspace"
1439        );
1440    }
1441
1442    #[tokio::test(flavor = "multi_thread")]
1443    async fn a_registration_carries_the_complete_current_list() {
1444        let f = fixture().await;
1445        let second_root = f.second_root();
1446
1447        let a = f.runtime.enable(&f.root).await.unwrap();
1448        let b = f.runtime.enable(&second_root).await.unwrap();
1449
1450        let mut seen = f.stub.last_workspaces();
1451        seen.sort();
1452        let mut want = vec![a, b];
1453        want.sort();
1454        assert_eq!(seen, want, "registration is the whole set, not a delta");
1455    }
1456
1457    #[tokio::test(flavor = "multi_thread")]
1458    async fn no_later_registration_clears_this_apps_routes() {
1459        // The device id is asked of the bridge once, on the first `enable`, with an empty
1460        // workspace list — the one moment that is safe, because no route exists yet. Every
1461        // later registration must name the complete set: one with an empty list would
1462        // briefly clear this application's routes, and a peer asking for a workspace this
1463        // server owns would be told nobody has it.
1464        let f = fixture().await;
1465        let second_root = f.second_root();
1466
1467        f.runtime.enable(&f.root).await.unwrap();
1468        f.runtime.enable(&second_root).await.unwrap();
1469        f.runtime.sync_now(&f.root).await.unwrap();
1470        f.runtime.disable(&f.root).await.unwrap();
1471
1472        let seen = f.stub.seen.lock().expect("stub");
1473        let first_full = seen
1474            .registrations
1475            .iter()
1476            .position(|r| !r.workspaces.is_empty())
1477            .expect("enabling registers the workspace");
1478        let emptied_afterwards = seen.registrations[first_full + 1..]
1479            .iter()
1480            .filter(|r| r.workspaces.is_empty())
1481            .count();
1482        assert_eq!(
1483            emptied_afterwards, 0,
1484            "a registration after the first workspace must still name the whole set"
1485        );
1486    }
1487
1488    #[tokio::test(flavor = "multi_thread")]
1489    async fn disabling_unregisters_and_leaves_the_files_alone() {
1490        let f = fixture().await;
1491        std::fs::write(f.root.join("keep.md"), "content").unwrap();
1492
1493        let id = f.runtime.enable(&f.root).await.unwrap();
1494        f.runtime.disable(&f.root).await.unwrap();
1495
1496        assert!(f.stub.last_workspaces().is_empty());
1497        assert!(
1498            f.root.join("keep.md").exists(),
1499            "disabling sync must not touch files"
1500        );
1501        assert!(
1502            crate::sync::id::sync_id_path("sapphire-synctest", &f.root).exists(),
1503            "the sync id stays, so re-enabling rejoins the same workspace"
1504        );
1505        let _ = id;
1506    }
1507
1508    #[tokio::test(flavor = "multi_thread")]
1509    async fn re_enabling_after_disabling_reuses_the_identity() {
1510        let f = fixture().await;
1511
1512        let first = f.runtime.enable(&f.root).await.unwrap();
1513        f.runtime.disable(&f.root).await.unwrap();
1514        let again = f.runtime.enable(&f.root).await.unwrap();
1515
1516        assert_eq!(first, again);
1517    }
1518
1519    #[tokio::test(flavor = "multi_thread")]
1520    async fn status_reports_disabled_for_a_workspace_that_was_never_enabled() {
1521        let f = fixture().await;
1522
1523        let status = f.runtime.status(&f.root).await;
1524        assert!(!status.enabled);
1525        assert!(status.workspace_id.is_none());
1526    }
1527
1528    #[tokio::test(flavor = "multi_thread")]
1529    async fn disabling_something_that_was_never_enabled_is_not_an_error() {
1530        let f = fixture().await;
1531        f.runtime
1532            .disable(&f.root)
1533            .await
1534            .expect("disabling twice is fine");
1535    }
1536
1537    #[tokio::test(flavor = "multi_thread")]
1538    async fn a_workspace_whose_root_vanished_is_reported_as_paused_not_deleted() {
1539        let f = fixture().await;
1540        std::fs::write(f.root.join("a.md"), "content").unwrap();
1541        f.runtime.enable(&f.root).await.unwrap();
1542        f.runtime.scan(&f.root).await.unwrap();
1543
1544        // The drive is unmounted: the marker directory goes with it.
1545        std::fs::remove_dir_all(f.root.join(".sapphire-synctest")).unwrap();
1546        let _ = f.runtime.scan(&f.root).await;
1547
1548        let status = f.runtime.status(&f.root).await;
1549        assert!(
1550            status.paused.is_some(),
1551            "a missing root must pause, not replicate as a mass deletion"
1552        );
1553    }
1554
1555    #[tokio::test(flavor = "multi_thread")]
1556    async fn a_directory_that_is_not_a_workspace_cannot_be_enabled() {
1557        let f = fixture().await;
1558        let plain = f._tmp.path().join("plain");
1559        std::fs::create_dir_all(&plain).unwrap();
1560
1561        assert!(f.runtime.enable(&plain).await.is_err());
1562    }
1563
1564    #[tokio::test(flavor = "multi_thread")]
1565    async fn a_write_through_the_workspace_router_is_committed_to_the_replica() {
1566        // The brief's "main path": a write the app server itself made is scanned straight
1567        // after, not left to the watcher's debounce. Asserted against the replica's own
1568        // state, which is the only thing that can tell the two paths apart — reading the
1569        // file back would pass even if the scan never ran.
1570        let f = fixture().await;
1571        f.runtime.enable(&f.root).await.unwrap();
1572
1573        let host = Arc::new(crate::WorkspaceHost::new(&CTX));
1574        host.select(&f.root, false).await.unwrap();
1575        let current = Arc::new(crate::Current::new(
1576            &CTX,
1577            host,
1578            Some(Arc::clone(&f.runtime)),
1579            crate::selection::SelectionFile::at(f._tmp.path().join("workspace.toml")),
1580        ));
1581        let router = Arc::new(crate::workspace_router(current));
1582        let (client_conn, server_conn) = sapphire_ipc::Connection::pair();
1583        tokio::spawn(async move {
1584            let info = sapphire_ipc::ServerInfo {
1585                version: "0.0.0".into(),
1586                api: sapphire_backend::protocol::API_VERSION,
1587                app_api: None,
1588                pid: std::process::id(),
1589                managed_by: ManagedBy::Spawned,
1590            };
1591            let _ = sapphire_ipc::serve(server_conn, router, "sapphire-synctest", info).await;
1592        });
1593        let info = sapphire_ipc::ClientInfo {
1594            kind: "test".into(),
1595            version: "0.0.0".into(),
1596            api: sapphire_backend::protocol::API_VERSION,
1597            app_api: None,
1598            pid: std::process::id(),
1599        };
1600        let (client, _) = sapphire_ipc::Client::handshake(client_conn, "sapphire-synctest", info)
1601            .await
1602            .unwrap();
1603
1604        let _: sapphire_backend::protocol::Ack = client
1605            .call(
1606                sapphire_backend::protocol::WRITE_FILE,
1607                sapphire_backend::protocol::ContentParams {
1608                    path: PathBuf::from("through-the-server.md"),
1609                    content: "written by the server".into(),
1610                },
1611            )
1612            .await
1613            .unwrap();
1614
1615        // `scan` runs before the reply is written, so by the time the call returns the
1616        // replica must already know the path.
1617        let synced = f.runtime.synced.lock().await;
1618        let entry = synced.get(&f.root).expect("enabled above");
1619        let replica = entry.replica.lock().await;
1620        let state = replica
1621            .state("through-the-server.md")
1622            .expect("a readable state");
1623        assert!(
1624            state.is_some(),
1625            "a write through the server must be scanned into the replica, not left to the watcher"
1626        );
1627    }
1628
1629    #[tokio::test(flavor = "multi_thread")]
1630    async fn an_announcement_for_an_unknown_workspace_is_ignored_not_fatal() {
1631        let f = fixture().await;
1632        f.runtime.enable(&f.root).await.unwrap();
1633
1634        let driver = Arc::clone(&f.runtime);
1635        let handle = tokio::spawn(async move { driver.run().await });
1636
1637        f.stub
1638            .announce(grain_id::GrainId::random(), "no-such-ticket")
1639            .await;
1640        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
1641
1642        assert!(
1643            !handle.is_finished(),
1644            "one bad announcement must not stop the runtime"
1645        );
1646        handle.abort();
1647    }
1648}