sapphire_framework_server/sync/mod.rs
1//! Replication, from the app server's side.
2//!
3//! One `Replica` per synced workspace, registered with the bridge, driven by announcements
4//! from it and by local edits.
5//!
6//! See `docs/superpowers/specs/2026-09-16-process-architecture-design.md` §4.2.
7
8use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use grain_id::GrainId;
13use sapphire_backend::protocol as proto;
14use sapphire_bridge_api::{PeersResult, RegisterParams, WorkspaceRegistration, WorkspacesResult};
15use sapphire_bridge_client::BridgeClient;
16use sapphire_ipc::ManagedBy;
17use sapphire_sync::{PathUpdate, PauseReason, Replica, ReplicaConfig, ScanOutcome, SystemClock};
18use sapphire_workspace::{AppContext, Workspace};
19use tokio::sync::{Mutex, OnceCell, broadcast, mpsc};
20
21pub mod id;
22mod live;
23mod methods;
24#[cfg(any(test, feature = "test-util"))]
25pub mod testing;
26mod topology;
27mod watch;
28
29pub use id::{SYNC_ID_FILE, WORKSPACE_MAP_FILE, sync_id, sync_id_path};
30pub use live::DIAL_BACKOFF_MAX;
31pub use methods::sync_router;
32pub use watch::{DEBOUNCE, Watcher};
33
34use live::{LivePeers, dial_backoff};
35
36use crate::error::{Error, Result};
37
38use crate::host::WorkspaceHost;
39
40/// What `sync.status` reports.
41#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
42pub struct SyncStatus {
43 /// Whether this workspace is synced at all.
44 pub enabled: bool,
45 /// Its identity across devices, when it is.
46 pub workspace_id: Option<GrainId>,
47 /// How many devices the workgroup has, this host excluded.
48 pub peers: usize,
49 /// Why replication is paused, if it is — a missing root or marker directory.
50 pub paused: Option<String>,
51 /// The last failure, if any.
52 pub last_error: Option<String>,
53 /// Whether the bridge is reachable. `false` is not an outage: the app server works.
54 pub bridge_available: bool,
55 /// How the workspace is wired to its peers, by the roles the bridge last reported.
56 pub topology: proto::Topology,
57 /// How far embedding has got, when this host embeds (#188).
58 pub embedding: Option<proto::EmbeddingProgress>,
59}
60
61/// One workspace this runtime syncs.
62struct Synced {
63 /// Its identity across devices, shared with every peer.
64 workspace_id: GrainId,
65 /// Its replica store. Held open because the store is a redb database.
66 replica: Arc<Mutex<Replica>>,
67 /// Why replication is paused, from the last scan.
68 paused: Option<PauseReason>,
69 /// The last failure, if any.
70 last_error: Option<String>,
71}
72
73/// How old a vector file no live content needs must be before the primary device removes
74/// it: a vector can arrive before its file, and a day is far longer than that window.
75const STALE_VECTOR_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
76
77/// Texts per embedding request in a server pass: small, so a search query waiting on the
78/// bridge's one worker is never stuck behind a long backfill (#188).
79const PASS_BATCH: usize = 4;
80
81/// How the primary device paces its backfill (#188).
82#[derive(Clone, Copy, Debug)]
83pub struct BackfillTiming {
84 /// How old another device's version must be before the primary embeds it: its author
85 /// has embedded it by then, or will not.
86 pub grace: std::time::Duration,
87 /// How often the primary device runs a pass with nothing prompting it.
88 pub interval: std::time::Duration,
89}
90
91impl Default for BackfillTiming {
92 fn default() -> Self {
93 BackfillTiming {
94 grace: std::time::Duration::from_secs(10 * 60),
95 interval: std::time::Duration::from_secs(10 * 60),
96 }
97 }
98}
99
100/// Replication for one application's workspaces.
101pub struct SyncRuntime {
102 ctx: &'static AppContext,
103 bridge: Arc<BridgeClient>,
104 exe_path: PathBuf,
105 managed_by: ManagedBy,
106 synced: Mutex<HashMap<PathBuf, Synced>>,
107 /// This host's device id. Asked of the bridge once, on the first [`enable`], and never
108 /// again: a second registration with an empty workspace list would briefly clear this
109 /// application's routes.
110 ///
111 /// [`enable`]: SyncRuntime::enable
112 device_id: OnceCell<GrainId>,
113 /// The file watcher, once [`watch_changes`] has started it.
114 ///
115 /// The runtime owns it because the spec puts it here: the sync runtime is a watcher and
116 /// a replica per registered root, and the two must agree on which roots are synced.
117 /// [`enable`] and [`disable`] keep them in step.
118 ///
119 /// [`watch_changes`]: SyncRuntime::watch_changes
120 /// [`enable`]: SyncRuntime::enable
121 /// [`disable`]: SyncRuntime::disable
122 watcher: OnceCell<Arc<Watcher>>,
123 /// The app server that owns these workspaces' caches, once it has told us.
124 ///
125 /// A replication session writes files, and the search index is not part of a session: the
126 /// index belongs to the app server, which holds the workspace open. Reaching it through
127 /// the host is what keeps a received file and its index moving together, and it is the
128 /// *same* [`WorkspaceState`](sapphire_workspace::WorkspaceState) the server serves from —
129 /// opening a second one would collide on the retrieve store's exclusive lock.
130 ///
131 /// Absent for a runtime nobody has wired to an app server (a test stub driving the
132 /// replica directly). Nothing then indexes, which is correct: nothing owns an index.
133 host: OnceCell<Arc<WorkspaceHost>>,
134 /// Serialises re-indexing, so two sessions finishing at once do not both sweep the
135 /// workspace through the same store.
136 reindexing: Mutex<()>,
137 /// Roots with an embedding pass waiting to run (#187). A request for a root already
138 /// waiting is folded into that pass.
139 embed_wanted: Mutex<std::collections::HashSet<PathBuf>>,
140 /// Serialises embedding passes: one model, one queue.
141 embedding: Mutex<()>,
142 /// Roots whose embedding pass is running now, for `sync.status`.
143 embed_running: Mutex<std::collections::HashSet<PathBuf>>,
144 /// The primary device's backfill pacing (#188).
145 backfill: std::sync::Mutex<BackfillTiming>,
146 /// The bridge an embedding pass asks for its model; `None` is the host's standard one.
147 bridge_endpoint: Option<sapphire_ipc::Endpoint>,
148 /// The live session table of every synced workspace, keyed by canonical root.
149 ///
150 /// One table per workspace rather than one for the runtime: a session is about one
151 /// workspace, and the two storm rules are per-workspace rules. A root that is not
152 /// synced has none, so a dial or a push for it is a no-op rather than an error.
153 live: Mutex<HashMap<PathBuf, Arc<LivePeers>>>,
154 /// The bridge's last answer to `peers`, roles included. The inbound guard and
155 /// [`is_primary`](SyncRuntime::is_primary) read it rather than asking again.
156 last_peers: Mutex<Option<PeersResult>>,
157}
158
159impl SyncRuntime {
160 /// A runtime for `ctx`'s application.
161 pub fn new(
162 ctx: &'static AppContext,
163 bridge: Arc<BridgeClient>,
164 exe_path: PathBuf,
165 managed_by: ManagedBy,
166 ) -> SyncRuntime {
167 SyncRuntime {
168 ctx,
169 bridge,
170 exe_path,
171 managed_by,
172 synced: Mutex::new(HashMap::new()),
173 device_id: OnceCell::new(),
174 watcher: OnceCell::new(),
175 host: OnceCell::new(),
176 reindexing: Mutex::new(()),
177 embed_wanted: Mutex::new(std::collections::HashSet::new()),
178 embedding: Mutex::new(()),
179 embed_running: Mutex::new(std::collections::HashSet::new()),
180 backfill: std::sync::Mutex::new(BackfillTiming::default()),
181 bridge_endpoint: None,
182 live: Mutex::new(HashMap::new()),
183 last_peers: Mutex::new(None),
184 }
185 }
186
187 /// Tell the runtime which app server owns these workspaces' caches.
188 ///
189 /// Called by [`AppServer::sync`](crate::AppServer::sync). First writer wins: a runtime
190 /// belongs to the server that wired it.
191 /// Embed through the bridge at `endpoint` rather than the host's standard one: for a
192 /// fixture running several hosts in one process.
193 pub fn with_bridge_endpoint(mut self, endpoint: sapphire_ipc::Endpoint) -> SyncRuntime {
194 self.bridge_endpoint = Some(endpoint);
195 self
196 }
197
198 /// Pace the primary device's backfill differently: for tests, which cannot wait ten
199 /// minutes.
200 pub fn set_backfill_timing(&self, timing: BackfillTiming) {
201 *self.backfill.lock().expect("backfill timing") = timing;
202 }
203
204 fn backfill_timing(&self) -> BackfillTiming {
205 *self.backfill.lock().expect("backfill timing")
206 }
207
208 pub(crate) fn set_host(&self, host: Arc<WorkspaceHost>) {
209 let _ = self.host.set(host);
210 }
211
212 /// Start syncing `root`, returning its identity across devices.
213 ///
214 /// Idempotent: enabling an already-synced workspace returns the same id and changes
215 /// nothing.
216 pub async fn enable(self: &Arc<Self>, root: &Path) -> Result<GrainId> {
217 let key = root.canonicalize().map_err(Error::Io)?;
218 let watch_key = key.clone();
219 let workspace_id = sync_id(self.ctx.app_name, &key)?;
220
221 // The map stays locked across opening the replica, so two simultaneous `enable`s for
222 // one root cannot both open it — the store is a redb database, and it is opened once.
223 {
224 let mut synced = self.synced.lock().await;
225 if let Some(existing) = synced.get(&key) {
226 return Ok(existing.workspace_id);
227 }
228 let device_id = self.device_id().await?;
229 let state_dir = self.state_dir(&key)?;
230 std::fs::create_dir_all(&state_dir).map_err(Error::Io)?;
231 let config = ReplicaConfig::new(self.ctx.app_name, key.clone(), device_id, &state_dir);
232 let replica = Replica::open(config, Arc::new(SystemClock))
233 .map_err(|e| Error::Sync(e.to_string()))?;
234 synced.insert(
235 key.clone(),
236 Synced {
237 workspace_id,
238 replica: Arc::new(Mutex::new(replica)),
239 paused: None,
240 last_error: None,
241 },
242 );
243 }
244
245 // Watch the new root too, so an edit made outside the server from now on is seen.
246 // Not fatal if it fails: the app server's own writes take the exact path in
247 // `handlers.rs`, and the next `watch_changes` re-reads the root set anyway.
248 if let Some(watcher) = self.watcher.get()
249 && let Err(err) = watcher.watch(&watch_key)
250 {
251 tracing::warn!(root = %watch_key.display(), "could not watch: {err}");
252 }
253
254 // Register only once the replica is open, so a registration never names a workspace
255 // that cannot serve a session.
256 self.reregister().await?;
257
258 // Dial now, so enabling a workspace converges it rather than waiting for the next
259 // local edit to trigger a session. This is what catches a host up when it returns:
260 // the peer that has been running is the one the returning host has to ask, and no
261 // watcher on either side fires for a file that was never written locally.
262 //
263 // Best-effort: a workspace is enabled whether or not any peer is up, and a failure
264 // here is not a failure to enable. A session is symmetric, so dialing is enough —
265 // this side sends its own view, and the peer answers with what this side lacks.
266 if let Err(err) = self.sync_now(&key).await {
267 tracing::warn!(root = %key.display(), "the first session after enabling failed: {err}");
268 }
269 Ok(workspace_id)
270 }
271
272 /// Stop syncing `root`. Files and the sync id stay, so re-enabling rejoins the same
273 /// workspace rather than creating a second one.
274 pub async fn disable(&self, root: &Path) -> Result<()> {
275 // A root that no longer exists cannot be canonicalized, but the keys of `synced` are
276 // canonical already, so the path as given (a registry root) still names the entry.
277 let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
278 let removed = self.synced.lock().await.remove(&key);
279 if removed.is_some()
280 && let Some(watcher) = self.watcher.get()
281 {
282 watcher.unwatch(&key);
283 }
284 // Any live session for a workspace this server no longer holds is closed: it would
285 // otherwise keep pushing files for a workspace nobody asked about any more.
286 if let Some(peers) = self.live.lock().await.remove(&key) {
287 peers.drop_connections().await;
288 }
289 if let Some(removed) = removed {
290 self.bridge
291 .unregister(removed.workspace_id)
292 .await
293 .map_err(|e| Error::Bridge(e.to_string()))?;
294 self.reregister().await?;
295 }
296 Ok(())
297 }
298
299 /// The workspaces the workgroup knows about.
300 ///
301 /// Read straight from the bridge, so it reflects every device's registrations. The
302 /// bridge being down is not an empty answer — that would read as "nothing to map" — so
303 /// it is an error, and `sync.map` reports it.
304 pub async fn workspaces(&self) -> Result<WorkspacesResult> {
305 self.bridge
306 .workspaces()
307 .await
308 .map_err(|e| Error::Bridge(e.to_string()))
309 }
310
311 /// Place a workgroup workspace at a directory of this host: write its identity and
312 /// map there, returning the canonical root. [`Current::map`](crate::Current::map)
313 /// then selects it and syncs it.
314 ///
315 /// `selector` is the workspace's name or id, as [`workspaces`] lists it; `dir` must
316 /// already be a workspace of this application — the directory is placed, not made.
317 ///
318 /// [`workspaces`]: SyncRuntime::workspaces
319 pub async fn place(&self, selector: &str, dir: &Path) -> Result<PathBuf> {
320 let workgroup = self.workspaces().await?;
321 let wanted = workgroup
322 .workspaces
323 .iter()
324 // A name is the user-facing handle; the id is the exact one. Two workspaces
325 // may not share a name, so a name names at most one.
326 .find(|w| w.name == selector || w.workspace_id.to_string() == selector)
327 .ok_or_else(|| Error::UnknownWorkspaceName(selector.to_owned()))?;
328
329 // Refuse another application's workspace: this server syncs its own application's
330 // workspaces, and `Workspace::from_root` would later refuse the directory anyway.
331 if wanted.app_name != self.ctx.app_name {
332 return Err(Error::WrongApp {
333 name: wanted.name.clone(),
334 app_name: wanted.app_name.clone(),
335 });
336 }
337
338 let root = dir.canonicalize().map_err(Error::Io)?;
339 let workspace = Workspace::from_root(self.ctx, &root)?;
340 if workspace.root != root {
341 return Err(Error::UnknownWorkspace(root, self.ctx.app_name));
342 }
343
344 // A directory already syncing under another identity is not remapped: the sync id
345 // is a device's word that it is the same workspace, and rewriting it silently
346 // would make two hosts disagree about what they share.
347 {
348 let synced = self.synced.lock().await;
349 if let Some(entry) = synced.get(&root)
350 && entry.workspace_id != wanted.workspace_id
351 {
352 return Err(Error::SyncId(format!(
353 "{} already syncs as {}, not {}",
354 root.display(),
355 entry.workspace_id,
356 wanted.workspace_id
357 )));
358 }
359 }
360
361 // The identity arrives from the workgroup, not from this host: two hosts that map
362 // the same workspace must sync as one, so the id is the one the workgroup lists,
363 // written before `enable` reads it. A different id already on disk is refused
364 // above; an absent one is written here.
365 let marker = workspace.marker_dir();
366 let id_path = marker.join(crate::sync::id::SYNC_ID_FILE);
367 match std::fs::read_to_string(&id_path) {
368 Ok(existing) if existing.trim() != wanted.workspace_id.to_string() => {
369 return Err(Error::SyncId(format!(
370 "{} already holds identity {}, not {}",
371 root.display(),
372 existing.trim(),
373 wanted.workspace_id
374 )));
375 }
376 Err(_) => {
377 std::fs::write(&id_path, format!("{}\n", wanted.workspace_id))
378 .map_err(Error::Io)?;
379 }
380 Ok(_) => {}
381 }
382
383 // The map is what tells this application, on a later start, which workgroup
384 // workspace a directory is.
385 let map_path = root
386 .join(format!(".{}", self.ctx.app_name))
387 .join(WORKSPACE_MAP_FILE);
388 let workspace_id = wanted.workspace_id.to_string();
389 std::fs::write(&map_path, format!("{workspace_id}\n")).map_err(Error::Io)?;
390 Ok(root)
391 }
392
393 /// What `sync.status` answers with.
394 pub async fn status(&self, root: &Path) -> SyncStatus {
395 let bridge_available = self.bridge.status().await.is_ok();
396 let answer = self.bridge.peers().await.ok();
397 if let Some(answer) = &answer {
398 *self.last_peers.lock().await = Some(answer.clone());
399 }
400 let peers = answer
401 .as_ref()
402 .map_or(0, |p| p.peers.len().saturating_sub(1));
403 let Ok(key) = root.canonicalize() else {
404 return SyncStatus {
405 enabled: false,
406 workspace_id: None,
407 peers,
408 paused: None,
409 last_error: None,
410 bridge_available,
411 topology: proto::Topology::Mesh,
412 embedding: None,
413 };
414 };
415 let synced = self.synced.lock().await;
416 match synced.get(&key) {
417 Some(entry) => SyncStatus {
418 enabled: true,
419 workspace_id: Some(entry.workspace_id),
420 peers,
421 paused: entry.paused.map(|r| format!("{r:?}")),
422 last_error: entry.last_error.clone(),
423 bridge_available,
424 topology: topology::topology(
425 answer
426 .as_ref()
427 .and_then(|p| p.roles_for(entry.workspace_id)),
428 ),
429 embedding: self.embedding_progress(&key).await,
430 },
431 None => SyncStatus {
432 enabled: false,
433 workspace_id: None,
434 peers,
435 paused: None,
436 last_error: None,
437 bridge_available,
438 topology: proto::Topology::Mesh,
439 embedding: None,
440 },
441 }
442 }
443
444 /// How far `key`'s embedding has got, when this host embeds it. Reads the open
445 /// workspace's index; never opens one, so a status call stays cheap.
446 async fn embedding_progress(&self, key: &Path) -> Option<proto::EmbeddingProgress> {
447 let host = self.host.get()?;
448 let backend = host.open_backend(key)?;
449 let state = backend.state();
450 state.embedder()?;
451 let info = state.db_info().ok()?;
452 Some(proto::EmbeddingProgress {
453 vectors: info.vector_count,
454 pending: info.pending_count,
455 running: self.embed_running.lock().await.contains(key),
456 })
457 }
458
459 /// Bring the replica's view of the files up to date.
460 ///
461 /// Called straight after the app server's own writes, and by the watcher for everything
462 /// else. A scan that finds nothing is cheap; a scan that is skipped loses an edit until
463 /// the next one.
464 pub async fn scan(&self, root: &Path) -> Result<()> {
465 let Ok(key) = root.canonicalize() else {
466 return Ok(());
467 };
468 let replica = {
469 let synced = self.synced.lock().await;
470 match synced.get(&key) {
471 Some(entry) => Arc::clone(&entry.replica),
472 None => return Ok(()),
473 }
474 };
475 let (outcome, updates) = {
476 let mut replica = replica.lock().await;
477 let outcome = replica.scan().map_err(|e| Error::Sync(e.to_string()))?;
478 // What the scan committed, as path updates, so it can go out on the sessions
479 // that are already open. This is the whole point of the exact-path scan in
480 // `handlers.rs`: a write through the server reaches a peer live, not at the next
481 // dial.
482 let updates = match &outcome {
483 ScanOutcome::Scanned(report) => {
484 // The unit of replication is the path state (design doc section 2.4):
485 // the `seen` a peer prunes against is the *path's* version vector - the
486 // context the recorded entry was built on plus its own dot - never the
487 // whole replica vv. Sending the global vv here made receivers prune
488 // causally-related versions they still held: a conflict copy on the
489 // first remote edit, and tombstones pruned so deletes stopped
490 // propagating (issue #157).
491 report
492 .recorded
493 .iter()
494 .map(|entry| PathUpdate {
495 path: entry.path.clone(),
496 versions: vec![entry.clone()],
497 seen: {
498 let mut seen = entry.context.clone();
499 seen.add_dot(&entry.dot);
500 seen
501 },
502 })
503 .collect()
504 }
505 ScanOutcome::Paused(_) => Vec::new(),
506 };
507 (outcome, updates)
508 };
509 {
510 let mut synced = self.synced.lock().await;
511 if let Some(entry) = synced.get_mut(&key) {
512 entry.paused = match outcome {
513 ScanOutcome::Paused(reason) => Some(reason),
514 ScanOutcome::Scanned(_) => None,
515 };
516 }
517 }
518 // After the map is unlocked: a push awaits every peer, and the table must not be
519 // held behind a slow one.
520 self.after_commit(&key, updates).await;
521 Ok(())
522 }
523
524 /// Bring the search index up to date with files a session has written.
525 ///
526 /// A replication session moves files, not index rows: the index belongs to the app
527 /// server, and the sync core is deliberately free of it. So the side that *applied* a
528 /// change re-indexes, which is what makes a file that arrived from a peer searchable
529 /// without a manual reindex — the claim the whole architecture rests on.
530 ///
531 /// Incremental, not a full rebuild: the workspace's mtime/size stamps mean only what
532 /// actually changed is read back. A file the session wrote has a new stamp, so it is
533 /// picked up; one that was already indexed and untouched is skipped.
534 ///
535 /// Best-effort by contract: the caller's session has already succeeded, and a failure
536 /// here must not be reported as the session's. It is logged and the next session, local
537 /// edit or explicit `workspace.reindex` tries again.
538 async fn reindex(&self, root: &Path) {
539 let Some(host) = self.host.get() else {
540 return;
541 };
542 let host = Arc::clone(host);
543 let root = root.to_owned();
544 // One sweep at a time: two sessions that finish together would otherwise both walk
545 // the workspace through the same retrieve store.
546 let _serialised = self.reindexing.lock().await;
547 // Opening the workspace is itself blocking work, and it is where the exclusive
548 // retrieve-store lock is taken, so it happens here rather than inside the sweep.
549 let backend = match host.backend_at(&root).await {
550 Ok(backend) => backend,
551 Err(err) => {
552 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
553 return;
554 }
555 };
556 let state = Arc::clone(backend.state());
557 let result = tokio::task::spawn_blocking(move || state.sync_retrieve()).await;
558 match result {
559 Ok(Ok(report)) => {
560 tracing::debug!(root = %root.display(), ?report, "re-indexed after a session");
561 }
562 Ok(Err(err)) => {
563 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
564 }
565 Err(err) => {
566 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
567 }
568 }
569 }
570
571 /// Run an embedding pass over `root` in the background, after the pass already
572 /// waiting for it if there is one (#187).
573 ///
574 /// The pass reads the vector files that arrived, and computes the missing ones this
575 /// host is responsible for: the files whose winning version this replica wrote, and —
576 /// on the primary device — every other one. It never holds up the caller.
577 pub fn request_embed(self: &Arc<Self>, root: &Path) {
578 let Ok(key) = root.canonicalize() else {
579 return;
580 };
581 let runtime = Arc::clone(self);
582 tokio::spawn(async move {
583 if !runtime.embed_wanted.lock().await.insert(key.clone()) {
584 return;
585 }
586 let _one_at_a_time = runtime.embedding.lock().await;
587 // Taken off the list as the pass starts: a change during the pass asks again.
588 runtime.embed_wanted.lock().await.remove(&key);
589 runtime.embed_running.lock().await.insert(key.clone());
590 runtime.embed_pass(&key).await;
591 runtime.embed_running.lock().await.remove(&key);
592 });
593 }
594
595 /// One embedding pass over `key`. Best-effort, like the re-index: logged, never fatal.
596 async fn embed_pass(&self, key: &Path) {
597 let Some(host) = self.host.get() else {
598 return;
599 };
600 let (workspace_id, replica) = {
601 let synced = self.synced.lock().await;
602 match synced.get(key) {
603 Some(entry) => (entry.workspace_id, Arc::clone(&entry.replica)),
604 None => return,
605 }
606 };
607 let backend = match host.backend_at(key).await {
608 Ok(backend) => backend,
609 Err(err) => {
610 tracing::warn!(root = %key.display(), "embedding pass: {err}");
611 return;
612 }
613 };
614 let state = Arc::clone(backend.state());
615 if let Err(err) = state
616 .refresh_embedder_at(self.bridge_endpoint.clone())
617 .await
618 {
619 tracing::warn!(root = %key.display(), "embedding pass: {err}");
620 return;
621 }
622 if state.embedder().is_none() {
623 return;
624 }
625 let primary = self.is_primary(workspace_id).await;
626 // The replica's view, read once: which content each path has, and which of those
627 // versions this replica wrote.
628 let grace_ms = self.backfill_timing().grace.as_millis() as u64;
629 let now_ms = std::time::SystemTime::now()
630 .duration_since(std::time::UNIX_EPOCH)
631 .map_or(0, |d| d.as_millis() as u64);
632 let (mine, settled, live) = {
633 let replica = replica.lock().await;
634 let me = replica.replica_id();
635 let states = match replica.states() {
636 Ok(states) => states,
637 Err(err) => {
638 tracing::warn!(root = %key.display(), "embedding pass: {err}");
639 return;
640 }
641 };
642 let mut mine = HashMap::new();
643 // Other devices' versions old enough for the primary to fill in.
644 let mut settled = HashMap::new();
645 let mut live = std::collections::HashSet::new();
646 for (path, path_state) in states {
647 let winner = path_state.winner();
648 let Some(hash) = winner.content.hash() else {
649 continue;
650 };
651 let hash = hash.to_hex();
652 if winner.dot.replica == me {
653 mine.insert(path, hash.clone());
654 } else if now_ms.saturating_sub(winner.hlc.wall_ms) >= grace_ms {
655 settled.insert(path, hash.clone());
656 }
657 live.insert(hash);
658 }
659 (mine, settled, live)
660 };
661 let root = key.to_owned();
662 let result = tokio::task::spawn_blocking(move || {
663 // The author embeds its own versions at once; the primary device fills in other
664 // devices' versions once they have settled, racing nobody.
665 let policy = |rel: &str, hash: &str| {
666 mine.get(rel).is_some_and(|h| h.as_str() == hash)
667 || (primary && settled.get(rel).is_some_and(|h| h.as_str() == hash))
668 };
669 let embedded = state.embed_pending_with(&policy, PASS_BATCH)?;
670 let removed = if primary {
671 state.remove_stale_vectors(&live, STALE_VECTOR_AGE)
672 } else {
673 0
674 };
675 Ok::<_, sapphire_workspace::Error>((embedded, removed))
676 })
677 .await;
678 match result {
679 Ok(Ok((embedded, removed))) if embedded > 0 || removed > 0 => tracing::info!(
680 root = %root.display(),
681 embedded,
682 removed,
683 "embedding pass"
684 ),
685 Ok(Ok(_)) => {}
686 Ok(Err(err)) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
687 Err(err) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
688 }
689 }
690
691 /// Open a session with every peer that will take one.
692 pub async fn sync_now(self: &Arc<Self>, root: &Path) -> Result<()> {
693 let Ok(key) = root.canonicalize() else {
694 return Ok(());
695 };
696 let workspace_id = {
697 let synced = self.synced.lock().await;
698 match synced.get(&key) {
699 Some(entry) => entry.workspace_id,
700 None => return Ok(()),
701 }
702 };
703 let peers = self
704 .bridge
705 .peers()
706 .await
707 .map_err(|e| Error::Bridge(e.to_string()))?;
708 *self.last_peers.lock().await = Some(peers.clone());
709 let me = self.device_id().await?;
710
711 // The same direction rule the dial loop follows: a host only dials peers with a
712 // greater device id. Dialling the other way from here would reintroduce the
713 // dial-both-ways deadlock the loop was cured of, just on a different trigger.
714 // Inside a star, a pair with neither hub in it does not link at all.
715 let roles = peers.roles_for(workspace_id).cloned();
716 for peer in peers
717 .peers
718 .iter()
719 .filter(|p| topology::link(me, p.device_id, roles.as_ref()) == topology::Link::Dial)
720 .cloned()
721 .collect::<Vec<_>>()
722 {
723 // A peer that already has a live session is caught up by it: a second exchange
724 // on the same workspace would be redundant, and a second stream between the same
725 // pair is exactly what the tie-break cannot settle.
726 if let Some(live) = self.live_peers(&key).await
727 && live.has(&peer.device_id).await
728 {
729 continue;
730 }
731 // One dial per peer at a time. `sync_now` runs beside the dial loop, and two
732 // overlapping outbound dials to one peer are the one case the tie-break rule
733 // cannot resolve — each end would see its own stream as `outbound`. Whoever
734 // arrives second simply leaves it to the next pass. The table is made here if
735 // the dial loop has not yet: the first dial after enabling must not depend on
736 // a tick that has not fired.
737 let Some((_, table)) = self.live_peers_or_create(&key).await else {
738 continue;
739 };
740 if !table.begin_dial(peer.device_id).await {
741 continue;
742 }
743 // A peer that does not host this workspace refuses, which is normal and cheap.
744 let result = self
745 .sync_one(&key, &peer.device_id, peer.name.clone(), workspace_id)
746 .await;
747 table.end_dial(peer.device_id).await;
748 if let Err(err) = result {
749 tracing::debug!(peer = %peer.name, "no session: {err}");
750 }
751 }
752 Ok(())
753 }
754
755 /// Dial one peer and adopt the stream as a live session.
756 ///
757 /// The one-shot path (`run_session`) would close the stream the moment the exchange
758 /// finished — while the accepting peer keeps its half open and pushes into it, a session
759 /// that dies with nothing reading it. So the dialled side is adopted live too, and both
760 /// ends of a pair agree that a session stays open.
761 async fn sync_one(
762 self: &Arc<Self>,
763 key: &Path,
764 device: &GrainId,
765 name: String,
766 workspace_id: GrainId,
767 ) -> Result<()> {
768 let stream = self.bridge.open_stream(workspace_id, *device).await?;
769 self.adopt_live_session(stream, key, *device, workspace_id)
770 .await
771 .map_err(|err| {
772 tracing::warn!(peer = %name, "a live session could not be started: {err}");
773 err
774 })
775 }
776
777 /// Answer the bridge's announcements until the connection closes.
778 pub async fn run(self: Arc<Self>) -> Result<()> {
779 let mut incoming = self.bridge.incoming();
780 loop {
781 let announcement = match incoming.recv().await {
782 Ok(a) => a,
783 Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
784 tracing::warn!(missed = n, "fell behind on bridge announcements");
785 continue;
786 }
787 Err(tokio::sync::broadcast::error::RecvError::Closed) => return Ok(()),
788 };
789
790 let found = {
791 let synced = self.synced.lock().await;
792 synced
793 .iter()
794 .find(|(_, s)| s.workspace_id == announcement.workspace_id)
795 .map(|(root, s)| (root.clone(), Arc::clone(&s.replica)))
796 };
797 // `replica` is still what the accepted session shares, and `root` is where its
798 // files and its index live.
799 let Some((root, replica)) = found else {
800 // The bridge routed to us for a workspace we no longer hold. Not fatal:
801 // ignore it and let the ticket expire.
802 tracing::debug!(
803 workspace = %announcement.workspace_id,
804 "an announcement for a workspace this server does not hold"
805 );
806 continue;
807 };
808
809 let bridge = Arc::clone(&self.bridge);
810 let workspace_id = announcement.workspace_id;
811 let peer = announcement.peer_device_id;
812 let driver = Arc::clone(&self);
813 tokio::spawn(async move {
814 let stream = match bridge.accept_stream(announcement.ticket).await {
815 Ok(stream) => stream,
816 Err(err) => {
817 tracing::warn!("could not claim an announced stream: {err}");
818 return;
819 }
820 };
821 if driver.should_skip(workspace_id, peer).await {
822 tracing::debug!(%peer, "dropped an inbound session: both ends are outside this workspace's star");
823 return;
824 }
825 // Live, not one-shot: the peer that dialled is keeping this session open and
826 // pushing what it commits, so this side has to keep reading. A session that
827 // closed after the exchange would leave the dialler pushing into a stream
828 // nobody reads.
829 let (_, session) = match sapphire_framework_session::open_live_session(
830 stream,
831 Arc::clone(&replica),
832 workspace_id,
833 )
834 .await
835 {
836 Ok(started) => started,
837 Err(err) => {
838 tracing::warn!("an inbound session failed: {err}");
839 return;
840 }
841 };
842 let Some((key, table)) = driver.live_peers_or_create(&root).await else {
843 return;
844 };
845 // The peer dialled: this stream survives only if the *peer* has the lower
846 // device id. A stream that loses is dropped, which closes it, and the
847 // connection ends tidily rather than being left half-read.
848 let Some(updates) = table.insert(peer, session, false).await else {
849 return;
850 };
851 // This is the receiving side: the files are on disk now and nobody else will
852 // index them.
853 driver.reindex(&key).await;
854 driver.request_embed(&key);
855 driver.spawn_reader(table, updates, key, peer);
856 });
857 }
858 }
859
860 /// Every synced root, for the watcher.
861 pub async fn roots(&self) -> Vec<PathBuf> {
862 self.synced.lock().await.keys().cloned().collect()
863 }
864
865 /// The live session table for `root`, if this runtime syncs it.
866 ///
867 /// `None` is not an error: a dial, a push or a drop for a workspace this server does not
868 /// hold is a no-op, and every caller below treats it that way.
869 async fn live_peers(&self, root: &Path) -> Option<Arc<LivePeers>> {
870 let key = root.canonicalize().ok()?;
871 self.live.lock().await.get(&key).cloned()
872 }
873
874 /// The live session table for `root`, creating it if this runtime syncs it.
875 ///
876 /// The two locks are taken one after the other, never nested: `synced` to decide whether
877 /// the workspace is ours at all, then `live` to get or make its table.
878 async fn live_peers_or_create(&self, root: &Path) -> Option<(PathBuf, Arc<LivePeers>)> {
879 let key = root.canonicalize().ok()?;
880 if !self.synced.lock().await.contains_key(&key) {
881 return None;
882 }
883 // A table is built by whoever walks the workspace first, and it needs this host's
884 // device id — the tie-break between two simultaneous streams is decided from it.
885 let me = self.device_id().await.ok()?;
886 let mut live = self.live.lock().await;
887 let peers = Arc::clone(
888 live.entry(key.clone())
889 .or_insert_with(|| Arc::new(LivePeers::new(me))),
890 );
891 Some((key, peers))
892 }
893
894 /// Push `updates` to every open session for `root`.
895 ///
896 /// Called with what a scan recorded. `from` is always `None` here: a local commit came
897 /// from this host, so no session is excluded. The forwarding path, which does name a
898 /// source, is [`LivePeers::fan_out`](live::LivePeers::fan_out) called directly by the
899 /// reader side of a session.
900 ///
901 /// Best-effort by contract: the commit already happened, and a peer that is gone is the
902 /// dialer's problem, not the writer's.
903 pub async fn after_commit(&self, root: &Path, updates: Vec<PathUpdate>) {
904 if updates.is_empty() {
905 return;
906 }
907 if let Some(peers) = self.live_peers(root).await {
908 peers.fan_out(&updates, None).await;
909 }
910 }
911
912 /// The devices with an open live session for `root`.
913 ///
914 /// Used by tests to wait for the dialer; empty for a workspace this server does not hold.
915 pub async fn live_session_devices(&self, root: &Path) -> Vec<GrainId> {
916 match self.live_peers(root).await {
917 Some(peers) => peers.devices().await,
918 None => Vec::new(),
919 }
920 }
921
922 /// Close every open live session, as if the connections had been cut.
923 ///
924 /// Called for a workspace being disabled, and by tests that want to watch the dialer
925 /// rebuild what was dropped.
926 pub async fn drop_connections(&self) {
927 let tables: Vec<Arc<LivePeers>> = self.live.lock().await.values().cloned().collect();
928 for peers in tables {
929 peers.drop_connections().await;
930 }
931 }
932
933 /// Keep a live session open to every peer, for every synced workspace.
934 ///
935 /// One walk per [`DIAL_INTERVAL`](live::DIAL_INTERVAL): for each synced workspace, ask
936 /// the bridge who is in the workgroup, and open a session to every peer that is connected
937 /// and does not already have one.
938 ///
939 /// A peer that cannot be reached is retried on a per-peer exponential backoff capped at
940 /// [`DIAL_BACKOFF_MAX`] — one failed attempt per walk, not one per spin — and the counter
941 /// resets when a session opens. An unreachable peer is therefore no obstacle to the
942 /// others: its dial fails, the walk moves on, and the next workspace is dialled.
943 ///
944 /// Runs until the task is aborted, which is what the app server's shutdown does with it.
945 /// It is allowed to fail like [`run`](SyncRuntime::run): only sync stops, and the app
946 /// server keeps serving files (spec §10).
947 pub async fn dial_loop(self: Arc<Self>) -> Result<()> {
948 let mut tick = tokio::time::interval(live::DIAL_INTERVAL);
949 // A missed tick is a late walk, not a burst of them: the interval is a pace, not a
950 // quota to make up.
951 tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
952 let mut waiting: HashMap<GrainId, std::time::Instant> = HashMap::new();
953 let mut failures: HashMap<GrainId, u32> = HashMap::new();
954 // The workspaces this host was the primary device of at the last walk.
955 let mut primary_of: std::collections::HashSet<GrainId> = Default::default();
956 // When the primary device last ran a backfill pass with nothing prompting it.
957 let mut last_backfill = std::time::Instant::now();
958 loop {
959 tick.tick().await;
960 let roots = self.roots().await;
961 if roots.is_empty() {
962 continue;
963 }
964 let peers = match self.bridge.peers().await {
965 Ok(peers) => {
966 *self.last_peers.lock().await = Some(peers.clone());
967 peers
968 }
969 // The bridge is down or has no workgroup: nothing to dial, and the next walk
970 // asks again. Not an error for the caller — the app server is still serving.
971 Err(err) => {
972 tracing::debug!("no peers to dial: {err}");
973 continue;
974 }
975 };
976 let me = match self.device_id().await {
977 Ok(me) => me,
978 Err(err) => {
979 tracing::warn!("no device id, so nobody to dial: {err}");
980 continue;
981 }
982 };
983 // A host that has just become a workspace's primary device owes it the vectors
984 // nobody else computes (#187): files that arrived before the election settled
985 // would otherwise wait for the next change.
986 let ids: Vec<(PathBuf, GrainId)> = self
987 .synced
988 .lock()
989 .await
990 .iter()
991 .map(|(root, entry)| (root.clone(), entry.workspace_id))
992 .collect();
993 // And every so often anyway: an author that never embedded leaves files only the
994 // primary will fill in, and nothing else would prompt it (#188).
995 let backfill_due = last_backfill.elapsed() >= self.backfill_timing().interval;
996 if backfill_due {
997 last_backfill = std::time::Instant::now();
998 }
999 for (root, workspace_id) in ids {
1000 let primary = peers
1001 .roles_for(workspace_id)
1002 .is_some_and(|r| r.primary == Some(me));
1003 if !primary {
1004 primary_of.remove(&workspace_id);
1005 } else if primary_of.insert(workspace_id) || backfill_due {
1006 self.request_embed(&root);
1007 }
1008 }
1009 let now = std::time::Instant::now();
1010 // Dial only peers whose device id is greater than ours. Every device dialling
1011 // every other can deadlock: two hosts that dial each other at once each hold
1012 // their replica's lock across their own exchange and wait for the other's
1013 // hello, which the other's dialler holds the lock for. With one direction per
1014 // pair the wait chain is a DAG — a higher-id host never dials, so it never
1015 // holds its lock out while a lower-id host's exchange needs it. The lower host
1016 // dials within `DIAL_INTERVAL`, and the exchange is symmetric, so nothing is
1017 // lost; the tie-break is left as the safety net it is.
1018 for root in roots {
1019 let Some((key, table)) = self.live_peers_or_create(&root).await else {
1020 continue;
1021 };
1022 let workspace_id = {
1023 let synced = self.synced.lock().await;
1024 match synced.get(&key) {
1025 Some(entry) => entry.workspace_id,
1026 None => continue,
1027 }
1028 };
1029 let roles = peers.roles_for(workspace_id);
1030 // Entering a star closes what it no longer needs. The hubs carry it all —
1031 // once one of them is reachable from here (#192).
1032 if topology::may_close_skipped(me, roles, &table.devices().await) {
1033 table
1034 .retain(|d| topology::link(me, *d, roles) != topology::Link::Skip)
1035 .await;
1036 }
1037 for peer in peers.peers.iter().filter(|p| {
1038 p.connected && topology::link(me, p.device_id, roles) == topology::Link::Dial
1039 }) {
1040 let device = peer.device_id;
1041 if table.has(&device).await {
1042 // Already talking; a redial would drop a working session.
1043 failures.remove(&device);
1044 waiting.remove(&device);
1045 continue;
1046 }
1047 if let Some(until) = waiting.get(&device)
1048 && now < *until
1049 {
1050 // Backing off after a failure; the walk skips it until then.
1051 continue;
1052 }
1053 // The same claim `sync_now` takes: this walk and it can overlap, and a
1054 // double dial in one direction is the one race the tie-break cannot
1055 // settle. Skipping is free — the next pass, or `sync_now` itself, dials.
1056 if !table.begin_dial(device).await {
1057 continue;
1058 }
1059 let opened = self.bridge.open_stream(workspace_id, device).await;
1060 if opened.is_err() {
1061 table.end_dial(device).await;
1062 }
1063 match opened {
1064 Ok(stream) => {
1065 let adopted = self
1066 .adopt_live_session(stream, &root, device, workspace_id)
1067 .await;
1068 // The claim is out either way: the attempt is over, and a
1069 // session that died mid-exchange is re-dialled next pass.
1070 table.end_dial(device).await;
1071 match adopted {
1072 Ok(()) => {
1073 failures.remove(&device);
1074 waiting.remove(&device);
1075 }
1076 Err(err) => {
1077 tracing::debug!(
1078 peer = %peer.name,
1079 err = %err,
1080 "a live session could not be started"
1081 );
1082 let count = failures.entry(device).or_insert(0);
1083 *count = count.saturating_add(1);
1084 waiting.insert(device, now + dial_backoff(*count));
1085 }
1086 }
1087 }
1088 Err(err) => {
1089 // Normal and cheap: a peer that does not host this workspace
1090 // refuses, and one that is unreachable cannot be opened.
1091 tracing::debug!("no stream to a peer: {err}");
1092 let count = failures.entry(device).or_insert(0);
1093 *count = count.saturating_add(1);
1094 waiting.insert(device, now + dial_backoff(*count));
1095 }
1096 }
1097 }
1098 }
1099 }
1100 }
1101
1102 /// Take ownership of an opened stream: run the exchange, then keep it live.
1103 ///
1104 /// The session is stored under `device` and its reader pumped until the stream ends. A
1105 /// half-open or refused stream is not stored; the dialer retries it with backoff.
1106 async fn adopt_live_session(
1107 self: &Arc<Self>,
1108 stream: sapphire_ipc::RawStream,
1109 root: &Path,
1110 device: GrainId,
1111 workspace_id: GrainId,
1112 ) -> Result<()> {
1113 let replica = self.replica_of(root).await?;
1114 let (_, session) = sapphire_framework_session::open_live_session(
1115 stream,
1116 Arc::clone(&replica),
1117 workspace_id,
1118 )
1119 .await
1120 .map_err(|e| Error::Sync(e.to_string()))?;
1121 let Some((key, table)) = self.live_peers_or_create(root).await else {
1122 return Ok(());
1123 };
1124 // This stream was dialled by us, so it is the one the pair keeps if this host's
1125 // device id is the lower of the two. If it lost the tie-break the session is
1126 // dropped here, which closes the stream, and the surviving one is already in the
1127 // table.
1128 let Some(updates) = table.insert(device, session, true).await else {
1129 return Ok(());
1130 };
1131 // The session wrote files during the exchange; the index has to catch up before the
1132 // workspace is searchable again.
1133 self.reindex(&key).await;
1134 self.request_embed(&key);
1135 self.spawn_reader(Arc::clone(&table), updates, key, device);
1136 Ok(())
1137 }
1138
1139 /// Forward what one session receives, and re-index it.
1140 ///
1141 /// This is the loop that makes A → S → B work: a batch that arrived on S's session with A
1142 /// is applied (the session crate does that before publishing it), the workspace behind
1143 /// `key` is re-indexed, and the batch is handed to every *other* session whose peer lacks
1144 /// it — `from` is the device it came from, so it never goes back to A.
1145 fn spawn_reader(
1146 self: &Arc<Self>,
1147 table: Arc<LivePeers>,
1148 mut updates: broadcast::Receiver<Vec<PathUpdate>>,
1149 key: PathBuf,
1150 from: GrainId,
1151 ) {
1152 let driver = Arc::clone(self);
1153 tokio::spawn(async move {
1154 loop {
1155 match updates.recv().await {
1156 Ok(batch) => {
1157 // The files are on disk now (the session guarantees it before
1158 // publishing), so the index is behind them until the sweep below.
1159 driver.reindex(&key).await;
1160 driver.request_embed(&key);
1161 table.fan_out(&batch, Some(from)).await;
1162 }
1163 // A burst bigger than the channel: the newest is kept and the gap is
1164 // covered by the next session's exchange. Losing a batch here cannot lose
1165 // an entry — the peer's vector does not cover it, so the next dial or push
1166 // carries it again.
1167 Err(broadcast::error::RecvError::Lagged(missed)) => {
1168 tracing::debug!(
1169 device = %from,
1170 missed,
1171 "fell behind on a live session; the next exchange catches up"
1172 );
1173 }
1174 // The session closed. Its table entry is reaped by the next fan-out or
1175 // dial pass, and the dialer opens a fresh one.
1176 Err(broadcast::error::RecvError::Closed) => {
1177 return;
1178 }
1179 }
1180 }
1181 });
1182 }
1183
1184 /// The replica of a synced `root`.
1185 async fn replica_of(&self, root: &Path) -> Result<Arc<Mutex<Replica>>> {
1186 let key = root.canonicalize().map_err(Error::Io)?;
1187 let synced = self.synced.lock().await;
1188 match synced.get(&key) {
1189 Some(entry) => Ok(Arc::clone(&entry.replica)),
1190 None => Err(Error::UnknownWorkspace(key, self.ctx.app_name)),
1191 }
1192 }
1193
1194 /// Watch the synced roots and, on a debounced report, scan and dial.
1195 ///
1196 /// This is the safety net for everything the app server did not do itself — a file
1197 /// edited in an editor, a `git checkout`. The exact path is [`scan`] called from
1198 /// `handlers.rs` right after a write the server made. A scan that finds nothing is
1199 /// cheap; a scan that is skipped loses an edit until the next one.
1200 ///
1201 /// Returns when the reporting channel closes — that is, when the [`Watcher`] this
1202 /// runtime holds is dropped, which happens when the runtime itself is.
1203 ///
1204 /// [`scan`]: SyncRuntime::scan
1205 pub async fn watch_changes(self: Arc<Self>) -> Result<()> {
1206 let (tx, mut rx) = mpsc::channel(64);
1207 let watcher = Arc::new(Watcher::start(self.roots().await, tx)?);
1208 let _ = self.watcher.set(Arc::clone(&watcher));
1209
1210 // A root enabled between `roots()` above and the `set` just now was not in the
1211 // starting set, and `enable` could not reach a watcher that did not exist yet.
1212 // Watching the current set again is idempotent on `notify`'s side.
1213 for root in self.roots().await {
1214 if let Err(err) = watcher.watch(&root) {
1215 tracing::warn!(root = %root.display(), "could not watch: {err}");
1216 }
1217 }
1218
1219 while let Some(root) = rx.recv().await {
1220 // Logged, not fatal: a failed scan or dial leaves the replica behind, and the
1221 // next report tries again. Neither may take the app server down (spec §10).
1222 if let Err(err) = self.scan(&root).await {
1223 tracing::warn!(root = %root.display(), "scan after a local edit failed: {err}");
1224 }
1225 self.request_embed(&root);
1226 if let Err(err) = self.sync_now(&root).await {
1227 tracing::warn!(root = %root.display(), "dial after a local edit failed: {err}");
1228 }
1229 }
1230 Ok(())
1231 }
1232
1233 /// Tell the bridge the complete current set. A registration is not a delta.
1234 async fn reregister(&self) -> Result<()> {
1235 let workspaces: Vec<WorkspaceRegistration> = {
1236 let synced = self.synced.lock().await;
1237 synced
1238 .iter()
1239 .map(|(root, entry)| WorkspaceRegistration {
1240 workspace_id: entry.workspace_id,
1241 root: root.clone(),
1242 })
1243 .collect()
1244 };
1245 self.bridge
1246 .register(RegisterParams {
1247 app_name: self.ctx.app_name.to_owned(),
1248 exe_path: self.exe_path.clone(),
1249 managed_by: wire_managed_by(self.managed_by),
1250 workspaces,
1251 })
1252 .await
1253 .map_err(|e| Error::Bridge(e.to_string()))?;
1254 Ok(())
1255 }
1256
1257 /// Whether the last roles the bridge reported say this pair does not link.
1258 ///
1259 /// Reads the cached device id only: asking the bridge here would register with an empty
1260 /// workspace list, racing `enable`'s registration. `false` (accept) when it is unknown.
1261 async fn should_skip(&self, workspace_id: GrainId, peer: GrainId) -> bool {
1262 let Some(&me) = self.device_id.get() else {
1263 return false;
1264 };
1265 let cached = self.last_peers.lock().await;
1266 let roles = cached.as_ref().and_then(|p| p.roles_for(workspace_id));
1267 topology::link(me, peer, roles) == topology::Link::Skip
1268 }
1269
1270 /// Whether this host is `workspace_id`'s primary device, by the last roles the
1271 /// bridge reported. `false` when the bridge has not answered yet, or when this host's
1272 /// device id is not known yet (no workspace was enabled). Reads the cached id only:
1273 /// asking the bridge here would register with an empty workspace list, racing
1274 /// `enable`'s registration.
1275 pub async fn is_primary(&self, workspace_id: GrainId) -> bool {
1276 let Some(&me) = self.device_id.get() else {
1277 return false;
1278 };
1279 let cached = self.last_peers.lock().await;
1280 cached
1281 .as_ref()
1282 .and_then(|p| p.roles_for(workspace_id))
1283 .is_some_and(|r| r.primary == Some(me))
1284 }
1285
1286 /// This host's device id, asked of the bridge once.
1287 async fn device_id(&self) -> Result<GrainId> {
1288 if let Some(id) = self.device_id.get() {
1289 return Ok(*id);
1290 }
1291 let result = self
1292 .bridge
1293 .register(RegisterParams {
1294 app_name: self.ctx.app_name.to_owned(),
1295 exe_path: self.exe_path.clone(),
1296 managed_by: wire_managed_by(self.managed_by),
1297 workspaces: Vec::new(),
1298 })
1299 .await
1300 .map_err(|e| Error::Bridge(e.to_string()))?;
1301 let _ = self.device_id.set(result.device_id);
1302 Ok(result.device_id)
1303 }
1304
1305 /// `<workspace cache dir>/sync/`.
1306 fn state_dir(&self, root: &Path) -> Result<PathBuf> {
1307 let workspace = Workspace::from_root(self.ctx, root)?;
1308 Ok(workspace.cache_dir().join("sync"))
1309 }
1310}
1311
1312/// The bridge API's copy of how this server was started, for [`RegisterParams`].
1313fn wire_managed_by(managed_by: ManagedBy) -> sapphire_bridge_api::ManagedBy {
1314 match managed_by {
1315 ManagedBy::Service => sapphire_bridge_api::ManagedBy::Service,
1316 ManagedBy::Spawned => sapphire_bridge_api::ManagedBy::Spawned,
1317 }
1318}
1319
1320#[cfg(test)]
1321mod tests {
1322 use super::*;
1323 use crate::sync::testing::StubBridge;
1324 use crate::test_support;
1325 use sapphire_workspace::AppKind;
1326 use std::ffi::OsString;
1327
1328 static CTX: AppContext = AppContext::new("sapphire-synctest");
1329
1330 /// The env vars `CTX.init` reads.
1331 const DIR_VARS: [&str; 3] = [
1332 "SAPPHIRE_SYNCTEST_CACHE_DIR",
1333 "SAPPHIRE_SYNCTEST_DATA_DIR",
1334 "SAPPHIRE_SYNCTEST_CONFIG_DIR",
1335 ];
1336
1337 /// Points the context's directories at the test's scratch tree, and restores the
1338 /// previous values when dropped — including while unwinding from a panic.
1339 struct EnvGuard {
1340 previous: [Option<OsString>; 3],
1341 _lock: std::sync::MutexGuard<'static, ()>,
1342 }
1343
1344 impl Drop for EnvGuard {
1345 fn drop(&mut self) {
1346 // SAFETY: `self._lock` still serialises the environment; it is dropped only
1347 // after this method returns.
1348 for (name, previous) in DIR_VARS.iter().zip(self.previous.iter_mut()) {
1349 match previous.take() {
1350 Some(value) => unsafe { std::env::set_var(name, value) },
1351 None => test_support::remove(name),
1352 }
1353 }
1354 }
1355 }
1356
1357 /// Everything one test needs, held for the test's whole body.
1358 ///
1359 /// The context is a `static` shared by this binary, so its directories are resolved from
1360 /// the environment by whichever test initialises it first. The environment lock is
1361 /// therefore held for the whole test, as in the crate's other test modules, and `_tmp`
1362 /// is declared before `_env` so the scratch tree is gone only once the environment no
1363 /// longer points at it.
1364 struct Fixture {
1365 _tmp: tempfile::TempDir,
1366 _env: EnvGuard,
1367 root: PathBuf,
1368 stub: StubBridge,
1369 runtime: Arc<SyncRuntime>,
1370 }
1371
1372 impl Fixture {
1373 /// A second workspace root beside the first, marker directory and all.
1374 fn second_root(&self) -> PathBuf {
1375 let root = self._tmp.path().join("ws2");
1376 std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
1377 root.canonicalize().unwrap()
1378 }
1379 }
1380
1381 async fn fixture() -> Fixture {
1382 let lock = test_support::lock();
1383 let tmp = tempfile::tempdir().unwrap();
1384 let previous = DIR_VARS.map(std::env::var_os);
1385 // SAFETY (via `test_support::set`): `lock` serialises every read and write of the
1386 // process environment in this test binary, and it is held until `drop` has restored
1387 // the old values.
1388 for (name, dir) in DIR_VARS
1389 .iter()
1390 .zip(["cache", "data", "config"].map(|cat| tmp.path().join(cat)))
1391 {
1392 test_support::set(name, &dir);
1393 }
1394 let env = EnvGuard {
1395 previous,
1396 _lock: lock,
1397 };
1398 CTX.init(AppKind::Server);
1399
1400 let root = tmp.path().join("ws");
1401 std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
1402 let root = root.canonicalize().unwrap();
1403
1404 let (stub, client) = StubBridge::start().await;
1405 let runtime = Arc::new(SyncRuntime::new(
1406 &CTX,
1407 client,
1408 "/bin/true".into(),
1409 ManagedBy::Service,
1410 ));
1411 Fixture {
1412 _tmp: tmp,
1413 _env: env,
1414 root,
1415 stub,
1416 runtime,
1417 }
1418 }
1419
1420 #[tokio::test(flavor = "multi_thread")]
1421 async fn enabling_registers_the_workspace_with_the_bridge() {
1422 let f = fixture().await;
1423
1424 let id = f.runtime.enable(&f.root).await.unwrap();
1425 assert_eq!(f.stub.last_workspaces(), vec![id]);
1426 }
1427
1428 #[tokio::test(flavor = "multi_thread")]
1429 async fn enabling_twice_is_idempotent_and_keeps_the_same_id() {
1430 let f = fixture().await;
1431
1432 let first = f.runtime.enable(&f.root).await.unwrap();
1433 let second = f.runtime.enable(&f.root).await.unwrap();
1434 assert_eq!(first, second);
1435 assert_eq!(
1436 f.stub.last_workspaces(),
1437 vec![first],
1438 "still exactly one workspace"
1439 );
1440 }
1441
1442 #[tokio::test(flavor = "multi_thread")]
1443 async fn a_registration_carries_the_complete_current_list() {
1444 let f = fixture().await;
1445 let second_root = f.second_root();
1446
1447 let a = f.runtime.enable(&f.root).await.unwrap();
1448 let b = f.runtime.enable(&second_root).await.unwrap();
1449
1450 let mut seen = f.stub.last_workspaces();
1451 seen.sort();
1452 let mut want = vec![a, b];
1453 want.sort();
1454 assert_eq!(seen, want, "registration is the whole set, not a delta");
1455 }
1456
1457 #[tokio::test(flavor = "multi_thread")]
1458 async fn no_later_registration_clears_this_apps_routes() {
1459 // The device id is asked of the bridge once, on the first `enable`, with an empty
1460 // workspace list — the one moment that is safe, because no route exists yet. Every
1461 // later registration must name the complete set: one with an empty list would
1462 // briefly clear this application's routes, and a peer asking for a workspace this
1463 // server owns would be told nobody has it.
1464 let f = fixture().await;
1465 let second_root = f.second_root();
1466
1467 f.runtime.enable(&f.root).await.unwrap();
1468 f.runtime.enable(&second_root).await.unwrap();
1469 f.runtime.sync_now(&f.root).await.unwrap();
1470 f.runtime.disable(&f.root).await.unwrap();
1471
1472 let seen = f.stub.seen.lock().expect("stub");
1473 let first_full = seen
1474 .registrations
1475 .iter()
1476 .position(|r| !r.workspaces.is_empty())
1477 .expect("enabling registers the workspace");
1478 let emptied_afterwards = seen.registrations[first_full + 1..]
1479 .iter()
1480 .filter(|r| r.workspaces.is_empty())
1481 .count();
1482 assert_eq!(
1483 emptied_afterwards, 0,
1484 "a registration after the first workspace must still name the whole set"
1485 );
1486 }
1487
1488 #[tokio::test(flavor = "multi_thread")]
1489 async fn disabling_unregisters_and_leaves_the_files_alone() {
1490 let f = fixture().await;
1491 std::fs::write(f.root.join("keep.md"), "content").unwrap();
1492
1493 let id = f.runtime.enable(&f.root).await.unwrap();
1494 f.runtime.disable(&f.root).await.unwrap();
1495
1496 assert!(f.stub.last_workspaces().is_empty());
1497 assert!(
1498 f.root.join("keep.md").exists(),
1499 "disabling sync must not touch files"
1500 );
1501 assert!(
1502 crate::sync::id::sync_id_path("sapphire-synctest", &f.root).exists(),
1503 "the sync id stays, so re-enabling rejoins the same workspace"
1504 );
1505 let _ = id;
1506 }
1507
1508 #[tokio::test(flavor = "multi_thread")]
1509 async fn re_enabling_after_disabling_reuses_the_identity() {
1510 let f = fixture().await;
1511
1512 let first = f.runtime.enable(&f.root).await.unwrap();
1513 f.runtime.disable(&f.root).await.unwrap();
1514 let again = f.runtime.enable(&f.root).await.unwrap();
1515
1516 assert_eq!(first, again);
1517 }
1518
1519 #[tokio::test(flavor = "multi_thread")]
1520 async fn status_reports_disabled_for_a_workspace_that_was_never_enabled() {
1521 let f = fixture().await;
1522
1523 let status = f.runtime.status(&f.root).await;
1524 assert!(!status.enabled);
1525 assert!(status.workspace_id.is_none());
1526 }
1527
1528 #[tokio::test(flavor = "multi_thread")]
1529 async fn disabling_something_that_was_never_enabled_is_not_an_error() {
1530 let f = fixture().await;
1531 f.runtime
1532 .disable(&f.root)
1533 .await
1534 .expect("disabling twice is fine");
1535 }
1536
1537 #[tokio::test(flavor = "multi_thread")]
1538 async fn a_workspace_whose_root_vanished_is_reported_as_paused_not_deleted() {
1539 let f = fixture().await;
1540 std::fs::write(f.root.join("a.md"), "content").unwrap();
1541 f.runtime.enable(&f.root).await.unwrap();
1542 f.runtime.scan(&f.root).await.unwrap();
1543
1544 // The drive is unmounted: the marker directory goes with it.
1545 std::fs::remove_dir_all(f.root.join(".sapphire-synctest")).unwrap();
1546 let _ = f.runtime.scan(&f.root).await;
1547
1548 let status = f.runtime.status(&f.root).await;
1549 assert!(
1550 status.paused.is_some(),
1551 "a missing root must pause, not replicate as a mass deletion"
1552 );
1553 }
1554
1555 #[tokio::test(flavor = "multi_thread")]
1556 async fn a_directory_that_is_not_a_workspace_cannot_be_enabled() {
1557 let f = fixture().await;
1558 let plain = f._tmp.path().join("plain");
1559 std::fs::create_dir_all(&plain).unwrap();
1560
1561 assert!(f.runtime.enable(&plain).await.is_err());
1562 }
1563
1564 #[tokio::test(flavor = "multi_thread")]
1565 async fn a_write_through_the_workspace_router_is_committed_to_the_replica() {
1566 // The brief's "main path": a write the app server itself made is scanned straight
1567 // after, not left to the watcher's debounce. Asserted against the replica's own
1568 // state, which is the only thing that can tell the two paths apart — reading the
1569 // file back would pass even if the scan never ran.
1570 let f = fixture().await;
1571 f.runtime.enable(&f.root).await.unwrap();
1572
1573 let host = Arc::new(crate::WorkspaceHost::new(&CTX));
1574 host.select(&f.root, false).await.unwrap();
1575 let current = Arc::new(crate::Current::new(
1576 &CTX,
1577 host,
1578 Some(Arc::clone(&f.runtime)),
1579 crate::selection::SelectionFile::at(f._tmp.path().join("workspace.toml")),
1580 ));
1581 let router = Arc::new(crate::workspace_router(current));
1582 let (client_conn, server_conn) = sapphire_ipc::Connection::pair();
1583 tokio::spawn(async move {
1584 let info = sapphire_ipc::ServerInfo {
1585 version: "0.0.0".into(),
1586 api: sapphire_backend::protocol::API_VERSION,
1587 app_api: None,
1588 pid: std::process::id(),
1589 managed_by: ManagedBy::Spawned,
1590 };
1591 let _ = sapphire_ipc::serve(server_conn, router, "sapphire-synctest", info).await;
1592 });
1593 let info = sapphire_ipc::ClientInfo {
1594 kind: "test".into(),
1595 version: "0.0.0".into(),
1596 api: sapphire_backend::protocol::API_VERSION,
1597 app_api: None,
1598 pid: std::process::id(),
1599 };
1600 let (client, _) = sapphire_ipc::Client::handshake(client_conn, "sapphire-synctest", info)
1601 .await
1602 .unwrap();
1603
1604 let _: sapphire_backend::protocol::Ack = client
1605 .call(
1606 sapphire_backend::protocol::WRITE_FILE,
1607 sapphire_backend::protocol::ContentParams {
1608 path: PathBuf::from("through-the-server.md"),
1609 content: "written by the server".into(),
1610 },
1611 )
1612 .await
1613 .unwrap();
1614
1615 // `scan` runs before the reply is written, so by the time the call returns the
1616 // replica must already know the path.
1617 let synced = f.runtime.synced.lock().await;
1618 let entry = synced.get(&f.root).expect("enabled above");
1619 let replica = entry.replica.lock().await;
1620 let state = replica
1621 .state("through-the-server.md")
1622 .expect("a readable state");
1623 assert!(
1624 state.is_some(),
1625 "a write through the server must be scanned into the replica, not left to the watcher"
1626 );
1627 }
1628
1629 #[tokio::test(flavor = "multi_thread")]
1630 async fn an_announcement_for_an_unknown_workspace_is_ignored_not_fatal() {
1631 let f = fixture().await;
1632 f.runtime.enable(&f.root).await.unwrap();
1633
1634 let driver = Arc::clone(&f.runtime);
1635 let handle = tokio::spawn(async move { driver.run().await });
1636
1637 f.stub
1638 .announce(grain_id::GrainId::random(), "no-such-ticket")
1639 .await;
1640 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
1641
1642 assert!(
1643 !handle.is_finished(),
1644 "one bad announcement must not stop the runtime"
1645 );
1646 handle.abort();
1647 }
1648}