sapphire_framework_server/sync/mod.rs
1//! Replication, from the app server's side.
2//!
3//! One `Replica` per synced workspace, registered with the bridge, driven by announcements
4//! from it and by local edits.
5//!
6//! See `docs/superpowers/specs/2026-09-16-process-architecture-design.md` §4.2.
7
8use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use grain_id::GrainId;
13use sapphire_backend::protocol as proto;
14use sapphire_bridge_api::{PeersResult, RegisterParams, WorkspaceRegistration, WorkspacesResult};
15use sapphire_bridge_client::BridgeClient;
16use sapphire_ipc::ManagedBy;
17use sapphire_sync::{PathUpdate, PauseReason, Replica, ReplicaConfig, ScanOutcome, SystemClock};
18use sapphire_workspace::{AppContext, Workspace};
19use tokio::sync::{Mutex, OnceCell, broadcast, mpsc};
20
21pub mod id;
22mod live;
23mod methods;
24#[cfg(any(test, feature = "test-util"))]
25pub mod testing;
26mod topology;
27mod watch;
28
29pub use id::{SYNC_ID_FILE, WORKSPACE_MAP_FILE, sync_id, sync_id_path};
30pub use live::{DIAL_BACKOFF_MAX, OPEN_TIMEOUT};
31pub use methods::sync_router;
32pub use watch::{DEBOUNCE, Watcher};
33
34use live::{LivePeers, dial_backoff};
35
36use crate::error::{Error, Result};
37
38use crate::host::WorkspaceHost;
39
40/// What `sync.status` reports.
41#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
42pub struct SyncStatus {
43 /// Whether this workspace is synced at all.
44 pub enabled: bool,
45 /// Its identity across devices, when it is.
46 pub workspace_id: Option<GrainId>,
47 /// How many devices the workgroup has, this host excluded.
48 pub peers: usize,
49 /// Why replication is paused, if it is — a missing root or marker directory.
50 pub paused: Option<String>,
51 /// The last failure, if any.
52 pub last_error: Option<String>,
53 /// Whether the bridge is reachable. `false` is not an outage: the app server works.
54 pub bridge_available: bool,
55 /// How the workspace is wired to its peers, by the roles the bridge last reported.
56 pub topology: proto::Topology,
57 /// How far embedding has got, when this host embeds (#188).
58 pub embedding: Option<proto::EmbeddingProgress>,
59}
60
61/// One workspace this runtime syncs.
62struct Synced {
63 /// Its identity across devices, shared with every peer.
64 workspace_id: GrainId,
65 /// Its replica store. Held open because the store is a redb database.
66 replica: Arc<Mutex<Replica>>,
67 /// Why replication is paused, from the last scan.
68 paused: Option<PauseReason>,
69 /// The last failure, if any.
70 last_error: Option<String>,
71}
72
73/// How old a vector file no live content needs must be before the primary device removes
74/// it: a vector can arrive before its file, and a day is far longer than that window.
75const STALE_VECTOR_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
76
77/// Texts per embedding request in a server pass: small, so a search query waiting on the
78/// bridge's one worker is never stuck behind a long backfill (#188).
79const PASS_BATCH: usize = 4;
80
81/// How the primary device paces its backfill (#188).
82#[derive(Clone, Copy, Debug)]
83pub struct BackfillTiming {
84 /// How old another device's version must be before the primary embeds it: its author
85 /// has embedded it by then, or will not.
86 pub grace: std::time::Duration,
87 /// How often the primary device runs a pass with nothing prompting it.
88 pub interval: std::time::Duration,
89}
90
91impl Default for BackfillTiming {
92 fn default() -> Self {
93 BackfillTiming {
94 grace: std::time::Duration::from_secs(10 * 60),
95 interval: std::time::Duration::from_secs(10 * 60),
96 }
97 }
98}
99
100/// Replication for one application's workspaces.
101pub struct SyncRuntime {
102 ctx: &'static AppContext,
103 bridge: Arc<BridgeClient>,
104 exe_path: PathBuf,
105 managed_by: ManagedBy,
106 synced: Mutex<HashMap<PathBuf, Synced>>,
107 /// This host's device id. Asked of the bridge once, on the first [`enable`], and never
108 /// again: a second registration with an empty workspace list would briefly clear this
109 /// application's routes.
110 ///
111 /// [`enable`]: SyncRuntime::enable
112 device_id: OnceCell<GrainId>,
113 /// The file watcher, once [`watch_changes`] has started it.
114 ///
115 /// The runtime owns it because the spec puts it here: the sync runtime is a watcher and
116 /// a replica per registered root, and the two must agree on which roots are synced.
117 /// [`enable`] and [`disable`] keep them in step.
118 ///
119 /// [`watch_changes`]: SyncRuntime::watch_changes
120 /// [`enable`]: SyncRuntime::enable
121 /// [`disable`]: SyncRuntime::disable
122 watcher: OnceCell<Arc<Watcher>>,
123 /// The app server that owns these workspaces' caches, once it has told us.
124 ///
125 /// A replication session writes files, and the search index is not part of a session: the
126 /// index belongs to the app server, which holds the workspace open. Reaching it through
127 /// the host is what keeps a received file and its index moving together, and it is the
128 /// *same* [`WorkspaceState`](sapphire_workspace::WorkspaceState) the server serves from —
129 /// opening a second one would collide on the retrieve store's exclusive lock.
130 ///
131 /// Absent for a runtime nobody has wired to an app server (a test stub driving the
132 /// replica directly). Nothing then indexes, which is correct: nothing owns an index.
133 host: OnceCell<Arc<WorkspaceHost>>,
134 /// Serialises re-indexing, so two sessions finishing at once do not both sweep the
135 /// workspace through the same store.
136 reindexing: Mutex<()>,
137 /// Roots with an embedding pass waiting to run (#187). A request for a root already
138 /// waiting is folded into that pass.
139 embed_wanted: Mutex<std::collections::HashSet<PathBuf>>,
140 /// Serialises embedding passes: one model, one queue.
141 embedding: Mutex<()>,
142 /// Roots whose embedding pass is running now, for `sync.status`.
143 embed_running: Mutex<std::collections::HashSet<PathBuf>>,
144 /// The primary device's backfill pacing (#188).
145 backfill: std::sync::Mutex<BackfillTiming>,
146 /// The bridge an embedding pass asks for its model; `None` is the host's standard one.
147 bridge_endpoint: Option<sapphire_ipc::Endpoint>,
148 /// The live session table of every synced workspace, keyed by canonical root.
149 ///
150 /// One table per workspace rather than one for the runtime: a session is about one
151 /// workspace, and the two storm rules are per-workspace rules. A root that is not
152 /// synced has none, so a dial or a push for it is a no-op rather than an error.
153 live: Mutex<HashMap<PathBuf, Arc<LivePeers>>>,
154 /// The bridge's last answer to `peers`, roles included. The inbound guard and
155 /// [`is_primary`](SyncRuntime::is_primary) read it rather than asking again.
156 last_peers: Mutex<Option<PeersResult>>,
157 /// How long a dial waits for the bridge to open a stream; [`OPEN_TIMEOUT`] by default.
158 open_timeout: std::time::Duration,
159 /// The first dials [`enable`](SyncRuntime::enable) leaves running. Each pins the
160 /// runtime, so [`stop_dials`](SyncRuntime::stop_dials) ends them with the server.
161 first_dials: std::sync::Mutex<tokio::task::JoinSet<()>>,
162}
163
164impl SyncRuntime {
165 /// A runtime for `ctx`'s application.
166 pub fn new(
167 ctx: &'static AppContext,
168 bridge: Arc<BridgeClient>,
169 exe_path: PathBuf,
170 managed_by: ManagedBy,
171 ) -> SyncRuntime {
172 SyncRuntime {
173 ctx,
174 bridge,
175 exe_path,
176 managed_by,
177 synced: Mutex::new(HashMap::new()),
178 device_id: OnceCell::new(),
179 watcher: OnceCell::new(),
180 host: OnceCell::new(),
181 reindexing: Mutex::new(()),
182 embed_wanted: Mutex::new(std::collections::HashSet::new()),
183 embedding: Mutex::new(()),
184 embed_running: Mutex::new(std::collections::HashSet::new()),
185 backfill: std::sync::Mutex::new(BackfillTiming::default()),
186 bridge_endpoint: None,
187 live: Mutex::new(HashMap::new()),
188 last_peers: Mutex::new(None),
189 open_timeout: OPEN_TIMEOUT,
190 first_dials: std::sync::Mutex::new(tokio::task::JoinSet::new()),
191 }
192 }
193
194 /// Give up on opening a stream to a peer after `timeout` rather than [`OPEN_TIMEOUT`]:
195 /// for tests, which cannot wait that long for a peer that never answers.
196 pub fn with_open_timeout(mut self, timeout: std::time::Duration) -> SyncRuntime {
197 self.open_timeout = timeout;
198 self
199 }
200
201 /// Tell the runtime which app server owns these workspaces' caches.
202 ///
203 /// Called by [`AppServer::sync`](crate::AppServer::sync). First writer wins: a runtime
204 /// belongs to the server that wired it.
205 /// Embed through the bridge at `endpoint` rather than the host's standard one: for a
206 /// fixture running several hosts in one process.
207 pub fn with_bridge_endpoint(mut self, endpoint: sapphire_ipc::Endpoint) -> SyncRuntime {
208 self.bridge_endpoint = Some(endpoint);
209 self
210 }
211
212 /// Pace the primary device's backfill differently: for tests, which cannot wait ten
213 /// minutes.
214 pub fn set_backfill_timing(&self, timing: BackfillTiming) {
215 *self.backfill.lock().expect("backfill timing") = timing;
216 }
217
218 fn backfill_timing(&self) -> BackfillTiming {
219 *self.backfill.lock().expect("backfill timing")
220 }
221
222 pub(crate) fn set_host(&self, host: Arc<WorkspaceHost>) {
223 let _ = self.host.set(host);
224 }
225
226 /// Start syncing `root`, returning its identity across devices.
227 ///
228 /// Idempotent: enabling an already-synced workspace returns the same id and changes
229 /// nothing.
230 pub async fn enable(self: &Arc<Self>, root: &Path) -> Result<GrainId> {
231 let key = root.canonicalize().map_err(Error::Io)?;
232 let watch_key = key.clone();
233 let workspace_id = sync_id(self.ctx.app_name, &key)?;
234
235 // The map stays locked across opening the replica, so two simultaneous `enable`s for
236 // one root cannot both open it — the store is a redb database, and it is opened once.
237 {
238 let mut synced = self.synced.lock().await;
239 if let Some(existing) = synced.get(&key) {
240 return Ok(existing.workspace_id);
241 }
242 let device_id = self.device_id().await?;
243 let state_dir = self.state_dir(&key)?;
244 std::fs::create_dir_all(&state_dir).map_err(Error::Io)?;
245 let config = ReplicaConfig::new(self.ctx.app_name, key.clone(), device_id, &state_dir);
246 let replica = Replica::open(config, Arc::new(SystemClock))
247 .map_err(|e| Error::Sync(e.to_string()))?;
248 synced.insert(
249 key.clone(),
250 Synced {
251 workspace_id,
252 replica: Arc::new(Mutex::new(replica)),
253 paused: None,
254 last_error: None,
255 },
256 );
257 }
258
259 // Watch the new root too, so an edit made outside the server from now on is seen.
260 // Not fatal if it fails: the app server's own writes take the exact path in
261 // `handlers.rs`, and the next `watch_changes` re-reads the root set anyway.
262 if let Some(watcher) = self.watcher.get()
263 && let Err(err) = watcher.watch(&watch_key)
264 {
265 tracing::warn!(root = %watch_key.display(), "could not watch: {err}");
266 }
267
268 // Register only once the replica is open, so a registration never names a workspace
269 // that cannot serve a session.
270 self.reregister().await?;
271
272 // Dial now, so enabling a workspace converges it rather than waiting for the next
273 // local edit to trigger a session. This is what catches a host up when it returns:
274 // the peer that has been running is the one the returning host has to ask, and no
275 // watcher on either side fires for a file that was never written locally.
276 //
277 // Best-effort: a workspace is enabled whether or not any peer is up, and a failure
278 // here is not a failure to enable. A session is symmetric, so dialing is enough —
279 // this side sends its own view, and the peer answers with what this side lacks.
280 //
281 // In the background, so the answer does not wait on it: a peer that is offline
282 // holds a dial up to `open_timeout`, longer than a client waits for `sync.enable`
283 // (#228).
284 let runtime = Arc::clone(self);
285 let mut dials = self.first_dials.lock().expect("first dials");
286 // Reap the finished ones, so the set holds only what is running.
287 while dials.try_join_next().is_some() {}
288 dials.spawn(async move {
289 if let Err(err) = runtime.sync_now(&key).await {
290 tracing::warn!(root = %key.display(), "the first session after enabling failed: {err}");
291 }
292 });
293 Ok(workspace_id)
294 }
295
296 /// Stop syncing `root`. Files and the sync id stay, so re-enabling rejoins the same
297 /// workspace rather than creating a second one.
298 pub async fn disable(&self, root: &Path) -> Result<()> {
299 self.stop(root).await.1
300 }
301
302 /// Stop syncing `root`, like [`disable`](Self::disable), then wait up to `limit` until
303 /// nothing holds its replica store open any more — a session, a scan or an embedding
304 /// pass that started before may still have it. A store about to be moved or deleted
305 /// must be closed first.
306 ///
307 /// Only a store still in use at `limit` is an error. The bridge not hearing of it is
308 /// logged, not returned: the store is closed all the same, and the next registration
309 /// replaces the bridge's view.
310 pub async fn release(&self, root: &Path, limit: std::time::Duration) -> Result<()> {
311 let (replica, unregistered) = self.stop(root).await;
312 if let Err(err) = unregistered {
313 tracing::warn!(root = %root.display(), "stopping sync: {err}");
314 }
315 let Some(replica) = replica else {
316 return Ok(());
317 };
318 let deadline = tokio::time::Instant::now() + limit;
319 while Arc::strong_count(&replica) > 1 {
320 if tokio::time::Instant::now() >= deadline {
321 return Err(Error::Sync(format!(
322 "the replica of {} is still in use; try again",
323 root.display()
324 )));
325 }
326 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
327 }
328 Ok(())
329 }
330
331 /// Stop syncing `root`, handing back its replica — `None` when it was not synced — and
332 /// how telling the bridge went.
333 async fn stop(&self, root: &Path) -> (Option<Arc<Mutex<Replica>>>, Result<()>) {
334 // A root that no longer exists cannot be canonicalized, but the keys of `synced` are
335 // canonical already, so the path as given (a registry root) still names the entry.
336 let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
337 let removed = self.synced.lock().await.remove(&key);
338 if removed.is_some()
339 && let Some(watcher) = self.watcher.get()
340 {
341 watcher.unwatch(&key);
342 }
343 // Any live session for a workspace this server no longer holds is closed: it would
344 // otherwise keep pushing files for a workspace nobody asked about any more.
345 if let Some(peers) = self.live.lock().await.remove(&key) {
346 peers.drop_connections().await;
347 }
348 let Some(removed) = removed else {
349 return (None, Ok(()));
350 };
351 let unregistered = async {
352 self.bridge
353 .unregister(removed.workspace_id)
354 .await
355 .map_err(|e| Error::Bridge(e.to_string()))?;
356 self.reregister().await
357 }
358 .await;
359 (Some(removed.replica), unregistered)
360 }
361
362 /// The workspaces the workgroup knows about.
363 ///
364 /// Read straight from the bridge, so it reflects every device's registrations. The
365 /// bridge being down is not an empty answer — that would read as "nothing to map" — so
366 /// it is an error, and `sync.map` reports it.
367 pub async fn workspaces(&self) -> Result<WorkspacesResult> {
368 self.bridge
369 .workspaces()
370 .await
371 .map_err(|e| Error::Bridge(e.to_string()))
372 }
373
374 /// Place a workgroup workspace at a directory of this host: write its identity and
375 /// map there, returning the canonical root. [`Current::map`](crate::Current::map)
376 /// then selects it and syncs it.
377 ///
378 /// `selector` is the workspace's name or id, as [`workspaces`] lists it; `dir` must
379 /// already be a workspace of this application — the directory is placed, not made.
380 ///
381 /// [`workspaces`]: SyncRuntime::workspaces
382 pub async fn place(&self, selector: &str, dir: &Path) -> Result<PathBuf> {
383 let workgroup = self.workspaces().await?;
384 let wanted = workgroup
385 .workspaces
386 .iter()
387 // A name is the user-facing handle; the id is the exact one. Two workspaces
388 // may not share a name, so a name names at most one.
389 .find(|w| w.name == selector || w.workspace_id.to_string() == selector)
390 .ok_or_else(|| Error::UnknownWorkspaceName(selector.to_owned()))?;
391
392 // Refuse another application's workspace: this server syncs its own application's
393 // workspaces, and `Workspace::from_root` would later refuse the directory anyway.
394 if wanted.app_name != self.ctx.app_name {
395 return Err(Error::WrongApp {
396 name: wanted.name.clone(),
397 app_name: wanted.app_name.clone(),
398 });
399 }
400
401 let root = dir.canonicalize().map_err(Error::Io)?;
402 let workspace = Workspace::from_root(self.ctx, &root)?;
403 if workspace.root != root {
404 return Err(Error::UnknownWorkspace(root, self.ctx.app_name));
405 }
406
407 // A directory already syncing under another identity is not remapped: the sync id
408 // is a device's word that it is the same workspace, and rewriting it silently
409 // would make two hosts disagree about what they share.
410 {
411 let synced = self.synced.lock().await;
412 if let Some(entry) = synced.get(&root)
413 && entry.workspace_id != wanted.workspace_id
414 {
415 return Err(Error::SyncId(format!(
416 "{} already syncs as {}, not {}",
417 root.display(),
418 entry.workspace_id,
419 wanted.workspace_id
420 )));
421 }
422 }
423
424 // The identity arrives from the workgroup, not from this host: two hosts that map
425 // the same workspace must sync as one, so the id is the one the workgroup lists,
426 // written before `enable` reads it. A different id already on disk is refused
427 // above; an absent one is written here.
428 let marker = workspace.marker_dir();
429 let id_path = marker.join(crate::sync::id::SYNC_ID_FILE);
430 match std::fs::read_to_string(&id_path) {
431 Ok(existing) if existing.trim() != wanted.workspace_id.to_string() => {
432 return Err(Error::SyncId(format!(
433 "{} already holds identity {}, not {}",
434 root.display(),
435 existing.trim(),
436 wanted.workspace_id
437 )));
438 }
439 Err(_) => {
440 std::fs::write(&id_path, format!("{}\n", wanted.workspace_id))
441 .map_err(Error::Io)?;
442 }
443 Ok(_) => {}
444 }
445
446 // The map is what tells this application, on a later start, which workgroup
447 // workspace a directory is.
448 let map_path = root
449 .join(format!(".{}", self.ctx.app_name))
450 .join(WORKSPACE_MAP_FILE);
451 let workspace_id = wanted.workspace_id.to_string();
452 std::fs::write(&map_path, format!("{workspace_id}\n")).map_err(Error::Io)?;
453 Ok(root)
454 }
455
456 /// What `sync.status` answers with.
457 pub async fn status(&self, root: &Path) -> SyncStatus {
458 let bridge_available = self.bridge.status().await.is_ok();
459 let answer = self.bridge.peers().await.ok();
460 if let Some(answer) = &answer {
461 *self.last_peers.lock().await = Some(answer.clone());
462 }
463 let peers = answer
464 .as_ref()
465 .map_or(0, |p| p.peers.len().saturating_sub(1));
466 let Ok(key) = root.canonicalize() else {
467 return SyncStatus {
468 enabled: false,
469 workspace_id: None,
470 peers,
471 paused: None,
472 last_error: None,
473 bridge_available,
474 topology: proto::Topology::Mesh,
475 embedding: None,
476 };
477 };
478 let synced = self.synced.lock().await;
479 match synced.get(&key) {
480 Some(entry) => SyncStatus {
481 enabled: true,
482 workspace_id: Some(entry.workspace_id),
483 peers,
484 paused: entry.paused.map(|r| format!("{r:?}")),
485 last_error: entry.last_error.clone(),
486 bridge_available,
487 topology: topology::topology(
488 answer
489 .as_ref()
490 .and_then(|p| p.roles_for(entry.workspace_id)),
491 ),
492 embedding: self.embedding_progress(&key).await,
493 },
494 None => SyncStatus {
495 enabled: false,
496 workspace_id: None,
497 peers,
498 paused: None,
499 last_error: None,
500 bridge_available,
501 topology: proto::Topology::Mesh,
502 embedding: None,
503 },
504 }
505 }
506
507 /// How far `key`'s embedding has got, when this host embeds it. Reads the open
508 /// workspace's index; never opens one, so a status call stays cheap.
509 async fn embedding_progress(&self, key: &Path) -> Option<proto::EmbeddingProgress> {
510 let host = self.host.get()?;
511 let backend = host.open_backend(key)?;
512 let state = backend.state();
513 state.embedder()?;
514 let info = state.db_info().ok()?;
515 Some(proto::EmbeddingProgress {
516 vectors: info.vector_count,
517 pending: info.pending_count,
518 running: self.embed_running.lock().await.contains(key),
519 })
520 }
521
522 /// Bring the replica's view of the files up to date.
523 ///
524 /// Called straight after the app server's own writes, and by the watcher for everything
525 /// else. A scan that finds nothing is cheap; a scan that is skipped loses an edit until
526 /// the next one.
527 pub async fn scan(&self, root: &Path) -> Result<()> {
528 let Ok(key) = root.canonicalize() else {
529 return Ok(());
530 };
531 let replica = {
532 let synced = self.synced.lock().await;
533 match synced.get(&key) {
534 Some(entry) => Arc::clone(&entry.replica),
535 None => return Ok(()),
536 }
537 };
538 let (outcome, updates) = {
539 let mut replica = replica.lock().await;
540 let outcome = replica.scan().map_err(|e| Error::Sync(e.to_string()))?;
541 // What the scan committed, as path updates, so it can go out on the sessions
542 // that are already open. This is the whole point of the exact-path scan in
543 // `handlers.rs`: a write through the server reaches a peer live, not at the next
544 // dial.
545 let updates = match &outcome {
546 ScanOutcome::Scanned(report) => {
547 // The unit of replication is the path state (design doc section 2.4):
548 // the `seen` a peer prunes against is the *path's* version vector - the
549 // context the recorded entry was built on plus its own dot - never the
550 // whole replica vv. Sending the global vv here made receivers prune
551 // causally-related versions they still held: a conflict copy on the
552 // first remote edit, and tombstones pruned so deletes stopped
553 // propagating (issue #157).
554 report
555 .recorded
556 .iter()
557 .map(|entry| PathUpdate {
558 path: entry.path.clone(),
559 versions: vec![entry.clone()],
560 seen: {
561 let mut seen = entry.context.clone();
562 seen.add_dot(&entry.dot);
563 seen
564 },
565 })
566 .collect()
567 }
568 ScanOutcome::Paused(_) => Vec::new(),
569 };
570 (outcome, updates)
571 };
572 {
573 let mut synced = self.synced.lock().await;
574 if let Some(entry) = synced.get_mut(&key) {
575 entry.paused = match outcome {
576 ScanOutcome::Paused(reason) => Some(reason),
577 ScanOutcome::Scanned(_) => None,
578 };
579 }
580 }
581 // After the map is unlocked: a push awaits every peer, and the table must not be
582 // held behind a slow one.
583 self.after_commit(&key, updates).await;
584 Ok(())
585 }
586
587 /// Bring the search index up to date with files a session has written.
588 ///
589 /// A replication session moves files, not index rows: the index belongs to the app
590 /// server, and the sync core is deliberately free of it. So the side that *applied* a
591 /// change re-indexes, which is what makes a file that arrived from a peer searchable
592 /// without a manual reindex — the claim the whole architecture rests on.
593 ///
594 /// Incremental, not a full rebuild: the workspace's mtime/size stamps mean only what
595 /// actually changed is read back. A file the session wrote has a new stamp, so it is
596 /// picked up; one that was already indexed and untouched is skipped.
597 ///
598 /// Best-effort by contract: the caller's session has already succeeded, and a failure
599 /// here must not be reported as the session's. It is logged and the next session, local
600 /// edit or explicit `workspace.reindex` tries again.
601 async fn reindex(&self, root: &Path) {
602 let Some(host) = self.host.get() else {
603 return;
604 };
605 let host = Arc::clone(host);
606 let root = root.to_owned();
607 // One sweep at a time: two sessions that finish together would otherwise both walk
608 // the workspace through the same retrieve store.
609 let _serialised = self.reindexing.lock().await;
610 // Opening the workspace is itself blocking work, and it is where the exclusive
611 // retrieve-store lock is taken, so it happens here rather than inside the sweep.
612 let backend = match host.backend_at(&root).await {
613 Ok(backend) => backend,
614 Err(err) => {
615 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
616 return;
617 }
618 };
619 let state = Arc::clone(backend.state());
620 let result = tokio::task::spawn_blocking(move || state.sync_retrieve()).await;
621 match result {
622 Ok(Ok(report)) => {
623 tracing::debug!(root = %root.display(), ?report, "re-indexed after a session");
624 }
625 Ok(Err(err)) => {
626 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
627 }
628 Err(err) => {
629 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
630 }
631 }
632 }
633
634 /// Run an embedding pass over `root` in the background, after the pass already
635 /// waiting for it if there is one (#187).
636 ///
637 /// The pass reads the vector files that arrived, and computes the missing ones this
638 /// host is responsible for: the files whose winning version this replica wrote, and —
639 /// on the primary device — every other one. It never holds up the caller.
640 pub fn request_embed(self: &Arc<Self>, root: &Path) {
641 let Ok(key) = root.canonicalize() else {
642 return;
643 };
644 let runtime = Arc::clone(self);
645 tokio::spawn(async move {
646 if !runtime.embed_wanted.lock().await.insert(key.clone()) {
647 return;
648 }
649 let _one_at_a_time = runtime.embedding.lock().await;
650 // Taken off the list as the pass starts: a change during the pass asks again.
651 runtime.embed_wanted.lock().await.remove(&key);
652 runtime.embed_running.lock().await.insert(key.clone());
653 runtime.embed_pass(&key).await;
654 runtime.embed_running.lock().await.remove(&key);
655 });
656 }
657
658 /// One embedding pass over `key`. Best-effort, like the re-index: logged, never fatal.
659 async fn embed_pass(&self, key: &Path) {
660 let Some(host) = self.host.get() else {
661 return;
662 };
663 let (workspace_id, replica) = {
664 let synced = self.synced.lock().await;
665 match synced.get(key) {
666 Some(entry) => (entry.workspace_id, Arc::clone(&entry.replica)),
667 None => return,
668 }
669 };
670 let backend = match host.backend_at(key).await {
671 Ok(backend) => backend,
672 Err(err) => {
673 tracing::warn!(root = %key.display(), "embedding pass: {err}");
674 return;
675 }
676 };
677 let state = Arc::clone(backend.state());
678 if let Err(err) = state
679 .refresh_embedder_at(self.bridge_endpoint.clone())
680 .await
681 {
682 tracing::warn!(root = %key.display(), "embedding pass: {err}");
683 return;
684 }
685 if state.embedder().is_none() {
686 return;
687 }
688 let primary = self.is_primary(workspace_id).await;
689 // The replica's view, read once: which content each path has, and which of those
690 // versions this replica wrote.
691 let grace_ms = self.backfill_timing().grace.as_millis() as u64;
692 let now_ms = std::time::SystemTime::now()
693 .duration_since(std::time::UNIX_EPOCH)
694 .map_or(0, |d| d.as_millis() as u64);
695 let (mine, settled, live) = {
696 let replica = replica.lock().await;
697 let me = replica.replica_id();
698 let states = match replica.states() {
699 Ok(states) => states,
700 Err(err) => {
701 tracing::warn!(root = %key.display(), "embedding pass: {err}");
702 return;
703 }
704 };
705 let mut mine = HashMap::new();
706 // Other devices' versions old enough for the primary to fill in.
707 let mut settled = HashMap::new();
708 let mut live = std::collections::HashSet::new();
709 for (path, path_state) in states {
710 let winner = path_state.winner();
711 let Some(hash) = winner.content.hash() else {
712 continue;
713 };
714 let hash = hash.to_hex();
715 if winner.dot.replica == me {
716 mine.insert(path, hash.clone());
717 } else if now_ms.saturating_sub(winner.hlc.wall_ms) >= grace_ms {
718 settled.insert(path, hash.clone());
719 }
720 live.insert(hash);
721 }
722 (mine, settled, live)
723 };
724 let root = key.to_owned();
725 let result = tokio::task::spawn_blocking(move || {
726 // The author embeds its own versions at once; the primary device fills in other
727 // devices' versions once they have settled, racing nobody.
728 let policy = |rel: &str, hash: &str| {
729 mine.get(rel).is_some_and(|h| h.as_str() == hash)
730 || (primary && settled.get(rel).is_some_and(|h| h.as_str() == hash))
731 };
732 let embedded = state.embed_pending_with(&policy, PASS_BATCH)?;
733 let removed = if primary {
734 state.remove_stale_vectors(&live, STALE_VECTOR_AGE)
735 } else {
736 0
737 };
738 Ok::<_, sapphire_workspace::Error>((embedded, removed))
739 })
740 .await;
741 match result {
742 Ok(Ok((embedded, removed))) if embedded > 0 || removed > 0 => tracing::info!(
743 root = %root.display(),
744 embedded,
745 removed,
746 "embedding pass"
747 ),
748 Ok(Ok(_)) => {}
749 Ok(Err(err)) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
750 Err(err) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
751 }
752 }
753
754 /// Open a session with every peer that will take one.
755 pub async fn sync_now(self: &Arc<Self>, root: &Path) -> Result<()> {
756 let Ok(key) = root.canonicalize() else {
757 return Ok(());
758 };
759 let workspace_id = {
760 let synced = self.synced.lock().await;
761 match synced.get(&key) {
762 Some(entry) => entry.workspace_id,
763 None => return Ok(()),
764 }
765 };
766 let peers = self
767 .bridge
768 .peers()
769 .await
770 .map_err(|e| Error::Bridge(e.to_string()))?;
771 *self.last_peers.lock().await = Some(peers.clone());
772 let me = self.device_id().await?;
773
774 // The same direction rule the dial loop follows: a host only dials peers with a
775 // greater device id. Dialling the other way from here would reintroduce the
776 // dial-both-ways deadlock the loop was cured of, just on a different trigger.
777 // Inside a star, a pair with neither hub in it does not link at all.
778 let roles = peers.roles_for(workspace_id).cloned();
779 // Every peer at once, so one that is offline costs the others nothing (#228). On
780 // this task, not spawned: cancelling the caller — the server's shutdown aborting the
781 // watcher — must drop every dial with it, since each one pins the runtime.
782 let mut dials: Vec<Dial<'_>> = Vec::new();
783 for peer in peers
784 .peers
785 .iter()
786 .filter(|p| topology::link(me, p.device_id, roles.as_ref()) == topology::Link::Dial)
787 .cloned()
788 .collect::<Vec<_>>()
789 {
790 // A peer that already has a live session is caught up by it: a second exchange
791 // on the same workspace would be redundant, and a second stream between the same
792 // pair is exactly what the tie-break cannot settle.
793 if let Some(live) = self.live_peers(&key).await
794 && live.has(&peer.device_id).await
795 {
796 continue;
797 }
798 // One dial per peer at a time. `sync_now` runs beside the dial loop, and two
799 // overlapping outbound dials to one peer are the one case the tie-break rule
800 // cannot resolve — each end would see its own stream as `outbound`. Whoever
801 // arrives second simply leaves it to the next pass. The table is made here if
802 // the dial loop has not yet: the first dial after enabling must not depend on
803 // a tick that has not fired.
804 let Some((_, table)) = self.live_peers_or_create(&key).await else {
805 continue;
806 };
807 if !table.begin_dial(peer.device_id).await {
808 continue;
809 }
810 let key = &key;
811 dials.push(Box::pin(async move {
812 // A peer that does not host this workspace refuses, which is normal and cheap.
813 let result = self
814 .sync_one(key, &peer.device_id, peer.name.clone(), workspace_id)
815 .await;
816 table.end_dial(peer.device_id).await;
817 if let Err(err) = result {
818 tracing::debug!(peer = %peer.name, "no session: {err}");
819 }
820 }));
821 }
822 join_all(dials).await;
823 Ok(())
824 }
825
826 /// Dial one peer and adopt the stream as a live session.
827 ///
828 /// The one-shot path (`run_session`) would close the stream the moment the exchange
829 /// finished — while the accepting peer keeps its half open and pushes into it, a session
830 /// that dies with nothing reading it. So the dialled side is adopted live too, and both
831 /// ends of a pair agree that a session stays open.
832 async fn sync_one(
833 self: &Arc<Self>,
834 key: &Path,
835 device: &GrainId,
836 name: String,
837 workspace_id: GrainId,
838 ) -> Result<()> {
839 let stream = self.open_stream(workspace_id, *device).await?;
840 self.adopt_live_session(stream, key, *device, workspace_id)
841 .await
842 .map_err(|err| {
843 tracing::warn!(peer = %name, "a live session could not be started: {err}");
844 err
845 })
846 }
847
848 /// Ask the bridge for a stream to `device`, giving up after `open_timeout`.
849 async fn open_stream(
850 &self,
851 workspace_id: GrainId,
852 device: GrainId,
853 ) -> Result<sapphire_ipc::RawStream> {
854 match tokio::time::timeout(
855 self.open_timeout,
856 self.bridge.open_stream(workspace_id, device),
857 )
858 .await
859 {
860 Ok(opened) => Ok(opened?),
861 Err(_) => Err(Error::Bridge(format!(
862 "no stream to {device} within {} s; it may be offline",
863 self.open_timeout.as_secs_f32()
864 ))),
865 }
866 }
867
868 /// Answer the bridge's announcements until the connection closes.
869 pub async fn run(self: Arc<Self>) -> Result<()> {
870 let mut incoming = self.bridge.incoming();
871 loop {
872 let announcement = match incoming.recv().await {
873 Ok(a) => a,
874 Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
875 tracing::warn!(missed = n, "fell behind on bridge announcements");
876 continue;
877 }
878 Err(tokio::sync::broadcast::error::RecvError::Closed) => return Ok(()),
879 };
880
881 let found = {
882 let synced = self.synced.lock().await;
883 synced
884 .iter()
885 .find(|(_, s)| s.workspace_id == announcement.workspace_id)
886 .map(|(root, s)| (root.clone(), Arc::clone(&s.replica)))
887 };
888 // `replica` is still what the accepted session shares, and `root` is where its
889 // files and its index live.
890 let Some((root, replica)) = found else {
891 // The bridge routed to us for a workspace we no longer hold. Not fatal:
892 // ignore it and let the ticket expire.
893 tracing::debug!(
894 workspace = %announcement.workspace_id,
895 "an announcement for a workspace this server does not hold"
896 );
897 continue;
898 };
899
900 let bridge = Arc::clone(&self.bridge);
901 let workspace_id = announcement.workspace_id;
902 let peer = announcement.peer_device_id;
903 let driver = Arc::clone(&self);
904 tokio::spawn(async move {
905 let stream = match bridge.accept_stream(announcement.ticket).await {
906 Ok(stream) => stream,
907 Err(err) => {
908 tracing::warn!("could not claim an announced stream: {err}");
909 return;
910 }
911 };
912 if driver.should_skip(workspace_id, peer).await {
913 tracing::debug!(%peer, "dropped an inbound session: both ends are outside this workspace's star");
914 return;
915 }
916 // Live, not one-shot: the peer that dialled is keeping this session open and
917 // pushing what it commits, so this side has to keep reading. A session that
918 // closed after the exchange would leave the dialler pushing into a stream
919 // nobody reads.
920 let (_, session) = match sapphire_framework_session::open_live_session(
921 stream,
922 Arc::clone(&replica),
923 workspace_id,
924 )
925 .await
926 {
927 Ok(started) => started,
928 Err(err) => {
929 tracing::warn!("an inbound session failed: {err}");
930 return;
931 }
932 };
933 let Some((key, table)) = driver.live_peers_or_create(&root).await else {
934 return;
935 };
936 // The peer dialled: this stream survives only if the *peer* has the lower
937 // device id. A stream that loses is dropped, which closes it, and the
938 // connection ends tidily rather than being left half-read.
939 let Some(updates) = table.insert(peer, session, false).await else {
940 return;
941 };
942 // This is the receiving side: the files are on disk now and nobody else will
943 // index them.
944 driver.reindex(&key).await;
945 driver.request_embed(&key);
946 driver.spawn_reader(table, updates, key, peer);
947 });
948 }
949 }
950
951 /// Every synced root, for the watcher.
952 pub async fn roots(&self) -> Vec<PathBuf> {
953 self.synced.lock().await.keys().cloned().collect()
954 }
955
956 /// The live session table for `root`, if this runtime syncs it.
957 ///
958 /// `None` is not an error: a dial, a push or a drop for a workspace this server does not
959 /// hold is a no-op, and every caller below treats it that way.
960 async fn live_peers(&self, root: &Path) -> Option<Arc<LivePeers>> {
961 let key = root.canonicalize().ok()?;
962 self.live.lock().await.get(&key).cloned()
963 }
964
965 /// The live session table for `root`, creating it if this runtime syncs it.
966 ///
967 /// The two locks are taken one after the other, never nested: `synced` to decide whether
968 /// the workspace is ours at all, then `live` to get or make its table.
969 async fn live_peers_or_create(&self, root: &Path) -> Option<(PathBuf, Arc<LivePeers>)> {
970 let key = root.canonicalize().ok()?;
971 if !self.synced.lock().await.contains_key(&key) {
972 return None;
973 }
974 // A table is built by whoever walks the workspace first, and it needs this host's
975 // device id — the tie-break between two simultaneous streams is decided from it.
976 let me = self.device_id().await.ok()?;
977 let mut live = self.live.lock().await;
978 let peers = Arc::clone(
979 live.entry(key.clone())
980 .or_insert_with(|| Arc::new(LivePeers::new(me))),
981 );
982 Some((key, peers))
983 }
984
985 /// Push `updates` to every open session for `root`.
986 ///
987 /// Called with what a scan recorded. `from` is always `None` here: a local commit came
988 /// from this host, so no session is excluded. The forwarding path, which does name a
989 /// source, is [`LivePeers::fan_out`](live::LivePeers::fan_out) called directly by the
990 /// reader side of a session.
991 ///
992 /// Best-effort by contract: the commit already happened, and a peer that is gone is the
993 /// dialer's problem, not the writer's.
994 pub async fn after_commit(&self, root: &Path, updates: Vec<PathUpdate>) {
995 if updates.is_empty() {
996 return;
997 }
998 if let Some(peers) = self.live_peers(root).await {
999 peers.fan_out(&updates, None).await;
1000 }
1001 }
1002
1003 /// The devices with an open live session for `root`.
1004 ///
1005 /// Used by tests to wait for the dialer; empty for a workspace this server does not hold.
1006 pub async fn live_session_devices(&self, root: &Path) -> Vec<GrainId> {
1007 match self.live_peers(root).await {
1008 Some(peers) => peers.devices().await,
1009 None => Vec::new(),
1010 }
1011 }
1012
1013 /// End the first dials [`enable`](SyncRuntime::enable) left running, waiting until
1014 /// they are gone.
1015 ///
1016 /// For the app server's shutdown: each one pins the runtime, and with it the replica
1017 /// stores, which are redb databases that stay locked — a dial outliving its server
1018 /// would lock the next one out.
1019 pub async fn stop_dials(&self) {
1020 let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1021 dials.abort_all();
1022 while dials.join_next().await.is_some() {}
1023 }
1024
1025 /// Wait for the first dials [`enable`](SyncRuntime::enable) left running to finish on
1026 /// their own: for a test that must not race them.
1027 #[cfg(test)]
1028 async fn settle_dials(&self) {
1029 let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1030 while dials.join_next().await.is_some() {}
1031 }
1032
1033 /// Close every open live session, as if the connections had been cut.
1034 ///
1035 /// Called for a workspace being disabled, and by tests that want to watch the dialer
1036 /// rebuild what was dropped.
1037 pub async fn drop_connections(&self) {
1038 let tables: Vec<Arc<LivePeers>> = self.live.lock().await.values().cloned().collect();
1039 for peers in tables {
1040 peers.drop_connections().await;
1041 }
1042 }
1043
1044 /// Keep a live session open to every peer, for every synced workspace.
1045 ///
1046 /// One walk per [`DIAL_INTERVAL`](live::DIAL_INTERVAL): for each synced workspace, ask
1047 /// the bridge who is in the workgroup, and open a session to every peer that is connected
1048 /// and does not already have one.
1049 ///
1050 /// A peer that cannot be reached is retried on a per-peer exponential backoff capped at
1051 /// [`DIAL_BACKOFF_MAX`] — one failed attempt per walk, not one per spin — and the counter
1052 /// resets when a session opens. An unreachable peer is therefore no obstacle to the
1053 /// others: its dial fails, the walk moves on, and the next workspace is dialled.
1054 ///
1055 /// Runs until the task is aborted, which is what the app server's shutdown does with it.
1056 /// It is allowed to fail like [`run`](SyncRuntime::run): only sync stops, and the app
1057 /// server keeps serving files (spec §10).
1058 pub async fn dial_loop(self: Arc<Self>) -> Result<()> {
1059 let mut tick = tokio::time::interval(live::DIAL_INTERVAL);
1060 // A missed tick is a late walk, not a burst of them: the interval is a pace, not a
1061 // quota to make up.
1062 tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1063 let mut waiting: HashMap<GrainId, std::time::Instant> = HashMap::new();
1064 let mut failures: HashMap<GrainId, u32> = HashMap::new();
1065 // The workspaces this host was the primary device of at the last walk.
1066 let mut primary_of: std::collections::HashSet<GrainId> = Default::default();
1067 // When the primary device last ran a backfill pass with nothing prompting it.
1068 let mut last_backfill = std::time::Instant::now();
1069 loop {
1070 tick.tick().await;
1071 let roots = self.roots().await;
1072 if roots.is_empty() {
1073 continue;
1074 }
1075 let peers = match self.bridge.peers().await {
1076 Ok(peers) => {
1077 *self.last_peers.lock().await = Some(peers.clone());
1078 peers
1079 }
1080 // The bridge is down or has no workgroup: nothing to dial, and the next walk
1081 // asks again. Not an error for the caller — the app server is still serving.
1082 Err(err) => {
1083 tracing::debug!("no peers to dial: {err}");
1084 continue;
1085 }
1086 };
1087 let me = match self.device_id().await {
1088 Ok(me) => me,
1089 Err(err) => {
1090 tracing::warn!("no device id, so nobody to dial: {err}");
1091 continue;
1092 }
1093 };
1094 // A host that has just become a workspace's primary device owes it the vectors
1095 // nobody else computes (#187): files that arrived before the election settled
1096 // would otherwise wait for the next change.
1097 let ids: Vec<(PathBuf, GrainId)> = self
1098 .synced
1099 .lock()
1100 .await
1101 .iter()
1102 .map(|(root, entry)| (root.clone(), entry.workspace_id))
1103 .collect();
1104 // And every so often anyway: an author that never embedded leaves files only the
1105 // primary will fill in, and nothing else would prompt it (#188).
1106 let backfill_due = last_backfill.elapsed() >= self.backfill_timing().interval;
1107 if backfill_due {
1108 last_backfill = std::time::Instant::now();
1109 }
1110 for (root, workspace_id) in ids {
1111 let primary = peers
1112 .roles_for(workspace_id)
1113 .is_some_and(|r| r.primary == Some(me));
1114 if !primary {
1115 primary_of.remove(&workspace_id);
1116 } else if primary_of.insert(workspace_id) || backfill_due {
1117 self.request_embed(&root);
1118 }
1119 }
1120 let now = std::time::Instant::now();
1121 // Dial only peers whose device id is greater than ours. Every device dialling
1122 // every other can deadlock: two hosts that dial each other at once each hold
1123 // their replica's lock across their own exchange and wait for the other's
1124 // hello, which the other's dialler holds the lock for. With one direction per
1125 // pair the wait chain is a DAG — a higher-id host never dials, so it never
1126 // holds its lock out while a lower-id host's exchange needs it. The lower host
1127 // dials within `DIAL_INTERVAL`, and the exchange is symmetric, so nothing is
1128 // lost; the tie-break is left as the safety net it is.
1129 for root in roots {
1130 let Some((key, table)) = self.live_peers_or_create(&root).await else {
1131 continue;
1132 };
1133 let workspace_id = {
1134 let synced = self.synced.lock().await;
1135 match synced.get(&key) {
1136 Some(entry) => entry.workspace_id,
1137 None => continue,
1138 }
1139 };
1140 let roles = peers.roles_for(workspace_id);
1141 // Entering a star closes what it no longer needs. The hubs carry it all —
1142 // once one of them is reachable from here (#192).
1143 if topology::may_close_skipped(me, roles, &table.devices().await) {
1144 table
1145 .retain(|d| topology::link(me, *d, roles) != topology::Link::Skip)
1146 .await;
1147 }
1148 for peer in peers.peers.iter().filter(|p| {
1149 p.connected && topology::link(me, p.device_id, roles) == topology::Link::Dial
1150 }) {
1151 let device = peer.device_id;
1152 if table.has(&device).await {
1153 // Already talking; a redial would drop a working session.
1154 failures.remove(&device);
1155 waiting.remove(&device);
1156 continue;
1157 }
1158 if let Some(until) = waiting.get(&device)
1159 && now < *until
1160 {
1161 // Backing off after a failure; the walk skips it until then.
1162 continue;
1163 }
1164 // The same claim `sync_now` takes: this walk and it can overlap, and a
1165 // double dial in one direction is the one race the tie-break cannot
1166 // settle. Skipping is free — the next pass, or `sync_now` itself, dials.
1167 if !table.begin_dial(device).await {
1168 continue;
1169 }
1170 let opened = self.open_stream(workspace_id, device).await;
1171 if opened.is_err() {
1172 table.end_dial(device).await;
1173 }
1174 match opened {
1175 Ok(stream) => {
1176 let adopted = self
1177 .adopt_live_session(stream, &root, device, workspace_id)
1178 .await;
1179 // The claim is out either way: the attempt is over, and a
1180 // session that died mid-exchange is re-dialled next pass.
1181 table.end_dial(device).await;
1182 match adopted {
1183 Ok(()) => {
1184 failures.remove(&device);
1185 waiting.remove(&device);
1186 }
1187 Err(err) => {
1188 tracing::debug!(
1189 peer = %peer.name,
1190 err = %err,
1191 "a live session could not be started"
1192 );
1193 let count = failures.entry(device).or_insert(0);
1194 *count = count.saturating_add(1);
1195 waiting.insert(device, now + dial_backoff(*count));
1196 }
1197 }
1198 }
1199 Err(err) => {
1200 // Normal and cheap: a peer that does not host this workspace
1201 // refuses, and one that is unreachable cannot be opened.
1202 tracing::debug!("no stream to a peer: {err}");
1203 let count = failures.entry(device).or_insert(0);
1204 *count = count.saturating_add(1);
1205 waiting.insert(device, now + dial_backoff(*count));
1206 }
1207 }
1208 }
1209 }
1210 }
1211 }
1212
1213 /// Take ownership of an opened stream: run the exchange, then keep it live.
1214 ///
1215 /// The session is stored under `device` and its reader pumped until the stream ends. A
1216 /// half-open or refused stream is not stored; the dialer retries it with backoff.
1217 async fn adopt_live_session(
1218 self: &Arc<Self>,
1219 stream: sapphire_ipc::RawStream,
1220 root: &Path,
1221 device: GrainId,
1222 workspace_id: GrainId,
1223 ) -> Result<()> {
1224 let replica = self.replica_of(root).await?;
1225 let (_, session) = sapphire_framework_session::open_live_session(
1226 stream,
1227 Arc::clone(&replica),
1228 workspace_id,
1229 )
1230 .await
1231 .map_err(|e| Error::Sync(e.to_string()))?;
1232 let Some((key, table)) = self.live_peers_or_create(root).await else {
1233 return Ok(());
1234 };
1235 // This stream was dialled by us, so it is the one the pair keeps if this host's
1236 // device id is the lower of the two. If it lost the tie-break the session is
1237 // dropped here, which closes the stream, and the surviving one is already in the
1238 // table.
1239 let Some(updates) = table.insert(device, session, true).await else {
1240 return Ok(());
1241 };
1242 // The session wrote files during the exchange; the index has to catch up before the
1243 // workspace is searchable again.
1244 self.reindex(&key).await;
1245 self.request_embed(&key);
1246 self.spawn_reader(Arc::clone(&table), updates, key, device);
1247 Ok(())
1248 }
1249
1250 /// Forward what one session receives, and re-index it.
1251 ///
1252 /// This is the loop that makes A → S → B work: a batch that arrived on S's session with A
1253 /// is applied (the session crate does that before publishing it), the workspace behind
1254 /// `key` is re-indexed, and the batch is handed to every *other* session whose peer lacks
1255 /// it — `from` is the device it came from, so it never goes back to A.
1256 fn spawn_reader(
1257 self: &Arc<Self>,
1258 table: Arc<LivePeers>,
1259 mut updates: broadcast::Receiver<Vec<PathUpdate>>,
1260 key: PathBuf,
1261 from: GrainId,
1262 ) {
1263 let driver = Arc::clone(self);
1264 tokio::spawn(async move {
1265 loop {
1266 match updates.recv().await {
1267 Ok(batch) => {
1268 // The files are on disk now (the session guarantees it before
1269 // publishing), so the index is behind them until the sweep below.
1270 driver.reindex(&key).await;
1271 driver.request_embed(&key);
1272 table.fan_out(&batch, Some(from)).await;
1273 }
1274 // A burst bigger than the channel: the newest is kept and the gap is
1275 // covered by the next session's exchange. Losing a batch here cannot lose
1276 // an entry — the peer's vector does not cover it, so the next dial or push
1277 // carries it again.
1278 Err(broadcast::error::RecvError::Lagged(missed)) => {
1279 tracing::debug!(
1280 device = %from,
1281 missed,
1282 "fell behind on a live session; the next exchange catches up"
1283 );
1284 }
1285 // The session closed. Its table entry is reaped by the next fan-out or
1286 // dial pass, and the dialer opens a fresh one.
1287 Err(broadcast::error::RecvError::Closed) => {
1288 return;
1289 }
1290 }
1291 }
1292 });
1293 }
1294
1295 /// The replica of a synced `root`.
1296 async fn replica_of(&self, root: &Path) -> Result<Arc<Mutex<Replica>>> {
1297 let key = root.canonicalize().map_err(Error::Io)?;
1298 let synced = self.synced.lock().await;
1299 match synced.get(&key) {
1300 Some(entry) => Ok(Arc::clone(&entry.replica)),
1301 None => Err(Error::UnknownWorkspace(key, self.ctx.app_name)),
1302 }
1303 }
1304
1305 /// Watch the synced roots and, on a debounced report, scan and dial.
1306 ///
1307 /// This is the safety net for everything the app server did not do itself — a file
1308 /// edited in an editor, a `git checkout`. The exact path is [`scan`] called from
1309 /// `handlers.rs` right after a write the server made. A scan that finds nothing is
1310 /// cheap; a scan that is skipped loses an edit until the next one.
1311 ///
1312 /// Returns when the reporting channel closes — that is, when the [`Watcher`] this
1313 /// runtime holds is dropped, which happens when the runtime itself is.
1314 ///
1315 /// [`scan`]: SyncRuntime::scan
1316 pub async fn watch_changes(self: Arc<Self>) -> Result<()> {
1317 let (tx, mut rx) = mpsc::channel(64);
1318 let watcher = Arc::new(Watcher::start(self.roots().await, tx)?);
1319 let _ = self.watcher.set(Arc::clone(&watcher));
1320
1321 // A root enabled between `roots()` above and the `set` just now was not in the
1322 // starting set, and `enable` could not reach a watcher that did not exist yet.
1323 // Watching the current set again is idempotent on `notify`'s side.
1324 for root in self.roots().await {
1325 if let Err(err) = watcher.watch(&root) {
1326 tracing::warn!(root = %root.display(), "could not watch: {err}");
1327 }
1328 }
1329
1330 while let Some(root) = rx.recv().await {
1331 // Logged, not fatal: a failed scan or dial leaves the replica behind, and the
1332 // next report tries again. Neither may take the app server down (spec §10).
1333 if let Err(err) = self.scan(&root).await {
1334 tracing::warn!(root = %root.display(), "scan after a local edit failed: {err}");
1335 }
1336 self.request_embed(&root);
1337 if let Err(err) = self.sync_now(&root).await {
1338 tracing::warn!(root = %root.display(), "dial after a local edit failed: {err}");
1339 }
1340 }
1341 Ok(())
1342 }
1343
1344 /// Tell the bridge the complete current set. A registration is not a delta.
1345 async fn reregister(&self) -> Result<()> {
1346 let workspaces: Vec<WorkspaceRegistration> = {
1347 let synced = self.synced.lock().await;
1348 synced
1349 .iter()
1350 .map(|(root, entry)| WorkspaceRegistration {
1351 workspace_id: entry.workspace_id,
1352 root: root.clone(),
1353 })
1354 .collect()
1355 };
1356 self.bridge
1357 .register(RegisterParams {
1358 app_name: self.ctx.app_name.to_owned(),
1359 exe_path: self.exe_path.clone(),
1360 managed_by: wire_managed_by(self.managed_by),
1361 workspaces,
1362 })
1363 .await
1364 .map_err(|e| Error::Bridge(e.to_string()))?;
1365 Ok(())
1366 }
1367
1368 /// Whether the last roles the bridge reported say this pair does not link.
1369 ///
1370 /// Reads the cached device id only: asking the bridge here would register with an empty
1371 /// workspace list, racing `enable`'s registration. `false` (accept) when it is unknown.
1372 async fn should_skip(&self, workspace_id: GrainId, peer: GrainId) -> bool {
1373 let Some(&me) = self.device_id.get() else {
1374 return false;
1375 };
1376 let cached = self.last_peers.lock().await;
1377 let roles = cached.as_ref().and_then(|p| p.roles_for(workspace_id));
1378 topology::link(me, peer, roles) == topology::Link::Skip
1379 }
1380
1381 /// Whether this host is `workspace_id`'s primary device, by the last roles the
1382 /// bridge reported. `false` when the bridge has not answered yet, or when this host's
1383 /// device id is not known yet (no workspace was enabled). Reads the cached id only:
1384 /// asking the bridge here would register with an empty workspace list, racing
1385 /// `enable`'s registration.
1386 pub async fn is_primary(&self, workspace_id: GrainId) -> bool {
1387 let Some(&me) = self.device_id.get() else {
1388 return false;
1389 };
1390 let cached = self.last_peers.lock().await;
1391 cached
1392 .as_ref()
1393 .and_then(|p| p.roles_for(workspace_id))
1394 .is_some_and(|r| r.primary == Some(me))
1395 }
1396
1397 /// This host's device id, asked of the bridge once.
1398 async fn device_id(&self) -> Result<GrainId> {
1399 if let Some(id) = self.device_id.get() {
1400 return Ok(*id);
1401 }
1402 let result = self
1403 .bridge
1404 .register(RegisterParams {
1405 app_name: self.ctx.app_name.to_owned(),
1406 exe_path: self.exe_path.clone(),
1407 managed_by: wire_managed_by(self.managed_by),
1408 workspaces: Vec::new(),
1409 })
1410 .await
1411 .map_err(|e| Error::Bridge(e.to_string()))?;
1412 let _ = self.device_id.set(result.device_id);
1413 Ok(result.device_id)
1414 }
1415
1416 /// `<workspace cache dir>/sync/`.
1417 fn state_dir(&self, root: &Path) -> Result<PathBuf> {
1418 let workspace = Workspace::from_root(self.ctx, root)?;
1419 Ok(workspace.cache_dir().join("sync"))
1420 }
1421}
1422
1423/// One dial of [`SyncRuntime::sync_now`].
1424type Dial<'a> = std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>>;
1425
1426/// Run `dials` concurrently on the calling task, until every one has finished.
1427async fn join_all(mut dials: Vec<Dial<'_>>) {
1428 std::future::poll_fn(|cx| {
1429 dials.retain_mut(|dial| dial.as_mut().poll(cx).is_pending());
1430 if dials.is_empty() {
1431 std::task::Poll::Ready(())
1432 } else {
1433 std::task::Poll::Pending
1434 }
1435 })
1436 .await
1437}
1438
1439/// The bridge API's copy of how this server was started, for [`RegisterParams`].
1440fn wire_managed_by(managed_by: ManagedBy) -> sapphire_bridge_api::ManagedBy {
1441 match managed_by {
1442 ManagedBy::Service => sapphire_bridge_api::ManagedBy::Service,
1443 ManagedBy::Spawned => sapphire_bridge_api::ManagedBy::Spawned,
1444 }
1445}
1446
1447#[cfg(test)]
1448mod tests {
1449 use super::*;
1450 use crate::sync::testing::StubBridge;
1451 use crate::test_support;
1452 use sapphire_workspace::AppKind;
1453 use std::ffi::OsString;
1454
1455 static CTX: AppContext = AppContext::new("sapphire-synctest");
1456
1457 /// The env vars `CTX.init` reads.
1458 const DIR_VARS: [&str; 3] = [
1459 "SAPPHIRE_SYNCTEST_CACHE_DIR",
1460 "SAPPHIRE_SYNCTEST_DATA_DIR",
1461 "SAPPHIRE_SYNCTEST_CONFIG_DIR",
1462 ];
1463
1464 /// Points the context's directories at the test's scratch tree, and restores the
1465 /// previous values when dropped — including while unwinding from a panic.
1466 struct EnvGuard {
1467 previous: [Option<OsString>; 3],
1468 _lock: std::sync::MutexGuard<'static, ()>,
1469 }
1470
1471 impl Drop for EnvGuard {
1472 fn drop(&mut self) {
1473 // SAFETY: `self._lock` still serialises the environment; it is dropped only
1474 // after this method returns.
1475 for (name, previous) in DIR_VARS.iter().zip(self.previous.iter_mut()) {
1476 match previous.take() {
1477 Some(value) => unsafe { std::env::set_var(name, value) },
1478 None => test_support::remove(name),
1479 }
1480 }
1481 }
1482 }
1483
1484 /// Everything one test needs, held for the test's whole body.
1485 ///
1486 /// The context is a `static` shared by this binary, so its directories are resolved from
1487 /// the environment by whichever test initialises it first. The environment lock is
1488 /// therefore held for the whole test, as in the crate's other test modules, and `_tmp`
1489 /// is declared before `_env` so the scratch tree is gone only once the environment no
1490 /// longer points at it.
1491 struct Fixture {
1492 _tmp: tempfile::TempDir,
1493 _env: EnvGuard,
1494 root: PathBuf,
1495 stub: StubBridge,
1496 runtime: Arc<SyncRuntime>,
1497 }
1498
1499 impl Fixture {
1500 /// A second workspace root beside the first, marker directory and all.
1501 fn second_root(&self) -> PathBuf {
1502 let root = self._tmp.path().join("ws2");
1503 std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
1504 root.canonicalize().unwrap()
1505 }
1506 }
1507
1508 /// How long the fixture's runtime waits on a stream to a peer.
1509 const FIXTURE_OPEN_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
1510
1511 /// `n` peers this host dials (a greater device id), as `bridge.peers` lists them.
1512 fn dialled_peers(me: GrainId, n: usize) -> Vec<sapphire_bridge_api::PeerInfo> {
1513 std::iter::repeat_with(GrainId::random)
1514 .filter(|id| *id > me)
1515 .take(n)
1516 .enumerate()
1517 .map(|(i, device_id)| sapphire_bridge_api::PeerInfo {
1518 device_id,
1519 name: format!("offline-{i}"),
1520 node_id: format!("node-{i}"),
1521 connected: false,
1522 priority: 1,
1523 availability: None,
1524 })
1525 .collect()
1526 }
1527
1528 #[tokio::test(flavor = "multi_thread")]
1529 async fn enabling_answers_at_once_with_a_peer_that_never_answers() {
1530 // #228: the first dial waited on a peer that was offline, past the client's limit.
1531 let f = fixture().await;
1532 let _hung = f.stub.hang_streams().await;
1533 f.stub.set_peers(dialled_peers(f.stub.device_id, 1));
1534
1535 let started = std::time::Instant::now();
1536 f.runtime.enable(&f.root).await.unwrap();
1537 assert!(
1538 started.elapsed() < FIXTURE_OPEN_TIMEOUT,
1539 "enable waited {:?} on the dial",
1540 started.elapsed()
1541 );
1542 }
1543
1544 #[tokio::test(flavor = "multi_thread")]
1545 async fn peers_that_never_answer_are_dialled_together_and_given_up_on() {
1546 let f = fixture().await;
1547 let _hung = f.stub.hang_streams().await;
1548 f.runtime.enable(&f.root).await.unwrap();
1549 // The first dial, with nobody to dial yet, must be over before the peers appear:
1550 // one still running would claim them, and `sync_now` would skip them all.
1551 f.runtime.settle_dials().await;
1552 f.stub.set_peers(dialled_peers(f.stub.device_id, 3));
1553
1554 let started = std::time::Instant::now();
1555 f.runtime.sync_now(&f.root).await.unwrap();
1556 let took = started.elapsed();
1557 assert!(took >= FIXTURE_OPEN_TIMEOUT, "gave up early: {took:?}");
1558 assert!(
1559 took < FIXTURE_OPEN_TIMEOUT * 2,
1560 "three dials one after another: {took:?}"
1561 );
1562 }
1563
1564 async fn fixture() -> Fixture {
1565 let lock = test_support::lock();
1566 let tmp = tempfile::tempdir().unwrap();
1567 let previous = DIR_VARS.map(std::env::var_os);
1568 // SAFETY (via `test_support::set`): `lock` serialises every read and write of the
1569 // process environment in this test binary, and it is held until `drop` has restored
1570 // the old values.
1571 for (name, dir) in DIR_VARS
1572 .iter()
1573 .zip(["cache", "data", "config"].map(|cat| tmp.path().join(cat)))
1574 {
1575 test_support::set(name, &dir);
1576 }
1577 let env = EnvGuard {
1578 previous,
1579 _lock: lock,
1580 };
1581 CTX.init(AppKind::Server);
1582
1583 let root = tmp.path().join("ws");
1584 std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
1585 let root = root.canonicalize().unwrap();
1586
1587 let (stub, client) = StubBridge::start_in(tmp.path().join("run")).await;
1588 let runtime = Arc::new(
1589 SyncRuntime::new(&CTX, client, "/bin/true".into(), ManagedBy::Service)
1590 .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
1591 );
1592 Fixture {
1593 _tmp: tmp,
1594 _env: env,
1595 root,
1596 stub,
1597 runtime,
1598 }
1599 }
1600
1601 #[tokio::test(flavor = "multi_thread")]
1602 async fn enabling_registers_the_workspace_with_the_bridge() {
1603 let f = fixture().await;
1604
1605 let id = f.runtime.enable(&f.root).await.unwrap();
1606 assert_eq!(f.stub.last_workspaces(), vec![id]);
1607 }
1608
1609 #[tokio::test(flavor = "multi_thread")]
1610 async fn enabling_twice_is_idempotent_and_keeps_the_same_id() {
1611 let f = fixture().await;
1612
1613 let first = f.runtime.enable(&f.root).await.unwrap();
1614 let second = f.runtime.enable(&f.root).await.unwrap();
1615 assert_eq!(first, second);
1616 assert_eq!(
1617 f.stub.last_workspaces(),
1618 vec![first],
1619 "still exactly one workspace"
1620 );
1621 }
1622
1623 #[tokio::test(flavor = "multi_thread")]
1624 async fn a_registration_carries_the_complete_current_list() {
1625 let f = fixture().await;
1626 let second_root = f.second_root();
1627
1628 let a = f.runtime.enable(&f.root).await.unwrap();
1629 let b = f.runtime.enable(&second_root).await.unwrap();
1630
1631 let mut seen = f.stub.last_workspaces();
1632 seen.sort();
1633 let mut want = vec![a, b];
1634 want.sort();
1635 assert_eq!(seen, want, "registration is the whole set, not a delta");
1636 }
1637
1638 #[tokio::test(flavor = "multi_thread")]
1639 async fn no_later_registration_clears_this_apps_routes() {
1640 // The device id is asked of the bridge once, on the first `enable`, with an empty
1641 // workspace list — the one moment that is safe, because no route exists yet. Every
1642 // later registration must name the complete set: one with an empty list would
1643 // briefly clear this application's routes, and a peer asking for a workspace this
1644 // server owns would be told nobody has it.
1645 let f = fixture().await;
1646 let second_root = f.second_root();
1647
1648 f.runtime.enable(&f.root).await.unwrap();
1649 f.runtime.enable(&second_root).await.unwrap();
1650 f.runtime.sync_now(&f.root).await.unwrap();
1651 f.runtime.disable(&f.root).await.unwrap();
1652
1653 let seen = f.stub.seen.lock().expect("stub");
1654 let first_full = seen
1655 .registrations
1656 .iter()
1657 .position(|r| !r.workspaces.is_empty())
1658 .expect("enabling registers the workspace");
1659 let emptied_afterwards = seen.registrations[first_full + 1..]
1660 .iter()
1661 .filter(|r| r.workspaces.is_empty())
1662 .count();
1663 assert_eq!(
1664 emptied_afterwards, 0,
1665 "a registration after the first workspace must still name the whole set"
1666 );
1667 }
1668
1669 #[tokio::test(flavor = "multi_thread")]
1670 async fn disabling_unregisters_and_leaves_the_files_alone() {
1671 let f = fixture().await;
1672 std::fs::write(f.root.join("keep.md"), "content").unwrap();
1673
1674 let id = f.runtime.enable(&f.root).await.unwrap();
1675 f.runtime.disable(&f.root).await.unwrap();
1676
1677 assert!(f.stub.last_workspaces().is_empty());
1678 assert!(
1679 f.root.join("keep.md").exists(),
1680 "disabling sync must not touch files"
1681 );
1682 assert!(
1683 crate::sync::id::sync_id_path("sapphire-synctest", &f.root).exists(),
1684 "the sync id stays, so re-enabling rejoins the same workspace"
1685 );
1686 let _ = id;
1687 }
1688
1689 #[tokio::test(flavor = "multi_thread")]
1690 async fn re_enabling_after_disabling_reuses_the_identity() {
1691 let f = fixture().await;
1692
1693 let first = f.runtime.enable(&f.root).await.unwrap();
1694 f.runtime.disable(&f.root).await.unwrap();
1695 let again = f.runtime.enable(&f.root).await.unwrap();
1696
1697 assert_eq!(first, again);
1698 }
1699
1700 #[tokio::test(flavor = "multi_thread")]
1701 async fn status_reports_disabled_for_a_workspace_that_was_never_enabled() {
1702 let f = fixture().await;
1703
1704 let status = f.runtime.status(&f.root).await;
1705 assert!(!status.enabled);
1706 assert!(status.workspace_id.is_none());
1707 }
1708
1709 #[tokio::test(flavor = "multi_thread")]
1710 async fn disabling_something_that_was_never_enabled_is_not_an_error() {
1711 let f = fixture().await;
1712 f.runtime
1713 .disable(&f.root)
1714 .await
1715 .expect("disabling twice is fine");
1716 }
1717
1718 #[tokio::test(flavor = "multi_thread")]
1719 async fn a_workspace_whose_root_vanished_is_reported_as_paused_not_deleted() {
1720 let f = fixture().await;
1721 std::fs::write(f.root.join("a.md"), "content").unwrap();
1722 f.runtime.enable(&f.root).await.unwrap();
1723 f.runtime.scan(&f.root).await.unwrap();
1724
1725 // The drive is unmounted: the marker directory goes with it.
1726 std::fs::remove_dir_all(f.root.join(".sapphire-synctest")).unwrap();
1727 let _ = f.runtime.scan(&f.root).await;
1728
1729 let status = f.runtime.status(&f.root).await;
1730 assert!(
1731 status.paused.is_some(),
1732 "a missing root must pause, not replicate as a mass deletion"
1733 );
1734 }
1735
1736 #[tokio::test(flavor = "multi_thread")]
1737 async fn a_directory_that_is_not_a_workspace_cannot_be_enabled() {
1738 let f = fixture().await;
1739 let plain = f._tmp.path().join("plain");
1740 std::fs::create_dir_all(&plain).unwrap();
1741
1742 assert!(f.runtime.enable(&plain).await.is_err());
1743 }
1744
1745 #[tokio::test(flavor = "multi_thread")]
1746 async fn a_write_through_the_workspace_router_is_committed_to_the_replica() {
1747 // The brief's "main path": a write the app server itself made is scanned straight
1748 // after, not left to the watcher's debounce. Asserted against the replica's own
1749 // state, which is the only thing that can tell the two paths apart — reading the
1750 // file back would pass even if the scan never ran.
1751 let f = fixture().await;
1752 f.runtime.enable(&f.root).await.unwrap();
1753
1754 let host = Arc::new(crate::WorkspaceHost::new(&CTX));
1755 host.select(&f.root, false).await.unwrap();
1756 let current = Arc::new(crate::Current::new(
1757 &CTX,
1758 host,
1759 Some(Arc::clone(&f.runtime)),
1760 crate::selection::SelectionFile::at(f._tmp.path().join("workspace.toml")),
1761 ));
1762 let router = Arc::new(crate::workspace_router(current));
1763 let (client_conn, server_conn) = sapphire_ipc::Connection::pair();
1764 tokio::spawn(async move {
1765 let info = sapphire_ipc::ServerInfo {
1766 version: "0.0.0".into(),
1767 api: sapphire_backend::protocol::API_VERSION,
1768 app_api: None,
1769 pid: std::process::id(),
1770 managed_by: ManagedBy::Spawned,
1771 };
1772 let _ = sapphire_ipc::serve(server_conn, router, "sapphire-synctest", info).await;
1773 });
1774 let info = sapphire_ipc::ClientInfo {
1775 kind: "test".into(),
1776 version: "0.0.0".into(),
1777 api: sapphire_backend::protocol::API_VERSION,
1778 app_api: None,
1779 pid: std::process::id(),
1780 };
1781 let (client, _) = sapphire_ipc::Client::handshake(client_conn, "sapphire-synctest", info)
1782 .await
1783 .unwrap();
1784
1785 let _: sapphire_backend::protocol::Ack = client
1786 .call(
1787 sapphire_backend::protocol::WRITE_FILE,
1788 sapphire_backend::protocol::ContentParams {
1789 path: PathBuf::from("through-the-server.md"),
1790 content: "written by the server".into(),
1791 },
1792 )
1793 .await
1794 .unwrap();
1795
1796 // `scan` runs before the reply is written, so by the time the call returns the
1797 // replica must already know the path.
1798 let synced = f.runtime.synced.lock().await;
1799 let entry = synced.get(&f.root).expect("enabled above");
1800 let replica = entry.replica.lock().await;
1801 let state = replica
1802 .state("through-the-server.md")
1803 .expect("a readable state");
1804 assert!(
1805 state.is_some(),
1806 "a write through the server must be scanned into the replica, not left to the watcher"
1807 );
1808 }
1809
1810 #[tokio::test(flavor = "multi_thread")]
1811 async fn an_announcement_for_an_unknown_workspace_is_ignored_not_fatal() {
1812 let f = fixture().await;
1813 f.runtime.enable(&f.root).await.unwrap();
1814
1815 let driver = Arc::clone(&f.runtime);
1816 let handle = tokio::spawn(async move { driver.run().await });
1817
1818 f.stub
1819 .announce(grain_id::GrainId::random(), "no-such-ticket")
1820 .await;
1821 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
1822
1823 assert!(
1824 !handle.is_finished(),
1825 "one bad announcement must not stop the runtime"
1826 );
1827 handle.abort();
1828 }
1829}