1use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use grain_id::GrainId;
13use sapphire_backend::protocol as proto;
14use sapphire_bridge_api::{
15 IncomingParams, RegisterParams, WorkspaceRegistration, WorkspacesResult,
16};
17use sapphire_bridge_client::BridgeClient;
18use sapphire_ipc::ManagedBy;
19use sapphire_sync::{PathUpdate, PauseReason, Replica, ReplicaConfig, ScanOutcome, SystemClock};
20use sapphire_workspace::{AppContext, Workspace};
21use tokio::sync::{Mutex, OnceCell, broadcast, mpsc};
22
23pub mod id;
24mod live;
25mod methods;
26#[cfg(any(test, feature = "test-util"))]
27pub mod testing;
28mod topology;
29mod watch;
30
31pub use id::{SYNC_ID_FILE, WORKSPACE_MAP_FILE, sync_id, sync_id_path};
32pub use live::{DIAL_BACKOFF_MAX, OPEN_TIMEOUT};
33pub use methods::sync_router;
34pub use sapphire_bridge_api::{PeerInfo, PeersResult};
36pub use watch::{DEBOUNCE, Watcher};
37
38use live::{LivePeers, dial_backoff};
39
40use crate::error::{Error, Result};
41
42use crate::host::WorkspaceHost;
43
44#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
46pub struct SyncStatus {
47 pub enabled: bool,
49 pub workspace_id: Option<GrainId>,
51 pub peers: usize,
53 pub paused: Option<String>,
55 pub last_error: Option<String>,
57 pub bridge_available: bool,
59 pub topology: proto::Topology,
61 pub embedding: Option<proto::EmbeddingProgress>,
63}
64
65struct Synced {
67 workspace_id: GrainId,
69 replica: Arc<Mutex<Replica>>,
71 paused: Option<PauseReason>,
73 last_error: Option<String>,
75}
76
77const STALE_VECTOR_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
80
81const PASS_BATCH: usize = 4;
84
85#[derive(Clone, Copy, Debug)]
87pub struct BackfillTiming {
88 pub grace: std::time::Duration,
91 pub interval: std::time::Duration,
93}
94
95impl Default for BackfillTiming {
96 fn default() -> Self {
97 BackfillTiming {
98 grace: std::time::Duration::from_secs(10 * 60),
99 interval: std::time::Duration::from_secs(10 * 60),
100 }
101 }
102}
103
104pub const RECONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_millis(500);
107
108pub const RECONNECT_BACKOFF_MAX: std::time::Duration = std::time::Duration::from_secs(30);
111
112fn reconnect_backoff(attempt: u32) -> std::time::Duration {
114 RECONNECT_BACKOFF
115 .saturating_mul(1u32 << attempt.min(16))
116 .min(RECONNECT_BACKOFF_MAX)
117}
118
119type Connecting = std::pin::Pin<
121 Box<dyn std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>> + Send>,
122>;
123
124#[derive(Clone)]
129pub struct BridgeConnector(Arc<dyn Fn() -> Connecting + Send + Sync>);
130
131impl BridgeConnector {
132 pub fn new<F, Fut>(connect: F) -> BridgeConnector
134 where
135 F: Fn() -> Fut + Send + Sync + 'static,
136 Fut: std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>>
137 + Send
138 + 'static,
139 {
140 BridgeConnector(Arc::new(move || Box::pin(connect())))
141 }
142
143 pub fn standard(kind: &'static str, version: &'static str) -> BridgeConnector {
146 BridgeConnector::new(move || async move {
147 match BridgeClient::connect(kind, version).await {
148 Ok(client) => Ok(Some(client)),
149 Err(sapphire_ipc::Error::NotRunning(_)) => Ok(None),
150 Err(err) => Err(err),
151 }
152 })
153 }
154
155 pub fn at(
157 endpoint: sapphire_ipc::Endpoint,
158 kind: &'static str,
159 version: &'static str,
160 ) -> BridgeConnector {
161 BridgeConnector::new(move || {
162 let endpoint = endpoint.clone();
163 async move { BridgeClient::connect_at(&endpoint, kind, version).await }
164 })
165 }
166
167 async fn connect(&self) -> sapphire_ipc::Result<Option<BridgeClient>> {
168 (self.0)().await
169 }
170}
171
172impl std::fmt::Debug for BridgeConnector {
173 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
174 f.write_str("BridgeConnector")
175 }
176}
177
178enum Reclaimed {
180 Open(Arc<Mutex<Replica>>),
182 Closing,
184 Closed,
186}
187
188const CLOSING_RETRIES: u32 = 50;
190const CLOSING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(20);
191
192pub struct SyncRuntime {
194 ctx: &'static AppContext,
195 bridge: tokio::sync::watch::Sender<Option<Arc<BridgeClient>>>,
198 connector: Option<BridgeConnector>,
200 pending: Mutex<HashMap<PathBuf, GrainId>>,
207 subscribed: std::sync::Mutex<Option<(Arc<BridgeClient>, broadcast::Receiver<IncomingParams>)>>,
212 exe_path: PathBuf,
213 managed_by: ManagedBy,
214 synced: Mutex<HashMap<PathBuf, Synced>>,
215 stopped: std::sync::Mutex<HashMap<PathBuf, std::sync::Weak<Mutex<Replica>>>>,
222 device_id: OnceCell<GrainId>,
228 watcher: OnceCell<Arc<Watcher>>,
238 host: OnceCell<Arc<WorkspaceHost>>,
249 reindexing: Mutex<()>,
252 embed_wanted: Mutex<std::collections::HashSet<PathBuf>>,
255 embedding: Mutex<()>,
257 embed_running: Mutex<std::collections::HashSet<PathBuf>>,
259 backfill: std::sync::Mutex<BackfillTiming>,
261 bridge_endpoint: Option<sapphire_ipc::Endpoint>,
263 live: Mutex<HashMap<PathBuf, Arc<LivePeers>>>,
269 last_peers: Mutex<Option<PeersResult>>,
272 open_timeout: std::time::Duration,
274 first_dials: std::sync::Mutex<tokio::task::JoinSet<()>>,
277 background: std::sync::Mutex<tokio::task::JoinSet<()>>,
282}
283
284impl SyncRuntime {
285 pub fn new(
290 ctx: &'static AppContext,
291 bridge: Arc<BridgeClient>,
292 exe_path: PathBuf,
293 managed_by: ManagedBy,
294 ) -> SyncRuntime {
295 SyncRuntime::build(ctx, Some(bridge), None, exe_path, managed_by)
296 }
297
298 pub fn connecting(
306 ctx: &'static AppContext,
307 connector: BridgeConnector,
308 exe_path: PathBuf,
309 managed_by: ManagedBy,
310 ) -> SyncRuntime {
311 SyncRuntime::build(ctx, None, Some(connector), exe_path, managed_by)
312 }
313
314 pub fn with_connector(mut self, connector: BridgeConnector) -> SyncRuntime {
316 self.connector = Some(connector);
317 self
318 }
319
320 fn build(
321 ctx: &'static AppContext,
322 bridge: Option<Arc<BridgeClient>>,
323 connector: Option<BridgeConnector>,
324 exe_path: PathBuf,
325 managed_by: ManagedBy,
326 ) -> SyncRuntime {
327 SyncRuntime {
328 ctx,
329 bridge: tokio::sync::watch::Sender::new(bridge),
330 connector,
331 pending: Mutex::new(HashMap::new()),
332 subscribed: std::sync::Mutex::new(None),
333 exe_path,
334 managed_by,
335 synced: Mutex::new(HashMap::new()),
336 stopped: std::sync::Mutex::new(HashMap::new()),
337 device_id: OnceCell::new(),
338 watcher: OnceCell::new(),
339 host: OnceCell::new(),
340 reindexing: Mutex::new(()),
341 embed_wanted: Mutex::new(std::collections::HashSet::new()),
342 embedding: Mutex::new(()),
343 embed_running: Mutex::new(std::collections::HashSet::new()),
344 backfill: std::sync::Mutex::new(BackfillTiming::default()),
345 bridge_endpoint: None,
346 live: Mutex::new(HashMap::new()),
347 last_peers: Mutex::new(None),
348 open_timeout: OPEN_TIMEOUT,
349 first_dials: std::sync::Mutex::new(tokio::task::JoinSet::new()),
350 background: std::sync::Mutex::new(tokio::task::JoinSet::new()),
351 }
352 }
353
354 fn bridge(&self) -> Result<Arc<BridgeClient>> {
356 match &*self.bridge.borrow() {
357 Some(bridge) if !bridge.is_closed() => Ok(Arc::clone(bridge)),
358 _ => Err(Error::BridgeDisconnected),
359 }
360 }
361
362 pub fn is_connected(&self) -> bool {
364 self.bridge().is_ok()
365 }
366
367 fn spawn_background(&self, task: impl std::future::Future<Output = ()> + Send + 'static) {
369 let mut background = self.background.lock().expect("background tasks");
370 while background.try_join_next().is_some() {}
372 background.spawn(task);
373 }
374
375 pub fn with_open_timeout(mut self, timeout: std::time::Duration) -> SyncRuntime {
378 self.open_timeout = timeout;
379 self
380 }
381
382 pub fn with_bridge_endpoint(mut self, endpoint: sapphire_ipc::Endpoint) -> SyncRuntime {
389 self.bridge_endpoint = Some(endpoint);
390 self
391 }
392
393 pub fn set_backfill_timing(&self, timing: BackfillTiming) {
396 *self.backfill.lock().expect("backfill timing") = timing;
397 }
398
399 fn backfill_timing(&self) -> BackfillTiming {
400 *self.backfill.lock().expect("backfill timing")
401 }
402
403 pub(crate) fn set_host(&self, host: Arc<WorkspaceHost>) {
404 let _ = self.host.set(host);
405 }
406
407 pub async fn enable(self: &Arc<Self>, root: &Path) -> Result<GrainId> {
412 let key = root.canonicalize().map_err(Error::Io)?;
413 let watch_key = key.clone();
414 let workspace_id = sync_id(self.ctx.app_name, &key)?;
415
416 {
419 let mut pending = self.pending.lock().await;
420 if self.bridge().is_err() {
421 pending.insert(key, workspace_id);
422 return Err(Error::BridgeDisconnected);
423 }
424 pending.remove(&key);
425 }
426
427 {
430 let mut synced = self.synced.lock().await;
431 if let Some(existing) = synced.get(&key) {
432 return Ok(existing.workspace_id);
433 }
434 let replica = match self.reclaim(&key) {
437 Reclaimed::Open(replica) => replica,
438 reclaimed => {
439 let device_id = self.device_id().await?;
440 let state_dir = self.state_dir(&key)?;
441 std::fs::create_dir_all(&state_dir).map_err(Error::Io)?;
442 let config =
443 ReplicaConfig::new(self.ctx.app_name, key.clone(), device_id, &state_dir);
444 let mut tries = match reclaimed {
447 Reclaimed::Closing => CLOSING_RETRIES,
448 _ => 0,
449 };
450 let replica = loop {
451 match Replica::open(config.clone(), Arc::new(SystemClock)) {
452 Ok(replica) => break replica,
453 Err(_) if tries > 0 => {
454 tries -= 1;
455 tokio::time::sleep(CLOSING_RETRY_DELAY).await;
456 }
457 Err(err) => return Err(Error::Sync(err.to_string())),
458 }
459 };
460 Arc::new(Mutex::new(replica))
461 }
462 };
463 synced.insert(
464 key.clone(),
465 Synced {
466 workspace_id,
467 replica,
468 paused: None,
469 last_error: None,
470 },
471 );
472 }
473
474 if let Some(watcher) = self.watcher.get()
478 && let Err(err) = watcher.watch(&watch_key)
479 {
480 tracing::warn!(root = %watch_key.display(), "could not watch: {err}");
481 }
482
483 self.reregister().await?;
486
487 let runtime = Arc::clone(self);
505 let mut dials = self.first_dials.lock().expect("first dials");
506 while dials.try_join_next().is_some() {}
508 dials.spawn(async move {
509 if let Err(err) = runtime.scan(&key).await {
510 tracing::warn!(root = %key.display(), "the first scan after enabling failed: {err}");
511 }
512 if let Err(err) = runtime.sync_now(&key).await {
513 tracing::warn!(root = %key.display(), "the first session after enabling failed: {err}");
514 }
515 });
516 Ok(workspace_id)
517 }
518
519 pub async fn disable(&self, root: &Path) -> Result<()> {
525 let (replica, unregistered) = self.stop(root).await;
526 if let Some(replica) = replica {
527 let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
528 let mut stopped = self.stopped.lock().expect("stopped replicas");
529 stopped.retain(|_, replica| replica.strong_count() > 0);
531 stopped.insert(key, Arc::downgrade(&replica));
532 }
533 unregistered
534 }
535
536 pub async fn release(&self, root: &Path, limit: std::time::Duration) -> Result<()> {
545 let (replica, unregistered) = self.stop(root).await;
546 if let Err(err) = unregistered {
547 tracing::warn!(root = %root.display(), "stopping sync: {err}");
548 }
549 let Some(replica) = replica else {
550 return Ok(());
551 };
552 let deadline = tokio::time::Instant::now() + limit;
553 while Arc::strong_count(&replica) > 1 {
554 if tokio::time::Instant::now() >= deadline {
555 return Err(Error::Sync(format!(
556 "the replica of {} is still in use; try again",
557 root.display()
558 )));
559 }
560 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
561 }
562 Ok(())
563 }
564
565 async fn stop(&self, root: &Path) -> (Option<Arc<Mutex<Replica>>>, Result<()>) {
568 let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
571 self.pending.lock().await.remove(&key);
572 let removed = self.synced.lock().await.remove(&key);
573 if removed.is_some()
574 && let Some(watcher) = self.watcher.get()
575 {
576 watcher.unwatch(&key);
577 }
578 if let Some(peers) = self.live.lock().await.remove(&key) {
581 peers.drop_connections().await;
582 }
583 let Some(removed) = removed else {
584 return (None, Ok(()));
585 };
586 let unregistered = async {
587 let Ok(bridge) = self.bridge() else {
590 return Ok(());
591 };
592 bridge
593 .unregister(removed.workspace_id)
594 .await
595 .map_err(|e| Error::Bridge(e.to_string()))?;
596 self.reregister().await
597 }
598 .await;
599 (Some(removed.replica), unregistered)
600 }
601
602 pub async fn workspaces(&self) -> Result<WorkspacesResult> {
608 self.bridge()?
609 .workspaces()
610 .await
611 .map_err(|e| Error::Bridge(e.to_string()))
612 }
613
614 pub async fn place(&self, selector: &str, dir: &Path) -> Result<PathBuf> {
625 let workgroup = self.workspaces().await?;
626 let wanted = workgroup
627 .workspaces
628 .iter()
629 .find(|w| w.name == selector || w.workspace_id.to_string() == selector)
632 .ok_or_else(|| Error::UnknownWorkspaceName(selector.to_owned()))?;
633
634 if wanted.app_name != self.ctx.app_name {
637 return Err(Error::WrongApp {
638 name: wanted.name.clone(),
639 app_name: wanted.app_name.clone(),
640 });
641 }
642
643 let root = dir.canonicalize().map_err(Error::Io)?;
644 let marker = root.join(format!(".{}", self.ctx.app_name));
645 if !marker.is_dir() {
646 std::fs::create_dir(&marker).map_err(Error::Io)?;
647 }
648 let workspace = Workspace::from_root(self.ctx, &root)?;
649 if workspace.root != root {
650 return Err(Error::UnknownWorkspace(root, self.ctx.app_name));
651 }
652
653 {
657 let synced = self.synced.lock().await;
658 if let Some(entry) = synced.get(&root)
659 && entry.workspace_id != wanted.workspace_id
660 {
661 return Err(Error::SyncId(format!(
662 "{} already syncs as {}, not {}",
663 root.display(),
664 entry.workspace_id,
665 wanted.workspace_id
666 )));
667 }
668 }
669
670 let marker = workspace.marker_dir();
675 let id_path = marker.join(crate::sync::id::SYNC_ID_FILE);
676 match std::fs::read_to_string(&id_path) {
677 Ok(existing) if existing.trim() != wanted.workspace_id.to_string() => {
678 return Err(Error::SyncId(format!(
679 "{} already holds identity {}, not {}",
680 root.display(),
681 existing.trim(),
682 wanted.workspace_id
683 )));
684 }
685 Err(_) => {
686 std::fs::write(&id_path, format!("{}\n", wanted.workspace_id))
687 .map_err(Error::Io)?;
688 }
689 Ok(_) => {}
690 }
691
692 let map_path = root
695 .join(format!(".{}", self.ctx.app_name))
696 .join(WORKSPACE_MAP_FILE);
697 let workspace_id = wanted.workspace_id.to_string();
698 std::fs::write(&map_path, format!("{workspace_id}\n")).map_err(Error::Io)?;
699 Ok(root)
700 }
701
702 pub async fn status(&self, root: &Path) -> SyncStatus {
704 let bridge = self.bridge().ok();
705 let bridge_available = match &bridge {
706 Some(bridge) => bridge.status().await.is_ok(),
707 None => false,
708 };
709 let answer = match &bridge {
710 Some(_) => self.peers().await.ok(),
711 None => None,
712 };
713 let peers = answer
714 .as_ref()
715 .map_or(0, |p| p.peers.len().saturating_sub(1));
716 let Ok(key) = root.canonicalize() else {
717 return SyncStatus {
718 enabled: false,
719 workspace_id: None,
720 peers,
721 paused: None,
722 last_error: None,
723 bridge_available,
724 topology: proto::Topology::Mesh,
725 embedding: None,
726 };
727 };
728 let synced = self.synced.lock().await;
729 match synced.get(&key) {
730 Some(entry) => SyncStatus {
731 enabled: true,
732 workspace_id: Some(entry.workspace_id),
733 peers,
734 paused: entry.paused.map(|r| format!("{r:?}")),
735 last_error: entry.last_error.clone(),
736 bridge_available,
737 topology: topology::topology(
738 answer
739 .as_ref()
740 .and_then(|p| p.roles_for(entry.workspace_id)),
741 ),
742 embedding: self.embedding_progress(&key).await,
743 },
744 None => {
745 drop(synced);
746 let waiting = self.pending.lock().await.get(&key).copied();
749 SyncStatus {
750 enabled: waiting.is_some(),
751 workspace_id: waiting,
752 peers,
753 paused: None,
754 last_error: None,
755 bridge_available,
756 topology: proto::Topology::Mesh,
757 embedding: None,
758 }
759 }
760 }
761 }
762
763 async fn embedding_progress(&self, key: &Path) -> Option<proto::EmbeddingProgress> {
766 let host = self.host.get()?;
767 let backend = host.open_backend(key)?;
768 let state = backend.state();
769 state.embedder()?;
770 let info = state.db_info().ok()?;
771 Some(proto::EmbeddingProgress {
772 vectors: info.vector_count,
773 pending: info.pending_count,
774 running: self.embed_running.lock().await.contains(key),
775 })
776 }
777
778 pub async fn scan(&self, root: &Path) -> Result<()> {
784 let Ok(key) = root.canonicalize() else {
785 return Ok(());
786 };
787 let replica = {
788 let synced = self.synced.lock().await;
789 match synced.get(&key) {
790 Some(entry) => Arc::clone(&entry.replica),
791 None => return Ok(()),
792 }
793 };
794 let (outcome, updates) = {
795 let mut replica = replica.lock().await;
796 let outcome = replica.scan().map_err(|e| Error::Sync(e.to_string()))?;
797 let updates = match &outcome {
802 ScanOutcome::Scanned(report) => {
803 report
811 .recorded
812 .iter()
813 .map(|entry| PathUpdate {
814 path: entry.path.clone(),
815 versions: vec![entry.clone()],
816 seen: {
817 let mut seen = entry.context.clone();
818 seen.add_dot(&entry.dot);
819 seen
820 },
821 })
822 .collect()
823 }
824 ScanOutcome::Paused(_) => Vec::new(),
825 };
826 (outcome, updates)
827 };
828 {
829 let mut synced = self.synced.lock().await;
830 if let Some(entry) = synced.get_mut(&key) {
831 entry.paused = match outcome {
832 ScanOutcome::Paused(reason) => Some(reason),
833 ScanOutcome::Scanned(_) => None,
834 };
835 }
836 }
837 self.after_commit(&key, updates).await;
840 Ok(())
841 }
842
843 async fn reindex(&self, root: &Path) {
858 let Some(host) = self.host.get() else {
859 return;
860 };
861 let host = Arc::clone(host);
862 let root = root.to_owned();
863 let _serialised = self.reindexing.lock().await;
866 let backend = match host.backend_at(&root).await {
869 Ok(backend) => backend,
870 Err(err) => {
871 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
872 return;
873 }
874 };
875 let state = Arc::clone(backend.state());
876 let result = tokio::task::spawn_blocking(move || state.sync_retrieve()).await;
877 match result {
878 Ok(Ok(report)) => {
879 tracing::debug!(root = %root.display(), ?report, "re-indexed after a session");
880 }
881 Ok(Err(err)) => {
882 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
883 }
884 Err(err) => {
885 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
886 }
887 }
888 }
889
890 pub fn request_embed(self: &Arc<Self>, root: &Path) {
897 let Ok(key) = root.canonicalize() else {
898 return;
899 };
900 let runtime = Arc::clone(self);
901 self.spawn_background(async move {
902 if !runtime.embed_wanted.lock().await.insert(key.clone()) {
903 return;
904 }
905 let _one_at_a_time = runtime.embedding.lock().await;
906 runtime.embed_wanted.lock().await.remove(&key);
908 runtime.embed_running.lock().await.insert(key.clone());
909 runtime.embed_pass(&key).await;
910 runtime.embed_running.lock().await.remove(&key);
911 });
912 }
913
914 async fn embed_pass(&self, key: &Path) {
916 let Some(host) = self.host.get() else {
917 return;
918 };
919 let (workspace_id, replica) = {
920 let synced = self.synced.lock().await;
921 match synced.get(key) {
922 Some(entry) => (entry.workspace_id, Arc::clone(&entry.replica)),
923 None => return,
924 }
925 };
926 let backend = match host.backend_at(key).await {
927 Ok(backend) => backend,
928 Err(err) => {
929 tracing::warn!(root = %key.display(), "embedding pass: {err}");
930 return;
931 }
932 };
933 let state = Arc::clone(backend.state());
934 if let Err(err) = state
935 .refresh_embedder_at(self.bridge_endpoint.clone())
936 .await
937 {
938 tracing::warn!(root = %key.display(), "embedding pass: {err}");
939 return;
940 }
941 if state.embedder().is_none() {
942 return;
943 }
944 let primary = self.is_primary(workspace_id).await;
945 let grace_ms = self.backfill_timing().grace.as_millis() as u64;
948 let now_ms = std::time::SystemTime::now()
949 .duration_since(std::time::UNIX_EPOCH)
950 .map_or(0, |d| d.as_millis() as u64);
951 let (mine, settled, live) = {
952 let replica = replica.lock().await;
953 let me = replica.replica_id();
954 let states = match replica.states() {
955 Ok(states) => states,
956 Err(err) => {
957 tracing::warn!(root = %key.display(), "embedding pass: {err}");
958 return;
959 }
960 };
961 let mut mine = HashMap::new();
962 let mut settled = HashMap::new();
964 let mut live = std::collections::HashSet::new();
965 for (path, path_state) in states {
966 let winner = path_state.winner();
967 let Some(hash) = winner.content.hash() else {
968 continue;
969 };
970 let hash = hash.to_hex();
971 if winner.dot.replica == me {
972 mine.insert(path, hash.clone());
973 } else if now_ms.saturating_sub(winner.hlc.wall_ms) >= grace_ms {
974 settled.insert(path, hash.clone());
975 }
976 live.insert(hash);
977 }
978 (mine, settled, live)
979 };
980 let root = key.to_owned();
981 let result = tokio::task::spawn_blocking(move || {
982 let policy = |rel: &str, hash: &str| {
985 mine.get(rel).is_some_and(|h| h.as_str() == hash)
986 || (primary && settled.get(rel).is_some_and(|h| h.as_str() == hash))
987 };
988 let embedded = state.embed_pending_with(&policy, PASS_BATCH)?;
989 let removed = if primary {
990 state.remove_stale_vectors(&live, STALE_VECTOR_AGE)
991 } else {
992 0
993 };
994 Ok::<_, sapphire_workspace::Error>((embedded, removed))
995 })
996 .await;
997 match result {
998 Ok(Ok((embedded, removed))) if embedded > 0 || removed > 0 => tracing::info!(
999 root = %root.display(),
1000 embedded,
1001 removed,
1002 "embedding pass"
1003 ),
1004 Ok(Ok(_)) => {}
1005 Ok(Err(err)) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
1006 Err(err) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
1007 }
1008 }
1009
1010 pub async fn sync_now(self: &Arc<Self>, root: &Path) -> Result<()> {
1012 let Ok(key) = root.canonicalize() else {
1013 return Ok(());
1014 };
1015 let workspace_id = {
1016 let synced = self.synced.lock().await;
1017 match synced.get(&key) {
1018 Some(entry) => entry.workspace_id,
1019 None => return Ok(()),
1020 }
1021 };
1022 let peers = self.peers().await?;
1023 let me = self.device_id().await?;
1024
1025 let roles = peers.roles_for(workspace_id).cloned();
1030 let mut dials: Vec<Dial<'_>> = Vec::new();
1034 for peer in peers
1035 .peers
1036 .iter()
1037 .filter(|p| topology::link(me, p.device_id, roles.as_ref()) == topology::Link::Dial)
1038 .cloned()
1039 .collect::<Vec<_>>()
1040 {
1041 if let Some(live) = self.live_peers(&key).await
1045 && live.has(&peer.device_id).await
1046 {
1047 continue;
1048 }
1049 let Some((_, table)) = self.live_peers_or_create(&key).await else {
1056 continue;
1057 };
1058 if !table.begin_dial(peer.device_id).await {
1059 continue;
1060 }
1061 let key = &key;
1062 dials.push(Box::pin(async move {
1063 let result = self
1065 .sync_one(key, &peer.device_id, peer.name.clone(), workspace_id)
1066 .await;
1067 table.end_dial(peer.device_id).await;
1068 if let Err(err) = result {
1069 tracing::debug!(peer = %peer.name, "no session: {err}");
1070 }
1071 }));
1072 }
1073 join_all(dials).await;
1074 Ok(())
1075 }
1076
1077 async fn sync_one(
1084 self: &Arc<Self>,
1085 key: &Path,
1086 device: &GrainId,
1087 name: String,
1088 workspace_id: GrainId,
1089 ) -> Result<()> {
1090 let stream = self.open_stream(workspace_id, *device).await?;
1091 self.adopt_live_session(stream, key, *device, workspace_id)
1092 .await
1093 .map_err(|err| {
1094 tracing::warn!(peer = %name, "a live session could not be started: {err}");
1095 err
1096 })
1097 }
1098
1099 async fn open_stream(
1101 &self,
1102 workspace_id: GrainId,
1103 device: GrainId,
1104 ) -> Result<sapphire_ipc::RawStream> {
1105 match tokio::time::timeout(
1106 self.open_timeout,
1107 self.bridge()?.open_stream(workspace_id, device),
1108 )
1109 .await
1110 {
1111 Ok(opened) => Ok(opened?),
1112 Err(_) => Err(Error::Bridge(format!(
1113 "no stream to {device} within {} s; it may be offline",
1114 self.open_timeout.as_secs_f32()
1115 ))),
1116 }
1117 }
1118
1119 pub async fn run(self: Arc<Self>) -> Result<()> {
1122 let mut slot = self.bridge.subscribe();
1123 loop {
1124 let bridge = slot.borrow_and_update().clone();
1125 match bridge {
1126 Some(bridge) if !bridge.is_closed() => self.answer(&bridge, &mut slot).await,
1127 _ => {
1129 if slot.changed().await.is_err() {
1130 return Ok(());
1131 }
1132 }
1133 }
1134 }
1135 }
1136
1137 async fn answer(
1139 self: &Arc<Self>,
1140 bridge: &Arc<BridgeClient>,
1141 slot: &mut tokio::sync::watch::Receiver<Option<Arc<BridgeClient>>>,
1142 ) {
1143 let early = self
1144 .subscribed
1145 .lock()
1146 .expect("subscribed")
1147 .take_if(|(b, _)| Arc::ptr_eq(b, bridge));
1148 let mut incoming = match early {
1149 Some((_, incoming)) => incoming,
1150 None => bridge.incoming(),
1151 };
1152 loop {
1153 let received = tokio::select! {
1154 received = incoming.recv() => received,
1155 () = bridge.closed() => return,
1156 _ = slot.changed() => return,
1157 };
1158 let announcement = match received {
1159 Ok(a) => a,
1160 Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
1161 tracing::warn!(missed = n, "fell behind on bridge announcements");
1162 continue;
1163 }
1164 Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1165 };
1166
1167 let found = {
1168 let synced = self.synced.lock().await;
1169 synced
1170 .iter()
1171 .find(|(_, s)| s.workspace_id == announcement.workspace_id)
1172 .map(|(root, s)| (root.clone(), Arc::clone(&s.replica)))
1173 };
1174 let Some((root, replica)) = found else {
1177 tracing::debug!(
1180 workspace = %announcement.workspace_id,
1181 "an announcement for a workspace this server does not hold"
1182 );
1183 continue;
1184 };
1185
1186 let bridge = Arc::clone(bridge);
1187 let workspace_id = announcement.workspace_id;
1188 let peer = announcement.peer_device_id;
1189 let driver = Arc::clone(self);
1190 self.spawn_background(async move {
1191 let stream = match bridge.accept_stream(announcement.ticket).await {
1192 Ok(stream) => stream,
1193 Err(err) => {
1194 tracing::warn!("could not claim an announced stream: {err}");
1195 return;
1196 }
1197 };
1198 if driver.should_skip(workspace_id, peer).await {
1199 tracing::debug!(%peer, "dropped an inbound session: both ends are outside this workspace's star");
1200 return;
1201 }
1202 let (_, session) = match sapphire_framework_session::open_live_session(
1207 stream,
1208 Arc::clone(&replica),
1209 workspace_id,
1210 )
1211 .await
1212 {
1213 Ok(started) => started,
1214 Err(err) => {
1215 tracing::warn!("an inbound session failed: {err}");
1216 return;
1217 }
1218 };
1219 let Some((key, table)) = driver.live_peers_or_create(&root).await else {
1220 return;
1221 };
1222 let Some(updates) = table.insert(peer, session, false).await else {
1226 return;
1227 };
1228 table.catch_up(peer, &replica).await;
1229 driver.reindex(&key).await;
1232 driver.request_embed(&key);
1233 driver.spawn_reader(table, updates, key, peer);
1234 });
1235 }
1236 }
1237
1238 pub async fn connect(self: &Arc<Self>) -> Result<()> {
1245 let Some(connector) = &self.connector else {
1246 return Ok(());
1247 };
1248 if self.is_connected() {
1249 return Ok(());
1250 }
1251 match connector.connect().await? {
1252 Some(client) => self.attach(client).await,
1253 None => tracing::warn!(
1254 "no sapphire-bridge is running; serving without sync until it starts"
1255 ),
1256 }
1257 Ok(())
1258 }
1259
1260 pub async fn keep_connected(self: Arc<Self>) {
1268 let mut attempt = 0;
1269 loop {
1270 if let Ok(bridge) = self.bridge() {
1271 bridge.closed().await;
1272 tracing::warn!("lost the connection to sapphire-bridge; reconnecting");
1273 self.bridge.send_if_modified(|slot| {
1274 let same = slot.as_ref().is_some_and(|b| Arc::ptr_eq(b, &bridge));
1275 if same {
1276 *slot = None;
1277 }
1278 same
1279 });
1280 attempt = 0;
1281 continue;
1282 }
1283 let Some(connector) = &self.connector else {
1284 return;
1285 };
1286 match connector.connect().await {
1287 Ok(Some(client)) => {
1288 tracing::info!("connected to sapphire-bridge");
1289 self.attach(client).await;
1290 attempt = 0;
1291 continue;
1292 }
1293 Ok(None) => tracing::debug!("no sapphire-bridge is running yet"),
1294 Err(err) if attempt == 0 => {
1296 tracing::warn!("could not connect to sapphire-bridge: {err}")
1297 }
1298 Err(err) => tracing::debug!("could not connect to sapphire-bridge: {err}"),
1299 }
1300 tokio::time::sleep(reconnect_backoff(attempt)).await;
1301 attempt = attempt.saturating_add(1);
1302 }
1303 }
1304
1305 async fn attach(self: &Arc<Self>, client: BridgeClient) {
1308 let client = Arc::new(client);
1311 *self.subscribed.lock().expect("subscribed") =
1312 Some((Arc::clone(&client), client.incoming()));
1313 let waiting = {
1314 let mut pending = self.pending.lock().await;
1315 self.bridge.send_replace(Some(client));
1316 std::mem::take(&mut *pending)
1317 };
1318 if !self.synced.lock().await.is_empty()
1320 && let Err(err) = self.reregister().await
1321 {
1322 tracing::warn!("could not register with sapphire-bridge: {err}");
1323 }
1324 for root in waiting.into_keys() {
1326 if let Err(err) = self.enable(&root).await {
1327 tracing::warn!(root = %root.display(), "could not start sync: {err}");
1328 }
1329 }
1330 }
1331
1332 pub async fn roots(&self) -> Vec<PathBuf> {
1334 self.synced.lock().await.keys().cloned().collect()
1335 }
1336
1337 async fn live_peers(&self, root: &Path) -> Option<Arc<LivePeers>> {
1342 let key = root.canonicalize().ok()?;
1343 self.live.lock().await.get(&key).cloned()
1344 }
1345
1346 async fn live_peers_or_create(&self, root: &Path) -> Option<(PathBuf, Arc<LivePeers>)> {
1351 let key = root.canonicalize().ok()?;
1352 if !self.synced.lock().await.contains_key(&key) {
1353 return None;
1354 }
1355 let me = self.device_id().await.ok()?;
1358 let mut live = self.live.lock().await;
1359 let peers = Arc::clone(
1360 live.entry(key.clone())
1361 .or_insert_with(|| Arc::new(LivePeers::new(me))),
1362 );
1363 Some((key, peers))
1364 }
1365
1366 pub async fn after_commit(&self, root: &Path, updates: Vec<PathUpdate>) {
1376 if updates.is_empty() {
1377 return;
1378 }
1379 if let Some(peers) = self.live_peers(root).await {
1380 peers.fan_out(&updates, None).await;
1381 }
1382 }
1383
1384 pub async fn live_session_devices(&self, root: &Path) -> Vec<GrainId> {
1388 match self.live_peers(root).await {
1389 Some(peers) => peers.devices().await,
1390 None => Vec::new(),
1391 }
1392 }
1393
1394 pub async fn stop_dials(&self) {
1401 let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1402 dials.abort_all();
1403 while dials.join_next().await.is_some() {}
1404 }
1405
1406 pub async fn stop_background(&self) {
1414 let mut background =
1415 std::mem::take(&mut *self.background.lock().expect("background tasks"));
1416 background.abort_all();
1417 while background.join_next().await.is_some() {}
1418 }
1419
1420 #[cfg(test)]
1423 async fn settle_dials(&self) {
1424 let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1425 while dials.join_next().await.is_some() {}
1426 }
1427
1428 pub async fn drop_connections(&self) {
1433 let tables: Vec<Arc<LivePeers>> = self.live.lock().await.values().cloned().collect();
1434 for peers in tables {
1435 peers.drop_connections().await;
1436 }
1437 }
1438
1439 pub async fn dial_loop(self: Arc<Self>) -> Result<()> {
1454 let mut tick = tokio::time::interval(live::DIAL_INTERVAL);
1455 tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1458 let mut waiting: HashMap<GrainId, std::time::Instant> = HashMap::new();
1459 let mut failures: HashMap<GrainId, u32> = HashMap::new();
1460 let mut primary_of: std::collections::HashSet<GrainId> = Default::default();
1462 let mut last_backfill = std::time::Instant::now();
1464 loop {
1465 tick.tick().await;
1466 let roots = self.roots().await;
1467 if roots.is_empty() {
1468 continue;
1469 }
1470 let peers = match self.peers().await {
1471 Ok(peers) => peers,
1472 Err(err) => {
1475 tracing::debug!("no peers to dial: {err}");
1476 continue;
1477 }
1478 };
1479 let me = match self.device_id().await {
1480 Ok(me) => me,
1481 Err(err) => {
1482 tracing::warn!("no device id, so nobody to dial: {err}");
1483 continue;
1484 }
1485 };
1486 let ids: Vec<(PathBuf, GrainId)> = self
1490 .synced
1491 .lock()
1492 .await
1493 .iter()
1494 .map(|(root, entry)| (root.clone(), entry.workspace_id))
1495 .collect();
1496 let backfill_due = last_backfill.elapsed() >= self.backfill_timing().interval;
1499 if backfill_due {
1500 last_backfill = std::time::Instant::now();
1501 }
1502 for (root, workspace_id) in ids {
1503 let primary = peers
1504 .roles_for(workspace_id)
1505 .is_some_and(|r| r.primary == Some(me));
1506 if !primary {
1507 primary_of.remove(&workspace_id);
1508 } else if primary_of.insert(workspace_id) || backfill_due {
1509 self.request_embed(&root);
1510 }
1511 }
1512 let now = std::time::Instant::now();
1513 for root in roots {
1522 let Some((key, table)) = self.live_peers_or_create(&root).await else {
1523 continue;
1524 };
1525 let workspace_id = {
1526 let synced = self.synced.lock().await;
1527 match synced.get(&key) {
1528 Some(entry) => entry.workspace_id,
1529 None => continue,
1530 }
1531 };
1532 let roles = peers.roles_for(workspace_id);
1533 if topology::may_close_skipped(me, roles, &table.devices().await) {
1536 table
1537 .retain(|d| topology::link(me, *d, roles) != topology::Link::Skip)
1538 .await;
1539 }
1540 for peer in peers.peers.iter().filter(|p| {
1541 p.connected && topology::link(me, p.device_id, roles) == topology::Link::Dial
1542 }) {
1543 let device = peer.device_id;
1544 if table.has(&device).await {
1545 failures.remove(&device);
1547 waiting.remove(&device);
1548 continue;
1549 }
1550 if let Some(until) = waiting.get(&device)
1551 && now < *until
1552 {
1553 continue;
1555 }
1556 if !table.begin_dial(device).await {
1560 continue;
1561 }
1562 let opened = self.open_stream(workspace_id, device).await;
1563 if opened.is_err() {
1564 table.end_dial(device).await;
1565 }
1566 match opened {
1567 Ok(stream) => {
1568 let adopted = self
1569 .adopt_live_session(stream, &root, device, workspace_id)
1570 .await;
1571 table.end_dial(device).await;
1574 match adopted {
1575 Ok(()) => {
1576 failures.remove(&device);
1577 waiting.remove(&device);
1578 }
1579 Err(err) => {
1580 tracing::debug!(
1581 peer = %peer.name,
1582 err = %err,
1583 "a live session could not be started"
1584 );
1585 let count = failures.entry(device).or_insert(0);
1586 *count = count.saturating_add(1);
1587 waiting.insert(device, now + dial_backoff(*count));
1588 }
1589 }
1590 }
1591 Err(err) => {
1592 tracing::debug!("no stream to a peer: {err}");
1595 let count = failures.entry(device).or_insert(0);
1596 *count = count.saturating_add(1);
1597 waiting.insert(device, now + dial_backoff(*count));
1598 }
1599 }
1600 }
1601 }
1602 }
1603 }
1604
1605 async fn adopt_live_session(
1610 self: &Arc<Self>,
1611 stream: sapphire_ipc::RawStream,
1612 root: &Path,
1613 device: GrainId,
1614 workspace_id: GrainId,
1615 ) -> Result<()> {
1616 let replica = self.replica_of(root).await?;
1617 let (_, session) = sapphire_framework_session::open_live_session(
1618 stream,
1619 Arc::clone(&replica),
1620 workspace_id,
1621 )
1622 .await
1623 .map_err(|e| Error::Sync(e.to_string()))?;
1624 let Some((key, table)) = self.live_peers_or_create(root).await else {
1625 return Ok(());
1626 };
1627 let Some(updates) = table.insert(device, session, true).await else {
1632 return Ok(());
1633 };
1634 table.catch_up(device, &replica).await;
1635 self.reindex(&key).await;
1638 self.request_embed(&key);
1639 self.spawn_reader(Arc::clone(&table), updates, key, device);
1640 Ok(())
1641 }
1642
1643 fn spawn_reader(
1650 self: &Arc<Self>,
1651 table: Arc<LivePeers>,
1652 mut updates: broadcast::Receiver<Vec<PathUpdate>>,
1653 key: PathBuf,
1654 from: GrainId,
1655 ) {
1656 let driver = Arc::clone(self);
1657 self.spawn_background(async move {
1658 loop {
1659 match updates.recv().await {
1660 Ok(batch) => {
1661 driver.reindex(&key).await;
1664 driver.request_embed(&key);
1665 table.fan_out(&batch, Some(from)).await;
1666 }
1667 Err(broadcast::error::RecvError::Lagged(missed)) => {
1672 tracing::debug!(
1673 device = %from,
1674 missed,
1675 "fell behind on a live session; the next exchange catches up"
1676 );
1677 }
1678 Err(broadcast::error::RecvError::Closed) => {
1681 return;
1682 }
1683 }
1684 }
1685 });
1686 }
1687
1688 fn reclaim(&self, key: &Path) -> Reclaimed {
1691 let stopped = self.stopped.lock().expect("stopped replicas").remove(key);
1692 match stopped {
1693 None => Reclaimed::Closed,
1694 Some(replica) => replica
1695 .upgrade()
1696 .map_or(Reclaimed::Closing, Reclaimed::Open),
1697 }
1698 }
1699
1700 async fn replica_of(&self, root: &Path) -> Result<Arc<Mutex<Replica>>> {
1702 let key = root.canonicalize().map_err(Error::Io)?;
1703 let synced = self.synced.lock().await;
1704 match synced.get(&key) {
1705 Some(entry) => Ok(Arc::clone(&entry.replica)),
1706 None => Err(Error::UnknownWorkspace(key, self.ctx.app_name)),
1707 }
1708 }
1709
1710 pub async fn watch_changes(self: Arc<Self>) -> Result<()> {
1722 let (tx, mut rx) = mpsc::channel(64);
1723 let watcher = Arc::new(Watcher::start(self.roots().await, tx)?);
1724 let _ = self.watcher.set(Arc::clone(&watcher));
1725
1726 for root in self.roots().await {
1730 if let Err(err) = watcher.watch(&root) {
1731 tracing::warn!(root = %root.display(), "could not watch: {err}");
1732 }
1733 }
1734
1735 while let Some(root) = rx.recv().await {
1736 if let Err(err) = self.scan(&root).await {
1739 tracing::warn!(root = %root.display(), "scan after a local edit failed: {err}");
1740 }
1741 self.request_embed(&root);
1742 if let Err(err) = self.sync_now(&root).await {
1743 tracing::warn!(root = %root.display(), "dial after a local edit failed: {err}");
1744 }
1745 }
1746 Ok(())
1747 }
1748
1749 async fn reregister(&self) -> Result<()> {
1751 let workspaces: Vec<WorkspaceRegistration> = {
1752 let synced = self.synced.lock().await;
1753 synced
1754 .iter()
1755 .map(|(root, entry)| WorkspaceRegistration {
1756 workspace_id: entry.workspace_id,
1757 root: root.clone(),
1758 })
1759 .collect()
1760 };
1761 self.bridge()?
1762 .register(RegisterParams {
1763 app_name: self.ctx.app_name.to_owned(),
1764 exe_path: self.exe_path.clone(),
1765 managed_by: wire_managed_by(self.managed_by),
1766 workspaces,
1767 })
1768 .await
1769 .map_err(|e| Error::Bridge(e.to_string()))?;
1770 Ok(())
1771 }
1772
1773 async fn should_skip(&self, workspace_id: GrainId, peer: GrainId) -> bool {
1778 let Some(&me) = self.device_id.get() else {
1779 return false;
1780 };
1781 let cached = self.last_peers.lock().await;
1782 let roles = cached.as_ref().and_then(|p| p.roles_for(workspace_id));
1783 topology::link(me, peer, roles) == topology::Link::Skip
1784 }
1785
1786 pub async fn is_primary(&self, workspace_id: GrainId) -> bool {
1792 let Some(&me) = self.device_id.get() else {
1793 return false;
1794 };
1795 let cached = self.last_peers.lock().await;
1796 cached
1797 .as_ref()
1798 .and_then(|p| p.roles_for(workspace_id))
1799 .is_some_and(|r| r.primary == Some(me))
1800 }
1801
1802 pub fn this_device(&self) -> Option<GrainId> {
1809 self.device_id.get().copied()
1810 }
1811
1812 pub async fn peers(&self) -> Result<PeersResult> {
1818 let peers = self
1819 .bridge()?
1820 .peers()
1821 .await
1822 .map_err(|e| Error::Bridge(e.to_string()))?;
1823 *self.last_peers.lock().await = Some(peers.clone());
1824 Ok(peers)
1825 }
1826
1827 async fn device_id(&self) -> Result<GrainId> {
1829 if let Some(id) = self.device_id.get() {
1830 return Ok(*id);
1831 }
1832 let result = self
1833 .bridge()?
1834 .register(RegisterParams {
1835 app_name: self.ctx.app_name.to_owned(),
1836 exe_path: self.exe_path.clone(),
1837 managed_by: wire_managed_by(self.managed_by),
1838 workspaces: Vec::new(),
1839 })
1840 .await
1841 .map_err(|e| Error::Bridge(e.to_string()))?;
1842 let _ = self.device_id.set(result.device_id);
1843 Ok(result.device_id)
1844 }
1845
1846 fn state_dir(&self, root: &Path) -> Result<PathBuf> {
1848 let workspace = Workspace::from_root(self.ctx, root)?;
1849 Ok(workspace.cache_dir().join("sync"))
1850 }
1851}
1852
1853type Dial<'a> = std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>>;
1855
1856async fn join_all(mut dials: Vec<Dial<'_>>) {
1858 std::future::poll_fn(|cx| {
1859 dials.retain_mut(|dial| dial.as_mut().poll(cx).is_pending());
1860 if dials.is_empty() {
1861 std::task::Poll::Ready(())
1862 } else {
1863 std::task::Poll::Pending
1864 }
1865 })
1866 .await
1867}
1868
1869fn wire_managed_by(managed_by: ManagedBy) -> sapphire_bridge_api::ManagedBy {
1871 match managed_by {
1872 ManagedBy::Service => sapphire_bridge_api::ManagedBy::Service,
1873 ManagedBy::Spawned => sapphire_bridge_api::ManagedBy::Spawned,
1874 }
1875}
1876
1877#[cfg(test)]
1878mod tests {
1879 use super::*;
1880 use crate::sync::testing::StubBridge;
1881 use crate::test_support;
1882 use sapphire_workspace::AppKind;
1883 use std::ffi::OsString;
1884
1885 static CTX: AppContext = AppContext::new("sapphire-synctest");
1886
1887 const DIR_VARS: [&str; 3] = [
1889 "SAPPHIRE_SYNCTEST_CACHE_DIR",
1890 "SAPPHIRE_SYNCTEST_DATA_DIR",
1891 "SAPPHIRE_SYNCTEST_CONFIG_DIR",
1892 ];
1893
1894 struct EnvGuard {
1897 previous: [Option<OsString>; 3],
1898 _lock: std::sync::MutexGuard<'static, ()>,
1899 }
1900
1901 impl Drop for EnvGuard {
1902 fn drop(&mut self) {
1903 for (name, previous) in DIR_VARS.iter().zip(self.previous.iter_mut()) {
1906 match previous.take() {
1907 Some(value) => unsafe { std::env::set_var(name, value) },
1908 None => test_support::remove(name),
1909 }
1910 }
1911 }
1912 }
1913
1914 struct Fixture {
1922 _tmp: tempfile::TempDir,
1923 _env: EnvGuard,
1924 root: PathBuf,
1925 stub: StubBridge,
1926 runtime: Arc<SyncRuntime>,
1927 }
1928
1929 impl Fixture {
1930 fn second_root(&self) -> PathBuf {
1932 let root = self._tmp.path().join("ws2");
1933 std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
1934 root.canonicalize().unwrap()
1935 }
1936 }
1937
1938 const FIXTURE_OPEN_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
1940
1941 fn dialled_peers(me: GrainId, n: usize) -> Vec<sapphire_bridge_api::PeerInfo> {
1943 std::iter::repeat_with(GrainId::random)
1944 .filter(|id| *id > me)
1945 .take(n)
1946 .enumerate()
1947 .map(|(i, device_id)| sapphire_bridge_api::PeerInfo {
1948 device_id,
1949 name: format!("offline-{i}"),
1950 node_id: format!("node-{i}"),
1951 connected: false,
1952 priority: 1,
1953 availability: None,
1954 })
1955 .collect()
1956 }
1957
1958 #[tokio::test(flavor = "multi_thread")]
1959 async fn enabling_answers_at_once_with_a_peer_that_never_answers() {
1960 let f = fixture().await;
1962 let _hung = f.stub.hang_streams().await;
1963 f.stub.set_peers(dialled_peers(f.stub.device_id, 1));
1964
1965 let started = std::time::Instant::now();
1966 f.runtime.enable(&f.root).await.unwrap();
1967 assert!(
1968 started.elapsed() < FIXTURE_OPEN_TIMEOUT,
1969 "enable waited {:?} on the dial",
1970 started.elapsed()
1971 );
1972 }
1973
1974 #[tokio::test(flavor = "multi_thread")]
1975 async fn peers_that_never_answer_are_dialled_together_and_given_up_on() {
1976 let f = fixture().await;
1977 let _hung = f.stub.hang_streams().await;
1978 f.runtime.enable(&f.root).await.unwrap();
1979 f.runtime.settle_dials().await;
1982 f.stub.set_peers(dialled_peers(f.stub.device_id, 3));
1983
1984 let started = std::time::Instant::now();
1985 f.runtime.sync_now(&f.root).await.unwrap();
1986 let took = started.elapsed();
1987 assert!(took >= FIXTURE_OPEN_TIMEOUT, "gave up early: {took:?}");
1988 assert!(
1989 took < FIXTURE_OPEN_TIMEOUT * 2,
1990 "three dials one after another: {took:?}"
1991 );
1992 }
1993
1994 async fn fixture() -> Fixture {
1995 let (tmp, env, root) = scratch();
1996 let (stub, client) = StubBridge::start_in(tmp.path().join("run")).await;
1997 let runtime = Arc::new(
1998 SyncRuntime::new(&CTX, client, "/bin/true".into(), ManagedBy::Service)
1999 .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
2000 );
2001 Fixture {
2002 _tmp: tmp,
2003 _env: env,
2004 root,
2005 stub,
2006 runtime,
2007 }
2008 }
2009
2010 type Bridges = Arc<std::sync::Mutex<std::collections::VecDeque<BridgeClient>>>;
2013
2014 async fn connecting() -> (Fixture, Bridges, BridgeClient) {
2017 let (tmp, env, root) = scratch();
2018 let (stub, client) = StubBridge::start_client_in(tmp.path().join("run")).await;
2019 let bridges: Bridges = Arc::default();
2020 let connector = BridgeConnector::new({
2021 let bridges = Arc::clone(&bridges);
2022 move || {
2023 let next = bridges.lock().unwrap().pop_front();
2024 async move { Ok(next) }
2025 }
2026 });
2027 let runtime = Arc::new(
2028 SyncRuntime::connecting(&CTX, connector, "/bin/true".into(), ManagedBy::Service)
2029 .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
2030 );
2031 let f = Fixture {
2032 _tmp: tmp,
2033 _env: env,
2034 root,
2035 stub,
2036 runtime,
2037 };
2038 (f, bridges, client)
2039 }
2040
2041 async fn eventually(what: &str, mut done: impl AsyncFnMut() -> bool) {
2043 let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
2044 while !done().await {
2045 assert!(tokio::time::Instant::now() < deadline, "never: {what}");
2046 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
2047 }
2048 }
2049
2050 #[tokio::test(flavor = "multi_thread")]
2051 async fn a_runtime_started_without_a_bridge_syncs_once_one_starts() {
2052 let (f, bridges, client) = connecting().await;
2054 f.runtime.connect().await.unwrap();
2055 assert!(!f.runtime.is_connected());
2056
2057 let err = f.runtime.enable(&f.root).await.unwrap_err();
2058 assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2059 let status = f.runtime.status(&f.root).await;
2060 assert!(status.enabled, "enabled, waiting for the bridge");
2061 assert!(!status.bridge_available);
2062 let id = status.workspace_id.expect("its id");
2063
2064 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2065 bridges.lock().unwrap().push_back(client);
2066 eventually("the waiting workspace is registered", async || {
2067 f.stub.last_workspaces() == vec![id]
2068 })
2069 .await;
2070 assert_eq!(f.runtime.roots().await, vec![f.root.clone()]);
2071 assert!(f.runtime.status(&f.root).await.bridge_available);
2072 keeping.abort();
2073 }
2074
2075 #[tokio::test(flavor = "multi_thread")]
2076 async fn a_runtime_that_loses_its_bridge_registers_with_the_next_one() {
2077 let (f, bridges, client) = connecting().await;
2078 bridges.lock().unwrap().push_back(client);
2079 f.runtime.connect().await.unwrap();
2080 assert!(f.runtime.is_connected());
2081 let id = f.runtime.enable(&f.root).await.unwrap();
2082 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2083
2084 f.stub.disconnect();
2085 eventually("the runtime notices", async || !f.runtime.is_connected()).await;
2086 let err = f.runtime.workspaces().await.unwrap_err();
2087 assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2088
2089 let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2090 bridges.lock().unwrap().push_back(client);
2091 eventually("the synced workspace is registered again", async || {
2092 next.last_workspaces() == vec![id]
2093 })
2094 .await;
2095 assert!(f.runtime.is_connected());
2096 keeping.abort();
2097 }
2098
2099 #[tokio::test(flavor = "multi_thread")]
2100 async fn this_device_is_known_once_a_workspace_is_enabled_and_never_asked_for() {
2101 let f = fixture().await;
2102 assert_eq!(f.runtime.this_device(), None);
2103 assert!(
2104 f.stub.seen.lock().unwrap().registrations.is_empty(),
2105 "asking must not register"
2106 );
2107
2108 f.runtime.enable(&f.root).await.unwrap();
2109 assert_eq!(f.runtime.this_device(), Some(f.stub.device_id));
2110 }
2111
2112 #[tokio::test(flavor = "multi_thread")]
2113 async fn peers_answers_the_workgroup_from_the_bridge() {
2114 let f = fixture().await;
2115 let listed = dialled_peers(f.stub.device_id, 2);
2116 f.stub.set_peers(listed.clone());
2117
2118 let peers = f.runtime.peers().await.unwrap();
2119 let ids: Vec<_> = peers.peers.iter().map(|p| p.device_id).collect();
2120 assert_eq!(ids, listed.iter().map(|p| p.device_id).collect::<Vec<_>>());
2121 }
2122
2123 #[tokio::test(flavor = "multi_thread")]
2124 async fn peers_says_so_while_the_bridge_is_away() {
2125 let (f, _bridges, _client) = connecting().await;
2126 let err = f.runtime.peers().await.unwrap_err();
2127 assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2128 }
2129
2130 #[tokio::test(flavor = "multi_thread")]
2131 async fn disabling_while_the_bridge_is_away_forgets_the_waiting_workspace() {
2132 let (f, bridges, client) = connecting().await;
2133 f.runtime.enable(&f.root).await.unwrap_err();
2134 f.runtime.disable(&f.root).await.unwrap();
2135 assert!(!f.runtime.status(&f.root).await.enabled);
2136
2137 bridges.lock().unwrap().push_back(client);
2138 f.runtime.connect().await.unwrap();
2139 assert!(f.runtime.roots().await.is_empty());
2140 assert!(f.stub.last_workspaces().is_empty());
2141 }
2142
2143 #[tokio::test(flavor = "multi_thread")]
2144 async fn announcements_are_answered_from_a_reconnected_bridge() {
2145 let (f, bridges, client) = connecting().await;
2146 bridges.lock().unwrap().push_back(client);
2147 f.runtime.connect().await.unwrap();
2148 let id = f.runtime.enable(&f.root).await.unwrap();
2149 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2150 let answering = tokio::spawn(Arc::clone(&f.runtime).run());
2151
2152 f.stub.disconnect();
2153 let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2154 bridges.lock().unwrap().push_back(client);
2155 eventually("reconnected", async || next.last_workspaces() == vec![id]).await;
2156
2157 next.announce(id, "ticket").await;
2160 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2161 assert!(!answering.is_finished(), "the announcement loop ended");
2162 answering.abort();
2163 keeping.abort();
2164 }
2165
2166 fn scratch() -> (tempfile::TempDir, EnvGuard, PathBuf) {
2168 let lock = test_support::lock();
2169 let tmp = tempfile::tempdir().unwrap();
2170 let previous = DIR_VARS.map(std::env::var_os);
2171 for (name, dir) in DIR_VARS
2175 .iter()
2176 .zip(["cache", "data", "config"].map(|cat| tmp.path().join(cat)))
2177 {
2178 test_support::set(name, &dir);
2179 }
2180 let env = EnvGuard {
2181 previous,
2182 _lock: lock,
2183 };
2184 CTX.init(AppKind::Server);
2185
2186 let root = tmp.path().join("ws");
2187 std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
2188 let root = root.canonicalize().unwrap();
2189 (tmp, env, root)
2190 }
2191
2192 #[tokio::test(flavor = "multi_thread")]
2193 async fn enabling_registers_the_workspace_with_the_bridge() {
2194 let f = fixture().await;
2195
2196 let id = f.runtime.enable(&f.root).await.unwrap();
2197 assert_eq!(f.stub.last_workspaces(), vec![id]);
2198 }
2199
2200 #[tokio::test(flavor = "multi_thread")]
2201 async fn enabling_twice_is_idempotent_and_keeps_the_same_id() {
2202 let f = fixture().await;
2203
2204 let first = f.runtime.enable(&f.root).await.unwrap();
2205 let second = f.runtime.enable(&f.root).await.unwrap();
2206 assert_eq!(first, second);
2207 assert_eq!(
2208 f.stub.last_workspaces(),
2209 vec![first],
2210 "still exactly one workspace"
2211 );
2212 }
2213
2214 #[tokio::test(flavor = "multi_thread")]
2215 async fn a_registration_carries_the_complete_current_list() {
2216 let f = fixture().await;
2217 let second_root = f.second_root();
2218
2219 let a = f.runtime.enable(&f.root).await.unwrap();
2220 let b = f.runtime.enable(&second_root).await.unwrap();
2221
2222 let mut seen = f.stub.last_workspaces();
2223 seen.sort();
2224 let mut want = vec![a, b];
2225 want.sort();
2226 assert_eq!(seen, want, "registration is the whole set, not a delta");
2227 }
2228
2229 #[tokio::test(flavor = "multi_thread")]
2230 async fn no_later_registration_clears_this_apps_routes() {
2231 let f = fixture().await;
2237 let second_root = f.second_root();
2238
2239 f.runtime.enable(&f.root).await.unwrap();
2240 f.runtime.enable(&second_root).await.unwrap();
2241 f.runtime.sync_now(&f.root).await.unwrap();
2242 f.runtime.disable(&f.root).await.unwrap();
2243
2244 let seen = f.stub.seen.lock().expect("stub");
2245 let first_full = seen
2246 .registrations
2247 .iter()
2248 .position(|r| !r.workspaces.is_empty())
2249 .expect("enabling registers the workspace");
2250 let emptied_afterwards = seen.registrations[first_full + 1..]
2251 .iter()
2252 .filter(|r| r.workspaces.is_empty())
2253 .count();
2254 assert_eq!(
2255 emptied_afterwards, 0,
2256 "a registration after the first workspace must still name the whole set"
2257 );
2258 }
2259
2260 #[tokio::test(flavor = "multi_thread")]
2261 async fn disabling_unregisters_and_leaves_the_files_alone() {
2262 let f = fixture().await;
2263 std::fs::write(f.root.join("keep.md"), "content").unwrap();
2264
2265 let id = f.runtime.enable(&f.root).await.unwrap();
2266 f.runtime.disable(&f.root).await.unwrap();
2267
2268 assert!(f.stub.last_workspaces().is_empty());
2269 assert!(
2270 f.root.join("keep.md").exists(),
2271 "disabling sync must not touch files"
2272 );
2273 assert!(
2274 crate::sync::id::sync_id_path("sapphire-synctest", &f.root).exists(),
2275 "the sync id stays, so re-enabling rejoins the same workspace"
2276 );
2277 let _ = id;
2278 }
2279
2280 #[tokio::test(flavor = "multi_thread")]
2281 async fn re_enabling_after_disabling_reuses_the_identity() {
2282 let f = fixture().await;
2283
2284 let first = f.runtime.enable(&f.root).await.unwrap();
2285 f.runtime.disable(&f.root).await.unwrap();
2286 let again = f.runtime.enable(&f.root).await.unwrap();
2287
2288 assert_eq!(first, again);
2289 }
2290
2291 #[tokio::test(flavor = "multi_thread")]
2292 async fn status_reports_disabled_for_a_workspace_that_was_never_enabled() {
2293 let f = fixture().await;
2294
2295 let status = f.runtime.status(&f.root).await;
2296 assert!(!status.enabled);
2297 assert!(status.workspace_id.is_none());
2298 }
2299
2300 #[tokio::test(flavor = "multi_thread")]
2301 async fn disabling_something_that_was_never_enabled_is_not_an_error() {
2302 let f = fixture().await;
2303 f.runtime
2304 .disable(&f.root)
2305 .await
2306 .expect("disabling twice is fine");
2307 }
2308
2309 #[tokio::test(flavor = "multi_thread")]
2310 async fn a_file_already_on_disk_is_recorded_on_enable() {
2311 let f = fixture().await;
2314 std::fs::write(f.root.join("a.md"), "content").unwrap();
2315 f.runtime.enable(&f.root).await.unwrap();
2316 f.runtime.settle_dials().await;
2317
2318 let replica = f.runtime.replica_of(&f.root).await.unwrap();
2319 let states = replica.lock().await.states().unwrap();
2320 assert!(
2321 states.iter().any(|(path, _)| path == "a.md"),
2322 "a file on disk before enable must be recorded: {states:?}"
2323 );
2324 }
2325
2326 #[tokio::test(flavor = "multi_thread")]
2327 async fn re_enabling_while_the_replica_is_still_held_takes_it_back() {
2328 let f = fixture().await;
2331 f.runtime.enable(&f.root).await.unwrap();
2332 f.runtime.settle_dials().await;
2333 let held = f.runtime.replica_of(&f.root).await.unwrap();
2334
2335 f.runtime.disable(&f.root).await.unwrap();
2336 f.runtime
2337 .enable(&f.root)
2338 .await
2339 .expect("a re-enable must not open the store a second time");
2340
2341 let again = f.runtime.replica_of(&f.root).await.unwrap();
2342 assert!(Arc::ptr_eq(&held, &again), "the open replica is taken back");
2343 }
2344
2345 #[tokio::test(flavor = "multi_thread")]
2346 async fn re_enabling_while_the_replica_is_still_closing_waits_for_it() {
2347 let f = fixture().await;
2350 f.runtime.enable(&f.root).await.unwrap();
2351 f.runtime.settle_dials().await;
2352 let config = {
2353 let replica = f.runtime.replica_of(&f.root).await.unwrap();
2354 replica.lock().await.config().clone()
2355 };
2356 f.runtime.disable(&f.root).await.unwrap();
2357
2358 let closing = Replica::open(config, Arc::new(SystemClock)).unwrap();
2360 let key = f.root.canonicalize().unwrap();
2361 f.runtime
2362 .stopped
2363 .lock()
2364 .unwrap()
2365 .insert(key, std::sync::Weak::new());
2366 let closed = tokio::spawn(async move {
2367 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2368 drop(closing);
2369 });
2370
2371 f.runtime
2372 .enable(&f.root)
2373 .await
2374 .expect("a re-enable must wait for a store that is closing");
2375 closed.await.unwrap();
2376 }
2377
2378 #[tokio::test(flavor = "multi_thread")]
2379 async fn a_workspace_whose_root_vanished_is_reported_as_paused_not_deleted() {
2380 let f = fixture().await;
2381 std::fs::write(f.root.join("a.md"), "content").unwrap();
2382 f.runtime.enable(&f.root).await.unwrap();
2383 f.runtime.scan(&f.root).await.unwrap();
2384
2385 std::fs::remove_dir_all(f.root.join(".sapphire-synctest")).unwrap();
2387 let _ = f.runtime.scan(&f.root).await;
2388
2389 let status = f.runtime.status(&f.root).await;
2390 assert!(
2391 status.paused.is_some(),
2392 "a missing root must pause, not replicate as a mass deletion"
2393 );
2394 }
2395
2396 #[tokio::test(flavor = "multi_thread")]
2397 async fn a_directory_that_is_not_a_workspace_cannot_be_enabled() {
2398 let f = fixture().await;
2399 let plain = f._tmp.path().join("plain");
2400 std::fs::create_dir_all(&plain).unwrap();
2401
2402 assert!(f.runtime.enable(&plain).await.is_err());
2403 }
2404
2405 #[tokio::test(flavor = "multi_thread")]
2406 async fn a_write_through_the_workspace_router_is_committed_to_the_replica() {
2407 let f = fixture().await;
2412 f.runtime.enable(&f.root).await.unwrap();
2413
2414 let host = Arc::new(crate::WorkspaceHost::new(&CTX));
2415 host.select(&f.root, false).await.unwrap();
2416 let current = Arc::new(crate::Current::new(
2417 &CTX,
2418 host,
2419 Some(Arc::clone(&f.runtime)),
2420 crate::selection::SelectionFile::at(f._tmp.path().join("workspace.toml")),
2421 ));
2422 let router = Arc::new(crate::workspace_router(current));
2423 let (client_conn, server_conn) = sapphire_ipc::Connection::pair();
2424 tokio::spawn(async move {
2425 let info = sapphire_ipc::ServerInfo {
2426 version: "0.0.0".into(),
2427 api: sapphire_backend::protocol::API_VERSION,
2428 app_api: None,
2429 pid: std::process::id(),
2430 managed_by: ManagedBy::Spawned,
2431 };
2432 let _ = sapphire_ipc::serve(server_conn, router, "sapphire-synctest", info).await;
2433 });
2434 let info = sapphire_ipc::ClientInfo {
2435 kind: "test".into(),
2436 version: "0.0.0".into(),
2437 api: sapphire_backend::protocol::API_VERSION,
2438 app_api: None,
2439 pid: std::process::id(),
2440 };
2441 let (client, _) = sapphire_ipc::Client::handshake(client_conn, "sapphire-synctest", info)
2442 .await
2443 .unwrap();
2444
2445 let _: sapphire_backend::protocol::Ack = client
2446 .call(
2447 sapphire_backend::protocol::WRITE_FILE,
2448 sapphire_backend::protocol::ContentParams {
2449 path: PathBuf::from("through-the-server.md"),
2450 content: "written by the server".into(),
2451 },
2452 )
2453 .await
2454 .unwrap();
2455
2456 let synced = f.runtime.synced.lock().await;
2459 let entry = synced.get(&f.root).expect("enabled above");
2460 let replica = entry.replica.lock().await;
2461 let state = replica
2462 .state("through-the-server.md")
2463 .expect("a readable state");
2464 assert!(
2465 state.is_some(),
2466 "a write through the server must be scanned into the replica, not left to the watcher"
2467 );
2468 }
2469
2470 #[tokio::test(flavor = "multi_thread")]
2471 async fn an_announcement_for_an_unknown_workspace_is_ignored_not_fatal() {
2472 let f = fixture().await;
2473 f.runtime.enable(&f.root).await.unwrap();
2474
2475 let driver = Arc::clone(&f.runtime);
2476 let handle = tokio::spawn(async move { driver.run().await });
2477
2478 f.stub
2479 .announce(grain_id::GrainId::random(), "no-such-ticket")
2480 .await;
2481 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
2482
2483 assert!(
2484 !handle.is_finished(),
2485 "one bad announcement must not stop the runtime"
2486 );
2487 handle.abort();
2488 }
2489}