1use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use grain_id::GrainId;
13use sapphire_backend::protocol as proto;
14use sapphire_bridge_api::{
15 IncomingParams, RegisterParams, WorkspaceRegistration, WorkspacesResult,
16};
17use sapphire_bridge_client::BridgeClient;
18use sapphire_ipc::ManagedBy;
19use sapphire_sync::{PathUpdate, PauseReason, Replica, ReplicaConfig, ScanOutcome, SystemClock};
20use sapphire_workspace::{AppContext, Workspace};
21use tokio::sync::{Mutex, OnceCell, broadcast, mpsc};
22
23pub mod id;
24mod live;
25mod methods;
26#[cfg(any(test, feature = "test-util"))]
27pub mod testing;
28mod topology;
29mod watch;
30
31pub use id::{SYNC_ID_FILE, WORKSPACE_MAP_FILE, sync_id, sync_id_path};
32pub use live::{DIAL_BACKOFF_MAX, OPEN_TIMEOUT};
33pub use methods::sync_router;
34pub use sapphire_bridge_api::{PeerInfo, PeersResult};
36pub use watch::{DEBOUNCE, Watcher};
37
38use live::{LivePeers, dial_backoff};
39
40use crate::error::{Error, Result};
41
42use crate::host::WorkspaceHost;
43
44#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
46pub struct SyncStatus {
47 pub enabled: bool,
49 pub workspace_id: Option<GrainId>,
51 pub peers: usize,
53 pub paused: Option<String>,
55 pub last_error: Option<String>,
57 pub bridge_available: bool,
59 pub topology: proto::Topology,
61 pub embedding: Option<proto::EmbeddingProgress>,
63}
64
65struct Synced {
67 workspace_id: GrainId,
69 replica: Arc<Mutex<Replica>>,
71 paused: Option<PauseReason>,
73 last_error: Option<String>,
75}
76
77const STALE_VECTOR_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
80
81const PASS_BATCH: usize = 4;
84
85#[derive(Clone, Copy, Debug)]
87pub struct BackfillTiming {
88 pub grace: std::time::Duration,
91 pub interval: std::time::Duration,
93}
94
95impl Default for BackfillTiming {
96 fn default() -> Self {
97 BackfillTiming {
98 grace: std::time::Duration::from_secs(10 * 60),
99 interval: std::time::Duration::from_secs(10 * 60),
100 }
101 }
102}
103
104pub const RECONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_millis(500);
107
108pub const RECONNECT_BACKOFF_MAX: std::time::Duration = std::time::Duration::from_secs(30);
111
112fn reconnect_backoff(attempt: u32) -> std::time::Duration {
114 RECONNECT_BACKOFF
115 .saturating_mul(1u32 << attempt.min(16))
116 .min(RECONNECT_BACKOFF_MAX)
117}
118
119type Connecting = std::pin::Pin<
121 Box<dyn std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>> + Send>,
122>;
123
124#[derive(Clone)]
129pub struct BridgeConnector(Arc<dyn Fn() -> Connecting + Send + Sync>);
130
131impl BridgeConnector {
132 pub fn new<F, Fut>(connect: F) -> BridgeConnector
134 where
135 F: Fn() -> Fut + Send + Sync + 'static,
136 Fut: std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>>
137 + Send
138 + 'static,
139 {
140 BridgeConnector(Arc::new(move || Box::pin(connect())))
141 }
142
143 pub fn standard(kind: &'static str, version: &'static str) -> BridgeConnector {
146 BridgeConnector::new(move || async move {
147 match BridgeClient::connect(kind, version).await {
148 Ok(client) => Ok(Some(client)),
149 Err(sapphire_ipc::Error::NotRunning(_)) => Ok(None),
150 Err(err) => Err(err),
151 }
152 })
153 }
154
155 pub fn at(
157 endpoint: sapphire_ipc::Endpoint,
158 kind: &'static str,
159 version: &'static str,
160 ) -> BridgeConnector {
161 BridgeConnector::new(move || {
162 let endpoint = endpoint.clone();
163 async move { BridgeClient::connect_at(&endpoint, kind, version).await }
164 })
165 }
166
167 async fn connect(&self) -> sapphire_ipc::Result<Option<BridgeClient>> {
168 (self.0)().await
169 }
170}
171
172impl std::fmt::Debug for BridgeConnector {
173 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
174 f.write_str("BridgeConnector")
175 }
176}
177
178pub struct SyncRuntime {
180 ctx: &'static AppContext,
181 bridge: tokio::sync::watch::Sender<Option<Arc<BridgeClient>>>,
184 connector: Option<BridgeConnector>,
186 pending: Mutex<HashMap<PathBuf, GrainId>>,
193 subscribed: std::sync::Mutex<Option<(Arc<BridgeClient>, broadcast::Receiver<IncomingParams>)>>,
198 exe_path: PathBuf,
199 managed_by: ManagedBy,
200 synced: Mutex<HashMap<PathBuf, Synced>>,
201 device_id: OnceCell<GrainId>,
207 watcher: OnceCell<Arc<Watcher>>,
217 host: OnceCell<Arc<WorkspaceHost>>,
228 reindexing: Mutex<()>,
231 embed_wanted: Mutex<std::collections::HashSet<PathBuf>>,
234 embedding: Mutex<()>,
236 embed_running: Mutex<std::collections::HashSet<PathBuf>>,
238 backfill: std::sync::Mutex<BackfillTiming>,
240 bridge_endpoint: Option<sapphire_ipc::Endpoint>,
242 live: Mutex<HashMap<PathBuf, Arc<LivePeers>>>,
248 last_peers: Mutex<Option<PeersResult>>,
251 open_timeout: std::time::Duration,
253 first_dials: std::sync::Mutex<tokio::task::JoinSet<()>>,
256 background: std::sync::Mutex<tokio::task::JoinSet<()>>,
261}
262
263impl SyncRuntime {
264 pub fn new(
269 ctx: &'static AppContext,
270 bridge: Arc<BridgeClient>,
271 exe_path: PathBuf,
272 managed_by: ManagedBy,
273 ) -> SyncRuntime {
274 SyncRuntime::build(ctx, Some(bridge), None, exe_path, managed_by)
275 }
276
277 pub fn connecting(
285 ctx: &'static AppContext,
286 connector: BridgeConnector,
287 exe_path: PathBuf,
288 managed_by: ManagedBy,
289 ) -> SyncRuntime {
290 SyncRuntime::build(ctx, None, Some(connector), exe_path, managed_by)
291 }
292
293 pub fn with_connector(mut self, connector: BridgeConnector) -> SyncRuntime {
295 self.connector = Some(connector);
296 self
297 }
298
299 fn build(
300 ctx: &'static AppContext,
301 bridge: Option<Arc<BridgeClient>>,
302 connector: Option<BridgeConnector>,
303 exe_path: PathBuf,
304 managed_by: ManagedBy,
305 ) -> SyncRuntime {
306 SyncRuntime {
307 ctx,
308 bridge: tokio::sync::watch::Sender::new(bridge),
309 connector,
310 pending: Mutex::new(HashMap::new()),
311 subscribed: std::sync::Mutex::new(None),
312 exe_path,
313 managed_by,
314 synced: Mutex::new(HashMap::new()),
315 device_id: OnceCell::new(),
316 watcher: OnceCell::new(),
317 host: OnceCell::new(),
318 reindexing: Mutex::new(()),
319 embed_wanted: Mutex::new(std::collections::HashSet::new()),
320 embedding: Mutex::new(()),
321 embed_running: Mutex::new(std::collections::HashSet::new()),
322 backfill: std::sync::Mutex::new(BackfillTiming::default()),
323 bridge_endpoint: None,
324 live: Mutex::new(HashMap::new()),
325 last_peers: Mutex::new(None),
326 open_timeout: OPEN_TIMEOUT,
327 first_dials: std::sync::Mutex::new(tokio::task::JoinSet::new()),
328 background: std::sync::Mutex::new(tokio::task::JoinSet::new()),
329 }
330 }
331
332 fn bridge(&self) -> Result<Arc<BridgeClient>> {
334 match &*self.bridge.borrow() {
335 Some(bridge) if !bridge.is_closed() => Ok(Arc::clone(bridge)),
336 _ => Err(Error::BridgeDisconnected),
337 }
338 }
339
340 pub fn is_connected(&self) -> bool {
342 self.bridge().is_ok()
343 }
344
345 fn spawn_background(&self, task: impl std::future::Future<Output = ()> + Send + 'static) {
347 let mut background = self.background.lock().expect("background tasks");
348 while background.try_join_next().is_some() {}
350 background.spawn(task);
351 }
352
353 pub fn with_open_timeout(mut self, timeout: std::time::Duration) -> SyncRuntime {
356 self.open_timeout = timeout;
357 self
358 }
359
360 pub fn with_bridge_endpoint(mut self, endpoint: sapphire_ipc::Endpoint) -> SyncRuntime {
367 self.bridge_endpoint = Some(endpoint);
368 self
369 }
370
371 pub fn set_backfill_timing(&self, timing: BackfillTiming) {
374 *self.backfill.lock().expect("backfill timing") = timing;
375 }
376
377 fn backfill_timing(&self) -> BackfillTiming {
378 *self.backfill.lock().expect("backfill timing")
379 }
380
381 pub(crate) fn set_host(&self, host: Arc<WorkspaceHost>) {
382 let _ = self.host.set(host);
383 }
384
385 pub async fn enable(self: &Arc<Self>, root: &Path) -> Result<GrainId> {
390 let key = root.canonicalize().map_err(Error::Io)?;
391 let watch_key = key.clone();
392 let workspace_id = sync_id(self.ctx.app_name, &key)?;
393
394 {
397 let mut pending = self.pending.lock().await;
398 if self.bridge().is_err() {
399 pending.insert(key, workspace_id);
400 return Err(Error::BridgeDisconnected);
401 }
402 pending.remove(&key);
403 }
404
405 {
408 let mut synced = self.synced.lock().await;
409 if let Some(existing) = synced.get(&key) {
410 return Ok(existing.workspace_id);
411 }
412 let device_id = self.device_id().await?;
413 let state_dir = self.state_dir(&key)?;
414 std::fs::create_dir_all(&state_dir).map_err(Error::Io)?;
415 let config = ReplicaConfig::new(self.ctx.app_name, key.clone(), device_id, &state_dir);
416 let replica = Replica::open(config, Arc::new(SystemClock))
417 .map_err(|e| Error::Sync(e.to_string()))?;
418 synced.insert(
419 key.clone(),
420 Synced {
421 workspace_id,
422 replica: Arc::new(Mutex::new(replica)),
423 paused: None,
424 last_error: None,
425 },
426 );
427 }
428
429 if let Some(watcher) = self.watcher.get()
433 && let Err(err) = watcher.watch(&watch_key)
434 {
435 tracing::warn!(root = %watch_key.display(), "could not watch: {err}");
436 }
437
438 self.reregister().await?;
441
442 let runtime = Arc::clone(self);
455 let mut dials = self.first_dials.lock().expect("first dials");
456 while dials.try_join_next().is_some() {}
458 dials.spawn(async move {
459 if let Err(err) = runtime.sync_now(&key).await {
460 tracing::warn!(root = %key.display(), "the first session after enabling failed: {err}");
461 }
462 });
463 Ok(workspace_id)
464 }
465
466 pub async fn disable(&self, root: &Path) -> Result<()> {
469 self.stop(root).await.1
470 }
471
472 pub async fn release(&self, root: &Path, limit: std::time::Duration) -> Result<()> {
481 let (replica, unregistered) = self.stop(root).await;
482 if let Err(err) = unregistered {
483 tracing::warn!(root = %root.display(), "stopping sync: {err}");
484 }
485 let Some(replica) = replica else {
486 return Ok(());
487 };
488 let deadline = tokio::time::Instant::now() + limit;
489 while Arc::strong_count(&replica) > 1 {
490 if tokio::time::Instant::now() >= deadline {
491 return Err(Error::Sync(format!(
492 "the replica of {} is still in use; try again",
493 root.display()
494 )));
495 }
496 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
497 }
498 Ok(())
499 }
500
501 async fn stop(&self, root: &Path) -> (Option<Arc<Mutex<Replica>>>, Result<()>) {
504 let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
507 self.pending.lock().await.remove(&key);
508 let removed = self.synced.lock().await.remove(&key);
509 if removed.is_some()
510 && let Some(watcher) = self.watcher.get()
511 {
512 watcher.unwatch(&key);
513 }
514 if let Some(peers) = self.live.lock().await.remove(&key) {
517 peers.drop_connections().await;
518 }
519 let Some(removed) = removed else {
520 return (None, Ok(()));
521 };
522 let unregistered = async {
523 let Ok(bridge) = self.bridge() else {
526 return Ok(());
527 };
528 bridge
529 .unregister(removed.workspace_id)
530 .await
531 .map_err(|e| Error::Bridge(e.to_string()))?;
532 self.reregister().await
533 }
534 .await;
535 (Some(removed.replica), unregistered)
536 }
537
538 pub async fn workspaces(&self) -> Result<WorkspacesResult> {
544 self.bridge()?
545 .workspaces()
546 .await
547 .map_err(|e| Error::Bridge(e.to_string()))
548 }
549
550 pub async fn place(&self, selector: &str, dir: &Path) -> Result<PathBuf> {
561 let workgroup = self.workspaces().await?;
562 let wanted = workgroup
563 .workspaces
564 .iter()
565 .find(|w| w.name == selector || w.workspace_id.to_string() == selector)
568 .ok_or_else(|| Error::UnknownWorkspaceName(selector.to_owned()))?;
569
570 if wanted.app_name != self.ctx.app_name {
573 return Err(Error::WrongApp {
574 name: wanted.name.clone(),
575 app_name: wanted.app_name.clone(),
576 });
577 }
578
579 let root = dir.canonicalize().map_err(Error::Io)?;
580 let marker = root.join(format!(".{}", self.ctx.app_name));
581 if !marker.is_dir() {
582 std::fs::create_dir(&marker).map_err(Error::Io)?;
583 }
584 let workspace = Workspace::from_root(self.ctx, &root)?;
585 if workspace.root != root {
586 return Err(Error::UnknownWorkspace(root, self.ctx.app_name));
587 }
588
589 {
593 let synced = self.synced.lock().await;
594 if let Some(entry) = synced.get(&root)
595 && entry.workspace_id != wanted.workspace_id
596 {
597 return Err(Error::SyncId(format!(
598 "{} already syncs as {}, not {}",
599 root.display(),
600 entry.workspace_id,
601 wanted.workspace_id
602 )));
603 }
604 }
605
606 let marker = workspace.marker_dir();
611 let id_path = marker.join(crate::sync::id::SYNC_ID_FILE);
612 match std::fs::read_to_string(&id_path) {
613 Ok(existing) if existing.trim() != wanted.workspace_id.to_string() => {
614 return Err(Error::SyncId(format!(
615 "{} already holds identity {}, not {}",
616 root.display(),
617 existing.trim(),
618 wanted.workspace_id
619 )));
620 }
621 Err(_) => {
622 std::fs::write(&id_path, format!("{}\n", wanted.workspace_id))
623 .map_err(Error::Io)?;
624 }
625 Ok(_) => {}
626 }
627
628 let map_path = root
631 .join(format!(".{}", self.ctx.app_name))
632 .join(WORKSPACE_MAP_FILE);
633 let workspace_id = wanted.workspace_id.to_string();
634 std::fs::write(&map_path, format!("{workspace_id}\n")).map_err(Error::Io)?;
635 Ok(root)
636 }
637
638 pub async fn status(&self, root: &Path) -> SyncStatus {
640 let bridge = self.bridge().ok();
641 let bridge_available = match &bridge {
642 Some(bridge) => bridge.status().await.is_ok(),
643 None => false,
644 };
645 let answer = match &bridge {
646 Some(_) => self.peers().await.ok(),
647 None => None,
648 };
649 let peers = answer
650 .as_ref()
651 .map_or(0, |p| p.peers.len().saturating_sub(1));
652 let Ok(key) = root.canonicalize() else {
653 return SyncStatus {
654 enabled: false,
655 workspace_id: None,
656 peers,
657 paused: None,
658 last_error: None,
659 bridge_available,
660 topology: proto::Topology::Mesh,
661 embedding: None,
662 };
663 };
664 let synced = self.synced.lock().await;
665 match synced.get(&key) {
666 Some(entry) => SyncStatus {
667 enabled: true,
668 workspace_id: Some(entry.workspace_id),
669 peers,
670 paused: entry.paused.map(|r| format!("{r:?}")),
671 last_error: entry.last_error.clone(),
672 bridge_available,
673 topology: topology::topology(
674 answer
675 .as_ref()
676 .and_then(|p| p.roles_for(entry.workspace_id)),
677 ),
678 embedding: self.embedding_progress(&key).await,
679 },
680 None => {
681 drop(synced);
682 let waiting = self.pending.lock().await.get(&key).copied();
685 SyncStatus {
686 enabled: waiting.is_some(),
687 workspace_id: waiting,
688 peers,
689 paused: None,
690 last_error: None,
691 bridge_available,
692 topology: proto::Topology::Mesh,
693 embedding: None,
694 }
695 }
696 }
697 }
698
699 async fn embedding_progress(&self, key: &Path) -> Option<proto::EmbeddingProgress> {
702 let host = self.host.get()?;
703 let backend = host.open_backend(key)?;
704 let state = backend.state();
705 state.embedder()?;
706 let info = state.db_info().ok()?;
707 Some(proto::EmbeddingProgress {
708 vectors: info.vector_count,
709 pending: info.pending_count,
710 running: self.embed_running.lock().await.contains(key),
711 })
712 }
713
714 pub async fn scan(&self, root: &Path) -> Result<()> {
720 let Ok(key) = root.canonicalize() else {
721 return Ok(());
722 };
723 let replica = {
724 let synced = self.synced.lock().await;
725 match synced.get(&key) {
726 Some(entry) => Arc::clone(&entry.replica),
727 None => return Ok(()),
728 }
729 };
730 let (outcome, updates) = {
731 let mut replica = replica.lock().await;
732 let outcome = replica.scan().map_err(|e| Error::Sync(e.to_string()))?;
733 let updates = match &outcome {
738 ScanOutcome::Scanned(report) => {
739 report
747 .recorded
748 .iter()
749 .map(|entry| PathUpdate {
750 path: entry.path.clone(),
751 versions: vec![entry.clone()],
752 seen: {
753 let mut seen = entry.context.clone();
754 seen.add_dot(&entry.dot);
755 seen
756 },
757 })
758 .collect()
759 }
760 ScanOutcome::Paused(_) => Vec::new(),
761 };
762 (outcome, updates)
763 };
764 {
765 let mut synced = self.synced.lock().await;
766 if let Some(entry) = synced.get_mut(&key) {
767 entry.paused = match outcome {
768 ScanOutcome::Paused(reason) => Some(reason),
769 ScanOutcome::Scanned(_) => None,
770 };
771 }
772 }
773 self.after_commit(&key, updates).await;
776 Ok(())
777 }
778
779 async fn reindex(&self, root: &Path) {
794 let Some(host) = self.host.get() else {
795 return;
796 };
797 let host = Arc::clone(host);
798 let root = root.to_owned();
799 let _serialised = self.reindexing.lock().await;
802 let backend = match host.backend_at(&root).await {
805 Ok(backend) => backend,
806 Err(err) => {
807 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
808 return;
809 }
810 };
811 let state = Arc::clone(backend.state());
812 let result = tokio::task::spawn_blocking(move || state.sync_retrieve()).await;
813 match result {
814 Ok(Ok(report)) => {
815 tracing::debug!(root = %root.display(), ?report, "re-indexed after a session");
816 }
817 Ok(Err(err)) => {
818 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
819 }
820 Err(err) => {
821 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
822 }
823 }
824 }
825
826 pub fn request_embed(self: &Arc<Self>, root: &Path) {
833 let Ok(key) = root.canonicalize() else {
834 return;
835 };
836 let runtime = Arc::clone(self);
837 self.spawn_background(async move {
838 if !runtime.embed_wanted.lock().await.insert(key.clone()) {
839 return;
840 }
841 let _one_at_a_time = runtime.embedding.lock().await;
842 runtime.embed_wanted.lock().await.remove(&key);
844 runtime.embed_running.lock().await.insert(key.clone());
845 runtime.embed_pass(&key).await;
846 runtime.embed_running.lock().await.remove(&key);
847 });
848 }
849
850 async fn embed_pass(&self, key: &Path) {
852 let Some(host) = self.host.get() else {
853 return;
854 };
855 let (workspace_id, replica) = {
856 let synced = self.synced.lock().await;
857 match synced.get(key) {
858 Some(entry) => (entry.workspace_id, Arc::clone(&entry.replica)),
859 None => return,
860 }
861 };
862 let backend = match host.backend_at(key).await {
863 Ok(backend) => backend,
864 Err(err) => {
865 tracing::warn!(root = %key.display(), "embedding pass: {err}");
866 return;
867 }
868 };
869 let state = Arc::clone(backend.state());
870 if let Err(err) = state
871 .refresh_embedder_at(self.bridge_endpoint.clone())
872 .await
873 {
874 tracing::warn!(root = %key.display(), "embedding pass: {err}");
875 return;
876 }
877 if state.embedder().is_none() {
878 return;
879 }
880 let primary = self.is_primary(workspace_id).await;
881 let grace_ms = self.backfill_timing().grace.as_millis() as u64;
884 let now_ms = std::time::SystemTime::now()
885 .duration_since(std::time::UNIX_EPOCH)
886 .map_or(0, |d| d.as_millis() as u64);
887 let (mine, settled, live) = {
888 let replica = replica.lock().await;
889 let me = replica.replica_id();
890 let states = match replica.states() {
891 Ok(states) => states,
892 Err(err) => {
893 tracing::warn!(root = %key.display(), "embedding pass: {err}");
894 return;
895 }
896 };
897 let mut mine = HashMap::new();
898 let mut settled = HashMap::new();
900 let mut live = std::collections::HashSet::new();
901 for (path, path_state) in states {
902 let winner = path_state.winner();
903 let Some(hash) = winner.content.hash() else {
904 continue;
905 };
906 let hash = hash.to_hex();
907 if winner.dot.replica == me {
908 mine.insert(path, hash.clone());
909 } else if now_ms.saturating_sub(winner.hlc.wall_ms) >= grace_ms {
910 settled.insert(path, hash.clone());
911 }
912 live.insert(hash);
913 }
914 (mine, settled, live)
915 };
916 let root = key.to_owned();
917 let result = tokio::task::spawn_blocking(move || {
918 let policy = |rel: &str, hash: &str| {
921 mine.get(rel).is_some_and(|h| h.as_str() == hash)
922 || (primary && settled.get(rel).is_some_and(|h| h.as_str() == hash))
923 };
924 let embedded = state.embed_pending_with(&policy, PASS_BATCH)?;
925 let removed = if primary {
926 state.remove_stale_vectors(&live, STALE_VECTOR_AGE)
927 } else {
928 0
929 };
930 Ok::<_, sapphire_workspace::Error>((embedded, removed))
931 })
932 .await;
933 match result {
934 Ok(Ok((embedded, removed))) if embedded > 0 || removed > 0 => tracing::info!(
935 root = %root.display(),
936 embedded,
937 removed,
938 "embedding pass"
939 ),
940 Ok(Ok(_)) => {}
941 Ok(Err(err)) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
942 Err(err) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
943 }
944 }
945
946 pub async fn sync_now(self: &Arc<Self>, root: &Path) -> Result<()> {
948 let Ok(key) = root.canonicalize() else {
949 return Ok(());
950 };
951 let workspace_id = {
952 let synced = self.synced.lock().await;
953 match synced.get(&key) {
954 Some(entry) => entry.workspace_id,
955 None => return Ok(()),
956 }
957 };
958 let peers = self.peers().await?;
959 let me = self.device_id().await?;
960
961 let roles = peers.roles_for(workspace_id).cloned();
966 let mut dials: Vec<Dial<'_>> = Vec::new();
970 for peer in peers
971 .peers
972 .iter()
973 .filter(|p| topology::link(me, p.device_id, roles.as_ref()) == topology::Link::Dial)
974 .cloned()
975 .collect::<Vec<_>>()
976 {
977 if let Some(live) = self.live_peers(&key).await
981 && live.has(&peer.device_id).await
982 {
983 continue;
984 }
985 let Some((_, table)) = self.live_peers_or_create(&key).await else {
992 continue;
993 };
994 if !table.begin_dial(peer.device_id).await {
995 continue;
996 }
997 let key = &key;
998 dials.push(Box::pin(async move {
999 let result = self
1001 .sync_one(key, &peer.device_id, peer.name.clone(), workspace_id)
1002 .await;
1003 table.end_dial(peer.device_id).await;
1004 if let Err(err) = result {
1005 tracing::debug!(peer = %peer.name, "no session: {err}");
1006 }
1007 }));
1008 }
1009 join_all(dials).await;
1010 Ok(())
1011 }
1012
1013 async fn sync_one(
1020 self: &Arc<Self>,
1021 key: &Path,
1022 device: &GrainId,
1023 name: String,
1024 workspace_id: GrainId,
1025 ) -> Result<()> {
1026 let stream = self.open_stream(workspace_id, *device).await?;
1027 self.adopt_live_session(stream, key, *device, workspace_id)
1028 .await
1029 .map_err(|err| {
1030 tracing::warn!(peer = %name, "a live session could not be started: {err}");
1031 err
1032 })
1033 }
1034
1035 async fn open_stream(
1037 &self,
1038 workspace_id: GrainId,
1039 device: GrainId,
1040 ) -> Result<sapphire_ipc::RawStream> {
1041 match tokio::time::timeout(
1042 self.open_timeout,
1043 self.bridge()?.open_stream(workspace_id, device),
1044 )
1045 .await
1046 {
1047 Ok(opened) => Ok(opened?),
1048 Err(_) => Err(Error::Bridge(format!(
1049 "no stream to {device} within {} s; it may be offline",
1050 self.open_timeout.as_secs_f32()
1051 ))),
1052 }
1053 }
1054
1055 pub async fn run(self: Arc<Self>) -> Result<()> {
1058 let mut slot = self.bridge.subscribe();
1059 loop {
1060 let bridge = slot.borrow_and_update().clone();
1061 match bridge {
1062 Some(bridge) if !bridge.is_closed() => self.answer(&bridge, &mut slot).await,
1063 _ => {
1065 if slot.changed().await.is_err() {
1066 return Ok(());
1067 }
1068 }
1069 }
1070 }
1071 }
1072
1073 async fn answer(
1075 self: &Arc<Self>,
1076 bridge: &Arc<BridgeClient>,
1077 slot: &mut tokio::sync::watch::Receiver<Option<Arc<BridgeClient>>>,
1078 ) {
1079 let early = self
1080 .subscribed
1081 .lock()
1082 .expect("subscribed")
1083 .take_if(|(b, _)| Arc::ptr_eq(b, bridge));
1084 let mut incoming = match early {
1085 Some((_, incoming)) => incoming,
1086 None => bridge.incoming(),
1087 };
1088 loop {
1089 let received = tokio::select! {
1090 received = incoming.recv() => received,
1091 () = bridge.closed() => return,
1092 _ = slot.changed() => return,
1093 };
1094 let announcement = match received {
1095 Ok(a) => a,
1096 Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
1097 tracing::warn!(missed = n, "fell behind on bridge announcements");
1098 continue;
1099 }
1100 Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1101 };
1102
1103 let found = {
1104 let synced = self.synced.lock().await;
1105 synced
1106 .iter()
1107 .find(|(_, s)| s.workspace_id == announcement.workspace_id)
1108 .map(|(root, s)| (root.clone(), Arc::clone(&s.replica)))
1109 };
1110 let Some((root, replica)) = found else {
1113 tracing::debug!(
1116 workspace = %announcement.workspace_id,
1117 "an announcement for a workspace this server does not hold"
1118 );
1119 continue;
1120 };
1121
1122 let bridge = Arc::clone(bridge);
1123 let workspace_id = announcement.workspace_id;
1124 let peer = announcement.peer_device_id;
1125 let driver = Arc::clone(self);
1126 self.spawn_background(async move {
1127 let stream = match bridge.accept_stream(announcement.ticket).await {
1128 Ok(stream) => stream,
1129 Err(err) => {
1130 tracing::warn!("could not claim an announced stream: {err}");
1131 return;
1132 }
1133 };
1134 if driver.should_skip(workspace_id, peer).await {
1135 tracing::debug!(%peer, "dropped an inbound session: both ends are outside this workspace's star");
1136 return;
1137 }
1138 let (_, session) = match sapphire_framework_session::open_live_session(
1143 stream,
1144 Arc::clone(&replica),
1145 workspace_id,
1146 )
1147 .await
1148 {
1149 Ok(started) => started,
1150 Err(err) => {
1151 tracing::warn!("an inbound session failed: {err}");
1152 return;
1153 }
1154 };
1155 let Some((key, table)) = driver.live_peers_or_create(&root).await else {
1156 return;
1157 };
1158 let Some(updates) = table.insert(peer, session, false).await else {
1162 return;
1163 };
1164 table.catch_up(peer, &replica).await;
1165 driver.reindex(&key).await;
1168 driver.request_embed(&key);
1169 driver.spawn_reader(table, updates, key, peer);
1170 });
1171 }
1172 }
1173
1174 pub async fn connect(self: &Arc<Self>) -> Result<()> {
1181 let Some(connector) = &self.connector else {
1182 return Ok(());
1183 };
1184 if self.is_connected() {
1185 return Ok(());
1186 }
1187 match connector.connect().await? {
1188 Some(client) => self.attach(client).await,
1189 None => tracing::warn!(
1190 "no sapphire-bridge is running; serving without sync until it starts"
1191 ),
1192 }
1193 Ok(())
1194 }
1195
1196 pub async fn keep_connected(self: Arc<Self>) {
1204 let mut attempt = 0;
1205 loop {
1206 if let Ok(bridge) = self.bridge() {
1207 bridge.closed().await;
1208 tracing::warn!("lost the connection to sapphire-bridge; reconnecting");
1209 self.bridge.send_if_modified(|slot| {
1210 let same = slot.as_ref().is_some_and(|b| Arc::ptr_eq(b, &bridge));
1211 if same {
1212 *slot = None;
1213 }
1214 same
1215 });
1216 attempt = 0;
1217 continue;
1218 }
1219 let Some(connector) = &self.connector else {
1220 return;
1221 };
1222 match connector.connect().await {
1223 Ok(Some(client)) => {
1224 tracing::info!("connected to sapphire-bridge");
1225 self.attach(client).await;
1226 attempt = 0;
1227 continue;
1228 }
1229 Ok(None) => tracing::debug!("no sapphire-bridge is running yet"),
1230 Err(err) if attempt == 0 => {
1232 tracing::warn!("could not connect to sapphire-bridge: {err}")
1233 }
1234 Err(err) => tracing::debug!("could not connect to sapphire-bridge: {err}"),
1235 }
1236 tokio::time::sleep(reconnect_backoff(attempt)).await;
1237 attempt = attempt.saturating_add(1);
1238 }
1239 }
1240
1241 async fn attach(self: &Arc<Self>, client: BridgeClient) {
1244 let client = Arc::new(client);
1247 *self.subscribed.lock().expect("subscribed") =
1248 Some((Arc::clone(&client), client.incoming()));
1249 let waiting = {
1250 let mut pending = self.pending.lock().await;
1251 self.bridge.send_replace(Some(client));
1252 std::mem::take(&mut *pending)
1253 };
1254 if !self.synced.lock().await.is_empty()
1256 && let Err(err) = self.reregister().await
1257 {
1258 tracing::warn!("could not register with sapphire-bridge: {err}");
1259 }
1260 for root in waiting.into_keys() {
1262 if let Err(err) = self.enable(&root).await {
1263 tracing::warn!(root = %root.display(), "could not start sync: {err}");
1264 }
1265 }
1266 }
1267
1268 pub async fn roots(&self) -> Vec<PathBuf> {
1270 self.synced.lock().await.keys().cloned().collect()
1271 }
1272
1273 async fn live_peers(&self, root: &Path) -> Option<Arc<LivePeers>> {
1278 let key = root.canonicalize().ok()?;
1279 self.live.lock().await.get(&key).cloned()
1280 }
1281
1282 async fn live_peers_or_create(&self, root: &Path) -> Option<(PathBuf, Arc<LivePeers>)> {
1287 let key = root.canonicalize().ok()?;
1288 if !self.synced.lock().await.contains_key(&key) {
1289 return None;
1290 }
1291 let me = self.device_id().await.ok()?;
1294 let mut live = self.live.lock().await;
1295 let peers = Arc::clone(
1296 live.entry(key.clone())
1297 .or_insert_with(|| Arc::new(LivePeers::new(me))),
1298 );
1299 Some((key, peers))
1300 }
1301
1302 pub async fn after_commit(&self, root: &Path, updates: Vec<PathUpdate>) {
1312 if updates.is_empty() {
1313 return;
1314 }
1315 if let Some(peers) = self.live_peers(root).await {
1316 peers.fan_out(&updates, None).await;
1317 }
1318 }
1319
1320 pub async fn live_session_devices(&self, root: &Path) -> Vec<GrainId> {
1324 match self.live_peers(root).await {
1325 Some(peers) => peers.devices().await,
1326 None => Vec::new(),
1327 }
1328 }
1329
1330 pub async fn stop_dials(&self) {
1337 let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1338 dials.abort_all();
1339 while dials.join_next().await.is_some() {}
1340 }
1341
1342 pub async fn stop_background(&self) {
1350 let mut background =
1351 std::mem::take(&mut *self.background.lock().expect("background tasks"));
1352 background.abort_all();
1353 while background.join_next().await.is_some() {}
1354 }
1355
1356 #[cfg(test)]
1359 async fn settle_dials(&self) {
1360 let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1361 while dials.join_next().await.is_some() {}
1362 }
1363
1364 pub async fn drop_connections(&self) {
1369 let tables: Vec<Arc<LivePeers>> = self.live.lock().await.values().cloned().collect();
1370 for peers in tables {
1371 peers.drop_connections().await;
1372 }
1373 }
1374
1375 pub async fn dial_loop(self: Arc<Self>) -> Result<()> {
1390 let mut tick = tokio::time::interval(live::DIAL_INTERVAL);
1391 tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1394 let mut waiting: HashMap<GrainId, std::time::Instant> = HashMap::new();
1395 let mut failures: HashMap<GrainId, u32> = HashMap::new();
1396 let mut primary_of: std::collections::HashSet<GrainId> = Default::default();
1398 let mut last_backfill = std::time::Instant::now();
1400 loop {
1401 tick.tick().await;
1402 let roots = self.roots().await;
1403 if roots.is_empty() {
1404 continue;
1405 }
1406 let peers = match self.peers().await {
1407 Ok(peers) => peers,
1408 Err(err) => {
1411 tracing::debug!("no peers to dial: {err}");
1412 continue;
1413 }
1414 };
1415 let me = match self.device_id().await {
1416 Ok(me) => me,
1417 Err(err) => {
1418 tracing::warn!("no device id, so nobody to dial: {err}");
1419 continue;
1420 }
1421 };
1422 let ids: Vec<(PathBuf, GrainId)> = self
1426 .synced
1427 .lock()
1428 .await
1429 .iter()
1430 .map(|(root, entry)| (root.clone(), entry.workspace_id))
1431 .collect();
1432 let backfill_due = last_backfill.elapsed() >= self.backfill_timing().interval;
1435 if backfill_due {
1436 last_backfill = std::time::Instant::now();
1437 }
1438 for (root, workspace_id) in ids {
1439 let primary = peers
1440 .roles_for(workspace_id)
1441 .is_some_and(|r| r.primary == Some(me));
1442 if !primary {
1443 primary_of.remove(&workspace_id);
1444 } else if primary_of.insert(workspace_id) || backfill_due {
1445 self.request_embed(&root);
1446 }
1447 }
1448 let now = std::time::Instant::now();
1449 for root in roots {
1458 let Some((key, table)) = self.live_peers_or_create(&root).await else {
1459 continue;
1460 };
1461 let workspace_id = {
1462 let synced = self.synced.lock().await;
1463 match synced.get(&key) {
1464 Some(entry) => entry.workspace_id,
1465 None => continue,
1466 }
1467 };
1468 let roles = peers.roles_for(workspace_id);
1469 if topology::may_close_skipped(me, roles, &table.devices().await) {
1472 table
1473 .retain(|d| topology::link(me, *d, roles) != topology::Link::Skip)
1474 .await;
1475 }
1476 for peer in peers.peers.iter().filter(|p| {
1477 p.connected && topology::link(me, p.device_id, roles) == topology::Link::Dial
1478 }) {
1479 let device = peer.device_id;
1480 if table.has(&device).await {
1481 failures.remove(&device);
1483 waiting.remove(&device);
1484 continue;
1485 }
1486 if let Some(until) = waiting.get(&device)
1487 && now < *until
1488 {
1489 continue;
1491 }
1492 if !table.begin_dial(device).await {
1496 continue;
1497 }
1498 let opened = self.open_stream(workspace_id, device).await;
1499 if opened.is_err() {
1500 table.end_dial(device).await;
1501 }
1502 match opened {
1503 Ok(stream) => {
1504 let adopted = self
1505 .adopt_live_session(stream, &root, device, workspace_id)
1506 .await;
1507 table.end_dial(device).await;
1510 match adopted {
1511 Ok(()) => {
1512 failures.remove(&device);
1513 waiting.remove(&device);
1514 }
1515 Err(err) => {
1516 tracing::debug!(
1517 peer = %peer.name,
1518 err = %err,
1519 "a live session could not be started"
1520 );
1521 let count = failures.entry(device).or_insert(0);
1522 *count = count.saturating_add(1);
1523 waiting.insert(device, now + dial_backoff(*count));
1524 }
1525 }
1526 }
1527 Err(err) => {
1528 tracing::debug!("no stream to a peer: {err}");
1531 let count = failures.entry(device).or_insert(0);
1532 *count = count.saturating_add(1);
1533 waiting.insert(device, now + dial_backoff(*count));
1534 }
1535 }
1536 }
1537 }
1538 }
1539 }
1540
1541 async fn adopt_live_session(
1546 self: &Arc<Self>,
1547 stream: sapphire_ipc::RawStream,
1548 root: &Path,
1549 device: GrainId,
1550 workspace_id: GrainId,
1551 ) -> Result<()> {
1552 let replica = self.replica_of(root).await?;
1553 let (_, session) = sapphire_framework_session::open_live_session(
1554 stream,
1555 Arc::clone(&replica),
1556 workspace_id,
1557 )
1558 .await
1559 .map_err(|e| Error::Sync(e.to_string()))?;
1560 let Some((key, table)) = self.live_peers_or_create(root).await else {
1561 return Ok(());
1562 };
1563 let Some(updates) = table.insert(device, session, true).await else {
1568 return Ok(());
1569 };
1570 table.catch_up(device, &replica).await;
1571 self.reindex(&key).await;
1574 self.request_embed(&key);
1575 self.spawn_reader(Arc::clone(&table), updates, key, device);
1576 Ok(())
1577 }
1578
1579 fn spawn_reader(
1586 self: &Arc<Self>,
1587 table: Arc<LivePeers>,
1588 mut updates: broadcast::Receiver<Vec<PathUpdate>>,
1589 key: PathBuf,
1590 from: GrainId,
1591 ) {
1592 let driver = Arc::clone(self);
1593 self.spawn_background(async move {
1594 loop {
1595 match updates.recv().await {
1596 Ok(batch) => {
1597 driver.reindex(&key).await;
1600 driver.request_embed(&key);
1601 table.fan_out(&batch, Some(from)).await;
1602 }
1603 Err(broadcast::error::RecvError::Lagged(missed)) => {
1608 tracing::debug!(
1609 device = %from,
1610 missed,
1611 "fell behind on a live session; the next exchange catches up"
1612 );
1613 }
1614 Err(broadcast::error::RecvError::Closed) => {
1617 return;
1618 }
1619 }
1620 }
1621 });
1622 }
1623
1624 async fn replica_of(&self, root: &Path) -> Result<Arc<Mutex<Replica>>> {
1626 let key = root.canonicalize().map_err(Error::Io)?;
1627 let synced = self.synced.lock().await;
1628 match synced.get(&key) {
1629 Some(entry) => Ok(Arc::clone(&entry.replica)),
1630 None => Err(Error::UnknownWorkspace(key, self.ctx.app_name)),
1631 }
1632 }
1633
1634 pub async fn watch_changes(self: Arc<Self>) -> Result<()> {
1646 let (tx, mut rx) = mpsc::channel(64);
1647 let watcher = Arc::new(Watcher::start(self.roots().await, tx)?);
1648 let _ = self.watcher.set(Arc::clone(&watcher));
1649
1650 for root in self.roots().await {
1654 if let Err(err) = watcher.watch(&root) {
1655 tracing::warn!(root = %root.display(), "could not watch: {err}");
1656 }
1657 }
1658
1659 while let Some(root) = rx.recv().await {
1660 if let Err(err) = self.scan(&root).await {
1663 tracing::warn!(root = %root.display(), "scan after a local edit failed: {err}");
1664 }
1665 self.request_embed(&root);
1666 if let Err(err) = self.sync_now(&root).await {
1667 tracing::warn!(root = %root.display(), "dial after a local edit failed: {err}");
1668 }
1669 }
1670 Ok(())
1671 }
1672
1673 async fn reregister(&self) -> Result<()> {
1675 let workspaces: Vec<WorkspaceRegistration> = {
1676 let synced = self.synced.lock().await;
1677 synced
1678 .iter()
1679 .map(|(root, entry)| WorkspaceRegistration {
1680 workspace_id: entry.workspace_id,
1681 root: root.clone(),
1682 })
1683 .collect()
1684 };
1685 self.bridge()?
1686 .register(RegisterParams {
1687 app_name: self.ctx.app_name.to_owned(),
1688 exe_path: self.exe_path.clone(),
1689 managed_by: wire_managed_by(self.managed_by),
1690 workspaces,
1691 })
1692 .await
1693 .map_err(|e| Error::Bridge(e.to_string()))?;
1694 Ok(())
1695 }
1696
1697 async fn should_skip(&self, workspace_id: GrainId, peer: GrainId) -> bool {
1702 let Some(&me) = self.device_id.get() else {
1703 return false;
1704 };
1705 let cached = self.last_peers.lock().await;
1706 let roles = cached.as_ref().and_then(|p| p.roles_for(workspace_id));
1707 topology::link(me, peer, roles) == topology::Link::Skip
1708 }
1709
1710 pub async fn is_primary(&self, workspace_id: GrainId) -> bool {
1716 let Some(&me) = self.device_id.get() else {
1717 return false;
1718 };
1719 let cached = self.last_peers.lock().await;
1720 cached
1721 .as_ref()
1722 .and_then(|p| p.roles_for(workspace_id))
1723 .is_some_and(|r| r.primary == Some(me))
1724 }
1725
1726 pub fn this_device(&self) -> Option<GrainId> {
1733 self.device_id.get().copied()
1734 }
1735
1736 pub async fn peers(&self) -> Result<PeersResult> {
1742 let peers = self
1743 .bridge()?
1744 .peers()
1745 .await
1746 .map_err(|e| Error::Bridge(e.to_string()))?;
1747 *self.last_peers.lock().await = Some(peers.clone());
1748 Ok(peers)
1749 }
1750
1751 async fn device_id(&self) -> Result<GrainId> {
1753 if let Some(id) = self.device_id.get() {
1754 return Ok(*id);
1755 }
1756 let result = self
1757 .bridge()?
1758 .register(RegisterParams {
1759 app_name: self.ctx.app_name.to_owned(),
1760 exe_path: self.exe_path.clone(),
1761 managed_by: wire_managed_by(self.managed_by),
1762 workspaces: Vec::new(),
1763 })
1764 .await
1765 .map_err(|e| Error::Bridge(e.to_string()))?;
1766 let _ = self.device_id.set(result.device_id);
1767 Ok(result.device_id)
1768 }
1769
1770 fn state_dir(&self, root: &Path) -> Result<PathBuf> {
1772 let workspace = Workspace::from_root(self.ctx, root)?;
1773 Ok(workspace.cache_dir().join("sync"))
1774 }
1775}
1776
1777type Dial<'a> = std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>>;
1779
1780async fn join_all(mut dials: Vec<Dial<'_>>) {
1782 std::future::poll_fn(|cx| {
1783 dials.retain_mut(|dial| dial.as_mut().poll(cx).is_pending());
1784 if dials.is_empty() {
1785 std::task::Poll::Ready(())
1786 } else {
1787 std::task::Poll::Pending
1788 }
1789 })
1790 .await
1791}
1792
1793fn wire_managed_by(managed_by: ManagedBy) -> sapphire_bridge_api::ManagedBy {
1795 match managed_by {
1796 ManagedBy::Service => sapphire_bridge_api::ManagedBy::Service,
1797 ManagedBy::Spawned => sapphire_bridge_api::ManagedBy::Spawned,
1798 }
1799}
1800
1801#[cfg(test)]
1802mod tests {
1803 use super::*;
1804 use crate::sync::testing::StubBridge;
1805 use crate::test_support;
1806 use sapphire_workspace::AppKind;
1807 use std::ffi::OsString;
1808
1809 static CTX: AppContext = AppContext::new("sapphire-synctest");
1810
1811 const DIR_VARS: [&str; 3] = [
1813 "SAPPHIRE_SYNCTEST_CACHE_DIR",
1814 "SAPPHIRE_SYNCTEST_DATA_DIR",
1815 "SAPPHIRE_SYNCTEST_CONFIG_DIR",
1816 ];
1817
1818 struct EnvGuard {
1821 previous: [Option<OsString>; 3],
1822 _lock: std::sync::MutexGuard<'static, ()>,
1823 }
1824
1825 impl Drop for EnvGuard {
1826 fn drop(&mut self) {
1827 for (name, previous) in DIR_VARS.iter().zip(self.previous.iter_mut()) {
1830 match previous.take() {
1831 Some(value) => unsafe { std::env::set_var(name, value) },
1832 None => test_support::remove(name),
1833 }
1834 }
1835 }
1836 }
1837
1838 struct Fixture {
1846 _tmp: tempfile::TempDir,
1847 _env: EnvGuard,
1848 root: PathBuf,
1849 stub: StubBridge,
1850 runtime: Arc<SyncRuntime>,
1851 }
1852
1853 impl Fixture {
1854 fn second_root(&self) -> PathBuf {
1856 let root = self._tmp.path().join("ws2");
1857 std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
1858 root.canonicalize().unwrap()
1859 }
1860 }
1861
1862 const FIXTURE_OPEN_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
1864
1865 fn dialled_peers(me: GrainId, n: usize) -> Vec<sapphire_bridge_api::PeerInfo> {
1867 std::iter::repeat_with(GrainId::random)
1868 .filter(|id| *id > me)
1869 .take(n)
1870 .enumerate()
1871 .map(|(i, device_id)| sapphire_bridge_api::PeerInfo {
1872 device_id,
1873 name: format!("offline-{i}"),
1874 node_id: format!("node-{i}"),
1875 connected: false,
1876 priority: 1,
1877 availability: None,
1878 })
1879 .collect()
1880 }
1881
1882 #[tokio::test(flavor = "multi_thread")]
1883 async fn enabling_answers_at_once_with_a_peer_that_never_answers() {
1884 let f = fixture().await;
1886 let _hung = f.stub.hang_streams().await;
1887 f.stub.set_peers(dialled_peers(f.stub.device_id, 1));
1888
1889 let started = std::time::Instant::now();
1890 f.runtime.enable(&f.root).await.unwrap();
1891 assert!(
1892 started.elapsed() < FIXTURE_OPEN_TIMEOUT,
1893 "enable waited {:?} on the dial",
1894 started.elapsed()
1895 );
1896 }
1897
1898 #[tokio::test(flavor = "multi_thread")]
1899 async fn peers_that_never_answer_are_dialled_together_and_given_up_on() {
1900 let f = fixture().await;
1901 let _hung = f.stub.hang_streams().await;
1902 f.runtime.enable(&f.root).await.unwrap();
1903 f.runtime.settle_dials().await;
1906 f.stub.set_peers(dialled_peers(f.stub.device_id, 3));
1907
1908 let started = std::time::Instant::now();
1909 f.runtime.sync_now(&f.root).await.unwrap();
1910 let took = started.elapsed();
1911 assert!(took >= FIXTURE_OPEN_TIMEOUT, "gave up early: {took:?}");
1912 assert!(
1913 took < FIXTURE_OPEN_TIMEOUT * 2,
1914 "three dials one after another: {took:?}"
1915 );
1916 }
1917
1918 async fn fixture() -> Fixture {
1919 let (tmp, env, root) = scratch();
1920 let (stub, client) = StubBridge::start_in(tmp.path().join("run")).await;
1921 let runtime = Arc::new(
1922 SyncRuntime::new(&CTX, client, "/bin/true".into(), ManagedBy::Service)
1923 .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
1924 );
1925 Fixture {
1926 _tmp: tmp,
1927 _env: env,
1928 root,
1929 stub,
1930 runtime,
1931 }
1932 }
1933
1934 type Bridges = Arc<std::sync::Mutex<std::collections::VecDeque<BridgeClient>>>;
1937
1938 async fn connecting() -> (Fixture, Bridges, BridgeClient) {
1941 let (tmp, env, root) = scratch();
1942 let (stub, client) = StubBridge::start_client_in(tmp.path().join("run")).await;
1943 let bridges: Bridges = Arc::default();
1944 let connector = BridgeConnector::new({
1945 let bridges = Arc::clone(&bridges);
1946 move || {
1947 let next = bridges.lock().unwrap().pop_front();
1948 async move { Ok(next) }
1949 }
1950 });
1951 let runtime = Arc::new(
1952 SyncRuntime::connecting(&CTX, connector, "/bin/true".into(), ManagedBy::Service)
1953 .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
1954 );
1955 let f = Fixture {
1956 _tmp: tmp,
1957 _env: env,
1958 root,
1959 stub,
1960 runtime,
1961 };
1962 (f, bridges, client)
1963 }
1964
1965 async fn eventually(what: &str, mut done: impl AsyncFnMut() -> bool) {
1967 let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
1968 while !done().await {
1969 assert!(tokio::time::Instant::now() < deadline, "never: {what}");
1970 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
1971 }
1972 }
1973
1974 #[tokio::test(flavor = "multi_thread")]
1975 async fn a_runtime_started_without_a_bridge_syncs_once_one_starts() {
1976 let (f, bridges, client) = connecting().await;
1978 f.runtime.connect().await.unwrap();
1979 assert!(!f.runtime.is_connected());
1980
1981 let err = f.runtime.enable(&f.root).await.unwrap_err();
1982 assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
1983 let status = f.runtime.status(&f.root).await;
1984 assert!(status.enabled, "enabled, waiting for the bridge");
1985 assert!(!status.bridge_available);
1986 let id = status.workspace_id.expect("its id");
1987
1988 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
1989 bridges.lock().unwrap().push_back(client);
1990 eventually("the waiting workspace is registered", async || {
1991 f.stub.last_workspaces() == vec![id]
1992 })
1993 .await;
1994 assert_eq!(f.runtime.roots().await, vec![f.root.clone()]);
1995 assert!(f.runtime.status(&f.root).await.bridge_available);
1996 keeping.abort();
1997 }
1998
1999 #[tokio::test(flavor = "multi_thread")]
2000 async fn a_runtime_that_loses_its_bridge_registers_with_the_next_one() {
2001 let (f, bridges, client) = connecting().await;
2002 bridges.lock().unwrap().push_back(client);
2003 f.runtime.connect().await.unwrap();
2004 assert!(f.runtime.is_connected());
2005 let id = f.runtime.enable(&f.root).await.unwrap();
2006 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2007
2008 f.stub.disconnect();
2009 eventually("the runtime notices", async || !f.runtime.is_connected()).await;
2010 let err = f.runtime.workspaces().await.unwrap_err();
2011 assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2012
2013 let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2014 bridges.lock().unwrap().push_back(client);
2015 eventually("the synced workspace is registered again", async || {
2016 next.last_workspaces() == vec![id]
2017 })
2018 .await;
2019 assert!(f.runtime.is_connected());
2020 keeping.abort();
2021 }
2022
2023 #[tokio::test(flavor = "multi_thread")]
2024 async fn this_device_is_known_once_a_workspace_is_enabled_and_never_asked_for() {
2025 let f = fixture().await;
2026 assert_eq!(f.runtime.this_device(), None);
2027 assert!(
2028 f.stub.seen.lock().unwrap().registrations.is_empty(),
2029 "asking must not register"
2030 );
2031
2032 f.runtime.enable(&f.root).await.unwrap();
2033 assert_eq!(f.runtime.this_device(), Some(f.stub.device_id));
2034 }
2035
2036 #[tokio::test(flavor = "multi_thread")]
2037 async fn peers_answers_the_workgroup_from_the_bridge() {
2038 let f = fixture().await;
2039 let listed = dialled_peers(f.stub.device_id, 2);
2040 f.stub.set_peers(listed.clone());
2041
2042 let peers = f.runtime.peers().await.unwrap();
2043 let ids: Vec<_> = peers.peers.iter().map(|p| p.device_id).collect();
2044 assert_eq!(ids, listed.iter().map(|p| p.device_id).collect::<Vec<_>>());
2045 }
2046
2047 #[tokio::test(flavor = "multi_thread")]
2048 async fn peers_says_so_while_the_bridge_is_away() {
2049 let (f, _bridges, _client) = connecting().await;
2050 let err = f.runtime.peers().await.unwrap_err();
2051 assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2052 }
2053
2054 #[tokio::test(flavor = "multi_thread")]
2055 async fn disabling_while_the_bridge_is_away_forgets_the_waiting_workspace() {
2056 let (f, bridges, client) = connecting().await;
2057 f.runtime.enable(&f.root).await.unwrap_err();
2058 f.runtime.disable(&f.root).await.unwrap();
2059 assert!(!f.runtime.status(&f.root).await.enabled);
2060
2061 bridges.lock().unwrap().push_back(client);
2062 f.runtime.connect().await.unwrap();
2063 assert!(f.runtime.roots().await.is_empty());
2064 assert!(f.stub.last_workspaces().is_empty());
2065 }
2066
2067 #[tokio::test(flavor = "multi_thread")]
2068 async fn announcements_are_answered_from_a_reconnected_bridge() {
2069 let (f, bridges, client) = connecting().await;
2070 bridges.lock().unwrap().push_back(client);
2071 f.runtime.connect().await.unwrap();
2072 let id = f.runtime.enable(&f.root).await.unwrap();
2073 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2074 let answering = tokio::spawn(Arc::clone(&f.runtime).run());
2075
2076 f.stub.disconnect();
2077 let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2078 bridges.lock().unwrap().push_back(client);
2079 eventually("reconnected", async || next.last_workspaces() == vec![id]).await;
2080
2081 next.announce(id, "ticket").await;
2084 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2085 assert!(!answering.is_finished(), "the announcement loop ended");
2086 answering.abort();
2087 keeping.abort();
2088 }
2089
2090 fn scratch() -> (tempfile::TempDir, EnvGuard, PathBuf) {
2092 let lock = test_support::lock();
2093 let tmp = tempfile::tempdir().unwrap();
2094 let previous = DIR_VARS.map(std::env::var_os);
2095 for (name, dir) in DIR_VARS
2099 .iter()
2100 .zip(["cache", "data", "config"].map(|cat| tmp.path().join(cat)))
2101 {
2102 test_support::set(name, &dir);
2103 }
2104 let env = EnvGuard {
2105 previous,
2106 _lock: lock,
2107 };
2108 CTX.init(AppKind::Server);
2109
2110 let root = tmp.path().join("ws");
2111 std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
2112 let root = root.canonicalize().unwrap();
2113 (tmp, env, root)
2114 }
2115
2116 #[tokio::test(flavor = "multi_thread")]
2117 async fn enabling_registers_the_workspace_with_the_bridge() {
2118 let f = fixture().await;
2119
2120 let id = f.runtime.enable(&f.root).await.unwrap();
2121 assert_eq!(f.stub.last_workspaces(), vec![id]);
2122 }
2123
2124 #[tokio::test(flavor = "multi_thread")]
2125 async fn enabling_twice_is_idempotent_and_keeps_the_same_id() {
2126 let f = fixture().await;
2127
2128 let first = f.runtime.enable(&f.root).await.unwrap();
2129 let second = f.runtime.enable(&f.root).await.unwrap();
2130 assert_eq!(first, second);
2131 assert_eq!(
2132 f.stub.last_workspaces(),
2133 vec![first],
2134 "still exactly one workspace"
2135 );
2136 }
2137
2138 #[tokio::test(flavor = "multi_thread")]
2139 async fn a_registration_carries_the_complete_current_list() {
2140 let f = fixture().await;
2141 let second_root = f.second_root();
2142
2143 let a = f.runtime.enable(&f.root).await.unwrap();
2144 let b = f.runtime.enable(&second_root).await.unwrap();
2145
2146 let mut seen = f.stub.last_workspaces();
2147 seen.sort();
2148 let mut want = vec![a, b];
2149 want.sort();
2150 assert_eq!(seen, want, "registration is the whole set, not a delta");
2151 }
2152
2153 #[tokio::test(flavor = "multi_thread")]
2154 async fn no_later_registration_clears_this_apps_routes() {
2155 let f = fixture().await;
2161 let second_root = f.second_root();
2162
2163 f.runtime.enable(&f.root).await.unwrap();
2164 f.runtime.enable(&second_root).await.unwrap();
2165 f.runtime.sync_now(&f.root).await.unwrap();
2166 f.runtime.disable(&f.root).await.unwrap();
2167
2168 let seen = f.stub.seen.lock().expect("stub");
2169 let first_full = seen
2170 .registrations
2171 .iter()
2172 .position(|r| !r.workspaces.is_empty())
2173 .expect("enabling registers the workspace");
2174 let emptied_afterwards = seen.registrations[first_full + 1..]
2175 .iter()
2176 .filter(|r| r.workspaces.is_empty())
2177 .count();
2178 assert_eq!(
2179 emptied_afterwards, 0,
2180 "a registration after the first workspace must still name the whole set"
2181 );
2182 }
2183
2184 #[tokio::test(flavor = "multi_thread")]
2185 async fn disabling_unregisters_and_leaves_the_files_alone() {
2186 let f = fixture().await;
2187 std::fs::write(f.root.join("keep.md"), "content").unwrap();
2188
2189 let id = f.runtime.enable(&f.root).await.unwrap();
2190 f.runtime.disable(&f.root).await.unwrap();
2191
2192 assert!(f.stub.last_workspaces().is_empty());
2193 assert!(
2194 f.root.join("keep.md").exists(),
2195 "disabling sync must not touch files"
2196 );
2197 assert!(
2198 crate::sync::id::sync_id_path("sapphire-synctest", &f.root).exists(),
2199 "the sync id stays, so re-enabling rejoins the same workspace"
2200 );
2201 let _ = id;
2202 }
2203
2204 #[tokio::test(flavor = "multi_thread")]
2205 async fn re_enabling_after_disabling_reuses_the_identity() {
2206 let f = fixture().await;
2207
2208 let first = f.runtime.enable(&f.root).await.unwrap();
2209 f.runtime.disable(&f.root).await.unwrap();
2210 let again = f.runtime.enable(&f.root).await.unwrap();
2211
2212 assert_eq!(first, again);
2213 }
2214
2215 #[tokio::test(flavor = "multi_thread")]
2216 async fn status_reports_disabled_for_a_workspace_that_was_never_enabled() {
2217 let f = fixture().await;
2218
2219 let status = f.runtime.status(&f.root).await;
2220 assert!(!status.enabled);
2221 assert!(status.workspace_id.is_none());
2222 }
2223
2224 #[tokio::test(flavor = "multi_thread")]
2225 async fn disabling_something_that_was_never_enabled_is_not_an_error() {
2226 let f = fixture().await;
2227 f.runtime
2228 .disable(&f.root)
2229 .await
2230 .expect("disabling twice is fine");
2231 }
2232
2233 #[tokio::test(flavor = "multi_thread")]
2234 async fn a_workspace_whose_root_vanished_is_reported_as_paused_not_deleted() {
2235 let f = fixture().await;
2236 std::fs::write(f.root.join("a.md"), "content").unwrap();
2237 f.runtime.enable(&f.root).await.unwrap();
2238 f.runtime.scan(&f.root).await.unwrap();
2239
2240 std::fs::remove_dir_all(f.root.join(".sapphire-synctest")).unwrap();
2242 let _ = f.runtime.scan(&f.root).await;
2243
2244 let status = f.runtime.status(&f.root).await;
2245 assert!(
2246 status.paused.is_some(),
2247 "a missing root must pause, not replicate as a mass deletion"
2248 );
2249 }
2250
2251 #[tokio::test(flavor = "multi_thread")]
2252 async fn a_directory_that_is_not_a_workspace_cannot_be_enabled() {
2253 let f = fixture().await;
2254 let plain = f._tmp.path().join("plain");
2255 std::fs::create_dir_all(&plain).unwrap();
2256
2257 assert!(f.runtime.enable(&plain).await.is_err());
2258 }
2259
2260 #[tokio::test(flavor = "multi_thread")]
2261 async fn a_write_through_the_workspace_router_is_committed_to_the_replica() {
2262 let f = fixture().await;
2267 f.runtime.enable(&f.root).await.unwrap();
2268
2269 let host = Arc::new(crate::WorkspaceHost::new(&CTX));
2270 host.select(&f.root, false).await.unwrap();
2271 let current = Arc::new(crate::Current::new(
2272 &CTX,
2273 host,
2274 Some(Arc::clone(&f.runtime)),
2275 crate::selection::SelectionFile::at(f._tmp.path().join("workspace.toml")),
2276 ));
2277 let router = Arc::new(crate::workspace_router(current));
2278 let (client_conn, server_conn) = sapphire_ipc::Connection::pair();
2279 tokio::spawn(async move {
2280 let info = sapphire_ipc::ServerInfo {
2281 version: "0.0.0".into(),
2282 api: sapphire_backend::protocol::API_VERSION,
2283 app_api: None,
2284 pid: std::process::id(),
2285 managed_by: ManagedBy::Spawned,
2286 };
2287 let _ = sapphire_ipc::serve(server_conn, router, "sapphire-synctest", info).await;
2288 });
2289 let info = sapphire_ipc::ClientInfo {
2290 kind: "test".into(),
2291 version: "0.0.0".into(),
2292 api: sapphire_backend::protocol::API_VERSION,
2293 app_api: None,
2294 pid: std::process::id(),
2295 };
2296 let (client, _) = sapphire_ipc::Client::handshake(client_conn, "sapphire-synctest", info)
2297 .await
2298 .unwrap();
2299
2300 let _: sapphire_backend::protocol::Ack = client
2301 .call(
2302 sapphire_backend::protocol::WRITE_FILE,
2303 sapphire_backend::protocol::ContentParams {
2304 path: PathBuf::from("through-the-server.md"),
2305 content: "written by the server".into(),
2306 },
2307 )
2308 .await
2309 .unwrap();
2310
2311 let synced = f.runtime.synced.lock().await;
2314 let entry = synced.get(&f.root).expect("enabled above");
2315 let replica = entry.replica.lock().await;
2316 let state = replica
2317 .state("through-the-server.md")
2318 .expect("a readable state");
2319 assert!(
2320 state.is_some(),
2321 "a write through the server must be scanned into the replica, not left to the watcher"
2322 );
2323 }
2324
2325 #[tokio::test(flavor = "multi_thread")]
2326 async fn an_announcement_for_an_unknown_workspace_is_ignored_not_fatal() {
2327 let f = fixture().await;
2328 f.runtime.enable(&f.root).await.unwrap();
2329
2330 let driver = Arc::clone(&f.runtime);
2331 let handle = tokio::spawn(async move { driver.run().await });
2332
2333 f.stub
2334 .announce(grain_id::GrainId::random(), "no-such-ticket")
2335 .await;
2336 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
2337
2338 assert!(
2339 !handle.is_finished(),
2340 "one bad announcement must not stop the runtime"
2341 );
2342 handle.abort();
2343 }
2344}