1use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use grain_id::GrainId;
13use sapphire_backend::protocol as proto;
14use sapphire_bridge_api::{
15 IncomingParams, PeersResult, RegisterParams, WorkspaceRegistration, WorkspacesResult,
16};
17use sapphire_bridge_client::BridgeClient;
18use sapphire_ipc::ManagedBy;
19use sapphire_sync::{PathUpdate, PauseReason, Replica, ReplicaConfig, ScanOutcome, SystemClock};
20use sapphire_workspace::{AppContext, Workspace};
21use tokio::sync::{Mutex, OnceCell, broadcast, mpsc};
22
23pub mod id;
24mod live;
25mod methods;
26#[cfg(any(test, feature = "test-util"))]
27pub mod testing;
28mod topology;
29mod watch;
30
31pub use id::{SYNC_ID_FILE, WORKSPACE_MAP_FILE, sync_id, sync_id_path};
32pub use live::{DIAL_BACKOFF_MAX, OPEN_TIMEOUT};
33pub use methods::sync_router;
34pub use watch::{DEBOUNCE, Watcher};
35
36use live::{LivePeers, dial_backoff};
37
38use crate::error::{Error, Result};
39
40use crate::host::WorkspaceHost;
41
42#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
44pub struct SyncStatus {
45 pub enabled: bool,
47 pub workspace_id: Option<GrainId>,
49 pub peers: usize,
51 pub paused: Option<String>,
53 pub last_error: Option<String>,
55 pub bridge_available: bool,
57 pub topology: proto::Topology,
59 pub embedding: Option<proto::EmbeddingProgress>,
61}
62
63struct Synced {
65 workspace_id: GrainId,
67 replica: Arc<Mutex<Replica>>,
69 paused: Option<PauseReason>,
71 last_error: Option<String>,
73}
74
75const STALE_VECTOR_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
78
79const PASS_BATCH: usize = 4;
82
83#[derive(Clone, Copy, Debug)]
85pub struct BackfillTiming {
86 pub grace: std::time::Duration,
89 pub interval: std::time::Duration,
91}
92
93impl Default for BackfillTiming {
94 fn default() -> Self {
95 BackfillTiming {
96 grace: std::time::Duration::from_secs(10 * 60),
97 interval: std::time::Duration::from_secs(10 * 60),
98 }
99 }
100}
101
102pub const RECONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_millis(500);
105
106pub const RECONNECT_BACKOFF_MAX: std::time::Duration = std::time::Duration::from_secs(30);
109
110fn reconnect_backoff(attempt: u32) -> std::time::Duration {
112 RECONNECT_BACKOFF
113 .saturating_mul(1u32 << attempt.min(16))
114 .min(RECONNECT_BACKOFF_MAX)
115}
116
117type Connecting = std::pin::Pin<
119 Box<dyn std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>> + Send>,
120>;
121
122#[derive(Clone)]
127pub struct BridgeConnector(Arc<dyn Fn() -> Connecting + Send + Sync>);
128
129impl BridgeConnector {
130 pub fn new<F, Fut>(connect: F) -> BridgeConnector
132 where
133 F: Fn() -> Fut + Send + Sync + 'static,
134 Fut: std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>>
135 + Send
136 + 'static,
137 {
138 BridgeConnector(Arc::new(move || Box::pin(connect())))
139 }
140
141 pub fn standard(kind: &'static str, version: &'static str) -> BridgeConnector {
144 BridgeConnector::new(move || async move {
145 match BridgeClient::connect(kind, version).await {
146 Ok(client) => Ok(Some(client)),
147 Err(sapphire_ipc::Error::NotRunning(_)) => Ok(None),
148 Err(err) => Err(err),
149 }
150 })
151 }
152
153 pub fn at(
155 endpoint: sapphire_ipc::Endpoint,
156 kind: &'static str,
157 version: &'static str,
158 ) -> BridgeConnector {
159 BridgeConnector::new(move || {
160 let endpoint = endpoint.clone();
161 async move { BridgeClient::connect_at(&endpoint, kind, version).await }
162 })
163 }
164
165 async fn connect(&self) -> sapphire_ipc::Result<Option<BridgeClient>> {
166 (self.0)().await
167 }
168}
169
170impl std::fmt::Debug for BridgeConnector {
171 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
172 f.write_str("BridgeConnector")
173 }
174}
175
176pub struct SyncRuntime {
178 ctx: &'static AppContext,
179 bridge: tokio::sync::watch::Sender<Option<Arc<BridgeClient>>>,
182 connector: Option<BridgeConnector>,
184 pending: Mutex<HashMap<PathBuf, GrainId>>,
191 subscribed: std::sync::Mutex<Option<(Arc<BridgeClient>, broadcast::Receiver<IncomingParams>)>>,
196 exe_path: PathBuf,
197 managed_by: ManagedBy,
198 synced: Mutex<HashMap<PathBuf, Synced>>,
199 device_id: OnceCell<GrainId>,
205 watcher: OnceCell<Arc<Watcher>>,
215 host: OnceCell<Arc<WorkspaceHost>>,
226 reindexing: Mutex<()>,
229 embed_wanted: Mutex<std::collections::HashSet<PathBuf>>,
232 embedding: Mutex<()>,
234 embed_running: Mutex<std::collections::HashSet<PathBuf>>,
236 backfill: std::sync::Mutex<BackfillTiming>,
238 bridge_endpoint: Option<sapphire_ipc::Endpoint>,
240 live: Mutex<HashMap<PathBuf, Arc<LivePeers>>>,
246 last_peers: Mutex<Option<PeersResult>>,
249 open_timeout: std::time::Duration,
251 first_dials: std::sync::Mutex<tokio::task::JoinSet<()>>,
254 background: std::sync::Mutex<tokio::task::JoinSet<()>>,
259}
260
261impl SyncRuntime {
262 pub fn new(
267 ctx: &'static AppContext,
268 bridge: Arc<BridgeClient>,
269 exe_path: PathBuf,
270 managed_by: ManagedBy,
271 ) -> SyncRuntime {
272 SyncRuntime::build(ctx, Some(bridge), None, exe_path, managed_by)
273 }
274
275 pub fn connecting(
283 ctx: &'static AppContext,
284 connector: BridgeConnector,
285 exe_path: PathBuf,
286 managed_by: ManagedBy,
287 ) -> SyncRuntime {
288 SyncRuntime::build(ctx, None, Some(connector), exe_path, managed_by)
289 }
290
291 pub fn with_connector(mut self, connector: BridgeConnector) -> SyncRuntime {
293 self.connector = Some(connector);
294 self
295 }
296
297 fn build(
298 ctx: &'static AppContext,
299 bridge: Option<Arc<BridgeClient>>,
300 connector: Option<BridgeConnector>,
301 exe_path: PathBuf,
302 managed_by: ManagedBy,
303 ) -> SyncRuntime {
304 SyncRuntime {
305 ctx,
306 bridge: tokio::sync::watch::Sender::new(bridge),
307 connector,
308 pending: Mutex::new(HashMap::new()),
309 subscribed: std::sync::Mutex::new(None),
310 exe_path,
311 managed_by,
312 synced: Mutex::new(HashMap::new()),
313 device_id: OnceCell::new(),
314 watcher: OnceCell::new(),
315 host: OnceCell::new(),
316 reindexing: Mutex::new(()),
317 embed_wanted: Mutex::new(std::collections::HashSet::new()),
318 embedding: Mutex::new(()),
319 embed_running: Mutex::new(std::collections::HashSet::new()),
320 backfill: std::sync::Mutex::new(BackfillTiming::default()),
321 bridge_endpoint: None,
322 live: Mutex::new(HashMap::new()),
323 last_peers: Mutex::new(None),
324 open_timeout: OPEN_TIMEOUT,
325 first_dials: std::sync::Mutex::new(tokio::task::JoinSet::new()),
326 background: std::sync::Mutex::new(tokio::task::JoinSet::new()),
327 }
328 }
329
330 fn bridge(&self) -> Result<Arc<BridgeClient>> {
332 match &*self.bridge.borrow() {
333 Some(bridge) if !bridge.is_closed() => Ok(Arc::clone(bridge)),
334 _ => Err(Error::BridgeDisconnected),
335 }
336 }
337
338 pub fn is_connected(&self) -> bool {
340 self.bridge().is_ok()
341 }
342
343 fn spawn_background(&self, task: impl std::future::Future<Output = ()> + Send + 'static) {
345 let mut background = self.background.lock().expect("background tasks");
346 while background.try_join_next().is_some() {}
348 background.spawn(task);
349 }
350
351 pub fn with_open_timeout(mut self, timeout: std::time::Duration) -> SyncRuntime {
354 self.open_timeout = timeout;
355 self
356 }
357
358 pub fn with_bridge_endpoint(mut self, endpoint: sapphire_ipc::Endpoint) -> SyncRuntime {
365 self.bridge_endpoint = Some(endpoint);
366 self
367 }
368
369 pub fn set_backfill_timing(&self, timing: BackfillTiming) {
372 *self.backfill.lock().expect("backfill timing") = timing;
373 }
374
375 fn backfill_timing(&self) -> BackfillTiming {
376 *self.backfill.lock().expect("backfill timing")
377 }
378
379 pub(crate) fn set_host(&self, host: Arc<WorkspaceHost>) {
380 let _ = self.host.set(host);
381 }
382
383 pub async fn enable(self: &Arc<Self>, root: &Path) -> Result<GrainId> {
388 let key = root.canonicalize().map_err(Error::Io)?;
389 let watch_key = key.clone();
390 let workspace_id = sync_id(self.ctx.app_name, &key)?;
391
392 {
395 let mut pending = self.pending.lock().await;
396 if self.bridge().is_err() {
397 pending.insert(key, workspace_id);
398 return Err(Error::BridgeDisconnected);
399 }
400 pending.remove(&key);
401 }
402
403 {
406 let mut synced = self.synced.lock().await;
407 if let Some(existing) = synced.get(&key) {
408 return Ok(existing.workspace_id);
409 }
410 let device_id = self.device_id().await?;
411 let state_dir = self.state_dir(&key)?;
412 std::fs::create_dir_all(&state_dir).map_err(Error::Io)?;
413 let config = ReplicaConfig::new(self.ctx.app_name, key.clone(), device_id, &state_dir);
414 let replica = Replica::open(config, Arc::new(SystemClock))
415 .map_err(|e| Error::Sync(e.to_string()))?;
416 synced.insert(
417 key.clone(),
418 Synced {
419 workspace_id,
420 replica: Arc::new(Mutex::new(replica)),
421 paused: None,
422 last_error: None,
423 },
424 );
425 }
426
427 if let Some(watcher) = self.watcher.get()
431 && let Err(err) = watcher.watch(&watch_key)
432 {
433 tracing::warn!(root = %watch_key.display(), "could not watch: {err}");
434 }
435
436 self.reregister().await?;
439
440 let runtime = Arc::clone(self);
453 let mut dials = self.first_dials.lock().expect("first dials");
454 while dials.try_join_next().is_some() {}
456 dials.spawn(async move {
457 if let Err(err) = runtime.sync_now(&key).await {
458 tracing::warn!(root = %key.display(), "the first session after enabling failed: {err}");
459 }
460 });
461 Ok(workspace_id)
462 }
463
464 pub async fn disable(&self, root: &Path) -> Result<()> {
467 self.stop(root).await.1
468 }
469
470 pub async fn release(&self, root: &Path, limit: std::time::Duration) -> Result<()> {
479 let (replica, unregistered) = self.stop(root).await;
480 if let Err(err) = unregistered {
481 tracing::warn!(root = %root.display(), "stopping sync: {err}");
482 }
483 let Some(replica) = replica else {
484 return Ok(());
485 };
486 let deadline = tokio::time::Instant::now() + limit;
487 while Arc::strong_count(&replica) > 1 {
488 if tokio::time::Instant::now() >= deadline {
489 return Err(Error::Sync(format!(
490 "the replica of {} is still in use; try again",
491 root.display()
492 )));
493 }
494 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
495 }
496 Ok(())
497 }
498
499 async fn stop(&self, root: &Path) -> (Option<Arc<Mutex<Replica>>>, Result<()>) {
502 let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
505 self.pending.lock().await.remove(&key);
506 let removed = self.synced.lock().await.remove(&key);
507 if removed.is_some()
508 && let Some(watcher) = self.watcher.get()
509 {
510 watcher.unwatch(&key);
511 }
512 if let Some(peers) = self.live.lock().await.remove(&key) {
515 peers.drop_connections().await;
516 }
517 let Some(removed) = removed else {
518 return (None, Ok(()));
519 };
520 let unregistered = async {
521 let Ok(bridge) = self.bridge() else {
524 return Ok(());
525 };
526 bridge
527 .unregister(removed.workspace_id)
528 .await
529 .map_err(|e| Error::Bridge(e.to_string()))?;
530 self.reregister().await
531 }
532 .await;
533 (Some(removed.replica), unregistered)
534 }
535
536 pub async fn workspaces(&self) -> Result<WorkspacesResult> {
542 self.bridge()?
543 .workspaces()
544 .await
545 .map_err(|e| Error::Bridge(e.to_string()))
546 }
547
548 pub async fn place(&self, selector: &str, dir: &Path) -> Result<PathBuf> {
557 let workgroup = self.workspaces().await?;
558 let wanted = workgroup
559 .workspaces
560 .iter()
561 .find(|w| w.name == selector || w.workspace_id.to_string() == selector)
564 .ok_or_else(|| Error::UnknownWorkspaceName(selector.to_owned()))?;
565
566 if wanted.app_name != self.ctx.app_name {
569 return Err(Error::WrongApp {
570 name: wanted.name.clone(),
571 app_name: wanted.app_name.clone(),
572 });
573 }
574
575 let root = dir.canonicalize().map_err(Error::Io)?;
576 let workspace = Workspace::from_root(self.ctx, &root)?;
577 if workspace.root != root {
578 return Err(Error::UnknownWorkspace(root, self.ctx.app_name));
579 }
580
581 {
585 let synced = self.synced.lock().await;
586 if let Some(entry) = synced.get(&root)
587 && entry.workspace_id != wanted.workspace_id
588 {
589 return Err(Error::SyncId(format!(
590 "{} already syncs as {}, not {}",
591 root.display(),
592 entry.workspace_id,
593 wanted.workspace_id
594 )));
595 }
596 }
597
598 let marker = workspace.marker_dir();
603 let id_path = marker.join(crate::sync::id::SYNC_ID_FILE);
604 match std::fs::read_to_string(&id_path) {
605 Ok(existing) if existing.trim() != wanted.workspace_id.to_string() => {
606 return Err(Error::SyncId(format!(
607 "{} already holds identity {}, not {}",
608 root.display(),
609 existing.trim(),
610 wanted.workspace_id
611 )));
612 }
613 Err(_) => {
614 std::fs::write(&id_path, format!("{}\n", wanted.workspace_id))
615 .map_err(Error::Io)?;
616 }
617 Ok(_) => {}
618 }
619
620 let map_path = root
623 .join(format!(".{}", self.ctx.app_name))
624 .join(WORKSPACE_MAP_FILE);
625 let workspace_id = wanted.workspace_id.to_string();
626 std::fs::write(&map_path, format!("{workspace_id}\n")).map_err(Error::Io)?;
627 Ok(root)
628 }
629
630 pub async fn status(&self, root: &Path) -> SyncStatus {
632 let bridge = self.bridge().ok();
633 let bridge_available = match &bridge {
634 Some(bridge) => bridge.status().await.is_ok(),
635 None => false,
636 };
637 let answer = match &bridge {
638 Some(bridge) => bridge.peers().await.ok(),
639 None => None,
640 };
641 if let Some(answer) = &answer {
642 *self.last_peers.lock().await = Some(answer.clone());
643 }
644 let peers = answer
645 .as_ref()
646 .map_or(0, |p| p.peers.len().saturating_sub(1));
647 let Ok(key) = root.canonicalize() else {
648 return SyncStatus {
649 enabled: false,
650 workspace_id: None,
651 peers,
652 paused: None,
653 last_error: None,
654 bridge_available,
655 topology: proto::Topology::Mesh,
656 embedding: None,
657 };
658 };
659 let synced = self.synced.lock().await;
660 match synced.get(&key) {
661 Some(entry) => SyncStatus {
662 enabled: true,
663 workspace_id: Some(entry.workspace_id),
664 peers,
665 paused: entry.paused.map(|r| format!("{r:?}")),
666 last_error: entry.last_error.clone(),
667 bridge_available,
668 topology: topology::topology(
669 answer
670 .as_ref()
671 .and_then(|p| p.roles_for(entry.workspace_id)),
672 ),
673 embedding: self.embedding_progress(&key).await,
674 },
675 None => {
676 drop(synced);
677 let waiting = self.pending.lock().await.get(&key).copied();
680 SyncStatus {
681 enabled: waiting.is_some(),
682 workspace_id: waiting,
683 peers,
684 paused: None,
685 last_error: None,
686 bridge_available,
687 topology: proto::Topology::Mesh,
688 embedding: None,
689 }
690 }
691 }
692 }
693
694 async fn embedding_progress(&self, key: &Path) -> Option<proto::EmbeddingProgress> {
697 let host = self.host.get()?;
698 let backend = host.open_backend(key)?;
699 let state = backend.state();
700 state.embedder()?;
701 let info = state.db_info().ok()?;
702 Some(proto::EmbeddingProgress {
703 vectors: info.vector_count,
704 pending: info.pending_count,
705 running: self.embed_running.lock().await.contains(key),
706 })
707 }
708
709 pub async fn scan(&self, root: &Path) -> Result<()> {
715 let Ok(key) = root.canonicalize() else {
716 return Ok(());
717 };
718 let replica = {
719 let synced = self.synced.lock().await;
720 match synced.get(&key) {
721 Some(entry) => Arc::clone(&entry.replica),
722 None => return Ok(()),
723 }
724 };
725 let (outcome, updates) = {
726 let mut replica = replica.lock().await;
727 let outcome = replica.scan().map_err(|e| Error::Sync(e.to_string()))?;
728 let updates = match &outcome {
733 ScanOutcome::Scanned(report) => {
734 report
742 .recorded
743 .iter()
744 .map(|entry| PathUpdate {
745 path: entry.path.clone(),
746 versions: vec![entry.clone()],
747 seen: {
748 let mut seen = entry.context.clone();
749 seen.add_dot(&entry.dot);
750 seen
751 },
752 })
753 .collect()
754 }
755 ScanOutcome::Paused(_) => Vec::new(),
756 };
757 (outcome, updates)
758 };
759 {
760 let mut synced = self.synced.lock().await;
761 if let Some(entry) = synced.get_mut(&key) {
762 entry.paused = match outcome {
763 ScanOutcome::Paused(reason) => Some(reason),
764 ScanOutcome::Scanned(_) => None,
765 };
766 }
767 }
768 self.after_commit(&key, updates).await;
771 Ok(())
772 }
773
774 async fn reindex(&self, root: &Path) {
789 let Some(host) = self.host.get() else {
790 return;
791 };
792 let host = Arc::clone(host);
793 let root = root.to_owned();
794 let _serialised = self.reindexing.lock().await;
797 let backend = match host.backend_at(&root).await {
800 Ok(backend) => backend,
801 Err(err) => {
802 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
803 return;
804 }
805 };
806 let state = Arc::clone(backend.state());
807 let result = tokio::task::spawn_blocking(move || state.sync_retrieve()).await;
808 match result {
809 Ok(Ok(report)) => {
810 tracing::debug!(root = %root.display(), ?report, "re-indexed after a session");
811 }
812 Ok(Err(err)) => {
813 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
814 }
815 Err(err) => {
816 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
817 }
818 }
819 }
820
821 pub fn request_embed(self: &Arc<Self>, root: &Path) {
828 let Ok(key) = root.canonicalize() else {
829 return;
830 };
831 let runtime = Arc::clone(self);
832 self.spawn_background(async move {
833 if !runtime.embed_wanted.lock().await.insert(key.clone()) {
834 return;
835 }
836 let _one_at_a_time = runtime.embedding.lock().await;
837 runtime.embed_wanted.lock().await.remove(&key);
839 runtime.embed_running.lock().await.insert(key.clone());
840 runtime.embed_pass(&key).await;
841 runtime.embed_running.lock().await.remove(&key);
842 });
843 }
844
845 async fn embed_pass(&self, key: &Path) {
847 let Some(host) = self.host.get() else {
848 return;
849 };
850 let (workspace_id, replica) = {
851 let synced = self.synced.lock().await;
852 match synced.get(key) {
853 Some(entry) => (entry.workspace_id, Arc::clone(&entry.replica)),
854 None => return,
855 }
856 };
857 let backend = match host.backend_at(key).await {
858 Ok(backend) => backend,
859 Err(err) => {
860 tracing::warn!(root = %key.display(), "embedding pass: {err}");
861 return;
862 }
863 };
864 let state = Arc::clone(backend.state());
865 if let Err(err) = state
866 .refresh_embedder_at(self.bridge_endpoint.clone())
867 .await
868 {
869 tracing::warn!(root = %key.display(), "embedding pass: {err}");
870 return;
871 }
872 if state.embedder().is_none() {
873 return;
874 }
875 let primary = self.is_primary(workspace_id).await;
876 let grace_ms = self.backfill_timing().grace.as_millis() as u64;
879 let now_ms = std::time::SystemTime::now()
880 .duration_since(std::time::UNIX_EPOCH)
881 .map_or(0, |d| d.as_millis() as u64);
882 let (mine, settled, live) = {
883 let replica = replica.lock().await;
884 let me = replica.replica_id();
885 let states = match replica.states() {
886 Ok(states) => states,
887 Err(err) => {
888 tracing::warn!(root = %key.display(), "embedding pass: {err}");
889 return;
890 }
891 };
892 let mut mine = HashMap::new();
893 let mut settled = HashMap::new();
895 let mut live = std::collections::HashSet::new();
896 for (path, path_state) in states {
897 let winner = path_state.winner();
898 let Some(hash) = winner.content.hash() else {
899 continue;
900 };
901 let hash = hash.to_hex();
902 if winner.dot.replica == me {
903 mine.insert(path, hash.clone());
904 } else if now_ms.saturating_sub(winner.hlc.wall_ms) >= grace_ms {
905 settled.insert(path, hash.clone());
906 }
907 live.insert(hash);
908 }
909 (mine, settled, live)
910 };
911 let root = key.to_owned();
912 let result = tokio::task::spawn_blocking(move || {
913 let policy = |rel: &str, hash: &str| {
916 mine.get(rel).is_some_and(|h| h.as_str() == hash)
917 || (primary && settled.get(rel).is_some_and(|h| h.as_str() == hash))
918 };
919 let embedded = state.embed_pending_with(&policy, PASS_BATCH)?;
920 let removed = if primary {
921 state.remove_stale_vectors(&live, STALE_VECTOR_AGE)
922 } else {
923 0
924 };
925 Ok::<_, sapphire_workspace::Error>((embedded, removed))
926 })
927 .await;
928 match result {
929 Ok(Ok((embedded, removed))) if embedded > 0 || removed > 0 => tracing::info!(
930 root = %root.display(),
931 embedded,
932 removed,
933 "embedding pass"
934 ),
935 Ok(Ok(_)) => {}
936 Ok(Err(err)) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
937 Err(err) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
938 }
939 }
940
941 pub async fn sync_now(self: &Arc<Self>, root: &Path) -> Result<()> {
943 let Ok(key) = root.canonicalize() else {
944 return Ok(());
945 };
946 let workspace_id = {
947 let synced = self.synced.lock().await;
948 match synced.get(&key) {
949 Some(entry) => entry.workspace_id,
950 None => return Ok(()),
951 }
952 };
953 let peers = self
954 .bridge()?
955 .peers()
956 .await
957 .map_err(|e| Error::Bridge(e.to_string()))?;
958 *self.last_peers.lock().await = Some(peers.clone());
959 let me = self.device_id().await?;
960
961 let roles = peers.roles_for(workspace_id).cloned();
966 let mut dials: Vec<Dial<'_>> = Vec::new();
970 for peer in peers
971 .peers
972 .iter()
973 .filter(|p| topology::link(me, p.device_id, roles.as_ref()) == topology::Link::Dial)
974 .cloned()
975 .collect::<Vec<_>>()
976 {
977 if let Some(live) = self.live_peers(&key).await
981 && live.has(&peer.device_id).await
982 {
983 continue;
984 }
985 let Some((_, table)) = self.live_peers_or_create(&key).await else {
992 continue;
993 };
994 if !table.begin_dial(peer.device_id).await {
995 continue;
996 }
997 let key = &key;
998 dials.push(Box::pin(async move {
999 let result = self
1001 .sync_one(key, &peer.device_id, peer.name.clone(), workspace_id)
1002 .await;
1003 table.end_dial(peer.device_id).await;
1004 if let Err(err) = result {
1005 tracing::debug!(peer = %peer.name, "no session: {err}");
1006 }
1007 }));
1008 }
1009 join_all(dials).await;
1010 Ok(())
1011 }
1012
1013 async fn sync_one(
1020 self: &Arc<Self>,
1021 key: &Path,
1022 device: &GrainId,
1023 name: String,
1024 workspace_id: GrainId,
1025 ) -> Result<()> {
1026 let stream = self.open_stream(workspace_id, *device).await?;
1027 self.adopt_live_session(stream, key, *device, workspace_id)
1028 .await
1029 .map_err(|err| {
1030 tracing::warn!(peer = %name, "a live session could not be started: {err}");
1031 err
1032 })
1033 }
1034
1035 async fn open_stream(
1037 &self,
1038 workspace_id: GrainId,
1039 device: GrainId,
1040 ) -> Result<sapphire_ipc::RawStream> {
1041 match tokio::time::timeout(
1042 self.open_timeout,
1043 self.bridge()?.open_stream(workspace_id, device),
1044 )
1045 .await
1046 {
1047 Ok(opened) => Ok(opened?),
1048 Err(_) => Err(Error::Bridge(format!(
1049 "no stream to {device} within {} s; it may be offline",
1050 self.open_timeout.as_secs_f32()
1051 ))),
1052 }
1053 }
1054
1055 pub async fn run(self: Arc<Self>) -> Result<()> {
1058 let mut slot = self.bridge.subscribe();
1059 loop {
1060 let bridge = slot.borrow_and_update().clone();
1061 match bridge {
1062 Some(bridge) if !bridge.is_closed() => self.answer(&bridge, &mut slot).await,
1063 _ => {
1065 if slot.changed().await.is_err() {
1066 return Ok(());
1067 }
1068 }
1069 }
1070 }
1071 }
1072
1073 async fn answer(
1075 self: &Arc<Self>,
1076 bridge: &Arc<BridgeClient>,
1077 slot: &mut tokio::sync::watch::Receiver<Option<Arc<BridgeClient>>>,
1078 ) {
1079 let early = self
1080 .subscribed
1081 .lock()
1082 .expect("subscribed")
1083 .take_if(|(b, _)| Arc::ptr_eq(b, bridge));
1084 let mut incoming = match early {
1085 Some((_, incoming)) => incoming,
1086 None => bridge.incoming(),
1087 };
1088 loop {
1089 let received = tokio::select! {
1090 received = incoming.recv() => received,
1091 () = bridge.closed() => return,
1092 _ = slot.changed() => return,
1093 };
1094 let announcement = match received {
1095 Ok(a) => a,
1096 Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
1097 tracing::warn!(missed = n, "fell behind on bridge announcements");
1098 continue;
1099 }
1100 Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1101 };
1102
1103 let found = {
1104 let synced = self.synced.lock().await;
1105 synced
1106 .iter()
1107 .find(|(_, s)| s.workspace_id == announcement.workspace_id)
1108 .map(|(root, s)| (root.clone(), Arc::clone(&s.replica)))
1109 };
1110 let Some((root, replica)) = found else {
1113 tracing::debug!(
1116 workspace = %announcement.workspace_id,
1117 "an announcement for a workspace this server does not hold"
1118 );
1119 continue;
1120 };
1121
1122 let bridge = Arc::clone(bridge);
1123 let workspace_id = announcement.workspace_id;
1124 let peer = announcement.peer_device_id;
1125 let driver = Arc::clone(self);
1126 self.spawn_background(async move {
1127 let stream = match bridge.accept_stream(announcement.ticket).await {
1128 Ok(stream) => stream,
1129 Err(err) => {
1130 tracing::warn!("could not claim an announced stream: {err}");
1131 return;
1132 }
1133 };
1134 if driver.should_skip(workspace_id, peer).await {
1135 tracing::debug!(%peer, "dropped an inbound session: both ends are outside this workspace's star");
1136 return;
1137 }
1138 let (_, session) = match sapphire_framework_session::open_live_session(
1143 stream,
1144 Arc::clone(&replica),
1145 workspace_id,
1146 )
1147 .await
1148 {
1149 Ok(started) => started,
1150 Err(err) => {
1151 tracing::warn!("an inbound session failed: {err}");
1152 return;
1153 }
1154 };
1155 let Some((key, table)) = driver.live_peers_or_create(&root).await else {
1156 return;
1157 };
1158 let Some(updates) = table.insert(peer, session, false).await else {
1162 return;
1163 };
1164 table.catch_up(peer, &replica).await;
1165 driver.reindex(&key).await;
1168 driver.request_embed(&key);
1169 driver.spawn_reader(table, updates, key, peer);
1170 });
1171 }
1172 }
1173
1174 pub async fn connect(self: &Arc<Self>) -> Result<()> {
1181 let Some(connector) = &self.connector else {
1182 return Ok(());
1183 };
1184 if self.is_connected() {
1185 return Ok(());
1186 }
1187 match connector.connect().await? {
1188 Some(client) => self.attach(client).await,
1189 None => tracing::warn!(
1190 "no sapphire-bridge is running; serving without sync until it starts"
1191 ),
1192 }
1193 Ok(())
1194 }
1195
1196 pub async fn keep_connected(self: Arc<Self>) {
1204 let mut attempt = 0;
1205 loop {
1206 if let Ok(bridge) = self.bridge() {
1207 bridge.closed().await;
1208 tracing::warn!("lost the connection to sapphire-bridge; reconnecting");
1209 self.bridge.send_if_modified(|slot| {
1210 let same = slot.as_ref().is_some_and(|b| Arc::ptr_eq(b, &bridge));
1211 if same {
1212 *slot = None;
1213 }
1214 same
1215 });
1216 attempt = 0;
1217 continue;
1218 }
1219 let Some(connector) = &self.connector else {
1220 return;
1221 };
1222 match connector.connect().await {
1223 Ok(Some(client)) => {
1224 tracing::info!("connected to sapphire-bridge");
1225 self.attach(client).await;
1226 attempt = 0;
1227 continue;
1228 }
1229 Ok(None) => tracing::debug!("no sapphire-bridge is running yet"),
1230 Err(err) if attempt == 0 => {
1232 tracing::warn!("could not connect to sapphire-bridge: {err}")
1233 }
1234 Err(err) => tracing::debug!("could not connect to sapphire-bridge: {err}"),
1235 }
1236 tokio::time::sleep(reconnect_backoff(attempt)).await;
1237 attempt = attempt.saturating_add(1);
1238 }
1239 }
1240
1241 async fn attach(self: &Arc<Self>, client: BridgeClient) {
1244 let client = Arc::new(client);
1247 *self.subscribed.lock().expect("subscribed") =
1248 Some((Arc::clone(&client), client.incoming()));
1249 let waiting = {
1250 let mut pending = self.pending.lock().await;
1251 self.bridge.send_replace(Some(client));
1252 std::mem::take(&mut *pending)
1253 };
1254 if !self.synced.lock().await.is_empty()
1256 && let Err(err) = self.reregister().await
1257 {
1258 tracing::warn!("could not register with sapphire-bridge: {err}");
1259 }
1260 for root in waiting.into_keys() {
1262 if let Err(err) = self.enable(&root).await {
1263 tracing::warn!(root = %root.display(), "could not start sync: {err}");
1264 }
1265 }
1266 }
1267
1268 pub async fn roots(&self) -> Vec<PathBuf> {
1270 self.synced.lock().await.keys().cloned().collect()
1271 }
1272
1273 async fn live_peers(&self, root: &Path) -> Option<Arc<LivePeers>> {
1278 let key = root.canonicalize().ok()?;
1279 self.live.lock().await.get(&key).cloned()
1280 }
1281
1282 async fn live_peers_or_create(&self, root: &Path) -> Option<(PathBuf, Arc<LivePeers>)> {
1287 let key = root.canonicalize().ok()?;
1288 if !self.synced.lock().await.contains_key(&key) {
1289 return None;
1290 }
1291 let me = self.device_id().await.ok()?;
1294 let mut live = self.live.lock().await;
1295 let peers = Arc::clone(
1296 live.entry(key.clone())
1297 .or_insert_with(|| Arc::new(LivePeers::new(me))),
1298 );
1299 Some((key, peers))
1300 }
1301
1302 pub async fn after_commit(&self, root: &Path, updates: Vec<PathUpdate>) {
1312 if updates.is_empty() {
1313 return;
1314 }
1315 if let Some(peers) = self.live_peers(root).await {
1316 peers.fan_out(&updates, None).await;
1317 }
1318 }
1319
1320 pub async fn live_session_devices(&self, root: &Path) -> Vec<GrainId> {
1324 match self.live_peers(root).await {
1325 Some(peers) => peers.devices().await,
1326 None => Vec::new(),
1327 }
1328 }
1329
1330 pub async fn stop_dials(&self) {
1337 let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1338 dials.abort_all();
1339 while dials.join_next().await.is_some() {}
1340 }
1341
1342 pub async fn stop_background(&self) {
1350 let mut background =
1351 std::mem::take(&mut *self.background.lock().expect("background tasks"));
1352 background.abort_all();
1353 while background.join_next().await.is_some() {}
1354 }
1355
1356 #[cfg(test)]
1359 async fn settle_dials(&self) {
1360 let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1361 while dials.join_next().await.is_some() {}
1362 }
1363
1364 pub async fn drop_connections(&self) {
1369 let tables: Vec<Arc<LivePeers>> = self.live.lock().await.values().cloned().collect();
1370 for peers in tables {
1371 peers.drop_connections().await;
1372 }
1373 }
1374
1375 pub async fn dial_loop(self: Arc<Self>) -> Result<()> {
1390 let mut tick = tokio::time::interval(live::DIAL_INTERVAL);
1391 tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1394 let mut waiting: HashMap<GrainId, std::time::Instant> = HashMap::new();
1395 let mut failures: HashMap<GrainId, u32> = HashMap::new();
1396 let mut primary_of: std::collections::HashSet<GrainId> = Default::default();
1398 let mut last_backfill = std::time::Instant::now();
1400 loop {
1401 tick.tick().await;
1402 let roots = self.roots().await;
1403 if roots.is_empty() {
1404 continue;
1405 }
1406 let peers = match async { self.bridge()?.peers().await.map_err(Error::from) }.await {
1407 Ok(peers) => {
1408 *self.last_peers.lock().await = Some(peers.clone());
1409 peers
1410 }
1411 Err(err) => {
1414 tracing::debug!("no peers to dial: {err}");
1415 continue;
1416 }
1417 };
1418 let me = match self.device_id().await {
1419 Ok(me) => me,
1420 Err(err) => {
1421 tracing::warn!("no device id, so nobody to dial: {err}");
1422 continue;
1423 }
1424 };
1425 let ids: Vec<(PathBuf, GrainId)> = self
1429 .synced
1430 .lock()
1431 .await
1432 .iter()
1433 .map(|(root, entry)| (root.clone(), entry.workspace_id))
1434 .collect();
1435 let backfill_due = last_backfill.elapsed() >= self.backfill_timing().interval;
1438 if backfill_due {
1439 last_backfill = std::time::Instant::now();
1440 }
1441 for (root, workspace_id) in ids {
1442 let primary = peers
1443 .roles_for(workspace_id)
1444 .is_some_and(|r| r.primary == Some(me));
1445 if !primary {
1446 primary_of.remove(&workspace_id);
1447 } else if primary_of.insert(workspace_id) || backfill_due {
1448 self.request_embed(&root);
1449 }
1450 }
1451 let now = std::time::Instant::now();
1452 for root in roots {
1461 let Some((key, table)) = self.live_peers_or_create(&root).await else {
1462 continue;
1463 };
1464 let workspace_id = {
1465 let synced = self.synced.lock().await;
1466 match synced.get(&key) {
1467 Some(entry) => entry.workspace_id,
1468 None => continue,
1469 }
1470 };
1471 let roles = peers.roles_for(workspace_id);
1472 if topology::may_close_skipped(me, roles, &table.devices().await) {
1475 table
1476 .retain(|d| topology::link(me, *d, roles) != topology::Link::Skip)
1477 .await;
1478 }
1479 for peer in peers.peers.iter().filter(|p| {
1480 p.connected && topology::link(me, p.device_id, roles) == topology::Link::Dial
1481 }) {
1482 let device = peer.device_id;
1483 if table.has(&device).await {
1484 failures.remove(&device);
1486 waiting.remove(&device);
1487 continue;
1488 }
1489 if let Some(until) = waiting.get(&device)
1490 && now < *until
1491 {
1492 continue;
1494 }
1495 if !table.begin_dial(device).await {
1499 continue;
1500 }
1501 let opened = self.open_stream(workspace_id, device).await;
1502 if opened.is_err() {
1503 table.end_dial(device).await;
1504 }
1505 match opened {
1506 Ok(stream) => {
1507 let adopted = self
1508 .adopt_live_session(stream, &root, device, workspace_id)
1509 .await;
1510 table.end_dial(device).await;
1513 match adopted {
1514 Ok(()) => {
1515 failures.remove(&device);
1516 waiting.remove(&device);
1517 }
1518 Err(err) => {
1519 tracing::debug!(
1520 peer = %peer.name,
1521 err = %err,
1522 "a live session could not be started"
1523 );
1524 let count = failures.entry(device).or_insert(0);
1525 *count = count.saturating_add(1);
1526 waiting.insert(device, now + dial_backoff(*count));
1527 }
1528 }
1529 }
1530 Err(err) => {
1531 tracing::debug!("no stream to a peer: {err}");
1534 let count = failures.entry(device).or_insert(0);
1535 *count = count.saturating_add(1);
1536 waiting.insert(device, now + dial_backoff(*count));
1537 }
1538 }
1539 }
1540 }
1541 }
1542 }
1543
1544 async fn adopt_live_session(
1549 self: &Arc<Self>,
1550 stream: sapphire_ipc::RawStream,
1551 root: &Path,
1552 device: GrainId,
1553 workspace_id: GrainId,
1554 ) -> Result<()> {
1555 let replica = self.replica_of(root).await?;
1556 let (_, session) = sapphire_framework_session::open_live_session(
1557 stream,
1558 Arc::clone(&replica),
1559 workspace_id,
1560 )
1561 .await
1562 .map_err(|e| Error::Sync(e.to_string()))?;
1563 let Some((key, table)) = self.live_peers_or_create(root).await else {
1564 return Ok(());
1565 };
1566 let Some(updates) = table.insert(device, session, true).await else {
1571 return Ok(());
1572 };
1573 table.catch_up(device, &replica).await;
1574 self.reindex(&key).await;
1577 self.request_embed(&key);
1578 self.spawn_reader(Arc::clone(&table), updates, key, device);
1579 Ok(())
1580 }
1581
1582 fn spawn_reader(
1589 self: &Arc<Self>,
1590 table: Arc<LivePeers>,
1591 mut updates: broadcast::Receiver<Vec<PathUpdate>>,
1592 key: PathBuf,
1593 from: GrainId,
1594 ) {
1595 let driver = Arc::clone(self);
1596 self.spawn_background(async move {
1597 loop {
1598 match updates.recv().await {
1599 Ok(batch) => {
1600 driver.reindex(&key).await;
1603 driver.request_embed(&key);
1604 table.fan_out(&batch, Some(from)).await;
1605 }
1606 Err(broadcast::error::RecvError::Lagged(missed)) => {
1611 tracing::debug!(
1612 device = %from,
1613 missed,
1614 "fell behind on a live session; the next exchange catches up"
1615 );
1616 }
1617 Err(broadcast::error::RecvError::Closed) => {
1620 return;
1621 }
1622 }
1623 }
1624 });
1625 }
1626
1627 async fn replica_of(&self, root: &Path) -> Result<Arc<Mutex<Replica>>> {
1629 let key = root.canonicalize().map_err(Error::Io)?;
1630 let synced = self.synced.lock().await;
1631 match synced.get(&key) {
1632 Some(entry) => Ok(Arc::clone(&entry.replica)),
1633 None => Err(Error::UnknownWorkspace(key, self.ctx.app_name)),
1634 }
1635 }
1636
1637 pub async fn watch_changes(self: Arc<Self>) -> Result<()> {
1649 let (tx, mut rx) = mpsc::channel(64);
1650 let watcher = Arc::new(Watcher::start(self.roots().await, tx)?);
1651 let _ = self.watcher.set(Arc::clone(&watcher));
1652
1653 for root in self.roots().await {
1657 if let Err(err) = watcher.watch(&root) {
1658 tracing::warn!(root = %root.display(), "could not watch: {err}");
1659 }
1660 }
1661
1662 while let Some(root) = rx.recv().await {
1663 if let Err(err) = self.scan(&root).await {
1666 tracing::warn!(root = %root.display(), "scan after a local edit failed: {err}");
1667 }
1668 self.request_embed(&root);
1669 if let Err(err) = self.sync_now(&root).await {
1670 tracing::warn!(root = %root.display(), "dial after a local edit failed: {err}");
1671 }
1672 }
1673 Ok(())
1674 }
1675
1676 async fn reregister(&self) -> Result<()> {
1678 let workspaces: Vec<WorkspaceRegistration> = {
1679 let synced = self.synced.lock().await;
1680 synced
1681 .iter()
1682 .map(|(root, entry)| WorkspaceRegistration {
1683 workspace_id: entry.workspace_id,
1684 root: root.clone(),
1685 })
1686 .collect()
1687 };
1688 self.bridge()?
1689 .register(RegisterParams {
1690 app_name: self.ctx.app_name.to_owned(),
1691 exe_path: self.exe_path.clone(),
1692 managed_by: wire_managed_by(self.managed_by),
1693 workspaces,
1694 })
1695 .await
1696 .map_err(|e| Error::Bridge(e.to_string()))?;
1697 Ok(())
1698 }
1699
1700 async fn should_skip(&self, workspace_id: GrainId, peer: GrainId) -> bool {
1705 let Some(&me) = self.device_id.get() else {
1706 return false;
1707 };
1708 let cached = self.last_peers.lock().await;
1709 let roles = cached.as_ref().and_then(|p| p.roles_for(workspace_id));
1710 topology::link(me, peer, roles) == topology::Link::Skip
1711 }
1712
1713 pub async fn is_primary(&self, workspace_id: GrainId) -> bool {
1719 let Some(&me) = self.device_id.get() else {
1720 return false;
1721 };
1722 let cached = self.last_peers.lock().await;
1723 cached
1724 .as_ref()
1725 .and_then(|p| p.roles_for(workspace_id))
1726 .is_some_and(|r| r.primary == Some(me))
1727 }
1728
1729 async fn device_id(&self) -> Result<GrainId> {
1731 if let Some(id) = self.device_id.get() {
1732 return Ok(*id);
1733 }
1734 let result = self
1735 .bridge()?
1736 .register(RegisterParams {
1737 app_name: self.ctx.app_name.to_owned(),
1738 exe_path: self.exe_path.clone(),
1739 managed_by: wire_managed_by(self.managed_by),
1740 workspaces: Vec::new(),
1741 })
1742 .await
1743 .map_err(|e| Error::Bridge(e.to_string()))?;
1744 let _ = self.device_id.set(result.device_id);
1745 Ok(result.device_id)
1746 }
1747
1748 fn state_dir(&self, root: &Path) -> Result<PathBuf> {
1750 let workspace = Workspace::from_root(self.ctx, root)?;
1751 Ok(workspace.cache_dir().join("sync"))
1752 }
1753}
1754
1755type Dial<'a> = std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>>;
1757
1758async fn join_all(mut dials: Vec<Dial<'_>>) {
1760 std::future::poll_fn(|cx| {
1761 dials.retain_mut(|dial| dial.as_mut().poll(cx).is_pending());
1762 if dials.is_empty() {
1763 std::task::Poll::Ready(())
1764 } else {
1765 std::task::Poll::Pending
1766 }
1767 })
1768 .await
1769}
1770
1771fn wire_managed_by(managed_by: ManagedBy) -> sapphire_bridge_api::ManagedBy {
1773 match managed_by {
1774 ManagedBy::Service => sapphire_bridge_api::ManagedBy::Service,
1775 ManagedBy::Spawned => sapphire_bridge_api::ManagedBy::Spawned,
1776 }
1777}
1778
1779#[cfg(test)]
1780mod tests {
1781 use super::*;
1782 use crate::sync::testing::StubBridge;
1783 use crate::test_support;
1784 use sapphire_workspace::AppKind;
1785 use std::ffi::OsString;
1786
1787 static CTX: AppContext = AppContext::new("sapphire-synctest");
1788
1789 const DIR_VARS: [&str; 3] = [
1791 "SAPPHIRE_SYNCTEST_CACHE_DIR",
1792 "SAPPHIRE_SYNCTEST_DATA_DIR",
1793 "SAPPHIRE_SYNCTEST_CONFIG_DIR",
1794 ];
1795
1796 struct EnvGuard {
1799 previous: [Option<OsString>; 3],
1800 _lock: std::sync::MutexGuard<'static, ()>,
1801 }
1802
1803 impl Drop for EnvGuard {
1804 fn drop(&mut self) {
1805 for (name, previous) in DIR_VARS.iter().zip(self.previous.iter_mut()) {
1808 match previous.take() {
1809 Some(value) => unsafe { std::env::set_var(name, value) },
1810 None => test_support::remove(name),
1811 }
1812 }
1813 }
1814 }
1815
1816 struct Fixture {
1824 _tmp: tempfile::TempDir,
1825 _env: EnvGuard,
1826 root: PathBuf,
1827 stub: StubBridge,
1828 runtime: Arc<SyncRuntime>,
1829 }
1830
1831 impl Fixture {
1832 fn second_root(&self) -> PathBuf {
1834 let root = self._tmp.path().join("ws2");
1835 std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
1836 root.canonicalize().unwrap()
1837 }
1838 }
1839
1840 const FIXTURE_OPEN_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
1842
1843 fn dialled_peers(me: GrainId, n: usize) -> Vec<sapphire_bridge_api::PeerInfo> {
1845 std::iter::repeat_with(GrainId::random)
1846 .filter(|id| *id > me)
1847 .take(n)
1848 .enumerate()
1849 .map(|(i, device_id)| sapphire_bridge_api::PeerInfo {
1850 device_id,
1851 name: format!("offline-{i}"),
1852 node_id: format!("node-{i}"),
1853 connected: false,
1854 priority: 1,
1855 availability: None,
1856 })
1857 .collect()
1858 }
1859
1860 #[tokio::test(flavor = "multi_thread")]
1861 async fn enabling_answers_at_once_with_a_peer_that_never_answers() {
1862 let f = fixture().await;
1864 let _hung = f.stub.hang_streams().await;
1865 f.stub.set_peers(dialled_peers(f.stub.device_id, 1));
1866
1867 let started = std::time::Instant::now();
1868 f.runtime.enable(&f.root).await.unwrap();
1869 assert!(
1870 started.elapsed() < FIXTURE_OPEN_TIMEOUT,
1871 "enable waited {:?} on the dial",
1872 started.elapsed()
1873 );
1874 }
1875
1876 #[tokio::test(flavor = "multi_thread")]
1877 async fn peers_that_never_answer_are_dialled_together_and_given_up_on() {
1878 let f = fixture().await;
1879 let _hung = f.stub.hang_streams().await;
1880 f.runtime.enable(&f.root).await.unwrap();
1881 f.runtime.settle_dials().await;
1884 f.stub.set_peers(dialled_peers(f.stub.device_id, 3));
1885
1886 let started = std::time::Instant::now();
1887 f.runtime.sync_now(&f.root).await.unwrap();
1888 let took = started.elapsed();
1889 assert!(took >= FIXTURE_OPEN_TIMEOUT, "gave up early: {took:?}");
1890 assert!(
1891 took < FIXTURE_OPEN_TIMEOUT * 2,
1892 "three dials one after another: {took:?}"
1893 );
1894 }
1895
1896 async fn fixture() -> Fixture {
1897 let (tmp, env, root) = scratch();
1898 let (stub, client) = StubBridge::start_in(tmp.path().join("run")).await;
1899 let runtime = Arc::new(
1900 SyncRuntime::new(&CTX, client, "/bin/true".into(), ManagedBy::Service)
1901 .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
1902 );
1903 Fixture {
1904 _tmp: tmp,
1905 _env: env,
1906 root,
1907 stub,
1908 runtime,
1909 }
1910 }
1911
1912 type Bridges = Arc<std::sync::Mutex<std::collections::VecDeque<BridgeClient>>>;
1915
1916 async fn connecting() -> (Fixture, Bridges, BridgeClient) {
1919 let (tmp, env, root) = scratch();
1920 let (stub, client) = StubBridge::start_client_in(tmp.path().join("run")).await;
1921 let bridges: Bridges = Arc::default();
1922 let connector = BridgeConnector::new({
1923 let bridges = Arc::clone(&bridges);
1924 move || {
1925 let next = bridges.lock().unwrap().pop_front();
1926 async move { Ok(next) }
1927 }
1928 });
1929 let runtime = Arc::new(
1930 SyncRuntime::connecting(&CTX, connector, "/bin/true".into(), ManagedBy::Service)
1931 .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
1932 );
1933 let f = Fixture {
1934 _tmp: tmp,
1935 _env: env,
1936 root,
1937 stub,
1938 runtime,
1939 };
1940 (f, bridges, client)
1941 }
1942
1943 async fn eventually(what: &str, mut done: impl AsyncFnMut() -> bool) {
1945 let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
1946 while !done().await {
1947 assert!(tokio::time::Instant::now() < deadline, "never: {what}");
1948 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
1949 }
1950 }
1951
1952 #[tokio::test(flavor = "multi_thread")]
1953 async fn a_runtime_started_without_a_bridge_syncs_once_one_starts() {
1954 let (f, bridges, client) = connecting().await;
1956 f.runtime.connect().await.unwrap();
1957 assert!(!f.runtime.is_connected());
1958
1959 let err = f.runtime.enable(&f.root).await.unwrap_err();
1960 assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
1961 let status = f.runtime.status(&f.root).await;
1962 assert!(status.enabled, "enabled, waiting for the bridge");
1963 assert!(!status.bridge_available);
1964 let id = status.workspace_id.expect("its id");
1965
1966 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
1967 bridges.lock().unwrap().push_back(client);
1968 eventually("the waiting workspace is registered", async || {
1969 f.stub.last_workspaces() == vec![id]
1970 })
1971 .await;
1972 assert_eq!(f.runtime.roots().await, vec![f.root.clone()]);
1973 assert!(f.runtime.status(&f.root).await.bridge_available);
1974 keeping.abort();
1975 }
1976
1977 #[tokio::test(flavor = "multi_thread")]
1978 async fn a_runtime_that_loses_its_bridge_registers_with_the_next_one() {
1979 let (f, bridges, client) = connecting().await;
1980 bridges.lock().unwrap().push_back(client);
1981 f.runtime.connect().await.unwrap();
1982 assert!(f.runtime.is_connected());
1983 let id = f.runtime.enable(&f.root).await.unwrap();
1984 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
1985
1986 f.stub.disconnect();
1987 eventually("the runtime notices", async || !f.runtime.is_connected()).await;
1988 let err = f.runtime.workspaces().await.unwrap_err();
1989 assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
1990
1991 let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
1992 bridges.lock().unwrap().push_back(client);
1993 eventually("the synced workspace is registered again", async || {
1994 next.last_workspaces() == vec![id]
1995 })
1996 .await;
1997 assert!(f.runtime.is_connected());
1998 keeping.abort();
1999 }
2000
2001 #[tokio::test(flavor = "multi_thread")]
2002 async fn disabling_while_the_bridge_is_away_forgets_the_waiting_workspace() {
2003 let (f, bridges, client) = connecting().await;
2004 f.runtime.enable(&f.root).await.unwrap_err();
2005 f.runtime.disable(&f.root).await.unwrap();
2006 assert!(!f.runtime.status(&f.root).await.enabled);
2007
2008 bridges.lock().unwrap().push_back(client);
2009 f.runtime.connect().await.unwrap();
2010 assert!(f.runtime.roots().await.is_empty());
2011 assert!(f.stub.last_workspaces().is_empty());
2012 }
2013
2014 #[tokio::test(flavor = "multi_thread")]
2015 async fn announcements_are_answered_from_a_reconnected_bridge() {
2016 let (f, bridges, client) = connecting().await;
2017 bridges.lock().unwrap().push_back(client);
2018 f.runtime.connect().await.unwrap();
2019 let id = f.runtime.enable(&f.root).await.unwrap();
2020 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2021 let answering = tokio::spawn(Arc::clone(&f.runtime).run());
2022
2023 f.stub.disconnect();
2024 let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2025 bridges.lock().unwrap().push_back(client);
2026 eventually("reconnected", async || next.last_workspaces() == vec![id]).await;
2027
2028 next.announce(id, "ticket").await;
2031 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2032 assert!(!answering.is_finished(), "the announcement loop ended");
2033 answering.abort();
2034 keeping.abort();
2035 }
2036
2037 fn scratch() -> (tempfile::TempDir, EnvGuard, PathBuf) {
2039 let lock = test_support::lock();
2040 let tmp = tempfile::tempdir().unwrap();
2041 let previous = DIR_VARS.map(std::env::var_os);
2042 for (name, dir) in DIR_VARS
2046 .iter()
2047 .zip(["cache", "data", "config"].map(|cat| tmp.path().join(cat)))
2048 {
2049 test_support::set(name, &dir);
2050 }
2051 let env = EnvGuard {
2052 previous,
2053 _lock: lock,
2054 };
2055 CTX.init(AppKind::Server);
2056
2057 let root = tmp.path().join("ws");
2058 std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
2059 let root = root.canonicalize().unwrap();
2060 (tmp, env, root)
2061 }
2062
2063 #[tokio::test(flavor = "multi_thread")]
2064 async fn enabling_registers_the_workspace_with_the_bridge() {
2065 let f = fixture().await;
2066
2067 let id = f.runtime.enable(&f.root).await.unwrap();
2068 assert_eq!(f.stub.last_workspaces(), vec![id]);
2069 }
2070
2071 #[tokio::test(flavor = "multi_thread")]
2072 async fn enabling_twice_is_idempotent_and_keeps_the_same_id() {
2073 let f = fixture().await;
2074
2075 let first = f.runtime.enable(&f.root).await.unwrap();
2076 let second = f.runtime.enable(&f.root).await.unwrap();
2077 assert_eq!(first, second);
2078 assert_eq!(
2079 f.stub.last_workspaces(),
2080 vec![first],
2081 "still exactly one workspace"
2082 );
2083 }
2084
2085 #[tokio::test(flavor = "multi_thread")]
2086 async fn a_registration_carries_the_complete_current_list() {
2087 let f = fixture().await;
2088 let second_root = f.second_root();
2089
2090 let a = f.runtime.enable(&f.root).await.unwrap();
2091 let b = f.runtime.enable(&second_root).await.unwrap();
2092
2093 let mut seen = f.stub.last_workspaces();
2094 seen.sort();
2095 let mut want = vec![a, b];
2096 want.sort();
2097 assert_eq!(seen, want, "registration is the whole set, not a delta");
2098 }
2099
2100 #[tokio::test(flavor = "multi_thread")]
2101 async fn no_later_registration_clears_this_apps_routes() {
2102 let f = fixture().await;
2108 let second_root = f.second_root();
2109
2110 f.runtime.enable(&f.root).await.unwrap();
2111 f.runtime.enable(&second_root).await.unwrap();
2112 f.runtime.sync_now(&f.root).await.unwrap();
2113 f.runtime.disable(&f.root).await.unwrap();
2114
2115 let seen = f.stub.seen.lock().expect("stub");
2116 let first_full = seen
2117 .registrations
2118 .iter()
2119 .position(|r| !r.workspaces.is_empty())
2120 .expect("enabling registers the workspace");
2121 let emptied_afterwards = seen.registrations[first_full + 1..]
2122 .iter()
2123 .filter(|r| r.workspaces.is_empty())
2124 .count();
2125 assert_eq!(
2126 emptied_afterwards, 0,
2127 "a registration after the first workspace must still name the whole set"
2128 );
2129 }
2130
2131 #[tokio::test(flavor = "multi_thread")]
2132 async fn disabling_unregisters_and_leaves_the_files_alone() {
2133 let f = fixture().await;
2134 std::fs::write(f.root.join("keep.md"), "content").unwrap();
2135
2136 let id = f.runtime.enable(&f.root).await.unwrap();
2137 f.runtime.disable(&f.root).await.unwrap();
2138
2139 assert!(f.stub.last_workspaces().is_empty());
2140 assert!(
2141 f.root.join("keep.md").exists(),
2142 "disabling sync must not touch files"
2143 );
2144 assert!(
2145 crate::sync::id::sync_id_path("sapphire-synctest", &f.root).exists(),
2146 "the sync id stays, so re-enabling rejoins the same workspace"
2147 );
2148 let _ = id;
2149 }
2150
2151 #[tokio::test(flavor = "multi_thread")]
2152 async fn re_enabling_after_disabling_reuses_the_identity() {
2153 let f = fixture().await;
2154
2155 let first = f.runtime.enable(&f.root).await.unwrap();
2156 f.runtime.disable(&f.root).await.unwrap();
2157 let again = f.runtime.enable(&f.root).await.unwrap();
2158
2159 assert_eq!(first, again);
2160 }
2161
2162 #[tokio::test(flavor = "multi_thread")]
2163 async fn status_reports_disabled_for_a_workspace_that_was_never_enabled() {
2164 let f = fixture().await;
2165
2166 let status = f.runtime.status(&f.root).await;
2167 assert!(!status.enabled);
2168 assert!(status.workspace_id.is_none());
2169 }
2170
2171 #[tokio::test(flavor = "multi_thread")]
2172 async fn disabling_something_that_was_never_enabled_is_not_an_error() {
2173 let f = fixture().await;
2174 f.runtime
2175 .disable(&f.root)
2176 .await
2177 .expect("disabling twice is fine");
2178 }
2179
2180 #[tokio::test(flavor = "multi_thread")]
2181 async fn a_workspace_whose_root_vanished_is_reported_as_paused_not_deleted() {
2182 let f = fixture().await;
2183 std::fs::write(f.root.join("a.md"), "content").unwrap();
2184 f.runtime.enable(&f.root).await.unwrap();
2185 f.runtime.scan(&f.root).await.unwrap();
2186
2187 std::fs::remove_dir_all(f.root.join(".sapphire-synctest")).unwrap();
2189 let _ = f.runtime.scan(&f.root).await;
2190
2191 let status = f.runtime.status(&f.root).await;
2192 assert!(
2193 status.paused.is_some(),
2194 "a missing root must pause, not replicate as a mass deletion"
2195 );
2196 }
2197
2198 #[tokio::test(flavor = "multi_thread")]
2199 async fn a_directory_that_is_not_a_workspace_cannot_be_enabled() {
2200 let f = fixture().await;
2201 let plain = f._tmp.path().join("plain");
2202 std::fs::create_dir_all(&plain).unwrap();
2203
2204 assert!(f.runtime.enable(&plain).await.is_err());
2205 }
2206
2207 #[tokio::test(flavor = "multi_thread")]
2208 async fn a_write_through_the_workspace_router_is_committed_to_the_replica() {
2209 let f = fixture().await;
2214 f.runtime.enable(&f.root).await.unwrap();
2215
2216 let host = Arc::new(crate::WorkspaceHost::new(&CTX));
2217 host.select(&f.root, false).await.unwrap();
2218 let current = Arc::new(crate::Current::new(
2219 &CTX,
2220 host,
2221 Some(Arc::clone(&f.runtime)),
2222 crate::selection::SelectionFile::at(f._tmp.path().join("workspace.toml")),
2223 ));
2224 let router = Arc::new(crate::workspace_router(current));
2225 let (client_conn, server_conn) = sapphire_ipc::Connection::pair();
2226 tokio::spawn(async move {
2227 let info = sapphire_ipc::ServerInfo {
2228 version: "0.0.0".into(),
2229 api: sapphire_backend::protocol::API_VERSION,
2230 app_api: None,
2231 pid: std::process::id(),
2232 managed_by: ManagedBy::Spawned,
2233 };
2234 let _ = sapphire_ipc::serve(server_conn, router, "sapphire-synctest", info).await;
2235 });
2236 let info = sapphire_ipc::ClientInfo {
2237 kind: "test".into(),
2238 version: "0.0.0".into(),
2239 api: sapphire_backend::protocol::API_VERSION,
2240 app_api: None,
2241 pid: std::process::id(),
2242 };
2243 let (client, _) = sapphire_ipc::Client::handshake(client_conn, "sapphire-synctest", info)
2244 .await
2245 .unwrap();
2246
2247 let _: sapphire_backend::protocol::Ack = client
2248 .call(
2249 sapphire_backend::protocol::WRITE_FILE,
2250 sapphire_backend::protocol::ContentParams {
2251 path: PathBuf::from("through-the-server.md"),
2252 content: "written by the server".into(),
2253 },
2254 )
2255 .await
2256 .unwrap();
2257
2258 let synced = f.runtime.synced.lock().await;
2261 let entry = synced.get(&f.root).expect("enabled above");
2262 let replica = entry.replica.lock().await;
2263 let state = replica
2264 .state("through-the-server.md")
2265 .expect("a readable state");
2266 assert!(
2267 state.is_some(),
2268 "a write through the server must be scanned into the replica, not left to the watcher"
2269 );
2270 }
2271
2272 #[tokio::test(flavor = "multi_thread")]
2273 async fn an_announcement_for_an_unknown_workspace_is_ignored_not_fatal() {
2274 let f = fixture().await;
2275 f.runtime.enable(&f.root).await.unwrap();
2276
2277 let driver = Arc::clone(&f.runtime);
2278 let handle = tokio::spawn(async move { driver.run().await });
2279
2280 f.stub
2281 .announce(grain_id::GrainId::random(), "no-such-ticket")
2282 .await;
2283 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
2284
2285 assert!(
2286 !handle.is_finished(),
2287 "one bad announcement must not stop the runtime"
2288 );
2289 handle.abort();
2290 }
2291}