1use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use grain_id::GrainId;
13use sapphire_backend::BackendEvent;
14use sapphire_backend::protocol as proto;
15use sapphire_bridge_api::{
16 IncomingParams, RegisterParams, WorkspaceRegistration, WorkspacesResult,
17};
18use sapphire_bridge_client::BridgeClient;
19use sapphire_ipc::ManagedBy;
20use sapphire_sync::{PathUpdate, PauseReason, Replica, ReplicaConfig, ScanOutcome, SystemClock};
21use sapphire_workspace::{AppContext, Workspace};
22use tokio::sync::{Mutex, OnceCell, broadcast, mpsc};
23
24pub mod id;
25mod live;
26mod methods;
27#[cfg(any(test, feature = "test-util"))]
28pub mod testing;
29mod topology;
30mod watch;
31
32pub use id::{SYNC_ID_FILE, WORKSPACE_MAP_FILE, sync_id, sync_id_path};
33pub use live::{DIAL_BACKOFF_MAX, OPEN_TIMEOUT};
34pub use methods::sync_router;
35pub use sapphire_bridge_api::{PeerInfo, PeersResult};
37pub use watch::{DEBOUNCE, Watcher};
38
39use live::{LivePeers, dial_backoff};
40
41use crate::error::{Error, Result};
42use crate::settings::Settings;
43
44use crate::host::WorkspaceHost;
45
46#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
48pub struct SyncStatus {
49 pub enabled: bool,
51 pub workspace_id: Option<GrainId>,
53 pub peers: usize,
55 pub paused: Option<String>,
57 pub last_error: Option<String>,
59 pub bridge_available: bool,
61 pub topology: proto::Topology,
63 pub embedding: Option<proto::EmbeddingProgress>,
65 pub pending_delivery: bool,
67 pub acknowledging_peers: usize,
69}
70
71struct Synced {
73 workspace_id: GrainId,
75 replica: Arc<Mutex<Replica>>,
77 paused: Option<PauseReason>,
79 last_error: Option<String>,
81 delivered_through: u64,
86}
87
88const STALE_VECTOR_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
91
92const PASS_BATCH: usize = 4;
95
96#[derive(Clone, Copy, Debug)]
98pub struct BackfillTiming {
99 pub grace: std::time::Duration,
102 pub interval: std::time::Duration,
104}
105
106impl Default for BackfillTiming {
107 fn default() -> Self {
108 BackfillTiming {
109 grace: std::time::Duration::from_secs(10 * 60),
110 interval: std::time::Duration::from_secs(10 * 60),
111 }
112 }
113}
114
115pub const RECONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_millis(500);
118
119pub const RECONNECT_BACKOFF_MAX: std::time::Duration = std::time::Duration::from_secs(30);
122
123fn reconnect_backoff(first: std::time::Duration, attempt: u32) -> std::time::Duration {
125 first
126 .saturating_mul(1u32 << attempt.min(16))
127 .min(RECONNECT_BACKOFF_MAX.max(first))
128}
129
130type Connecting = std::pin::Pin<
132 Box<dyn std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>> + Send>,
133>;
134
135#[derive(Clone)]
140pub struct BridgeConnector(Arc<dyn Fn() -> Connecting + Send + Sync>);
141
142impl BridgeConnector {
143 pub fn new<F, Fut>(connect: F) -> BridgeConnector
145 where
146 F: Fn() -> Fut + Send + Sync + 'static,
147 Fut: std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>>
148 + Send
149 + 'static,
150 {
151 BridgeConnector(Arc::new(move || Box::pin(connect())))
152 }
153
154 pub fn standard(kind: &'static str, version: &'static str) -> BridgeConnector {
157 BridgeConnector::new(move || async move {
158 match BridgeClient::connect(kind, version).await {
159 Ok(client) => Ok(Some(client)),
160 Err(sapphire_ipc::Error::NotRunning(_)) => Ok(None),
161 Err(err) => Err(err),
162 }
163 })
164 }
165
166 pub fn at(
168 endpoint: sapphire_ipc::Endpoint,
169 kind: &'static str,
170 version: &'static str,
171 ) -> BridgeConnector {
172 BridgeConnector::new(move || {
173 let endpoint = endpoint.clone();
174 async move { BridgeClient::connect_at(&endpoint, kind, version).await }
175 })
176 }
177
178 pub(crate) async fn connect(&self) -> sapphire_ipc::Result<Option<BridgeClient>> {
180 (self.0)().await
181 }
182}
183
184impl std::fmt::Debug for BridgeConnector {
185 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
186 f.write_str("BridgeConnector")
187 }
188}
189
190enum Reclaimed {
192 Open(Arc<Mutex<Replica>>),
194 Closing,
196 Closed,
198}
199
200const CLOSING_RETRIES: u32 = 50;
202const CLOSING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(20);
203
204pub struct SyncRuntime {
206 ctx: &'static AppContext,
207 bridge: tokio::sync::watch::Sender<Option<Arc<BridgeClient>>>,
210 connector: Option<BridgeConnector>,
212 reconnect_now: tokio::sync::Notify,
215 pending: Mutex<HashMap<PathBuf, GrainId>>,
222 subscribed: std::sync::Mutex<Option<(Arc<BridgeClient>, broadcast::Receiver<IncomingParams>)>>,
227 exe_path: PathBuf,
228 managed_by: ManagedBy,
229 synced: Mutex<HashMap<PathBuf, Synced>>,
230 stopped: std::sync::Mutex<HashMap<PathBuf, std::sync::Weak<Mutex<Replica>>>>,
237 device_id: OnceCell<GrainId>,
243 watcher: OnceCell<Arc<Watcher>>,
253 host: OnceCell<Arc<WorkspaceHost>>,
264 settings: std::sync::Mutex<Option<Settings>>,
267 reindexing: Mutex<()>,
270 embed_wanted: Mutex<std::collections::HashSet<PathBuf>>,
273 embedding: Mutex<()>,
275 embed_running: Mutex<std::collections::HashSet<PathBuf>>,
277 backfill: std::sync::Mutex<BackfillTiming>,
279 bridge_endpoint: Option<sapphire_ipc::Endpoint>,
281 live: Mutex<HashMap<PathBuf, Arc<LivePeers>>>,
287 last_peers: Mutex<Option<PeersResult>>,
290 open_timeout: std::time::Duration,
292 reconnect_first: std::time::Duration,
295 first_dials: std::sync::Mutex<tokio::task::JoinSet<()>>,
298 background: std::sync::Mutex<tokio::task::JoinSet<()>>,
303}
304
305impl SyncRuntime {
306 pub fn new(
311 ctx: &'static AppContext,
312 bridge: Arc<BridgeClient>,
313 exe_path: PathBuf,
314 managed_by: ManagedBy,
315 ) -> SyncRuntime {
316 SyncRuntime::build(ctx, Some(bridge), None, exe_path, managed_by)
317 }
318
319 pub fn connecting(
327 ctx: &'static AppContext,
328 connector: BridgeConnector,
329 exe_path: PathBuf,
330 managed_by: ManagedBy,
331 ) -> SyncRuntime {
332 SyncRuntime::build(ctx, None, Some(connector), exe_path, managed_by)
333 }
334
335 pub fn with_connector(mut self, connector: BridgeConnector) -> SyncRuntime {
337 self.connector = Some(connector);
338 self
339 }
340
341 fn build(
342 ctx: &'static AppContext,
343 bridge: Option<Arc<BridgeClient>>,
344 connector: Option<BridgeConnector>,
345 exe_path: PathBuf,
346 managed_by: ManagedBy,
347 ) -> SyncRuntime {
348 SyncRuntime {
349 ctx,
350 bridge: tokio::sync::watch::Sender::new(bridge),
351 connector,
352 reconnect_now: tokio::sync::Notify::new(),
353 pending: Mutex::new(HashMap::new()),
354 subscribed: std::sync::Mutex::new(None),
355 exe_path,
356 managed_by,
357 synced: Mutex::new(HashMap::new()),
358 stopped: std::sync::Mutex::new(HashMap::new()),
359 device_id: OnceCell::new(),
360 watcher: OnceCell::new(),
361 host: OnceCell::new(),
362 settings: std::sync::Mutex::new(None),
363 reindexing: Mutex::new(()),
364 embed_wanted: Mutex::new(std::collections::HashSet::new()),
365 embedding: Mutex::new(()),
366 embed_running: Mutex::new(std::collections::HashSet::new()),
367 backfill: std::sync::Mutex::new(BackfillTiming::default()),
368 bridge_endpoint: None,
369 live: Mutex::new(HashMap::new()),
370 last_peers: Mutex::new(None),
371 open_timeout: OPEN_TIMEOUT,
372 reconnect_first: RECONNECT_BACKOFF,
373 first_dials: std::sync::Mutex::new(tokio::task::JoinSet::new()),
374 background: std::sync::Mutex::new(tokio::task::JoinSet::new()),
375 }
376 }
377
378 fn bridge(&self) -> Result<Arc<BridgeClient>> {
380 match &*self.bridge.borrow() {
381 Some(bridge) if !bridge.is_closed() => Ok(Arc::clone(bridge)),
382 _ => Err(Error::BridgeDisconnected),
383 }
384 }
385
386 pub fn is_connected(&self) -> bool {
388 self.bridge().is_ok()
389 }
390
391 pub(crate) fn connected_bridge(&self) -> Option<Arc<BridgeClient>> {
393 self.bridge().ok()
394 }
395
396 pub(crate) fn reconnect_now(&self) {
401 self.reconnect_now.notify_one();
402 }
403
404 pub(crate) fn bridge_changes(&self) -> tokio::sync::watch::Receiver<Option<Arc<BridgeClient>>> {
407 self.bridge.subscribe()
408 }
409
410 fn spawn_background(&self, task: impl std::future::Future<Output = ()> + Send + 'static) {
412 let mut background = self.background.lock().expect("background tasks");
413 while background.try_join_next().is_some() {}
415 background.spawn(task);
416 }
417
418 pub fn with_open_timeout(mut self, timeout: std::time::Duration) -> SyncRuntime {
421 self.open_timeout = timeout;
422 self
423 }
424
425 #[cfg(test)]
428 pub(crate) fn with_reconnect_backoff(mut self, first: std::time::Duration) -> SyncRuntime {
429 self.reconnect_first = first;
430 self
431 }
432
433 pub fn with_bridge_endpoint(mut self, endpoint: sapphire_ipc::Endpoint) -> SyncRuntime {
440 self.bridge_endpoint = Some(endpoint);
441 self
442 }
443
444 pub fn set_backfill_timing(&self, timing: BackfillTiming) {
447 *self.backfill.lock().expect("backfill timing") = timing;
448 }
449
450 fn backfill_timing(&self) -> BackfillTiming {
451 *self.backfill.lock().expect("backfill timing")
452 }
453
454 pub(crate) fn set_host(&self, host: Arc<WorkspaceHost>) {
455 let _ = self.host.set(host);
456 }
457
458 pub(crate) fn set_settings(&self, settings: Option<Settings>) {
462 *self.settings.lock().unwrap_or_else(|e| e.into_inner()) = settings;
463 }
464
465 fn refresh_settings(&self) {
468 let settings = self
469 .settings
470 .lock()
471 .unwrap_or_else(|e| e.into_inner())
472 .clone();
473 if let Some(settings) = settings
474 && settings.refresh().is_err()
475 {
476 tracing::warn!("could not read the settings again after a change to the files");
478 }
479 }
480
481 pub async fn enable(self: &Arc<Self>, root: &Path) -> Result<GrainId> {
486 let key = root.canonicalize().map_err(Error::Io)?;
487 let watch_key = key.clone();
488 let workspace_id = sync_id(self.ctx.app_name, &key)?;
489
490 {
493 let mut pending = self.pending.lock().await;
494 if self.bridge().is_err() {
495 pending.insert(key, workspace_id);
496 return Err(Error::BridgeDisconnected);
497 }
498 pending.remove(&key);
499 }
500
501 {
504 let mut synced = self.synced.lock().await;
505 if let Some(existing) = synced.get(&key) {
506 return Ok(existing.workspace_id);
507 }
508 let replica = match self.reclaim(&key) {
511 Reclaimed::Open(replica) => replica,
512 reclaimed => {
513 let device_id = self.device_id().await?;
514 let state_dir = self.state_dir(&key)?;
515 std::fs::create_dir_all(&state_dir).map_err(Error::Io)?;
516 let config =
517 ReplicaConfig::new(self.ctx.app_name, key.clone(), device_id, &state_dir);
518 let mut tries = match reclaimed {
521 Reclaimed::Closing => CLOSING_RETRIES,
522 _ => 0,
523 };
524 let replica = loop {
525 match Replica::open(config.clone(), Arc::new(SystemClock)) {
526 Ok(replica) => break replica,
527 Err(_) if tries > 0 => {
528 tries -= 1;
529 tokio::time::sleep(CLOSING_RETRY_DELAY).await;
530 }
531 Err(err) => return Err(Error::Sync(err.to_string())),
532 }
533 };
534 Arc::new(Mutex::new(replica))
535 }
536 };
537 let delivered_through = own_counter(&*replica.lock().await);
540 synced.insert(
541 key.clone(),
542 Synced {
543 workspace_id,
544 replica,
545 paused: None,
546 last_error: None,
547 delivered_through,
548 },
549 );
550 }
551
552 if let Some(watcher) = self.watcher.get()
556 && let Err(err) = watcher.watch(&watch_key)
557 {
558 tracing::warn!(root = %watch_key.display(), "could not watch: {err}");
559 }
560
561 self.reregister().await?;
564
565 let runtime = Arc::clone(self);
583 let mut dials = self.first_dials.lock().expect("first dials");
584 while dials.try_join_next().is_some() {}
586 dials.spawn(async move {
587 if let Err(err) = runtime.scan(&key).await {
588 tracing::warn!(root = %key.display(), "the first scan after enabling failed: {err}");
589 }
590 if let Err(err) = runtime.sync_now(&key).await {
591 tracing::warn!(root = %key.display(), "the first session after enabling failed: {err}");
592 }
593 });
594 Ok(workspace_id)
595 }
596
597 pub async fn disable(&self, root: &Path) -> Result<()> {
603 let (replica, unregistered) = self.stop(root).await;
604 if let Some(replica) = replica {
605 let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
606 let mut stopped = self.stopped.lock().expect("stopped replicas");
607 stopped.retain(|_, replica| replica.strong_count() > 0);
609 stopped.insert(key, Arc::downgrade(&replica));
610 }
611 unregistered
612 }
613
614 pub async fn release(&self, root: &Path, limit: std::time::Duration) -> Result<()> {
623 let (replica, unregistered) = self.stop(root).await;
624 if let Err(err) = unregistered {
625 tracing::warn!(root = %root.display(), "stopping sync: {err}");
626 }
627 let Some(replica) = replica else {
628 return Ok(());
629 };
630 let deadline = tokio::time::Instant::now() + limit;
631 while Arc::strong_count(&replica) > 1 {
632 if tokio::time::Instant::now() >= deadline {
633 return Err(Error::Sync(format!(
634 "the replica of {} is still in use; try again",
635 root.display()
636 )));
637 }
638 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
639 }
640 Ok(())
641 }
642
643 async fn stop(&self, root: &Path) -> (Option<Arc<Mutex<Replica>>>, Result<()>) {
646 let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
649 self.pending.lock().await.remove(&key);
650 let removed = self.synced.lock().await.remove(&key);
651 if removed.is_some()
652 && let Some(watcher) = self.watcher.get()
653 {
654 watcher.unwatch(&key);
655 }
656 if let Some(peers) = self.live.lock().await.remove(&key) {
659 peers.drop_connections().await;
660 }
661 let Some(removed) = removed else {
662 return (None, Ok(()));
663 };
664 let unregistered = async {
665 let Ok(bridge) = self.bridge() else {
668 return Ok(());
669 };
670 bridge
671 .unregister(removed.workspace_id)
672 .await
673 .map_err(|e| Error::Bridge(e.to_string()))?;
674 self.reregister().await
675 }
676 .await;
677 (Some(removed.replica), unregistered)
678 }
679
680 pub async fn workspaces(&self) -> Result<WorkspacesResult> {
686 self.bridge()?
687 .workspaces()
688 .await
689 .map_err(|e| Error::Bridge(e.to_string()))
690 }
691
692 pub async fn place(&self, selector: &str, dir: &Path) -> Result<PathBuf> {
703 let workgroup = self.workspaces().await?;
704 let wanted = workgroup
705 .workspaces
706 .iter()
707 .find(|w| w.name == selector || w.workspace_id.to_string() == selector)
710 .ok_or_else(|| Error::UnknownWorkspaceName(selector.to_owned()))?;
711
712 if wanted.app_name != self.ctx.app_name {
715 return Err(Error::WrongApp {
716 name: wanted.name.clone(),
717 app_name: wanted.app_name.clone(),
718 });
719 }
720
721 let root = dir.canonicalize().map_err(Error::Io)?;
722 let marker = root.join(format!(".{}", self.ctx.app_name));
723 if !marker.is_dir() {
724 std::fs::create_dir(&marker).map_err(Error::Io)?;
725 }
726 let workspace = Workspace::from_root(self.ctx, &root)?;
727 if workspace.root != root {
728 return Err(Error::UnknownWorkspace(root, self.ctx.app_name));
729 }
730
731 {
735 let synced = self.synced.lock().await;
736 if let Some(entry) = synced.get(&root)
737 && entry.workspace_id != wanted.workspace_id
738 {
739 return Err(Error::SyncId(format!(
740 "{} already syncs as {}, not {}",
741 root.display(),
742 entry.workspace_id,
743 wanted.workspace_id
744 )));
745 }
746 }
747
748 let marker = workspace.marker_dir();
753 let id_path = marker.join(crate::sync::id::SYNC_ID_FILE);
754 match std::fs::read_to_string(&id_path) {
755 Ok(existing) if existing.trim() != wanted.workspace_id.to_string() => {
756 return Err(Error::SyncId(format!(
757 "{} already holds identity {}, not {}",
758 root.display(),
759 existing.trim(),
760 wanted.workspace_id
761 )));
762 }
763 Err(_) => {
764 std::fs::write(&id_path, format!("{}\n", wanted.workspace_id))
765 .map_err(Error::Io)?;
766 }
767 Ok(_) => {}
768 }
769
770 let map_path = root
773 .join(format!(".{}", self.ctx.app_name))
774 .join(WORKSPACE_MAP_FILE);
775 let workspace_id = wanted.workspace_id.to_string();
776 std::fs::write(&map_path, format!("{workspace_id}\n")).map_err(Error::Io)?;
777 Ok(root)
778 }
779
780 pub async fn status(&self, root: &Path) -> SyncStatus {
782 let bridge = self.bridge().ok();
783 let bridge_available = match &bridge {
784 Some(bridge) => bridge.status().await.is_ok(),
785 None => false,
786 };
787 let answer = match &bridge {
788 Some(_) => self.peers().await.ok(),
789 None => None,
790 };
791 let peers = answer
792 .as_ref()
793 .map_or(0, |p| p.peers.len().saturating_sub(1));
794 let Ok(key) = root.canonicalize() else {
795 return SyncStatus {
796 enabled: false,
797 workspace_id: None,
798 peers,
799 paused: None,
800 last_error: None,
801 bridge_available,
802 topology: proto::Topology::Mesh,
803 embedding: None,
804 pending_delivery: false,
805 acknowledging_peers: 0,
806 };
807 };
808 let synced = self.synced.lock().await;
809 match synced.get(&key) {
810 Some(entry) => {
811 let replica = Arc::clone(&entry.replica);
812 let delivered_through = entry.delivered_through;
813 let base = SyncStatus {
814 enabled: true,
815 workspace_id: Some(entry.workspace_id),
816 peers,
817 paused: entry.paused.map(|r| format!("{r:?}")),
818 last_error: entry.last_error.clone(),
819 bridge_available,
820 topology: topology::topology(
821 answer
822 .as_ref()
823 .and_then(|p| p.roles_for(entry.workspace_id)),
824 ),
825 embedding: None,
826 pending_delivery: false,
827 acknowledging_peers: 0,
828 };
829 drop(synced);
830 let own = own_counter(&*replica.lock().await);
833 let acknowledging_peers = match self.live_peers(&key).await {
834 Some(table) => table.acknowledging().await,
835 None => 0,
836 };
837 SyncStatus {
838 embedding: self.embedding_progress(&key).await,
839 pending_delivery: own > delivered_through,
840 acknowledging_peers,
841 ..base
842 }
843 }
844 None => {
845 drop(synced);
846 let waiting = self.pending.lock().await.get(&key).copied();
849 SyncStatus {
850 enabled: waiting.is_some(),
851 workspace_id: waiting,
852 peers,
853 paused: None,
854 last_error: None,
855 bridge_available,
856 topology: proto::Topology::Mesh,
857 embedding: None,
858 pending_delivery: false,
859 acknowledging_peers: 0,
860 }
861 }
862 }
863 }
864
865 async fn embedding_progress(&self, key: &Path) -> Option<proto::EmbeddingProgress> {
868 let host = self.host.get()?;
869 let backend = host.open_backend(key)?;
870 let state = backend.state();
871 state.embedder()?;
872 let info = state.db_info().ok()?;
873 Some(proto::EmbeddingProgress {
874 vectors: info.vector_count,
875 pending: info.pending_count,
876 running: self.embed_running.lock().await.contains(key),
877 })
878 }
879
880 pub async fn scan(&self, root: &Path) -> Result<()> {
886 let Ok(key) = root.canonicalize() else {
887 return Ok(());
888 };
889 let replica = {
890 let synced = self.synced.lock().await;
891 match synced.get(&key) {
892 Some(entry) => Arc::clone(&entry.replica),
893 None => return Ok(()),
894 }
895 };
896 let (outcome, updates) = {
897 let mut replica = replica.lock().await;
898 let outcome = replica.scan().map_err(|e| Error::Sync(e.to_string()))?;
899 let updates = match &outcome {
904 ScanOutcome::Scanned(report) => {
905 report
913 .recorded
914 .iter()
915 .map(|entry| PathUpdate {
916 path: entry.path.clone(),
917 versions: vec![entry.clone()],
918 seen: {
919 let mut seen = entry.context.clone();
920 seen.add_dot(&entry.dot);
921 seen
922 },
923 })
924 .collect()
925 }
926 ScanOutcome::Paused(_) => Vec::new(),
927 };
928 (outcome, updates)
929 };
930 {
931 let mut synced = self.synced.lock().await;
932 if let Some(entry) = synced.get_mut(&key) {
933 entry.paused = match outcome {
934 ScanOutcome::Paused(reason) => Some(reason),
935 ScanOutcome::Scanned(_) => None,
936 };
937 }
938 }
939 self.after_commit(&key, updates).await;
942 Ok(())
943 }
944
945 async fn reindex(&self, root: &Path) {
960 self.refresh_settings();
962 let Some(host) = self.host.get() else {
963 return;
964 };
965 let host = Arc::clone(host);
966 let root = root.to_owned();
967 let _serialised = self.reindexing.lock().await;
970 let started = std::time::Instant::now();
972 let backend = match host.backend_at(&root).await {
975 Ok(backend) => backend,
976 Err(err) => {
977 crate::instrument::reindexed(started, false);
978 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
979 return;
980 }
981 };
982 let state = Arc::clone(backend.state());
983 let result = tokio::task::spawn_blocking(move || state.sync_retrieve()).await;
984 crate::instrument::reindexed(started, matches!(result, Ok(Ok(_))));
985 match result {
986 Ok(Ok(report)) => {
987 tracing::debug!(root = %root.display(), ?report, "re-indexed after a session");
988 }
989 Ok(Err(err)) => {
990 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
991 }
992 Err(err) => {
993 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
994 }
995 }
996 }
997
998 pub fn request_embed(self: &Arc<Self>, root: &Path) {
1005 let Ok(key) = root.canonicalize() else {
1006 return;
1007 };
1008 let runtime = Arc::clone(self);
1009 self.spawn_background(async move {
1010 if !runtime.embed_wanted.lock().await.insert(key.clone()) {
1011 return;
1012 }
1013 let _one_at_a_time = runtime.embedding.lock().await;
1014 runtime.embed_wanted.lock().await.remove(&key);
1016 runtime.embed_running.lock().await.insert(key.clone());
1017 runtime.embed_pass(&key).await;
1018 runtime.embed_running.lock().await.remove(&key);
1019 });
1020 }
1021
1022 async fn embed_pass(&self, key: &Path) {
1024 let Some(host) = self.host.get() else {
1025 return;
1026 };
1027 let (workspace_id, replica) = {
1028 let synced = self.synced.lock().await;
1029 match synced.get(key) {
1030 Some(entry) => (entry.workspace_id, Arc::clone(&entry.replica)),
1031 None => return,
1032 }
1033 };
1034 let backend = match host.backend_at(key).await {
1035 Ok(backend) => backend,
1036 Err(err) => {
1037 tracing::warn!(root = %key.display(), "embedding pass: {err}");
1038 return;
1039 }
1040 };
1041 let state = Arc::clone(backend.state());
1042 if let Err(err) = state
1043 .refresh_embedder_at(self.bridge_endpoint.clone())
1044 .await
1045 {
1046 tracing::warn!(root = %key.display(), "embedding pass: {err}");
1047 return;
1048 }
1049 if state.embedder().is_none() {
1050 return;
1051 }
1052 let primary = self.is_primary(workspace_id).await;
1053 let grace_ms = self.backfill_timing().grace.as_millis() as u64;
1056 let now_ms = std::time::SystemTime::now()
1057 .duration_since(std::time::UNIX_EPOCH)
1058 .map_or(0, |d| d.as_millis() as u64);
1059 let (mine, settled, live) = {
1060 let replica = replica.lock().await;
1061 let me = replica.replica_id();
1062 let states = match replica.states() {
1063 Ok(states) => states,
1064 Err(err) => {
1065 tracing::warn!(root = %key.display(), "embedding pass: {err}");
1066 return;
1067 }
1068 };
1069 let mut mine = HashMap::new();
1070 let mut settled = HashMap::new();
1072 let mut live = std::collections::HashSet::new();
1073 for (path, path_state) in states {
1074 let winner = path_state.winner();
1075 let Some(hash) = winner.content.hash() else {
1076 continue;
1077 };
1078 let hash = hash.to_hex();
1079 if winner.dot.replica == me {
1080 mine.insert(path, hash.clone());
1081 } else if now_ms.saturating_sub(winner.hlc.wall_ms) >= grace_ms {
1082 settled.insert(path, hash.clone());
1083 }
1084 live.insert(hash);
1085 }
1086 (mine, settled, live)
1087 };
1088 let root = key.to_owned();
1089 let result = tokio::task::spawn_blocking(move || {
1090 let policy = |rel: &str, hash: &str| {
1093 mine.get(rel).is_some_and(|h| h.as_str() == hash)
1094 || (primary && settled.get(rel).is_some_and(|h| h.as_str() == hash))
1095 };
1096 let embedded = state.embed_pending_with(&policy, PASS_BATCH)?;
1097 let removed = if primary {
1098 state.remove_stale_vectors(&live, STALE_VECTOR_AGE)
1099 } else {
1100 0
1101 };
1102 Ok::<_, sapphire_workspace::Error>((embedded, removed))
1103 })
1104 .await;
1105 match result {
1106 Ok(Ok((embedded, removed))) if embedded > 0 || removed > 0 => tracing::info!(
1107 root = %root.display(),
1108 embedded,
1109 removed,
1110 "embedding pass"
1111 ),
1112 Ok(Ok(_)) => {}
1113 Ok(Err(err)) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
1114 Err(err) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
1115 }
1116 }
1117
1118 pub async fn sync_now(self: &Arc<Self>, root: &Path) -> Result<()> {
1120 let Ok(key) = root.canonicalize() else {
1121 return Ok(());
1122 };
1123 let workspace_id = {
1124 let synced = self.synced.lock().await;
1125 match synced.get(&key) {
1126 Some(entry) => entry.workspace_id,
1127 None => return Ok(()),
1128 }
1129 };
1130 let peers = self.peers().await?;
1131 let me = self.device_id().await?;
1132
1133 let roles = peers.roles_for(workspace_id).cloned();
1138 let mut dials: Vec<Dial<'_>> = Vec::new();
1142 for peer in peers
1143 .peers
1144 .iter()
1145 .filter(|p| topology::link(me, p.device_id, roles.as_ref()) == topology::Link::Dial)
1146 .cloned()
1147 .collect::<Vec<_>>()
1148 {
1149 if let Some(live) = self.live_peers(&key).await
1153 && live.has(&peer.device_id).await
1154 {
1155 continue;
1156 }
1157 let Some((_, table)) = self.live_peers_or_create(&key).await else {
1164 continue;
1165 };
1166 if !table.begin_dial(peer.device_id).await {
1167 continue;
1168 }
1169 let key = &key;
1170 dials.push(Box::pin(async move {
1171 let result = self
1173 .sync_one(key, &peer.device_id, peer.name.clone(), workspace_id)
1174 .await;
1175 table.end_dial(peer.device_id).await;
1176 if let Err(err) = result {
1177 tracing::debug!(peer = %peer.name, "no session: {err}");
1178 }
1179 }));
1180 }
1181 join_all(dials).await;
1182 Ok(())
1183 }
1184
1185 async fn sync_one(
1192 self: &Arc<Self>,
1193 key: &Path,
1194 device: &GrainId,
1195 name: String,
1196 workspace_id: GrainId,
1197 ) -> Result<()> {
1198 let stream = self.open_stream(workspace_id, *device).await?;
1199 self.adopt_live_session(stream, key, *device, workspace_id)
1200 .await
1201 .map_err(|err| {
1202 tracing::warn!(peer = %name, "a live session could not be started: {err}");
1203 err
1204 })
1205 }
1206
1207 async fn open_stream(
1209 &self,
1210 workspace_id: GrainId,
1211 device: GrainId,
1212 ) -> Result<sapphire_ipc::RawStream> {
1213 match tokio::time::timeout(
1214 self.open_timeout,
1215 self.bridge()?.open_stream(workspace_id, device),
1216 )
1217 .await
1218 {
1219 Ok(opened) => Ok(opened?),
1220 Err(_) => Err(Error::Bridge(format!(
1221 "no stream to {device} within {} s; it may be offline",
1222 self.open_timeout.as_secs_f32()
1223 ))),
1224 }
1225 }
1226
1227 pub async fn run(self: Arc<Self>) -> Result<()> {
1230 let mut slot = self.bridge.subscribe();
1231 loop {
1232 let bridge = slot.borrow_and_update().clone();
1233 match bridge {
1234 Some(bridge) if !bridge.is_closed() => self.answer(&bridge, &mut slot).await,
1235 _ => {
1237 if slot.changed().await.is_err() {
1238 return Ok(());
1239 }
1240 }
1241 }
1242 }
1243 }
1244
1245 async fn answer(
1247 self: &Arc<Self>,
1248 bridge: &Arc<BridgeClient>,
1249 slot: &mut tokio::sync::watch::Receiver<Option<Arc<BridgeClient>>>,
1250 ) {
1251 let early = self
1252 .subscribed
1253 .lock()
1254 .expect("subscribed")
1255 .take_if(|(b, _)| Arc::ptr_eq(b, bridge));
1256 let mut incoming = match early {
1257 Some((_, incoming)) => incoming,
1258 None => bridge.incoming(),
1259 };
1260 loop {
1261 let received = tokio::select! {
1262 received = incoming.recv() => received,
1263 () = bridge.closed() => return,
1264 _ = slot.changed() => return,
1265 };
1266 let announcement = match received {
1267 Ok(a) => a,
1268 Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
1269 tracing::warn!(missed = n, "fell behind on bridge announcements");
1270 continue;
1271 }
1272 Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1273 };
1274
1275 let found = {
1276 let synced = self.synced.lock().await;
1277 synced
1278 .iter()
1279 .find(|(_, s)| s.workspace_id == announcement.workspace_id)
1280 .map(|(root, s)| (root.clone(), Arc::clone(&s.replica)))
1281 };
1282 let Some((root, replica)) = found else {
1285 tracing::debug!(
1288 workspace = %announcement.workspace_id,
1289 "an announcement for a workspace this server does not hold"
1290 );
1291 continue;
1292 };
1293
1294 let bridge = Arc::clone(bridge);
1295 let workspace_id = announcement.workspace_id;
1296 let peer = announcement.peer_device_id;
1297 let driver = Arc::clone(self);
1298 self.spawn_background(async move {
1299 let stream = match bridge.accept_stream(announcement.ticket).await {
1300 Ok(stream) => stream,
1301 Err(err) => {
1302 tracing::warn!("could not claim an announced stream: {err}");
1303 return;
1304 }
1305 };
1306 if driver.should_skip(workspace_id, peer).await {
1307 tracing::debug!(%peer, "dropped an inbound session: both ends are outside this workspace's star");
1308 return;
1309 }
1310 let (_, session) = match sapphire_framework_session::open_live_session(
1315 stream,
1316 Arc::clone(&replica),
1317 workspace_id,
1318 )
1319 .await
1320 {
1321 Ok(started) => started,
1322 Err(err) => {
1323 crate::instrument::sync_session(false);
1324 tracing::warn!("an inbound session failed: {err}");
1325 return;
1326 }
1327 };
1328 let Some((key, table)) = driver.live_peers_or_create(&root).await else {
1329 return;
1330 };
1331 let acknowledged = session.acknowledged();
1332 let Some(updates) = table.insert(peer, session, false).await else {
1336 return;
1337 };
1338 driver.watch_delivery(key.clone(), peer, acknowledged);
1339 table.catch_up(peer, &replica).await;
1340 crate::instrument::sync_session(true);
1341 driver.reindex(&key).await;
1344 driver.request_embed(&key);
1345 driver.spawn_reader(table, updates, key, peer);
1346 });
1347 }
1348 }
1349
1350 pub async fn connect(self: &Arc<Self>) -> Result<()> {
1357 let Some(connector) = &self.connector else {
1358 return Ok(());
1359 };
1360 if self.is_connected() {
1361 return Ok(());
1362 }
1363 match connector.connect().await? {
1364 Some(client) => self.attach(client).await,
1365 None => tracing::warn!(
1366 "no sapphire-bridge is running; serving without sync until it starts"
1367 ),
1368 }
1369 Ok(())
1370 }
1371
1372 pub async fn keep_connected(self: Arc<Self>) {
1381 let mut attempt = 0;
1382 let mut warned = false;
1385 loop {
1386 if let Ok(bridge) = self.bridge() {
1387 bridge.closed().await;
1388 tracing::warn!("lost the connection to sapphire-bridge; reconnecting");
1389 self.bridge.send_if_modified(|slot| {
1390 let same = slot.as_ref().is_some_and(|b| Arc::ptr_eq(b, &bridge));
1391 if same {
1392 *slot = None;
1393 }
1394 same
1395 });
1396 attempt = 0;
1397 warned = false;
1398 continue;
1399 }
1400 let Some(connector) = &self.connector else {
1401 return;
1402 };
1403 match connector.connect().await {
1404 Ok(Some(client)) => {
1405 tracing::info!("connected to sapphire-bridge");
1406 self.attach(client).await;
1407 attempt = 0;
1408 warned = false;
1409 continue;
1410 }
1411 Ok(None) => tracing::debug!("no sapphire-bridge is running yet"),
1412 Err(err) if !warned => {
1414 tracing::warn!("could not connect to sapphire-bridge: {err}");
1415 warned = true;
1416 }
1417 Err(err) => tracing::debug!("could not connect to sapphire-bridge: {err}"),
1418 }
1419 tokio::select! {
1420 () = tokio::time::sleep(reconnect_backoff(self.reconnect_first, attempt)) => {
1421 attempt = attempt.saturating_add(1);
1422 }
1423 () = self.reconnect_now.notified() => attempt = 0,
1424 }
1425 }
1426 }
1427
1428 async fn attach(self: &Arc<Self>, client: BridgeClient) {
1431 let client = Arc::new(client);
1434 *self.subscribed.lock().expect("subscribed") =
1435 Some((Arc::clone(&client), client.incoming()));
1436 let waiting = {
1437 let mut pending = self.pending.lock().await;
1438 self.bridge.send_replace(Some(client));
1439 std::mem::take(&mut *pending)
1440 };
1441 if !self.synced.lock().await.is_empty()
1443 && let Err(err) = self.reregister().await
1444 {
1445 tracing::warn!("could not register with sapphire-bridge: {err}");
1446 }
1447 for root in waiting.into_keys() {
1449 if let Err(err) = self.enable(&root).await {
1450 tracing::warn!(root = %root.display(), "could not start sync: {err}");
1451 }
1452 }
1453 }
1454
1455 pub async fn roots(&self) -> Vec<PathBuf> {
1457 self.synced.lock().await.keys().cloned().collect()
1458 }
1459
1460 async fn live_peers(&self, root: &Path) -> Option<Arc<LivePeers>> {
1465 let key = root.canonicalize().ok()?;
1466 self.live.lock().await.get(&key).cloned()
1467 }
1468
1469 async fn live_peers_or_create(&self, root: &Path) -> Option<(PathBuf, Arc<LivePeers>)> {
1474 let key = root.canonicalize().ok()?;
1475 if !self.synced.lock().await.contains_key(&key) {
1476 return None;
1477 }
1478 let me = self.device_id().await.ok()?;
1481 let mut live = self.live.lock().await;
1482 let peers = Arc::clone(
1483 live.entry(key.clone())
1484 .or_insert_with(|| Arc::new(LivePeers::new(me))),
1485 );
1486 Some((key, peers))
1487 }
1488
1489 pub async fn after_commit(&self, root: &Path, updates: Vec<PathUpdate>) {
1499 if updates.is_empty() {
1500 return;
1501 }
1502 if let Some(peers) = self.live_peers(root).await {
1503 peers.fan_out(&updates, None).await;
1504 }
1505 }
1506
1507 pub async fn live_session_devices(&self, root: &Path) -> Vec<GrainId> {
1511 match self.live_peers(root).await {
1512 Some(peers) => peers.devices().await,
1513 None => Vec::new(),
1514 }
1515 }
1516
1517 pub async fn stop_dials(&self) {
1524 let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1525 dials.abort_all();
1526 while dials.join_next().await.is_some() {}
1527 }
1528
1529 pub async fn stop_background(&self) {
1537 let mut background =
1538 std::mem::take(&mut *self.background.lock().expect("background tasks"));
1539 background.abort_all();
1540 while background.join_next().await.is_some() {}
1541 }
1542
1543 #[cfg(test)]
1546 async fn settle_dials(&self) {
1547 let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1548 while dials.join_next().await.is_some() {}
1549 }
1550
1551 pub async fn drop_connections(&self) {
1556 let tables: Vec<Arc<LivePeers>> = self.live.lock().await.values().cloned().collect();
1557 for peers in tables {
1558 peers.drop_connections().await;
1559 }
1560 }
1561
1562 pub async fn dial_loop(self: Arc<Self>) -> Result<()> {
1577 let mut tick = tokio::time::interval(live::DIAL_INTERVAL);
1578 tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1581 let mut waiting: HashMap<GrainId, std::time::Instant> = HashMap::new();
1582 let mut failures: HashMap<GrainId, u32> = HashMap::new();
1583 let mut primary_of: std::collections::HashSet<GrainId> = Default::default();
1585 let mut last_backfill = std::time::Instant::now();
1587 loop {
1588 tick.tick().await;
1589 let roots = self.roots().await;
1590 if roots.is_empty() {
1591 continue;
1592 }
1593 let peers = match self.peers().await {
1594 Ok(peers) => peers,
1595 Err(err) => {
1598 tracing::debug!("no peers to dial: {err}");
1599 continue;
1600 }
1601 };
1602 let me = match self.device_id().await {
1603 Ok(me) => me,
1604 Err(err) => {
1605 tracing::warn!("no device id, so nobody to dial: {err}");
1606 continue;
1607 }
1608 };
1609 let ids: Vec<(PathBuf, GrainId)> = self
1613 .synced
1614 .lock()
1615 .await
1616 .iter()
1617 .map(|(root, entry)| (root.clone(), entry.workspace_id))
1618 .collect();
1619 let backfill_due = last_backfill.elapsed() >= self.backfill_timing().interval;
1622 if backfill_due {
1623 last_backfill = std::time::Instant::now();
1624 }
1625 for (root, workspace_id) in ids {
1626 let primary = peers
1627 .roles_for(workspace_id)
1628 .is_some_and(|r| r.primary == Some(me));
1629 if !primary {
1630 primary_of.remove(&workspace_id);
1631 } else if primary_of.insert(workspace_id) || backfill_due {
1632 self.request_embed(&root);
1633 }
1634 }
1635 let now = std::time::Instant::now();
1636 for root in roots {
1645 let Some((key, table)) = self.live_peers_or_create(&root).await else {
1646 continue;
1647 };
1648 let workspace_id = {
1649 let synced = self.synced.lock().await;
1650 match synced.get(&key) {
1651 Some(entry) => entry.workspace_id,
1652 None => continue,
1653 }
1654 };
1655 let roles = peers.roles_for(workspace_id);
1656 if topology::may_close_skipped(me, roles, &table.devices().await) {
1659 table
1660 .retain(|d| topology::link(me, *d, roles) != topology::Link::Skip)
1661 .await;
1662 }
1663 for peer in peers.peers.iter().filter(|p| {
1664 p.connected && topology::link(me, p.device_id, roles) == topology::Link::Dial
1665 }) {
1666 let device = peer.device_id;
1667 if table.has(&device).await {
1668 failures.remove(&device);
1670 waiting.remove(&device);
1671 continue;
1672 }
1673 if let Some(until) = waiting.get(&device)
1674 && now < *until
1675 {
1676 continue;
1678 }
1679 if !table.begin_dial(device).await {
1683 continue;
1684 }
1685 let opened = self.open_stream(workspace_id, device).await;
1686 if opened.is_err() {
1687 table.end_dial(device).await;
1688 }
1689 match opened {
1690 Ok(stream) => {
1691 let adopted = self
1692 .adopt_live_session(stream, &root, device, workspace_id)
1693 .await;
1694 table.end_dial(device).await;
1697 match adopted {
1698 Ok(()) => {
1699 failures.remove(&device);
1700 waiting.remove(&device);
1701 }
1702 Err(err) => {
1703 tracing::debug!(
1704 peer = %peer.name,
1705 err = %err,
1706 "a live session could not be started"
1707 );
1708 let count = failures.entry(device).or_insert(0);
1709 *count = count.saturating_add(1);
1710 waiting.insert(device, now + dial_backoff(*count));
1711 }
1712 }
1713 }
1714 Err(err) => {
1715 tracing::debug!("no stream to a peer: {err}");
1718 let count = failures.entry(device).or_insert(0);
1719 *count = count.saturating_add(1);
1720 waiting.insert(device, now + dial_backoff(*count));
1721 }
1722 }
1723 }
1724 }
1725 }
1726 }
1727
1728 async fn adopt_live_session(
1733 self: &Arc<Self>,
1734 stream: sapphire_ipc::RawStream,
1735 root: &Path,
1736 device: GrainId,
1737 workspace_id: GrainId,
1738 ) -> Result<()> {
1739 let replica = self.replica_of(root).await?;
1740 let (_, session) = sapphire_framework_session::open_live_session(
1741 stream,
1742 Arc::clone(&replica),
1743 workspace_id,
1744 )
1745 .await
1746 .map_err(|e| {
1747 crate::instrument::sync_session(false);
1748 Error::Sync(e.to_string())
1749 })?;
1750 let Some((key, table)) = self.live_peers_or_create(root).await else {
1751 return Ok(());
1752 };
1753 let acknowledged = session.acknowledged();
1754 let Some(updates) = table.insert(device, session, true).await else {
1759 return Ok(());
1760 };
1761 self.watch_delivery(key.clone(), device, acknowledged);
1762 table.catch_up(device, &replica).await;
1763 crate::instrument::sync_session(true);
1764 self.reindex(&key).await;
1767 self.request_embed(&key);
1768 self.spawn_reader(Arc::clone(&table), updates, key, device);
1769 Ok(())
1770 }
1771
1772 fn spawn_reader(
1779 self: &Arc<Self>,
1780 table: Arc<LivePeers>,
1781 mut updates: broadcast::Receiver<Vec<PathUpdate>>,
1782 key: PathBuf,
1783 from: GrainId,
1784 ) {
1785 let driver = Arc::clone(self);
1786 self.spawn_background(async move {
1787 loop {
1788 match updates.recv().await {
1789 Ok(batch) => {
1790 driver.reindex(&key).await;
1793 driver.request_embed(&key);
1794 table.fan_out(&batch, Some(from)).await;
1795 }
1796 Err(broadcast::error::RecvError::Lagged(missed)) => {
1801 tracing::debug!(
1802 device = %from,
1803 missed,
1804 "fell behind on a live session; the next exchange catches up"
1805 );
1806 }
1807 Err(broadcast::error::RecvError::Closed) => {
1810 return;
1811 }
1812 }
1813 }
1814 });
1815 }
1816
1817 fn watch_delivery(
1823 self: &Arc<Self>,
1824 key: PathBuf,
1825 device: GrainId,
1826 mut acknowledged: tokio::sync::watch::Receiver<sapphire_sync::VersionVector>,
1827 ) {
1828 let driver = Arc::clone(self);
1829 self.spawn_background(async move {
1830 loop {
1831 let known = acknowledged.borrow_and_update().clone();
1832 driver.check_delivered(&key, device, &known).await;
1833 if acknowledged.changed().await.is_err() {
1834 return;
1835 }
1836 }
1837 });
1838 }
1839
1840 async fn check_delivered(
1843 &self,
1844 key: &Path,
1845 device: GrainId,
1846 known: &sapphire_sync::VersionVector,
1847 ) {
1848 let Some(replica) = self
1849 .synced
1850 .lock()
1851 .await
1852 .get(key)
1853 .map(|entry| Arc::clone(&entry.replica))
1854 else {
1855 return;
1856 };
1857 let (me, own) = {
1858 let replica = replica.lock().await;
1859 (replica.replica_id(), own_counter(&replica))
1860 };
1861 if known.get(&me) < own {
1862 return;
1863 }
1864 {
1865 let mut synced = self.synced.lock().await;
1866 let Some(entry) = synced.get_mut(key) else {
1867 return;
1868 };
1869 if own <= entry.delivered_through {
1872 return;
1873 }
1874 entry.delivered_through = own;
1875 }
1876 let device_name = self
1877 .last_peers
1878 .lock()
1879 .await
1880 .as_ref()
1881 .and_then(|answer| answer.peers.iter().find(|p| p.device_id == device))
1882 .map_or_else(|| device.to_string(), |peer| peer.name.clone());
1883 tracing::info!(root = %key.display(), peer = %device_name, "local edits delivered");
1884 if let Some(host) = self.host.get() {
1885 host.emit(
1886 key,
1887 BackendEvent::SyncDelivered {
1888 device_id: device,
1889 device_name,
1890 },
1891 );
1892 }
1893 }
1894
1895 fn reclaim(&self, key: &Path) -> Reclaimed {
1898 let stopped = self.stopped.lock().expect("stopped replicas").remove(key);
1899 match stopped {
1900 None => Reclaimed::Closed,
1901 Some(replica) => replica
1902 .upgrade()
1903 .map_or(Reclaimed::Closing, Reclaimed::Open),
1904 }
1905 }
1906
1907 async fn replica_of(&self, root: &Path) -> Result<Arc<Mutex<Replica>>> {
1909 let key = root.canonicalize().map_err(Error::Io)?;
1910 let synced = self.synced.lock().await;
1911 match synced.get(&key) {
1912 Some(entry) => Ok(Arc::clone(&entry.replica)),
1913 None => Err(Error::UnknownWorkspace(key, self.ctx.app_name)),
1914 }
1915 }
1916
1917 pub async fn watch_changes(self: Arc<Self>) -> Result<()> {
1929 let (tx, mut rx) = mpsc::channel(64);
1930 let watcher = Arc::new(Watcher::start(self.roots().await, tx)?);
1931 let _ = self.watcher.set(Arc::clone(&watcher));
1932
1933 for root in self.roots().await {
1937 if let Err(err) = watcher.watch(&root) {
1938 tracing::warn!(root = %root.display(), "could not watch: {err}");
1939 }
1940 }
1941
1942 while let Some(root) = rx.recv().await {
1943 self.refresh_settings();
1945 if let Err(err) = self.scan(&root).await {
1948 tracing::warn!(root = %root.display(), "scan after a local edit failed: {err}");
1949 }
1950 self.request_embed(&root);
1951 if let Err(err) = self.sync_now(&root).await {
1952 tracing::warn!(root = %root.display(), "dial after a local edit failed: {err}");
1953 }
1954 }
1955 Ok(())
1956 }
1957
1958 async fn reregister(&self) -> Result<()> {
1960 let workspaces: Vec<WorkspaceRegistration> = {
1961 let synced = self.synced.lock().await;
1962 synced
1963 .iter()
1964 .map(|(root, entry)| WorkspaceRegistration {
1965 workspace_id: entry.workspace_id,
1966 root: root.clone(),
1967 })
1968 .collect()
1969 };
1970 self.bridge()?
1971 .register(RegisterParams {
1972 app_name: self.ctx.app_name.to_owned(),
1973 exe_path: self.exe_path.clone(),
1974 managed_by: wire_managed_by(self.managed_by),
1975 workspaces,
1976 })
1977 .await
1978 .map_err(|e| Error::Bridge(e.to_string()))?;
1979 Ok(())
1980 }
1981
1982 async fn should_skip(&self, workspace_id: GrainId, peer: GrainId) -> bool {
1987 let Some(&me) = self.device_id.get() else {
1988 return false;
1989 };
1990 let cached = self.last_peers.lock().await;
1991 let roles = cached.as_ref().and_then(|p| p.roles_for(workspace_id));
1992 topology::link(me, peer, roles) == topology::Link::Skip
1993 }
1994
1995 pub async fn is_primary(&self, workspace_id: GrainId) -> bool {
2001 let Some(&me) = self.device_id.get() else {
2002 return false;
2003 };
2004 let cached = self.last_peers.lock().await;
2005 cached
2006 .as_ref()
2007 .and_then(|p| p.roles_for(workspace_id))
2008 .is_some_and(|r| r.primary == Some(me))
2009 }
2010
2011 pub fn this_device(&self) -> Option<GrainId> {
2018 self.device_id.get().copied()
2019 }
2020
2021 pub async fn peers(&self) -> Result<PeersResult> {
2027 let peers = self
2028 .bridge()?
2029 .peers()
2030 .await
2031 .map_err(|e| Error::Bridge(e.to_string()))?;
2032 *self.last_peers.lock().await = Some(peers.clone());
2033 Ok(peers)
2034 }
2035
2036 async fn device_id(&self) -> Result<GrainId> {
2038 if let Some(id) = self.device_id.get() {
2039 return Ok(*id);
2040 }
2041 let result = self
2042 .bridge()?
2043 .register(RegisterParams {
2044 app_name: self.ctx.app_name.to_owned(),
2045 exe_path: self.exe_path.clone(),
2046 managed_by: wire_managed_by(self.managed_by),
2047 workspaces: Vec::new(),
2048 })
2049 .await
2050 .map_err(|e| Error::Bridge(e.to_string()))?;
2051 let _ = self.device_id.set(result.device_id);
2052 Ok(result.device_id)
2053 }
2054
2055 fn state_dir(&self, root: &Path) -> Result<PathBuf> {
2057 let workspace = Workspace::from_root(self.ctx, root)?;
2058 Ok(workspace.cache_dir().join("sync"))
2059 }
2060}
2061
2062type Dial<'a> = std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>>;
2064
2065fn own_counter(replica: &Replica) -> u64 {
2068 replica.vv().get(&replica.replica_id())
2069}
2070
2071async fn join_all(mut dials: Vec<Dial<'_>>) {
2072 std::future::poll_fn(|cx| {
2073 dials.retain_mut(|dial| dial.as_mut().poll(cx).is_pending());
2074 if dials.is_empty() {
2075 std::task::Poll::Ready(())
2076 } else {
2077 std::task::Poll::Pending
2078 }
2079 })
2080 .await
2081}
2082
2083fn wire_managed_by(managed_by: ManagedBy) -> sapphire_bridge_api::ManagedBy {
2085 match managed_by {
2086 ManagedBy::Service => sapphire_bridge_api::ManagedBy::Service,
2087 ManagedBy::Spawned => sapphire_bridge_api::ManagedBy::Spawned,
2088 }
2089}
2090
2091#[cfg(test)]
2092mod tests {
2093 use super::*;
2094 use crate::sync::testing::StubBridge;
2095 use crate::test_support;
2096 use sapphire_workspace::AppKind;
2097 use std::ffi::OsString;
2098
2099 static CTX: AppContext = AppContext::new("sapphire-synctest");
2100
2101 const DIR_VARS: [&str; 3] = [
2103 "SAPPHIRE_SYNCTEST_CACHE_DIR",
2104 "SAPPHIRE_SYNCTEST_DATA_DIR",
2105 "SAPPHIRE_SYNCTEST_CONFIG_DIR",
2106 ];
2107
2108 struct EnvGuard {
2111 previous: [Option<OsString>; 3],
2112 _lock: std::sync::MutexGuard<'static, ()>,
2113 }
2114
2115 impl Drop for EnvGuard {
2116 fn drop(&mut self) {
2117 for (name, previous) in DIR_VARS.iter().zip(self.previous.iter_mut()) {
2120 match previous.take() {
2121 Some(value) => unsafe { std::env::set_var(name, value) },
2122 None => test_support::remove(name),
2123 }
2124 }
2125 }
2126 }
2127
2128 struct Fixture {
2136 _tmp: tempfile::TempDir,
2137 _env: EnvGuard,
2138 root: PathBuf,
2139 stub: StubBridge,
2140 runtime: Arc<SyncRuntime>,
2141 }
2142
2143 impl Fixture {
2144 fn second_root(&self) -> PathBuf {
2146 let root = self._tmp.path().join("ws2");
2147 std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
2148 root.canonicalize().unwrap()
2149 }
2150 }
2151
2152 const FIXTURE_OPEN_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
2154
2155 fn dialled_peers(me: GrainId, n: usize) -> Vec<sapphire_bridge_api::PeerInfo> {
2157 std::iter::repeat_with(GrainId::random)
2158 .filter(|id| *id > me)
2159 .take(n)
2160 .enumerate()
2161 .map(|(i, device_id)| sapphire_bridge_api::PeerInfo {
2162 device_id,
2163 name: format!("offline-{i}"),
2164 node_id: format!("node-{i}"),
2165 connected: false,
2166 priority: 1,
2167 availability: None,
2168 })
2169 .collect()
2170 }
2171
2172 #[tokio::test(flavor = "multi_thread")]
2173 async fn enabling_answers_at_once_with_a_peer_that_never_answers() {
2174 let f = fixture().await;
2176 let _hung = f.stub.hang_streams().await;
2177 f.stub.set_peers(dialled_peers(f.stub.device_id, 1));
2178
2179 let started = std::time::Instant::now();
2180 f.runtime.enable(&f.root).await.unwrap();
2181 assert!(
2182 started.elapsed() < FIXTURE_OPEN_TIMEOUT,
2183 "enable waited {:?} on the dial",
2184 started.elapsed()
2185 );
2186 }
2187
2188 #[tokio::test(flavor = "multi_thread")]
2189 async fn peers_that_never_answer_are_dialled_together_and_given_up_on() {
2190 let f = fixture().await;
2191 let _hung = f.stub.hang_streams().await;
2192 f.runtime.enable(&f.root).await.unwrap();
2193 f.runtime.settle_dials().await;
2196 f.stub.set_peers(dialled_peers(f.stub.device_id, 3));
2197
2198 let started = std::time::Instant::now();
2199 f.runtime.sync_now(&f.root).await.unwrap();
2200 let took = started.elapsed();
2201 assert!(took >= FIXTURE_OPEN_TIMEOUT, "gave up early: {took:?}");
2202 assert!(
2203 took < FIXTURE_OPEN_TIMEOUT * 2,
2204 "three dials one after another: {took:?}"
2205 );
2206 }
2207
2208 async fn fixture() -> Fixture {
2209 let (tmp, env, root) = scratch();
2210 let (stub, client) = StubBridge::start_in(tmp.path().join("run")).await;
2211 let runtime = Arc::new(
2212 SyncRuntime::new(&CTX, client, "/bin/true".into(), ManagedBy::Service)
2213 .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
2214 );
2215 Fixture {
2216 _tmp: tmp,
2217 _env: env,
2218 root,
2219 stub,
2220 runtime,
2221 }
2222 }
2223
2224 type Bridges = Arc<std::sync::Mutex<std::collections::VecDeque<BridgeClient>>>;
2227
2228 async fn connecting() -> (Fixture, Bridges, BridgeClient) {
2231 let (tmp, env, root) = scratch();
2232 let (stub, client) = StubBridge::start_client_in(tmp.path().join("run")).await;
2233 let bridges: Bridges = Arc::default();
2234 let connector = BridgeConnector::new({
2235 let bridges = Arc::clone(&bridges);
2236 move || {
2237 let next = bridges.lock().unwrap().pop_front();
2238 async move { Ok(next) }
2239 }
2240 });
2241 let runtime = Arc::new(
2242 SyncRuntime::connecting(&CTX, connector, "/bin/true".into(), ManagedBy::Service)
2243 .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
2244 );
2245 let f = Fixture {
2246 _tmp: tmp,
2247 _env: env,
2248 root,
2249 stub,
2250 runtime,
2251 };
2252 (f, bridges, client)
2253 }
2254
2255 async fn eventually(what: &str, mut done: impl AsyncFnMut() -> bool) {
2257 let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
2258 while !done().await {
2259 assert!(tokio::time::Instant::now() < deadline, "never: {what}");
2260 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
2261 }
2262 }
2263
2264 #[tokio::test(flavor = "multi_thread")]
2265 async fn a_runtime_started_without_a_bridge_syncs_once_one_starts() {
2266 let (f, bridges, client) = connecting().await;
2268 f.runtime.connect().await.unwrap();
2269 assert!(!f.runtime.is_connected());
2270
2271 let err = f.runtime.enable(&f.root).await.unwrap_err();
2272 assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2273 let status = f.runtime.status(&f.root).await;
2274 assert!(status.enabled, "enabled, waiting for the bridge");
2275 assert!(!status.bridge_available);
2276 let id = status.workspace_id.expect("its id");
2277
2278 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2279 bridges.lock().unwrap().push_back(client);
2280 eventually("the waiting workspace is registered", async || {
2281 f.stub.last_workspaces() == vec![id]
2282 })
2283 .await;
2284 assert_eq!(f.runtime.roots().await, vec![f.root.clone()]);
2285 assert!(f.runtime.status(&f.root).await.bridge_available);
2286 keeping.abort();
2287 }
2288
2289 #[tokio::test(flavor = "multi_thread")]
2290 async fn a_runtime_that_loses_its_bridge_registers_with_the_next_one() {
2291 let (f, bridges, client) = connecting().await;
2292 bridges.lock().unwrap().push_back(client);
2293 f.runtime.connect().await.unwrap();
2294 assert!(f.runtime.is_connected());
2295 let id = f.runtime.enable(&f.root).await.unwrap();
2296 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2297
2298 f.stub.disconnect();
2299 eventually("the runtime notices", async || !f.runtime.is_connected()).await;
2300 let err = f.runtime.workspaces().await.unwrap_err();
2301 assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2302
2303 let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2304 bridges.lock().unwrap().push_back(client);
2305 eventually("the synced workspace is registered again", async || {
2306 next.last_workspaces() == vec![id]
2307 })
2308 .await;
2309 assert!(f.runtime.is_connected());
2310 keeping.abort();
2311 }
2312
2313 #[tokio::test(flavor = "multi_thread")]
2314 async fn this_device_is_known_once_a_workspace_is_enabled_and_never_asked_for() {
2315 let f = fixture().await;
2316 assert_eq!(f.runtime.this_device(), None);
2317 assert!(
2318 f.stub.seen.lock().unwrap().registrations.is_empty(),
2319 "asking must not register"
2320 );
2321
2322 f.runtime.enable(&f.root).await.unwrap();
2323 assert_eq!(f.runtime.this_device(), Some(f.stub.device_id));
2324 }
2325
2326 #[tokio::test(flavor = "multi_thread")]
2327 async fn peers_answers_the_workgroup_from_the_bridge() {
2328 let f = fixture().await;
2329 let listed = dialled_peers(f.stub.device_id, 2);
2330 f.stub.set_peers(listed.clone());
2331
2332 let peers = f.runtime.peers().await.unwrap();
2333 let ids: Vec<_> = peers.peers.iter().map(|p| p.device_id).collect();
2334 assert_eq!(ids, listed.iter().map(|p| p.device_id).collect::<Vec<_>>());
2335 }
2336
2337 #[tokio::test(flavor = "multi_thread")]
2338 async fn peers_says_so_while_the_bridge_is_away() {
2339 let (f, _bridges, _client) = connecting().await;
2340 let err = f.runtime.peers().await.unwrap_err();
2341 assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2342 }
2343
2344 #[tokio::test(flavor = "multi_thread")]
2345 async fn disabling_while_the_bridge_is_away_forgets_the_waiting_workspace() {
2346 let (f, bridges, client) = connecting().await;
2347 f.runtime.enable(&f.root).await.unwrap_err();
2348 f.runtime.disable(&f.root).await.unwrap();
2349 assert!(!f.runtime.status(&f.root).await.enabled);
2350
2351 bridges.lock().unwrap().push_back(client);
2352 f.runtime.connect().await.unwrap();
2353 assert!(f.runtime.roots().await.is_empty());
2354 assert!(f.stub.last_workspaces().is_empty());
2355 }
2356
2357 #[tokio::test(flavor = "multi_thread")]
2358 async fn announcements_are_answered_from_a_reconnected_bridge() {
2359 let (f, bridges, client) = connecting().await;
2360 bridges.lock().unwrap().push_back(client);
2361 f.runtime.connect().await.unwrap();
2362 let id = f.runtime.enable(&f.root).await.unwrap();
2363 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2364 let answering = tokio::spawn(Arc::clone(&f.runtime).run());
2365
2366 f.stub.disconnect();
2367 let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2368 bridges.lock().unwrap().push_back(client);
2369 eventually("reconnected", async || next.last_workspaces() == vec![id]).await;
2370
2371 next.announce(id, "ticket").await;
2374 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2375 assert!(!answering.is_finished(), "the announcement loop ended");
2376 answering.abort();
2377 keeping.abort();
2378 }
2379
2380 fn scratch() -> (tempfile::TempDir, EnvGuard, PathBuf) {
2382 let lock = test_support::lock();
2383 let tmp = tempfile::tempdir().unwrap();
2384 let previous = DIR_VARS.map(std::env::var_os);
2385 for (name, dir) in DIR_VARS
2389 .iter()
2390 .zip(["cache", "data", "config"].map(|cat| tmp.path().join(cat)))
2391 {
2392 test_support::set(name, &dir);
2393 }
2394 let env = EnvGuard {
2395 previous,
2396 _lock: lock,
2397 };
2398 CTX.init(AppKind::Server);
2399
2400 let root = tmp.path().join("ws");
2401 std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
2402 let root = root.canonicalize().unwrap();
2403 (tmp, env, root)
2404 }
2405
2406 struct NoteApp;
2408
2409 impl crate::settings::AppSettings for NoteApp {
2410 fn defs(&self) -> Vec<sapphire_framework_settings::SettingDef> {
2411 vec![sapphire_framework_settings::SettingDef {
2412 key: "sapphire-synctest.note".into(),
2413 kind: sapphire_framework_settings::Kind::String { non_empty: false },
2414 scopes: vec![sapphire_framework_settings::Scope::Workspace],
2415 overridable: false,
2416 apply: sapphire_framework_settings::Apply::Live,
2417 default: None,
2418 section: "Test".into(),
2419 doc: "test".into(),
2420 process: sapphire_framework_settings::Process::App,
2421 }]
2422 }
2423 }
2424
2425 #[tokio::test(flavor = "multi_thread")]
2426 async fn a_workspace_config_changed_under_the_watcher_reaches_the_settings() {
2427 let f = fixture().await;
2428 let ctx: &'static AppContext = Box::leak(Box::new(AppContext::new("sapphire-synctest")));
2431 ctx.set_config_dir(f._tmp.path().join("settings"));
2432 let selection = crate::selection::SelectionFile::at(
2433 f._tmp.path().join("settings").join("workspace.toml"),
2434 );
2435 selection
2436 .save(&crate::selection::Selection {
2437 root: f.root.clone(),
2438 sync: true,
2439 })
2440 .unwrap();
2441 let settings =
2442 crate::settings::Settings::load_with(ctx, Arc::new(NoteApp), selection).unwrap();
2443 let changed = settings.changed();
2444 f.runtime.set_settings(Some(settings.clone()));
2445 f.runtime.enable(&f.root).await.unwrap();
2446 let watching = tokio::spawn(Arc::clone(&f.runtime).watch_changes());
2447 let config = f.root.join(".sapphire-synctest").join("config.toml");
2449 eventually("the settings hear of the new value", async || {
2450 std::fs::write(&config, "[sapphire-synctest]\nnote = \"synced\"\n").unwrap();
2451 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
2452 changed.has_changed().unwrap()
2453 })
2454 .await;
2455 assert_eq!(
2456 settings
2457 .get_as::<String>("sapphire-synctest.note")
2458 .as_deref(),
2459 Some("synced")
2460 );
2461 watching.abort();
2462 }
2463
2464 #[tokio::test(flavor = "multi_thread")]
2465 async fn enabling_registers_the_workspace_with_the_bridge() {
2466 let f = fixture().await;
2467
2468 let id = f.runtime.enable(&f.root).await.unwrap();
2469 assert_eq!(f.stub.last_workspaces(), vec![id]);
2470 }
2471
2472 #[tokio::test(flavor = "multi_thread")]
2473 async fn enabling_twice_is_idempotent_and_keeps_the_same_id() {
2474 let f = fixture().await;
2475
2476 let first = f.runtime.enable(&f.root).await.unwrap();
2477 let second = f.runtime.enable(&f.root).await.unwrap();
2478 assert_eq!(first, second);
2479 assert_eq!(
2480 f.stub.last_workspaces(),
2481 vec![first],
2482 "still exactly one workspace"
2483 );
2484 }
2485
2486 #[tokio::test(flavor = "multi_thread")]
2487 async fn a_registration_carries_the_complete_current_list() {
2488 let f = fixture().await;
2489 let second_root = f.second_root();
2490
2491 let a = f.runtime.enable(&f.root).await.unwrap();
2492 let b = f.runtime.enable(&second_root).await.unwrap();
2493
2494 let mut seen = f.stub.last_workspaces();
2495 seen.sort();
2496 let mut want = vec![a, b];
2497 want.sort();
2498 assert_eq!(seen, want, "registration is the whole set, not a delta");
2499 }
2500
2501 #[tokio::test(flavor = "multi_thread")]
2502 async fn no_later_registration_clears_this_apps_routes() {
2503 let f = fixture().await;
2509 let second_root = f.second_root();
2510
2511 f.runtime.enable(&f.root).await.unwrap();
2512 f.runtime.enable(&second_root).await.unwrap();
2513 f.runtime.sync_now(&f.root).await.unwrap();
2514 f.runtime.disable(&f.root).await.unwrap();
2515
2516 let seen = f.stub.seen.lock().expect("stub");
2517 let first_full = seen
2518 .registrations
2519 .iter()
2520 .position(|r| !r.workspaces.is_empty())
2521 .expect("enabling registers the workspace");
2522 let emptied_afterwards = seen.registrations[first_full + 1..]
2523 .iter()
2524 .filter(|r| r.workspaces.is_empty())
2525 .count();
2526 assert_eq!(
2527 emptied_afterwards, 0,
2528 "a registration after the first workspace must still name the whole set"
2529 );
2530 }
2531
2532 #[tokio::test(flavor = "multi_thread")]
2533 async fn disabling_unregisters_and_leaves_the_files_alone() {
2534 let f = fixture().await;
2535 std::fs::write(f.root.join("keep.md"), "content").unwrap();
2536
2537 let id = f.runtime.enable(&f.root).await.unwrap();
2538 f.runtime.disable(&f.root).await.unwrap();
2539
2540 assert!(f.stub.last_workspaces().is_empty());
2541 assert!(
2542 f.root.join("keep.md").exists(),
2543 "disabling sync must not touch files"
2544 );
2545 assert!(
2546 crate::sync::id::sync_id_path("sapphire-synctest", &f.root).exists(),
2547 "the sync id stays, so re-enabling rejoins the same workspace"
2548 );
2549 let _ = id;
2550 }
2551
2552 #[tokio::test(flavor = "multi_thread")]
2553 async fn re_enabling_after_disabling_reuses_the_identity() {
2554 let f = fixture().await;
2555
2556 let first = f.runtime.enable(&f.root).await.unwrap();
2557 f.runtime.disable(&f.root).await.unwrap();
2558 let again = f.runtime.enable(&f.root).await.unwrap();
2559
2560 assert_eq!(first, again);
2561 }
2562
2563 #[tokio::test(flavor = "multi_thread")]
2564 async fn status_reports_disabled_for_a_workspace_that_was_never_enabled() {
2565 let f = fixture().await;
2566
2567 let status = f.runtime.status(&f.root).await;
2568 assert!(!status.enabled);
2569 assert!(status.workspace_id.is_none());
2570 }
2571
2572 #[tokio::test(flavor = "multi_thread")]
2573 async fn disabling_something_that_was_never_enabled_is_not_an_error() {
2574 let f = fixture().await;
2575 f.runtime
2576 .disable(&f.root)
2577 .await
2578 .expect("disabling twice is fine");
2579 }
2580
2581 #[tokio::test(flavor = "multi_thread")]
2582 async fn a_file_already_on_disk_is_recorded_on_enable() {
2583 let f = fixture().await;
2586 std::fs::write(f.root.join("a.md"), "content").unwrap();
2587 f.runtime.enable(&f.root).await.unwrap();
2588 f.runtime.settle_dials().await;
2589
2590 let replica = f.runtime.replica_of(&f.root).await.unwrap();
2591 let states = replica.lock().await.states().unwrap();
2592 assert!(
2593 states.iter().any(|(path, _)| path == "a.md"),
2594 "a file on disk before enable must be recorded: {states:?}"
2595 );
2596 }
2597
2598 #[tokio::test(flavor = "multi_thread")]
2599 async fn re_enabling_while_the_replica_is_still_held_takes_it_back() {
2600 let f = fixture().await;
2603 f.runtime.enable(&f.root).await.unwrap();
2604 f.runtime.settle_dials().await;
2605 let held = f.runtime.replica_of(&f.root).await.unwrap();
2606
2607 f.runtime.disable(&f.root).await.unwrap();
2608 f.runtime
2609 .enable(&f.root)
2610 .await
2611 .expect("a re-enable must not open the store a second time");
2612
2613 let again = f.runtime.replica_of(&f.root).await.unwrap();
2614 assert!(Arc::ptr_eq(&held, &again), "the open replica is taken back");
2615 }
2616
2617 #[tokio::test(flavor = "multi_thread")]
2618 async fn re_enabling_while_the_replica_is_still_closing_waits_for_it() {
2619 let f = fixture().await;
2622 f.runtime.enable(&f.root).await.unwrap();
2623 f.runtime.settle_dials().await;
2624 let config = {
2625 let replica = f.runtime.replica_of(&f.root).await.unwrap();
2626 replica.lock().await.config().clone()
2627 };
2628 f.runtime.disable(&f.root).await.unwrap();
2629
2630 let closing = Replica::open(config, Arc::new(SystemClock)).unwrap();
2632 let key = f.root.canonicalize().unwrap();
2633 f.runtime
2634 .stopped
2635 .lock()
2636 .unwrap()
2637 .insert(key, std::sync::Weak::new());
2638 let closed = tokio::spawn(async move {
2639 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2640 drop(closing);
2641 });
2642
2643 f.runtime
2644 .enable(&f.root)
2645 .await
2646 .expect("a re-enable must wait for a store that is closing");
2647 closed.await.unwrap();
2648 }
2649
2650 #[tokio::test(flavor = "multi_thread")]
2651 async fn a_workspace_whose_root_vanished_is_reported_as_paused_not_deleted() {
2652 let f = fixture().await;
2653 std::fs::write(f.root.join("a.md"), "content").unwrap();
2654 f.runtime.enable(&f.root).await.unwrap();
2655 f.runtime.scan(&f.root).await.unwrap();
2656
2657 std::fs::remove_dir_all(f.root.join(".sapphire-synctest")).unwrap();
2659 let _ = f.runtime.scan(&f.root).await;
2660
2661 let status = f.runtime.status(&f.root).await;
2662 assert!(
2663 status.paused.is_some(),
2664 "a missing root must pause, not replicate as a mass deletion"
2665 );
2666 }
2667
2668 #[tokio::test(flavor = "multi_thread")]
2669 async fn a_directory_that_is_not_a_workspace_cannot_be_enabled() {
2670 let f = fixture().await;
2671 let plain = f._tmp.path().join("plain");
2672 std::fs::create_dir_all(&plain).unwrap();
2673
2674 assert!(f.runtime.enable(&plain).await.is_err());
2675 }
2676
2677 #[tokio::test(flavor = "multi_thread")]
2678 async fn a_write_through_the_workspace_router_is_committed_to_the_replica() {
2679 let f = fixture().await;
2684 f.runtime.enable(&f.root).await.unwrap();
2685
2686 let host = Arc::new(crate::WorkspaceHost::new(&CTX));
2687 host.select(&f.root, false).await.unwrap();
2688 let current = Arc::new(crate::Current::new(
2689 &CTX,
2690 host,
2691 Some(Arc::clone(&f.runtime)),
2692 crate::selection::SelectionFile::at(f._tmp.path().join("workspace.toml")),
2693 ));
2694 let router = Arc::new(crate::workspace_router(current));
2695 let (client_conn, server_conn) = sapphire_ipc::Connection::pair();
2696 tokio::spawn(async move {
2697 let info = sapphire_ipc::ServerInfo {
2698 version: "0.0.0".into(),
2699 api: sapphire_backend::protocol::API_VERSION,
2700 app_api: None,
2701 pid: std::process::id(),
2702 managed_by: ManagedBy::Spawned,
2703 };
2704 let _ = sapphire_ipc::serve(server_conn, router, "sapphire-synctest", info).await;
2705 });
2706 let info = sapphire_ipc::ClientInfo {
2707 kind: "test".into(),
2708 version: "0.0.0".into(),
2709 api: sapphire_backend::protocol::API_VERSION,
2710 app_api: None,
2711 pid: std::process::id(),
2712 };
2713 let (client, _) = sapphire_ipc::Client::handshake(client_conn, "sapphire-synctest", info)
2714 .await
2715 .unwrap();
2716
2717 let _: sapphire_backend::protocol::Ack = client
2718 .call(
2719 sapphire_backend::protocol::WRITE_FILE,
2720 sapphire_backend::protocol::ContentParams {
2721 path: PathBuf::from("through-the-server.md"),
2722 content: "written by the server".into(),
2723 },
2724 )
2725 .await
2726 .unwrap();
2727
2728 let synced = f.runtime.synced.lock().await;
2731 let entry = synced.get(&f.root).expect("enabled above");
2732 let replica = entry.replica.lock().await;
2733 let state = replica
2734 .state("through-the-server.md")
2735 .expect("a readable state");
2736 assert!(
2737 state.is_some(),
2738 "a write through the server must be scanned into the replica, not left to the watcher"
2739 );
2740 }
2741
2742 #[tokio::test(flavor = "multi_thread")]
2743 async fn an_announcement_for_an_unknown_workspace_is_ignored_not_fatal() {
2744 let f = fixture().await;
2745 f.runtime.enable(&f.root).await.unwrap();
2746
2747 let driver = Arc::clone(&f.runtime);
2748 let handle = tokio::spawn(async move { driver.run().await });
2749
2750 f.stub
2751 .announce(grain_id::GrainId::random(), "no-such-ticket")
2752 .await;
2753 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
2754
2755 assert!(
2756 !handle.is_finished(),
2757 "one bad announcement must not stop the runtime"
2758 );
2759 handle.abort();
2760 }
2761}