1use std::collections::HashMap;
9use std::path::{Path, PathBuf};
10use std::sync::Arc;
11
12use grain_id::GrainId;
13use sapphire_backend::protocol as proto;
14use sapphire_bridge_api::{
15 IncomingParams, RegisterParams, WorkspaceRegistration, WorkspacesResult,
16};
17use sapphire_bridge_client::BridgeClient;
18use sapphire_ipc::ManagedBy;
19use sapphire_sync::{PathUpdate, PauseReason, Replica, ReplicaConfig, ScanOutcome, SystemClock};
20use sapphire_workspace::{AppContext, Workspace};
21use tokio::sync::{Mutex, OnceCell, broadcast, mpsc};
22
23pub mod id;
24mod live;
25mod methods;
26#[cfg(any(test, feature = "test-util"))]
27pub mod testing;
28mod topology;
29mod watch;
30
31pub use id::{SYNC_ID_FILE, WORKSPACE_MAP_FILE, sync_id, sync_id_path};
32pub use live::{DIAL_BACKOFF_MAX, OPEN_TIMEOUT};
33pub use methods::sync_router;
34pub use sapphire_bridge_api::{PeerInfo, PeersResult};
36pub use watch::{DEBOUNCE, Watcher};
37
38use live::{LivePeers, dial_backoff};
39
40use crate::error::{Error, Result};
41use crate::settings::Settings;
42
43use crate::host::WorkspaceHost;
44
45#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
47pub struct SyncStatus {
48 pub enabled: bool,
50 pub workspace_id: Option<GrainId>,
52 pub peers: usize,
54 pub paused: Option<String>,
56 pub last_error: Option<String>,
58 pub bridge_available: bool,
60 pub topology: proto::Topology,
62 pub embedding: Option<proto::EmbeddingProgress>,
64}
65
66struct Synced {
68 workspace_id: GrainId,
70 replica: Arc<Mutex<Replica>>,
72 paused: Option<PauseReason>,
74 last_error: Option<String>,
76}
77
78const STALE_VECTOR_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
81
82const PASS_BATCH: usize = 4;
85
86#[derive(Clone, Copy, Debug)]
88pub struct BackfillTiming {
89 pub grace: std::time::Duration,
92 pub interval: std::time::Duration,
94}
95
96impl Default for BackfillTiming {
97 fn default() -> Self {
98 BackfillTiming {
99 grace: std::time::Duration::from_secs(10 * 60),
100 interval: std::time::Duration::from_secs(10 * 60),
101 }
102 }
103}
104
105pub const RECONNECT_BACKOFF: std::time::Duration = std::time::Duration::from_millis(500);
108
109pub const RECONNECT_BACKOFF_MAX: std::time::Duration = std::time::Duration::from_secs(30);
112
113fn reconnect_backoff(first: std::time::Duration, attempt: u32) -> std::time::Duration {
115 first
116 .saturating_mul(1u32 << attempt.min(16))
117 .min(RECONNECT_BACKOFF_MAX.max(first))
118}
119
120type Connecting = std::pin::Pin<
122 Box<dyn std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>> + Send>,
123>;
124
125#[derive(Clone)]
130pub struct BridgeConnector(Arc<dyn Fn() -> Connecting + Send + Sync>);
131
132impl BridgeConnector {
133 pub fn new<F, Fut>(connect: F) -> BridgeConnector
135 where
136 F: Fn() -> Fut + Send + Sync + 'static,
137 Fut: std::future::Future<Output = sapphire_ipc::Result<Option<BridgeClient>>>
138 + Send
139 + 'static,
140 {
141 BridgeConnector(Arc::new(move || Box::pin(connect())))
142 }
143
144 pub fn standard(kind: &'static str, version: &'static str) -> BridgeConnector {
147 BridgeConnector::new(move || async move {
148 match BridgeClient::connect(kind, version).await {
149 Ok(client) => Ok(Some(client)),
150 Err(sapphire_ipc::Error::NotRunning(_)) => Ok(None),
151 Err(err) => Err(err),
152 }
153 })
154 }
155
156 pub fn at(
158 endpoint: sapphire_ipc::Endpoint,
159 kind: &'static str,
160 version: &'static str,
161 ) -> BridgeConnector {
162 BridgeConnector::new(move || {
163 let endpoint = endpoint.clone();
164 async move { BridgeClient::connect_at(&endpoint, kind, version).await }
165 })
166 }
167
168 pub(crate) async fn connect(&self) -> sapphire_ipc::Result<Option<BridgeClient>> {
170 (self.0)().await
171 }
172}
173
174impl std::fmt::Debug for BridgeConnector {
175 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
176 f.write_str("BridgeConnector")
177 }
178}
179
180enum Reclaimed {
182 Open(Arc<Mutex<Replica>>),
184 Closing,
186 Closed,
188}
189
190const CLOSING_RETRIES: u32 = 50;
192const CLOSING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(20);
193
194pub struct SyncRuntime {
196 ctx: &'static AppContext,
197 bridge: tokio::sync::watch::Sender<Option<Arc<BridgeClient>>>,
200 connector: Option<BridgeConnector>,
202 reconnect_now: tokio::sync::Notify,
205 pending: Mutex<HashMap<PathBuf, GrainId>>,
212 subscribed: std::sync::Mutex<Option<(Arc<BridgeClient>, broadcast::Receiver<IncomingParams>)>>,
217 exe_path: PathBuf,
218 managed_by: ManagedBy,
219 synced: Mutex<HashMap<PathBuf, Synced>>,
220 stopped: std::sync::Mutex<HashMap<PathBuf, std::sync::Weak<Mutex<Replica>>>>,
227 device_id: OnceCell<GrainId>,
233 watcher: OnceCell<Arc<Watcher>>,
243 host: OnceCell<Arc<WorkspaceHost>>,
254 settings: std::sync::Mutex<Option<Settings>>,
257 reindexing: Mutex<()>,
260 embed_wanted: Mutex<std::collections::HashSet<PathBuf>>,
263 embedding: Mutex<()>,
265 embed_running: Mutex<std::collections::HashSet<PathBuf>>,
267 backfill: std::sync::Mutex<BackfillTiming>,
269 bridge_endpoint: Option<sapphire_ipc::Endpoint>,
271 live: Mutex<HashMap<PathBuf, Arc<LivePeers>>>,
277 last_peers: Mutex<Option<PeersResult>>,
280 open_timeout: std::time::Duration,
282 reconnect_first: std::time::Duration,
285 first_dials: std::sync::Mutex<tokio::task::JoinSet<()>>,
288 background: std::sync::Mutex<tokio::task::JoinSet<()>>,
293}
294
295impl SyncRuntime {
296 pub fn new(
301 ctx: &'static AppContext,
302 bridge: Arc<BridgeClient>,
303 exe_path: PathBuf,
304 managed_by: ManagedBy,
305 ) -> SyncRuntime {
306 SyncRuntime::build(ctx, Some(bridge), None, exe_path, managed_by)
307 }
308
309 pub fn connecting(
317 ctx: &'static AppContext,
318 connector: BridgeConnector,
319 exe_path: PathBuf,
320 managed_by: ManagedBy,
321 ) -> SyncRuntime {
322 SyncRuntime::build(ctx, None, Some(connector), exe_path, managed_by)
323 }
324
325 pub fn with_connector(mut self, connector: BridgeConnector) -> SyncRuntime {
327 self.connector = Some(connector);
328 self
329 }
330
331 fn build(
332 ctx: &'static AppContext,
333 bridge: Option<Arc<BridgeClient>>,
334 connector: Option<BridgeConnector>,
335 exe_path: PathBuf,
336 managed_by: ManagedBy,
337 ) -> SyncRuntime {
338 SyncRuntime {
339 ctx,
340 bridge: tokio::sync::watch::Sender::new(bridge),
341 connector,
342 reconnect_now: tokio::sync::Notify::new(),
343 pending: Mutex::new(HashMap::new()),
344 subscribed: std::sync::Mutex::new(None),
345 exe_path,
346 managed_by,
347 synced: Mutex::new(HashMap::new()),
348 stopped: std::sync::Mutex::new(HashMap::new()),
349 device_id: OnceCell::new(),
350 watcher: OnceCell::new(),
351 host: OnceCell::new(),
352 settings: std::sync::Mutex::new(None),
353 reindexing: Mutex::new(()),
354 embed_wanted: Mutex::new(std::collections::HashSet::new()),
355 embedding: Mutex::new(()),
356 embed_running: Mutex::new(std::collections::HashSet::new()),
357 backfill: std::sync::Mutex::new(BackfillTiming::default()),
358 bridge_endpoint: None,
359 live: Mutex::new(HashMap::new()),
360 last_peers: Mutex::new(None),
361 open_timeout: OPEN_TIMEOUT,
362 reconnect_first: RECONNECT_BACKOFF,
363 first_dials: std::sync::Mutex::new(tokio::task::JoinSet::new()),
364 background: std::sync::Mutex::new(tokio::task::JoinSet::new()),
365 }
366 }
367
368 fn bridge(&self) -> Result<Arc<BridgeClient>> {
370 match &*self.bridge.borrow() {
371 Some(bridge) if !bridge.is_closed() => Ok(Arc::clone(bridge)),
372 _ => Err(Error::BridgeDisconnected),
373 }
374 }
375
376 pub fn is_connected(&self) -> bool {
378 self.bridge().is_ok()
379 }
380
381 pub(crate) fn connected_bridge(&self) -> Option<Arc<BridgeClient>> {
383 self.bridge().ok()
384 }
385
386 pub(crate) fn reconnect_now(&self) {
391 self.reconnect_now.notify_one();
392 }
393
394 pub(crate) fn bridge_changes(&self) -> tokio::sync::watch::Receiver<Option<Arc<BridgeClient>>> {
397 self.bridge.subscribe()
398 }
399
400 fn spawn_background(&self, task: impl std::future::Future<Output = ()> + Send + 'static) {
402 let mut background = self.background.lock().expect("background tasks");
403 while background.try_join_next().is_some() {}
405 background.spawn(task);
406 }
407
408 pub fn with_open_timeout(mut self, timeout: std::time::Duration) -> SyncRuntime {
411 self.open_timeout = timeout;
412 self
413 }
414
415 #[cfg(test)]
418 pub(crate) fn with_reconnect_backoff(mut self, first: std::time::Duration) -> SyncRuntime {
419 self.reconnect_first = first;
420 self
421 }
422
423 pub fn with_bridge_endpoint(mut self, endpoint: sapphire_ipc::Endpoint) -> SyncRuntime {
430 self.bridge_endpoint = Some(endpoint);
431 self
432 }
433
434 pub fn set_backfill_timing(&self, timing: BackfillTiming) {
437 *self.backfill.lock().expect("backfill timing") = timing;
438 }
439
440 fn backfill_timing(&self) -> BackfillTiming {
441 *self.backfill.lock().expect("backfill timing")
442 }
443
444 pub(crate) fn set_host(&self, host: Arc<WorkspaceHost>) {
445 let _ = self.host.set(host);
446 }
447
448 pub(crate) fn set_settings(&self, settings: Option<Settings>) {
452 *self.settings.lock().unwrap_or_else(|e| e.into_inner()) = settings;
453 }
454
455 fn refresh_settings(&self) {
458 let settings = self
459 .settings
460 .lock()
461 .unwrap_or_else(|e| e.into_inner())
462 .clone();
463 if let Some(settings) = settings
464 && settings.refresh().is_err()
465 {
466 tracing::warn!("could not read the settings again after a change to the files");
468 }
469 }
470
471 pub async fn enable(self: &Arc<Self>, root: &Path) -> Result<GrainId> {
476 let key = root.canonicalize().map_err(Error::Io)?;
477 let watch_key = key.clone();
478 let workspace_id = sync_id(self.ctx.app_name, &key)?;
479
480 {
483 let mut pending = self.pending.lock().await;
484 if self.bridge().is_err() {
485 pending.insert(key, workspace_id);
486 return Err(Error::BridgeDisconnected);
487 }
488 pending.remove(&key);
489 }
490
491 {
494 let mut synced = self.synced.lock().await;
495 if let Some(existing) = synced.get(&key) {
496 return Ok(existing.workspace_id);
497 }
498 let replica = match self.reclaim(&key) {
501 Reclaimed::Open(replica) => replica,
502 reclaimed => {
503 let device_id = self.device_id().await?;
504 let state_dir = self.state_dir(&key)?;
505 std::fs::create_dir_all(&state_dir).map_err(Error::Io)?;
506 let config =
507 ReplicaConfig::new(self.ctx.app_name, key.clone(), device_id, &state_dir);
508 let mut tries = match reclaimed {
511 Reclaimed::Closing => CLOSING_RETRIES,
512 _ => 0,
513 };
514 let replica = loop {
515 match Replica::open(config.clone(), Arc::new(SystemClock)) {
516 Ok(replica) => break replica,
517 Err(_) if tries > 0 => {
518 tries -= 1;
519 tokio::time::sleep(CLOSING_RETRY_DELAY).await;
520 }
521 Err(err) => return Err(Error::Sync(err.to_string())),
522 }
523 };
524 Arc::new(Mutex::new(replica))
525 }
526 };
527 synced.insert(
528 key.clone(),
529 Synced {
530 workspace_id,
531 replica,
532 paused: None,
533 last_error: None,
534 },
535 );
536 }
537
538 if let Some(watcher) = self.watcher.get()
542 && let Err(err) = watcher.watch(&watch_key)
543 {
544 tracing::warn!(root = %watch_key.display(), "could not watch: {err}");
545 }
546
547 self.reregister().await?;
550
551 let runtime = Arc::clone(self);
569 let mut dials = self.first_dials.lock().expect("first dials");
570 while dials.try_join_next().is_some() {}
572 dials.spawn(async move {
573 if let Err(err) = runtime.scan(&key).await {
574 tracing::warn!(root = %key.display(), "the first scan after enabling failed: {err}");
575 }
576 if let Err(err) = runtime.sync_now(&key).await {
577 tracing::warn!(root = %key.display(), "the first session after enabling failed: {err}");
578 }
579 });
580 Ok(workspace_id)
581 }
582
583 pub async fn disable(&self, root: &Path) -> Result<()> {
589 let (replica, unregistered) = self.stop(root).await;
590 if let Some(replica) = replica {
591 let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
592 let mut stopped = self.stopped.lock().expect("stopped replicas");
593 stopped.retain(|_, replica| replica.strong_count() > 0);
595 stopped.insert(key, Arc::downgrade(&replica));
596 }
597 unregistered
598 }
599
600 pub async fn release(&self, root: &Path, limit: std::time::Duration) -> Result<()> {
609 let (replica, unregistered) = self.stop(root).await;
610 if let Err(err) = unregistered {
611 tracing::warn!(root = %root.display(), "stopping sync: {err}");
612 }
613 let Some(replica) = replica else {
614 return Ok(());
615 };
616 let deadline = tokio::time::Instant::now() + limit;
617 while Arc::strong_count(&replica) > 1 {
618 if tokio::time::Instant::now() >= deadline {
619 return Err(Error::Sync(format!(
620 "the replica of {} is still in use; try again",
621 root.display()
622 )));
623 }
624 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
625 }
626 Ok(())
627 }
628
629 async fn stop(&self, root: &Path) -> (Option<Arc<Mutex<Replica>>>, Result<()>) {
632 let key = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
635 self.pending.lock().await.remove(&key);
636 let removed = self.synced.lock().await.remove(&key);
637 if removed.is_some()
638 && let Some(watcher) = self.watcher.get()
639 {
640 watcher.unwatch(&key);
641 }
642 if let Some(peers) = self.live.lock().await.remove(&key) {
645 peers.drop_connections().await;
646 }
647 let Some(removed) = removed else {
648 return (None, Ok(()));
649 };
650 let unregistered = async {
651 let Ok(bridge) = self.bridge() else {
654 return Ok(());
655 };
656 bridge
657 .unregister(removed.workspace_id)
658 .await
659 .map_err(|e| Error::Bridge(e.to_string()))?;
660 self.reregister().await
661 }
662 .await;
663 (Some(removed.replica), unregistered)
664 }
665
666 pub async fn workspaces(&self) -> Result<WorkspacesResult> {
672 self.bridge()?
673 .workspaces()
674 .await
675 .map_err(|e| Error::Bridge(e.to_string()))
676 }
677
678 pub async fn place(&self, selector: &str, dir: &Path) -> Result<PathBuf> {
689 let workgroup = self.workspaces().await?;
690 let wanted = workgroup
691 .workspaces
692 .iter()
693 .find(|w| w.name == selector || w.workspace_id.to_string() == selector)
696 .ok_or_else(|| Error::UnknownWorkspaceName(selector.to_owned()))?;
697
698 if wanted.app_name != self.ctx.app_name {
701 return Err(Error::WrongApp {
702 name: wanted.name.clone(),
703 app_name: wanted.app_name.clone(),
704 });
705 }
706
707 let root = dir.canonicalize().map_err(Error::Io)?;
708 let marker = root.join(format!(".{}", self.ctx.app_name));
709 if !marker.is_dir() {
710 std::fs::create_dir(&marker).map_err(Error::Io)?;
711 }
712 let workspace = Workspace::from_root(self.ctx, &root)?;
713 if workspace.root != root {
714 return Err(Error::UnknownWorkspace(root, self.ctx.app_name));
715 }
716
717 {
721 let synced = self.synced.lock().await;
722 if let Some(entry) = synced.get(&root)
723 && entry.workspace_id != wanted.workspace_id
724 {
725 return Err(Error::SyncId(format!(
726 "{} already syncs as {}, not {}",
727 root.display(),
728 entry.workspace_id,
729 wanted.workspace_id
730 )));
731 }
732 }
733
734 let marker = workspace.marker_dir();
739 let id_path = marker.join(crate::sync::id::SYNC_ID_FILE);
740 match std::fs::read_to_string(&id_path) {
741 Ok(existing) if existing.trim() != wanted.workspace_id.to_string() => {
742 return Err(Error::SyncId(format!(
743 "{} already holds identity {}, not {}",
744 root.display(),
745 existing.trim(),
746 wanted.workspace_id
747 )));
748 }
749 Err(_) => {
750 std::fs::write(&id_path, format!("{}\n", wanted.workspace_id))
751 .map_err(Error::Io)?;
752 }
753 Ok(_) => {}
754 }
755
756 let map_path = root
759 .join(format!(".{}", self.ctx.app_name))
760 .join(WORKSPACE_MAP_FILE);
761 let workspace_id = wanted.workspace_id.to_string();
762 std::fs::write(&map_path, format!("{workspace_id}\n")).map_err(Error::Io)?;
763 Ok(root)
764 }
765
766 pub async fn status(&self, root: &Path) -> SyncStatus {
768 let bridge = self.bridge().ok();
769 let bridge_available = match &bridge {
770 Some(bridge) => bridge.status().await.is_ok(),
771 None => false,
772 };
773 let answer = match &bridge {
774 Some(_) => self.peers().await.ok(),
775 None => None,
776 };
777 let peers = answer
778 .as_ref()
779 .map_or(0, |p| p.peers.len().saturating_sub(1));
780 let Ok(key) = root.canonicalize() else {
781 return SyncStatus {
782 enabled: false,
783 workspace_id: None,
784 peers,
785 paused: None,
786 last_error: None,
787 bridge_available,
788 topology: proto::Topology::Mesh,
789 embedding: None,
790 };
791 };
792 let synced = self.synced.lock().await;
793 match synced.get(&key) {
794 Some(entry) => SyncStatus {
795 enabled: true,
796 workspace_id: Some(entry.workspace_id),
797 peers,
798 paused: entry.paused.map(|r| format!("{r:?}")),
799 last_error: entry.last_error.clone(),
800 bridge_available,
801 topology: topology::topology(
802 answer
803 .as_ref()
804 .and_then(|p| p.roles_for(entry.workspace_id)),
805 ),
806 embedding: self.embedding_progress(&key).await,
807 },
808 None => {
809 drop(synced);
810 let waiting = self.pending.lock().await.get(&key).copied();
813 SyncStatus {
814 enabled: waiting.is_some(),
815 workspace_id: waiting,
816 peers,
817 paused: None,
818 last_error: None,
819 bridge_available,
820 topology: proto::Topology::Mesh,
821 embedding: None,
822 }
823 }
824 }
825 }
826
827 async fn embedding_progress(&self, key: &Path) -> Option<proto::EmbeddingProgress> {
830 let host = self.host.get()?;
831 let backend = host.open_backend(key)?;
832 let state = backend.state();
833 state.embedder()?;
834 let info = state.db_info().ok()?;
835 Some(proto::EmbeddingProgress {
836 vectors: info.vector_count,
837 pending: info.pending_count,
838 running: self.embed_running.lock().await.contains(key),
839 })
840 }
841
842 pub async fn scan(&self, root: &Path) -> Result<()> {
848 let Ok(key) = root.canonicalize() else {
849 return Ok(());
850 };
851 let replica = {
852 let synced = self.synced.lock().await;
853 match synced.get(&key) {
854 Some(entry) => Arc::clone(&entry.replica),
855 None => return Ok(()),
856 }
857 };
858 let (outcome, updates) = {
859 let mut replica = replica.lock().await;
860 let outcome = replica.scan().map_err(|e| Error::Sync(e.to_string()))?;
861 let updates = match &outcome {
866 ScanOutcome::Scanned(report) => {
867 report
875 .recorded
876 .iter()
877 .map(|entry| PathUpdate {
878 path: entry.path.clone(),
879 versions: vec![entry.clone()],
880 seen: {
881 let mut seen = entry.context.clone();
882 seen.add_dot(&entry.dot);
883 seen
884 },
885 })
886 .collect()
887 }
888 ScanOutcome::Paused(_) => Vec::new(),
889 };
890 (outcome, updates)
891 };
892 {
893 let mut synced = self.synced.lock().await;
894 if let Some(entry) = synced.get_mut(&key) {
895 entry.paused = match outcome {
896 ScanOutcome::Paused(reason) => Some(reason),
897 ScanOutcome::Scanned(_) => None,
898 };
899 }
900 }
901 self.after_commit(&key, updates).await;
904 Ok(())
905 }
906
907 async fn reindex(&self, root: &Path) {
922 self.refresh_settings();
924 let Some(host) = self.host.get() else {
925 return;
926 };
927 let host = Arc::clone(host);
928 let root = root.to_owned();
929 let _serialised = self.reindexing.lock().await;
932 let started = std::time::Instant::now();
934 let backend = match host.backend_at(&root).await {
937 Ok(backend) => backend,
938 Err(err) => {
939 crate::instrument::reindexed(started, false);
940 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
941 return;
942 }
943 };
944 let state = Arc::clone(backend.state());
945 let result = tokio::task::spawn_blocking(move || state.sync_retrieve()).await;
946 crate::instrument::reindexed(started, matches!(result, Ok(Ok(_))));
947 match result {
948 Ok(Ok(report)) => {
949 tracing::debug!(root = %root.display(), ?report, "re-indexed after a session");
950 }
951 Ok(Err(err)) => {
952 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
953 }
954 Err(err) => {
955 tracing::warn!(root = %root.display(), "re-indexing after a session failed: {err}");
956 }
957 }
958 }
959
960 pub fn request_embed(self: &Arc<Self>, root: &Path) {
967 let Ok(key) = root.canonicalize() else {
968 return;
969 };
970 let runtime = Arc::clone(self);
971 self.spawn_background(async move {
972 if !runtime.embed_wanted.lock().await.insert(key.clone()) {
973 return;
974 }
975 let _one_at_a_time = runtime.embedding.lock().await;
976 runtime.embed_wanted.lock().await.remove(&key);
978 runtime.embed_running.lock().await.insert(key.clone());
979 runtime.embed_pass(&key).await;
980 runtime.embed_running.lock().await.remove(&key);
981 });
982 }
983
984 async fn embed_pass(&self, key: &Path) {
986 let Some(host) = self.host.get() else {
987 return;
988 };
989 let (workspace_id, replica) = {
990 let synced = self.synced.lock().await;
991 match synced.get(key) {
992 Some(entry) => (entry.workspace_id, Arc::clone(&entry.replica)),
993 None => return,
994 }
995 };
996 let backend = match host.backend_at(key).await {
997 Ok(backend) => backend,
998 Err(err) => {
999 tracing::warn!(root = %key.display(), "embedding pass: {err}");
1000 return;
1001 }
1002 };
1003 let state = Arc::clone(backend.state());
1004 if let Err(err) = state
1005 .refresh_embedder_at(self.bridge_endpoint.clone())
1006 .await
1007 {
1008 tracing::warn!(root = %key.display(), "embedding pass: {err}");
1009 return;
1010 }
1011 if state.embedder().is_none() {
1012 return;
1013 }
1014 let primary = self.is_primary(workspace_id).await;
1015 let grace_ms = self.backfill_timing().grace.as_millis() as u64;
1018 let now_ms = std::time::SystemTime::now()
1019 .duration_since(std::time::UNIX_EPOCH)
1020 .map_or(0, |d| d.as_millis() as u64);
1021 let (mine, settled, live) = {
1022 let replica = replica.lock().await;
1023 let me = replica.replica_id();
1024 let states = match replica.states() {
1025 Ok(states) => states,
1026 Err(err) => {
1027 tracing::warn!(root = %key.display(), "embedding pass: {err}");
1028 return;
1029 }
1030 };
1031 let mut mine = HashMap::new();
1032 let mut settled = HashMap::new();
1034 let mut live = std::collections::HashSet::new();
1035 for (path, path_state) in states {
1036 let winner = path_state.winner();
1037 let Some(hash) = winner.content.hash() else {
1038 continue;
1039 };
1040 let hash = hash.to_hex();
1041 if winner.dot.replica == me {
1042 mine.insert(path, hash.clone());
1043 } else if now_ms.saturating_sub(winner.hlc.wall_ms) >= grace_ms {
1044 settled.insert(path, hash.clone());
1045 }
1046 live.insert(hash);
1047 }
1048 (mine, settled, live)
1049 };
1050 let root = key.to_owned();
1051 let result = tokio::task::spawn_blocking(move || {
1052 let policy = |rel: &str, hash: &str| {
1055 mine.get(rel).is_some_and(|h| h.as_str() == hash)
1056 || (primary && settled.get(rel).is_some_and(|h| h.as_str() == hash))
1057 };
1058 let embedded = state.embed_pending_with(&policy, PASS_BATCH)?;
1059 let removed = if primary {
1060 state.remove_stale_vectors(&live, STALE_VECTOR_AGE)
1061 } else {
1062 0
1063 };
1064 Ok::<_, sapphire_workspace::Error>((embedded, removed))
1065 })
1066 .await;
1067 match result {
1068 Ok(Ok((embedded, removed))) if embedded > 0 || removed > 0 => tracing::info!(
1069 root = %root.display(),
1070 embedded,
1071 removed,
1072 "embedding pass"
1073 ),
1074 Ok(Ok(_)) => {}
1075 Ok(Err(err)) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
1076 Err(err) => tracing::warn!(root = %root.display(), "embedding pass: {err}"),
1077 }
1078 }
1079
1080 pub async fn sync_now(self: &Arc<Self>, root: &Path) -> Result<()> {
1082 let Ok(key) = root.canonicalize() else {
1083 return Ok(());
1084 };
1085 let workspace_id = {
1086 let synced = self.synced.lock().await;
1087 match synced.get(&key) {
1088 Some(entry) => entry.workspace_id,
1089 None => return Ok(()),
1090 }
1091 };
1092 let peers = self.peers().await?;
1093 let me = self.device_id().await?;
1094
1095 let roles = peers.roles_for(workspace_id).cloned();
1100 let mut dials: Vec<Dial<'_>> = Vec::new();
1104 for peer in peers
1105 .peers
1106 .iter()
1107 .filter(|p| topology::link(me, p.device_id, roles.as_ref()) == topology::Link::Dial)
1108 .cloned()
1109 .collect::<Vec<_>>()
1110 {
1111 if let Some(live) = self.live_peers(&key).await
1115 && live.has(&peer.device_id).await
1116 {
1117 continue;
1118 }
1119 let Some((_, table)) = self.live_peers_or_create(&key).await else {
1126 continue;
1127 };
1128 if !table.begin_dial(peer.device_id).await {
1129 continue;
1130 }
1131 let key = &key;
1132 dials.push(Box::pin(async move {
1133 let result = self
1135 .sync_one(key, &peer.device_id, peer.name.clone(), workspace_id)
1136 .await;
1137 table.end_dial(peer.device_id).await;
1138 if let Err(err) = result {
1139 tracing::debug!(peer = %peer.name, "no session: {err}");
1140 }
1141 }));
1142 }
1143 join_all(dials).await;
1144 Ok(())
1145 }
1146
1147 async fn sync_one(
1154 self: &Arc<Self>,
1155 key: &Path,
1156 device: &GrainId,
1157 name: String,
1158 workspace_id: GrainId,
1159 ) -> Result<()> {
1160 let stream = self.open_stream(workspace_id, *device).await?;
1161 self.adopt_live_session(stream, key, *device, workspace_id)
1162 .await
1163 .map_err(|err| {
1164 tracing::warn!(peer = %name, "a live session could not be started: {err}");
1165 err
1166 })
1167 }
1168
1169 async fn open_stream(
1171 &self,
1172 workspace_id: GrainId,
1173 device: GrainId,
1174 ) -> Result<sapphire_ipc::RawStream> {
1175 match tokio::time::timeout(
1176 self.open_timeout,
1177 self.bridge()?.open_stream(workspace_id, device),
1178 )
1179 .await
1180 {
1181 Ok(opened) => Ok(opened?),
1182 Err(_) => Err(Error::Bridge(format!(
1183 "no stream to {device} within {} s; it may be offline",
1184 self.open_timeout.as_secs_f32()
1185 ))),
1186 }
1187 }
1188
1189 pub async fn run(self: Arc<Self>) -> Result<()> {
1192 let mut slot = self.bridge.subscribe();
1193 loop {
1194 let bridge = slot.borrow_and_update().clone();
1195 match bridge {
1196 Some(bridge) if !bridge.is_closed() => self.answer(&bridge, &mut slot).await,
1197 _ => {
1199 if slot.changed().await.is_err() {
1200 return Ok(());
1201 }
1202 }
1203 }
1204 }
1205 }
1206
1207 async fn answer(
1209 self: &Arc<Self>,
1210 bridge: &Arc<BridgeClient>,
1211 slot: &mut tokio::sync::watch::Receiver<Option<Arc<BridgeClient>>>,
1212 ) {
1213 let early = self
1214 .subscribed
1215 .lock()
1216 .expect("subscribed")
1217 .take_if(|(b, _)| Arc::ptr_eq(b, bridge));
1218 let mut incoming = match early {
1219 Some((_, incoming)) => incoming,
1220 None => bridge.incoming(),
1221 };
1222 loop {
1223 let received = tokio::select! {
1224 received = incoming.recv() => received,
1225 () = bridge.closed() => return,
1226 _ = slot.changed() => return,
1227 };
1228 let announcement = match received {
1229 Ok(a) => a,
1230 Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
1231 tracing::warn!(missed = n, "fell behind on bridge announcements");
1232 continue;
1233 }
1234 Err(tokio::sync::broadcast::error::RecvError::Closed) => return,
1235 };
1236
1237 let found = {
1238 let synced = self.synced.lock().await;
1239 synced
1240 .iter()
1241 .find(|(_, s)| s.workspace_id == announcement.workspace_id)
1242 .map(|(root, s)| (root.clone(), Arc::clone(&s.replica)))
1243 };
1244 let Some((root, replica)) = found else {
1247 tracing::debug!(
1250 workspace = %announcement.workspace_id,
1251 "an announcement for a workspace this server does not hold"
1252 );
1253 continue;
1254 };
1255
1256 let bridge = Arc::clone(bridge);
1257 let workspace_id = announcement.workspace_id;
1258 let peer = announcement.peer_device_id;
1259 let driver = Arc::clone(self);
1260 self.spawn_background(async move {
1261 let stream = match bridge.accept_stream(announcement.ticket).await {
1262 Ok(stream) => stream,
1263 Err(err) => {
1264 tracing::warn!("could not claim an announced stream: {err}");
1265 return;
1266 }
1267 };
1268 if driver.should_skip(workspace_id, peer).await {
1269 tracing::debug!(%peer, "dropped an inbound session: both ends are outside this workspace's star");
1270 return;
1271 }
1272 let (_, session) = match sapphire_framework_session::open_live_session(
1277 stream,
1278 Arc::clone(&replica),
1279 workspace_id,
1280 )
1281 .await
1282 {
1283 Ok(started) => started,
1284 Err(err) => {
1285 crate::instrument::sync_session(false);
1286 tracing::warn!("an inbound session failed: {err}");
1287 return;
1288 }
1289 };
1290 let Some((key, table)) = driver.live_peers_or_create(&root).await else {
1291 return;
1292 };
1293 let Some(updates) = table.insert(peer, session, false).await else {
1297 return;
1298 };
1299 table.catch_up(peer, &replica).await;
1300 crate::instrument::sync_session(true);
1301 driver.reindex(&key).await;
1304 driver.request_embed(&key);
1305 driver.spawn_reader(table, updates, key, peer);
1306 });
1307 }
1308 }
1309
1310 pub async fn connect(self: &Arc<Self>) -> Result<()> {
1317 let Some(connector) = &self.connector else {
1318 return Ok(());
1319 };
1320 if self.is_connected() {
1321 return Ok(());
1322 }
1323 match connector.connect().await? {
1324 Some(client) => self.attach(client).await,
1325 None => tracing::warn!(
1326 "no sapphire-bridge is running; serving without sync until it starts"
1327 ),
1328 }
1329 Ok(())
1330 }
1331
1332 pub async fn keep_connected(self: Arc<Self>) {
1341 let mut attempt = 0;
1342 let mut warned = false;
1345 loop {
1346 if let Ok(bridge) = self.bridge() {
1347 bridge.closed().await;
1348 tracing::warn!("lost the connection to sapphire-bridge; reconnecting");
1349 self.bridge.send_if_modified(|slot| {
1350 let same = slot.as_ref().is_some_and(|b| Arc::ptr_eq(b, &bridge));
1351 if same {
1352 *slot = None;
1353 }
1354 same
1355 });
1356 attempt = 0;
1357 warned = false;
1358 continue;
1359 }
1360 let Some(connector) = &self.connector else {
1361 return;
1362 };
1363 match connector.connect().await {
1364 Ok(Some(client)) => {
1365 tracing::info!("connected to sapphire-bridge");
1366 self.attach(client).await;
1367 attempt = 0;
1368 warned = false;
1369 continue;
1370 }
1371 Ok(None) => tracing::debug!("no sapphire-bridge is running yet"),
1372 Err(err) if !warned => {
1374 tracing::warn!("could not connect to sapphire-bridge: {err}");
1375 warned = true;
1376 }
1377 Err(err) => tracing::debug!("could not connect to sapphire-bridge: {err}"),
1378 }
1379 tokio::select! {
1380 () = tokio::time::sleep(reconnect_backoff(self.reconnect_first, attempt)) => {
1381 attempt = attempt.saturating_add(1);
1382 }
1383 () = self.reconnect_now.notified() => attempt = 0,
1384 }
1385 }
1386 }
1387
1388 async fn attach(self: &Arc<Self>, client: BridgeClient) {
1391 let client = Arc::new(client);
1394 *self.subscribed.lock().expect("subscribed") =
1395 Some((Arc::clone(&client), client.incoming()));
1396 let waiting = {
1397 let mut pending = self.pending.lock().await;
1398 self.bridge.send_replace(Some(client));
1399 std::mem::take(&mut *pending)
1400 };
1401 if !self.synced.lock().await.is_empty()
1403 && let Err(err) = self.reregister().await
1404 {
1405 tracing::warn!("could not register with sapphire-bridge: {err}");
1406 }
1407 for root in waiting.into_keys() {
1409 if let Err(err) = self.enable(&root).await {
1410 tracing::warn!(root = %root.display(), "could not start sync: {err}");
1411 }
1412 }
1413 }
1414
1415 pub async fn roots(&self) -> Vec<PathBuf> {
1417 self.synced.lock().await.keys().cloned().collect()
1418 }
1419
1420 async fn live_peers(&self, root: &Path) -> Option<Arc<LivePeers>> {
1425 let key = root.canonicalize().ok()?;
1426 self.live.lock().await.get(&key).cloned()
1427 }
1428
1429 async fn live_peers_or_create(&self, root: &Path) -> Option<(PathBuf, Arc<LivePeers>)> {
1434 let key = root.canonicalize().ok()?;
1435 if !self.synced.lock().await.contains_key(&key) {
1436 return None;
1437 }
1438 let me = self.device_id().await.ok()?;
1441 let mut live = self.live.lock().await;
1442 let peers = Arc::clone(
1443 live.entry(key.clone())
1444 .or_insert_with(|| Arc::new(LivePeers::new(me))),
1445 );
1446 Some((key, peers))
1447 }
1448
1449 pub async fn after_commit(&self, root: &Path, updates: Vec<PathUpdate>) {
1459 if updates.is_empty() {
1460 return;
1461 }
1462 if let Some(peers) = self.live_peers(root).await {
1463 peers.fan_out(&updates, None).await;
1464 }
1465 }
1466
1467 pub async fn live_session_devices(&self, root: &Path) -> Vec<GrainId> {
1471 match self.live_peers(root).await {
1472 Some(peers) => peers.devices().await,
1473 None => Vec::new(),
1474 }
1475 }
1476
1477 pub async fn stop_dials(&self) {
1484 let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1485 dials.abort_all();
1486 while dials.join_next().await.is_some() {}
1487 }
1488
1489 pub async fn stop_background(&self) {
1497 let mut background =
1498 std::mem::take(&mut *self.background.lock().expect("background tasks"));
1499 background.abort_all();
1500 while background.join_next().await.is_some() {}
1501 }
1502
1503 #[cfg(test)]
1506 async fn settle_dials(&self) {
1507 let mut dials = std::mem::take(&mut *self.first_dials.lock().expect("first dials"));
1508 while dials.join_next().await.is_some() {}
1509 }
1510
1511 pub async fn drop_connections(&self) {
1516 let tables: Vec<Arc<LivePeers>> = self.live.lock().await.values().cloned().collect();
1517 for peers in tables {
1518 peers.drop_connections().await;
1519 }
1520 }
1521
1522 pub async fn dial_loop(self: Arc<Self>) -> Result<()> {
1537 let mut tick = tokio::time::interval(live::DIAL_INTERVAL);
1538 tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
1541 let mut waiting: HashMap<GrainId, std::time::Instant> = HashMap::new();
1542 let mut failures: HashMap<GrainId, u32> = HashMap::new();
1543 let mut primary_of: std::collections::HashSet<GrainId> = Default::default();
1545 let mut last_backfill = std::time::Instant::now();
1547 loop {
1548 tick.tick().await;
1549 let roots = self.roots().await;
1550 if roots.is_empty() {
1551 continue;
1552 }
1553 let peers = match self.peers().await {
1554 Ok(peers) => peers,
1555 Err(err) => {
1558 tracing::debug!("no peers to dial: {err}");
1559 continue;
1560 }
1561 };
1562 let me = match self.device_id().await {
1563 Ok(me) => me,
1564 Err(err) => {
1565 tracing::warn!("no device id, so nobody to dial: {err}");
1566 continue;
1567 }
1568 };
1569 let ids: Vec<(PathBuf, GrainId)> = self
1573 .synced
1574 .lock()
1575 .await
1576 .iter()
1577 .map(|(root, entry)| (root.clone(), entry.workspace_id))
1578 .collect();
1579 let backfill_due = last_backfill.elapsed() >= self.backfill_timing().interval;
1582 if backfill_due {
1583 last_backfill = std::time::Instant::now();
1584 }
1585 for (root, workspace_id) in ids {
1586 let primary = peers
1587 .roles_for(workspace_id)
1588 .is_some_and(|r| r.primary == Some(me));
1589 if !primary {
1590 primary_of.remove(&workspace_id);
1591 } else if primary_of.insert(workspace_id) || backfill_due {
1592 self.request_embed(&root);
1593 }
1594 }
1595 let now = std::time::Instant::now();
1596 for root in roots {
1605 let Some((key, table)) = self.live_peers_or_create(&root).await else {
1606 continue;
1607 };
1608 let workspace_id = {
1609 let synced = self.synced.lock().await;
1610 match synced.get(&key) {
1611 Some(entry) => entry.workspace_id,
1612 None => continue,
1613 }
1614 };
1615 let roles = peers.roles_for(workspace_id);
1616 if topology::may_close_skipped(me, roles, &table.devices().await) {
1619 table
1620 .retain(|d| topology::link(me, *d, roles) != topology::Link::Skip)
1621 .await;
1622 }
1623 for peer in peers.peers.iter().filter(|p| {
1624 p.connected && topology::link(me, p.device_id, roles) == topology::Link::Dial
1625 }) {
1626 let device = peer.device_id;
1627 if table.has(&device).await {
1628 failures.remove(&device);
1630 waiting.remove(&device);
1631 continue;
1632 }
1633 if let Some(until) = waiting.get(&device)
1634 && now < *until
1635 {
1636 continue;
1638 }
1639 if !table.begin_dial(device).await {
1643 continue;
1644 }
1645 let opened = self.open_stream(workspace_id, device).await;
1646 if opened.is_err() {
1647 table.end_dial(device).await;
1648 }
1649 match opened {
1650 Ok(stream) => {
1651 let adopted = self
1652 .adopt_live_session(stream, &root, device, workspace_id)
1653 .await;
1654 table.end_dial(device).await;
1657 match adopted {
1658 Ok(()) => {
1659 failures.remove(&device);
1660 waiting.remove(&device);
1661 }
1662 Err(err) => {
1663 tracing::debug!(
1664 peer = %peer.name,
1665 err = %err,
1666 "a live session could not be started"
1667 );
1668 let count = failures.entry(device).or_insert(0);
1669 *count = count.saturating_add(1);
1670 waiting.insert(device, now + dial_backoff(*count));
1671 }
1672 }
1673 }
1674 Err(err) => {
1675 tracing::debug!("no stream to a peer: {err}");
1678 let count = failures.entry(device).or_insert(0);
1679 *count = count.saturating_add(1);
1680 waiting.insert(device, now + dial_backoff(*count));
1681 }
1682 }
1683 }
1684 }
1685 }
1686 }
1687
1688 async fn adopt_live_session(
1693 self: &Arc<Self>,
1694 stream: sapphire_ipc::RawStream,
1695 root: &Path,
1696 device: GrainId,
1697 workspace_id: GrainId,
1698 ) -> Result<()> {
1699 let replica = self.replica_of(root).await?;
1700 let (_, session) = sapphire_framework_session::open_live_session(
1701 stream,
1702 Arc::clone(&replica),
1703 workspace_id,
1704 )
1705 .await
1706 .map_err(|e| {
1707 crate::instrument::sync_session(false);
1708 Error::Sync(e.to_string())
1709 })?;
1710 let Some((key, table)) = self.live_peers_or_create(root).await else {
1711 return Ok(());
1712 };
1713 let Some(updates) = table.insert(device, session, true).await else {
1718 return Ok(());
1719 };
1720 table.catch_up(device, &replica).await;
1721 crate::instrument::sync_session(true);
1722 self.reindex(&key).await;
1725 self.request_embed(&key);
1726 self.spawn_reader(Arc::clone(&table), updates, key, device);
1727 Ok(())
1728 }
1729
1730 fn spawn_reader(
1737 self: &Arc<Self>,
1738 table: Arc<LivePeers>,
1739 mut updates: broadcast::Receiver<Vec<PathUpdate>>,
1740 key: PathBuf,
1741 from: GrainId,
1742 ) {
1743 let driver = Arc::clone(self);
1744 self.spawn_background(async move {
1745 loop {
1746 match updates.recv().await {
1747 Ok(batch) => {
1748 driver.reindex(&key).await;
1751 driver.request_embed(&key);
1752 table.fan_out(&batch, Some(from)).await;
1753 }
1754 Err(broadcast::error::RecvError::Lagged(missed)) => {
1759 tracing::debug!(
1760 device = %from,
1761 missed,
1762 "fell behind on a live session; the next exchange catches up"
1763 );
1764 }
1765 Err(broadcast::error::RecvError::Closed) => {
1768 return;
1769 }
1770 }
1771 }
1772 });
1773 }
1774
1775 fn reclaim(&self, key: &Path) -> Reclaimed {
1778 let stopped = self.stopped.lock().expect("stopped replicas").remove(key);
1779 match stopped {
1780 None => Reclaimed::Closed,
1781 Some(replica) => replica
1782 .upgrade()
1783 .map_or(Reclaimed::Closing, Reclaimed::Open),
1784 }
1785 }
1786
1787 async fn replica_of(&self, root: &Path) -> Result<Arc<Mutex<Replica>>> {
1789 let key = root.canonicalize().map_err(Error::Io)?;
1790 let synced = self.synced.lock().await;
1791 match synced.get(&key) {
1792 Some(entry) => Ok(Arc::clone(&entry.replica)),
1793 None => Err(Error::UnknownWorkspace(key, self.ctx.app_name)),
1794 }
1795 }
1796
1797 pub async fn watch_changes(self: Arc<Self>) -> Result<()> {
1809 let (tx, mut rx) = mpsc::channel(64);
1810 let watcher = Arc::new(Watcher::start(self.roots().await, tx)?);
1811 let _ = self.watcher.set(Arc::clone(&watcher));
1812
1813 for root in self.roots().await {
1817 if let Err(err) = watcher.watch(&root) {
1818 tracing::warn!(root = %root.display(), "could not watch: {err}");
1819 }
1820 }
1821
1822 while let Some(root) = rx.recv().await {
1823 self.refresh_settings();
1825 if let Err(err) = self.scan(&root).await {
1828 tracing::warn!(root = %root.display(), "scan after a local edit failed: {err}");
1829 }
1830 self.request_embed(&root);
1831 if let Err(err) = self.sync_now(&root).await {
1832 tracing::warn!(root = %root.display(), "dial after a local edit failed: {err}");
1833 }
1834 }
1835 Ok(())
1836 }
1837
1838 async fn reregister(&self) -> Result<()> {
1840 let workspaces: Vec<WorkspaceRegistration> = {
1841 let synced = self.synced.lock().await;
1842 synced
1843 .iter()
1844 .map(|(root, entry)| WorkspaceRegistration {
1845 workspace_id: entry.workspace_id,
1846 root: root.clone(),
1847 })
1848 .collect()
1849 };
1850 self.bridge()?
1851 .register(RegisterParams {
1852 app_name: self.ctx.app_name.to_owned(),
1853 exe_path: self.exe_path.clone(),
1854 managed_by: wire_managed_by(self.managed_by),
1855 workspaces,
1856 })
1857 .await
1858 .map_err(|e| Error::Bridge(e.to_string()))?;
1859 Ok(())
1860 }
1861
1862 async fn should_skip(&self, workspace_id: GrainId, peer: GrainId) -> bool {
1867 let Some(&me) = self.device_id.get() else {
1868 return false;
1869 };
1870 let cached = self.last_peers.lock().await;
1871 let roles = cached.as_ref().and_then(|p| p.roles_for(workspace_id));
1872 topology::link(me, peer, roles) == topology::Link::Skip
1873 }
1874
1875 pub async fn is_primary(&self, workspace_id: GrainId) -> bool {
1881 let Some(&me) = self.device_id.get() else {
1882 return false;
1883 };
1884 let cached = self.last_peers.lock().await;
1885 cached
1886 .as_ref()
1887 .and_then(|p| p.roles_for(workspace_id))
1888 .is_some_and(|r| r.primary == Some(me))
1889 }
1890
1891 pub fn this_device(&self) -> Option<GrainId> {
1898 self.device_id.get().copied()
1899 }
1900
1901 pub async fn peers(&self) -> Result<PeersResult> {
1907 let peers = self
1908 .bridge()?
1909 .peers()
1910 .await
1911 .map_err(|e| Error::Bridge(e.to_string()))?;
1912 *self.last_peers.lock().await = Some(peers.clone());
1913 Ok(peers)
1914 }
1915
1916 async fn device_id(&self) -> Result<GrainId> {
1918 if let Some(id) = self.device_id.get() {
1919 return Ok(*id);
1920 }
1921 let result = self
1922 .bridge()?
1923 .register(RegisterParams {
1924 app_name: self.ctx.app_name.to_owned(),
1925 exe_path: self.exe_path.clone(),
1926 managed_by: wire_managed_by(self.managed_by),
1927 workspaces: Vec::new(),
1928 })
1929 .await
1930 .map_err(|e| Error::Bridge(e.to_string()))?;
1931 let _ = self.device_id.set(result.device_id);
1932 Ok(result.device_id)
1933 }
1934
1935 fn state_dir(&self, root: &Path) -> Result<PathBuf> {
1937 let workspace = Workspace::from_root(self.ctx, root)?;
1938 Ok(workspace.cache_dir().join("sync"))
1939 }
1940}
1941
1942type Dial<'a> = std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send + 'a>>;
1944
1945async fn join_all(mut dials: Vec<Dial<'_>>) {
1947 std::future::poll_fn(|cx| {
1948 dials.retain_mut(|dial| dial.as_mut().poll(cx).is_pending());
1949 if dials.is_empty() {
1950 std::task::Poll::Ready(())
1951 } else {
1952 std::task::Poll::Pending
1953 }
1954 })
1955 .await
1956}
1957
1958fn wire_managed_by(managed_by: ManagedBy) -> sapphire_bridge_api::ManagedBy {
1960 match managed_by {
1961 ManagedBy::Service => sapphire_bridge_api::ManagedBy::Service,
1962 ManagedBy::Spawned => sapphire_bridge_api::ManagedBy::Spawned,
1963 }
1964}
1965
1966#[cfg(test)]
1967mod tests {
1968 use super::*;
1969 use crate::sync::testing::StubBridge;
1970 use crate::test_support;
1971 use sapphire_workspace::AppKind;
1972 use std::ffi::OsString;
1973
1974 static CTX: AppContext = AppContext::new("sapphire-synctest");
1975
1976 const DIR_VARS: [&str; 3] = [
1978 "SAPPHIRE_SYNCTEST_CACHE_DIR",
1979 "SAPPHIRE_SYNCTEST_DATA_DIR",
1980 "SAPPHIRE_SYNCTEST_CONFIG_DIR",
1981 ];
1982
1983 struct EnvGuard {
1986 previous: [Option<OsString>; 3],
1987 _lock: std::sync::MutexGuard<'static, ()>,
1988 }
1989
1990 impl Drop for EnvGuard {
1991 fn drop(&mut self) {
1992 for (name, previous) in DIR_VARS.iter().zip(self.previous.iter_mut()) {
1995 match previous.take() {
1996 Some(value) => unsafe { std::env::set_var(name, value) },
1997 None => test_support::remove(name),
1998 }
1999 }
2000 }
2001 }
2002
2003 struct Fixture {
2011 _tmp: tempfile::TempDir,
2012 _env: EnvGuard,
2013 root: PathBuf,
2014 stub: StubBridge,
2015 runtime: Arc<SyncRuntime>,
2016 }
2017
2018 impl Fixture {
2019 fn second_root(&self) -> PathBuf {
2021 let root = self._tmp.path().join("ws2");
2022 std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
2023 root.canonicalize().unwrap()
2024 }
2025 }
2026
2027 const FIXTURE_OPEN_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500);
2029
2030 fn dialled_peers(me: GrainId, n: usize) -> Vec<sapphire_bridge_api::PeerInfo> {
2032 std::iter::repeat_with(GrainId::random)
2033 .filter(|id| *id > me)
2034 .take(n)
2035 .enumerate()
2036 .map(|(i, device_id)| sapphire_bridge_api::PeerInfo {
2037 device_id,
2038 name: format!("offline-{i}"),
2039 node_id: format!("node-{i}"),
2040 connected: false,
2041 priority: 1,
2042 availability: None,
2043 })
2044 .collect()
2045 }
2046
2047 #[tokio::test(flavor = "multi_thread")]
2048 async fn enabling_answers_at_once_with_a_peer_that_never_answers() {
2049 let f = fixture().await;
2051 let _hung = f.stub.hang_streams().await;
2052 f.stub.set_peers(dialled_peers(f.stub.device_id, 1));
2053
2054 let started = std::time::Instant::now();
2055 f.runtime.enable(&f.root).await.unwrap();
2056 assert!(
2057 started.elapsed() < FIXTURE_OPEN_TIMEOUT,
2058 "enable waited {:?} on the dial",
2059 started.elapsed()
2060 );
2061 }
2062
2063 #[tokio::test(flavor = "multi_thread")]
2064 async fn peers_that_never_answer_are_dialled_together_and_given_up_on() {
2065 let f = fixture().await;
2066 let _hung = f.stub.hang_streams().await;
2067 f.runtime.enable(&f.root).await.unwrap();
2068 f.runtime.settle_dials().await;
2071 f.stub.set_peers(dialled_peers(f.stub.device_id, 3));
2072
2073 let started = std::time::Instant::now();
2074 f.runtime.sync_now(&f.root).await.unwrap();
2075 let took = started.elapsed();
2076 assert!(took >= FIXTURE_OPEN_TIMEOUT, "gave up early: {took:?}");
2077 assert!(
2078 took < FIXTURE_OPEN_TIMEOUT * 2,
2079 "three dials one after another: {took:?}"
2080 );
2081 }
2082
2083 async fn fixture() -> Fixture {
2084 let (tmp, env, root) = scratch();
2085 let (stub, client) = StubBridge::start_in(tmp.path().join("run")).await;
2086 let runtime = Arc::new(
2087 SyncRuntime::new(&CTX, client, "/bin/true".into(), ManagedBy::Service)
2088 .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
2089 );
2090 Fixture {
2091 _tmp: tmp,
2092 _env: env,
2093 root,
2094 stub,
2095 runtime,
2096 }
2097 }
2098
2099 type Bridges = Arc<std::sync::Mutex<std::collections::VecDeque<BridgeClient>>>;
2102
2103 async fn connecting() -> (Fixture, Bridges, BridgeClient) {
2106 let (tmp, env, root) = scratch();
2107 let (stub, client) = StubBridge::start_client_in(tmp.path().join("run")).await;
2108 let bridges: Bridges = Arc::default();
2109 let connector = BridgeConnector::new({
2110 let bridges = Arc::clone(&bridges);
2111 move || {
2112 let next = bridges.lock().unwrap().pop_front();
2113 async move { Ok(next) }
2114 }
2115 });
2116 let runtime = Arc::new(
2117 SyncRuntime::connecting(&CTX, connector, "/bin/true".into(), ManagedBy::Service)
2118 .with_open_timeout(FIXTURE_OPEN_TIMEOUT),
2119 );
2120 let f = Fixture {
2121 _tmp: tmp,
2122 _env: env,
2123 root,
2124 stub,
2125 runtime,
2126 };
2127 (f, bridges, client)
2128 }
2129
2130 async fn eventually(what: &str, mut done: impl AsyncFnMut() -> bool) {
2132 let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10);
2133 while !done().await {
2134 assert!(tokio::time::Instant::now() < deadline, "never: {what}");
2135 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
2136 }
2137 }
2138
2139 #[tokio::test(flavor = "multi_thread")]
2140 async fn a_runtime_started_without_a_bridge_syncs_once_one_starts() {
2141 let (f, bridges, client) = connecting().await;
2143 f.runtime.connect().await.unwrap();
2144 assert!(!f.runtime.is_connected());
2145
2146 let err = f.runtime.enable(&f.root).await.unwrap_err();
2147 assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2148 let status = f.runtime.status(&f.root).await;
2149 assert!(status.enabled, "enabled, waiting for the bridge");
2150 assert!(!status.bridge_available);
2151 let id = status.workspace_id.expect("its id");
2152
2153 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2154 bridges.lock().unwrap().push_back(client);
2155 eventually("the waiting workspace is registered", async || {
2156 f.stub.last_workspaces() == vec![id]
2157 })
2158 .await;
2159 assert_eq!(f.runtime.roots().await, vec![f.root.clone()]);
2160 assert!(f.runtime.status(&f.root).await.bridge_available);
2161 keeping.abort();
2162 }
2163
2164 #[tokio::test(flavor = "multi_thread")]
2165 async fn a_runtime_that_loses_its_bridge_registers_with_the_next_one() {
2166 let (f, bridges, client) = connecting().await;
2167 bridges.lock().unwrap().push_back(client);
2168 f.runtime.connect().await.unwrap();
2169 assert!(f.runtime.is_connected());
2170 let id = f.runtime.enable(&f.root).await.unwrap();
2171 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2172
2173 f.stub.disconnect();
2174 eventually("the runtime notices", async || !f.runtime.is_connected()).await;
2175 let err = f.runtime.workspaces().await.unwrap_err();
2176 assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2177
2178 let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2179 bridges.lock().unwrap().push_back(client);
2180 eventually("the synced workspace is registered again", async || {
2181 next.last_workspaces() == vec![id]
2182 })
2183 .await;
2184 assert!(f.runtime.is_connected());
2185 keeping.abort();
2186 }
2187
2188 #[tokio::test(flavor = "multi_thread")]
2189 async fn this_device_is_known_once_a_workspace_is_enabled_and_never_asked_for() {
2190 let f = fixture().await;
2191 assert_eq!(f.runtime.this_device(), None);
2192 assert!(
2193 f.stub.seen.lock().unwrap().registrations.is_empty(),
2194 "asking must not register"
2195 );
2196
2197 f.runtime.enable(&f.root).await.unwrap();
2198 assert_eq!(f.runtime.this_device(), Some(f.stub.device_id));
2199 }
2200
2201 #[tokio::test(flavor = "multi_thread")]
2202 async fn peers_answers_the_workgroup_from_the_bridge() {
2203 let f = fixture().await;
2204 let listed = dialled_peers(f.stub.device_id, 2);
2205 f.stub.set_peers(listed.clone());
2206
2207 let peers = f.runtime.peers().await.unwrap();
2208 let ids: Vec<_> = peers.peers.iter().map(|p| p.device_id).collect();
2209 assert_eq!(ids, listed.iter().map(|p| p.device_id).collect::<Vec<_>>());
2210 }
2211
2212 #[tokio::test(flavor = "multi_thread")]
2213 async fn peers_says_so_while_the_bridge_is_away() {
2214 let (f, _bridges, _client) = connecting().await;
2215 let err = f.runtime.peers().await.unwrap_err();
2216 assert!(matches!(err, Error::BridgeDisconnected), "{err:?}");
2217 }
2218
2219 #[tokio::test(flavor = "multi_thread")]
2220 async fn disabling_while_the_bridge_is_away_forgets_the_waiting_workspace() {
2221 let (f, bridges, client) = connecting().await;
2222 f.runtime.enable(&f.root).await.unwrap_err();
2223 f.runtime.disable(&f.root).await.unwrap();
2224 assert!(!f.runtime.status(&f.root).await.enabled);
2225
2226 bridges.lock().unwrap().push_back(client);
2227 f.runtime.connect().await.unwrap();
2228 assert!(f.runtime.roots().await.is_empty());
2229 assert!(f.stub.last_workspaces().is_empty());
2230 }
2231
2232 #[tokio::test(flavor = "multi_thread")]
2233 async fn announcements_are_answered_from_a_reconnected_bridge() {
2234 let (f, bridges, client) = connecting().await;
2235 bridges.lock().unwrap().push_back(client);
2236 f.runtime.connect().await.unwrap();
2237 let id = f.runtime.enable(&f.root).await.unwrap();
2238 let keeping = tokio::spawn(Arc::clone(&f.runtime).keep_connected());
2239 let answering = tokio::spawn(Arc::clone(&f.runtime).run());
2240
2241 f.stub.disconnect();
2242 let (next, client) = StubBridge::start_client_in(f._tmp.path().join("run")).await;
2243 bridges.lock().unwrap().push_back(client);
2244 eventually("reconnected", async || next.last_workspaces() == vec![id]).await;
2245
2246 next.announce(id, "ticket").await;
2249 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2250 assert!(!answering.is_finished(), "the announcement loop ended");
2251 answering.abort();
2252 keeping.abort();
2253 }
2254
2255 fn scratch() -> (tempfile::TempDir, EnvGuard, PathBuf) {
2257 let lock = test_support::lock();
2258 let tmp = tempfile::tempdir().unwrap();
2259 let previous = DIR_VARS.map(std::env::var_os);
2260 for (name, dir) in DIR_VARS
2264 .iter()
2265 .zip(["cache", "data", "config"].map(|cat| tmp.path().join(cat)))
2266 {
2267 test_support::set(name, &dir);
2268 }
2269 let env = EnvGuard {
2270 previous,
2271 _lock: lock,
2272 };
2273 CTX.init(AppKind::Server);
2274
2275 let root = tmp.path().join("ws");
2276 std::fs::create_dir_all(root.join(".sapphire-synctest")).unwrap();
2277 let root = root.canonicalize().unwrap();
2278 (tmp, env, root)
2279 }
2280
2281 struct NoteApp;
2283
2284 impl crate::settings::AppSettings for NoteApp {
2285 fn defs(&self) -> Vec<sapphire_framework_settings::SettingDef> {
2286 vec![sapphire_framework_settings::SettingDef {
2287 key: "sapphire-synctest.note".into(),
2288 kind: sapphire_framework_settings::Kind::String { non_empty: false },
2289 scopes: vec![sapphire_framework_settings::Scope::Workspace],
2290 overridable: false,
2291 apply: sapphire_framework_settings::Apply::Live,
2292 default: None,
2293 section: "Test".into(),
2294 doc: "test".into(),
2295 process: sapphire_framework_settings::Process::App,
2296 }]
2297 }
2298 }
2299
2300 #[tokio::test(flavor = "multi_thread")]
2301 async fn a_workspace_config_changed_under_the_watcher_reaches_the_settings() {
2302 let f = fixture().await;
2303 let ctx: &'static AppContext = Box::leak(Box::new(AppContext::new("sapphire-synctest")));
2306 ctx.set_config_dir(f._tmp.path().join("settings"));
2307 let selection = crate::selection::SelectionFile::at(
2308 f._tmp.path().join("settings").join("workspace.toml"),
2309 );
2310 selection
2311 .save(&crate::selection::Selection {
2312 root: f.root.clone(),
2313 sync: true,
2314 })
2315 .unwrap();
2316 let settings =
2317 crate::settings::Settings::load_with(ctx, Arc::new(NoteApp), selection).unwrap();
2318 let changed = settings.changed();
2319 f.runtime.set_settings(Some(settings.clone()));
2320 f.runtime.enable(&f.root).await.unwrap();
2321 let watching = tokio::spawn(Arc::clone(&f.runtime).watch_changes());
2322 let config = f.root.join(".sapphire-synctest").join("config.toml");
2324 eventually("the settings hear of the new value", async || {
2325 std::fs::write(&config, "[sapphire-synctest]\nnote = \"synced\"\n").unwrap();
2326 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
2327 changed.has_changed().unwrap()
2328 })
2329 .await;
2330 assert_eq!(
2331 settings
2332 .get_as::<String>("sapphire-synctest.note")
2333 .as_deref(),
2334 Some("synced")
2335 );
2336 watching.abort();
2337 }
2338
2339 #[tokio::test(flavor = "multi_thread")]
2340 async fn enabling_registers_the_workspace_with_the_bridge() {
2341 let f = fixture().await;
2342
2343 let id = f.runtime.enable(&f.root).await.unwrap();
2344 assert_eq!(f.stub.last_workspaces(), vec![id]);
2345 }
2346
2347 #[tokio::test(flavor = "multi_thread")]
2348 async fn enabling_twice_is_idempotent_and_keeps_the_same_id() {
2349 let f = fixture().await;
2350
2351 let first = f.runtime.enable(&f.root).await.unwrap();
2352 let second = f.runtime.enable(&f.root).await.unwrap();
2353 assert_eq!(first, second);
2354 assert_eq!(
2355 f.stub.last_workspaces(),
2356 vec![first],
2357 "still exactly one workspace"
2358 );
2359 }
2360
2361 #[tokio::test(flavor = "multi_thread")]
2362 async fn a_registration_carries_the_complete_current_list() {
2363 let f = fixture().await;
2364 let second_root = f.second_root();
2365
2366 let a = f.runtime.enable(&f.root).await.unwrap();
2367 let b = f.runtime.enable(&second_root).await.unwrap();
2368
2369 let mut seen = f.stub.last_workspaces();
2370 seen.sort();
2371 let mut want = vec![a, b];
2372 want.sort();
2373 assert_eq!(seen, want, "registration is the whole set, not a delta");
2374 }
2375
2376 #[tokio::test(flavor = "multi_thread")]
2377 async fn no_later_registration_clears_this_apps_routes() {
2378 let f = fixture().await;
2384 let second_root = f.second_root();
2385
2386 f.runtime.enable(&f.root).await.unwrap();
2387 f.runtime.enable(&second_root).await.unwrap();
2388 f.runtime.sync_now(&f.root).await.unwrap();
2389 f.runtime.disable(&f.root).await.unwrap();
2390
2391 let seen = f.stub.seen.lock().expect("stub");
2392 let first_full = seen
2393 .registrations
2394 .iter()
2395 .position(|r| !r.workspaces.is_empty())
2396 .expect("enabling registers the workspace");
2397 let emptied_afterwards = seen.registrations[first_full + 1..]
2398 .iter()
2399 .filter(|r| r.workspaces.is_empty())
2400 .count();
2401 assert_eq!(
2402 emptied_afterwards, 0,
2403 "a registration after the first workspace must still name the whole set"
2404 );
2405 }
2406
2407 #[tokio::test(flavor = "multi_thread")]
2408 async fn disabling_unregisters_and_leaves_the_files_alone() {
2409 let f = fixture().await;
2410 std::fs::write(f.root.join("keep.md"), "content").unwrap();
2411
2412 let id = f.runtime.enable(&f.root).await.unwrap();
2413 f.runtime.disable(&f.root).await.unwrap();
2414
2415 assert!(f.stub.last_workspaces().is_empty());
2416 assert!(
2417 f.root.join("keep.md").exists(),
2418 "disabling sync must not touch files"
2419 );
2420 assert!(
2421 crate::sync::id::sync_id_path("sapphire-synctest", &f.root).exists(),
2422 "the sync id stays, so re-enabling rejoins the same workspace"
2423 );
2424 let _ = id;
2425 }
2426
2427 #[tokio::test(flavor = "multi_thread")]
2428 async fn re_enabling_after_disabling_reuses_the_identity() {
2429 let f = fixture().await;
2430
2431 let first = f.runtime.enable(&f.root).await.unwrap();
2432 f.runtime.disable(&f.root).await.unwrap();
2433 let again = f.runtime.enable(&f.root).await.unwrap();
2434
2435 assert_eq!(first, again);
2436 }
2437
2438 #[tokio::test(flavor = "multi_thread")]
2439 async fn status_reports_disabled_for_a_workspace_that_was_never_enabled() {
2440 let f = fixture().await;
2441
2442 let status = f.runtime.status(&f.root).await;
2443 assert!(!status.enabled);
2444 assert!(status.workspace_id.is_none());
2445 }
2446
2447 #[tokio::test(flavor = "multi_thread")]
2448 async fn disabling_something_that_was_never_enabled_is_not_an_error() {
2449 let f = fixture().await;
2450 f.runtime
2451 .disable(&f.root)
2452 .await
2453 .expect("disabling twice is fine");
2454 }
2455
2456 #[tokio::test(flavor = "multi_thread")]
2457 async fn a_file_already_on_disk_is_recorded_on_enable() {
2458 let f = fixture().await;
2461 std::fs::write(f.root.join("a.md"), "content").unwrap();
2462 f.runtime.enable(&f.root).await.unwrap();
2463 f.runtime.settle_dials().await;
2464
2465 let replica = f.runtime.replica_of(&f.root).await.unwrap();
2466 let states = replica.lock().await.states().unwrap();
2467 assert!(
2468 states.iter().any(|(path, _)| path == "a.md"),
2469 "a file on disk before enable must be recorded: {states:?}"
2470 );
2471 }
2472
2473 #[tokio::test(flavor = "multi_thread")]
2474 async fn re_enabling_while_the_replica_is_still_held_takes_it_back() {
2475 let f = fixture().await;
2478 f.runtime.enable(&f.root).await.unwrap();
2479 f.runtime.settle_dials().await;
2480 let held = f.runtime.replica_of(&f.root).await.unwrap();
2481
2482 f.runtime.disable(&f.root).await.unwrap();
2483 f.runtime
2484 .enable(&f.root)
2485 .await
2486 .expect("a re-enable must not open the store a second time");
2487
2488 let again = f.runtime.replica_of(&f.root).await.unwrap();
2489 assert!(Arc::ptr_eq(&held, &again), "the open replica is taken back");
2490 }
2491
2492 #[tokio::test(flavor = "multi_thread")]
2493 async fn re_enabling_while_the_replica_is_still_closing_waits_for_it() {
2494 let f = fixture().await;
2497 f.runtime.enable(&f.root).await.unwrap();
2498 f.runtime.settle_dials().await;
2499 let config = {
2500 let replica = f.runtime.replica_of(&f.root).await.unwrap();
2501 replica.lock().await.config().clone()
2502 };
2503 f.runtime.disable(&f.root).await.unwrap();
2504
2505 let closing = Replica::open(config, Arc::new(SystemClock)).unwrap();
2507 let key = f.root.canonicalize().unwrap();
2508 f.runtime
2509 .stopped
2510 .lock()
2511 .unwrap()
2512 .insert(key, std::sync::Weak::new());
2513 let closed = tokio::spawn(async move {
2514 tokio::time::sleep(std::time::Duration::from_millis(100)).await;
2515 drop(closing);
2516 });
2517
2518 f.runtime
2519 .enable(&f.root)
2520 .await
2521 .expect("a re-enable must wait for a store that is closing");
2522 closed.await.unwrap();
2523 }
2524
2525 #[tokio::test(flavor = "multi_thread")]
2526 async fn a_workspace_whose_root_vanished_is_reported_as_paused_not_deleted() {
2527 let f = fixture().await;
2528 std::fs::write(f.root.join("a.md"), "content").unwrap();
2529 f.runtime.enable(&f.root).await.unwrap();
2530 f.runtime.scan(&f.root).await.unwrap();
2531
2532 std::fs::remove_dir_all(f.root.join(".sapphire-synctest")).unwrap();
2534 let _ = f.runtime.scan(&f.root).await;
2535
2536 let status = f.runtime.status(&f.root).await;
2537 assert!(
2538 status.paused.is_some(),
2539 "a missing root must pause, not replicate as a mass deletion"
2540 );
2541 }
2542
2543 #[tokio::test(flavor = "multi_thread")]
2544 async fn a_directory_that_is_not_a_workspace_cannot_be_enabled() {
2545 let f = fixture().await;
2546 let plain = f._tmp.path().join("plain");
2547 std::fs::create_dir_all(&plain).unwrap();
2548
2549 assert!(f.runtime.enable(&plain).await.is_err());
2550 }
2551
2552 #[tokio::test(flavor = "multi_thread")]
2553 async fn a_write_through_the_workspace_router_is_committed_to_the_replica() {
2554 let f = fixture().await;
2559 f.runtime.enable(&f.root).await.unwrap();
2560
2561 let host = Arc::new(crate::WorkspaceHost::new(&CTX));
2562 host.select(&f.root, false).await.unwrap();
2563 let current = Arc::new(crate::Current::new(
2564 &CTX,
2565 host,
2566 Some(Arc::clone(&f.runtime)),
2567 crate::selection::SelectionFile::at(f._tmp.path().join("workspace.toml")),
2568 ));
2569 let router = Arc::new(crate::workspace_router(current));
2570 let (client_conn, server_conn) = sapphire_ipc::Connection::pair();
2571 tokio::spawn(async move {
2572 let info = sapphire_ipc::ServerInfo {
2573 version: "0.0.0".into(),
2574 api: sapphire_backend::protocol::API_VERSION,
2575 app_api: None,
2576 pid: std::process::id(),
2577 managed_by: ManagedBy::Spawned,
2578 };
2579 let _ = sapphire_ipc::serve(server_conn, router, "sapphire-synctest", info).await;
2580 });
2581 let info = sapphire_ipc::ClientInfo {
2582 kind: "test".into(),
2583 version: "0.0.0".into(),
2584 api: sapphire_backend::protocol::API_VERSION,
2585 app_api: None,
2586 pid: std::process::id(),
2587 };
2588 let (client, _) = sapphire_ipc::Client::handshake(client_conn, "sapphire-synctest", info)
2589 .await
2590 .unwrap();
2591
2592 let _: sapphire_backend::protocol::Ack = client
2593 .call(
2594 sapphire_backend::protocol::WRITE_FILE,
2595 sapphire_backend::protocol::ContentParams {
2596 path: PathBuf::from("through-the-server.md"),
2597 content: "written by the server".into(),
2598 },
2599 )
2600 .await
2601 .unwrap();
2602
2603 let synced = f.runtime.synced.lock().await;
2606 let entry = synced.get(&f.root).expect("enabled above");
2607 let replica = entry.replica.lock().await;
2608 let state = replica
2609 .state("through-the-server.md")
2610 .expect("a readable state");
2611 assert!(
2612 state.is_some(),
2613 "a write through the server must be scanned into the replica, not left to the watcher"
2614 );
2615 }
2616
2617 #[tokio::test(flavor = "multi_thread")]
2618 async fn an_announcement_for_an_unknown_workspace_is_ignored_not_fatal() {
2619 let f = fixture().await;
2620 f.runtime.enable(&f.root).await.unwrap();
2621
2622 let driver = Arc::clone(&f.runtime);
2623 let handle = tokio::spawn(async move { driver.run().await });
2624
2625 f.stub
2626 .announce(grain_id::GrainId::random(), "no-such-ticket")
2627 .await;
2628 tokio::time::sleep(std::time::Duration::from_millis(200)).await;
2629
2630 assert!(
2631 !handle.is_finished(),
2632 "one bad announcement must not stop the runtime"
2633 );
2634 handle.abort();
2635 }
2636}