1use super::*;
4use crate::request::DEFAULT_IQ_TIMEOUT;
5
6const APPSTATE_BLOB_DOWNLOAD_CONCURRENCY: usize = 4;
11const APP_STATE_KEY_REQUEST_DEDUP: Duration = Duration::from_secs(24 * 3600);
12const APP_STATE_KEY_REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
13const APP_STATE_KEY_PARTIAL_RETRY: Duration = Duration::from_secs(10);
14const APP_STATE_KEY_RETRY_MAX: Duration = Duration::from_secs(60);
15const APP_STATE_PATCH_SEND_ATTEMPTS: usize = 5;
19const APP_STATE_RESERVATION_WAIT: Duration =
32 Duration::from_secs(DEFAULT_IQ_TIMEOUT.as_secs() * (APP_STATE_PATCH_SEND_ATTEMPTS as u64 + 1));
33const APP_STATE_RETRY_BACKOFF_MIN: Duration = Duration::from_secs(1);
37const APP_STATE_RETRY_BACKOFF_MAX: Duration = Duration::from_secs(60 * 60);
38const APP_STATE_RETRY_MAX_ROUNDS: u32 = 8;
44const APP_STATE_RETRY_ROUND_SLACK: u32 = 4;
47
48fn app_state_retry_backoff(attempts: u32) -> Duration {
57 APP_STATE_RETRY_BACKOFF_MIN
58 .saturating_mul(2u32.saturating_pow(attempts))
59 .min(APP_STATE_RETRY_BACKOFF_MAX)
60}
61
62#[derive(Debug, Clone, Copy, PartialEq, Eq)]
75pub(crate) enum SyncOutcome {
76 Completed,
78 Deferred,
82}
83
84fn sync_still_owed(outcome: &Result<SyncOutcome>) -> bool {
93 !matches!(outcome, Ok(SyncOutcome::Completed))
94}
95
96#[derive(Debug, Clone, Copy, PartialEq, Eq)]
107pub(crate) struct SyncScope {
108 generation: u64,
110 deadline: Option<wacore::time::Instant>,
113}
114
115#[derive(Debug, Clone, Copy, PartialEq, Eq)]
117pub(crate) enum ScopeLost {
118 Retired,
121 Expired,
124}
125
126impl SyncScope {
127 #[cfg_attr(not(test), allow(dead_code))]
130 pub(crate) fn generation(self) -> u64 {
131 self.generation
132 }
133
134 pub(crate) fn remaining(self) -> Option<Duration> {
136 self.deadline
137 .map(|d| d.saturating_duration_since(wacore::time::Instant::now()))
138 }
139
140 pub(crate) fn is_bootstrap(self) -> bool {
143 self.deadline.is_some()
144 }
145
146 pub(crate) fn rebind(&mut self, to: u64) -> bool {
152 let moved = self.generation != to;
153 self.generation = to;
154 moved
155 }
156}
157
158#[derive(Debug)]
168pub(crate) struct BootstrapGate(AtomicU64);
169
170impl BootstrapGate {
171 pub(crate) fn new(outstanding: bool) -> Self {
174 Self(AtomicU64::new(Self::encode(0, outstanding)))
175 }
176
177 const fn encode(generation: u64, outstanding: bool) -> u64 {
178 (generation << 1) | outstanding as u64
179 }
180
181 pub(crate) fn is_armed(&self) -> bool {
183 self.0.load(Ordering::Acquire) & 1 == 1
184 }
185
186 pub(crate) fn arm_for_pairing(&self, current_generation: u64) {
211 let mut current = self.0.load(Ordering::Acquire);
212 loop {
213 let generation = (current >> 1).max(current_generation).saturating_add(1);
218 match self.0.compare_exchange_weak(
219 current,
220 Self::encode(generation, true),
221 Ordering::AcqRel,
222 Ordering::Acquire,
223 ) {
224 Ok(_) => return,
225 Err(observed) => current = observed,
226 }
227 }
228 }
229
230 pub(crate) fn settle(&self, generation: u64, outstanding: bool) -> bool {
236 let mut current = self.0.load(Ordering::Acquire);
237 loop {
238 if (current >> 1) > generation {
241 return false;
242 }
243 match self.0.compare_exchange_weak(
244 current,
245 Self::encode(generation, outstanding),
246 Ordering::AcqRel,
247 Ordering::Acquire,
248 ) {
249 Ok(_) => return true,
250 Err(observed) => current = observed,
251 }
252 }
253 }
254}
255
256#[derive(Debug, Clone, Copy, PartialEq, Eq)]
262pub(crate) enum SyncHolder {
263 Sync,
266 PatchSend,
268}
269
270#[derive(Debug, Clone, Copy, PartialEq, Eq)]
272pub(crate) enum ReservationSkip {
273 EquivalentSyncInFlight,
275 WaitTimedOut,
277}
278
279#[derive(Debug, Clone, Copy, PartialEq, Eq)]
282pub(crate) enum SyncSettles {
283 JustTheCollections,
287 InitialSync,
291}
292
293#[derive(Debug, Clone, Copy, PartialEq, Eq)]
295pub(crate) enum ReservationWait {
296 SkipBehindSync,
300 Always,
305}
306
307#[derive(Debug, Default, Clone, PartialEq, Eq)]
315pub(crate) struct BatchedSyncOutcome {
316 pub(crate) synced: Vec<WAPatchName>,
318 pub(crate) fatal: Vec<WAPatchName>,
321 pub(crate) retryable: Vec<WAPatchName>,
324 pub(crate) skipped: Vec<WAPatchName>,
326 reached_server: bool,
335}
336
337impl BatchedSyncOutcome {
338 pub(crate) fn reached_server(&self) -> bool {
344 self.reached_server
345 }
346
347 fn note_reached_server(&mut self) {
349 self.reached_server = true;
350 }
351
352 pub(crate) fn unsynced(&self) -> impl Iterator<Item = WAPatchName> + '_ {
354 self.fatal
355 .iter()
356 .chain(&self.retryable)
357 .chain(&self.skipped)
358 .copied()
359 }
360
361 pub(crate) fn all_synced(&self) -> bool {
363 self.unsynced().next().is_none()
364 }
365}
366
367pub(crate) struct SyncInFlight {
376 entries: std::sync::Mutex<HashMap<WAPatchName, (u64, SyncHolder)>>,
377 next_token: AtomicU64,
378 released: event_listener::Event,
381}
382
383impl SyncInFlight {
384 pub(crate) fn new() -> Arc<Self> {
385 Arc::new(Self {
386 entries: std::sync::Mutex::new(HashMap::new()),
387 next_token: AtomicU64::new(0),
388 released: event_listener::Event::new(),
389 })
390 }
391
392 pub(crate) fn try_begin_as(
394 self: &Arc<Self>,
395 name: WAPatchName,
396 holder: SyncHolder,
397 ) -> Result<SyncInFlightGuard, SyncHolder> {
398 let token = self.next_token.fetch_add(1, Ordering::Relaxed);
399 let mut entries = self.entries.lock().unwrap_or_else(|p| p.into_inner());
400 if let Some(&(_, current)) = entries.get(&name) {
401 return Err(current);
402 }
403 entries.insert(name, (token, holder));
404 Ok(SyncInFlightGuard {
405 registry: Arc::clone(self),
406 name,
407 token,
408 })
409 }
410
411 #[cfg(test)]
419 pub(crate) fn try_begin(self: &Arc<Self>, name: WAPatchName) -> Option<SyncInFlightGuard> {
420 self.try_begin_as(name, SyncHolder::Sync).ok()
421 }
422
423 pub(crate) async fn begin(
430 self: &Arc<Self>,
431 name: WAPatchName,
432 holder: SyncHolder,
433 ) -> SyncInFlightGuard {
434 loop {
435 let released = self.released.listen();
438 if let Ok(guard) = self.try_begin_as(name, holder) {
439 return guard;
440 }
441 released.await;
442 }
443 }
444
445 pub(crate) fn clear(&self) {
448 *self.entries.lock().unwrap_or_else(|p| p.into_inner()) = HashMap::new();
449 self.released.notify(usize::MAX);
450 }
451
452 pub(crate) fn len(&self) -> usize {
453 self.entries.lock().unwrap_or_else(|p| p.into_inner()).len()
454 }
455}
456
457pub(crate) struct SyncInFlightGuard {
458 registry: Arc<SyncInFlight>,
459 name: WAPatchName,
460 token: u64,
461}
462
463impl Drop for SyncInFlightGuard {
464 fn drop(&mut self) {
465 let mut entries = self
466 .registry
467 .entries
468 .lock()
469 .unwrap_or_else(|p| p.into_inner());
470 if entries
471 .get(&self.name)
472 .is_some_and(|&(t, _)| t == self.token)
473 {
474 entries.remove(&self.name);
475 }
476 drop(entries);
477 self.registry.released.notify(usize::MAX);
480 }
481}
482
483fn initial_app_state_key_retry(timeout: Duration) -> Duration {
484 (timeout / 2)
485 .max(Duration::from_millis(1))
486 .min(APP_STATE_KEY_PARTIAL_RETRY)
487}
488
489#[derive(Clone, Copy, Debug, Eq, PartialEq)]
490enum AppStateKeyRequestDelivery {
491 AllPeers,
492 SomePeers,
493}
494
495struct AppStateKeyRequestSchedule {
496 retry_at: wacore::time::Instant,
497 sent: bool,
498}
499
500enum AppStateKeyRequestProgress {
501 Scheduled(AppStateKeyRequestSchedule),
502 KeysReady,
503 TimedOut,
504}
505
506#[cold]
507#[inline(never)]
508fn classify_app_state_key_request_failures(
509 peer_count: usize,
510 failure_count: usize,
511 failures: &str,
512) -> Result<AppStateKeyRequestDelivery, anyhow::Error> {
513 if failure_count == peer_count {
514 return Err(anyhow::anyhow!(
515 "app-state key request failed for all {peer_count} peer device(s): {failures}"
516 ));
517 }
518 warn!(
519 "App-state key request failed for {failure_count}/{peer_count} peer device(s): {failures}"
520 );
521 Ok(AppStateKeyRequestDelivery::SomePeers)
522}
523
524#[cold]
525#[inline(never)]
526fn append_app_state_key_request_failure(
527 failures: &mut Option<String>,
528 message: std::fmt::Arguments<'_>,
529) {
530 let failures = failures.get_or_insert_with(String::new);
531 if !failures.is_empty() {
532 failures.push_str(", ");
533 }
534 let _ = std::fmt::Write::write_fmt(failures, message);
535}
536
537async fn collect_app_state_key_request_results<F, E>(
538 runtime: &dyn Runtime,
539 mut requests: futures::stream::FuturesUnordered<F>,
540 timeout: Duration,
541) -> Result<AppStateKeyRequestDelivery, anyhow::Error>
542where
543 F: Future<Output = (u16, std::result::Result<(), E>)>,
544 E: std::fmt::Display,
545{
546 use futures::StreamExt;
547 use futures::future::Either;
548
549 let peer_count = requests.len();
550 let mut failure_count = 0;
551 let mut failures = None;
552 let mut deadline = runtime.sleep(timeout);
553 while !requests.is_empty() {
554 match futures::future::select(requests.next(), deadline.as_mut()).await {
555 Either::Left((Some((device, result)), _)) => {
556 if let Err(error) = result {
557 failure_count += 1;
558 append_app_state_key_request_failure(
559 &mut failures,
560 format_args!("device {device}: {error}"),
561 );
562 }
563 }
564 Either::Left((None, _)) => break,
565 Either::Right(((), _)) => {
566 let timed_out = requests.len();
567 failure_count += timed_out;
568 append_app_state_key_request_failure(
569 &mut failures,
570 format_args!("{timed_out} peer request(s) timed out"),
571 );
572 break;
573 }
574 }
575 }
576
577 if failure_count != 0 {
578 return classify_app_state_key_request_failures(
579 peer_count,
580 failure_count,
581 failures.as_deref().unwrap_or_default(),
582 );
583 }
584 Ok(AppStateKeyRequestDelivery::AllPeers)
585}
586
587async fn app_state_keys_available(
588 backend: &dyn crate::store::traits::Backend,
589 key_ids: &[Vec<u8>],
590) -> bool {
591 for key_id in key_ids {
592 if backend.get_sync_key(key_id).await.ok().flatten().is_none() {
593 return false;
594 }
595 }
596 true
597}
598
599async fn remove_available_app_state_keys(
600 backend: &dyn crate::store::traits::Backend,
601 missing: &mut Vec<Vec<u8>>,
602) {
603 let mut index = 0;
604 while index < missing.len() {
605 if backend
606 .get_sync_key(&missing[index])
607 .await
608 .ok()
609 .flatten()
610 .is_some()
611 {
612 missing.swap_remove(index);
613 } else {
614 index += 1;
615 }
616 }
617}
618
619fn finalize_app_state_key_request_peers(
620 mut peers: Vec<Jid>,
621 current_device: u16,
622 primary: Jid,
623) -> Result<Vec<Jid>, anyhow::Error> {
624 for peer in &mut peers {
626 peer.user.clone_from(&primary.user);
627 peer.server = primary.server;
628 peer.agent = primary.agent;
629 peer.integrator = primary.integrator;
630 }
631 peers.retain(|jid| jid.device != current_device);
632 wacore::types::jid::sort_dedup_by_device(&mut peers);
633 if peers.is_empty() && current_device != primary.device {
634 peers.push(primary);
635 }
636 if peers.is_empty() {
637 return Err(anyhow::anyhow!(
638 "no peer devices available for app-state key request"
639 ));
640 }
641 Ok(peers)
642}
643
644impl Client {
645 pub(crate) async fn get_app_state_processor(&self) -> Arc<AppStateProcessor> {
646 let mut guard = self.app_state_processor.lock().await;
647 if let Some(proc) = guard.as_ref() {
648 return proc.clone();
649 }
650 debug!("Initializing AppStateProcessor for the first time.");
651 let proc = Arc::new(AppStateProcessor::new(
652 self.persistence_manager.backend(),
653 self.runtime.clone(),
654 ));
655 *guard = Some(proc.clone());
656 proc
657 }
658
659 async fn pre_download_external_blobs(
665 &self,
666 patch_lists: &[wacore::appstate::patch_decode::PatchList],
667 ) -> HashMap<String, Vec<u8>> {
668 use futures::StreamExt;
669
670 enum BlobKind {
672 Snapshot(WAPatchName),
673 Mutation(u64),
674 }
675
676 let mut jobs: Vec<(wa::ExternalBlobReference, BlobKind)> = Vec::new();
681 let mut seen_paths: HashSet<&str> = HashSet::new();
682 for pl in patch_lists {
683 if let Some(ext) = &pl.snapshot_ref
684 && let Some(path) = ext.direct_path.as_deref()
685 && seen_paths.insert(path)
686 {
687 jobs.push((ext.clone(), BlobKind::Snapshot(pl.name)));
688 }
689 for patch in &pl.patches {
690 if let Some(ext) = patch.external_mutations.as_option()
691 && let Some(path) = ext.direct_path.as_deref()
692 && seen_paths.insert(path)
693 {
694 let v = patch
695 .version
696 .as_option()
697 .and_then(|v| v.version)
698 .unwrap_or(0);
699 jobs.push((ext.clone(), BlobKind::Mutation(v)));
700 }
701 }
702 }
703
704 if jobs.is_empty() {
705 return HashMap::new();
706 }
707
708 let mut pre_downloaded = HashMap::with_capacity(jobs.len());
709 let results = futures::stream::iter(jobs.into_iter().map(|(ext, kind)| async move {
710 let bytes = self.download(&ext).await;
711 (ext.direct_path, kind, bytes)
713 }))
714 .buffer_unordered(APPSTATE_BLOB_DOWNLOAD_CONCURRENCY)
715 .collect::<Vec<_>>()
716 .await;
717
718 for (path, kind, res) in results {
719 match res {
720 Ok(bytes) => {
721 if let BlobKind::Mutation(v) = kind {
722 debug!(target: "Client/AppState", "Downloaded external mutations for patch v{} ({} bytes)", v, bytes.len());
723 } else {
724 debug!(target: "Client/AppState", "Downloaded external snapshot ({} bytes)", bytes.len());
725 }
726 if let Some(path) = path {
727 pre_downloaded.insert(path, bytes);
728 }
729 }
730 Err(e) => match kind {
731 BlobKind::Snapshot(name) => {
732 warn!("Failed to download external snapshot for {:?}: {e}", name)
733 }
734 BlobKind::Mutation(v) => {
735 warn!(
736 "Failed to download external mutations for patch v{}: {e}",
737 v
738 )
739 }
740 },
741 }
742 }
743
744 pre_downloaded
745 }
746
747 pub(crate) fn start_sync_task_worker(
748 self: &Arc<Self>,
749 receiver: async_channel::Receiver<MajorSyncTask>,
750 ) {
751 const HISTORY_SYNC_CONCURRENCY: usize = 2;
752
753 let worker_client = Arc::downgrade(self);
754 let history_permits = Arc::new(async_lock::Semaphore::new(HISTORY_SYNC_CONCURRENCY));
755 self.runtime
756 .spawn(Box::pin(async move {
757 while let Ok(task) = receiver.recv().await {
758 let Some(worker_client) = worker_client.upgrade() else {
759 break;
760 };
761
762 if matches!(task, MajorSyncTask::HistorySync { .. }) {
763 let permit = history_permits.acquire_arc().await;
764 let task_client = worker_client.clone();
765 worker_client
766 .runtime
767 .spawn(Box::pin(async move {
768 let _permit = permit;
769 task_client.process_sync_task(task).await;
770 }))
771 .detach();
772 } else {
773 worker_client.process_sync_task(task).await;
774 }
775 }
776 info!(
777 "Sync worker intake loop finished (detached history-sync tasks may still be running)."
778 );
779 }))
780 .detach();
781 }
782
783 #[cfg_attr(
785 feature = "tracing",
786 tracing::instrument(name = "wa.appstate.sync_task", level = "debug", skip_all)
787 )]
788 pub async fn process_sync_task(self: &Arc<Self>, task: MajorSyncTask) {
789 match task {
790 MajorSyncTask::HistorySync {
791 message_id,
792 notification,
793 mut tracker,
794 } => {
795 self.process_history_sync_task_tracked(message_id, *notification, &mut tracker)
796 .await;
797 }
798 MajorSyncTask::AppStateSync { name, full_sync } => {
799 let _guard = match self
808 .reserve_for_sync(name, ReservationWait::Always, self.sync_scope(None))
809 .await
810 {
811 Ok(guard) => guard,
812 Err(ReservationSkip::EquivalentSyncInFlight) => {
813 debug!(target: "Client/AppState", "Skipping app state sync task {name:?}: an equivalent sync holds it");
814 return;
815 }
816 Err(ReservationSkip::WaitTimedOut) => {
824 warn!(target: "Client/AppState", "Gave up waiting to sync {name:?}; scheduling a retry");
825 self.schedule_app_state_task_retry(name, full_sync);
826 return;
827 }
828 };
829 let outcome = self.process_app_state_sync_task(name, full_sync).await;
836 match &outcome {
837 Err(e) => self.log_sync_error(&format!("app state sync for {name:?}"), e),
838 Ok(SyncOutcome::Deferred) => {
839 debug!(target: "Client/AppState", "App state sync for {name:?} was deferred")
840 }
841 Ok(SyncOutcome::Completed) => {}
842 }
843 if sync_still_owed(&outcome) {
844 self.schedule_app_state_task_retry(name, full_sync);
845 }
846 }
847 }
848 }
849
850 #[cfg_attr(feature = "tracing", tracing::instrument(name = "wa.appstate.fetch", level = "debug", skip_all, fields(name = ?name), err(Debug)))]
858 async fn fetch_app_state_with_retry_inner(&self, name: WAPatchName) -> Result<()> {
859 let _t = wacore::telemetry::timer(wacore::telemetry::APPSTATE_SYNC_DURATION);
860 let mut attempt = 0u32;
861 loop {
862 attempt += 1;
863 let res = self.process_app_state_sync_task(name, false).await;
867 match res {
868 Ok(SyncOutcome::Completed) => {
869 wacore::telemetry::appstate_sync("ok");
870 return Ok(());
871 }
872 Ok(SyncOutcome::Deferred) => {
878 wacore::telemetry::appstate_sync("deferred");
879 if let Some(client) = self.self_weak.get().and_then(|w| w.upgrade()) {
880 client.schedule_app_state_task_retry(name, false);
881 }
882 return Ok(());
883 }
884 Err(e) => {
885 if e.downcast_ref::<crate::appstate_sync::AppStateSyncError>()
886 .is_some_and(|ase| {
887 matches!(ase, crate::appstate_sync::AppStateSyncError::KeyNotFound(_))
888 })
889 && attempt == 1
890 {
891 if !self.initial_app_state_keys_received.load(Ordering::Relaxed) {
892 debug!(target: "Client/AppState", "App state key missing for {:?}; waiting up to 10s for key share then retrying", name);
893 if rt_timeout(
894 &*self.runtime,
895 Duration::from_secs(10),
896 self.initial_keys_synced_notifier.listen(),
897 )
898 .await
899 .is_err()
900 {
901 warn!(target: "Client/AppState", "Timeout waiting for key share for {:?}; retrying anyway", name);
902 }
903 }
904 continue;
905 }
906 let is_db_locked = e
907 .downcast_ref::<wacore::store::error::StoreError>()
908 .is_some_and(|se| se.is_database_busy_or_locked())
909 || e.downcast_ref::<crate::appstate_sync::AppStateSyncError>()
910 .is_some_and(|ase| match ase {
911 crate::appstate_sync::AppStateSyncError::Store(se) => {
912 se.is_database_busy_or_locked()
913 }
914 _ => false,
915 });
916 if is_db_locked && attempt < APP_STATE_RETRY_MAX_ATTEMPTS {
917 let backoff = Duration::from_millis(200 * attempt as u64 + 150);
918 warn!(target: "Client/AppState", "Attempt {} for {:?} failed due to locked DB; backing off {:?} and retrying", attempt, name, backoff);
919 self.runtime.sleep(backoff).await;
920 continue;
921 }
922 wacore::telemetry::appstate_sync("fail");
923 return Err(e);
924 }
925 }
926 }
927 }
928
929 pub(crate) fn report_background_sync(
951 self: &Arc<Self>,
952 label: &str,
953 scope: SyncScope,
954 settles: SyncSettles,
955 requested: &[WAPatchName],
956 result: Result<BatchedSyncOutcome>,
957 ) {
958 self.report_background_sync_stranded(label, scope, settles, requested, false, result)
959 }
960
961 pub(crate) fn report_background_sync_stranded(
968 self: &Arc<Self>,
969 label: &str,
970 scope: SyncScope,
971 settles: SyncSettles,
972 requested: &[WAPatchName],
973 stranded_elsewhere: bool,
974 result: Result<BatchedSyncOutcome>,
975 ) {
976 if let Err(lost) = self.admits(scope) {
977 debug!(target: "Client/AppState", "{label}: outcome dropped ({lost:?})");
978 return;
979 }
980 match result {
981 Ok(outcome) if outcome.all_synced() => {}
982 Ok(outcome) => {
983 warn!(
984 target: "Client/AppState",
985 "{label}: incomplete (fatal={:?} retryable={:?} skipped={:?})",
986 outcome.fatal, outcome.retryable, outcome.skipped
987 );
988 self.dispatch_app_state_sync_failed(
989 &outcome,
990 self.is_ready.load(Ordering::Relaxed),
991 );
992 let already_stranded =
998 stranded_elsewhere || !outcome.fatal.is_empty() || !outcome.skipped.is_empty();
999 self.schedule_app_state_retry(outcome.retryable, scope, settles, already_stranded);
1000 }
1001 Err(e) => {
1002 self.log_sync_error(label, &e);
1003 self.schedule_app_state_retry(
1014 requested.to_vec(),
1015 scope,
1016 settles,
1017 stranded_elsewhere,
1018 );
1019 }
1020 }
1021 }
1022
1023 pub(crate) async fn await_connection(&self) -> bool {
1036 loop {
1037 if let Some(verdict) = self.connection_wait_verdict() {
1038 return verdict;
1039 }
1040 let ready = self.socket_ready_notifier.listen();
1047 let session = self.session_state_notifier.listen();
1048 if let Some(verdict) = self.connection_wait_verdict() {
1049 return verdict;
1050 }
1051 futures::pin_mut!(ready);
1052 futures::pin_mut!(session);
1053 futures::future::select(ready, session).await;
1056 }
1057 }
1058
1059 fn connection_wait_verdict(&self) -> Option<bool> {
1061 if self.is_terminal() {
1066 return Some(false);
1067 }
1068 if self.can_reach_server() {
1069 return Some(true);
1070 }
1071 if !self.is_running.load(Ordering::Relaxed) {
1076 return Some(false);
1077 }
1078 None
1079 }
1080
1081 pub(crate) fn sync_scope(&self, deadline: Option<wacore::time::Instant>) -> SyncScope {
1083 SyncScope {
1084 generation: self.connection_generation.load(Ordering::SeqCst),
1085 deadline,
1086 }
1087 }
1088
1089 pub(crate) fn admits(&self, scope: SyncScope) -> Result<(), ScopeLost> {
1096 if self.connection_generation.load(Ordering::SeqCst) != scope.generation {
1097 return Err(ScopeLost::Retired);
1098 }
1099 if let Some(deadline) = scope.deadline
1100 && wacore::time::Instant::now() >= deadline
1101 {
1102 return Err(ScopeLost::Expired);
1103 }
1104 Ok(())
1105 }
1106
1107 pub(crate) fn settle_bootstrap(&self, scope: SyncScope, outstanding: bool) {
1115 if self.admits(scope) == Err(ScopeLost::Retired) {
1129 debug!(
1130 target: "Client/AppState",
1131 "Bootstrap gate left alone: connection {} retired", scope.generation
1132 );
1133 return;
1134 }
1135 if !self
1136 .needs_initial_full_sync
1137 .settle(scope.generation, outstanding)
1138 {
1139 debug!(
1140 target: "Client/AppState",
1141 "Bootstrap gate left to a newer connection than {}", scope.generation
1142 );
1143 return;
1144 }
1145 if outstanding {
1146 warn!(target: "Client/AppState", "Initial App State Sync incomplete; bootstrap stays armed");
1147 } else {
1148 debug!(target: "Client/AppState", "Initial App State Sync completed.");
1149 }
1150 }
1151
1152 pub(crate) fn dispatch_app_state_sync_failed(
1158 &self,
1159 outcome: &BatchedSyncOutcome,
1160 connected: bool,
1161 ) {
1162 let names = |v: &[WAPatchName]| v.iter().map(|n| n.as_str().to_string()).collect();
1163 self.core.event_bus.dispatch(Event::AppStateSyncFailed(
1164 crate::types::events::AppStateSyncFailed::builder()
1165 .fatal(names(&outcome.fatal))
1166 .retryable(names(&outcome.retryable))
1167 .skipped(names(&outcome.skipped))
1168 .connected(connected)
1169 .build(),
1170 ));
1171 }
1172
1173 fn schedule_app_state_task_retry(self: &Arc<Self>, name: WAPatchName, full_sync: bool) {
1181 let mut scope = self.sync_scope(None);
1182 let client = self.clone();
1183 self.runtime.spawn_detached(Box::pin(async move {
1184 let mut attempts = 0u32;
1189 for _ in 0..APP_STATE_RETRY_MAX_ROUNDS * APP_STATE_RETRY_ROUND_SLACK {
1190 if attempts >= APP_STATE_RETRY_MAX_ROUNDS {
1191 break;
1192 }
1193 client.runtime.sleep(app_state_retry_backoff(attempts)).await;
1194 if client.is_terminal() {
1195 debug!(target: "Client/AppState", "App state task retry cancelled: client is finished");
1196 return;
1197 }
1198 if !client.await_connection().await {
1208 debug!(target: "Client/AppState", "App state task retry cancelled: client is finished");
1209 return;
1210 }
1211
1212 scope.rebind(client.connection_generation.load(Ordering::SeqCst));
1222
1223 let guard = match client
1224 .reserve_for_sync(name, ReservationWait::Always, scope)
1225 .await
1226 {
1227 Ok(guard) => guard,
1228 Err(ReservationSkip::EquivalentSyncInFlight) => return,
1230 Err(ReservationSkip::WaitTimedOut) => {
1231 warn!(target: "Client/AppState", "Still waiting on the writer holding {name:?}");
1232 continue;
1233 }
1234 };
1235 if !client.can_reach_server() || client.admits(scope).is_err() {
1240 debug!(target: "Client/AppState", "Dropping the {name:?} attempt: state moved while reserving");
1241 drop(guard);
1242 continue;
1243 }
1244 attempts += 1;
1245 let outcome = client.process_app_state_sync_task(name, full_sync).await;
1246 if let Err(e) = &outcome {
1247 drop(guard);
1248 client.log_sync_error("app state task retry", e);
1249 }
1250 if !sync_still_owed(&outcome) && client.admits(scope).is_ok() {
1260 return;
1261 }
1262 debug!(target: "Client/AppState", "The {name:?} attempt did not settle it; keeping it queued");
1263 }
1264 warn!(
1265 target: "Client/AppState",
1266 "App state task for {name:?} still unsynced after {attempts} attempts"
1267 );
1268 }));
1269 }
1270
1271 pub(crate) fn schedule_app_state_retry(
1290 self: &Arc<Self>,
1291 collections: Vec<WAPatchName>,
1292 scope: SyncScope,
1293 settles: SyncSettles,
1294 already_stranded: bool,
1295 ) {
1296 if collections.is_empty() {
1297 return;
1298 }
1299 let client = self.clone();
1300 self.runtime.spawn_detached(Box::pin(async move {
1301 let mut scope = scope;
1302 let mut settles = settles;
1303 let mut pending = collections;
1304 let mut left_unresolved = already_stranded;
1311 let mut attempts = 0u32;
1317 for _ in 0..APP_STATE_RETRY_MAX_ROUNDS * APP_STATE_RETRY_ROUND_SLACK {
1318 if attempts >= APP_STATE_RETRY_MAX_ROUNDS {
1319 break;
1320 }
1321 client.runtime.sleep(app_state_retry_backoff(attempts)).await;
1322 if !client.await_connection().await {
1327 debug!(target: "Client/AppState", "App state retry cancelled: client is finished");
1328 return;
1329 }
1330
1331 if scope.rebind(client.connection_generation.load(Ordering::SeqCst)) {
1334 settles = SyncSettles::JustTheCollections;
1338 }
1339
1340 if !client.can_reach_server() {
1346 debug!(target: "Client/AppState", "Dropping the batched {pending:?} attempt: the connection is retiring");
1347 continue;
1348 }
1349
1350 debug!(
1351 target: "Client/AppState",
1352 "Retrying app state {pending:?} (attempt {}/{APP_STATE_RETRY_MAX_ROUNDS})",
1353 attempts + 1
1354 );
1355 let result = client
1356 .sync_collections_batched(pending.clone(), scope)
1357 .await;
1358 let reached_server = match &result {
1363 Ok(outcome) => outcome.reached_server(),
1364 Err(_) => true,
1365 };
1366 if reached_server {
1367 attempts += 1;
1368 }
1369
1370 if scope.rebind(client.connection_generation.load(Ordering::SeqCst)) {
1374 debug!(target: "Client/AppState", "App state retry outcome dropped; rebound");
1375 settles = SyncSettles::JustTheCollections;
1376 continue;
1377 }
1378
1379 match result {
1380 Ok(outcome) => {
1381 if !outcome.all_synced() {
1382 client.dispatch_app_state_sync_failed(
1383 &outcome,
1384 client.is_ready.load(Ordering::Relaxed),
1385 );
1386 }
1387 if !outcome.fatal.is_empty() || !outcome.skipped.is_empty() {
1388 left_unresolved = true;
1389 }
1390 pending = outcome.retryable;
1391 if pending.is_empty() {
1392 if settles == SyncSettles::InitialSync {
1393 client.settle_bootstrap(scope, left_unresolved);
1394 }
1395 return;
1396 }
1397 }
1398 Err(e) => client.log_sync_error("app state retry", &e),
1399 }
1400 }
1401 warn!(
1402 target: "Client/AppState",
1403 "App state {pending:?} still unsynced after {attempts} attempts; \
1404 leaving them to the next sync trigger"
1405 );
1406 if client.admits(scope).is_ok() {
1412 let exhausted = BatchedSyncOutcome {
1413 retryable: pending,
1414 ..Default::default()
1415 };
1416 client.dispatch_app_state_sync_failed(
1417 &exhausted,
1418 client.is_ready.load(Ordering::Relaxed),
1419 );
1420 }
1421 }));
1422 }
1423
1424 async fn reserve_for_sync(
1438 &self,
1439 name: WAPatchName,
1440 wait: ReservationWait,
1441 scope: SyncScope,
1442 ) -> Result<SyncInFlightGuard, ReservationSkip> {
1443 match self.app_state_syncing.try_begin_as(name, SyncHolder::Sync) {
1444 Ok(guard) => return Ok(guard),
1445 Err(SyncHolder::Sync) if wait == ReservationWait::SkipBehindSync => {
1446 return Err(ReservationSkip::EquivalentSyncInFlight);
1447 }
1448 Err(holder) => {
1449 debug!(target: "Client/AppState", "Waiting for the {holder:?} holding {name:?}");
1450 }
1451 }
1452 let bound = match scope.remaining() {
1453 Some(remaining) => APP_STATE_RESERVATION_WAIT.min(remaining),
1454 None => APP_STATE_RESERVATION_WAIT,
1455 };
1456 rt_timeout(
1457 &*self.runtime,
1458 bound,
1459 self.app_state_syncing.begin(name, SyncHolder::Sync),
1460 )
1461 .await
1462 .map_err(|_| ReservationSkip::WaitTimedOut)
1463 }
1464
1465 #[cfg_attr(feature = "tracing", tracing::instrument(name = "wa.appstate.sync_batched", level = "debug", skip_all, fields(count = collections.len()), err(Debug)))]
1476 pub(crate) async fn sync_collections_batched(
1477 &self,
1478 collections: Vec<WAPatchName>,
1479 scope: SyncScope,
1480 ) -> Result<BatchedSyncOutcome> {
1481 let mut outcome = BatchedSyncOutcome::default();
1482 if collections.is_empty() {
1483 return Ok(outcome);
1484 }
1485
1486 let mut seen = HashSet::with_capacity(collections.len());
1496 let collections: Vec<WAPatchName> = collections
1497 .into_iter()
1498 .filter(|name| seen.insert(*name))
1499 .collect();
1500
1501 let wait = if scope.is_bootstrap() {
1509 ReservationWait::Always
1510 } else {
1511 ReservationWait::SkipBehindSync
1512 };
1513
1514 let mut guards = Vec::with_capacity(collections.len());
1515 let mut pending = Vec::with_capacity(collections.len());
1516 for name in collections {
1517 if let Err(lost) = self.admits(scope) {
1521 warn!(target: "Client/AppState", "Not reserving {name:?}: {lost:?}");
1522 outcome.retryable.push(name);
1523 continue;
1524 }
1525 match self.reserve_for_sync(name, wait, scope).await {
1526 Ok(guard) => {
1527 guards.push(guard);
1528 pending.push(name);
1529 }
1530 Err(ReservationSkip::EquivalentSyncInFlight) => {
1535 debug!(target: "Client/AppState", "Skipping {name:?} in batch: an equivalent sync holds it");
1536 outcome.skipped.push(name);
1537 }
1538 Err(ReservationSkip::WaitTimedOut) => {
1539 warn!(target: "Client/AppState", "Gave up waiting for the writer holding {name:?}");
1540 outcome.retryable.push(name);
1541 }
1542 }
1543 }
1544
1545 if pending.is_empty() {
1546 return Ok(outcome);
1547 }
1548
1549 self.sync_collections_batched_inner(pending, scope, &mut outcome)
1550 .await?;
1551
1552 if !outcome.synced.is_empty()
1560 && let Err(lost @ ScopeLost::Expired) = self.admits(scope)
1561 {
1562 warn!(
1563 target: "Client/AppState",
1564 "Batched sync: {:?} applied but the run outlived its scope ({lost:?})",
1565 outcome.synced
1566 );
1567 let applied = std::mem::take(&mut outcome.synced);
1568 outcome.retryable.extend(applied);
1569 }
1570
1571 Ok(outcome)
1572 }
1573
1574 async fn sync_collections_batched_inner(
1575 &self,
1576 mut pending: Vec<WAPatchName>,
1577 scope: SyncScope,
1578 outcome: &mut BatchedSyncOutcome,
1579 ) -> Result<()> {
1580 use wacore::appstate::patch_decode::CollectionSyncError;
1581 const MAX_ITERATIONS: usize = 500;
1588 let mut iteration = 0;
1589
1590 while !pending.is_empty() && iteration < MAX_ITERATIONS {
1591 if let Err(lost) = self.admits(scope) {
1597 warn!(
1598 target: "Client/AppState",
1599 "Batched sync: stopping with {pending:?} still paging ({lost:?})"
1600 );
1601 outcome.retryable.extend(pending);
1602 return Ok(());
1603 }
1604 iteration += 1;
1605 debug!(
1606 target: "Client/AppState",
1607 "Batched sync iteration {}/{}: {:?}",
1608 iteration, MAX_ITERATIONS, pending
1609 );
1610
1611 let backend = self.persistence_manager.backend();
1612
1613 let mut collection_nodes = Vec::with_capacity(pending.len());
1615 let mut was_snapshot = HashSet::new();
1616 for &name in &pending {
1617 let state = backend.get_version(name.as_str()).await?;
1618 let want_snapshot = state.version == 0;
1619 if want_snapshot {
1620 was_snapshot.insert(name);
1621 }
1622 let mut builder = NodeBuilder::new("collection")
1623 .attr("name", name.as_str())
1624 .attr(
1625 "return_snapshot",
1626 if want_snapshot { "true" } else { "false" },
1627 );
1628 if !want_snapshot {
1629 builder = builder.attr("version", state.version);
1630 }
1631 collection_nodes.push(builder.build());
1632 }
1633
1634 let sync_node = NodeBuilder::new("sync").children(collection_nodes).build();
1635 let iq = crate::request::InfoQuery {
1636 namespace: "w:sync:app:state",
1637 query_type: crate::request::InfoQueryType::Set,
1638 to: server_jid().clone(),
1639 target: None,
1640 id: None,
1641 content: Some(wacore_binary::NodeContent::Nodes(vec![sync_node])),
1642 timeout: Some(Duration::from_secs(30)),
1643 };
1644
1645 outcome.note_reached_server();
1649 let resp = self.send_iq(iq).await?;
1650
1651 if let Err(lost) = self.admits(scope) {
1654 warn!(
1655 target: "Client/AppState",
1656 "Batched sync: dropping the response for {pending:?} ({lost:?})"
1657 );
1658 outcome.retryable.extend(pending);
1659 return Ok(());
1660 }
1661
1662 let mut patch_lists =
1665 wacore::appstate::patch_decode::parse_patch_lists_ref(resp.get())?;
1666
1667 {
1678 let requested: HashSet<WAPatchName> = pending.iter().copied().collect();
1679 let mut seen: HashSet<WAPatchName> = HashSet::new();
1680 patch_lists.retain(|pl| {
1681 if !requested.contains(&pl.name) {
1682 warn!(
1683 target: "Client/AppState",
1684 "Batched sync: response carried unrequested collection {:?}; dropping it",
1685 pl.name
1686 );
1687 return false;
1688 }
1689 if seen.insert(pl.name) {
1690 return true;
1691 }
1692 warn!(
1693 target: "Client/AppState",
1694 "Batched sync: response repeated collection {:?}; dropping the duplicate",
1695 pl.name
1696 );
1697 false
1698 });
1699 }
1700
1701 let proc = self.get_app_state_processor().await;
1702 let pre_downloaded = self.pre_download_external_blobs(&patch_lists).await;
1705
1706 let download = |ext: &wa::ExternalBlobReference| -> Result<Vec<u8>> {
1707 if let Some(path) = &ext.direct_path {
1708 if let Some(bytes) = pre_downloaded.get(path) {
1709 Ok(bytes.clone())
1710 } else {
1711 Err(anyhow::anyhow!(
1712 "external blob not pre-downloaded: {}",
1713 path
1714 ))
1715 }
1716 } else {
1717 Err(anyhow::anyhow!("external blob has no directPath"))
1718 }
1719 };
1720
1721 let mut missing_all: Vec<Vec<u8>> = Vec::new();
1728 for pl in &mut patch_lists {
1729 if let Ok(m) = proc.missing_key_ids_after_inline(pl, &download).await {
1730 missing_all.extend(m);
1731 }
1732 }
1733 let key_wait = scope.remaining().unwrap_or(APP_STATE_KEY_REQUEST_TIMEOUT);
1737 if !missing_all.is_empty() && !self.request_keys_and_wait(missing_all, key_wait).await {
1738 warn!(
1744 target: "Client/AppState",
1745 "Batched sync: decode key(s) still missing after re-request, deferring {pending:?}"
1746 );
1747 outcome.retryable.extend(pending);
1748 return Ok(());
1749 }
1750
1751 if let Err(lost) = self.admits(scope) {
1756 warn!(
1757 target: "Client/AppState",
1758 "Batched sync: not applying {pending:?} ({lost:?})"
1759 );
1760 outcome.retryable.extend(pending);
1761 return Ok(());
1762 }
1763
1764 let mut results = Vec::with_capacity(patch_lists.len());
1771 for pl in patch_lists {
1772 if let Err(lost) = self.admits(scope) {
1773 warn!(
1774 target: "Client/AppState",
1775 "Batched sync: stopping before {:?} ({lost:?})", pl.name
1776 );
1777 break;
1778 }
1779 results.push(proc.process_one_patch_list(pl, &download, true).await?);
1780 }
1781
1782 let mut needs_refetch = Vec::new();
1783 let mut answered: HashSet<WAPatchName> = HashSet::new();
1790
1791 for (mutations, new_state, list) in results {
1792 let name = list.name;
1793 answered.insert(name);
1794
1795 if let Some(ref err) = list.error {
1807 match err {
1808 CollectionSyncError::Conflict { has_more } => {
1809 if *has_more {
1810 warn!(target: "Client/AppState", "Collection {:?} conflict (has_more=true), will refetch", name);
1812 needs_refetch.push(name);
1813 } else {
1814 debug!(target: "Client/AppState", "Collection {:?} conflict (has_more=false), treating as success (no pending mutations)", name);
1818 outcome.synced.push(name);
1819 }
1820 continue;
1821 }
1822 CollectionSyncError::Fatal { code, text } => {
1823 warn!(target: "Client/AppState", "Collection {:?} fatal error {}: {}", name, code, text);
1824 outcome.fatal.push(name);
1825 continue;
1826 }
1827 CollectionSyncError::Retry { code, text } => {
1828 warn!(target: "Client/AppState", "Collection {:?} retryable error {}: {}", name, code, text);
1835 outcome.retryable.push(name);
1836 continue;
1837 }
1838 }
1839 }
1840
1841 let missing = match proc.get_missing_key_ids(&list).await {
1843 Ok(v) => v,
1844 Err(e) => {
1845 warn!("Failed to get missing key IDs for {:?}: {}", name, e);
1846 Vec::new()
1847 }
1848 };
1849 self.request_missing_keys_with_dedup(&missing, APP_STATE_KEY_REQUEST_DEDUP)
1850 .await;
1851
1852 let full_sync = was_snapshot.contains(&name);
1856 wacore::telemetry::appstate_mutations(mutations.len() as u64);
1857 for m in mutations {
1858 self.dispatch_app_state_mutation(&m, full_sync).await;
1859 }
1860
1861 backend
1863 .set_version(name.as_str(), new_state.clone())
1864 .await?;
1865
1866 if list.has_more_patches {
1868 needs_refetch.push(name);
1869 } else {
1870 outcome.synced.push(name);
1871 }
1872
1873 debug!(
1874 target: "Client/AppState",
1875 "Batched sync: {:?} done (version={}, has_more={})",
1876 name, new_state.version, list.has_more_patches
1877 );
1878 }
1879
1880 for name in pending {
1884 if !answered.contains(&name) {
1885 warn!(
1886 target: "Client/AppState",
1887 "Batched sync: response omitted collection {name:?}"
1888 );
1889 outcome.retryable.push(name);
1890 }
1891 }
1892
1893 pending = needs_refetch;
1894 }
1895
1896 if !pending.is_empty() {
1897 warn!(
1902 target: "Client/AppState",
1903 "Batched sync: max iterations ({}) reached for {:?}",
1904 MAX_ITERATIONS, pending
1905 );
1906 outcome.retryable.extend(pending);
1907 }
1908
1909 Ok(())
1910 }
1911
1912 #[cfg_attr(feature = "tracing", tracing::instrument(name = "wa.appstate.sync", level = "debug", skip_all, fields(name = ?name, full_sync = full_sync), err(Debug)))]
1913 pub(crate) async fn process_app_state_sync_task(
1914 &self,
1915 name: WAPatchName,
1916 full_sync: bool,
1917 ) -> Result<SyncOutcome> {
1918 if self.is_terminal() || !self.can_reach_server() {
1923 debug!(
1924 target: "Client/AppState",
1925 "Skipping app state sync task {name:?}: no usable connection"
1926 );
1927 return Ok(SyncOutcome::Deferred);
1928 }
1929
1930 let backend = self.persistence_manager.backend();
1931 let mut full_sync = full_sync;
1932
1933 let mut state = backend.get_version(name.as_str()).await?;
1934 if state.version == 0 {
1935 full_sync = true;
1936 }
1937
1938 let mut has_more = true;
1939 let mut want_snapshot = full_sync;
1940 const MAX_PAGINATION_ITERATIONS: u32 = 500;
1943 let mut iteration = 0u32;
1944 let mut outcome = SyncOutcome::Completed;
1948
1949 while has_more {
1950 if self.is_terminal() || !self.can_reach_server() {
1951 debug!(target: "Client/AppState", "Stopping app state sync task {name:?}: no usable connection");
1952 outcome = SyncOutcome::Deferred;
1953 break;
1954 }
1955 iteration += 1;
1956 if iteration > MAX_PAGINATION_ITERATIONS {
1957 warn!(target: "Client/AppState", "App state sync for {:?} exceeded {} iterations, aborting", name, MAX_PAGINATION_ITERATIONS);
1958 outcome = SyncOutcome::Deferred;
1969 break;
1970 }
1971 debug!(target: "Client/AppState", "Fetching app state patch batch: name={:?} want_snapshot={want_snapshot} version={} full_sync={} has_more_previous={}", name, state.version, full_sync, has_more);
1972
1973 let mut collection_builder = NodeBuilder::new("collection")
1974 .attr("name", name.as_str())
1975 .attr(
1976 "return_snapshot",
1977 if want_snapshot { "true" } else { "false" },
1978 );
1979 if !want_snapshot {
1980 collection_builder = collection_builder.attr("version", state.version);
1981 }
1982 let sync_node = NodeBuilder::new("sync")
1983 .children([collection_builder.build()])
1984 .build();
1985 let iq = crate::request::InfoQuery {
1986 namespace: "w:sync:app:state",
1987 query_type: crate::request::InfoQueryType::Set,
1988 to: server_jid().clone(),
1989 target: None,
1990 id: None,
1991 content: Some(wacore_binary::NodeContent::Nodes(vec![sync_node])),
1992 timeout: None,
1993 };
1994
1995 let resp = self.send_iq(iq).await?;
1996 if self.is_terminal() || !self.can_reach_server() {
1997 debug!(target: "Client/AppState", "Discarding app state sync response for {name:?}: no usable connection");
1998 outcome = SyncOutcome::Deferred;
1999 break;
2000 }
2001 debug!(target: "Client/AppState", "Received IQ response for {:?}; decoding patches", name);
2002
2003 let _decode_start = wacore::time::Instant::now();
2004
2005 let mut pl = wacore::appstate::patch_decode::parse_patch_list_ref(resp.get())?;
2008 debug!(target: "Client/AppState", "Parsed patch list for {:?}: has_snapshot_ref={} has_more_patches={} patches_count={}",
2009 name, pl.snapshot_ref.is_some(), pl.has_more_patches, pl.patches.len());
2010
2011 let proc = self.get_app_state_processor().await;
2012
2013 let pre_downloaded = self
2016 .pre_download_external_blobs(std::slice::from_ref(&pl))
2017 .await;
2018
2019 let download = |ext: &wa::ExternalBlobReference| -> Result<Vec<u8>> {
2020 if let Some(path) = &ext.direct_path {
2021 if let Some(bytes) = pre_downloaded.get(path) {
2022 Ok(bytes.clone())
2023 } else {
2024 Err(anyhow::anyhow!(
2025 "external blob not pre-downloaded: {}",
2026 path
2027 ))
2028 }
2029 } else {
2030 Err(anyhow::anyhow!("external blob has no directPath"))
2031 }
2032 };
2033
2034 let missing = proc
2040 .missing_key_ids_after_inline(&mut pl, &download)
2041 .await
2042 .unwrap_or_default();
2043 if !missing.is_empty()
2044 && !self
2045 .request_keys_and_wait(missing, APP_STATE_KEY_REQUEST_TIMEOUT)
2046 .await
2047 {
2048 return Err(anyhow::anyhow!(
2052 "app-state decode key(s) for {name:?} still missing after re-request; deferring sync"
2053 ));
2054 }
2055
2056 let (mutations, new_state, list) =
2057 proc.process_parsed_patch_list(pl, &download, true).await?;
2058 let decode_elapsed = _decode_start.elapsed();
2059 if decode_elapsed.as_millis() > 500 {
2060 debug!(target: "Client/AppState", "Patch decode for {:?} took {:?}", name, decode_elapsed);
2061 }
2062
2063 let missing = match proc.get_missing_key_ids(&list).await {
2064 Ok(v) => v,
2065 Err(e) => {
2066 warn!("Failed to get missing key IDs for {:?}: {}", name, e);
2067 Vec::new()
2068 }
2069 };
2070 self.request_missing_keys_with_dedup(&missing, APP_STATE_KEY_REQUEST_DEDUP)
2071 .await;
2072
2073 wacore::telemetry::appstate_mutations(mutations.len() as u64);
2074 for m in mutations {
2075 debug!(target: "Client/AppState", "Dispatching mutation kind={} index_len={} full_sync={}", m.index.first().map(|s| s.as_str()).unwrap_or(""), m.index.len(), full_sync);
2076 self.dispatch_app_state_mutation(&m, full_sync).await;
2077 }
2078
2079 state = new_state;
2080 has_more = list.has_more_patches;
2081 want_snapshot = false;
2083 debug!(target: "Client/AppState", "After processing batch name={:?} has_more={has_more} new_version={}", name, state.version);
2084 }
2085
2086 backend.set_version(name.as_str(), state.clone()).await?;
2087
2088 debug!(target: "Client/AppState", "Finished app state sync for {name:?} as {outcome:?} (final version={})", state.version);
2089 Ok(outcome)
2090 }
2091
2092 async fn request_keys_and_wait(&self, mut missing: Vec<Vec<u8>>, timeout: Duration) -> bool {
2102 if missing.is_empty() {
2103 return true;
2104 }
2105 let deadline = wacore::time::Instant::now() + timeout;
2106 let backend = self.persistence_manager.backend();
2107 let mut retry_after = initial_app_state_key_retry(timeout);
2108 loop {
2109 let listener = self.initial_keys_synced_notifier.listen();
2110 remove_available_app_state_keys(&*backend, &mut missing).await;
2111 if missing.is_empty() {
2112 return true;
2113 }
2114
2115 let request = self.request_missing_keys_with_dedup(&missing, retry_after);
2116 let schedule = match self
2117 .await_app_state_key_request(&*backend, &missing, deadline, listener, request)
2118 .await
2119 {
2120 AppStateKeyRequestProgress::Scheduled(schedule) => schedule,
2121 AppStateKeyRequestProgress::KeysReady => return true,
2122 AppStateKeyRequestProgress::TimedOut => return false,
2123 };
2124 if schedule.sent {
2125 debug!(target: "Client/AppState", "Requested {} missing app-state key(s); retrying after {retry_after:?} if no share arrives", missing.len());
2126 retry_after = retry_after.saturating_mul(2).min(APP_STATE_KEY_RETRY_MAX);
2127 }
2128
2129 let listener = self.initial_keys_synced_notifier.listen();
2130 remove_available_app_state_keys(&*backend, &mut missing).await;
2131 if missing.is_empty() {
2132 return true;
2133 }
2134
2135 let remaining = deadline.saturating_duration_since(wacore::time::Instant::now());
2136 if remaining.is_zero() {
2137 return false;
2138 }
2139
2140 let retry_wait = schedule
2141 .retry_at
2142 .saturating_duration_since(wacore::time::Instant::now());
2143 let wait = remaining.min(retry_wait);
2144 if !wait.is_zero() {
2145 let _ = rt_timeout(&*self.runtime, wait, listener).await;
2146 }
2147 }
2148 }
2149
2150 async fn await_app_state_key_request<F>(
2151 &self,
2152 backend: &dyn crate::store::traits::Backend,
2153 missing: &[Vec<u8>],
2154 deadline: wacore::time::Instant,
2155 mut listener: event_listener::EventListener,
2156 request: F,
2157 ) -> AppStateKeyRequestProgress
2158 where
2159 F: Future<Output = AppStateKeyRequestSchedule>,
2160 {
2161 futures::pin_mut!(request);
2162 loop {
2163 let remaining = deadline.saturating_duration_since(wacore::time::Instant::now());
2164 if remaining.is_zero() {
2165 return if app_state_keys_available(backend, missing).await {
2166 AppStateKeyRequestProgress::KeysReady
2167 } else {
2168 AppStateKeyRequestProgress::TimedOut
2169 };
2170 }
2171
2172 let notified = rt_timeout(&*self.runtime, remaining, listener);
2173 futures::pin_mut!(notified);
2174 match futures::future::select(request.as_mut(), notified.as_mut()).await {
2175 futures::future::Either::Left((schedule, _)) => {
2176 return AppStateKeyRequestProgress::Scheduled(schedule);
2177 }
2178 futures::future::Either::Right((notification, _)) => {
2179 let next_listener = self.initial_keys_synced_notifier.listen();
2180 if app_state_keys_available(backend, missing).await {
2181 return AppStateKeyRequestProgress::KeysReady;
2182 }
2183 if notification.is_err() {
2184 return AppStateKeyRequestProgress::TimedOut;
2185 }
2186 listener = next_listener;
2187 }
2188 }
2189 }
2190 }
2191
2192 async fn request_missing_keys_with_dedup(
2195 &self,
2196 missing: &[Vec<u8>],
2197 retry_after: Duration,
2198 ) -> AppStateKeyRequestSchedule {
2199 if missing.is_empty() {
2200 return AppStateKeyRequestSchedule {
2201 retry_at: wacore::time::Instant::now() + retry_after,
2202 sent: false,
2203 };
2204 }
2205 let mut guard = self.app_state_key_requests.lock().await;
2206 let now = wacore::time::Instant::now();
2207 let requested_retry_at = now + retry_after;
2208 guard.retain(|_, retry_at| now < *retry_at);
2209
2210 let mut to_request: Option<Vec<&[u8]>> = None;
2211 let mut next_retry_at = requested_retry_at;
2212 for key_id in missing {
2213 if let Some(retry_at) = guard.get_mut(key_id.as_slice()) {
2214 if *retry_at > requested_retry_at {
2215 *retry_at = requested_retry_at;
2216 }
2217 next_retry_at = next_retry_at.min(*retry_at);
2218 } else {
2219 guard.insert(key_id.clone(), requested_retry_at);
2220 to_request
2221 .get_or_insert_with(|| Vec::with_capacity(missing.len()))
2222 .push(key_id.as_slice());
2223 }
2224 }
2225 drop(guard);
2226
2227 let Some(to_request) = to_request else {
2228 return AppStateKeyRequestSchedule {
2229 retry_at: next_retry_at,
2230 sent: false,
2231 };
2232 };
2233
2234 match self
2235 .request_app_state_keys(&to_request, retry_after.min(APP_STATE_KEY_REQUEST_TIMEOUT))
2236 .await
2237 {
2238 Ok(AppStateKeyRequestDelivery::AllPeers) => AppStateKeyRequestSchedule {
2239 retry_at: next_retry_at,
2240 sent: true,
2241 },
2242 Ok(AppStateKeyRequestDelivery::SomePeers) => {
2243 let retry_at = wacore::time::Instant::now() + APP_STATE_KEY_PARTIAL_RETRY;
2244 let mut guard = self.app_state_key_requests.lock().await;
2245 for key_id in &to_request {
2246 if let Some(deadline) = guard.get_mut(*key_id) {
2247 *deadline = (*deadline).min(retry_at);
2248 }
2249 }
2250 AppStateKeyRequestSchedule {
2251 retry_at: next_retry_at.min(retry_at),
2252 sent: true,
2253 }
2254 }
2255 Err(e) => {
2256 warn!("Failed to send app state key request: {e}");
2257 let mut guard = self.app_state_key_requests.lock().await;
2258 for key_id in &to_request {
2259 if guard
2260 .get(*key_id)
2261 .is_some_and(|deadline| *deadline == requested_retry_at)
2262 {
2263 guard.remove(*key_id);
2264 }
2265 }
2266 AppStateKeyRequestSchedule {
2267 retry_at: requested_retry_at,
2268 sent: false,
2269 }
2270 }
2271 }
2272 }
2273
2274 async fn app_state_key_request_peers(&self) -> Result<Vec<Jid>, anyhow::Error> {
2275 let device_snapshot = self.persistence_manager.get_device_snapshot();
2276 let own_jid = device_snapshot
2277 .pn
2278 .as_ref()
2279 .ok_or_else(|| anyhow::anyhow!("no own JID available for app-state key request"))?;
2280 let current_device = own_jid.device;
2281 let primary = own_jid.to_non_ad();
2282 drop(device_snapshot);
2283
2284 let peers = match self.get_user_devices(std::slice::from_ref(&primary)).await {
2285 Ok(devices) => devices,
2286 Err(error) => {
2287 warn!(
2288 "Own device-list query failed; requesting app-state keys from primary only: {error}"
2289 );
2290 Vec::new()
2291 }
2292 };
2293 finalize_app_state_key_request_peers(peers, current_device, primary)
2294 }
2295
2296 #[cfg_attr(feature = "tracing", tracing::instrument(name = "wa.appstate.request_keys", level = "debug", skip_all, fields(count = raw_key_ids.len()), err(Debug)))]
2297 async fn request_app_state_keys(
2298 &self,
2299 raw_key_ids: &[&[u8]],
2300 fanout_timeout: Duration,
2301 ) -> Result<AppStateKeyRequestDelivery, anyhow::Error> {
2302 if raw_key_ids.is_empty() {
2303 return Ok(AppStateKeyRequestDelivery::AllPeers);
2304 }
2305 let peers = self.app_state_key_request_peers().await?;
2306 let key_ids: Vec<wa::message::AppStateSyncKeyId> = raw_key_ids
2307 .iter()
2308 .map(|k| wa::message::AppStateSyncKeyId {
2309 key_id: Some(k.to_vec()),
2310 })
2311 .collect();
2312 let msg = wa::Message {
2313 protocol_message: buffa::MessageField::some(wa::message::ProtocolMessage {
2314 r#type: Some(wa::message::protocol_message::Type::AppStateSyncKeyRequest),
2315 app_state_sync_key_request: buffa::MessageField::some(
2316 wa::message::AppStateSyncKeyRequest { key_ids },
2317 ),
2318 ..Default::default()
2319 }),
2320 ..Default::default()
2321 };
2322
2323 let requests = futures::stream::FuturesUnordered::new();
2324 for peer in peers {
2325 let msg = &msg;
2326 requests.push(async move {
2327 let device = peer.device;
2328 let result = async {
2329 self.ensure_e2e_sessions(std::slice::from_ref(&peer))
2330 .await?;
2331 let request_id = self.generate_message_id();
2332 self.send_message_impl(
2333 peer,
2334 msg,
2335 crate::send::SendPipelineOptions {
2336 request_id: Some(&request_id),
2337 peer: true,
2338 ..Default::default()
2339 },
2340 )
2341 .await
2342 }
2343 .await;
2344 (device, result)
2345 });
2346 }
2347
2348 collect_app_state_key_request_results(&*self.runtime, requests, fanout_timeout).await
2349 }
2350
2351 #[cfg_attr(feature = "tracing", tracing::instrument(name = "wa.appstate.send_patch", level = "debug", skip_all, fields(name = %collection_name, count = mutations.len()), err(Debug)))]
2365 pub(crate) async fn send_app_state_patch(
2366 &self,
2367 collection_name: &str,
2368 mutations: Vec<wa::SyncdMutation>,
2369 ) -> Result<()> {
2370 use wacore::appstate::patch_decode::CollectionSyncError;
2371
2372 let patch_name = collection_name.parse::<WAPatchName>().ok();
2373 let _send_guard = self.app_state_send_lock.lock().await;
2381 let _collection_guard = match patch_name {
2390 Some(name) => Some(
2391 self.app_state_syncing
2392 .begin(name, SyncHolder::PatchSend)
2393 .await,
2394 ),
2395 None => None,
2396 };
2397 let proc = self.get_app_state_processor().await;
2398
2399 for attempt in 1..=APP_STATE_PATCH_SEND_ATTEMPTS {
2400 let (patch_bytes, base_version) =
2404 proc.build_patch(collection_name, mutations.clone()).await?;
2405
2406 let collection_node = NodeBuilder::new("collection")
2407 .attr("name", collection_name)
2408 .attr("version", base_version)
2409 .attr("return_snapshot", "false")
2410 .children([NodeBuilder::new("patch").bytes(patch_bytes).build()])
2411 .build();
2412 let sync_node = NodeBuilder::new("sync").children([collection_node]).build();
2413 let iq = crate::request::InfoQuery {
2414 namespace: "w:sync:app:state",
2415 query_type: crate::request::InfoQueryType::Set,
2416 to: server_jid().clone(),
2417 target: None,
2418 id: None,
2419 content: Some(wacore_binary::NodeContent::Nodes(vec![sync_node])),
2420 timeout: None,
2421 };
2422
2423 let resp = self.send_iq(iq).await?;
2424 let resp = resp.get().to_owned();
2425 let list = match wacore::appstate::patch_decode::parse_patch_list(&resp) {
2433 Ok(list) => list,
2434 Err(e)
2435 if resp
2436 .get_optional_child_by_tag(&["sync", "collection"])
2437 .is_none() =>
2438 {
2439 debug!(
2440 target: "Client/AppState",
2441 "Patch response for {collection_name} carried no collection verdict ({e}); treating as accepted"
2442 );
2443 wacore::appstate::patch_decode::PatchList {
2444 name: patch_name.unwrap_or(WAPatchName::Unknown),
2445 has_more_patches: false,
2446 patches: Vec::new(),
2447 snapshot: None,
2448 snapshot_ref: None,
2449 error: None,
2450 }
2451 }
2452 Err(e) => {
2453 return Err(e.context(format!(
2454 "unreadable app-state patch response for {collection_name}"
2455 )));
2456 }
2457 };
2458 if Some(list.name) != patch_name {
2459 return Err(anyhow::anyhow!(
2460 "app-state patch response collection mismatch: requested {collection_name}, got {}",
2461 list.name.as_str()
2462 ));
2463 }
2464
2465 match list.error {
2466 None => {
2467 if let Some(patch_name) = patch_name
2470 && let Err(e) = self.fetch_app_state_with_retry_inner(patch_name).await
2471 {
2472 log::warn!("Failed to re-sync {collection_name} after patch send: {e}");
2473 }
2474 return Ok(());
2475 }
2476 Some(CollectionSyncError::Conflict { has_more }) => {
2477 warn!(
2478 target: "Client/AppState",
2479 "Patch for {collection_name} conflicted on v{base_version} \
2480 (attempt {attempt}/{APP_STATE_PATCH_SEND_ATTEMPTS}, has_more={has_more}); \
2481 applying the conflicting patches and rebuilding"
2482 );
2483 self.absorb_conflicting_patches(collection_name, patch_name, list, has_more)
2484 .await;
2485 }
2486 Some(error) => {
2487 return Err(anyhow::anyhow!(
2488 "app-state patch for {collection_name} rejected: {error}"
2489 ));
2490 }
2491 }
2492 }
2493
2494 Err(anyhow::anyhow!(
2495 "app-state patch for {collection_name} still conflicting after \
2496 {APP_STATE_PATCH_SEND_ATTEMPTS} attempts"
2497 ))
2498 }
2499
2500 async fn absorb_conflicting_patches(
2509 &self,
2510 collection_name: &str,
2511 patch_name: Option<WAPatchName>,
2512 mut list: wacore::appstate::patch_decode::PatchList,
2513 has_more: bool,
2514 ) {
2515 list.error = None;
2518 let applied = if list.patches.is_empty() && list.snapshot_ref.is_none() {
2519 false
2520 } else {
2521 let pre_downloaded = self
2522 .pre_download_external_blobs(std::slice::from_ref(&list))
2523 .await;
2524 let download = |ext: &wa::ExternalBlobReference| -> Result<Vec<u8>> {
2525 let path = ext
2526 .direct_path
2527 .as_ref()
2528 .ok_or_else(|| anyhow::anyhow!("external blob has no directPath"))?;
2529 pre_downloaded
2530 .get(path)
2531 .cloned()
2532 .ok_or_else(|| anyhow::anyhow!("external blob not pre-downloaded: {path}"))
2533 };
2534 let proc = self.get_app_state_processor().await;
2535 match proc.process_parsed_patch_list(list, &download, true).await {
2536 Ok((mutations, _, _)) => {
2537 wacore::telemetry::appstate_mutations(mutations.len() as u64);
2538 for m in &mutations {
2539 self.dispatch_app_state_mutation(m, false).await;
2540 }
2541 true
2542 }
2543 Err(e) => {
2544 warn!(
2545 target: "Client/AppState",
2546 "Failed to apply the patches {collection_name} conflicted with: {e:#}"
2547 );
2548 false
2549 }
2550 }
2551 };
2552
2553 if (!applied || has_more)
2556 && let Some(patch_name) = patch_name
2557 && let Err(e) = self.fetch_app_state_with_retry_inner(patch_name).await
2558 {
2559 warn!(
2560 target: "Client/AppState",
2561 "Failed to re-sync {collection_name} after a patch conflict: {e}"
2562 );
2563 }
2564 }
2565
2566 async fn dispatch_app_state_mutation(
2567 &self,
2568 m: &crate::appstate_sync::Mutation,
2569 full_sync: bool,
2570 ) {
2571 use wacore::types::events::Event;
2572
2573 if m.index.is_empty() {
2574 return;
2575 }
2576
2577 if m.index[0] == "nct_salt_sync" {
2580 if m.operation == wa::syncd_mutation::SyncdOperation::Remove {
2581 debug!(target: "Client/AppState", "Removing NCT salt via app state sync");
2582 self.persistence_manager
2583 .process_command(DeviceCommand::SetNctSalt(None))
2584 .await;
2585 } else if let Some(val) = &m.action_value
2586 && let Some(act) = val.nct_salt_sync_action.as_option()
2587 && let Some(salt) = &act.salt
2588 {
2589 if salt.is_empty() {
2590 warn!(target: "Client/AppState", "nct_salt_sync mutation has empty salt, ignoring");
2591 } else {
2592 debug!(target: "Client/AppState", "Stored NCT salt via app state sync ({} bytes)", salt.len());
2593 self.persistence_manager
2594 .process_command(DeviceCommand::SetNctSalt(Some(salt.clone())))
2595 .await;
2596 }
2597 } else {
2598 warn!(target: "Client/AppState", "nct_salt_sync mutation missing salt in action value");
2599 }
2600 return;
2601 }
2602
2603 if m.operation != wa::syncd_mutation::SyncdOperation::Set {
2605 return;
2606 }
2607
2608 if crate::features::chat_actions::dispatch_chat_mutation(&self.core.event_bus, m, full_sync)
2610 {
2611 return;
2612 }
2613
2614 if crate::features::labels::dispatch_label_mutation(&self.core.event_bus, m, full_sync) {
2617 return;
2618 }
2619
2620 if m.index[0] == "setting_pushName"
2622 && let Some(val) = &m.action_value
2623 && let Some(act) = val.push_name_setting.as_option()
2624 && let Some(new_name) = &act.name
2625 {
2626 let new_name = new_name.clone();
2627 let bus = self.core.event_bus.clone();
2628
2629 let snapshot = self.persistence_manager.get_device_snapshot();
2630 let old = snapshot.push_name.clone();
2631 if old != new_name {
2632 debug!(target: "Client/AppState", "Persisting push name from app state mutation: '{}' (old='{}')", new_name, old);
2633 self.persistence_manager
2634 .process_command(DeviceCommand::SetPushName(new_name.clone()))
2635 .await;
2636 bus.dispatch(Event::SelfPushNameUpdated(
2637 crate::types::events::SelfPushNameUpdated::builder()
2638 .from_server(true)
2639 .old_name(old.clone())
2640 .new_name(new_name.clone())
2641 .build(),
2642 ));
2643
2644 if old.is_empty() && !new_name.is_empty() {
2646 debug!(target: "Client/AppState", "Sending presence after receiving initial pushname from app state sync");
2647 if let Err(e) = self.presence().set_available().await {
2648 warn!(target: "Client/AppState", "Failed to send presence after pushname sync: {e:?}");
2649 }
2650 }
2651 } else {
2652 debug!(target: "Client/AppState", "Push name mutation received but name unchanged: '{}'", new_name);
2653 }
2654 }
2655 }
2656
2657 #[cfg_attr(feature = "tracing", tracing::instrument(name = "wa.appstate.clean_dirty", level = "debug", skip_all, fields(bit = ?bit), err(Debug)))]
2658 pub async fn clean_dirty_bits(
2659 &self,
2660 bit: wacore::iq::dirty::DirtyBit,
2661 ) -> Result<(), crate::request::IqError> {
2662 use wacore::iq::dirty::CleanDirtyBitsSpec;
2663
2664 let spec = CleanDirtyBitsSpec::single(bit);
2665 self.execute(spec).await
2666 }
2667}
2668
2669#[cfg(test)]
2670mod tests {
2671 use super::*;
2672
2673 #[tokio::test]
2674 async fn key_arrival_finishes_before_a_slow_fanout() {
2675 let client = crate::test_utils::create_test_client_with_name("appstate_slow_peer").await;
2676 let backend = client.persistence_manager.backend();
2677 let key_id = vec![7, 8, 9, 10];
2678 let listener = client.initial_keys_synced_notifier.listen();
2679 let notifier = client.initial_keys_synced_notifier.clone();
2680 let writer = backend.clone();
2681 let stored_id = key_id.clone();
2682 let (fanout_polled_tx, fanout_polled_rx) = tokio::sync::oneshot::channel();
2683 tokio::spawn(async move {
2684 fanout_polled_rx.await.expect("fanout must be polled");
2685 writer
2686 .set_sync_key(
2687 &stored_id,
2688 crate::store::traits::AppStateSyncKey {
2689 key_data: vec![7; 32],
2690 ..Default::default()
2691 },
2692 )
2693 .await
2694 .expect("store recovered key");
2695 notifier.notify(usize::MAX);
2696 });
2697
2698 let slow_fanout = async move {
2699 let _ = fanout_polled_tx.send(());
2700 std::future::pending::<AppStateKeyRequestSchedule>().await
2701 };
2702
2703 let progress = client
2704 .await_app_state_key_request(
2705 &*backend,
2706 std::slice::from_ref(&key_id),
2707 wacore::time::Instant::now() + Duration::from_secs(1),
2708 listener,
2709 slow_fanout,
2710 )
2711 .await;
2712
2713 assert!(matches!(progress, AppStateKeyRequestProgress::KeysReady));
2714 }
2715
2716 #[tokio::test]
2717 async fn passive_key_request_fanout_is_bounded() {
2718 async fn peer_request(
2719 device: u16,
2720 completes: bool,
2721 ) -> (u16, std::result::Result<(), anyhow::Error>) {
2722 if !completes {
2723 std::future::pending::<()>().await;
2724 }
2725 (device, Ok(()))
2726 }
2727
2728 let client =
2729 crate::test_utils::create_test_client_with_name("appstate_fanout_timeout").await;
2730 let requests = futures::stream::FuturesUnordered::new();
2731 requests.push(peer_request(1, true));
2732 requests.push(peer_request(2, false));
2733
2734 let delivery = tokio::time::timeout(
2735 Duration::from_secs(1),
2736 collect_app_state_key_request_results(
2737 &*client.runtime,
2738 requests,
2739 Duration::from_millis(20),
2740 ),
2741 )
2742 .await
2743 .expect("fanout collection must finish")
2744 .expect("one completed peer must preserve partial delivery");
2745
2746 assert_eq!(delivery, AppStateKeyRequestDelivery::SomePeers);
2747 }
2748
2749 #[test]
2750 fn empty_companion_discovery_falls_back_to_primary() {
2751 let primary: Jid = "5511000000000@s.whatsapp.net".parse().expect("primary jid");
2752 let peers = finalize_app_state_key_request_peers(Vec::new(), 7, primary.clone())
2753 .expect("companion fallback");
2754 assert_eq!(peers, vec![primary.clone()]);
2755 assert!(finalize_app_state_key_request_peers(Vec::new(), 0, primary).is_err());
2756 }
2757
2758 #[test]
2759 fn app_state_peers_use_the_own_pn_namespace() {
2760 let primary = Jid::pn("5511000000000");
2761 let peers = finalize_app_state_key_request_peers(
2762 vec![
2763 Jid::lid_device("100000000000001", 0),
2764 Jid::lid_device("100000000000001", 7),
2765 Jid::pn_device("5511000000000", 7),
2766 ],
2767 33,
2768 primary.clone(),
2769 )
2770 .expect("peer devices");
2771
2772 assert_eq!(peers, vec![primary, Jid::pn_device("5511000000000", 7)]);
2773 }
2774
2775 #[tokio::test]
2776 async fn active_key_wait_shortens_a_passive_dedup_stamp() {
2777 let client = crate::test_utils::create_test_client_with_name("appstate_retry_stamp").await;
2778 let key_id = vec![1, 2, 3, 4];
2779 client.app_state_key_requests.lock().await.insert(
2780 key_id.clone(),
2781 wacore::time::Instant::now() + APP_STATE_KEY_REQUEST_DEDUP,
2782 );
2783
2784 let started = wacore::time::Instant::now();
2785 let schedule = client
2786 .request_missing_keys_with_dedup(
2787 std::slice::from_ref(&key_id),
2788 APP_STATE_KEY_PARTIAL_RETRY,
2789 )
2790 .await;
2791
2792 assert!(
2793 !schedule.sent,
2794 "an in-flight request must not be duplicated"
2795 );
2796 assert!(schedule.retry_at > started);
2797 assert!(
2798 schedule.retry_at.saturating_duration_since(started)
2799 <= APP_STATE_KEY_PARTIAL_RETRY + Duration::from_millis(100),
2800 "an active waiter must retry before the passive 24-hour deadline"
2801 );
2802 assert_eq!(
2803 client
2804 .app_state_key_requests
2805 .lock()
2806 .await
2807 .get(key_id.as_slice())
2808 .copied(),
2809 Some(schedule.retry_at)
2810 );
2811 }
2812
2813 #[test]
2814 fn ordinary_key_wait_leaves_time_for_a_retry() {
2815 let retry = initial_app_state_key_retry(APP_STATE_KEY_REQUEST_TIMEOUT);
2816
2817 assert_eq!(retry, Duration::from_secs(5));
2818 assert!(retry < APP_STATE_KEY_REQUEST_TIMEOUT);
2819 assert_eq!(
2820 initial_app_state_key_retry(Duration::from_secs(180)),
2821 APP_STATE_KEY_PARTIAL_RETRY
2822 );
2823 }
2824}
2825
2826#[cfg(test)]
2845mod send_patch_response_tests {
2846 use super::*;
2847 use std::sync::atomic::AtomicUsize;
2848 use wacore_binary::node::Node;
2849
2850 async fn seed_collection(client: &Arc<Client>, collection: &str) -> Vec<u8> {
2853 let backend = client.persistence_manager.backend();
2854 let key_id = b"send-patch-key".to_vec();
2855 backend
2856 .set_sync_key(
2857 &key_id,
2858 crate::store::traits::AppStateSyncKey {
2859 key_data: vec![5u8; 32],
2860 ..Default::default()
2861 },
2862 )
2863 .await
2864 .expect("test backend should accept a sync key");
2865 backend
2866 .set_version(
2867 collection,
2868 wacore::appstate::hash::HashState {
2869 version: 7,
2870 ..Default::default()
2871 },
2872 )
2873 .await
2874 .expect("test backend should accept a version");
2875 key_id
2876 }
2877
2878 fn collection_error_result(request_id: &str, collection: &str, code: &str) -> Node {
2881 NodeBuilder::new("iq")
2882 .attr("type", "result")
2883 .attr("id", request_id)
2884 .attr("from", "s.whatsapp.net")
2885 .children([NodeBuilder::new("sync")
2886 .children([NodeBuilder::new("collection")
2887 .attr("name", collection)
2888 .attr("type", "error")
2889 .children([NodeBuilder::new("error")
2890 .attr("code", code)
2891 .attr("text", "")
2892 .build()])
2893 .build()])
2894 .build()])
2895 .build()
2896 }
2897
2898 fn empty_sync_result(request_id: &str, collection: &str) -> Node {
2900 NodeBuilder::new("iq")
2901 .attr("type", "result")
2902 .attr("id", request_id)
2903 .attr("from", "s.whatsapp.net")
2904 .children([NodeBuilder::new("sync")
2905 .children([NodeBuilder::new("collection")
2906 .attr("name", collection)
2907 .build()])
2908 .build()])
2909 .build()
2910 }
2911
2912 const COLLECTION: &str = "regular_low";
2913
2914 async fn serve_iqs(
2923 client: &Arc<Client>,
2924 transport: &Arc<crate::transport::mock::CapturingMockTransport>,
2925 patch_attempts: &AtomicUsize,
2926 response_collection: &str,
2927 mut reply: impl FnMut(usize) -> Option<&'static str>,
2928 ) {
2929 let mut frame = 0usize;
2930 loop {
2931 let node = crate::test_utils::decode_sent_iq(transport, frame).await;
2932 let node = node.get().to_owned();
2933 let id = node
2934 .attrs()
2935 .optional_string("id")
2936 .expect("every IQ carries an id")
2937 .into_owned();
2938 let attempt = if node
2939 .get_optional_child_by_tag(&["sync", "collection", "patch"])
2940 .is_some()
2941 {
2942 patch_attempts.fetch_add(1, Ordering::Relaxed) + 1
2943 } else {
2944 0
2945 };
2946 let response = match reply(attempt) {
2947 Some(code) => collection_error_result(&id, response_collection, code),
2948 None => empty_sync_result(&id, response_collection),
2949 };
2950 crate::test_utils::answer_iq(client, &id, &response).await;
2951 frame += 1;
2952 }
2953 }
2954
2955 async fn send_against(reply: impl FnMut(usize) -> Option<&'static str>) -> (Result<()>, usize) {
2958 send_against_collection(COLLECTION, reply).await
2959 }
2960
2961 async fn send_against_collection(
2962 response_collection: &'static str,
2963 reply: impl FnMut(usize) -> Option<&'static str>,
2964 ) -> (Result<()>, usize) {
2965 let (client, transport) = crate::test_utils::create_iq_test_client().await;
2966 seed_collection(&client, COLLECTION).await;
2967
2968 let mut send = {
2969 let client = Arc::clone(&client);
2970 tokio::spawn(async move {
2971 client
2972 .send_app_state_patch(COLLECTION, vec![wa::SyncdMutation::default()])
2973 .await
2974 })
2975 };
2976
2977 let patch_attempts = AtomicUsize::new(0);
2978 let server = serve_iqs(
2979 &client,
2980 &transport,
2981 &patch_attempts,
2982 response_collection,
2983 reply,
2984 );
2985 futures::pin_mut!(server);
2986 let result = futures::select! {
2987 result = (&mut send).fuse() => result.expect("the send task should not panic"),
2988 () = server.as_mut().fuse() => unreachable!("the responder never completes"),
2989 };
2990
2991 (result, patch_attempts.load(Ordering::Relaxed))
2992 }
2993
2994 #[tokio::test]
2995 async fn response_for_a_different_collection_is_rejected() {
2996 for error in [None, Some("409")] {
2997 let (result, patches) = send_against_collection("regular_high", move |_| error).await;
2998 assert!(
2999 result.is_err(),
3000 "a response for another collection must not accept or absorb this send"
3001 );
3002 assert_eq!(
3003 patches, 1,
3004 "a mismatched response must fail before retrying the mutation"
3005 );
3006 }
3007 }
3008
3009 #[tokio::test]
3013 async fn unresolvable_conflict_is_not_reported_as_success() {
3014 let (result, patches) = send_against(|_| Some("409")).await;
3015 assert!(
3016 result.is_err(),
3017 "a 409 conflict means the mutation was dropped; reporting Ok hides the loss"
3018 );
3019 assert_eq!(
3020 patches, APP_STATE_PATCH_SEND_ATTEMPTS,
3021 "the send must exhaust its rebuild attempts before giving up"
3022 );
3023 }
3024
3025 #[tokio::test]
3029 async fn conflict_is_resolved_by_rebuilding_and_resending() {
3030 let (result, patches) =
3031 send_against(|attempt| if attempt == 1 { Some("409") } else { None }).await;
3032 result.expect("a conflict the server later accepts must succeed, not fail");
3033 assert_eq!(
3034 patches, 2,
3035 "the losing patch must be rebuilt and re-sent exactly once"
3036 );
3037 }
3038
3039 #[tokio::test]
3043 async fn response_without_a_collection_verdict_is_accepted() {
3044 let (client, transport) = crate::test_utils::create_iq_test_client().await;
3045 seed_collection(&client, COLLECTION).await;
3046
3047 let mut send = {
3048 let client = Arc::clone(&client);
3049 tokio::spawn(async move {
3050 client
3051 .send_app_state_patch(COLLECTION, vec![wa::SyncdMutation::default()])
3052 .await
3053 })
3054 };
3055
3056 let bare = async {
3057 let mut frame = 0usize;
3058 loop {
3059 let node = crate::test_utils::decode_sent_iq(&transport, frame).await;
3060 let id = node
3061 .get()
3062 .attrs()
3063 .optional_string("id")
3064 .expect("every IQ carries an id")
3065 .into_owned();
3066 crate::test_utils::answer_iq(
3067 &client,
3068 &id,
3069 &NodeBuilder::new("iq")
3070 .attr("type", "result")
3071 .attr("id", &id)
3072 .attr("from", "s.whatsapp.net")
3073 .build(),
3074 )
3075 .await;
3076 frame += 1;
3077 }
3078 };
3079 futures::pin_mut!(bare);
3080
3081 let result = futures::select! {
3082 result = (&mut send).fuse() => result.expect("the send task should not panic"),
3083 () = bare.as_mut().fuse() => unreachable!("the responder never completes"),
3084 };
3085 result.expect("a terse but successful response must not read as a rejection");
3086 }
3087
3088 #[tokio::test]
3092 async fn unreadable_collection_is_not_mistaken_for_an_absent_one() {
3093 let (client, transport) = crate::test_utils::create_iq_test_client().await;
3094 seed_collection(&client, COLLECTION).await;
3095
3096 let mut send = {
3097 let client = Arc::clone(&client);
3098 tokio::spawn(async move {
3099 client
3100 .send_app_state_patch(COLLECTION, vec![wa::SyncdMutation::default()])
3101 .await
3102 })
3103 };
3104
3105 let malformed = async {
3106 let mut frame = 0usize;
3107 loop {
3108 let node = crate::test_utils::decode_sent_iq(&transport, frame).await;
3109 let id = node
3110 .get()
3111 .attrs()
3112 .optional_string("id")
3113 .expect("every IQ carries an id")
3114 .into_owned();
3115 crate::test_utils::answer_iq(
3117 &client,
3118 &id,
3119 &NodeBuilder::new("iq")
3120 .attr("type", "result")
3121 .attr("id", &id)
3122 .attr("from", "s.whatsapp.net")
3123 .children([NodeBuilder::new("sync")
3124 .children([NodeBuilder::new("collection")
3125 .attr("type", "error")
3126 .build()])
3127 .build()])
3128 .build(),
3129 )
3130 .await;
3131 frame += 1;
3132 }
3133 };
3134 futures::pin_mut!(malformed);
3135
3136 let result = futures::select! {
3137 result = (&mut send).fuse() => result.expect("the send task should not panic"),
3138 () = malformed.as_mut().fuse() => unreachable!("the responder never completes"),
3139 };
3140 assert!(
3141 result.is_err(),
3142 "a collection we cannot read may be the rejection; it must not read as success"
3143 );
3144 }
3145
3146 #[tokio::test]
3149 async fn fatal_collection_error_is_not_reported_as_success() {
3150 let (result, patches) = send_against(|_| Some("400")).await;
3151 assert!(
3152 result.is_err(),
3153 "a fatal collection error must surface to the caller, not read as success"
3154 );
3155 assert_eq!(patches, 1, "a fatal error must not be retried");
3156 }
3157}
3158
3159#[cfg(test)]
3160mod sync_in_flight_tests {
3161 use super::*;
3162
3163 #[tokio::test]
3168 async fn a_full_sync_task_waits_for_the_collection() {
3169 let client = crate::test_utils::create_test_client_with_name("appstate-task-wait").await;
3170 let held = client
3171 .app_state_syncing
3172 .try_begin(WAPatchName::CriticalBlock)
3173 .expect("reserve the collection first");
3174
3175 let task = tokio::spawn({
3176 let client = Arc::clone(&client);
3177 async move {
3178 client
3179 .process_sync_task(MajorSyncTask::AppStateSync {
3180 name: WAPatchName::CriticalBlock,
3181 full_sync: true,
3182 })
3183 .await;
3184 }
3185 });
3186
3187 for _ in 0..8 {
3188 tokio::task::yield_now().await;
3189 }
3190 assert!(
3191 !task.is_finished(),
3192 "the task ran while the collection was reserved"
3193 );
3194 assert_eq!(
3195 client.app_state_syncing.len(),
3196 1,
3197 "only the held reservation"
3198 );
3199
3200 drop(held);
3201 tokio::time::timeout(Duration::from_secs(5), task)
3204 .await
3205 .expect("released collection must let the task proceed")
3206 .expect("task panicked");
3207 assert_eq!(
3208 client.app_state_syncing.len(),
3209 0,
3210 "the task's own reservation must be released"
3211 );
3212 }
3213
3214 #[tokio::test]
3217 async fn an_incremental_sync_task_also_waits() {
3218 let client = crate::test_utils::create_test_client_with_name("appstate-task-skip").await;
3219 let held = client
3220 .app_state_syncing
3221 .try_begin(WAPatchName::Regular)
3222 .expect("reserve the collection first");
3223
3224 let task = tokio::spawn({
3225 let client = Arc::clone(&client);
3226 async move {
3227 client
3228 .process_sync_task(MajorSyncTask::AppStateSync {
3229 name: WAPatchName::Regular,
3230 full_sync: false,
3231 })
3232 .await;
3233 }
3234 });
3235
3236 for _ in 0..8 {
3237 tokio::task::yield_now().await;
3238 }
3239 assert!(
3240 !task.is_finished(),
3241 "the task ran while the collection was reserved"
3242 );
3243
3244 drop(held);
3245 tokio::time::timeout(Duration::from_secs(5), task)
3246 .await
3247 .expect("released collection must let the task proceed")
3248 .expect("task panicked");
3249 assert_eq!(client.app_state_syncing.len(), 0, "reservation released");
3250 }
3251
3252 #[test]
3253 fn second_begin_blocked_until_release() {
3254 let registry = SyncInFlight::new();
3255 let guard = registry
3256 .try_begin(WAPatchName::Regular)
3257 .expect("first begin must reserve");
3258 assert!(
3259 registry.try_begin(WAPatchName::Regular).is_none(),
3260 "in-flight collection must dedup"
3261 );
3262 assert!(registry.try_begin(WAPatchName::CriticalBlock).is_some());
3264
3265 drop(guard);
3266 assert!(
3267 registry.try_begin(WAPatchName::Regular).is_some(),
3268 "release (including cancellation drop) must free the slot"
3269 );
3270 }
3271
3272 #[test]
3273 fn stale_guard_does_not_clobber_new_generation() {
3274 let registry = SyncInFlight::new();
3275 let stale = registry
3278 .try_begin(WAPatchName::Regular)
3279 .expect("gen-1 reserve");
3280 registry.clear();
3281
3282 let fresh = registry
3284 .try_begin(WAPatchName::Regular)
3285 .expect("post-clear reserve");
3286
3287 drop(stale);
3289 assert!(
3290 registry.try_begin(WAPatchName::Regular).is_none(),
3291 "stale release clobbered the new generation's reservation"
3292 );
3293
3294 drop(fresh);
3295 assert!(registry.try_begin(WAPatchName::Regular).is_some());
3296 }
3297
3298 #[tokio::test]
3302 async fn begin_waits_for_the_holder_instead_of_skipping() {
3303 let registry = SyncInFlight::new();
3304 let held = registry
3305 .try_begin(WAPatchName::Regular)
3306 .expect("first reserve");
3307
3308 let (reserved_tx, mut reserved_rx) = tokio::sync::oneshot::channel();
3309 let waiter = {
3310 let registry = Arc::clone(®istry);
3311 tokio::spawn(async move {
3312 let guard = registry.begin(WAPatchName::Regular, SyncHolder::Sync).await;
3313 let _ = reserved_tx.send(());
3314 guard
3315 })
3316 };
3317
3318 crate::test_utils::poll_until("the waiter to park on the registry", || {
3321 registry.released.total_listeners() >= 1
3322 })
3323 .await;
3324 assert!(
3325 reserved_rx.try_recv().is_err(),
3326 "begin must not resolve while the collection is held"
3327 );
3328
3329 drop(held);
3330 let guard = waiter.await.expect("the waiter should not panic");
3331 assert!(
3332 registry.try_begin(WAPatchName::Regular).is_none(),
3333 "the waiter must now hold the reservation, not merely have observed it free"
3334 );
3335
3336 drop(guard);
3337 assert!(registry.try_begin(WAPatchName::Regular).is_some());
3338 }
3339}
3340
3341#[cfg(test)]
3342pub(crate) mod batched_sync_outcome_tests {
3343 use super::*;
3344 use wacore_binary::node::Node;
3345
3346 pub(crate) fn batch_result(request_id: &str, collections: &[(&str, Option<&str>)]) -> Node {
3350 let children: Vec<Node> = collections
3351 .iter()
3352 .map(|(name, error)| {
3353 let builder = NodeBuilder::new("collection").attr("name", *name);
3354 match error {
3355 Some(code) => builder
3356 .attr("type", "error")
3357 .children([NodeBuilder::new("error")
3358 .attr("code", *code)
3359 .attr("text", "")
3360 .build()])
3361 .build(),
3362 None => builder.build(),
3363 }
3364 })
3365 .collect();
3366 NodeBuilder::new("iq")
3367 .attr("type", "result")
3368 .attr("id", request_id)
3369 .attr("from", "s.whatsapp.net")
3370 .children([NodeBuilder::new("sync").children(children).build()])
3371 .build()
3372 }
3373
3374 pub(crate) async fn sync_against(
3378 request: Vec<WAPatchName>,
3379 collections: &'static [(&'static str, Option<&'static str>)],
3380 ) -> (BatchedSyncOutcome, usize) {
3381 use futures::FutureExt;
3382 let (client, transport) = crate::test_utils::create_iq_test_client().await;
3383
3384 let mut sync = {
3385 let client = Arc::clone(&client);
3386 tokio::spawn(async move {
3387 let scope = client.sync_scope(None);
3388 client.sync_collections_batched(request, scope).await
3389 })
3390 };
3391
3392 let sent = AtomicU64::new(0);
3393 let server = async {
3394 let mut frame = 0usize;
3395 loop {
3396 let node = crate::test_utils::decode_sent_iq(&transport, frame).await;
3397 let node = node.get().to_owned();
3398 let id = node
3399 .attrs()
3400 .optional_string("id")
3401 .expect("every IQ carries an id")
3402 .into_owned();
3403 sent.fetch_add(1, Ordering::Relaxed);
3404 let response = batch_result(&id, collections);
3405 crate::test_utils::answer_iq(&client, &id, &response).await;
3406 frame += 1;
3407 }
3408 };
3409 futures::pin_mut!(server);
3410 let outcome = futures::select! {
3411 result = (&mut sync).fuse() => result
3412 .expect("the sync task should not panic")
3413 .expect("a per-collection error is an outcome, not a transport failure"),
3414 () = server.as_mut().fuse() => unreachable!("the responder never completes"),
3415 };
3416
3417 (outcome, sent.load(Ordering::Relaxed) as usize)
3418 }
3419
3420 #[tokio::test]
3424 async fn a_refused_collection_is_reported_fatal_not_synced() {
3425 let (outcome, _) = sync_against(
3426 vec![WAPatchName::CriticalBlock, WAPatchName::CriticalUnblockLow],
3427 &[
3428 ("critical_block", Some("404")),
3429 ("critical_unblock_low", None),
3430 ],
3431 )
3432 .await;
3433
3434 assert_eq!(outcome.fatal, vec![WAPatchName::CriticalBlock]);
3435 assert_eq!(outcome.synced, vec![WAPatchName::CriticalUnblockLow]);
3436 assert!(!outcome.all_synced(), "the batch did not fully sync");
3437 }
3438
3439 #[tokio::test]
3444 async fn a_retryable_collection_is_not_refetched_in_the_same_run() {
3445 let (outcome, iqs) =
3446 sync_against(vec![WAPatchName::Regular], &[("regular", Some("500"))]).await;
3447
3448 assert_eq!(outcome.retryable, vec![WAPatchName::Regular]);
3449 assert!(outcome.fatal.is_empty(), "500 is not terminal");
3450 assert_eq!(iqs, 1, "a retryable error must not be re-asked in this run");
3451 }
3452
3453 #[tokio::test]
3456 async fn collections_held_by_another_sync_are_reported_skipped() {
3457 let (client, transport) = crate::test_utils::create_iq_test_client().await;
3458 let _held = client
3459 .app_state_syncing
3460 .try_begin_as(WAPatchName::CriticalBlock, SyncHolder::Sync)
3461 .expect("reserve the collection first");
3462
3463 let outcome = client
3464 .sync_collections_batched(vec![WAPatchName::CriticalBlock], client.sync_scope(None))
3465 .await
3466 .expect("skipping is an outcome, not an error");
3467
3468 assert_eq!(outcome.skipped, vec![WAPatchName::CriticalBlock]);
3469 assert!(outcome.synced.is_empty());
3470 assert!(!outcome.all_synced(), "a skipped collection did not sync");
3471 assert!(
3472 transport.sent().is_empty(),
3473 "a skipped batch must not reach the wire"
3474 );
3475 }
3476
3477 #[tokio::test]
3481 async fn a_patch_send_holder_is_waited_out_not_skipped() {
3482 let (client, _transport) = crate::test_utils::create_iq_test_client().await;
3483 let held = client
3484 .app_state_syncing
3485 .try_begin_as(WAPatchName::Regular, SyncHolder::PatchSend)
3486 .expect("reserve the collection first");
3487
3488 let reserve = {
3489 let client = Arc::clone(&client);
3490 tokio::spawn(async move {
3491 client
3492 .reserve_for_sync(
3493 WAPatchName::Regular,
3494 ReservationWait::SkipBehindSync,
3495 client.sync_scope(None),
3496 )
3497 .await
3498 .map(drop)
3499 })
3500 };
3501
3502 crate::test_utils::poll_until("the sync to park behind the patch send", || {
3503 client.app_state_syncing.released.total_listeners() >= 1
3504 })
3505 .await;
3506 assert!(
3507 !reserve.is_finished(),
3508 "a sync must wait for a patch send, not skip it"
3509 );
3510
3511 drop(held);
3512 tokio::time::timeout(Duration::from_secs(5), reserve)
3513 .await
3514 .expect("releasing the send must let the sync proceed")
3515 .expect("the reserve task should not panic")
3516 .expect("the sync must get the reservation, not time out");
3517 }
3518
3519 #[test]
3520 fn all_synced_is_false_for_every_kind_of_miss() {
3521 let mut outcome = BatchedSyncOutcome::default();
3522 assert!(outcome.all_synced(), "an empty batch missed nothing");
3523
3524 outcome.synced.push(WAPatchName::Regular);
3525 assert!(outcome.all_synced());
3526
3527 let buckets: [fn(&mut BatchedSyncOutcome) -> &mut Vec<WAPatchName>; 3] =
3528 [|o| &mut o.fatal, |o| &mut o.retryable, |o| &mut o.skipped];
3529 for bucket in buckets {
3530 bucket(&mut outcome).push(WAPatchName::CriticalBlock);
3531 assert!(!outcome.all_synced());
3532 bucket(&mut outcome).clear();
3533 }
3534 }
3535}
3536
3537#[cfg(test)]
3538mod batched_sync_reconciliation_tests {
3539 use super::*;
3540 use crate::client::app_state::batched_sync_outcome_tests::{batch_result, sync_against};
3541
3542 #[tokio::test]
3546 async fn a_collection_the_response_omits_is_not_reported_synced() {
3547 let (outcome, _) = sync_against(
3548 vec![WAPatchName::CriticalBlock, WAPatchName::CriticalUnblockLow],
3549 &[("critical_unblock_low", None)],
3550 )
3551 .await;
3552
3553 assert_eq!(outcome.synced, vec![WAPatchName::CriticalUnblockLow]);
3554 assert_eq!(
3555 outcome.retryable,
3556 vec![WAPatchName::CriticalBlock],
3557 "an omitted collection is a miss, not a success"
3558 );
3559 assert!(!outcome.all_synced());
3560 }
3561
3562 #[tokio::test]
3564 async fn an_empty_response_leaves_every_collection_unsynced() {
3565 let (outcome, _) = sync_against(vec![WAPatchName::Regular], &[]).await;
3566
3567 assert!(outcome.synced.is_empty());
3568 assert_eq!(outcome.retryable, vec![WAPatchName::Regular]);
3569 assert!(!outcome.all_synced());
3570 }
3571
3572 #[tokio::test]
3574 async fn a_repeated_collection_is_counted_once() {
3575 let (outcome, _) = sync_against(
3576 vec![WAPatchName::Regular],
3577 &[("regular", None), ("regular", None)],
3578 )
3579 .await;
3580
3581 assert_eq!(outcome.synced, vec![WAPatchName::Regular]);
3582 assert!(outcome.all_synced());
3583 }
3584
3585 #[tokio::test(start_paused = true)]
3592 async fn a_wait_that_runs_out_reports_the_collection_retryable() {
3593 let (client, transport) = crate::test_utils::create_iq_test_client().await;
3594 let _held = client
3597 .app_state_syncing
3598 .try_begin_as(WAPatchName::Regular, SyncHolder::PatchSend)
3599 .expect("reserve the collection first");
3600
3601 let outcome = client
3602 .sync_collections_batched(vec![WAPatchName::Regular], client.sync_scope(None))
3603 .await
3604 .expect("a wait that ran out is an outcome, not a transport failure");
3605
3606 assert_eq!(
3607 outcome.retryable,
3608 vec![WAPatchName::Regular],
3609 "nobody is covering it, so it has to come back around"
3610 );
3611 assert!(
3612 outcome.skipped.is_empty(),
3613 "skipped means an equivalent sync has it, which is not the case here"
3614 );
3615 assert!(
3616 transport.sent().is_empty(),
3617 "the sync never got its turn, so nothing should reach the wire"
3618 );
3619 }
3620
3621 #[test]
3624 fn batch_result_marks_errors_on_the_named_collection() {
3625 let node = batch_result("id-1", &[("regular", Some("500"))]);
3626 let collection = node
3627 .get_optional_child_by_tag(&["sync", "collection"])
3628 .expect("the helper builds sync/collection");
3629 assert_eq!(
3630 collection.attrs().optional_string("type").as_deref(),
3631 Some("error")
3632 );
3633 }
3634}
3635
3636#[cfg(test)]
3637mod duplicate_collection_tests {
3638 use super::*;
3639 use crate::client::app_state::batched_sync_outcome_tests::sync_against;
3640
3641 #[tokio::test]
3647 async fn a_duplicate_collection_is_dropped_before_it_is_applied() {
3648 let (outcome, _) = sync_against(
3649 vec![WAPatchName::Regular],
3650 &[("regular", None), ("regular", None)],
3651 )
3652 .await;
3653
3654 assert_eq!(
3655 outcome.synced,
3656 vec![WAPatchName::Regular],
3657 "the collection is accounted for exactly once"
3658 );
3659 assert!(outcome.all_synced());
3660 }
3661
3662 #[tokio::test]
3665 async fn a_duplicate_does_not_leave_the_batch_unsynced() {
3666 let (outcome, _) = sync_against(
3667 vec![WAPatchName::CriticalBlock, WAPatchName::CriticalUnblockLow],
3668 &[
3669 ("critical_block", None),
3670 ("critical_block", None),
3671 ("critical_unblock_low", None),
3672 ],
3673 )
3674 .await;
3675
3676 assert!(outcome.retryable.is_empty(), "both were answered");
3677 assert!(outcome.all_synced());
3678 }
3679}
3680
3681#[cfg(test)]
3682mod background_report_tests {
3683 use super::*;
3684 use crate::types::events::{EventHandler, EventInterest, EventKind};
3685
3686 struct FailureCounter(Arc<AtomicU64>);
3687
3688 impl EventHandler for FailureCounter {
3689 fn handle_event(&self, event: Arc<Event>) {
3690 if matches!(&*event, Event::AppStateSyncFailed(_)) {
3691 self.0.fetch_add(1, Ordering::Relaxed);
3692 }
3693 }
3694 }
3695
3696 #[tokio::test]
3701 async fn an_outcome_from_a_retired_connection_is_not_published() {
3702 let client = crate::test_utils::create_test_client_with_name("bg-report-gen").await;
3703 let retired_scope = client.sync_scope(None);
3704
3705 let seen = Arc::new(AtomicU64::new(0));
3706 let _subscription = client.subscribe(
3707 EventInterest::of(&[EventKind::AppStateSyncFailed]),
3708 Arc::new(FailureCounter(Arc::clone(&seen))),
3709 );
3710
3711 let mut outcome = BatchedSyncOutcome::default();
3712 outcome.fatal.push(WAPatchName::CriticalBlock);
3713
3714 client
3716 .connection_generation
3717 .store(retired_scope.generation() + 1, Ordering::SeqCst);
3718 client.report_background_sync(
3719 "test",
3720 retired_scope,
3721 SyncSettles::JustTheCollections,
3722 &[],
3723 Ok(outcome.clone()),
3724 );
3725 assert_eq!(
3726 seen.load(Ordering::Relaxed),
3727 0,
3728 "a retired connection's refusal must not reach consumers"
3729 );
3730
3731 client.report_background_sync(
3733 "test",
3734 client.sync_scope(None),
3735 SyncSettles::JustTheCollections,
3736 &[],
3737 Ok(outcome),
3738 );
3739 assert_eq!(seen.load(Ordering::Relaxed), 1);
3740 }
3741}
3742
3743#[cfg(test)]
3744mod retry_gate_tests {
3745 use super::*;
3746
3747 #[test]
3753 fn only_a_fully_synced_outcome_settles_the_bootstrap() {
3754 let mut fatal = BatchedSyncOutcome::default();
3755 fatal.fatal.push(WAPatchName::CriticalBlock);
3756 assert!(fatal.retryable.is_empty(), "nothing left to retry");
3757 assert!(
3758 !fatal.all_synced(),
3759 "but the collection is not synced, so the gate must stay armed"
3760 );
3761
3762 let mut skipped = BatchedSyncOutcome::default();
3763 skipped.skipped.push(WAPatchName::CriticalBlock);
3764 assert!(skipped.retryable.is_empty());
3765 assert!(
3766 !skipped.all_synced(),
3767 "another sync holding it is not proof it succeeded"
3768 );
3769
3770 let mut synced = BatchedSyncOutcome::default();
3771 synced.synced.push(WAPatchName::CriticalBlock);
3772 assert!(synced.all_synced(), "this is the only case that settles it");
3773 }
3774
3775 #[test]
3787 fn retry_backoff_doubles_then_clamps() {
3788 assert_eq!(app_state_retry_backoff(0), APP_STATE_RETRY_BACKOFF_MIN);
3789 assert_eq!(app_state_retry_backoff(1), APP_STATE_RETRY_BACKOFF_MIN * 2);
3790 assert_eq!(app_state_retry_backoff(3), APP_STATE_RETRY_BACKOFF_MIN * 8);
3791 assert_eq!(
3792 app_state_retry_backoff(u32::MAX),
3793 APP_STATE_RETRY_BACKOFF_MAX,
3794 "an absurd round must clamp, not overflow"
3795 );
3796 }
3797}
3798
3799#[cfg(test)]
3800mod request_hygiene_tests {
3801 use super::*;
3802 use crate::client::app_state::batched_sync_outcome_tests::sync_against;
3803
3804 #[tokio::test]
3810 async fn a_collection_requested_twice_is_reserved_once() {
3811 let (outcome, _) = sync_against(
3812 vec![WAPatchName::Regular, WAPatchName::Regular],
3813 &[("regular", None)],
3814 )
3815 .await;
3816
3817 assert_eq!(outcome.synced, vec![WAPatchName::Regular]);
3818 assert!(
3819 outcome.skipped.is_empty(),
3820 "the only holder was this same call"
3821 );
3822 assert!(outcome.all_synced());
3823 }
3824
3825 #[tokio::test]
3829 async fn an_unrequested_collection_in_the_response_is_dropped() {
3830 let (outcome, _) = sync_against(
3831 vec![WAPatchName::Regular],
3832 &[("regular", None), ("critical_block", None)],
3833 )
3834 .await;
3835
3836 assert_eq!(outcome.synced, vec![WAPatchName::Regular]);
3837 assert!(
3838 !outcome.synced.contains(&WAPatchName::CriticalBlock),
3839 "an unrequested collection must not be applied or reported"
3840 );
3841 assert!(outcome.all_synced());
3842 }
3843}
3844
3845#[cfg(test)]
3846mod deadline_tests {
3847 use super::*;
3848
3849 #[tokio::test]
3857 async fn an_expired_deadline_stops_the_batch_before_it_reserves() {
3858 let (client, transport) = crate::test_utils::create_iq_test_client().await;
3859
3860 let sync = {
3861 let client = Arc::clone(&client);
3862 tokio::spawn(async move {
3863 let scope = client.sync_scope(Some(wacore::time::Instant::now()));
3864 client
3865 .sync_collections_batched(vec![WAPatchName::Regular], scope)
3866 .await
3867 })
3868 };
3869
3870 let outcome = tokio::time::timeout(Duration::from_secs(5), sync)
3871 .await
3872 .expect("an expired deadline must not block")
3873 .expect("the sync task should not panic")
3874 .expect("a deadline is an outcome, not a transport failure");
3875
3876 assert_eq!(outcome.retryable, vec![WAPatchName::Regular]);
3877 assert!(outcome.synced.is_empty());
3878 assert!(
3879 transport.sent().is_empty(),
3880 "nothing may reach the wire past the deadline"
3881 );
3882 }
3883}
3884
3885#[cfg(test)]
3886mod sync_scope_tests {
3887 use super::*;
3888
3889 #[tokio::test]
3893 async fn a_scope_stops_admitting_when_its_connection_or_clock_goes() {
3894 let client = crate::test_utils::create_test_client_with_name("scope-admits").await;
3895
3896 let live = client.sync_scope(None);
3897 assert_eq!(client.admits(live), Ok(()));
3898
3899 let expired = client.sync_scope(Some(wacore::time::Instant::now()));
3900 assert_eq!(client.admits(expired), Err(ScopeLost::Expired));
3901
3902 let generous = client.sync_scope(Some(
3903 wacore::time::Instant::now() + Duration::from_secs(600),
3904 ));
3905 assert_eq!(client.admits(generous), Ok(()));
3906
3907 client
3908 .connection_generation
3909 .store(live.generation() + 1, Ordering::SeqCst);
3910 assert_eq!(client.admits(live), Err(ScopeLost::Retired));
3911 assert_eq!(
3912 client.admits(generous),
3913 Err(ScopeLost::Retired),
3914 "a retired connection outranks having time left"
3915 );
3916 }
3917
3918 #[tokio::test]
3925 async fn a_retired_scope_cannot_move_the_bootstrap_gate() {
3926 let client = crate::test_utils::create_test_client_with_name("scope-gate").await;
3927 let retired = client.sync_scope(None);
3928 client
3929 .connection_generation
3930 .store(retired.generation() + 1, Ordering::SeqCst);
3931
3932 for armed in [true, false] {
3933 client
3937 .needs_initial_full_sync
3938 .settle(client.connection_generation.load(Ordering::SeqCst), armed);
3939 client.settle_bootstrap(retired, !armed);
3940 assert_eq!(
3941 client.needs_initial_full_sync.is_armed(),
3942 armed,
3943 "a retired scope must leave the gate exactly as it found it"
3944 );
3945 }
3946
3947 let live = client.sync_scope(None);
3949 client.settle_bootstrap(live, true);
3950 assert!(client.needs_initial_full_sync.is_armed());
3951 client.settle_bootstrap(live, false);
3952 assert!(!client.needs_initial_full_sync.is_armed());
3953 }
3954
3955 #[tokio::test]
3958 async fn an_expired_scope_may_still_arm_the_gate() {
3959 let client = crate::test_utils::create_test_client_with_name("scope-expired-gate").await;
3960 let expired = client.sync_scope(Some(wacore::time::Instant::now()));
3961 client
3962 .needs_initial_full_sync
3963 .settle(expired.generation(), false);
3964
3965 client.settle_bootstrap(expired, true);
3966 assert!(
3967 client.needs_initial_full_sync.is_armed(),
3968 "an expired bootstrap is unfinished, and must say so"
3969 );
3970 }
3971
3972 #[tokio::test]
3976 async fn rebinding_reports_whether_the_connection_moved() {
3977 let client = crate::test_utils::create_test_client_with_name("scope-rebind").await;
3978 let mut scope = client.sync_scope(None);
3979 let original = scope.generation();
3980
3981 assert!(!scope.rebind(original), "same connection is not a move");
3982 assert_eq!(client.admits(scope), Ok(()));
3983
3984 assert!(scope.rebind(original + 1), "a different connection is");
3985 assert_eq!(scope.generation(), original + 1);
3986 }
3987
3988 #[tokio::test]
3993 async fn only_a_deadline_marks_the_bootstrap() {
3994 let client = crate::test_utils::create_test_client_with_name("scope-kind").await;
3995 assert!(!client.sync_scope(None).is_bootstrap());
3996 assert!(
3997 client
3998 .sync_scope(Some(wacore::time::Instant::now()))
3999 .is_bootstrap()
4000 );
4001 }
4002}
4003
4004#[cfg(test)]
4005mod task_retry_tests {
4006 use super::*;
4007
4008 #[tokio::test]
4014 async fn a_planned_reconnect_is_not_a_completed_sync() {
4015 let client = crate::test_utils::create_test_client_with_name("task-retry-reconnect").await;
4016
4017 client.is_running.store(true, Ordering::Relaxed);
4022 client.expected_disconnect.store(true, Ordering::Relaxed);
4023 assert!(
4024 client.is_running.load(Ordering::Relaxed),
4025 "the retry loop's own guard still admits this state"
4026 );
4027 assert!(
4028 client.is_shutting_down(),
4029 "but it does make the callee's shutdown guard true"
4030 );
4031 assert!(
4032 client
4033 .process_app_state_sync_task(WAPatchName::Regular, true)
4034 .await
4035 .is_ok(),
4036 "the callee reports Ok without doing anything, which is the trap"
4037 );
4038
4039 client.expected_disconnect.store(false, Ordering::Relaxed);
4040 assert!(
4041 !client.is_shutting_down(),
4042 "and the hold lifts once the reconnect settles"
4043 );
4044 }
4045}
4046
4047#[cfg(test)]
4048mod apply_boundary_tests {
4049 use super::*;
4050 use crate::client::app_state::batched_sync_outcome_tests::sync_against;
4051
4052 #[tokio::test]
4067 async fn an_applied_collection_is_never_reported_retryable() {
4068 let (outcome, _) = sync_against(
4069 vec![WAPatchName::CriticalBlock, WAPatchName::CriticalUnblockLow],
4070 &[
4071 ("critical_block", None),
4072 ("critical_unblock_low", Some("500")),
4073 ],
4074 )
4075 .await;
4076
4077 assert_eq!(
4078 outcome.synced,
4079 vec![WAPatchName::CriticalBlock],
4080 "an applied collection is synced"
4081 );
4082 assert!(
4083 !outcome.retryable.contains(&WAPatchName::CriticalBlock),
4084 "and must never also be queued for a retry that cannot re-fetch it"
4085 );
4086 assert_eq!(outcome.retryable, vec![WAPatchName::CriticalUnblockLow]);
4087 }
4088}
4089
4090#[cfg(test)]
4091mod bootstrap_gate_tests {
4092 use super::*;
4093
4094 #[test]
4099 fn an_older_connection_cannot_overwrite_a_newer_one() {
4100 let gate = BootstrapGate::new(false);
4101
4102 assert!(gate.settle(7, true), "the newest writer wins");
4103 assert!(gate.is_armed());
4104
4105 assert!(
4106 !gate.settle(6, false),
4107 "an older connection is refused outright"
4108 );
4109 assert!(gate.is_armed(), "and leaves the newer answer standing");
4110
4111 assert!(
4112 gate.settle(7, false),
4113 "the same connection may revise itself"
4114 );
4115 assert!(!gate.is_armed());
4116
4117 assert!(gate.settle(8, true), "and a newer one always may");
4118 assert!(gate.is_armed());
4119 }
4120
4121 #[test]
4130 fn pairing_arms_over_live_connections_but_not_the_next_one() {
4131 let gate = BootstrapGate::new(false);
4132 assert!(gate.settle(9, false));
4133 assert!(!gate.is_armed());
4134
4135 gate.arm_for_pairing(9);
4137 assert!(gate.is_armed());
4138
4139 assert!(
4140 !gate.settle(9, false),
4141 "a bootstrap already in flight on the pairing connection must not \
4142 answer for the sync pairing just asked for"
4143 );
4144 assert!(gate.is_armed(), "so the arm survives it");
4145
4146 assert!(
4147 gate.settle(10, false),
4148 "and the connection the forced 515 brings up can clear it"
4149 );
4150 assert!(
4151 !gate.is_armed(),
4152 "a pairing that outranked every connection would never clear, and the \
4153 client would re-run the critical bootstrap forever"
4154 );
4155 }
4156
4157 #[test]
4165 fn pairing_never_lowers_the_gate() {
4166 let gate = BootstrapGate::new(false);
4167 assert!(gate.settle(20, false));
4168
4169 gate.arm_for_pairing(3);
4171
4172 assert!(gate.is_armed(), "pairing always owes a bootstrap");
4173 assert!(
4174 !gate.settle(20, false),
4175 "and connection 20 still cannot answer for it"
4176 );
4177 assert!(gate.settle(21, false), "only something newer can");
4178 assert!(!gate.is_armed());
4179 }
4180
4181 #[test]
4184 fn the_armed_bit_round_trips() {
4185 let gate = BootstrapGate::new(true);
4186 assert!(gate.is_armed());
4187 let gate = BootstrapGate::new(false);
4188 assert!(!gate.is_armed());
4189 }
4190}
4191
4192#[cfg(test)]
4193mod lifecycle_signal_tests {
4194 use super::*;
4195
4196 #[tokio::test]
4201 async fn only_a_finished_client_looks_terminal() {
4202 let client = crate::test_utils::create_test_client_with_name("lifecycle-terminal").await;
4203
4204 assert!(
4207 !client.is_running.load(Ordering::Relaxed),
4208 "the fixture models a client that never called run()"
4209 );
4210 assert!(!client.is_terminal(), "which is not the same as finished");
4211
4212 client.expected_disconnect.store(true, Ordering::Relaxed);
4214 assert!(
4215 client.is_shutting_down(),
4216 "the old predicate cannot tell this apart"
4217 );
4218 assert!(!client.is_terminal(), "the new one can");
4219 client.expected_disconnect.store(false, Ordering::Relaxed);
4220
4221 client.is_running.store(true, Ordering::Relaxed);
4227 client.enable_auto_reconnect.store(false, Ordering::Relaxed);
4228 assert!(
4229 !client.is_terminal(),
4230 "a reconnect preference is not a verdict on the current session"
4231 );
4232
4233 client.expected_disconnect.store(true, Ordering::Relaxed);
4237 assert!(client.is_terminal());
4238 client.expected_disconnect.store(false, Ordering::Relaxed);
4239
4240 client.is_running.store(false, Ordering::Relaxed);
4246 client.set_connected_for_test(false);
4247 assert!(
4248 client.is_terminal(),
4249 "the supervision loop ending with auto-reconnect off is terminal"
4250 );
4251
4252 client.set_connected_for_test(true);
4256 assert!(
4257 !client.is_terminal(),
4258 "a live direct-connect client never started the loop that would have ended"
4259 );
4260 client.set_connected_for_test(false);
4261
4262 client.enable_auto_reconnect.store(true, Ordering::Relaxed);
4263 client.is_running.store(true, Ordering::Relaxed);
4264 assert!(!client.is_terminal());
4265
4266 client.signal_shutdown_sync();
4268 assert!(client.is_terminal());
4269 }
4270}
4271
4272#[cfg(test)]
4273mod connection_guard_tests {
4274 use super::*;
4275
4276 #[tokio::test]
4289 async fn a_planned_reconnect_does_not_look_like_a_stop() {
4290 let client = crate::test_utils::create_test_client_with_name("conn-guard").await;
4291 client.is_running.store(true, Ordering::Relaxed);
4292 client.set_connected_for_test(true);
4293
4294 assert!(!client.is_terminal() && client.is_connected());
4295
4296 client.expected_disconnect.store(true, Ordering::Relaxed);
4298 assert!(
4299 client.is_shutting_down(),
4300 "which the old guard could not tell from a stop"
4301 );
4302 assert!(
4303 !client.is_terminal(),
4304 "so work that outlives a connection stays alive"
4305 );
4306 }
4307}
4308
4309#[cfg(test)]
4310mod await_connection_tests {
4311 use super::*;
4312
4313 #[tokio::test]
4318 async fn a_stale_notification_does_not_end_the_wait() {
4319 let client = crate::test_utils::create_test_client_with_name("await-stale").await;
4320 client.is_running.store(true, Ordering::Relaxed);
4321
4322 let waiter = {
4323 let client = Arc::clone(&client);
4324 tokio::spawn(async move { client.await_connection().await })
4325 };
4326
4327 crate::test_utils::poll_until("the waiter to park on the notifier", || {
4328 client.socket_ready_notifier.total_listeners() >= 1
4329 })
4330 .await;
4331
4332 client.socket_ready_notifier.notify(usize::MAX);
4334 for _ in 0..8 {
4335 tokio::task::yield_now().await;
4336 }
4337 assert!(
4338 !waiter.is_finished(),
4339 "an event without a connection is not an answer"
4340 );
4341
4342 client.set_connected_for_test(true);
4346 client.socket_ready_notifier.notify(usize::MAX);
4347 for _ in 0..8 {
4348 tokio::task::yield_now().await;
4349 }
4350 assert!(
4351 !waiter.is_finished(),
4352 "a socket without an authenticated session is not one either"
4353 );
4354
4355 client.is_logged_in.store(true, Ordering::Relaxed);
4360 client.authenticated_generation.store(
4361 client.connection_generation.load(Ordering::SeqCst),
4362 Ordering::SeqCst,
4363 );
4364 client.notify_session_state();
4365 assert!(
4366 tokio::time::timeout(Duration::from_secs(5), waiter)
4367 .await
4368 .expect("a usable connection must end the wait")
4369 .expect("the waiter should not panic"),
4370 "and it reports that one arrived"
4371 );
4372 }
4373
4374 #[tokio::test]
4384 async fn a_finished_client_ends_the_wait() {
4385 let client = crate::test_utils::create_test_client_with_name("await-terminal").await;
4386 client.is_running.store(true, Ordering::Relaxed);
4387
4388 let waiter = {
4389 let client = Arc::clone(&client);
4390 tokio::spawn(async move { client.await_connection().await })
4391 };
4392
4393 crate::test_utils::poll_until("the waiter to park on the notifier", || {
4394 client.socket_ready_notifier.total_listeners() >= 1
4395 })
4396 .await;
4397
4398 client.signal_shutdown_sync();
4399 assert!(
4400 !tokio::time::timeout(Duration::from_secs(5), waiter)
4401 .await
4402 .expect("a finished client must end the wait, with nothing else nudging it")
4403 .expect("the waiter should not panic"),
4404 "and it reports that none arrived"
4405 );
4406 }
4407
4408 #[tokio::test]
4412 async fn the_run_loop_giving_up_ends_the_wait() {
4413 let client = crate::test_utils::create_test_client_with_name("await-runloop").await;
4414 client.is_running.store(true, Ordering::Relaxed);
4415
4416 let waiter = {
4417 let client = Arc::clone(&client);
4418 tokio::spawn(async move { client.await_connection().await })
4419 };
4420
4421 crate::test_utils::poll_until("the waiter to park on the notifier", || {
4422 client.socket_ready_notifier.total_listeners() >= 1
4423 })
4424 .await;
4425
4426 client.enable_auto_reconnect.store(false, Ordering::Relaxed);
4430 client.stop_supervision_loop();
4431
4432 assert!(
4433 !tokio::time::timeout(Duration::from_secs(5), waiter)
4434 .await
4435 .expect("the supervision loop ending must end the wait")
4436 .expect("the waiter should not panic"),
4437 "and it reports that none arrived"
4438 );
4439 }
4440
4441 #[tokio::test]
4445 async fn a_client_without_a_reader_is_not_worth_waiting_for() {
4446 let client = crate::test_utils::create_test_client_with_name("await-direct").await;
4447 client.set_connected_for_test(true);
4448
4449 assert!(
4450 !client.is_terminal(),
4451 "a live direct-connect client is fine"
4452 );
4453 assert!(
4454 !tokio::time::timeout(Duration::from_secs(5), client.await_connection())
4455 .await
4456 .expect("the wait must not park on a connection that cannot answer"),
4457 "it just cannot carry the work"
4458 );
4459 }
4460}
4461
4462#[cfg(test)]
4463mod sync_outcome_tests {
4464 use super::*;
4465
4466 async fn reachable_client(name: &str) -> Arc<Client> {
4469 let client = crate::test_utils::create_test_client_with_name(name).await;
4470 client.is_running.store(true, Ordering::Relaxed);
4471 client.set_connected_for_test(true);
4472 client.is_logged_in.store(true, Ordering::Relaxed);
4473 client.authenticated_generation.store(
4474 client.connection_generation.load(Ordering::SeqCst),
4475 Ordering::SeqCst,
4476 );
4477 assert!(client.can_reach_server(), "the fixture itself is usable");
4478 client
4479 }
4480
4481 #[tokio::test]
4488 async fn a_rate_limited_session_defers_rather_than_completes() {
4489 let client = reachable_client("outcome-429").await;
4490
4491 client.is_logged_in.store(false, Ordering::Relaxed);
4493
4494 assert!(
4495 !client.is_terminal(),
4496 "a rate limit is not the client being finished"
4497 );
4498 assert!(
4499 !client.is_shutting_down(),
4500 "nor is it anything the old proxy could see"
4501 );
4502 assert_eq!(
4503 client
4504 .process_app_state_sync_task(WAPatchName::Regular, true)
4505 .await
4506 .expect("skipping is not an error"),
4507 SyncOutcome::Deferred,
4508 "nothing was asked, so nothing was completed"
4509 );
4510 }
4511
4512 #[tokio::test]
4515 async fn a_reconnect_defers_rather_than_completes() {
4516 let client = reachable_client("outcome-reconnect").await;
4517 client.set_connected_for_test(false);
4518
4519 assert_eq!(
4520 client
4521 .process_app_state_sync_task(WAPatchName::Regular, false)
4522 .await
4523 .expect("skipping is not an error"),
4524 SyncOutcome::Deferred
4525 );
4526 }
4527
4528 #[tokio::test]
4532 async fn a_terminal_client_is_not_a_usable_one() {
4533 let client = reachable_client("verdict-order").await;
4534
4535 client.enable_auto_reconnect.store(false, Ordering::Relaxed);
4540 client.expected_disconnect.store(true, Ordering::Relaxed);
4541
4542 assert!(client.is_terminal());
4543 assert_eq!(client.connection_wait_verdict(), Some(false));
4544
4545 assert!(
4552 !client.can_reach_server(),
4553 "a finished client is never a reachable one"
4554 );
4555
4556 client.expected_disconnect.store(false, Ordering::Relaxed);
4557 client.signal_shutdown_sync();
4558 assert!(client.is_terminal() && !client.can_reach_server());
4559 }
4560
4561 #[tokio::test]
4566 async fn the_gap_inside_success_is_not_an_authenticated_connection() {
4567 let client = crate::test_utils::create_test_client_with_name("auth-window").await;
4568 client.is_running.store(true, Ordering::Relaxed);
4569
4570 client.set_connected_for_test(true);
4571
4572 client.is_logged_in.store(true, Ordering::Relaxed);
4578 assert!(
4579 client.is_logged_in() && client.is_connected(),
4580 "which is why the flags alone said yes"
4581 );
4582 assert!(
4583 !client.can_reach_server(),
4584 "this connection has not authenticated anything yet"
4585 );
4586
4587 let current = client.connection_generation.fetch_add(1, Ordering::SeqCst) + 1;
4589 assert!(!client.can_reach_server());
4590 assert_eq!(
4591 client.connection_wait_verdict(),
4592 None,
4593 "the wait carries on"
4594 );
4595
4596 client
4598 .authenticated_generation
4599 .store(current, Ordering::SeqCst);
4600 assert_eq!(client.connection_wait_verdict(), Some(true));
4601 }
4602}
4603
4604#[cfg(test)]
4605mod sync_owed_tests {
4606 use super::*;
4607
4608 #[test]
4617 fn everything_that_is_not_a_completed_sync_is_still_owed() {
4618 assert!(!sync_still_owed(&Ok(SyncOutcome::Completed)));
4619 assert!(sync_still_owed(&Ok(SyncOutcome::Deferred)));
4620 assert!(sync_still_owed(&Err(anyhow::anyhow!(
4621 "the socket died under the collection IQ"
4622 ))));
4623 }
4624}
4625
4626#[cfg(test)]
4627mod terminal_wake_tests {
4628 use super::*;
4629
4630 #[tokio::test]
4639 async fn a_fatal_stream_error_ends_the_wait() {
4640 let client = crate::test_utils::create_test_client_with_name("await-fatal").await;
4641 client.is_running.store(true, Ordering::Relaxed);
4642
4643 let waiter = {
4644 let client = Arc::clone(&client);
4645 tokio::spawn(async move { client.await_connection().await })
4646 };
4647
4648 crate::test_utils::poll_until("the waiter to park on the notifier", || {
4649 client.socket_ready_notifier.total_listeners() >= 1
4650 })
4651 .await;
4652
4653 client.expected_disconnect.store(true, Ordering::Relaxed);
4655 client.enable_auto_reconnect.store(false, Ordering::Relaxed);
4656 client.notify_connection_shutdown();
4657
4658 assert!(
4659 !tokio::time::timeout(Duration::from_secs(5), waiter)
4660 .await
4661 .expect("a fatal stream error must end the wait where it happens")
4662 .expect("the waiter should not panic"),
4663 "and it reports that no connection arrived"
4664 );
4665 }
4666}
4667
4668#[cfg(test)]
4669mod reconnect_wake_tests {
4670 use super::*;
4671
4672 #[tokio::test]
4684 async fn a_planned_reconnect_teardown_does_not_end_the_wait() {
4685 let client = crate::test_utils::create_test_client_with_name("await-replan").await;
4686 client.is_running.store(true, Ordering::Relaxed);
4687
4688 let waiter = {
4689 let client = Arc::clone(&client);
4690 tokio::spawn(async move { client.await_connection().await })
4691 };
4692
4693 crate::test_utils::poll_until("the waiter to park on the notifier", || {
4694 client.session_state_notifier.total_listeners() >= 1
4695 })
4696 .await;
4697
4698 client.expected_disconnect.store(true, Ordering::Relaxed);
4701 client.notify_connection_shutdown();
4702 for _ in 0..8 {
4703 tokio::task::yield_now().await;
4704 }
4705 assert!(
4706 !waiter.is_finished(),
4707 "a teardown is not a connection, however loudly it is announced"
4708 );
4709
4710 client.expected_disconnect.store(false, Ordering::Relaxed);
4712 client.set_connected_for_test(true);
4713 client.is_logged_in.store(true, Ordering::Relaxed);
4714 client.authenticated_generation.store(
4715 client.connection_generation.load(Ordering::SeqCst),
4716 Ordering::SeqCst,
4717 );
4718 client.notify_session_state();
4719 assert!(
4720 tokio::time::timeout(Duration::from_secs(5), waiter)
4721 .await
4722 .expect("the replacement connection must end the wait")
4723 .expect("the waiter should not panic")
4724 );
4725 }
4726}
4727
4728#[cfg(test)]
4729mod retiring_socket_tests {
4730 use super::*;
4731
4732 #[tokio::test]
4742 async fn a_socket_marked_for_reconnect_cannot_carry_work() {
4743 let client = crate::test_utils::create_test_client_with_name("retiring").await;
4744 client.is_running.store(true, Ordering::Relaxed);
4745 client.set_connected_for_test(true);
4746 client.is_logged_in.store(true, Ordering::Relaxed);
4747 client.authenticated_generation.store(
4748 client.connection_generation.load(Ordering::SeqCst),
4749 Ordering::SeqCst,
4750 );
4751 assert!(client.can_reach_server(), "healthy to begin with");
4752
4753 client.expected_disconnect.store(true, Ordering::Relaxed);
4755
4756 assert!(
4757 client.is_connected() && client.is_logged_in(),
4758 "and every other signal still says the socket is fine"
4759 );
4760 assert!(
4761 !client.can_reach_server(),
4762 "but it is going away, so nothing sent on it comes back"
4763 );
4764 assert!(
4765 !client.is_terminal(),
4766 "which is not the same as the client being finished"
4767 );
4768 assert_eq!(
4769 client.connection_wait_verdict(),
4770 None,
4771 "so the wait carries on to the replacement"
4772 );
4773 }
4774}
4775
4776#[cfg(test)]
4777mod batched_attempt_tests {
4778 use super::*;
4779
4780 #[test]
4794 fn only_a_sent_iq_counts_as_reaching_the_server() {
4795 let timed_out = BatchedSyncOutcome {
4798 retryable: vec![WAPatchName::Regular, WAPatchName::RegularHigh],
4799 ..Default::default()
4800 };
4801 assert!(
4802 !timed_out.reached_server(),
4803 "a reservation timeout never reached the wire, whatever bucket it lands in"
4804 );
4805
4806 let scope_lost = BatchedSyncOutcome {
4808 retryable: vec![WAPatchName::Regular],
4809 ..Default::default()
4810 };
4811 assert!(!scope_lost.reached_server());
4812
4813 let held = BatchedSyncOutcome {
4815 skipped: vec![WAPatchName::Regular],
4816 ..Default::default()
4817 };
4818 assert!(!held.reached_server());
4819 assert!(!BatchedSyncOutcome::default().reached_server());
4820
4821 let mut sent = BatchedSyncOutcome {
4825 retryable: vec![WAPatchName::Regular],
4826 ..Default::default()
4827 };
4828 sent.note_reached_server();
4829 assert!(sent.reached_server());
4830 }
4831}