Skip to main content

whatsapp_rust/client/
app_state.rs

1//! App-state collection sync and mutation dispatch.
2
3use super::*;
4use crate::request::DEFAULT_IQ_TIMEOUT;
5
6/// Concurrency cap for pre-downloading app-state external blobs (independent CDN
7/// GETs, keyed by directPath — LTHash ordering is in patch application, not blob
8/// fetching). WA Web fans these out under `Promise.all` (`Syncd/CollectionHandler`);
9/// bounded here because a snapshot can be multi-MB and a batch carries several.
10const APPSTATE_BLOB_DOWNLOAD_CONCURRENCY: usize = 4;
11const APP_STATE_KEY_REQUEST_DEDUP: Duration = Duration::from_secs(24 * 3600);
12const APP_STATE_KEY_REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
13const APP_STATE_KEY_PARTIAL_RETRY: Duration = Duration::from_secs(10);
14const APP_STATE_KEY_RETRY_MAX: Duration = Duration::from_secs(60);
15/// How many times an outgoing patch is rebuilt against a newer base before the
16/// send gives up. WA Web's `serverSync` runs the same resolve-and-retry loop
17/// with `y = 5` (`WAWebSyncdServerSync`).
18const APP_STATE_PATCH_SEND_ATTEMPTS: usize = 5;
19/// How long a sync waits for another writer to release a collection.
20///
21/// A holder's honest worst case is not derivable: a patch send keeps the
22/// reservation across `fetch_app_state_with_retry_inner`, which may page up to
23/// `MAX_PAGINATION_ITERATIONS` IQs, so any bound short enough to be useful can
24/// expire on healthy work. What makes the bound safe is not its size but that
25/// running out is never lossy — the collection comes back `retryable` and the
26/// retry scheduler picks it up. This value covers the common case
27/// ([`APP_STATE_PATCH_SEND_ATTEMPTS`] attempts of [`DEFAULT_IQ_TIMEOUT`], plus
28/// one) so the scheduler is the exception rather than the rule. The bound exists
29/// because the sync worker's intake loop runs non-history tasks inline, so a
30/// reservation that never releases would stall everything queued behind it.
31const APP_STATE_RESERVATION_WAIT: Duration =
32    Duration::from_secs(DEFAULT_IQ_TIMEOUT.as_secs() * (APP_STATE_PATCH_SEND_ATTEMPTS as u64 + 1));
33/// Spacing between re-syncs of a collection a run left retryable, mirroring the
34/// syncd backoff WA Web applies to exactly this case (`WASyncdConst`:
35/// `BACKOFF_MIN_TIMEOUT` 1s, `BACKOFF_BASE` 2, `BACKOFF_MAX_TIMEOUT` 1h).
36const APP_STATE_RETRY_BACKOFF_MIN: Duration = Duration::from_secs(1);
37const APP_STATE_RETRY_BACKOFF_MAX: Duration = Duration::from_secs(60 * 60);
38/// How many spaced attempts one connection makes before leaving the collection
39/// to the next sync trigger. WA Web keeps retrying against a persisted
40/// first-failure timestamp and only gives up after two days; without that column
41/// this is what a single connection can promise, and the doubling already puts
42/// the last wait minutes out.
43const APP_STATE_RETRY_MAX_ROUNDS: u32 = 8;
44/// How many extra rounds the loop may burn waiting for a writer to release a
45/// collection before the attempt budget is spent.
46const APP_STATE_RETRY_ROUND_SLACK: u32 = 4;
47
48/// Delay before the attempt after `attempts` failures, doubling from
49/// [`APP_STATE_RETRY_BACKOFF_MIN`] and clamped at
50/// [`APP_STATE_RETRY_BACKOFF_MAX`].
51///
52/// Indexed by failed attempts rather than by loop iterations: rounds also pass
53/// while waiting for a socket or for another writer, and those are not failures
54/// to back off from. Letting them advance the exponent would put the next real
55/// attempt an hour away once the clamp is reached.
56fn app_state_retry_backoff(attempts: u32) -> Duration {
57    APP_STATE_RETRY_BACKOFF_MIN
58        .saturating_mul(2u32.saturating_pow(attempts))
59        .min(APP_STATE_RETRY_BACKOFF_MAX)
60}
61
62/// What a sync run actually achieved.
63///
64/// `Result<()>` could not tell "the collection is current" from "the connection
65/// went away before anything was asked", so every caller re-derived the
66/// difference from lifecycle flags — each with its own approximation, each
67/// missing a case the next one caught. The 429 and 503 stream errors are the
68/// ones they all missed: those clear `is_logged_in` without setting
69/// `expected_disconnect` or retiring the generation, so a run cut short there
70/// answered `Ok(())` and every proxy read it as done. The trigger is consumed
71/// by then, and nothing asks again.
72///
73/// Skipping is not a variant of finishing, so it is not a variant of `Ok(())`.
74#[derive(Debug, Clone, Copy, PartialEq, Eq)]
75pub(crate) enum SyncOutcome {
76    /// The server was asked, and answered until it had nothing more to send.
77    Completed,
78    /// Nothing was asked, or the run stopped with pages outstanding. Whatever
79    /// arrived is persisted, but the trigger has not been honoured and the work
80    /// is still owed.
81    Deferred,
82}
83
84/// Whether an attempt that ended this way leaves the collection still owed a
85/// sync.
86///
87/// One function rather than a condition rewritten at each caller, and phrased
88/// so that only [`SyncOutcome::Completed`] discharges the request: a deferral,
89/// an error, a variant added later — all keep it. The alternative is to
90/// enumerate the ways to fail, and the ways to fail is exactly the list that
91/// kept turning out to be one short.
92fn sync_still_owed(outcome: &Result<SyncOutcome>) -> bool {
93    !matches!(outcome, Ok(SyncOutcome::Completed))
94}
95
96/// The connection a piece of app-state work belongs to, and the clock it runs
97/// against.
98///
99/// Every sync path awaits round trips, and between them the socket can retire
100/// or the bootstrap's watchdog can fire. Checking that by hand at each await
101/// boundary is what this replaces: the checks drifted apart, some paths grew a
102/// check the next one forgot, and the same load answered two different
103/// questions. A scope makes the question single — [`Client::admits`] — and the
104/// answer typed, so a new boundary that forgets to ask is a boundary that
105/// cannot compile against the API.
106#[derive(Debug, Clone, Copy, PartialEq, Eq)]
107pub(crate) struct SyncScope {
108    /// The connection this work was started for.
109    generation: u64,
110    /// When the work stops being worth doing. Only the initial bootstrap sets
111    /// one, because only it runs under a watchdog that reconnects underneath.
112    deadline: Option<wacore::time::Instant>,
113}
114
115/// Why a scope no longer admits its work.
116#[derive(Debug, Clone, Copy, PartialEq, Eq)]
117pub(crate) enum ScopeLost {
118    /// The connection was replaced. Anything this work would publish or persist
119    /// belongs to a socket that is gone.
120    Retired,
121    /// The deadline passed. The watchdog either has reconnected or is about to,
122    /// so finishing would race it.
123    Expired,
124}
125
126impl SyncScope {
127    /// The generation this scope is pinned to. Used by logs and by tests that
128    /// need to retire a connection out from under a scope.
129    #[cfg_attr(not(test), allow(dead_code))]
130    pub(crate) fn generation(self) -> u64 {
131        self.generation
132    }
133
134    /// How long the work has left, or `None` when it is not on a clock.
135    pub(crate) fn remaining(self) -> Option<Duration> {
136        self.deadline
137            .map(|d| d.saturating_duration_since(wacore::time::Instant::now()))
138    }
139
140    /// Whether this scope is bound to a deadline at all, which is what
141    /// distinguishes the bootstrap from every background trigger.
142    pub(crate) fn is_bootstrap(self) -> bool {
143        self.deadline.is_some()
144    }
145
146    /// Move to the live connection, for work that must outlive a reconnect.
147    ///
148    /// Returns whether it moved. The caller decides what that costs: an outcome
149    /// computed for the old socket must not be published, and a retry that
150    /// rebinds can no longer settle the bootstrap it was scheduled by.
151    pub(crate) fn rebind(&mut self, to: u64) -> bool {
152        let moved = self.generation != to;
153        self.generation = to;
154        moved
155    }
156}
157
158/// The initial-bootstrap flag, tagged with the connection that last wrote it.
159///
160/// A plain flag cannot be settled safely. Deciding whether the writer still owns
161/// the connection and then writing are two operations, and every attempt to
162/// bridge them failed a different way: checking first missed a retirement in the
163/// gap, and rolling back afterwards clobbered whatever the replacement had
164/// written in the meantime. Packing the generation into the same word makes the
165/// pair a single compare-and-swap, so a writer from a retired connection simply
166/// loses — there is no window left to lose in.
167#[derive(Debug)]
168pub(crate) struct BootstrapGate(AtomicU64);
169
170impl BootstrapGate {
171    /// Armed by pairing before any connection exists, so generation zero owns
172    /// the first write and every later connection outranks it.
173    pub(crate) fn new(outstanding: bool) -> Self {
174        Self(AtomicU64::new(Self::encode(0, outstanding)))
175    }
176
177    const fn encode(generation: u64, outstanding: bool) -> u64 {
178        (generation << 1) | outstanding as u64
179    }
180
181    /// Whether the bootstrap still owes work, whoever last said so.
182    pub(crate) fn is_armed(&self) -> bool {
183        self.0.load(Ordering::Acquire) & 1 == 1
184    }
185
186    /// Arm for a fresh pairing, above every connection that already exists.
187    ///
188    /// Both bounds matter, and each was wrong on its own:
189    ///
190    /// Tagging above *every* generation made the gate unclearable. A freshly
191    /// paired client re-ran the 180s critical bootstrap on every connect for the
192    /// life of the session, however many times that bootstrap succeeded.
193    ///
194    /// Tagging at zero made it clearable by anything, including a scope opened
195    /// on the connection that is live when `pair-success` arrives. Its
196    /// `settle_bootstrap(scope, false)` would clear the arm before the pairing
197    /// reconnect ever happens, and the replacement connection would find nothing
198    /// owed and skip the sync pairing exists to request.
199    ///
200    /// One past the current generation is the bound that says what is meant:
201    /// nothing already in flight can clear this, and the next connection — the
202    /// one the forced 515 brings up — can.
203    ///
204    /// A floor rather than an assignment, because `current_generation` is a
205    /// sample and the tag is shared with [`Self::settle`]. An unconditional
206    /// store can lower a tag a newer connection already set — the one case where
207    /// arming would make the gate *easier* to clear — and it broke the rule
208    /// `settle_bootstrap` relies on, that the tag only ever moves forward. Both
209    /// writers now obey it.
210    pub(crate) fn arm_for_pairing(&self, current_generation: u64) {
211        let mut current = self.0.load(Ordering::Acquire);
212        loop {
213            // One past *both*, not just past the sample. `settle` admits an
214            // equal generation — that is a connection revising its own answer,
215            // which it is entitled to do — so a tag merely level with an
216            // existing one still lets that connection clear the arm.
217            let generation = (current >> 1).max(current_generation).saturating_add(1);
218            match self.0.compare_exchange_weak(
219                current,
220                Self::encode(generation, true),
221                Ordering::AcqRel,
222                Ordering::Acquire,
223            ) {
224                Ok(_) => return,
225                Err(observed) => current = observed,
226            }
227        }
228    }
229
230    /// Record `outstanding` on behalf of `generation`, unless a newer connection
231    /// has already had its say.
232    ///
233    /// Returns whether the write took. A stale writer losing here is the point,
234    /// not a failure.
235    pub(crate) fn settle(&self, generation: u64, outstanding: bool) -> bool {
236        let mut current = self.0.load(Ordering::Acquire);
237        loop {
238            // Strictly newer wins. Equal is the same connection revising its own
239            // answer, which it is entitled to do.
240            if (current >> 1) > generation {
241                return false;
242            }
243            match self.0.compare_exchange_weak(
244                current,
245                Self::encode(generation, outstanding),
246                Ordering::AcqRel,
247                Ordering::Acquire,
248            ) {
249                Ok(_) => return true,
250                Err(observed) => current = observed,
251            }
252        }
253    }
254}
255
256/// What kind of work holds a collection's reservation.
257///
258/// Skipping behind a holder is only sound when that holder is doing the same
259/// fetch. A patch send takes the same reservation and never fetches, so a sync
260/// that skipped behind one would silently drop the work its caller asked for.
261#[derive(Debug, Clone, Copy, PartialEq, Eq)]
262pub(crate) enum SyncHolder {
263    /// A collection sync: fetches from the server, then writes the collection's
264    /// version and mutation MACs.
265    Sync,
266    /// A patch send: writes the same rows, but never fetches.
267    PatchSend,
268}
269
270/// Why a sync did not get the collection reserved.
271#[derive(Debug, Clone, Copy, PartialEq, Eq)]
272pub(crate) enum ReservationSkip {
273    /// An equivalent sync already holds it, so it is already doing this work.
274    EquivalentSyncInFlight,
275    /// The holder did not release within the bound.
276    WaitTimedOut,
277}
278
279/// What finishing a sync, or the retries it schedules, settles beyond the
280/// collections themselves.
281#[derive(Debug, Clone, Copy, PartialEq, Eq)]
282pub(crate) enum SyncSettles {
283    /// Only the collections. Every trigger except the initial bootstrap: a
284    /// `server_sync` or a dirty bit says nothing about whether the first full
285    /// sync ever finished.
286    JustTheCollections,
287    /// The initial bootstrap too. Its gate stays armed while anything it asked
288    /// for is still outstanding, so recovering the last of them — even rounds
289    /// later — is what stands it down.
290    InitialSync,
291}
292
293/// What a sync should do when the collection is already reserved.
294#[derive(Debug, Clone, Copy, PartialEq, Eq)]
295pub(crate) enum ReservationWait {
296    /// Skip when an equivalent sync holds it. The batched sync asks for whatever
297    /// the server has for a set of collections, so another sync of the same
298    /// collection does this call's work and dropping it costs nothing.
299    SkipBehindSync,
300    /// Wait for whoever holds it. A consumer asking for one specific collection
301    /// is not made redundant by a sync already in flight: a full sync asks for
302    /// the snapshot while an incremental one asks for patches after the
303    /// persisted version, so skipping would turn the request into a no-op.
304    Always,
305}
306
307/// What a batched sync achieved, collection by collection.
308///
309/// A single `Ok`/`Err` for the whole batch cannot carry this. The initial
310/// bootstrap must not read "one collection was refused" as permission to
311/// dispatch Connected, while a background `server_sync` only wants to log it,
312/// and both read the same return value. So the call reports what happened and
313/// each caller decides what a partial result means for it.
314#[derive(Debug, Default, Clone, PartialEq, Eq)]
315pub(crate) struct BatchedSyncOutcome {
316    /// Applied and persisted.
317    pub(crate) synced: Vec<WAPatchName>,
318    /// The server refused the collection outright (400/404). Repeating the same
319    /// request gets the same answer, so retrying on its own never clears this.
320    pub(crate) fatal: Vec<WAPatchName>,
321    /// Did not sync, but a later attempt can: a retryable server error, a decode
322    /// key that never landed, or the iteration cap.
323    pub(crate) retryable: Vec<WAPatchName>,
324    /// Another holder had it reserved, so this call did nothing for it.
325    pub(crate) skipped: Vec<WAPatchName>,
326    /// Whether a collection IQ actually went out.
327    ///
328    /// Recorded at the send, not inferred from the buckets above. Inferring it
329    /// was wrong in a way that is worth keeping written down: `retryable` looks
330    /// like it means "the server was asked and the answer was retryable", and it
331    /// does hold those — but it also holds the collections a scope loss or a
332    /// reservation timeout dropped *before* the wire. A batch of nothing but
333    /// those reads as a real attempt under any bucket-based test.
334    reached_server: bool,
335}
336
337impl BatchedSyncOutcome {
338    /// Whether this call got as far as asking the server about anything.
339    ///
340    /// A round that reserved nothing sent no IQ and learned nothing. A retry
341    /// that charges an attempt for it spends its budget on whoever is holding
342    /// the collection — waiting again, dressed as trying.
343    pub(crate) fn reached_server(&self) -> bool {
344        self.reached_server
345    }
346
347    /// Record that a collection IQ went out. The single place that decides it.
348    fn note_reached_server(&mut self) {
349        self.reached_server = true;
350    }
351
352    /// Every collection this call did not leave synced, whatever the reason.
353    pub(crate) fn unsynced(&self) -> impl Iterator<Item = WAPatchName> + '_ {
354        self.fatal
355            .iter()
356            .chain(&self.retryable)
357            .chain(&self.skipped)
358            .copied()
359    }
360
361    /// True when every collection asked for came back synced.
362    pub(crate) fn all_synced(&self) -> bool {
363        self.unsynced().next().is_none()
364    }
365}
366
367/// In-flight dedup registry for app-state collection syncs.
368///
369/// Reservations carry a per-begin token so a release can only ever remove the
370/// reservation it belongs to: a stale task finishing after a reconnect cleared
371/// the registry cannot evict the newer generation's reservation for the same
372/// collection. Releases run from the guard's `Drop`, so a cancelled sync
373/// (timeout, abort, teardown) can never strand a collection as "in flight".
374/// The mutex is synchronous and never held across an await.
375pub(crate) struct SyncInFlight {
376    entries: std::sync::Mutex<HashMap<WAPatchName, (u64, SyncHolder)>>,
377    next_token: AtomicU64,
378    /// Notified whenever a reservation is released, so [`SyncInFlight::begin`]
379    /// can wait for one instead of spinning.
380    released: event_listener::Event,
381}
382
383impl SyncInFlight {
384    pub(crate) fn new() -> Arc<Self> {
385        Arc::new(Self {
386            entries: std::sync::Mutex::new(HashMap::new()),
387            next_token: AtomicU64::new(0),
388            released: event_listener::Event::new(),
389        })
390    }
391
392    /// Reserve `name` for `holder`, or report what already holds it.
393    pub(crate) fn try_begin_as(
394        self: &Arc<Self>,
395        name: WAPatchName,
396        holder: SyncHolder,
397    ) -> Result<SyncInFlightGuard, SyncHolder> {
398        let token = self.next_token.fetch_add(1, Ordering::Relaxed);
399        let mut entries = self.entries.lock().unwrap_or_else(|p| p.into_inner());
400        if let Some(&(_, current)) = entries.get(&name) {
401            return Err(current);
402        }
403        entries.insert(name, (token, holder));
404        Ok(SyncInFlightGuard {
405            registry: Arc::clone(self),
406            name,
407            token,
408        })
409    }
410
411    /// Reserve `name` for a sync, or `None` when anything already holds it.
412    ///
413    /// Test-only: production callers go through
414    /// [`Client::reserve_for_sync`](crate::client::Client::reserve_for_sync),
415    /// which has to tell an equivalent sync apart from a patch send. Keeping the
416    /// shorthand here lets the registry's own tests stay about the token and
417    /// wake-up rules rather than repeating a holder kind they do not exercise.
418    #[cfg(test)]
419    pub(crate) fn try_begin(self: &Arc<Self>, name: WAPatchName) -> Option<SyncInFlightGuard> {
420        self.try_begin_as(name, SyncHolder::Sync).ok()
421    }
422
423    /// Reserve `name`, waiting for the current holder to finish.
424    ///
425    /// A patch send cannot skip: it must not write the collection's version and
426    /// mutation MACs while a sync is writing them, and it needs the base a
427    /// concurrent sync is about to move. Cancelling this future simply stops
428    /// waiting; nothing is reserved until the guard is returned.
429    pub(crate) async fn begin(
430        self: &Arc<Self>,
431        name: WAPatchName,
432        holder: SyncHolder,
433    ) -> SyncInFlightGuard {
434        loop {
435            // Register the listener before re-checking, so a release landing
436            // between the check and the wait cannot be missed.
437            let released = self.released.listen();
438            if let Ok(guard) = self.try_begin_as(name, holder) {
439                return guard;
440            }
441            released.await;
442        }
443    }
444
445    /// Drop every reservation, releasing backing storage. Guards from before
446    /// the clear become no-ops thanks to the token check.
447    pub(crate) fn clear(&self) {
448        *self.entries.lock().unwrap_or_else(|p| p.into_inner()) = HashMap::new();
449        self.released.notify(usize::MAX);
450    }
451
452    pub(crate) fn len(&self) -> usize {
453        self.entries.lock().unwrap_or_else(|p| p.into_inner()).len()
454    }
455}
456
457pub(crate) struct SyncInFlightGuard {
458    registry: Arc<SyncInFlight>,
459    name: WAPatchName,
460    token: u64,
461}
462
463impl Drop for SyncInFlightGuard {
464    fn drop(&mut self) {
465        let mut entries = self
466            .registry
467            .entries
468            .lock()
469            .unwrap_or_else(|p| p.into_inner());
470        if entries
471            .get(&self.name)
472            .is_some_and(|&(t, _)| t == self.token)
473        {
474            entries.remove(&self.name);
475        }
476        drop(entries);
477        // Waiters are keyed by nothing, so wake all of them and let each
478        // re-check its own collection.
479        self.registry.released.notify(usize::MAX);
480    }
481}
482
483fn initial_app_state_key_retry(timeout: Duration) -> Duration {
484    (timeout / 2)
485        .max(Duration::from_millis(1))
486        .min(APP_STATE_KEY_PARTIAL_RETRY)
487}
488
489#[derive(Clone, Copy, Debug, Eq, PartialEq)]
490enum AppStateKeyRequestDelivery {
491    AllPeers,
492    SomePeers,
493}
494
495struct AppStateKeyRequestSchedule {
496    retry_at: wacore::time::Instant,
497    sent: bool,
498}
499
500enum AppStateKeyRequestProgress {
501    Scheduled(AppStateKeyRequestSchedule),
502    KeysReady,
503    TimedOut,
504}
505
506#[cold]
507#[inline(never)]
508fn classify_app_state_key_request_failures(
509    peer_count: usize,
510    failure_count: usize,
511    failures: &str,
512) -> Result<AppStateKeyRequestDelivery, anyhow::Error> {
513    if failure_count == peer_count {
514        return Err(anyhow::anyhow!(
515            "app-state key request failed for all {peer_count} peer device(s): {failures}"
516        ));
517    }
518    warn!(
519        "App-state key request failed for {failure_count}/{peer_count} peer device(s): {failures}"
520    );
521    Ok(AppStateKeyRequestDelivery::SomePeers)
522}
523
524#[cold]
525#[inline(never)]
526fn append_app_state_key_request_failure(
527    failures: &mut Option<String>,
528    message: std::fmt::Arguments<'_>,
529) {
530    let failures = failures.get_or_insert_with(String::new);
531    if !failures.is_empty() {
532        failures.push_str(", ");
533    }
534    let _ = std::fmt::Write::write_fmt(failures, message);
535}
536
537async fn collect_app_state_key_request_results<F, E>(
538    runtime: &dyn Runtime,
539    mut requests: futures::stream::FuturesUnordered<F>,
540    timeout: Duration,
541) -> Result<AppStateKeyRequestDelivery, anyhow::Error>
542where
543    F: Future<Output = (u16, std::result::Result<(), E>)>,
544    E: std::fmt::Display,
545{
546    use futures::StreamExt;
547    use futures::future::Either;
548
549    let peer_count = requests.len();
550    let mut failure_count = 0;
551    let mut failures = None;
552    let mut deadline = runtime.sleep(timeout);
553    while !requests.is_empty() {
554        match futures::future::select(requests.next(), deadline.as_mut()).await {
555            Either::Left((Some((device, result)), _)) => {
556                if let Err(error) = result {
557                    failure_count += 1;
558                    append_app_state_key_request_failure(
559                        &mut failures,
560                        format_args!("device {device}: {error}"),
561                    );
562                }
563            }
564            Either::Left((None, _)) => break,
565            Either::Right(((), _)) => {
566                let timed_out = requests.len();
567                failure_count += timed_out;
568                append_app_state_key_request_failure(
569                    &mut failures,
570                    format_args!("{timed_out} peer request(s) timed out"),
571                );
572                break;
573            }
574        }
575    }
576
577    if failure_count != 0 {
578        return classify_app_state_key_request_failures(
579            peer_count,
580            failure_count,
581            failures.as_deref().unwrap_or_default(),
582        );
583    }
584    Ok(AppStateKeyRequestDelivery::AllPeers)
585}
586
587async fn app_state_keys_available(
588    backend: &dyn crate::store::traits::Backend,
589    key_ids: &[Vec<u8>],
590) -> bool {
591    for key_id in key_ids {
592        if backend.get_sync_key(key_id).await.ok().flatten().is_none() {
593            return false;
594        }
595    }
596    true
597}
598
599async fn remove_available_app_state_keys(
600    backend: &dyn crate::store::traits::Backend,
601    missing: &mut Vec<Vec<u8>>,
602) {
603    let mut index = 0;
604    while index < missing.len() {
605        if backend
606            .get_sync_key(&missing[index])
607            .await
608            .ok()
609            .flatten()
610            .is_some()
611        {
612            missing.swap_remove(index);
613        } else {
614            index += 1;
615        }
616    }
617}
618
619fn finalize_app_state_key_request_peers(
620    mut peers: Vec<Jid>,
621    current_device: u16,
622    primary: Jid,
623) -> Result<Vec<Jid>, anyhow::Error> {
624    // WA Web derives every sibling address from the account's PN namespace.
625    for peer in &mut peers {
626        peer.user.clone_from(&primary.user);
627        peer.server = primary.server;
628        peer.agent = primary.agent;
629        peer.integrator = primary.integrator;
630    }
631    peers.retain(|jid| jid.device != current_device);
632    wacore::types::jid::sort_dedup_by_device(&mut peers);
633    if peers.is_empty() && current_device != primary.device {
634        peers.push(primary);
635    }
636    if peers.is_empty() {
637        return Err(anyhow::anyhow!(
638            "no peer devices available for app-state key request"
639        ));
640    }
641    Ok(peers)
642}
643
644impl Client {
645    pub(crate) async fn get_app_state_processor(&self) -> Arc<AppStateProcessor> {
646        let mut guard = self.app_state_processor.lock().await;
647        if let Some(proc) = guard.as_ref() {
648            return proc.clone();
649        }
650        debug!("Initializing AppStateProcessor for the first time.");
651        let proc = Arc::new(AppStateProcessor::new(
652            self.persistence_manager.backend(),
653            self.runtime.clone(),
654        ));
655        *guard = Some(proc.clone());
656        proc
657    }
658
659    /// Pre-download every external blob (snapshots + patch external mutations)
660    /// referenced by `patch_lists`, keyed by directPath, fetching concurrently
661    /// (bounded by [`APPSTATE_BLOB_DOWNLOAD_CONCURRENCY`]). A failed download is
662    /// logged and omitted; the later inline step surfaces the missing blob as
663    /// before. Mirrors WA Web's parallel syncd blob fetch.
664    async fn pre_download_external_blobs(
665        &self,
666        patch_lists: &[wacore::appstate::patch_decode::PatchList],
667    ) -> HashMap<String, Vec<u8>> {
668        use futures::StreamExt;
669
670        // Kept only so a failed download logs the right message (snapshot vs patch).
671        enum BlobKind {
672            Snapshot(WAPatchName),
673            Mutation(u64),
674        }
675
676        // Clone the (small) blob ref into each job so the task owns its input and
677        // captures only `&self` (keeps the future Send); the directPath is
678        // recovered from the moved `ext` after the fetch. Dedup by directPath so
679        // patches sharing a blob don't fetch it twice into the same map key.
680        let mut jobs: Vec<(wa::ExternalBlobReference, BlobKind)> = Vec::new();
681        let mut seen_paths: HashSet<&str> = HashSet::new();
682        for pl in patch_lists {
683            if let Some(ext) = &pl.snapshot_ref
684                && let Some(path) = ext.direct_path.as_deref()
685                && seen_paths.insert(path)
686            {
687                jobs.push((ext.clone(), BlobKind::Snapshot(pl.name)));
688            }
689            for patch in &pl.patches {
690                if let Some(ext) = patch.external_mutations.as_option()
691                    && let Some(path) = ext.direct_path.as_deref()
692                    && seen_paths.insert(path)
693                {
694                    let v = patch
695                        .version
696                        .as_option()
697                        .and_then(|v| v.version)
698                        .unwrap_or(0);
699                    jobs.push((ext.clone(), BlobKind::Mutation(v)));
700                }
701            }
702        }
703
704        if jobs.is_empty() {
705            return HashMap::new();
706        }
707
708        let mut pre_downloaded = HashMap::with_capacity(jobs.len());
709        let results = futures::stream::iter(jobs.into_iter().map(|(ext, kind)| async move {
710            let bytes = self.download(&ext).await;
711            // directPath presence was checked when the job was built.
712            (ext.direct_path, kind, bytes)
713        }))
714        .buffer_unordered(APPSTATE_BLOB_DOWNLOAD_CONCURRENCY)
715        .collect::<Vec<_>>()
716        .await;
717
718        for (path, kind, res) in results {
719            match res {
720                Ok(bytes) => {
721                    if let BlobKind::Mutation(v) = kind {
722                        debug!(target: "Client/AppState", "Downloaded external mutations for patch v{} ({} bytes)", v, bytes.len());
723                    } else {
724                        debug!(target: "Client/AppState", "Downloaded external snapshot ({} bytes)", bytes.len());
725                    }
726                    if let Some(path) = path {
727                        pre_downloaded.insert(path, bytes);
728                    }
729                }
730                Err(e) => match kind {
731                    BlobKind::Snapshot(name) => {
732                        warn!("Failed to download external snapshot for {:?}: {e}", name)
733                    }
734                    BlobKind::Mutation(v) => {
735                        warn!(
736                            "Failed to download external mutations for patch v{}: {e}",
737                            v
738                        )
739                    }
740                },
741            }
742        }
743
744        pre_downloaded
745    }
746
747    pub(crate) fn start_sync_task_worker(
748        self: &Arc<Self>,
749        receiver: async_channel::Receiver<MajorSyncTask>,
750    ) {
751        const HISTORY_SYNC_CONCURRENCY: usize = 2;
752
753        let worker_client = Arc::downgrade(self);
754        let history_permits = Arc::new(async_lock::Semaphore::new(HISTORY_SYNC_CONCURRENCY));
755        self.runtime
756            .spawn(Box::pin(async move {
757                while let Ok(task) = receiver.recv().await {
758                    let Some(worker_client) = worker_client.upgrade() else {
759                        break;
760                    };
761
762                    if matches!(task, MajorSyncTask::HistorySync { .. }) {
763                        let permit = history_permits.acquire_arc().await;
764                        let task_client = worker_client.clone();
765                        worker_client
766                            .runtime
767                            .spawn(Box::pin(async move {
768                                let _permit = permit;
769                                task_client.process_sync_task(task).await;
770                            }))
771                            .detach();
772                    } else {
773                        worker_client.process_sync_task(task).await;
774                    }
775                }
776                info!(
777                    "Sync worker intake loop finished (detached history-sync tasks may still be running)."
778                );
779            }))
780            .detach();
781    }
782
783    /// Public entry point for processing [`MajorSyncTask`] from the sync channel.
784    #[cfg_attr(
785        feature = "tracing",
786        tracing::instrument(name = "wa.appstate.sync_task", level = "debug", skip_all)
787    )]
788    pub async fn process_sync_task(self: &Arc<Self>, task: MajorSyncTask) {
789        match task {
790            MajorSyncTask::HistorySync {
791                message_id,
792                notification,
793                mut tracker,
794            } => {
795                self.process_history_sync_task_tracked(message_id, *notification, &mut tracker)
796                    .await;
797            }
798            MajorSyncTask::AppStateSync { name, full_sync } => {
799                // Reserve the collection like every other sync path does.
800                // Unreserved, this writes the same version and mutation-MAC
801                // rows as a concurrent `sync_collections_batched`, leaving the
802                // ltHash disagreeing with the MAC store.
803                //
804                // Waits for any holder: this is a consumer asking for one named
805                // collection, and a sync already in flight is not necessarily
806                // fetching what it asked for.
807                let _guard = match self
808                    .reserve_for_sync(name, ReservationWait::Always, self.sync_scope(None))
809                    .await
810                {
811                    Ok(guard) => guard,
812                    Err(ReservationSkip::EquivalentSyncInFlight) => {
813                        debug!(target: "Client/AppState", "Skipping app state sync task {name:?}: an equivalent sync holds it");
814                        return;
815                    }
816                    // The bound ran out, so nobody is covering this collection
817                    // and the consumer asked for it. Retried as the task it was,
818                    // not as a plain collection sync: the batched path asks for a
819                    // snapshot only when the persisted version is zero, so
820                    // rescheduling a `full_sync` request that way would quietly
821                    // downgrade it to incremental and the snapshot would never
822                    // happen.
823                    Err(ReservationSkip::WaitTimedOut) => {
824                        warn!(target: "Client/AppState", "Gave up waiting to sync {name:?}; scheduling a retry");
825                        self.schedule_app_state_task_retry(name, full_sync);
826                        return;
827                    }
828                };
829                // The consumer asked once and nothing else will ask again, so
830                // this is the point that has to keep the request alive — for
831                // every way of not having synced, not just the ones the guards
832                // catch. A connection lost while the collection IQ is in flight
833                // is reported by `send_iq` as an error rather than as a
834                // deferral, and an error here used to be logged and dropped.
835                let outcome = self.process_app_state_sync_task(name, full_sync).await;
836                match &outcome {
837                    Err(e) => self.log_sync_error(&format!("app state sync for {name:?}"), e),
838                    Ok(SyncOutcome::Deferred) => {
839                        debug!(target: "Client/AppState", "App state sync for {name:?} was deferred")
840                    }
841                    Ok(SyncOutcome::Completed) => {}
842                }
843                if sync_still_owed(&outcome) {
844                    self.schedule_app_state_task_retry(name, full_sync);
845                }
846            }
847        }
848    }
849
850    /// Sync one collection, retrying a missing decode key and a locked DB.
851    ///
852    /// Takes no in-flight reservation of its own: the only caller is the patch
853    /// send, which already holds the collection's reservation for the whole
854    /// build-send-resolve cycle and would deadlock on its own guard. The
855    /// batched path reserves its collections in
856    /// [`sync_collections_batched`](Self::sync_collections_batched).
857    #[cfg_attr(feature = "tracing", tracing::instrument(name = "wa.appstate.fetch", level = "debug", skip_all, fields(name = ?name), err(Debug)))]
858    async fn fetch_app_state_with_retry_inner(&self, name: WAPatchName) -> Result<()> {
859        let _t = wacore::telemetry::timer(wacore::telemetry::APPSTATE_SYNC_DURATION);
860        let mut attempt = 0u32;
861        loop {
862            attempt += 1;
863            // full_sync=false lets process_app_state_sync_task auto-detect:
864            // version 0 → snapshot (full sync), version > 0 → incremental patches.
865            // Matches WA Web which only requests snapshot when version is undefined.
866            let res = self.process_app_state_sync_task(name, false).await;
867            match res {
868                Ok(SyncOutcome::Completed) => {
869                    wacore::telemetry::appstate_sync("ok");
870                    return Ok(());
871                }
872                // The send succeeded and the collection is now behind its own
873                // head, which is precisely the state this re-sync exists to
874                // repair. Counting it as `ok` reported a repair that did not
875                // happen; the retry is what actually carries it to the
876                // replacement connection.
877                Ok(SyncOutcome::Deferred) => {
878                    wacore::telemetry::appstate_sync("deferred");
879                    if let Some(client) = self.self_weak.get().and_then(|w| w.upgrade()) {
880                        client.schedule_app_state_task_retry(name, false);
881                    }
882                    return Ok(());
883                }
884                Err(e) => {
885                    if e.downcast_ref::<crate::appstate_sync::AppStateSyncError>()
886                        .is_some_and(|ase| {
887                            matches!(ase, crate::appstate_sync::AppStateSyncError::KeyNotFound(_))
888                        })
889                        && attempt == 1
890                    {
891                        if !self.initial_app_state_keys_received.load(Ordering::Relaxed) {
892                            debug!(target: "Client/AppState", "App state key missing for {:?}; waiting up to 10s for key share then retrying", name);
893                            if rt_timeout(
894                                &*self.runtime,
895                                Duration::from_secs(10),
896                                self.initial_keys_synced_notifier.listen(),
897                            )
898                            .await
899                            .is_err()
900                            {
901                                warn!(target: "Client/AppState", "Timeout waiting for key share for {:?}; retrying anyway", name);
902                            }
903                        }
904                        continue;
905                    }
906                    let is_db_locked = e
907                        .downcast_ref::<wacore::store::error::StoreError>()
908                        .is_some_and(|se| se.is_database_busy_or_locked())
909                        || e.downcast_ref::<crate::appstate_sync::AppStateSyncError>()
910                            .is_some_and(|ase| match ase {
911                                crate::appstate_sync::AppStateSyncError::Store(se) => {
912                                    se.is_database_busy_or_locked()
913                                }
914                                _ => false,
915                            });
916                    if is_db_locked && attempt < APP_STATE_RETRY_MAX_ATTEMPTS {
917                        let backoff = Duration::from_millis(200 * attempt as u64 + 150);
918                        warn!(target: "Client/AppState", "Attempt {} for {:?} failed due to locked DB; backing off {:?} and retrying", attempt, name, backoff);
919                        self.runtime.sleep(backoff).await;
920                        continue;
921                    }
922                    wacore::telemetry::appstate_sync("fail");
923                    return Err(e);
924                }
925            }
926        }
927    }
928
929    /// Log and surface a sync whose caller has no decision to make.
930    ///
931    /// The initial bootstrap is the only path that changes what it does based on
932    /// the outcome. Every other caller just needs an incomplete sync to be
933    /// visible instead of swallowed, which is how a collection could stop
934    /// syncing without anything saying so.
935    ///
936    /// Readiness is read, not assumed: a `syncd_app_state` dirty bit can start a
937    /// sync while offline stanzas are still being processed, so this can run
938    /// before the connection ever reaches `Connected`.
939    ///
940    /// `generation` is the connection the sync was started on. Checking it here
941    /// rather than at each call site is deliberate: every caller awaits a round
942    /// trip before reporting, and one that forgot would publish a retired
943    /// socket's refusal to a consumer whose documented response is to log out or
944    /// force a recovery — on the live session.
945    ///
946    /// `requested` is what the sync was asked to cover. It is only needed for
947    /// the failure arm: a top-level error produces no outcome at all, so there
948    /// are no per-collection buckets to retry from, and for a dirty-bit request
949    /// the trigger is already consumed — nothing would ask again.
950    pub(crate) fn report_background_sync(
951        self: &Arc<Self>,
952        label: &str,
953        scope: SyncScope,
954        settles: SyncSettles,
955        requested: &[WAPatchName],
956        result: Result<BatchedSyncOutcome>,
957    ) {
958        self.report_background_sync_stranded(label, scope, settles, requested, false, result)
959    }
960
961    /// [`report_background_sync`](Self::report_background_sync) for a caller that
962    /// already knows something is unrecoverable outside this result.
963    ///
964    /// A collection the server refused is not in `requested` — retrying it is
965    /// pointless — but it is still why the bootstrap is unfinished, and a later
966    /// clean round would otherwise settle the gate on its behalf.
967    pub(crate) fn report_background_sync_stranded(
968        self: &Arc<Self>,
969        label: &str,
970        scope: SyncScope,
971        settles: SyncSettles,
972        requested: &[WAPatchName],
973        stranded_elsewhere: bool,
974        result: Result<BatchedSyncOutcome>,
975    ) {
976        if let Err(lost) = self.admits(scope) {
977            debug!(target: "Client/AppState", "{label}: outcome dropped ({lost:?})");
978            return;
979        }
980        match result {
981            Ok(outcome) if outcome.all_synced() => {}
982            Ok(outcome) => {
983                warn!(
984                    target: "Client/AppState",
985                    "{label}: incomplete (fatal={:?} retryable={:?} skipped={:?})",
986                    outcome.fatal, outcome.retryable, outcome.skipped
987                );
988                self.dispatch_app_state_sync_failed(
989                    &outcome,
990                    self.is_ready.load(Ordering::Relaxed),
991                );
992                // Seeded with what this outcome already stranded. A fatal or
993                // skipped collection here is not in the retryable list the
994                // scheduler carries, so without this the scheduler would start
995                // clean and let a later successful round settle the bootstrap
996                // for collections that never synced and will not be retried.
997                let already_stranded =
998                    stranded_elsewhere || !outcome.fatal.is_empty() || !outcome.skipped.is_empty();
999                self.schedule_app_state_retry(outcome.retryable, scope, settles, already_stranded);
1000            }
1001            Err(e) => {
1002                self.log_sync_error(label, &e);
1003                // An IQ timeout, a malformed response, a failed blob fetch or a
1004                // store error takes the whole batch down without producing
1005                // buckets, so nothing above reaches the scheduler. Retry what
1006                // was asked for: the collections are no less stale for the
1007                // failure having been global, and the request that prompted it
1008                // is not coming back.
1009                //
1010                // Unasserted: the observable is a detached task that sleeps
1011                // before doing anything, and the two probes tried for it both
1012                // passed with the requeue removed.
1013                self.schedule_app_state_retry(
1014                    requested.to_vec(),
1015                    scope,
1016                    settles,
1017                    stranded_elsewhere,
1018                );
1019            }
1020        }
1021    }
1022
1023    /// Wait until there is a connection to work on, or the client is finished.
1024    ///
1025    /// Returns whether one arrived. Bounded by the client's own lifetime rather
1026    /// than by a duration: every number tried here was wrong in one direction or
1027    /// the other, because the reconnect backoff is jittered, capped at 900s, and
1028    /// followed by a handshake — there is no honest constant. Terminal is the
1029    /// condition that actually means "stop waiting", and it is already tracked.
1030    ///
1031    /// Waits for [`Self::can_reach_server`], not for `Connected`: the caller's
1032    /// question is whether its IQ can be sent and answered, and the `Connected`
1033    /// notifier additionally waits for the critical sync, so a retry would sit
1034    /// through a bootstrap it may itself be part of.
1035    pub(crate) async fn await_connection(&self) -> bool {
1036        loop {
1037            if let Some(verdict) = self.connection_wait_verdict() {
1038                return verdict;
1039            }
1040            // Both registered before the re-check, so a transition landing in the
1041            // gap is not missed. Socket readiness alone is not enough to wait on:
1042            // it fires before login, and nothing fires at all when the client
1043            // stops without a replacement socket — a wait on it alone parks
1044            // forever, holding the `Arc<Client>` whose drop is the only other
1045            // way this task ends.
1046            let ready = self.socket_ready_notifier.listen();
1047            let session = self.session_state_notifier.listen();
1048            if let Some(verdict) = self.connection_wait_verdict() {
1049                return verdict;
1050            }
1051            futures::pin_mut!(ready);
1052            futures::pin_mut!(session);
1053            // A notification that does not settle the question simply loops: the
1054            // wait ends on the state, never on one event.
1055            futures::future::select(ready, session).await;
1056        }
1057    }
1058
1059    /// Whether [`Self::await_connection`] can stop, and with what answer.
1060    fn connection_wait_verdict(&self) -> Option<bool> {
1061        // Terminal first. The two are not mutually exclusive during a teardown:
1062        // the stream-error paths set the terminal flags before they clear
1063        // `is_logged_in` and close the transport, so asking about reachability
1064        // first hands out a connection that is already ending.
1065        if self.is_terminal() {
1066            return Some(false);
1067        }
1068        if self.can_reach_server() {
1069            return Some(true);
1070        }
1071        // A client with no supervision loop has no reader, so nothing will ever
1072        // answer an IQ and no `<success>` will ever arrive. That is not terminal
1073        // — the connection is fine and the application may still use it — but it
1074        // is unwaitable, and the alternative is parking until the process ends.
1075        if !self.is_running.load(Ordering::Relaxed) {
1076            return Some(false);
1077        }
1078        None
1079    }
1080
1081    /// Open a scope for work starting now on the live connection.
1082    pub(crate) fn sync_scope(&self, deadline: Option<wacore::time::Instant>) -> SyncScope {
1083        SyncScope {
1084            generation: self.connection_generation.load(Ordering::SeqCst),
1085            deadline,
1086        }
1087    }
1088
1089    /// Whether `scope`'s work may still proceed.
1090    ///
1091    /// The single place either question is asked. Call it at every boundary that
1092    /// follows an await and precedes something observable — a write, a dispatch,
1093    /// a scheduled retry — and nowhere else, so there is one answer per boundary
1094    /// rather than one per author.
1095    pub(crate) fn admits(&self, scope: SyncScope) -> Result<(), ScopeLost> {
1096        if self.connection_generation.load(Ordering::SeqCst) != scope.generation {
1097            return Err(ScopeLost::Retired);
1098        }
1099        if let Some(deadline) = scope.deadline
1100            && wacore::time::Instant::now() >= deadline
1101        {
1102            return Err(ScopeLost::Expired);
1103        }
1104        Ok(())
1105    }
1106
1107    /// Record whether the initial bootstrap still has work outstanding.
1108    ///
1109    /// The gate is shared across connections, so a task from a retired one must
1110    /// not touch it: clearing would let the live connection skip a bootstrap it
1111    /// still needs, and arming would cost it one it does not. Routing every
1112    /// write through here is what keeps that check from being the caller's job —
1113    /// it was forgotten twice when it was.
1114    pub(crate) fn settle_bootstrap(&self, scope: SyncScope, outstanding: bool) {
1115        // Two guards, closing two different holes, which is why neither alone
1116        // was enough on the previous attempts.
1117        //
1118        // The admission check keeps a writer whose connection is already gone
1119        // from having a say at all. The compare-and-swap keeps any write from
1120        // clobbering one made on behalf of a newer connection. Between them the
1121        // worst case is a stale write that slips through the check and lands
1122        // before the replacement settles — and the replacement's own settle then
1123        // outranks it permanently, because the tag only ever moves forward.
1124        //
1125        // Expiry is deliberately not consulted: running out of time is exactly
1126        // when the bootstrap has to stay armed, and that is a write this
1127        // connection is still entitled to make.
1128        if self.admits(scope) == Err(ScopeLost::Retired) {
1129            debug!(
1130                target: "Client/AppState",
1131                "Bootstrap gate left alone: connection {} retired", scope.generation
1132            );
1133            return;
1134        }
1135        if !self
1136            .needs_initial_full_sync
1137            .settle(scope.generation, outstanding)
1138        {
1139            debug!(
1140                target: "Client/AppState",
1141                "Bootstrap gate left to a newer connection than {}", scope.generation
1142            );
1143            return;
1144        }
1145        if outstanding {
1146            warn!(target: "Client/AppState", "Initial App State Sync incomplete; bootstrap stays armed");
1147        } else {
1148            debug!(target: "Client/AppState", "Initial App State Sync completed.");
1149        }
1150    }
1151
1152    /// Report an incomplete batched sync to consumers.
1153    ///
1154    /// `connected` says whether the client went on to dispatch `Connected`
1155    /// anyway, which is the difference between "degraded but usable" and "still
1156    /// retrying", and is the only part a consumer cannot infer from the buckets.
1157    pub(crate) fn dispatch_app_state_sync_failed(
1158        &self,
1159        outcome: &BatchedSyncOutcome,
1160        connected: bool,
1161    ) {
1162        let names = |v: &[WAPatchName]| v.iter().map(|n| n.as_str().to_string()).collect();
1163        self.core.event_bus.dispatch(Event::AppStateSyncFailed(
1164            crate::types::events::AppStateSyncFailed::builder()
1165                .fatal(names(&outcome.fatal))
1166                .retryable(names(&outcome.retryable))
1167                .skipped(names(&outcome.skipped))
1168                .connected(connected)
1169                .build(),
1170        ));
1171    }
1172
1173    /// Retry one consumer-issued sync task, preserving the mode it asked for.
1174    ///
1175    /// Separate from [`schedule_app_state_retry`](Self::schedule_app_state_retry)
1176    /// because a task carries `full_sync`, and the batched path cannot express
1177    /// it: that one requests a snapshot only when the persisted version is zero,
1178    /// so a full sync routed through it becomes incremental and the caller's
1179    /// snapshot silently never happens.
1180    fn schedule_app_state_task_retry(self: &Arc<Self>, name: WAPatchName, full_sync: bool) {
1181        let mut scope = self.sync_scope(None);
1182        let client = self.clone();
1183        self.runtime.spawn_detached(Box::pin(async move {
1184            // Attempts and rounds are counted separately: a wait that ran out
1185            // never reached the server, so spending an attempt on it would let a
1186            // long-lived holder burn the budget without the sync being tried
1187            // once. Rounds still bound the loop overall.
1188            let mut attempts = 0u32;
1189            for _ in 0..APP_STATE_RETRY_MAX_ROUNDS * APP_STATE_RETRY_ROUND_SLACK {
1190                if attempts >= APP_STATE_RETRY_MAX_ROUNDS {
1191                    break;
1192                }
1193                client.runtime.sleep(app_state_retry_backoff(attempts)).await;
1194                if client.is_terminal() {
1195                    debug!(target: "Client/AppState", "App state task retry cancelled: client is finished");
1196                    return;
1197                }
1198                // Wait the planned reconnect out rather than spending an attempt
1199                // on it. `process_app_state_sync_task` defers at its own guard
1200                // without contacting the server, so attempting here would burn
1201                // the budget on rounds that never asked anything.
1202                //
1203                // Waited on rather than polled, and on the connection itself
1204                // rather than on any particular reason there is not one:
1205                // `reconnect()` tears down without setting `expected_disconnect`,
1206                // so asking about the reason misses the ordinary case.
1207                if !client.await_connection().await {
1208                    debug!(target: "Client/AppState", "App state task retry cancelled: client is finished");
1209                    return;
1210                }
1211
1212                // Rebound after the wait, not before: the generation is only
1213                // final once `<success>` has landed, which is what the wait
1214                // waits for. Binding first pinned the outgoing connection's
1215                // generation, and `admits` then rejected every attempt made on
1216                // the replacement — rounds spent without one request being sent.
1217                //
1218                // Rebound rather than discarded, because nothing on the new
1219                // connection re-issues a consumer's task, and a `full_sync` one
1220                // is the snapshot request this scheduler exists to keep alive.
1221                scope.rebind(client.connection_generation.load(Ordering::SeqCst));
1222
1223                let guard = match client
1224                    .reserve_for_sync(name, ReservationWait::Always, scope)
1225                    .await
1226                {
1227                    Ok(guard) => guard,
1228                    // Someone equivalent picked it up, so the request is covered.
1229                    Err(ReservationSkip::EquivalentSyncInFlight) => return,
1230                    Err(ReservationSkip::WaitTimedOut) => {
1231                        warn!(target: "Client/AppState", "Still waiting on the writer holding {name:?}");
1232                        continue;
1233                    }
1234                };
1235                // The reservation wait is itself an await, so the connection can
1236                // go inside it and the answer above is already stale. Asked
1237                // again before an attempt is counted, so a round spent waiting
1238                // is not charged as one spent asking.
1239                if !client.can_reach_server() || client.admits(scope).is_err() {
1240                    debug!(target: "Client/AppState", "Dropping the {name:?} attempt: state moved while reserving");
1241                    drop(guard);
1242                    continue;
1243                }
1244                attempts += 1;
1245                let outcome = client.process_app_state_sync_task(name, full_sync).await;
1246                if let Err(e) = &outcome {
1247                    drop(guard);
1248                    client.log_sync_error("app state task retry", e);
1249                }
1250                // The callee says which happened, so this no longer
1251                // reconstructs it from lifecycle flags. That proxy read `Ok(())`
1252                // as done for every cut-short run whose reason it did not model
1253                // — a 429 or 503 clears `is_logged_in` without setting
1254                // `expected_disconnect` — and the request was lost with the
1255                // `full_sync` snapshot it carried.
1256                //
1257                // `admits` still gates it: a run that completed against a
1258                // retired socket completed for somebody else.
1259                if !sync_still_owed(&outcome) && client.admits(scope).is_ok() {
1260                    return;
1261                }
1262                debug!(target: "Client/AppState", "The {name:?} attempt did not settle it; keeping it queued");
1263            }
1264            warn!(
1265                target: "Client/AppState",
1266                "App state task for {name:?} still unsynced after {attempts} attempts"
1267            );
1268        }));
1269    }
1270
1271    /// Re-sync collections a run left retryable, spaced the way WA Web spaces
1272    /// the same case (`WASyncdConst`: 1s base, doubling, capped at an hour).
1273    ///
1274    /// A transient error takes the collection out of the batched loop rather
1275    /// than being re-asked inside it, which is what WA Web does — but WA Web
1276    /// hands it to a retry state machine afterwards, and without one a single
1277    /// 500 would leave the collection stale until some unrelated trigger came
1278    /// along. This is that machine, minus the persisted two-day expiry.
1279    ///
1280    /// The scope is taken from the caller, which has already validated it, so
1281    /// dispatching the failure event in between — consumer handlers run
1282    /// synchronously and may disconnect — cannot silently rebind these retries
1283    /// to whatever replaced the connection.
1284    ///
1285    /// `already_stranded` carries forward what the originating outcome left
1286    /// behind but did not hand over: a refusal, or a collection another writer
1287    /// held. Those are not in `collections`, so without it a later clean round
1288    /// would look like everything recovered.
1289    pub(crate) fn schedule_app_state_retry(
1290        self: &Arc<Self>,
1291        collections: Vec<WAPatchName>,
1292        scope: SyncScope,
1293        settles: SyncSettles,
1294        already_stranded: bool,
1295    ) {
1296        if collections.is_empty() {
1297            return;
1298        }
1299        let client = self.clone();
1300        self.runtime.spawn_detached(Box::pin(async move {
1301            let mut scope = scope;
1302            let mut settles = settles;
1303            let mut pending = collections;
1304            // Sticky, and only for misses that do not come back. A round can
1305            // strand one collection as fatal or skipped while another stays
1306            // retryable; if that last one then succeeds, its own `all_synced()`
1307            // is true even though the first never synced. Retryable ones are
1308            // exactly what the next round carries, so counting them would keep
1309            // the gate armed forever after any transient round.
1310            let mut left_unresolved = already_stranded;
1311            // Attempts and rounds are counted separately. A round spent waiting
1312            // for a socket never reached the server, and the reconnect backoff
1313            // runs far longer than these delays do, so charging it as an attempt
1314            // would exhaust the budget before the replacement connection exists
1315            // and drop a trigger that is already consumed.
1316            let mut attempts = 0u32;
1317            for _ in 0..APP_STATE_RETRY_MAX_ROUNDS * APP_STATE_RETRY_ROUND_SLACK {
1318                if attempts >= APP_STATE_RETRY_MAX_ROUNDS {
1319                    break;
1320                }
1321                client.runtime.sleep(app_state_retry_backoff(attempts)).await;
1322                // Waited for, not charged for. Without this the loop spends an
1323                // attempt on every offline round, and the whole budget is gone
1324                // long before a reconnect that backs off in minutes returns —
1325                // taking a trigger the server already considers handled.
1326                if !client.await_connection().await {
1327                    debug!(target: "Client/AppState", "App state retry cancelled: client is finished");
1328                    return;
1329                }
1330
1331                // Rebound after the wait, not before: the connection that
1332                // arrives is the one this attempt belongs to.
1333                if scope.rebind(client.connection_generation.load(Ordering::SeqCst)) {
1334                    // The work carries over; the authority to settle does not.
1335                    // This run no longer belongs to the bootstrap that scheduled
1336                    // it, so it must never stand that gate down.
1337                    settles = SyncSettles::JustTheCollections;
1338                }
1339
1340                // The same guard the task retry makes, and for the same reason:
1341                // the wait can resolve just as a planned reconnect begins, and
1342                // the generation does not bump until cleanup. `admits(scope)` is
1343                // fresh from the rebind above and would say yes to that retiring
1344                // socket, so reachability is the question that catches it.
1345                if !client.can_reach_server() {
1346                    debug!(target: "Client/AppState", "Dropping the batched {pending:?} attempt: the connection is retiring");
1347                    continue;
1348                }
1349
1350                debug!(
1351                    target: "Client/AppState",
1352                    "Retrying app state {pending:?} (attempt {}/{APP_STATE_RETRY_MAX_ROUNDS})",
1353                    attempts + 1
1354                );
1355                let result = client
1356                    .sync_collections_batched(pending.clone(), scope)
1357                    .await;
1358                // Charged after the call, for what reached the server. A round
1359                // where every collection was held by another writer sent no IQ;
1360                // eight of those in a row would otherwise spend the whole budget
1361                // on someone else's patch send and drop a consumed trigger.
1362                let reached_server = match &result {
1363                    Ok(outcome) => outcome.reached_server(),
1364                    Err(_) => true,
1365                };
1366                if reached_server {
1367                    attempts += 1;
1368                }
1369
1370                // Rebinding here drops the outcome, not the work: publishing a
1371                // retired socket's refusal could have a consumer log out the
1372                // live session, while abandoning the names would strand them.
1373                if scope.rebind(client.connection_generation.load(Ordering::SeqCst)) {
1374                    debug!(target: "Client/AppState", "App state retry outcome dropped; rebound");
1375                    settles = SyncSettles::JustTheCollections;
1376                    continue;
1377                }
1378
1379                match result {
1380                    Ok(outcome) => {
1381                        if !outcome.all_synced() {
1382                            client.dispatch_app_state_sync_failed(
1383                                &outcome,
1384                                client.is_ready.load(Ordering::Relaxed),
1385                            );
1386                        }
1387                        if !outcome.fatal.is_empty() || !outcome.skipped.is_empty() {
1388                            left_unresolved = true;
1389                        }
1390                        pending = outcome.retryable;
1391                        if pending.is_empty() {
1392                            if settles == SyncSettles::InitialSync {
1393                                client.settle_bootstrap(scope, left_unresolved);
1394                            }
1395                            return;
1396                        }
1397                    }
1398                    Err(e) => client.log_sync_error("app state retry", &e),
1399                }
1400            }
1401            warn!(
1402                target: "Client/AppState",
1403                "App state {pending:?} still unsynced after {attempts} attempts; \
1404                 leaving them to the next sync trigger"
1405            );
1406            // Consumers heard about every round that produced buckets, but a
1407            // sequence that ends here — or one whose rounds all failed before
1408            // producing any — would otherwise finish in silence, with the
1409            // collections still stale. `retryable` is exactly what these are:
1410            // not synced, and a later trigger can still fix them.
1411            if client.admits(scope).is_ok() {
1412                let exhausted = BatchedSyncOutcome {
1413                    retryable: pending,
1414                    ..Default::default()
1415                };
1416                client.dispatch_app_state_sync_failed(
1417                    &exhausted,
1418                    client.is_ready.load(Ordering::Relaxed),
1419                );
1420            }
1421        }));
1422    }
1423
1424    /// Reserve `name` for a sync.
1425    ///
1426    /// Skipping is only ever sound behind an equivalent sync, and only for a
1427    /// caller whose request that sync subsumes — see [`ReservationWait`]. A
1428    /// patch send is never equivalent: it takes the same reservation and never
1429    /// fetches, so a sync that skipped behind one would be dropped silently and
1430    /// the patches that prompted it would go unfetched. Every wait is bounded by
1431    /// [`APP_STATE_RESERVATION_WAIT`], because the sync worker's intake loop
1432    /// runs non-history tasks inline and a wedged holder would otherwise stall
1433    /// everything queued behind it.
1434    /// The scope caps the wait further: the bootstrap runs under a watchdog that
1435    /// reconnects on wall-clock, so waiting past its deadline only guarantees the
1436    /// work lands on a socket that is already gone.
1437    async fn reserve_for_sync(
1438        &self,
1439        name: WAPatchName,
1440        wait: ReservationWait,
1441        scope: SyncScope,
1442    ) -> Result<SyncInFlightGuard, ReservationSkip> {
1443        match self.app_state_syncing.try_begin_as(name, SyncHolder::Sync) {
1444            Ok(guard) => return Ok(guard),
1445            Err(SyncHolder::Sync) if wait == ReservationWait::SkipBehindSync => {
1446                return Err(ReservationSkip::EquivalentSyncInFlight);
1447            }
1448            Err(holder) => {
1449                debug!(target: "Client/AppState", "Waiting for the {holder:?} holding {name:?}");
1450            }
1451        }
1452        let bound = match scope.remaining() {
1453            Some(remaining) => APP_STATE_RESERVATION_WAIT.min(remaining),
1454            None => APP_STATE_RESERVATION_WAIT,
1455        };
1456        rt_timeout(
1457            &*self.runtime,
1458            bound,
1459            self.app_state_syncing.begin(name, SyncHolder::Sync),
1460        )
1461        .await
1462        .map_err(|_| ReservationSkip::WaitTimedOut)
1463    }
1464
1465    /// Sync multiple collections in a single IQ request, re-fetching those with `has_more_patches`.
1466    /// Mirrors WA Web's `serverSync()` outer loop (`WAWebSyncdServerSync`).
1467    ///
1468    /// `scope` pins the work to the connection that asked for it and, for the
1469    /// initial bootstrap, to the 180s critical-sync deadline. Everything that
1470    /// follows an await re-asks [`Client::admits`] before writing, publishing or
1471    /// scheduling, so a batch cannot outlive the socket it belongs to. The
1472    /// deadline also bounds the missing-key wait, letting the explicit
1473    /// `AppStateSyncKeyRequest` fallback recover a late key on this connection
1474    /// without running past the watchdog.
1475    #[cfg_attr(feature = "tracing", tracing::instrument(name = "wa.appstate.sync_batched", level = "debug", skip_all, fields(count = collections.len()), err(Debug)))]
1476    pub(crate) async fn sync_collections_batched(
1477        &self,
1478        collections: Vec<WAPatchName>,
1479        scope: SyncScope,
1480    ) -> Result<BatchedSyncOutcome> {
1481        let mut outcome = BatchedSyncOutcome::default();
1482        if collections.is_empty() {
1483            return Ok(outcome);
1484        }
1485
1486        // In-flight dedup. The guards release on every exit path, including
1487        // cancellation. A collection we could not reserve is reported skipped
1488        // rather than silently dropped: this call did nothing for it, and only
1489        // the caller knows whether that is acceptable.
1490        // A caller can name the same collection twice — a `server_sync`
1491        // notification may repeat a `<collection>` child. Reserving it once and
1492        // then hitting our own reservation on the second pass would file one
1493        // collection under both `synced` and `skipped`, making `all_synced()`
1494        // false and publishing a failure that blames a writer who never existed.
1495        let mut seen = HashSet::with_capacity(collections.len());
1496        let collections: Vec<WAPatchName> = collections
1497            .into_iter()
1498            .filter(|name| seen.insert(*name))
1499            .collect();
1500
1501        // The bootstrap cannot skip: it has to know the collection is synced
1502        // before it dispatches Connected, and an equivalent sync in flight only
1503        // tells it someone else is trying. So when a deadline is supplied — which
1504        // is what marks the critical path — it waits for whoever holds it, and
1505        // finding the collection already synced afterwards is the fast case.
1506        // Background callers still skip, because for them the in-flight sync
1507        // genuinely does the work.
1508        let wait = if scope.is_bootstrap() {
1509            ReservationWait::Always
1510        } else {
1511            ReservationWait::SkipBehindSync
1512        };
1513
1514        let mut guards = Vec::with_capacity(collections.len());
1515        let mut pending = Vec::with_capacity(collections.len());
1516        for name in collections {
1517            // Asked per reservation, not once for the batch: each wait can burn
1518            // the remaining deadline, and the watchdog fires on wall-clock
1519            // regardless of which collection the batch is stuck on.
1520            if let Err(lost) = self.admits(scope) {
1521                warn!(target: "Client/AppState", "Not reserving {name:?}: {lost:?}");
1522                outcome.retryable.push(name);
1523                continue;
1524            }
1525            match self.reserve_for_sync(name, wait, scope).await {
1526                Ok(guard) => {
1527                    guards.push(guard);
1528                    pending.push(name);
1529                }
1530                // An equivalent sync in flight is doing this work, so the
1531                // collection is covered and only worth reporting. A wait that ran
1532                // out is not covered by anyone, so it belongs with the misses
1533                // that deserve another attempt.
1534                Err(ReservationSkip::EquivalentSyncInFlight) => {
1535                    debug!(target: "Client/AppState", "Skipping {name:?} in batch: an equivalent sync holds it");
1536                    outcome.skipped.push(name);
1537                }
1538                Err(ReservationSkip::WaitTimedOut) => {
1539                    warn!(target: "Client/AppState", "Gave up waiting for the writer holding {name:?}");
1540                    outcome.retryable.push(name);
1541                }
1542            }
1543        }
1544
1545        if pending.is_empty() {
1546            return Ok(outcome);
1547        }
1548
1549        self.sync_collections_batched_inner(pending, scope, &mut outcome)
1550            .await?;
1551
1552        // A run that crossed its deadline is not a clean one, even if every
1553        // collection it touched came back applied. Reporting it as fully synced
1554        // would let the bootstrap abort its watchdog and dispatch Connected on a
1555        // scope that has already expired, which is the outcome the deadline
1556        // exists to prevent. Moving the synced names to `retryable` sends it
1557        // down the retry path instead; the versions are persisted, so the next
1558        // attempt resumes rather than repeats.
1559        if !outcome.synced.is_empty()
1560            && let Err(lost @ ScopeLost::Expired) = self.admits(scope)
1561        {
1562            warn!(
1563                target: "Client/AppState",
1564                "Batched sync: {:?} applied but the run outlived its scope ({lost:?})",
1565                outcome.synced
1566            );
1567            let applied = std::mem::take(&mut outcome.synced);
1568            outcome.retryable.extend(applied);
1569        }
1570
1571        Ok(outcome)
1572    }
1573
1574    async fn sync_collections_batched_inner(
1575        &self,
1576        mut pending: Vec<WAPatchName>,
1577        scope: SyncScope,
1578        outcome: &mut BatchedSyncOutcome,
1579    ) -> Result<()> {
1580        use wacore::appstate::patch_decode::CollectionSyncError;
1581        // WA Web's own bound. Its loop reads `(l < y || (i.length > 0 && l < C))`
1582        // with `y = 5` and `C = 500`, and since the body only runs while there
1583        // is something left to refetch, that collapses to `l < 500` — the `y`
1584        // never bites. Rounds are back-to-back there too; the syncd backoff
1585        // applies to retrying a *failed* collection later, not to paging one
1586        // that is still making progress.
1587        const MAX_ITERATIONS: usize = 500;
1588        let mut iteration = 0;
1589
1590        while !pending.is_empty() && iteration < MAX_ITERATIONS {
1591            // With the cap at WA Web's 500, a collection paging healthily can
1592            // outlast the bootstrap's watchdog and be cut off mid-page on every
1593            // attempt, never reaching readiness though every page succeeded.
1594            // Stopping here instead keeps the progress: the versions applied so
1595            // far are persisted and the reconnect resumes from them.
1596            if let Err(lost) = self.admits(scope) {
1597                warn!(
1598                    target: "Client/AppState",
1599                    "Batched sync: stopping with {pending:?} still paging ({lost:?})"
1600                );
1601                outcome.retryable.extend(pending);
1602                return Ok(());
1603            }
1604            iteration += 1;
1605            debug!(
1606                target: "Client/AppState",
1607                "Batched sync iteration {}/{}: {:?}",
1608                iteration, MAX_ITERATIONS, pending
1609            );
1610
1611            let backend = self.persistence_manager.backend();
1612
1613            // Build multi-collection IQ, tracking which collections need a snapshot
1614            let mut collection_nodes = Vec::with_capacity(pending.len());
1615            let mut was_snapshot = HashSet::new();
1616            for &name in &pending {
1617                let state = backend.get_version(name.as_str()).await?;
1618                let want_snapshot = state.version == 0;
1619                if want_snapshot {
1620                    was_snapshot.insert(name);
1621                }
1622                let mut builder = NodeBuilder::new("collection")
1623                    .attr("name", name.as_str())
1624                    .attr(
1625                        "return_snapshot",
1626                        if want_snapshot { "true" } else { "false" },
1627                    );
1628                if !want_snapshot {
1629                    builder = builder.attr("version", state.version);
1630                }
1631                collection_nodes.push(builder.build());
1632            }
1633
1634            let sync_node = NodeBuilder::new("sync").children(collection_nodes).build();
1635            let iq = crate::request::InfoQuery {
1636                namespace: "w:sync:app:state",
1637                query_type: crate::request::InfoQueryType::Set,
1638                to: server_jid().clone(),
1639                target: None,
1640                id: None,
1641                content: Some(wacore_binary::NodeContent::Nodes(vec![sync_node])),
1642                timeout: Some(Duration::from_secs(30)),
1643            };
1644
1645            // Before the await, not after: the send is what spends an attempt,
1646            // and an IQ that errors or times out spent one just as much as one
1647            // that answered.
1648            outcome.note_reached_server();
1649            let resp = self.send_iq(iq).await?;
1650
1651            // The IQ can outrun the scope, so the round is re-admitted before
1652            // any of it is trusted.
1653            if let Err(lost) = self.admits(scope) {
1654                warn!(
1655                    target: "Client/AppState",
1656                    "Batched sync: dropping the response for {pending:?} ({lost:?})"
1657                );
1658                outcome.retryable.extend(pending);
1659                return Ok(());
1660            }
1661
1662            // Parse the response once here for pre-download; the same parsed
1663            // lists are handed to the processor below (no second parse).
1664            let mut patch_lists =
1665                wacore::appstate::patch_decode::parse_patch_lists_ref(resp.get())?;
1666
1667            // Drop a repeated collection before anything is applied. The
1668            // processor persists each list it is handed — mutation MACs and the
1669            // version — so two entries for one collection would be applied
1670            // twice, and the second could advance the MAC store past the version
1671            // the first then writes back, leaving the ltHash disagreeing with
1672            // the MACs. A collection outside `pending` is worse still: nothing
1673            // reserved it, so applying it can interleave with a concurrent sync
1674            // or patch send for that collection, and it dispatches mutations
1675            // nobody asked for. Checking after the fact only fixes the
1676            // bookkeeping.
1677            {
1678                let requested: HashSet<WAPatchName> = pending.iter().copied().collect();
1679                let mut seen: HashSet<WAPatchName> = HashSet::new();
1680                patch_lists.retain(|pl| {
1681                    if !requested.contains(&pl.name) {
1682                        warn!(
1683                            target: "Client/AppState",
1684                            "Batched sync: response carried unrequested collection {:?}; dropping it",
1685                            pl.name
1686                        );
1687                        return false;
1688                    }
1689                    if seen.insert(pl.name) {
1690                        return true;
1691                    }
1692                    warn!(
1693                        target: "Client/AppState",
1694                        "Batched sync: response repeated collection {:?}; dropping the duplicate",
1695                        pl.name
1696                    );
1697                    false
1698                });
1699            }
1700
1701            let proc = self.get_app_state_processor().await;
1702            // Pre-download all external blobs for all collections in the response,
1703            // concurrently (independent CDN GETs, keyed by directPath).
1704            let pre_downloaded = self.pre_download_external_blobs(&patch_lists).await;
1705
1706            let download = |ext: &wa::ExternalBlobReference| -> Result<Vec<u8>> {
1707                if let Some(path) = &ext.direct_path {
1708                    if let Some(bytes) = pre_downloaded.get(path) {
1709                        Ok(bytes.clone())
1710                    } else {
1711                        Err(anyhow::anyhow!(
1712                            "external blob not pre-downloaded: {}",
1713                            path
1714                        ))
1715                    }
1716                } else {
1717                    Err(anyhow::anyhow!("external blob has no directPath"))
1718                }
1719            };
1720
1721            // Request any missing decode keys and wait for them BEFORE processing. Inline
1722            // each list's external blobs first so the SNAPSHOT's key_id (inside the blob,
1723            // not the patch metadata) is visible -- else process_patch_lists aborts with
1724            // KeyNotFound on the snapshot key. If the share doesn't land in time, skip
1725            // this batch instead of aborting; it re-syncs on a later cycle once the key
1726            // arrives (process_patch_lists is all-or-nothing on a missing key anyway).
1727            let mut missing_all: Vec<Vec<u8>> = Vec::new();
1728            for pl in &mut patch_lists {
1729                if let Ok(m) = proc.missing_key_ids_after_inline(pl, &download).await {
1730                    missing_all.extend(m);
1731                }
1732            }
1733            // Bound the key wait by the critical-sync deadline when one was given
1734            // (initial bootstrap), so a late/never-auto-shared key still recovers via
1735            // the explicit request on this connection; otherwise a fixed short wait.
1736            let key_wait = scope.remaining().unwrap_or(APP_STATE_KEY_REQUEST_TIMEOUT);
1737            if !missing_all.is_empty() && !self.request_keys_and_wait(missing_all, key_wait).await {
1738                // The re-shared key didn't land in time. Nothing in this round can
1739                // be decoded, so report every collection still pending as
1740                // retryable rather than as synced: they re-sync on a later cycle
1741                // once the share arrives, and the keys we DID repair are already
1742                // persisted.
1743                warn!(
1744                    target: "Client/AppState",
1745                    "Batched sync: decode key(s) still missing after re-request, deferring {pending:?}"
1746                );
1747                outcome.retryable.extend(pending);
1748                return Ok(());
1749            }
1750
1751            // The last gate before anything is written. Blob downloads and the
1752            // key wait both sit between the previous check and here, and neither
1753            // is bounded by the scope, so this is where a response that is no
1754            // longer ours stops being applied.
1755            if let Err(lost) = self.admits(scope) {
1756                warn!(
1757                    target: "Client/AppState",
1758                    "Batched sync: not applying {pending:?} ({lost:?})"
1759                );
1760                outcome.retryable.extend(pending);
1761                return Ok(());
1762            }
1763
1764            // Applied one collection at a time rather than handing the whole
1765            // batch over, because the processor persists each list as it goes:
1766            // a batch that starts inside the scope can still be writing its
1767            // third collection well outside it. Per-list is the finest boundary
1768            // available without teaching `wacore` about connections, and it caps
1769            // what a retired scope can commit at one collection instead of five.
1770            let mut results = Vec::with_capacity(patch_lists.len());
1771            for pl in patch_lists {
1772                if let Err(lost) = self.admits(scope) {
1773                    warn!(
1774                        target: "Client/AppState",
1775                        "Batched sync: stopping before {:?} ({lost:?})", pl.name
1776                    );
1777                    break;
1778                }
1779                results.push(proc.process_one_patch_list(pl, &download, true).await?);
1780            }
1781
1782            let mut needs_refetch = Vec::new();
1783            // A `<sync>` that simply omits a requested `<collection>` parses
1784            // fine, so without this the collection lands in no bucket at all and
1785            // `all_synced()` reports a batch that never covered it. Track what
1786            // the response actually accounted for and reconcile below.
1787            // Duplicates are already gone, dropped above before anything applied
1788            // them.
1789            let mut answered: HashSet<WAPatchName> = HashSet::new();
1790
1791            for (mutations, new_state, list) in results {
1792                let name = list.name;
1793                answered.insert(name);
1794
1795                // No admission check here, deliberately. `process_one_patch_list`
1796                // persists the collection's version and mutation MACs before it
1797                // returns, so by this point the cursor has already moved. A
1798                // collection dropped here would never be re-sent — the retry
1799                // asks from the advanced version — and its mutations would be
1800                // lost for good, `setting_pushName` and the NCT salt included.
1801                // The scope is checked before the apply, which is the last
1802                // moment a collection can still be declined; after it,
1803                // dispatching is not optional.
1804
1805                // Handle per-collection errors
1806                if let Some(ref err) = list.error {
1807                    match err {
1808                        CollectionSyncError::Conflict { has_more } => {
1809                            if *has_more {
1810                                // ConflictHasMore: server has more patches, must refetch.
1811                                warn!(target: "Client/AppState", "Collection {:?} conflict (has_more=true), will refetch", name);
1812                                needs_refetch.push(name);
1813                            } else {
1814                                // Conflict without has_more: WA Web treats this as success
1815                                // when there are no pending mutations to push (which is
1816                                // always the case for us since we don't push app state).
1817                                debug!(target: "Client/AppState", "Collection {:?} conflict (has_more=false), treating as success (no pending mutations)", name);
1818                                outcome.synced.push(name);
1819                            }
1820                            continue;
1821                        }
1822                        CollectionSyncError::Fatal { code, text } => {
1823                            warn!(target: "Client/AppState", "Collection {:?} fatal error {}: {}", name, code, text);
1824                            outcome.fatal.push(name);
1825                            continue;
1826                        }
1827                        CollectionSyncError::Retry { code, text } => {
1828                            // Done for this run, not refetched inside it. WA Web
1829                            // routes ErrorRetry to `doneCollections` and leaves
1830                            // the next attempt to its retry state machine, which
1831                            // spaces them; refetching here would instead hammer
1832                            // the same failing collection for every iteration
1833                            // the cap allows.
1834                            warn!(target: "Client/AppState", "Collection {:?} retryable error {}: {}", name, code, text);
1835                            outcome.retryable.push(name);
1836                            continue;
1837                        }
1838                    }
1839                }
1840
1841                // Handle missing keys
1842                let missing = match proc.get_missing_key_ids(&list).await {
1843                    Ok(v) => v,
1844                    Err(e) => {
1845                        warn!("Failed to get missing key IDs for {:?}: {}", name, e);
1846                        Vec::new()
1847                    }
1848                };
1849                self.request_missing_keys_with_dedup(&missing, APP_STATE_KEY_REQUEST_DEDUP)
1850                    .await;
1851
1852                // full_sync is true only when this collection had a snapshot
1853                // (version was 0 before sync). This prevents server_sync-triggered
1854                // incremental syncs from being incorrectly marked as full syncs.
1855                let full_sync = was_snapshot.contains(&name);
1856                wacore::telemetry::appstate_mutations(mutations.len() as u64);
1857                for m in mutations {
1858                    self.dispatch_app_state_mutation(&m, full_sync).await;
1859                }
1860
1861                // Save version
1862                backend
1863                    .set_version(name.as_str(), new_state.clone())
1864                    .await?;
1865
1866                // Check if this collection needs more patches
1867                if list.has_more_patches {
1868                    needs_refetch.push(name);
1869                } else {
1870                    outcome.synced.push(name);
1871                }
1872
1873                debug!(
1874                    target: "Client/AppState",
1875                    "Batched sync: {:?} done (version={}, has_more={})",
1876                    name, new_state.version, list.has_more_patches
1877                );
1878            }
1879
1880            // Anything asked for that the response did not mention is not synced
1881            // and did not fail either; treat it as retryable so it is neither
1882            // reported as done nor re-asked immediately in this loop.
1883            for name in pending {
1884                if !answered.contains(&name) {
1885                    warn!(
1886                        target: "Client/AppState",
1887                        "Batched sync: response omitted collection {name:?}"
1888                    );
1889                    outcome.retryable.push(name);
1890                }
1891            }
1892
1893            pending = needs_refetch;
1894        }
1895
1896        if !pending.is_empty() {
1897            // Still paging when the cap ran out. Retryable, not fatal: the
1898            // versions applied so far are persisted, so a later sync resumes
1899            // where this one stopped. WA Web classifies the same exhaustion as
1900            // `ErrorRetry`.
1901            warn!(
1902                target: "Client/AppState",
1903                "Batched sync: max iterations ({}) reached for {:?}",
1904                MAX_ITERATIONS, pending
1905            );
1906            outcome.retryable.extend(pending);
1907        }
1908
1909        Ok(())
1910    }
1911
1912    #[cfg_attr(feature = "tracing", tracing::instrument(name = "wa.appstate.sync", level = "debug", skip_all, fields(name = ?name, full_sync = full_sync), err(Debug)))]
1913    pub(crate) async fn process_app_state_sync_task(
1914        &self,
1915        name: WAPatchName,
1916        full_sync: bool,
1917    ) -> Result<SyncOutcome> {
1918        // Two questions, where `is_shutting_down()` answered a blend of them: is
1919        // the client finished, and can a request reach the server at all. A
1920        // planned reconnect is neither — it clears `is_connected` and leaves
1921        // everything else alone — so the work waits instead of stopping.
1922        if self.is_terminal() || !self.can_reach_server() {
1923            debug!(
1924                target: "Client/AppState",
1925                "Skipping app state sync task {name:?}: no usable connection"
1926            );
1927            return Ok(SyncOutcome::Deferred);
1928        }
1929
1930        let backend = self.persistence_manager.backend();
1931        let mut full_sync = full_sync;
1932
1933        let mut state = backend.get_version(name.as_str()).await?;
1934        if state.version == 0 {
1935            full_sync = true;
1936        }
1937
1938        let mut has_more = true;
1939        let mut want_snapshot = full_sync;
1940        // Safety cap to prevent infinite loops if the server keeps returning
1941        // has_more_patches=true without advancing the version (WA Web uses 500).
1942        const MAX_PAGINATION_ITERATIONS: u32 = 500;
1943        let mut iteration = 0u32;
1944        // Every exit below still falls through to `set_version`: the pages
1945        // already applied are durable whether or not the rest arrived. Only
1946        // what the caller is told changes.
1947        let mut outcome = SyncOutcome::Completed;
1948
1949        while has_more {
1950            if self.is_terminal() || !self.can_reach_server() {
1951                debug!(target: "Client/AppState", "Stopping app state sync task {name:?}: no usable connection");
1952                outcome = SyncOutcome::Deferred;
1953                break;
1954            }
1955            iteration += 1;
1956            if iteration > MAX_PAGINATION_ITERATIONS {
1957                warn!(target: "Client/AppState", "App state sync for {:?} exceeded {} iterations, aborting", name, MAX_PAGINATION_ITERATIONS);
1958                // `has_more` is still set, so the persisted version is below the
1959                // server's head — which is the definition of deferred, whatever
1960                // the reason for stopping. Reporting completion here would have
1961                // callers drop the trigger and leave it there for good.
1962                //
1963                // The cost is a retry that may re-page against the same
1964                // non-progress, bounded by the attempt budget and its backoff.
1965                // That is the cheaper mistake: this cap is a should-never-happen
1966                // guard, and if it fires because the server was briefly wedged,
1967                // a later attempt is the only thing that ever fixes it.
1968                outcome = SyncOutcome::Deferred;
1969                break;
1970            }
1971            debug!(target: "Client/AppState", "Fetching app state patch batch: name={:?} want_snapshot={want_snapshot} version={} full_sync={} has_more_previous={}", name, state.version, full_sync, has_more);
1972
1973            let mut collection_builder = NodeBuilder::new("collection")
1974                .attr("name", name.as_str())
1975                .attr(
1976                    "return_snapshot",
1977                    if want_snapshot { "true" } else { "false" },
1978                );
1979            if !want_snapshot {
1980                collection_builder = collection_builder.attr("version", state.version);
1981            }
1982            let sync_node = NodeBuilder::new("sync")
1983                .children([collection_builder.build()])
1984                .build();
1985            let iq = crate::request::InfoQuery {
1986                namespace: "w:sync:app:state",
1987                query_type: crate::request::InfoQueryType::Set,
1988                to: server_jid().clone(),
1989                target: None,
1990                id: None,
1991                content: Some(wacore_binary::NodeContent::Nodes(vec![sync_node])),
1992                timeout: None,
1993            };
1994
1995            let resp = self.send_iq(iq).await?;
1996            if self.is_terminal() || !self.can_reach_server() {
1997                debug!(target: "Client/AppState", "Discarding app state sync response for {name:?}: no usable connection");
1998                outcome = SyncOutcome::Deferred;
1999                break;
2000            }
2001            debug!(target: "Client/AppState", "Received IQ response for {:?}; decoding patches", name);
2002
2003            let _decode_start = wacore::time::Instant::now();
2004
2005            // Parse the response once here; the same parsed list is handed to the
2006            // processor below (no second parse).
2007            let mut pl = wacore::appstate::patch_decode::parse_patch_list_ref(resp.get())?;
2008            debug!(target: "Client/AppState", "Parsed patch list for {:?}: has_snapshot_ref={} has_more_patches={} patches_count={}",
2009                name, pl.snapshot_ref.is_some(), pl.has_more_patches, pl.patches.len());
2010
2011            let proc = self.get_app_state_processor().await;
2012
2013            // Pre-download all external blobs (snapshot and patch mutations),
2014            // concurrently, keyed by directPath.
2015            let pre_downloaded = self
2016                .pre_download_external_blobs(std::slice::from_ref(&pl))
2017                .await;
2018
2019            let download = |ext: &wa::ExternalBlobReference| -> Result<Vec<u8>> {
2020                if let Some(path) = &ext.direct_path {
2021                    if let Some(bytes) = pre_downloaded.get(path) {
2022                        Ok(bytes.clone())
2023                    } else {
2024                        Err(anyhow::anyhow!(
2025                            "external blob not pre-downloaded: {}",
2026                            path
2027                        ))
2028                    }
2029                } else {
2030                    Err(anyhow::anyhow!("external blob has no directPath"))
2031                }
2032            };
2033
2034            // Request any missing decode keys and wait for them BEFORE processing. Inline
2035            // the blobs first so the SNAPSHOT's key_id (inside its external blob, not the
2036            // patch metadata) is visible -- else process aborts with KeyNotFound on the
2037            // snapshot key. If the share doesn't land in time, skip this collection
2038            // instead of aborting; it re-syncs on a later cycle once the key arrives.
2039            let missing = proc
2040                .missing_key_ids_after_inline(&mut pl, &download)
2041                .await
2042                .unwrap_or_default();
2043            if !missing.is_empty()
2044                && !self
2045                    .request_keys_and_wait(missing, APP_STATE_KEY_REQUEST_TIMEOUT)
2046                    .await
2047            {
2048                // Report failure (not a partial success) so the caller retries instead of
2049                // treating the collection as synced; it re-syncs once the share lands.
2050                // Pages already decoded this run have their version persisted.
2051                return Err(anyhow::anyhow!(
2052                    "app-state decode key(s) for {name:?} still missing after re-request; deferring sync"
2053                ));
2054            }
2055
2056            let (mutations, new_state, list) =
2057                proc.process_parsed_patch_list(pl, &download, true).await?;
2058            let decode_elapsed = _decode_start.elapsed();
2059            if decode_elapsed.as_millis() > 500 {
2060                debug!(target: "Client/AppState", "Patch decode for {:?} took {:?}", name, decode_elapsed);
2061            }
2062
2063            let missing = match proc.get_missing_key_ids(&list).await {
2064                Ok(v) => v,
2065                Err(e) => {
2066                    warn!("Failed to get missing key IDs for {:?}: {}", name, e);
2067                    Vec::new()
2068                }
2069            };
2070            self.request_missing_keys_with_dedup(&missing, APP_STATE_KEY_REQUEST_DEDUP)
2071                .await;
2072
2073            wacore::telemetry::appstate_mutations(mutations.len() as u64);
2074            for m in mutations {
2075                debug!(target: "Client/AppState", "Dispatching mutation kind={} index_len={} full_sync={}", m.index.first().map(|s| s.as_str()).unwrap_or(""), m.index.len(), full_sync);
2076                self.dispatch_app_state_mutation(&m, full_sync).await;
2077            }
2078
2079            state = new_state;
2080            has_more = list.has_more_patches;
2081            // After the first batch, never request a snapshot again — only incremental patches.
2082            want_snapshot = false;
2083            debug!(target: "Client/AppState", "After processing batch name={:?} has_more={has_more} new_version={}", name, state.version);
2084        }
2085
2086        backend.set_version(name.as_str(), state.clone()).await?;
2087
2088        debug!(target: "Client/AppState", "Finished app state sync for {name:?} as {outcome:?} (final version={})", state.version);
2089        Ok(outcome)
2090    }
2091
2092    /// Request the missing decode keys, wait up to `timeout` for the re-share, then
2093    /// VERIFY they actually landed. Returns true only when every requested key is now
2094    /// stored (the caller may process); false means the share didn't arrive in time and
2095    /// the caller must NOT process -- doing so would abort with KeyNotFound -- and should
2096    /// skip the collection so it re-syncs on a later cycle. Empty input returns true
2097    /// (nothing to wait for). Waits even when the per-key dedup suppressed the send: a
2098    /// deduped request means an earlier one is still in flight, so the key may yet land
2099    /// here, and a re-verify that fails can't be masked by treating "request sent" as
2100    /// success or by a wake from an unrelated key share.
2101    async fn request_keys_and_wait(&self, mut missing: Vec<Vec<u8>>, timeout: Duration) -> bool {
2102        if missing.is_empty() {
2103            return true;
2104        }
2105        let deadline = wacore::time::Instant::now() + timeout;
2106        let backend = self.persistence_manager.backend();
2107        let mut retry_after = initial_app_state_key_retry(timeout);
2108        loop {
2109            let listener = self.initial_keys_synced_notifier.listen();
2110            remove_available_app_state_keys(&*backend, &mut missing).await;
2111            if missing.is_empty() {
2112                return true;
2113            }
2114
2115            let request = self.request_missing_keys_with_dedup(&missing, retry_after);
2116            let schedule = match self
2117                .await_app_state_key_request(&*backend, &missing, deadline, listener, request)
2118                .await
2119            {
2120                AppStateKeyRequestProgress::Scheduled(schedule) => schedule,
2121                AppStateKeyRequestProgress::KeysReady => return true,
2122                AppStateKeyRequestProgress::TimedOut => return false,
2123            };
2124            if schedule.sent {
2125                debug!(target: "Client/AppState", "Requested {} missing app-state key(s); retrying after {retry_after:?} if no share arrives", missing.len());
2126                retry_after = retry_after.saturating_mul(2).min(APP_STATE_KEY_RETRY_MAX);
2127            }
2128
2129            let listener = self.initial_keys_synced_notifier.listen();
2130            remove_available_app_state_keys(&*backend, &mut missing).await;
2131            if missing.is_empty() {
2132                return true;
2133            }
2134
2135            let remaining = deadline.saturating_duration_since(wacore::time::Instant::now());
2136            if remaining.is_zero() {
2137                return false;
2138            }
2139
2140            let retry_wait = schedule
2141                .retry_at
2142                .saturating_duration_since(wacore::time::Instant::now());
2143            let wait = remaining.min(retry_wait);
2144            if !wait.is_zero() {
2145                let _ = rt_timeout(&*self.runtime, wait, listener).await;
2146            }
2147        }
2148    }
2149
2150    async fn await_app_state_key_request<F>(
2151        &self,
2152        backend: &dyn crate::store::traits::Backend,
2153        missing: &[Vec<u8>],
2154        deadline: wacore::time::Instant,
2155        mut listener: event_listener::EventListener,
2156        request: F,
2157    ) -> AppStateKeyRequestProgress
2158    where
2159        F: Future<Output = AppStateKeyRequestSchedule>,
2160    {
2161        futures::pin_mut!(request);
2162        loop {
2163            let remaining = deadline.saturating_duration_since(wacore::time::Instant::now());
2164            if remaining.is_zero() {
2165                return if app_state_keys_available(backend, missing).await {
2166                    AppStateKeyRequestProgress::KeysReady
2167                } else {
2168                    AppStateKeyRequestProgress::TimedOut
2169                };
2170            }
2171
2172            let notified = rt_timeout(&*self.runtime, remaining, listener);
2173            futures::pin_mut!(notified);
2174            match futures::future::select(request.as_mut(), notified.as_mut()).await {
2175                futures::future::Either::Left((schedule, _)) => {
2176                    return AppStateKeyRequestProgress::Scheduled(schedule);
2177                }
2178                futures::future::Either::Right((notification, _)) => {
2179                    let next_listener = self.initial_keys_synced_notifier.listen();
2180                    if app_state_keys_available(backend, missing).await {
2181                        return AppStateKeyRequestProgress::KeysReady;
2182                    }
2183                    if notification.is_err() {
2184                        return AppStateKeyRequestProgress::TimedOut;
2185                    }
2186                    listener = next_listener;
2187                }
2188            }
2189        }
2190    }
2191
2192    /// Request missing app-state keys with dedup stamps.
2193    /// Total failure removes stamps; partial fanout gets a short retry deadline.
2194    async fn request_missing_keys_with_dedup(
2195        &self,
2196        missing: &[Vec<u8>],
2197        retry_after: Duration,
2198    ) -> AppStateKeyRequestSchedule {
2199        if missing.is_empty() {
2200            return AppStateKeyRequestSchedule {
2201                retry_at: wacore::time::Instant::now() + retry_after,
2202                sent: false,
2203            };
2204        }
2205        let mut guard = self.app_state_key_requests.lock().await;
2206        let now = wacore::time::Instant::now();
2207        let requested_retry_at = now + retry_after;
2208        guard.retain(|_, retry_at| now < *retry_at);
2209
2210        let mut to_request: Option<Vec<&[u8]>> = None;
2211        let mut next_retry_at = requested_retry_at;
2212        for key_id in missing {
2213            if let Some(retry_at) = guard.get_mut(key_id.as_slice()) {
2214                if *retry_at > requested_retry_at {
2215                    *retry_at = requested_retry_at;
2216                }
2217                next_retry_at = next_retry_at.min(*retry_at);
2218            } else {
2219                guard.insert(key_id.clone(), requested_retry_at);
2220                to_request
2221                    .get_or_insert_with(|| Vec::with_capacity(missing.len()))
2222                    .push(key_id.as_slice());
2223            }
2224        }
2225        drop(guard);
2226
2227        let Some(to_request) = to_request else {
2228            return AppStateKeyRequestSchedule {
2229                retry_at: next_retry_at,
2230                sent: false,
2231            };
2232        };
2233
2234        match self
2235            .request_app_state_keys(&to_request, retry_after.min(APP_STATE_KEY_REQUEST_TIMEOUT))
2236            .await
2237        {
2238            Ok(AppStateKeyRequestDelivery::AllPeers) => AppStateKeyRequestSchedule {
2239                retry_at: next_retry_at,
2240                sent: true,
2241            },
2242            Ok(AppStateKeyRequestDelivery::SomePeers) => {
2243                let retry_at = wacore::time::Instant::now() + APP_STATE_KEY_PARTIAL_RETRY;
2244                let mut guard = self.app_state_key_requests.lock().await;
2245                for key_id in &to_request {
2246                    if let Some(deadline) = guard.get_mut(*key_id) {
2247                        *deadline = (*deadline).min(retry_at);
2248                    }
2249                }
2250                AppStateKeyRequestSchedule {
2251                    retry_at: next_retry_at.min(retry_at),
2252                    sent: true,
2253                }
2254            }
2255            Err(e) => {
2256                warn!("Failed to send app state key request: {e}");
2257                let mut guard = self.app_state_key_requests.lock().await;
2258                for key_id in &to_request {
2259                    if guard
2260                        .get(*key_id)
2261                        .is_some_and(|deadline| *deadline == requested_retry_at)
2262                    {
2263                        guard.remove(*key_id);
2264                    }
2265                }
2266                AppStateKeyRequestSchedule {
2267                    retry_at: requested_retry_at,
2268                    sent: false,
2269                }
2270            }
2271        }
2272    }
2273
2274    async fn app_state_key_request_peers(&self) -> Result<Vec<Jid>, anyhow::Error> {
2275        let device_snapshot = self.persistence_manager.get_device_snapshot();
2276        let own_jid = device_snapshot
2277            .pn
2278            .as_ref()
2279            .ok_or_else(|| anyhow::anyhow!("no own JID available for app-state key request"))?;
2280        let current_device = own_jid.device;
2281        let primary = own_jid.to_non_ad();
2282        drop(device_snapshot);
2283
2284        let peers = match self.get_user_devices(std::slice::from_ref(&primary)).await {
2285            Ok(devices) => devices,
2286            Err(error) => {
2287                warn!(
2288                    "Own device-list query failed; requesting app-state keys from primary only: {error}"
2289                );
2290                Vec::new()
2291            }
2292        };
2293        finalize_app_state_key_request_peers(peers, current_device, primary)
2294    }
2295
2296    #[cfg_attr(feature = "tracing", tracing::instrument(name = "wa.appstate.request_keys", level = "debug", skip_all, fields(count = raw_key_ids.len()), err(Debug)))]
2297    async fn request_app_state_keys(
2298        &self,
2299        raw_key_ids: &[&[u8]],
2300        fanout_timeout: Duration,
2301    ) -> Result<AppStateKeyRequestDelivery, anyhow::Error> {
2302        if raw_key_ids.is_empty() {
2303            return Ok(AppStateKeyRequestDelivery::AllPeers);
2304        }
2305        let peers = self.app_state_key_request_peers().await?;
2306        let key_ids: Vec<wa::message::AppStateSyncKeyId> = raw_key_ids
2307            .iter()
2308            .map(|k| wa::message::AppStateSyncKeyId {
2309                key_id: Some(k.to_vec()),
2310            })
2311            .collect();
2312        let msg = wa::Message {
2313            protocol_message: buffa::MessageField::some(wa::message::ProtocolMessage {
2314                r#type: Some(wa::message::protocol_message::Type::AppStateSyncKeyRequest),
2315                app_state_sync_key_request: buffa::MessageField::some(
2316                    wa::message::AppStateSyncKeyRequest { key_ids },
2317                ),
2318                ..Default::default()
2319            }),
2320            ..Default::default()
2321        };
2322
2323        let requests = futures::stream::FuturesUnordered::new();
2324        for peer in peers {
2325            let msg = &msg;
2326            requests.push(async move {
2327                let device = peer.device;
2328                let result = async {
2329                    self.ensure_e2e_sessions(std::slice::from_ref(&peer))
2330                        .await?;
2331                    let request_id = self.generate_message_id();
2332                    self.send_message_impl(
2333                        peer,
2334                        msg,
2335                        crate::send::SendPipelineOptions {
2336                            request_id: Some(&request_id),
2337                            peer: true,
2338                            ..Default::default()
2339                        },
2340                    )
2341                    .await
2342                }
2343                .await;
2344                (device, result)
2345            });
2346        }
2347
2348        collect_app_state_key_request_results(&*self.runtime, requests, fanout_timeout).await
2349    }
2350
2351    /// Send an app state patch to the server for a given collection.
2352    ///
2353    /// The server enforces optimistic concurrency on the collection `version`:
2354    /// a patch built on a base another device has already moved past is refused
2355    /// with `<collection type="error"><error code="409">`, *inside an otherwise
2356    /// successful IQ*, together with the patches that won. WA Web resolves that
2357    /// by applying the winners and letting `serverSync` re-queue the collection
2358    /// while pending mutations remain, so the mutation is re-sent on the new
2359    /// base instead of being dropped; this mirrors that, bounded by the same
2360    /// iteration cap WA Web uses (`ServerSync.js`, `y = 5`).
2361    ///
2362    /// `400`/`404` are fatal and anything else retryable, per
2363    /// `WAWebSyncdResponseParser`. All of them are errors here — never `Ok`.
2364    #[cfg_attr(feature = "tracing", tracing::instrument(name = "wa.appstate.send_patch", level = "debug", skip_all, fields(name = %collection_name, count = mutations.len()), err(Debug)))]
2365    pub(crate) async fn send_app_state_patch(
2366        &self,
2367        collection_name: &str,
2368        mutations: Vec<wa::SyncdMutation>,
2369    ) -> Result<()> {
2370        use wacore::appstate::patch_decode::CollectionSyncError;
2371
2372        let patch_name = collection_name.parse::<WAPatchName>().ok();
2373        // Held across the whole build-send-resolve cycle: the base version is
2374        // read at build time and only stops being valid once the send lands, so
2375        // releasing earlier would let a second verb build on a base this one is
2376        // about to consume. Deliberately held over the trailing re-sync too —
2377        // dropping it there would let the next send start from a base the
2378        // re-sync is about to move, trading a short wait for the 409s this
2379        // whole path exists to avoid.
2380        let _send_guard = self.app_state_send_lock.lock().await;
2381        // The send lock only orders sends against each other. This one orders
2382        // the send against the sync worker, which writes the same version and
2383        // mutation-MAC rows: without it, a conflict response for vN could be
2384        // absorbed while a sync is persisting vN+1, and the interleaved writes
2385        // would leave the ltHash disagreeing with the MAC store — the very
2386        // divergence #1156 is about. Waits rather than skipping, and the
2387        // re-syncs below go through `_inner` because this task already holds
2388        // the reservation they would otherwise take.
2389        let _collection_guard = match patch_name {
2390            Some(name) => Some(
2391                self.app_state_syncing
2392                    .begin(name, SyncHolder::PatchSend)
2393                    .await,
2394            ),
2395            None => None,
2396        };
2397        let proc = self.get_app_state_processor().await;
2398
2399        for attempt in 1..=APP_STATE_PATCH_SEND_ATTEMPTS {
2400            // Cloned per attempt because a conflict rebuilds the patch against
2401            // the winner's base; verbs carry one or two mutations, and this only
2402            // runs on the (rare) conflict path after the first attempt.
2403            let (patch_bytes, base_version) =
2404                proc.build_patch(collection_name, mutations.clone()).await?;
2405
2406            let collection_node = NodeBuilder::new("collection")
2407                .attr("name", collection_name)
2408                .attr("version", base_version)
2409                .attr("return_snapshot", "false")
2410                .children([NodeBuilder::new("patch").bytes(patch_bytes).build()])
2411                .build();
2412            let sync_node = NodeBuilder::new("sync").children([collection_node]).build();
2413            let iq = crate::request::InfoQuery {
2414                namespace: "w:sync:app:state",
2415                query_type: crate::request::InfoQueryType::Set,
2416                to: server_jid().clone(),
2417                target: None,
2418                id: None,
2419                content: Some(wacore_binary::NodeContent::Nodes(vec![sync_node])),
2420                timeout: None,
2421            };
2422
2423            let resp = self.send_iq(iq).await?;
2424            let resp = resp.get().to_owned();
2425            // Absence and malformation are different answers. A response with no
2426            // `<sync><collection>` at all carries no per-collection verdict —
2427            // a transport-level failure would have come back as
2428            // `<iq type="error">` and been raised by send_iq already — so it is
2429            // an accepted patch. A collection that IS present but does not parse
2430            // may well be carrying the rejection, and manufacturing an empty
2431            // success from it would drop the mutation exactly as before.
2432            let list = match wacore::appstate::patch_decode::parse_patch_list(&resp) {
2433                Ok(list) => list,
2434                Err(e)
2435                    if resp
2436                        .get_optional_child_by_tag(&["sync", "collection"])
2437                        .is_none() =>
2438                {
2439                    debug!(
2440                        target: "Client/AppState",
2441                        "Patch response for {collection_name} carried no collection verdict ({e}); treating as accepted"
2442                    );
2443                    wacore::appstate::patch_decode::PatchList {
2444                        name: patch_name.unwrap_or(WAPatchName::Unknown),
2445                        has_more_patches: false,
2446                        patches: Vec::new(),
2447                        snapshot: None,
2448                        snapshot_ref: None,
2449                        error: None,
2450                    }
2451                }
2452                Err(e) => {
2453                    return Err(e.context(format!(
2454                        "unreadable app-state patch response for {collection_name}"
2455                    )));
2456                }
2457            };
2458            if Some(list.name) != patch_name {
2459                return Err(anyhow::anyhow!(
2460                    "app-state patch response collection mismatch: requested {collection_name}, got {}",
2461                    list.name.as_str()
2462                ));
2463            }
2464
2465            match list.error {
2466                None => {
2467                    // Re-sync to pick up whatever else moved while we were sending.
2468                    // Matches whatsmeow's fetchAppState after a successful send.
2469                    if let Some(patch_name) = patch_name
2470                        && let Err(e) = self.fetch_app_state_with_retry_inner(patch_name).await
2471                    {
2472                        log::warn!("Failed to re-sync {collection_name} after patch send: {e}");
2473                    }
2474                    return Ok(());
2475                }
2476                Some(CollectionSyncError::Conflict { has_more }) => {
2477                    warn!(
2478                        target: "Client/AppState",
2479                        "Patch for {collection_name} conflicted on v{base_version} \
2480                         (attempt {attempt}/{APP_STATE_PATCH_SEND_ATTEMPTS}, has_more={has_more}); \
2481                         applying the conflicting patches and rebuilding"
2482                    );
2483                    self.absorb_conflicting_patches(collection_name, patch_name, list, has_more)
2484                        .await;
2485                }
2486                Some(error) => {
2487                    return Err(anyhow::anyhow!(
2488                        "app-state patch for {collection_name} rejected: {error}"
2489                    ));
2490                }
2491            }
2492        }
2493
2494        Err(anyhow::anyhow!(
2495            "app-state patch for {collection_name} still conflicting after \
2496             {APP_STATE_PATCH_SEND_ATTEMPTS} attempts"
2497        ))
2498    }
2499
2500    /// Fold the patches a 409 response carried into local state, so the retry
2501    /// builds on the base that actually won.
2502    ///
2503    /// Best-effort by design: if the response carried nothing usable (or failed
2504    /// to apply — a missing decode key, a bad blob), a plain re-sync is the
2505    /// fallback that advances the base. Either way the caller retries; the only
2506    /// unrecoverable outcome is making no progress, which the attempt cap turns
2507    /// into an error rather than a silent drop.
2508    async fn absorb_conflicting_patches(
2509        &self,
2510        collection_name: &str,
2511        patch_name: Option<WAPatchName>,
2512        mut list: wacore::appstate::patch_decode::PatchList,
2513        has_more: bool,
2514    ) {
2515        // The error tag described the send; the patches under it are ordinary
2516        // inbound data, so clear it before handing the list to the processor.
2517        list.error = None;
2518        let applied = if list.patches.is_empty() && list.snapshot_ref.is_none() {
2519            false
2520        } else {
2521            let pre_downloaded = self
2522                .pre_download_external_blobs(std::slice::from_ref(&list))
2523                .await;
2524            let download = |ext: &wa::ExternalBlobReference| -> Result<Vec<u8>> {
2525                let path = ext
2526                    .direct_path
2527                    .as_ref()
2528                    .ok_or_else(|| anyhow::anyhow!("external blob has no directPath"))?;
2529                pre_downloaded
2530                    .get(path)
2531                    .cloned()
2532                    .ok_or_else(|| anyhow::anyhow!("external blob not pre-downloaded: {path}"))
2533            };
2534            let proc = self.get_app_state_processor().await;
2535            match proc.process_parsed_patch_list(list, &download, true).await {
2536                Ok((mutations, _, _)) => {
2537                    wacore::telemetry::appstate_mutations(mutations.len() as u64);
2538                    for m in &mutations {
2539                        self.dispatch_app_state_mutation(m, false).await;
2540                    }
2541                    true
2542                }
2543                Err(e) => {
2544                    warn!(
2545                        target: "Client/AppState",
2546                        "Failed to apply the patches {collection_name} conflicted with: {e:#}"
2547                    );
2548                    false
2549                }
2550            }
2551        };
2552
2553        // `has_more` means the server held patches back, so even a clean apply
2554        // leaves the base short of the head.
2555        if (!applied || has_more)
2556            && let Some(patch_name) = patch_name
2557            && let Err(e) = self.fetch_app_state_with_retry_inner(patch_name).await
2558        {
2559            warn!(
2560                target: "Client/AppState",
2561                "Failed to re-sync {collection_name} after a patch conflict: {e}"
2562            );
2563        }
2564    }
2565
2566    async fn dispatch_app_state_mutation(
2567        &self,
2568        m: &crate::appstate_sync::Mutation,
2569        full_sync: bool,
2570    ) {
2571        use wacore::types::events::Event;
2572
2573        if m.index.is_empty() {
2574            return;
2575        }
2576
2577        // NCT salt sync — handles both "set" (store salt) and "remove" (clear salt).
2578        // Source: WAWebNctSaltSync, syncd collection RegularHigh, action "nct_salt_sync".
2579        if m.index[0] == "nct_salt_sync" {
2580            if m.operation == wa::syncd_mutation::SyncdOperation::Remove {
2581                debug!(target: "Client/AppState", "Removing NCT salt via app state sync");
2582                self.persistence_manager
2583                    .process_command(DeviceCommand::SetNctSalt(None))
2584                    .await;
2585            } else if let Some(val) = &m.action_value
2586                && let Some(act) = val.nct_salt_sync_action.as_option()
2587                && let Some(salt) = &act.salt
2588            {
2589                if salt.is_empty() {
2590                    warn!(target: "Client/AppState", "nct_salt_sync mutation has empty salt, ignoring");
2591                } else {
2592                    debug!(target: "Client/AppState", "Stored NCT salt via app state sync ({} bytes)", salt.len());
2593                    self.persistence_manager
2594                        .process_command(DeviceCommand::SetNctSalt(Some(salt.clone())))
2595                        .await;
2596                }
2597            } else {
2598                warn!(target: "Client/AppState", "nct_salt_sync mutation missing salt in action value");
2599            }
2600            return;
2601        }
2602
2603        // All remaining mutations only care about Set operations
2604        if m.operation != wa::syncd_mutation::SyncdOperation::Set {
2605            return;
2606        }
2607
2608        // Delegate chat-related mutations (mute, pin, archive, star, contact, etc.)
2609        if crate::features::chat_actions::dispatch_chat_mutation(&self.core.event_bus, m, full_sync)
2610        {
2611            return;
2612        }
2613
2614        // Label mutations have their own index shape (labelId, not a chat JID at
2615        // index[1]), so they are dispatched separately from chat actions.
2616        if crate::features::labels::dispatch_label_mutation(&self.core.event_bus, m, full_sync) {
2617            return;
2618        }
2619
2620        // Handle client-internal mutations that need persistence/presence access
2621        if m.index[0] == "setting_pushName"
2622            && let Some(val) = &m.action_value
2623            && let Some(act) = val.push_name_setting.as_option()
2624            && let Some(new_name) = &act.name
2625        {
2626            let new_name = new_name.clone();
2627            let bus = self.core.event_bus.clone();
2628
2629            let snapshot = self.persistence_manager.get_device_snapshot();
2630            let old = snapshot.push_name.clone();
2631            if old != new_name {
2632                debug!(target: "Client/AppState", "Persisting push name from app state mutation: '{}' (old='{}')", new_name, old);
2633                self.persistence_manager
2634                    .process_command(DeviceCommand::SetPushName(new_name.clone()))
2635                    .await;
2636                bus.dispatch(Event::SelfPushNameUpdated(
2637                    crate::types::events::SelfPushNameUpdated::builder()
2638                        .from_server(true)
2639                        .old_name(old.clone())
2640                        .new_name(new_name.clone())
2641                        .build(),
2642                ));
2643
2644                // WhatsApp Web sends presence immediately when receiving pushname
2645                if old.is_empty() && !new_name.is_empty() {
2646                    debug!(target: "Client/AppState", "Sending presence after receiving initial pushname from app state sync");
2647                    if let Err(e) = self.presence().set_available().await {
2648                        warn!(target: "Client/AppState", "Failed to send presence after pushname sync: {e:?}");
2649                    }
2650                }
2651            } else {
2652                debug!(target: "Client/AppState", "Push name mutation received but name unchanged: '{}'", new_name);
2653            }
2654        }
2655    }
2656
2657    #[cfg_attr(feature = "tracing", tracing::instrument(name = "wa.appstate.clean_dirty", level = "debug", skip_all, fields(bit = ?bit), err(Debug)))]
2658    pub async fn clean_dirty_bits(
2659        &self,
2660        bit: wacore::iq::dirty::DirtyBit,
2661    ) -> Result<(), crate::request::IqError> {
2662        use wacore::iq::dirty::CleanDirtyBitsSpec;
2663
2664        let spec = CleanDirtyBitsSpec::single(bit);
2665        self.execute(spec).await
2666    }
2667}
2668
2669#[cfg(test)]
2670mod tests {
2671    use super::*;
2672
2673    #[tokio::test]
2674    async fn key_arrival_finishes_before_a_slow_fanout() {
2675        let client = crate::test_utils::create_test_client_with_name("appstate_slow_peer").await;
2676        let backend = client.persistence_manager.backend();
2677        let key_id = vec![7, 8, 9, 10];
2678        let listener = client.initial_keys_synced_notifier.listen();
2679        let notifier = client.initial_keys_synced_notifier.clone();
2680        let writer = backend.clone();
2681        let stored_id = key_id.clone();
2682        let (fanout_polled_tx, fanout_polled_rx) = tokio::sync::oneshot::channel();
2683        tokio::spawn(async move {
2684            fanout_polled_rx.await.expect("fanout must be polled");
2685            writer
2686                .set_sync_key(
2687                    &stored_id,
2688                    crate::store::traits::AppStateSyncKey {
2689                        key_data: vec![7; 32],
2690                        ..Default::default()
2691                    },
2692                )
2693                .await
2694                .expect("store recovered key");
2695            notifier.notify(usize::MAX);
2696        });
2697
2698        let slow_fanout = async move {
2699            let _ = fanout_polled_tx.send(());
2700            std::future::pending::<AppStateKeyRequestSchedule>().await
2701        };
2702
2703        let progress = client
2704            .await_app_state_key_request(
2705                &*backend,
2706                std::slice::from_ref(&key_id),
2707                wacore::time::Instant::now() + Duration::from_secs(1),
2708                listener,
2709                slow_fanout,
2710            )
2711            .await;
2712
2713        assert!(matches!(progress, AppStateKeyRequestProgress::KeysReady));
2714    }
2715
2716    #[tokio::test]
2717    async fn passive_key_request_fanout_is_bounded() {
2718        async fn peer_request(
2719            device: u16,
2720            completes: bool,
2721        ) -> (u16, std::result::Result<(), anyhow::Error>) {
2722            if !completes {
2723                std::future::pending::<()>().await;
2724            }
2725            (device, Ok(()))
2726        }
2727
2728        let client =
2729            crate::test_utils::create_test_client_with_name("appstate_fanout_timeout").await;
2730        let requests = futures::stream::FuturesUnordered::new();
2731        requests.push(peer_request(1, true));
2732        requests.push(peer_request(2, false));
2733
2734        let delivery = tokio::time::timeout(
2735            Duration::from_secs(1),
2736            collect_app_state_key_request_results(
2737                &*client.runtime,
2738                requests,
2739                Duration::from_millis(20),
2740            ),
2741        )
2742        .await
2743        .expect("fanout collection must finish")
2744        .expect("one completed peer must preserve partial delivery");
2745
2746        assert_eq!(delivery, AppStateKeyRequestDelivery::SomePeers);
2747    }
2748
2749    #[test]
2750    fn empty_companion_discovery_falls_back_to_primary() {
2751        let primary: Jid = "5511000000000@s.whatsapp.net".parse().expect("primary jid");
2752        let peers = finalize_app_state_key_request_peers(Vec::new(), 7, primary.clone())
2753            .expect("companion fallback");
2754        assert_eq!(peers, vec![primary.clone()]);
2755        assert!(finalize_app_state_key_request_peers(Vec::new(), 0, primary).is_err());
2756    }
2757
2758    #[test]
2759    fn app_state_peers_use_the_own_pn_namespace() {
2760        let primary = Jid::pn("5511000000000");
2761        let peers = finalize_app_state_key_request_peers(
2762            vec![
2763                Jid::lid_device("100000000000001", 0),
2764                Jid::lid_device("100000000000001", 7),
2765                Jid::pn_device("5511000000000", 7),
2766            ],
2767            33,
2768            primary.clone(),
2769        )
2770        .expect("peer devices");
2771
2772        assert_eq!(peers, vec![primary, Jid::pn_device("5511000000000", 7)]);
2773    }
2774
2775    #[tokio::test]
2776    async fn active_key_wait_shortens_a_passive_dedup_stamp() {
2777        let client = crate::test_utils::create_test_client_with_name("appstate_retry_stamp").await;
2778        let key_id = vec![1, 2, 3, 4];
2779        client.app_state_key_requests.lock().await.insert(
2780            key_id.clone(),
2781            wacore::time::Instant::now() + APP_STATE_KEY_REQUEST_DEDUP,
2782        );
2783
2784        let started = wacore::time::Instant::now();
2785        let schedule = client
2786            .request_missing_keys_with_dedup(
2787                std::slice::from_ref(&key_id),
2788                APP_STATE_KEY_PARTIAL_RETRY,
2789            )
2790            .await;
2791
2792        assert!(
2793            !schedule.sent,
2794            "an in-flight request must not be duplicated"
2795        );
2796        assert!(schedule.retry_at > started);
2797        assert!(
2798            schedule.retry_at.saturating_duration_since(started)
2799                <= APP_STATE_KEY_PARTIAL_RETRY + Duration::from_millis(100),
2800            "an active waiter must retry before the passive 24-hour deadline"
2801        );
2802        assert_eq!(
2803            client
2804                .app_state_key_requests
2805                .lock()
2806                .await
2807                .get(key_id.as_slice())
2808                .copied(),
2809            Some(schedule.retry_at)
2810        );
2811    }
2812
2813    #[test]
2814    fn ordinary_key_wait_leaves_time_for_a_retry() {
2815        let retry = initial_app_state_key_retry(APP_STATE_KEY_REQUEST_TIMEOUT);
2816
2817        assert_eq!(retry, Duration::from_secs(5));
2818        assert!(retry < APP_STATE_KEY_REQUEST_TIMEOUT);
2819        assert_eq!(
2820            initial_app_state_key_retry(Duration::from_secs(180)),
2821            APP_STATE_KEY_PARTIAL_RETRY
2822        );
2823    }
2824}
2825
2826// ─── #1157: the app-state send path must read the server's answer ───────────
2827//
2828// `w:sync:app:state` enforces optimistic concurrency on the collection's
2829// `version`. A patch built against a stale base is not rejected at the IQ
2830// level: the IQ succeeds and the failure is reported *inside* it, as
2831// `<collection type="error"><error code="409"/>`, carrying the patches that
2832// won. WA Web reads exactly that (`WAWebSyncdResponseParser`, fn `h`) and maps
2833// it onto `CollectionState.Conflict{,HasMore}`; the collection then goes
2834// through `applyAppStateSyncResponse` like any other, and `serverSync` re-queues
2835// it for another round as long as pending mutations remain — so the mutation is
2836// re-sent on the winner's base instead of being dropped. `400`/`404` map to
2837// `ErrorFatal`, anything else to `ErrorRetry`.
2838//
2839// These tests pin what the send path must make of each response shape: a 409 it
2840// can resolve (rebuild and resend), a 409 it cannot (an error, after exhausting
2841// the rebuild attempts), a fatal code (an error, not retried), and a response
2842// carrying no collection verdict at all (accepted). Discarding the response —
2843// which is what made a 409 indistinguishable from success — fails all four.
2844#[cfg(test)]
2845mod send_patch_response_tests {
2846    use super::*;
2847    use std::sync::atomic::AtomicUsize;
2848    use wacore_binary::node::Node;
2849
2850    /// Seed the client's store with an app-state key so `build_patch` can sign,
2851    /// and give the collection a non-zero base so the IQ carries a `version`.
2852    async fn seed_collection(client: &Arc<Client>, collection: &str) -> Vec<u8> {
2853        let backend = client.persistence_manager.backend();
2854        let key_id = b"send-patch-key".to_vec();
2855        backend
2856            .set_sync_key(
2857                &key_id,
2858                crate::store::traits::AppStateSyncKey {
2859                    key_data: vec![5u8; 32],
2860                    ..Default::default()
2861                },
2862            )
2863            .await
2864            .expect("test backend should accept a sync key");
2865        backend
2866            .set_version(
2867                collection,
2868                wacore::appstate::hash::HashState {
2869                    version: 7,
2870                    ..Default::default()
2871                },
2872            )
2873            .await
2874            .expect("test backend should accept a version");
2875        key_id
2876    }
2877
2878    /// A `<collection>` the server marks as failed, mirroring the shape
2879    /// `WAWebSyncdResponseParser` reads.
2880    fn collection_error_result(request_id: &str, collection: &str, code: &str) -> Node {
2881        NodeBuilder::new("iq")
2882            .attr("type", "result")
2883            .attr("id", request_id)
2884            .attr("from", "s.whatsapp.net")
2885            .children([NodeBuilder::new("sync")
2886                .children([NodeBuilder::new("collection")
2887                    .attr("name", collection)
2888                    .attr("type", "error")
2889                    .children([NodeBuilder::new("error")
2890                        .attr("code", code)
2891                        .attr("text", "")
2892                        .build()])
2893                    .build()])
2894                .build()])
2895            .build()
2896    }
2897
2898    /// A collection the server reports as clean and up to date.
2899    fn empty_sync_result(request_id: &str, collection: &str) -> Node {
2900        NodeBuilder::new("iq")
2901            .attr("type", "result")
2902            .attr("id", request_id)
2903            .attr("from", "s.whatsapp.net")
2904            .children([NodeBuilder::new("sync")
2905                .children([NodeBuilder::new("collection")
2906                    .attr("name", collection)
2907                    .build()])
2908                .build()])
2909            .build()
2910    }
2911
2912    const COLLECTION: &str = "regular_low";
2913
2914    /// Answers every IQ the client writes, in order, with whatever `reply`
2915    /// returns for it — `Some(code)` for a `<collection type="error">`, `None`
2916    /// for a clean result. Runs forever: callers race it against the send, so a
2917    /// send that stops writing simply drops this future.
2918    ///
2919    /// `reply` is told the send-attempt number for patch IQs (0 for the
2920    /// re-syncs in between), which is what lets a test answer "conflict once,
2921    /// then accept".
2922    async fn serve_iqs(
2923        client: &Arc<Client>,
2924        transport: &Arc<crate::transport::mock::CapturingMockTransport>,
2925        patch_attempts: &AtomicUsize,
2926        response_collection: &str,
2927        mut reply: impl FnMut(usize) -> Option<&'static str>,
2928    ) {
2929        let mut frame = 0usize;
2930        loop {
2931            let node = crate::test_utils::decode_sent_iq(transport, frame).await;
2932            let node = node.get().to_owned();
2933            let id = node
2934                .attrs()
2935                .optional_string("id")
2936                .expect("every IQ carries an id")
2937                .into_owned();
2938            let attempt = if node
2939                .get_optional_child_by_tag(&["sync", "collection", "patch"])
2940                .is_some()
2941            {
2942                patch_attempts.fetch_add(1, Ordering::Relaxed) + 1
2943            } else {
2944                0
2945            };
2946            let response = match reply(attempt) {
2947                Some(code) => collection_error_result(&id, response_collection, code),
2948                None => empty_sync_result(&id, response_collection),
2949            };
2950            crate::test_utils::answer_iq(client, &id, &response).await;
2951            frame += 1;
2952        }
2953    }
2954
2955    /// Drives one `send_app_state_patch` to completion against `reply`, and
2956    /// reports how many patch IQs reached the wire.
2957    async fn send_against(reply: impl FnMut(usize) -> Option<&'static str>) -> (Result<()>, usize) {
2958        send_against_collection(COLLECTION, reply).await
2959    }
2960
2961    async fn send_against_collection(
2962        response_collection: &'static str,
2963        reply: impl FnMut(usize) -> Option<&'static str>,
2964    ) -> (Result<()>, usize) {
2965        let (client, transport) = crate::test_utils::create_iq_test_client().await;
2966        seed_collection(&client, COLLECTION).await;
2967
2968        let mut send = {
2969            let client = Arc::clone(&client);
2970            tokio::spawn(async move {
2971                client
2972                    .send_app_state_patch(COLLECTION, vec![wa::SyncdMutation::default()])
2973                    .await
2974            })
2975        };
2976
2977        let patch_attempts = AtomicUsize::new(0);
2978        let server = serve_iqs(
2979            &client,
2980            &transport,
2981            &patch_attempts,
2982            response_collection,
2983            reply,
2984        );
2985        futures::pin_mut!(server);
2986        let result = futures::select! {
2987            result = (&mut send).fuse() => result.expect("the send task should not panic"),
2988            () = server.as_mut().fuse() => unreachable!("the responder never completes"),
2989        };
2990
2991        (result, patch_attempts.load(Ordering::Relaxed))
2992    }
2993
2994    #[tokio::test]
2995    async fn response_for_a_different_collection_is_rejected() {
2996        for error in [None, Some("409")] {
2997            let (result, patches) = send_against_collection("regular_high", move |_| error).await;
2998            assert!(
2999                result.is_err(),
3000                "a response for another collection must not accept or absorb this send"
3001            );
3002            assert_eq!(
3003                patches, 1,
3004                "a mismatched response must fail before retrying the mutation"
3005            );
3006        }
3007    }
3008
3009    /// A 409 means the patch was built on a stale base and did NOT land. A
3010    /// server that keeps rejecting must end as an error, never as success — a
3011    /// `markChatAsRead` that silently lost must not be reported as done.
3012    #[tokio::test]
3013    async fn unresolvable_conflict_is_not_reported_as_success() {
3014        let (result, patches) = send_against(|_| Some("409")).await;
3015        assert!(
3016            result.is_err(),
3017            "a 409 conflict means the mutation was dropped; reporting Ok hides the loss"
3018        );
3019        assert_eq!(
3020            patches, APP_STATE_PATCH_SEND_ATTEMPTS,
3021            "the send must exhaust its rebuild attempts before giving up"
3022        );
3023    }
3024
3025    /// The resolution path: the first attempt loses the race, the client
3026    /// rebuilds against the new base, and the second attempt lands. That is WA
3027    /// Web's conflict loop, and the mutation survives it.
3028    #[tokio::test]
3029    async fn conflict_is_resolved_by_rebuilding_and_resending() {
3030        let (result, patches) =
3031            send_against(|attempt| if attempt == 1 { Some("409") } else { None }).await;
3032        result.expect("a conflict the server later accepts must succeed, not fail");
3033        assert_eq!(
3034            patches, 2,
3035            "the losing patch must be rebuilt and re-sent exactly once"
3036        );
3037    }
3038
3039    /// A bare `<iq type="result"/>` carries no per-collection verdict, so there
3040    /// is nothing to reject: reading the response must not turn a peer that
3041    /// answers tersely into a failing send.
3042    #[tokio::test]
3043    async fn response_without_a_collection_verdict_is_accepted() {
3044        let (client, transport) = crate::test_utils::create_iq_test_client().await;
3045        seed_collection(&client, COLLECTION).await;
3046
3047        let mut send = {
3048            let client = Arc::clone(&client);
3049            tokio::spawn(async move {
3050                client
3051                    .send_app_state_patch(COLLECTION, vec![wa::SyncdMutation::default()])
3052                    .await
3053            })
3054        };
3055
3056        let bare = async {
3057            let mut frame = 0usize;
3058            loop {
3059                let node = crate::test_utils::decode_sent_iq(&transport, frame).await;
3060                let id = node
3061                    .get()
3062                    .attrs()
3063                    .optional_string("id")
3064                    .expect("every IQ carries an id")
3065                    .into_owned();
3066                crate::test_utils::answer_iq(
3067                    &client,
3068                    &id,
3069                    &NodeBuilder::new("iq")
3070                        .attr("type", "result")
3071                        .attr("id", &id)
3072                        .attr("from", "s.whatsapp.net")
3073                        .build(),
3074                )
3075                .await;
3076                frame += 1;
3077            }
3078        };
3079        futures::pin_mut!(bare);
3080
3081        let result = futures::select! {
3082            result = (&mut send).fuse() => result.expect("the send task should not panic"),
3083            () = bare.as_mut().fuse() => unreachable!("the responder never completes"),
3084        };
3085        result.expect("a terse but successful response must not read as a rejection");
3086    }
3087
3088    /// A `<collection>` that IS present but does not parse may be carrying the
3089    /// rejection. Manufacturing an empty success from it would drop the
3090    /// mutation exactly as discarding the response did.
3091    #[tokio::test]
3092    async fn unreadable_collection_is_not_mistaken_for_an_absent_one() {
3093        let (client, transport) = crate::test_utils::create_iq_test_client().await;
3094        seed_collection(&client, COLLECTION).await;
3095
3096        let mut send = {
3097            let client = Arc::clone(&client);
3098            tokio::spawn(async move {
3099                client
3100                    .send_app_state_patch(COLLECTION, vec![wa::SyncdMutation::default()])
3101                    .await
3102            })
3103        };
3104
3105        let malformed = async {
3106            let mut frame = 0usize;
3107            loop {
3108                let node = crate::test_utils::decode_sent_iq(&transport, frame).await;
3109                let id = node
3110                    .get()
3111                    .attrs()
3112                    .optional_string("id")
3113                    .expect("every IQ carries an id")
3114                    .into_owned();
3115                // A collection with no `name`: present, unreadable.
3116                crate::test_utils::answer_iq(
3117                    &client,
3118                    &id,
3119                    &NodeBuilder::new("iq")
3120                        .attr("type", "result")
3121                        .attr("id", &id)
3122                        .attr("from", "s.whatsapp.net")
3123                        .children([NodeBuilder::new("sync")
3124                            .children([NodeBuilder::new("collection")
3125                                .attr("type", "error")
3126                                .build()])
3127                            .build()])
3128                        .build(),
3129                )
3130                .await;
3131                frame += 1;
3132            }
3133        };
3134        futures::pin_mut!(malformed);
3135
3136        let result = futures::select! {
3137            result = (&mut send).fuse() => result.expect("the send task should not panic"),
3138            () = malformed.as_mut().fuse() => unreachable!("the responder never completes"),
3139        };
3140        assert!(
3141            result.is_err(),
3142            "a collection we cannot read may be the rejection; it must not read as success"
3143        );
3144    }
3145
3146    /// 400/404 are `ErrorFatal` in WA Web and `ErrAppStateUpdate` in whatsmeow —
3147    /// never success, and never retried.
3148    #[tokio::test]
3149    async fn fatal_collection_error_is_not_reported_as_success() {
3150        let (result, patches) = send_against(|_| Some("400")).await;
3151        assert!(
3152            result.is_err(),
3153            "a fatal collection error must surface to the caller, not read as success"
3154        );
3155        assert_eq!(patches, 1, "a fatal error must not be retried");
3156    }
3157}
3158
3159#[cfg(test)]
3160mod sync_in_flight_tests {
3161    use super::*;
3162
3163    /// A consumer-issued full sync must not run alongside a sync already
3164    /// writing the collection's version and mutation MACs — and must not be
3165    /// dropped either, since the snapshot it asks for is not what an
3166    /// incremental sync in flight is fetching.
3167    #[tokio::test]
3168    async fn a_full_sync_task_waits_for_the_collection() {
3169        let client = crate::test_utils::create_test_client_with_name("appstate-task-wait").await;
3170        let held = client
3171            .app_state_syncing
3172            .try_begin(WAPatchName::CriticalBlock)
3173            .expect("reserve the collection first");
3174
3175        let task = tokio::spawn({
3176            let client = Arc::clone(&client);
3177            async move {
3178                client
3179                    .process_sync_task(MajorSyncTask::AppStateSync {
3180                        name: WAPatchName::CriticalBlock,
3181                        full_sync: true,
3182                    })
3183                    .await;
3184            }
3185        });
3186
3187        for _ in 0..8 {
3188            tokio::task::yield_now().await;
3189        }
3190        assert!(
3191            !task.is_finished(),
3192            "the task ran while the collection was reserved"
3193        );
3194        assert_eq!(
3195            client.app_state_syncing.len(),
3196            1,
3197            "only the held reservation"
3198        );
3199
3200        drop(held);
3201        // The client has no socket, so the sync itself fails fast with
3202        // NotConnected; what matters is that it got to run and released.
3203        tokio::time::timeout(Duration::from_secs(5), task)
3204            .await
3205            .expect("released collection must let the task proceed")
3206            .expect("task panicked");
3207        assert_eq!(
3208            client.app_state_syncing.len(),
3209            0,
3210            "the task's own reservation must be released"
3211        );
3212    }
3213
3214    /// An incremental task waits too: the holder may be a patch send, which
3215    /// never fetches, so skipping would drop the requested sync entirely.
3216    #[tokio::test]
3217    async fn an_incremental_sync_task_also_waits() {
3218        let client = crate::test_utils::create_test_client_with_name("appstate-task-skip").await;
3219        let held = client
3220            .app_state_syncing
3221            .try_begin(WAPatchName::Regular)
3222            .expect("reserve the collection first");
3223
3224        let task = tokio::spawn({
3225            let client = Arc::clone(&client);
3226            async move {
3227                client
3228                    .process_sync_task(MajorSyncTask::AppStateSync {
3229                        name: WAPatchName::Regular,
3230                        full_sync: false,
3231                    })
3232                    .await;
3233            }
3234        });
3235
3236        for _ in 0..8 {
3237            tokio::task::yield_now().await;
3238        }
3239        assert!(
3240            !task.is_finished(),
3241            "the task ran while the collection was reserved"
3242        );
3243
3244        drop(held);
3245        tokio::time::timeout(Duration::from_secs(5), task)
3246            .await
3247            .expect("released collection must let the task proceed")
3248            .expect("task panicked");
3249        assert_eq!(client.app_state_syncing.len(), 0, "reservation released");
3250    }
3251
3252    #[test]
3253    fn second_begin_blocked_until_release() {
3254        let registry = SyncInFlight::new();
3255        let guard = registry
3256            .try_begin(WAPatchName::Regular)
3257            .expect("first begin must reserve");
3258        assert!(
3259            registry.try_begin(WAPatchName::Regular).is_none(),
3260            "in-flight collection must dedup"
3261        );
3262        // Other collections are independent.
3263        assert!(registry.try_begin(WAPatchName::CriticalBlock).is_some());
3264
3265        drop(guard);
3266        assert!(
3267            registry.try_begin(WAPatchName::Regular).is_some(),
3268            "release (including cancellation drop) must free the slot"
3269        );
3270    }
3271
3272    #[test]
3273    fn stale_guard_does_not_clobber_new_generation() {
3274        let registry = SyncInFlight::new();
3275        // Generation 1 reserves, then a reconnect clears the registry while
3276        // the task is still in flight.
3277        let stale = registry
3278            .try_begin(WAPatchName::Regular)
3279            .expect("gen-1 reserve");
3280        registry.clear();
3281
3282        // Generation 2 reserves the same collection.
3283        let fresh = registry
3284            .try_begin(WAPatchName::Regular)
3285            .expect("post-clear reserve");
3286
3287        // The stale task finishing must NOT evict generation 2's reservation.
3288        drop(stale);
3289        assert!(
3290            registry.try_begin(WAPatchName::Regular).is_none(),
3291            "stale release clobbered the new generation's reservation"
3292        );
3293
3294        drop(fresh);
3295        assert!(registry.try_begin(WAPatchName::Regular).is_some());
3296    }
3297
3298    /// A patch send cannot treat "already in flight" as "nothing to do": it has
3299    /// to write the same version and mutation-MAC rows the sync writes, so it
3300    /// waits for the holder instead of skipping.
3301    #[tokio::test]
3302    async fn begin_waits_for_the_holder_instead_of_skipping() {
3303        let registry = SyncInFlight::new();
3304        let held = registry
3305            .try_begin(WAPatchName::Regular)
3306            .expect("first reserve");
3307
3308        let (reserved_tx, mut reserved_rx) = tokio::sync::oneshot::channel();
3309        let waiter = {
3310            let registry = Arc::clone(&registry);
3311            tokio::spawn(async move {
3312                let guard = registry.begin(WAPatchName::Regular, SyncHolder::Sync).await;
3313                let _ = reserved_tx.send(());
3314                guard
3315            })
3316        };
3317
3318        // A parked listener is proof the waiter reached its await point — the
3319        // observable a "still waiting" assertion needs instead of a sleep.
3320        crate::test_utils::poll_until("the waiter to park on the registry", || {
3321            registry.released.total_listeners() >= 1
3322        })
3323        .await;
3324        assert!(
3325            reserved_rx.try_recv().is_err(),
3326            "begin must not resolve while the collection is held"
3327        );
3328
3329        drop(held);
3330        let guard = waiter.await.expect("the waiter should not panic");
3331        assert!(
3332            registry.try_begin(WAPatchName::Regular).is_none(),
3333            "the waiter must now hold the reservation, not merely have observed it free"
3334        );
3335
3336        drop(guard);
3337        assert!(registry.try_begin(WAPatchName::Regular).is_some());
3338    }
3339}
3340
3341#[cfg(test)]
3342pub(crate) mod batched_sync_outcome_tests {
3343    use super::*;
3344    use wacore_binary::node::Node;
3345
3346    /// One `<iq result>` answering a whole batch, with each collection either
3347    /// clean or carrying an `<error code>` — the shape
3348    /// `WAWebSyncdResponseParser` reads.
3349    pub(crate) fn batch_result(request_id: &str, collections: &[(&str, Option<&str>)]) -> Node {
3350        let children: Vec<Node> = collections
3351            .iter()
3352            .map(|(name, error)| {
3353                let builder = NodeBuilder::new("collection").attr("name", *name);
3354                match error {
3355                    Some(code) => builder
3356                        .attr("type", "error")
3357                        .children([NodeBuilder::new("error")
3358                            .attr("code", *code)
3359                            .attr("text", "")
3360                            .build()])
3361                        .build(),
3362                    None => builder.build(),
3363                }
3364            })
3365            .collect();
3366        NodeBuilder::new("iq")
3367            .attr("type", "result")
3368            .attr("id", request_id)
3369            .attr("from", "s.whatsapp.net")
3370            .children([NodeBuilder::new("sync").children(children).build()])
3371            .build()
3372    }
3373
3374    /// Runs one batched sync against a server that answers every IQ with
3375    /// `collections`, and reports the outcome plus how many IQs reached the
3376    /// wire. The responder never completes, so it is raced against the sync.
3377    pub(crate) async fn sync_against(
3378        request: Vec<WAPatchName>,
3379        collections: &'static [(&'static str, Option<&'static str>)],
3380    ) -> (BatchedSyncOutcome, usize) {
3381        use futures::FutureExt;
3382        let (client, transport) = crate::test_utils::create_iq_test_client().await;
3383
3384        let mut sync = {
3385            let client = Arc::clone(&client);
3386            tokio::spawn(async move {
3387                let scope = client.sync_scope(None);
3388                client.sync_collections_batched(request, scope).await
3389            })
3390        };
3391
3392        let sent = AtomicU64::new(0);
3393        let server = async {
3394            let mut frame = 0usize;
3395            loop {
3396                let node = crate::test_utils::decode_sent_iq(&transport, frame).await;
3397                let node = node.get().to_owned();
3398                let id = node
3399                    .attrs()
3400                    .optional_string("id")
3401                    .expect("every IQ carries an id")
3402                    .into_owned();
3403                sent.fetch_add(1, Ordering::Relaxed);
3404                let response = batch_result(&id, collections);
3405                crate::test_utils::answer_iq(&client, &id, &response).await;
3406                frame += 1;
3407            }
3408        };
3409        futures::pin_mut!(server);
3410        let outcome = futures::select! {
3411            result = (&mut sync).fuse() => result
3412                .expect("the sync task should not panic")
3413                .expect("a per-collection error is an outcome, not a transport failure"),
3414            () = server.as_mut().fuse() => unreachable!("the responder never completes"),
3415        };
3416
3417        (outcome, sent.load(Ordering::Relaxed) as usize)
3418    }
3419
3420    /// A refused collection used to be logged and dropped, and the batch still
3421    /// reported success — which the initial bootstrap reads as permission to
3422    /// dispatch Connected.
3423    #[tokio::test]
3424    async fn a_refused_collection_is_reported_fatal_not_synced() {
3425        let (outcome, _) = sync_against(
3426            vec![WAPatchName::CriticalBlock, WAPatchName::CriticalUnblockLow],
3427            &[
3428                ("critical_block", Some("404")),
3429                ("critical_unblock_low", None),
3430            ],
3431        )
3432        .await;
3433
3434        assert_eq!(outcome.fatal, vec![WAPatchName::CriticalBlock]);
3435        assert_eq!(outcome.synced, vec![WAPatchName::CriticalUnblockLow]);
3436        assert!(!outcome.all_synced(), "the batch did not fully sync");
3437    }
3438
3439    /// A retryable collection is done for this run. WA Web routes ErrorRetry to
3440    /// `doneCollections`, never to `refetchCollections`, so it must not be
3441    /// re-asked inside the same loop — with a 500-iteration cap that would mean
3442    /// hammering a failing collection 500 times.
3443    #[tokio::test]
3444    async fn a_retryable_collection_is_not_refetched_in_the_same_run() {
3445        let (outcome, iqs) =
3446            sync_against(vec![WAPatchName::Regular], &[("regular", Some("500"))]).await;
3447
3448        assert_eq!(outcome.retryable, vec![WAPatchName::Regular]);
3449        assert!(outcome.fatal.is_empty(), "500 is not terminal");
3450        assert_eq!(iqs, 1, "a retryable error must not be re-asked in this run");
3451    }
3452
3453    /// The batch reported `Ok(())` when every collection was already in flight,
3454    /// which reads identically to "all synced" at the call site.
3455    #[tokio::test]
3456    async fn collections_held_by_another_sync_are_reported_skipped() {
3457        let (client, transport) = crate::test_utils::create_iq_test_client().await;
3458        let _held = client
3459            .app_state_syncing
3460            .try_begin_as(WAPatchName::CriticalBlock, SyncHolder::Sync)
3461            .expect("reserve the collection first");
3462
3463        let outcome = client
3464            .sync_collections_batched(vec![WAPatchName::CriticalBlock], client.sync_scope(None))
3465            .await
3466            .expect("skipping is an outcome, not an error");
3467
3468        assert_eq!(outcome.skipped, vec![WAPatchName::CriticalBlock]);
3469        assert!(outcome.synced.is_empty());
3470        assert!(!outcome.all_synced(), "a skipped collection did not sync");
3471        assert!(
3472            transport.sent().is_empty(),
3473            "a skipped batch must not reach the wire"
3474        );
3475    }
3476
3477    /// A patch send holds the same reservation and never fetches, so a sync
3478    /// that skipped behind one would be dropped and the patches that prompted
3479    /// it would go unfetched.
3480    #[tokio::test]
3481    async fn a_patch_send_holder_is_waited_out_not_skipped() {
3482        let (client, _transport) = crate::test_utils::create_iq_test_client().await;
3483        let held = client
3484            .app_state_syncing
3485            .try_begin_as(WAPatchName::Regular, SyncHolder::PatchSend)
3486            .expect("reserve the collection first");
3487
3488        let reserve = {
3489            let client = Arc::clone(&client);
3490            tokio::spawn(async move {
3491                client
3492                    .reserve_for_sync(
3493                        WAPatchName::Regular,
3494                        ReservationWait::SkipBehindSync,
3495                        client.sync_scope(None),
3496                    )
3497                    .await
3498                    .map(drop)
3499            })
3500        };
3501
3502        crate::test_utils::poll_until("the sync to park behind the patch send", || {
3503            client.app_state_syncing.released.total_listeners() >= 1
3504        })
3505        .await;
3506        assert!(
3507            !reserve.is_finished(),
3508            "a sync must wait for a patch send, not skip it"
3509        );
3510
3511        drop(held);
3512        tokio::time::timeout(Duration::from_secs(5), reserve)
3513            .await
3514            .expect("releasing the send must let the sync proceed")
3515            .expect("the reserve task should not panic")
3516            .expect("the sync must get the reservation, not time out");
3517    }
3518
3519    #[test]
3520    fn all_synced_is_false_for_every_kind_of_miss() {
3521        let mut outcome = BatchedSyncOutcome::default();
3522        assert!(outcome.all_synced(), "an empty batch missed nothing");
3523
3524        outcome.synced.push(WAPatchName::Regular);
3525        assert!(outcome.all_synced());
3526
3527        let buckets: [fn(&mut BatchedSyncOutcome) -> &mut Vec<WAPatchName>; 3] =
3528            [|o| &mut o.fatal, |o| &mut o.retryable, |o| &mut o.skipped];
3529        for bucket in buckets {
3530            bucket(&mut outcome).push(WAPatchName::CriticalBlock);
3531            assert!(!outcome.all_synced());
3532            bucket(&mut outcome).clear();
3533        }
3534    }
3535}
3536
3537#[cfg(test)]
3538mod batched_sync_reconciliation_tests {
3539    use super::*;
3540    use crate::client::app_state::batched_sync_outcome_tests::{batch_result, sync_against};
3541
3542    /// A `<sync>` that simply leaves a requested collection out parses fine, so
3543    /// nothing used to record it and `all_synced()` reported a batch that never
3544    /// covered it — the same false success this module exists to stop.
3545    #[tokio::test]
3546    async fn a_collection_the_response_omits_is_not_reported_synced() {
3547        let (outcome, _) = sync_against(
3548            vec![WAPatchName::CriticalBlock, WAPatchName::CriticalUnblockLow],
3549            &[("critical_unblock_low", None)],
3550        )
3551        .await;
3552
3553        assert_eq!(outcome.synced, vec![WAPatchName::CriticalUnblockLow]);
3554        assert_eq!(
3555            outcome.retryable,
3556            vec![WAPatchName::CriticalBlock],
3557            "an omitted collection is a miss, not a success"
3558        );
3559        assert!(!outcome.all_synced());
3560    }
3561
3562    /// An empty `<sync/>` accounts for nothing at all.
3563    #[tokio::test]
3564    async fn an_empty_response_leaves_every_collection_unsynced() {
3565        let (outcome, _) = sync_against(vec![WAPatchName::Regular], &[]).await;
3566
3567        assert!(outcome.synced.is_empty());
3568        assert_eq!(outcome.retryable, vec![WAPatchName::Regular]);
3569        assert!(!outcome.all_synced());
3570    }
3571
3572    /// A repeated collection must be applied once, not twice.
3573    #[tokio::test]
3574    async fn a_repeated_collection_is_counted_once() {
3575        let (outcome, _) = sync_against(
3576            vec![WAPatchName::Regular],
3577            &[("regular", None), ("regular", None)],
3578        )
3579        .await;
3580
3581        assert_eq!(outcome.synced, vec![WAPatchName::Regular]);
3582        assert!(outcome.all_synced());
3583    }
3584
3585    /// A wait that runs out leaves the collection uncovered by anyone, so it is
3586    /// a miss worth retrying rather than a skip someone else is handling.
3587    ///
3588    /// Paused time so the bound actually elapses instead of the test sitting
3589    /// there for [`APP_STATE_RESERVATION_WAIT`]; the holder never releases, so
3590    /// the timeout is the only way out.
3591    #[tokio::test(start_paused = true)]
3592    async fn a_wait_that_runs_out_reports_the_collection_retryable() {
3593        let (client, transport) = crate::test_utils::create_iq_test_client().await;
3594        // A patch send is never equivalent work, so the batched sync waits for
3595        // it rather than skipping — which is what puts the bound in play.
3596        let _held = client
3597            .app_state_syncing
3598            .try_begin_as(WAPatchName::Regular, SyncHolder::PatchSend)
3599            .expect("reserve the collection first");
3600
3601        let outcome = client
3602            .sync_collections_batched(vec![WAPatchName::Regular], client.sync_scope(None))
3603            .await
3604            .expect("a wait that ran out is an outcome, not a transport failure");
3605
3606        assert_eq!(
3607            outcome.retryable,
3608            vec![WAPatchName::Regular],
3609            "nobody is covering it, so it has to come back around"
3610        );
3611        assert!(
3612            outcome.skipped.is_empty(),
3613            "skipped means an equivalent sync has it, which is not the case here"
3614        );
3615        assert!(
3616            transport.sent().is_empty(),
3617            "the sync never got its turn, so nothing should reach the wire"
3618        );
3619    }
3620
3621    /// `batch_result` is shared with the outcome tests; this keeps the helper
3622    /// honest about the shape it builds.
3623    #[test]
3624    fn batch_result_marks_errors_on_the_named_collection() {
3625        let node = batch_result("id-1", &[("regular", Some("500"))]);
3626        let collection = node
3627            .get_optional_child_by_tag(&["sync", "collection"])
3628            .expect("the helper builds sync/collection");
3629        assert_eq!(
3630            collection.attrs().optional_string("type").as_deref(),
3631            Some("error")
3632        );
3633    }
3634}
3635
3636#[cfg(test)]
3637mod duplicate_collection_tests {
3638    use super::*;
3639    use crate::client::app_state::batched_sync_outcome_tests::sync_against;
3640
3641    /// The processor persists every list it is handed, so a repeated collection
3642    /// has to be dropped before it is processed, not reconciled afterwards.
3643    /// Reconciling late still applies the collection twice, and the second
3644    /// application can move the MAC store past the version the first writes
3645    /// back.
3646    #[tokio::test]
3647    async fn a_duplicate_collection_is_dropped_before_it_is_applied() {
3648        let (outcome, _) = sync_against(
3649            vec![WAPatchName::Regular],
3650            &[("regular", None), ("regular", None)],
3651        )
3652        .await;
3653
3654        assert_eq!(
3655            outcome.synced,
3656            vec![WAPatchName::Regular],
3657            "the collection is accounted for exactly once"
3658        );
3659        assert!(outcome.all_synced());
3660    }
3661
3662    /// A duplicate must not make the batch look incomplete either: it is the
3663    /// same collection, already answered.
3664    #[tokio::test]
3665    async fn a_duplicate_does_not_leave_the_batch_unsynced() {
3666        let (outcome, _) = sync_against(
3667            vec![WAPatchName::CriticalBlock, WAPatchName::CriticalUnblockLow],
3668            &[
3669                ("critical_block", None),
3670                ("critical_block", None),
3671                ("critical_unblock_low", None),
3672            ],
3673        )
3674        .await;
3675
3676        assert!(outcome.retryable.is_empty(), "both were answered");
3677        assert!(outcome.all_synced());
3678    }
3679}
3680
3681#[cfg(test)]
3682mod background_report_tests {
3683    use super::*;
3684    use crate::types::events::{EventHandler, EventInterest, EventKind};
3685
3686    struct FailureCounter(Arc<AtomicU64>);
3687
3688    impl EventHandler for FailureCounter {
3689        fn handle_event(&self, event: Arc<Event>) {
3690            if matches!(&*event, Event::AppStateSyncFailed(_)) {
3691                self.0.fetch_add(1, Ordering::Relaxed);
3692            }
3693        }
3694    }
3695
3696    /// A background sync awaits a round trip before it reports, so its outcome
3697    /// can belong to a socket that has since been replaced. Publishing it would
3698    /// hand a consumer a refusal whose documented response — logout, or forcing
3699    /// a recovery — would land on the healthy session that took its place.
3700    #[tokio::test]
3701    async fn an_outcome_from_a_retired_connection_is_not_published() {
3702        let client = crate::test_utils::create_test_client_with_name("bg-report-gen").await;
3703        let retired_scope = client.sync_scope(None);
3704
3705        let seen = Arc::new(AtomicU64::new(0));
3706        let _subscription = client.subscribe(
3707            EventInterest::of(&[EventKind::AppStateSyncFailed]),
3708            Arc::new(FailureCounter(Arc::clone(&seen))),
3709        );
3710
3711        let mut outcome = BatchedSyncOutcome::default();
3712        outcome.fatal.push(WAPatchName::CriticalBlock);
3713
3714        // The connection this sync belonged to is gone.
3715        client
3716            .connection_generation
3717            .store(retired_scope.generation() + 1, Ordering::SeqCst);
3718        client.report_background_sync(
3719            "test",
3720            retired_scope,
3721            SyncSettles::JustTheCollections,
3722            &[],
3723            Ok(outcome.clone()),
3724        );
3725        assert_eq!(
3726            seen.load(Ordering::Relaxed),
3727            0,
3728            "a retired connection's refusal must not reach consumers"
3729        );
3730
3731        // The live one still reports.
3732        client.report_background_sync(
3733            "test",
3734            client.sync_scope(None),
3735            SyncSettles::JustTheCollections,
3736            &[],
3737            Ok(outcome),
3738        );
3739        assert_eq!(seen.load(Ordering::Relaxed), 1);
3740    }
3741}
3742
3743#[cfg(test)]
3744mod retry_gate_tests {
3745    use super::*;
3746
3747    /// The bootstrap gate stands down on proof of sync, not on an empty
3748    /// retryable bucket. A round that ends in a refusal, or behind another sync,
3749    /// also leaves nothing to retry while the collection is still not synced,
3750    /// and clearing there lets the next connection skip the only thing that
3751    /// guarantees another attempt.
3752    #[test]
3753    fn only_a_fully_synced_outcome_settles_the_bootstrap() {
3754        let mut fatal = BatchedSyncOutcome::default();
3755        fatal.fatal.push(WAPatchName::CriticalBlock);
3756        assert!(fatal.retryable.is_empty(), "nothing left to retry");
3757        assert!(
3758            !fatal.all_synced(),
3759            "but the collection is not synced, so the gate must stay armed"
3760        );
3761
3762        let mut skipped = BatchedSyncOutcome::default();
3763        skipped.skipped.push(WAPatchName::CriticalBlock);
3764        assert!(skipped.retryable.is_empty());
3765        assert!(
3766            !skipped.all_synced(),
3767            "another sync holding it is not proof it succeeded"
3768        );
3769
3770        let mut synced = BatchedSyncOutcome::default();
3771        synced.synced.push(WAPatchName::CriticalBlock);
3772        assert!(synced.all_synced(), "this is the only case that settles it");
3773    }
3774
3775    // Not covered: that the scheduler refuses to run for a retired generation.
3776    // Two attempts at it passed with the guard removed — the gate stays armed
3777    // because the wrongly-attempted sync fails anyway on a socketless client,
3778    // and the attempt never reaches the capturing transport either. Asserting
3779    // on the gate or on the wire both hold for the wrong reason, and a test
3780    // that passes without the fix is worse than none. It needs a connected
3781    // fixture that can complete a sync, which the report-path test below
3782    // already has for its own case.
3783
3784    /// The backoff doubles from the minimum and clamps, matching the syncd
3785    /// spacing WA Web applies to the same case.
3786    #[test]
3787    fn retry_backoff_doubles_then_clamps() {
3788        assert_eq!(app_state_retry_backoff(0), APP_STATE_RETRY_BACKOFF_MIN);
3789        assert_eq!(app_state_retry_backoff(1), APP_STATE_RETRY_BACKOFF_MIN * 2);
3790        assert_eq!(app_state_retry_backoff(3), APP_STATE_RETRY_BACKOFF_MIN * 8);
3791        assert_eq!(
3792            app_state_retry_backoff(u32::MAX),
3793            APP_STATE_RETRY_BACKOFF_MAX,
3794            "an absurd round must clamp, not overflow"
3795        );
3796    }
3797}
3798
3799#[cfg(test)]
3800mod request_hygiene_tests {
3801    use super::*;
3802    use crate::client::app_state::batched_sync_outcome_tests::sync_against;
3803
3804    /// A `server_sync` notification can repeat a `<collection>` child. Reserving
3805    /// the name once and then tripping over our own reservation on the second
3806    /// pass filed one collection under both `synced` and `skipped`, which made
3807    /// `all_synced()` false and published a failure blaming a writer that never
3808    /// existed.
3809    #[tokio::test]
3810    async fn a_collection_requested_twice_is_reserved_once() {
3811        let (outcome, _) = sync_against(
3812            vec![WAPatchName::Regular, WAPatchName::Regular],
3813            &[("regular", None)],
3814        )
3815        .await;
3816
3817        assert_eq!(outcome.synced, vec![WAPatchName::Regular]);
3818        assert!(
3819            outcome.skipped.is_empty(),
3820            "the only holder was this same call"
3821        );
3822        assert!(outcome.all_synced());
3823    }
3824
3825    /// Nothing reserved a collection we did not ask for, so applying it can
3826    /// interleave with a concurrent writer for that collection — and it would
3827    /// dispatch mutations nobody requested.
3828    #[tokio::test]
3829    async fn an_unrequested_collection_in_the_response_is_dropped() {
3830        let (outcome, _) = sync_against(
3831            vec![WAPatchName::Regular],
3832            &[("regular", None), ("critical_block", None)],
3833        )
3834        .await;
3835
3836        assert_eq!(outcome.synced, vec![WAPatchName::Regular]);
3837        assert!(
3838            !outcome.synced.contains(&WAPatchName::CriticalBlock),
3839            "an unrequested collection must not be applied or reported"
3840        );
3841        assert!(outcome.all_synced());
3842    }
3843}
3844
3845#[cfg(test)]
3846mod deadline_tests {
3847    use super::*;
3848
3849    /// An expired deadline stops the batch before it reserves anything, so no
3850    /// collection is reported synced and nothing reaches the wire.
3851    ///
3852    /// Covers the pre-reservation check only. The second check — after the IQ
3853    /// returns and before the response is applied — needs a deadline that
3854    /// expires mid-round, which takes a responder driving a paused clock; that
3855    /// path is unasserted.
3856    #[tokio::test]
3857    async fn an_expired_deadline_stops_the_batch_before_it_reserves() {
3858        let (client, transport) = crate::test_utils::create_iq_test_client().await;
3859
3860        let sync = {
3861            let client = Arc::clone(&client);
3862            tokio::spawn(async move {
3863                let scope = client.sync_scope(Some(wacore::time::Instant::now()));
3864                client
3865                    .sync_collections_batched(vec![WAPatchName::Regular], scope)
3866                    .await
3867            })
3868        };
3869
3870        let outcome = tokio::time::timeout(Duration::from_secs(5), sync)
3871            .await
3872            .expect("an expired deadline must not block")
3873            .expect("the sync task should not panic")
3874            .expect("a deadline is an outcome, not a transport failure");
3875
3876        assert_eq!(outcome.retryable, vec![WAPatchName::Regular]);
3877        assert!(outcome.synced.is_empty());
3878        assert!(
3879            transport.sent().is_empty(),
3880            "nothing may reach the wire past the deadline"
3881        );
3882    }
3883}
3884
3885#[cfg(test)]
3886mod sync_scope_tests {
3887    use super::*;
3888
3889    /// The predicate every boundary asks. Both answers matter and they are not
3890    /// interchangeable: a retired scope must never write or publish, while an
3891    /// expired one is simply out of time on a connection that is still live.
3892    #[tokio::test]
3893    async fn a_scope_stops_admitting_when_its_connection_or_clock_goes() {
3894        let client = crate::test_utils::create_test_client_with_name("scope-admits").await;
3895
3896        let live = client.sync_scope(None);
3897        assert_eq!(client.admits(live), Ok(()));
3898
3899        let expired = client.sync_scope(Some(wacore::time::Instant::now()));
3900        assert_eq!(client.admits(expired), Err(ScopeLost::Expired));
3901
3902        let generous = client.sync_scope(Some(
3903            wacore::time::Instant::now() + Duration::from_secs(600),
3904        ));
3905        assert_eq!(client.admits(generous), Ok(()));
3906
3907        client
3908            .connection_generation
3909            .store(live.generation() + 1, Ordering::SeqCst);
3910        assert_eq!(client.admits(live), Err(ScopeLost::Retired));
3911        assert_eq!(
3912            client.admits(generous),
3913            Err(ScopeLost::Retired),
3914            "a retired connection outranks having time left"
3915        );
3916    }
3917
3918    /// The bootstrap gate is shared across connections, so a task from a retired
3919    /// one must not touch it in either direction: clearing lets the live
3920    /// connection skip a bootstrap it still needs, arming costs it one it does
3921    /// not. Routing every write through `settle_bootstrap` is what makes that
3922    /// check impossible to forget — it was forgotten twice when it was the
3923    /// caller's job.
3924    #[tokio::test]
3925    async fn a_retired_scope_cannot_move_the_bootstrap_gate() {
3926        let client = crate::test_utils::create_test_client_with_name("scope-gate").await;
3927        let retired = client.sync_scope(None);
3928        client
3929            .connection_generation
3930            .store(retired.generation() + 1, Ordering::SeqCst);
3931
3932        for armed in [true, false] {
3933            // Seeded as the connection that is live at seeding time, so the
3934            // retired scope below is genuinely outranked rather than merely
3935            // equal to the tag it left behind.
3936            client
3937                .needs_initial_full_sync
3938                .settle(client.connection_generation.load(Ordering::SeqCst), armed);
3939            client.settle_bootstrap(retired, !armed);
3940            assert_eq!(
3941                client.needs_initial_full_sync.is_armed(),
3942                armed,
3943                "a retired scope must leave the gate exactly as it found it"
3944            );
3945        }
3946
3947        // The live connection still owns it, both ways.
3948        let live = client.sync_scope(None);
3949        client.settle_bootstrap(live, true);
3950        assert!(client.needs_initial_full_sync.is_armed());
3951        client.settle_bootstrap(live, false);
3952        assert!(!client.needs_initial_full_sync.is_armed());
3953    }
3954
3955    /// An expired scope still owns its connection, so it may settle the gate —
3956    /// running out of time is exactly when the bootstrap needs to stay armed.
3957    #[tokio::test]
3958    async fn an_expired_scope_may_still_arm_the_gate() {
3959        let client = crate::test_utils::create_test_client_with_name("scope-expired-gate").await;
3960        let expired = client.sync_scope(Some(wacore::time::Instant::now()));
3961        client
3962            .needs_initial_full_sync
3963            .settle(expired.generation(), false);
3964
3965        client.settle_bootstrap(expired, true);
3966        assert!(
3967            client.needs_initial_full_sync.is_armed(),
3968            "an expired bootstrap is unfinished, and must say so"
3969        );
3970    }
3971
3972    /// Rebinding is what lets work outlive a planned reconnect, and it reports
3973    /// whether it moved so the caller can drop the authority that does not
3974    /// carry over with it.
3975    #[tokio::test]
3976    async fn rebinding_reports_whether_the_connection_moved() {
3977        let client = crate::test_utils::create_test_client_with_name("scope-rebind").await;
3978        let mut scope = client.sync_scope(None);
3979        let original = scope.generation();
3980
3981        assert!(!scope.rebind(original), "same connection is not a move");
3982        assert_eq!(client.admits(scope), Ok(()));
3983
3984        assert!(scope.rebind(original + 1), "a different connection is");
3985        assert_eq!(scope.generation(), original + 1);
3986    }
3987
3988    /// A scope with no deadline is a background trigger; one with a deadline is
3989    /// the bootstrap. That distinction decides whether the batch waits behind an
3990    /// equivalent sync or skips it, so it has to be readable from the scope
3991    /// alone rather than re-derived at each call site.
3992    #[tokio::test]
3993    async fn only_a_deadline_marks_the_bootstrap() {
3994        let client = crate::test_utils::create_test_client_with_name("scope-kind").await;
3995        assert!(!client.sync_scope(None).is_bootstrap());
3996        assert!(
3997            client
3998                .sync_scope(Some(wacore::time::Instant::now()))
3999                .is_bootstrap()
4000        );
4001    }
4002}
4003
4004#[cfg(test)]
4005mod task_retry_tests {
4006    use super::*;
4007
4008    /// `process_app_state_sync_task` answers `Ok(())` at its own shutdown guard
4009    /// without contacting the server, and `expected_disconnect` makes that guard
4010    /// true for an ordinary reconnect as well as a stop. A retry that attempted
4011    /// there would read the no-op as a completed sync and drop the consumer's
4012    /// request — a `full_sync` snapshot included — without ever asking for it.
4013    #[tokio::test]
4014    async fn a_planned_reconnect_is_not_a_completed_sync() {
4015        let client = crate::test_utils::create_test_client_with_name("task-retry-reconnect").await;
4016
4017        // A live client with a planned reconnect in flight: the run loop is up,
4018        // and only `expected_disconnect` is set. This is the state
4019        // `reconnect_immediately()` leaves behind, and the one the retry loop
4020        // used to walk straight through.
4021        client.is_running.store(true, Ordering::Relaxed);
4022        client.expected_disconnect.store(true, Ordering::Relaxed);
4023        assert!(
4024            client.is_running.load(Ordering::Relaxed),
4025            "the retry loop's own guard still admits this state"
4026        );
4027        assert!(
4028            client.is_shutting_down(),
4029            "but it does make the callee's shutdown guard true"
4030        );
4031        assert!(
4032            client
4033                .process_app_state_sync_task(WAPatchName::Regular, true)
4034                .await
4035                .is_ok(),
4036            "the callee reports Ok without doing anything, which is the trap"
4037        );
4038
4039        client.expected_disconnect.store(false, Ordering::Relaxed);
4040        assert!(
4041            !client.is_shutting_down(),
4042            "and the hold lifts once the reconnect settles"
4043        );
4044    }
4045}
4046
4047#[cfg(test)]
4048mod apply_boundary_tests {
4049    use super::*;
4050    use crate::client::app_state::batched_sync_outcome_tests::sync_against;
4051
4052    /// `process_one_patch_list` persists the version and mutation MACs before it
4053    /// returns, so once a collection is applied its cursor has moved and the
4054    /// server will not send those mutations again. Declining it after that point
4055    /// loses them for good — `setting_pushName` and the NCT salt included — so
4056    /// the admission check has to sit before the apply, and dispatching after it
4057    /// is not optional.
4058    ///
4059    /// Asserted as the property that matters: a collection the batch applied is
4060    /// reported synced, never retryable, because "retry" cannot recover it.
4061    ///
4062    /// This pins the invariant, not the race that violated it. Reproducing that
4063    /// needs the scope to be lost between two collections' applies, and the only
4064    /// hook for it would be inside the loop; a scope retired any earlier is
4065    /// caught by the pre-apply check and nothing is applied at all.
4066    #[tokio::test]
4067    async fn an_applied_collection_is_never_reported_retryable() {
4068        let (outcome, _) = sync_against(
4069            vec![WAPatchName::CriticalBlock, WAPatchName::CriticalUnblockLow],
4070            &[
4071                ("critical_block", None),
4072                ("critical_unblock_low", Some("500")),
4073            ],
4074        )
4075        .await;
4076
4077        assert_eq!(
4078            outcome.synced,
4079            vec![WAPatchName::CriticalBlock],
4080            "an applied collection is synced"
4081        );
4082        assert!(
4083            !outcome.retryable.contains(&WAPatchName::CriticalBlock),
4084            "and must never also be queued for a retry that cannot re-fetch it"
4085        );
4086        assert_eq!(outcome.retryable, vec![WAPatchName::CriticalUnblockLow]);
4087    }
4088}
4089
4090#[cfg(test)]
4091mod bootstrap_gate_tests {
4092    use super::*;
4093
4094    /// The tag only moves forward, so a write on behalf of an older connection
4095    /// can never overwrite one made for a newer. This is the half that the
4096    /// admission check cannot provide: by the time a stale writer is inside
4097    /// `settle`, the replacement may already have had its say.
4098    #[test]
4099    fn an_older_connection_cannot_overwrite_a_newer_one() {
4100        let gate = BootstrapGate::new(false);
4101
4102        assert!(gate.settle(7, true), "the newest writer wins");
4103        assert!(gate.is_armed());
4104
4105        assert!(
4106            !gate.settle(6, false),
4107            "an older connection is refused outright"
4108        );
4109        assert!(gate.is_armed(), "and leaves the newer answer standing");
4110
4111        assert!(
4112            gate.settle(7, false),
4113            "the same connection may revise itself"
4114        );
4115        assert!(!gate.is_armed());
4116
4117        assert!(gate.settle(8, true), "and a newer one always may");
4118        assert!(gate.is_armed());
4119    }
4120
4121    /// A fresh pairing owes a bootstrap whatever the live connection concluded,
4122    /// and only the connection that comes *after* it may say otherwise.
4123    ///
4124    /// Both halves have been wrong here. Arming above every generation left the
4125    /// gate unclearable and re-ran the 180s critical bootstrap on every connect
4126    /// forever; arming at zero let a scope already in flight on the pairing
4127    /// connection clear it before the forced reconnect, so the connection that
4128    /// was supposed to run the sync found nothing owed.
4129    #[test]
4130    fn pairing_arms_over_live_connections_but_not_the_next_one() {
4131        let gate = BootstrapGate::new(false);
4132        assert!(gate.settle(9, false));
4133        assert!(!gate.is_armed());
4134
4135        // `pair-success` arrives while connection 9 is live.
4136        gate.arm_for_pairing(9);
4137        assert!(gate.is_armed());
4138
4139        assert!(
4140            !gate.settle(9, false),
4141            "a bootstrap already in flight on the pairing connection must not \
4142             answer for the sync pairing just asked for"
4143        );
4144        assert!(gate.is_armed(), "so the arm survives it");
4145
4146        assert!(
4147            gate.settle(10, false),
4148            "and the connection the forced 515 brings up can clear it"
4149        );
4150        assert!(
4151            !gate.is_armed(),
4152            "a pairing that outranked every connection would never clear, and the \
4153             client would re-run the critical bootstrap forever"
4154        );
4155    }
4156
4157    /// The arm is a floor, never an assignment.
4158    ///
4159    /// `current_generation` is a sample, and the tag is shared with `settle`. An
4160    /// unconditional store could lower a tag a newer connection had already set,
4161    /// which is the one way arming could make the gate *easier* to clear than it
4162    /// was. It also broke the rule `settle_bootstrap` leans on — that the tag
4163    /// only ever moves forward — for one of the two writers.
4164    #[test]
4165    fn pairing_never_lowers_the_gate() {
4166        let gate = BootstrapGate::new(false);
4167        assert!(gate.settle(20, false));
4168
4169        // A `pair-success` carrying a sample from well before that.
4170        gate.arm_for_pairing(3);
4171
4172        assert!(gate.is_armed(), "pairing always owes a bootstrap");
4173        assert!(
4174            !gate.settle(20, false),
4175            "and connection 20 still cannot answer for it"
4176        );
4177        assert!(gate.settle(21, false), "only something newer can");
4178        assert!(!gate.is_armed());
4179    }
4180
4181    /// The flag survives the round trip through the tag, which is the only part
4182    /// readers see.
4183    #[test]
4184    fn the_armed_bit_round_trips() {
4185        let gate = BootstrapGate::new(true);
4186        assert!(gate.is_armed());
4187        let gate = BootstrapGate::new(false);
4188        assert!(!gate.is_armed());
4189    }
4190}
4191
4192#[cfg(test)]
4193mod lifecycle_signal_tests {
4194    use super::*;
4195
4196    /// The predicate app-state retries end on. It has to mean "finished", not
4197    /// "not currently connected": a planned reconnect, or a direct-connect
4198    /// client that never starts the supervision loop, must not look terminal or
4199    /// the retries throw away work nothing else will redo.
4200    #[tokio::test]
4201    async fn only_a_finished_client_looks_terminal() {
4202        let client = crate::test_utils::create_test_client_with_name("lifecycle-terminal").await;
4203
4204        // A direct-connect client never runs the supervision loop. Reading
4205        // `is_running` here would call a perfectly healthy client stopped.
4206        assert!(
4207            !client.is_running.load(Ordering::Relaxed),
4208            "the fixture models a client that never called run()"
4209        );
4210        assert!(!client.is_terminal(), "which is not the same as finished");
4211
4212        // A planned reconnect is not the end either.
4213        client.expected_disconnect.store(true, Ordering::Relaxed);
4214        assert!(
4215            client.is_shutting_down(),
4216            "the old predicate cannot tell this apart"
4217        );
4218        assert!(!client.is_terminal(), "the new one can");
4219        client.expected_disconnect.store(false, Ordering::Relaxed);
4220
4221        // Turning auto-reconnect off is a preference an application may express
4222        // on a healthy connection — "do not come back after this one ends" — and
4223        // the run loop does not act on it until the socket exits. On its own it
4224        // says nothing about the session being over, so the supervision loop has
4225        // to be up for the scenario to be the one described.
4226        client.is_running.store(true, Ordering::Relaxed);
4227        client.enable_auto_reconnect.store(false, Ordering::Relaxed);
4228        assert!(
4229            !client.is_terminal(),
4230            "a reconnect preference is not a verdict on the current session"
4231        );
4232
4233        // The stream errors that really do end one — conflict, 516, an
4234        // unrecoverable connect failure — set both, and the pair is what
4235        // separates them from the preference above.
4236        client.expected_disconnect.store(true, Ordering::Relaxed);
4237        assert!(client.is_terminal());
4238        client.expected_disconnect.store(false, Ordering::Relaxed);
4239
4240        // And the run loop's own exit: it stops by clearing `is_running` alone,
4241        // without firing the notifier or setting `expected_disconnect`, so that
4242        // pairing has to count too or retries wait for a connection that is
4243        // never coming. `cleanup_connection_state` has already run by then, which
4244        // is why the socket is down here.
4245        client.is_running.store(false, Ordering::Relaxed);
4246        client.set_connected_for_test(false);
4247        assert!(
4248            client.is_terminal(),
4249            "the supervision loop ending with auto-reconnect off is terminal"
4250        );
4251
4252        // But `is_running` is false for a direct-connect client too, which never
4253        // had a loop to end. One of those with a live socket is not finished, and
4254        // reading it as finished is what the first version of this predicate did.
4255        client.set_connected_for_test(true);
4256        assert!(
4257            !client.is_terminal(),
4258            "a live direct-connect client never started the loop that would have ended"
4259        );
4260        client.set_connected_for_test(false);
4261
4262        client.enable_auto_reconnect.store(true, Ordering::Relaxed);
4263        client.is_running.store(true, Ordering::Relaxed);
4264        assert!(!client.is_terminal());
4265
4266        // And an explicit shutdown, which reconnects deliberately leave alone.
4267        client.signal_shutdown_sync();
4268        assert!(client.is_terminal());
4269    }
4270}
4271
4272#[cfg(test)]
4273mod connection_guard_tests {
4274    use super::*;
4275
4276    /// The guards ask whether the client is finished and whether there is a
4277    /// socket, rather than `is_shutting_down()`.
4278    ///
4279    /// The point is the reconnect: `is_shutting_down()` is true for a planned
4280    /// one, so a task reading it stops for a connection that is coming back.
4281    ///
4282    /// Not a claim about direct-connect clients. `connect()` without `run()`
4283    /// leaves `is_running` false, and `send_and_wait_iq` rejects every IQ in
4284    /// that state (`request.rs`), so no such client can reach the server at all
4285    /// — for app state or anything else. An earlier version of this test
4286    /// asserted the sync errored and called that support; it errored one layer
4287    /// down, for that reason, and proved nothing.
4288    #[tokio::test]
4289    async fn a_planned_reconnect_does_not_look_like_a_stop() {
4290        let client = crate::test_utils::create_test_client_with_name("conn-guard").await;
4291        client.is_running.store(true, Ordering::Relaxed);
4292        client.set_connected_for_test(true);
4293
4294        assert!(!client.is_terminal() && client.is_connected());
4295
4296        // The state `reconnect_immediately()` leaves behind.
4297        client.expected_disconnect.store(true, Ordering::Relaxed);
4298        assert!(
4299            client.is_shutting_down(),
4300            "which the old guard could not tell from a stop"
4301        );
4302        assert!(
4303            !client.is_terminal(),
4304            "so work that outlives a connection stays alive"
4305        );
4306    }
4307}
4308
4309#[cfg(test)]
4310mod await_connection_tests {
4311    use super::*;
4312
4313    /// A notification that does not leave a live connection must not end the
4314    /// wait. The socket can be announced and gone again before the check reads
4315    /// it, and treating that as an answer dropped the retry while the client was
4316    /// still perfectly able to reconnect.
4317    #[tokio::test]
4318    async fn a_stale_notification_does_not_end_the_wait() {
4319        let client = crate::test_utils::create_test_client_with_name("await-stale").await;
4320        client.is_running.store(true, Ordering::Relaxed);
4321
4322        let waiter = {
4323            let client = Arc::clone(&client);
4324            tokio::spawn(async move { client.await_connection().await })
4325        };
4326
4327        crate::test_utils::poll_until("the waiter to park on the notifier", || {
4328            client.socket_ready_notifier.total_listeners() >= 1
4329        })
4330        .await;
4331
4332        // Announced, but nothing is connected: the wait has to carry on.
4333        client.socket_ready_notifier.notify(usize::MAX);
4334        for _ in 0..8 {
4335            tokio::task::yield_now().await;
4336        }
4337        assert!(
4338            !waiter.is_finished(),
4339            "an event without a connection is not an answer"
4340        );
4341
4342        // Nor is a socket on its own. `connect()` announces one before login, and
4343        // an IQ sent in that gap is answered by nobody and retired by the
4344        // `<success>` that follows.
4345        client.set_connected_for_test(true);
4346        client.socket_ready_notifier.notify(usize::MAX);
4347        for _ in 0..8 {
4348            tokio::task::yield_now().await;
4349        }
4350        assert!(
4351            !waiter.is_finished(),
4352            "a socket without an authenticated session is not one either"
4353        );
4354
4355        // Both stores that `handle_success` makes, in that order: the session,
4356        // then the generation it is authenticated under. Only the pair is an
4357        // answer — the marker alone would leave the wait on a socket that has
4358        // not authenticated, which is the state above.
4359        client.is_logged_in.store(true, Ordering::Relaxed);
4360        client.authenticated_generation.store(
4361            client.connection_generation.load(Ordering::SeqCst),
4362            Ordering::SeqCst,
4363        );
4364        client.notify_session_state();
4365        assert!(
4366            tokio::time::timeout(Duration::from_secs(5), waiter)
4367                .await
4368                .expect("a usable connection must end the wait")
4369                .expect("the waiter should not panic"),
4370            "and it reports that one arrived"
4371        );
4372    }
4373
4374    /// The wait has no duration bound, so the terminal state is the only thing
4375    /// that ends it when no connection is coming. Every duration tried here was
4376    /// wrong in one direction or the other.
4377    ///
4378    /// Shutdown alone has to end it. An earlier version of this test nudged
4379    /// `socket_ready_notifier` afterwards and passed on that nudge, hiding a wait
4380    /// that in production had nothing left to wake it — no socket is ever
4381    /// announced again after a shutdown, and the parked task holds the
4382    /// `Arc<Client>` whose drop would otherwise have been the way out.
4383    #[tokio::test]
4384    async fn a_finished_client_ends_the_wait() {
4385        let client = crate::test_utils::create_test_client_with_name("await-terminal").await;
4386        client.is_running.store(true, Ordering::Relaxed);
4387
4388        let waiter = {
4389            let client = Arc::clone(&client);
4390            tokio::spawn(async move { client.await_connection().await })
4391        };
4392
4393        crate::test_utils::poll_until("the waiter to park on the notifier", || {
4394            client.socket_ready_notifier.total_listeners() >= 1
4395        })
4396        .await;
4397
4398        client.signal_shutdown_sync();
4399        assert!(
4400            !tokio::time::timeout(Duration::from_secs(5), waiter)
4401                .await
4402                .expect("a finished client must end the wait, with nothing else nudging it")
4403                .expect("the waiter should not panic"),
4404            "and it reports that none arrived"
4405        );
4406    }
4407
4408    /// The run loop's own exit is the terminal transition with no signal of its
4409    /// own: it fires no notifier and announces no socket, so a wait parked
4410    /// through it is parked for good unless that branch says so itself.
4411    #[tokio::test]
4412    async fn the_run_loop_giving_up_ends_the_wait() {
4413        let client = crate::test_utils::create_test_client_with_name("await-runloop").await;
4414        client.is_running.store(true, Ordering::Relaxed);
4415
4416        let waiter = {
4417            let client = Arc::clone(&client);
4418            tokio::spawn(async move { client.await_connection().await })
4419        };
4420
4421        crate::test_utils::poll_until("the waiter to park on the notifier", || {
4422            client.socket_ready_notifier.total_listeners() >= 1
4423        })
4424        .await;
4425
4426        // The branch itself, not a re-enactment of it: `run()` reads this flag
4427        // and calls exactly this, and a version of the transition that forgets
4428        // to announce itself fails here.
4429        client.enable_auto_reconnect.store(false, Ordering::Relaxed);
4430        client.stop_supervision_loop();
4431
4432        assert!(
4433            !tokio::time::timeout(Duration::from_secs(5), waiter)
4434                .await
4435                .expect("the supervision loop ending must end the wait")
4436                .expect("the waiter should not panic"),
4437            "and it reports that none arrived"
4438        );
4439    }
4440
4441    /// A direct-connect client is not finished — its connection is fine and its
4442    /// application may still use it — but no `<success>` will ever arrive without
4443    /// a reader, so waiting for one is waiting forever.
4444    #[tokio::test]
4445    async fn a_client_without_a_reader_is_not_worth_waiting_for() {
4446        let client = crate::test_utils::create_test_client_with_name("await-direct").await;
4447        client.set_connected_for_test(true);
4448
4449        assert!(
4450            !client.is_terminal(),
4451            "a live direct-connect client is fine"
4452        );
4453        assert!(
4454            !tokio::time::timeout(Duration::from_secs(5), client.await_connection())
4455                .await
4456                .expect("the wait must not park on a connection that cannot answer"),
4457            "it just cannot carry the work"
4458        );
4459    }
4460}
4461
4462#[cfg(test)]
4463mod sync_outcome_tests {
4464    use super::*;
4465
4466    /// Sets up a client that can reach the server, so a test can then take one
4467    /// signal away and see what the guard makes of it.
4468    async fn reachable_client(name: &str) -> Arc<Client> {
4469        let client = crate::test_utils::create_test_client_with_name(name).await;
4470        client.is_running.store(true, Ordering::Relaxed);
4471        client.set_connected_for_test(true);
4472        client.is_logged_in.store(true, Ordering::Relaxed);
4473        client.authenticated_generation.store(
4474            client.connection_generation.load(Ordering::SeqCst),
4475            Ordering::SeqCst,
4476        );
4477        assert!(client.can_reach_server(), "the fixture itself is usable");
4478        client
4479    }
4480
4481    /// The case every lifecycle-flag proxy missed, because it is the one state
4482    /// none of them modelled: rate-limited, still connected, still supervised,
4483    /// no `expected_disconnect` and no generation change — and no session.
4484    ///
4485    /// `Ok(())` here read as a completed sync, and the caller returned. The
4486    /// trigger was consumed by then and nothing asks a second time.
4487    #[tokio::test]
4488    async fn a_rate_limited_session_defers_rather_than_completes() {
4489        let client = reachable_client("outcome-429").await;
4490
4491        // Exactly what `handle_stream_error` does for 429 and 503.
4492        client.is_logged_in.store(false, Ordering::Relaxed);
4493
4494        assert!(
4495            !client.is_terminal(),
4496            "a rate limit is not the client being finished"
4497        );
4498        assert!(
4499            !client.is_shutting_down(),
4500            "nor is it anything the old proxy could see"
4501        );
4502        assert_eq!(
4503            client
4504                .process_app_state_sync_task(WAPatchName::Regular, true)
4505                .await
4506                .expect("skipping is not an error"),
4507            SyncOutcome::Deferred,
4508            "nothing was asked, so nothing was completed"
4509        );
4510    }
4511
4512    /// A planned reconnect reaches the same guard by a different route, and has
4513    /// to answer the same way.
4514    #[tokio::test]
4515    async fn a_reconnect_defers_rather_than_completes() {
4516        let client = reachable_client("outcome-reconnect").await;
4517        client.set_connected_for_test(false);
4518
4519        assert_eq!(
4520            client
4521                .process_app_state_sync_task(WAPatchName::Regular, false)
4522                .await
4523                .expect("skipping is not an error"),
4524            SyncOutcome::Deferred
4525        );
4526    }
4527
4528    /// Terminal and reachable are not mutually exclusive: the stream-error paths
4529    /// set the terminal flags before they clear the session and close the
4530    /// socket. Asking about reachability first hands out that window.
4531    #[tokio::test]
4532    async fn a_terminal_client_is_not_a_usable_one() {
4533        let client = reachable_client("verdict-order").await;
4534
4535        // A conflict or 516: both flags set together, and the socket not yet
4536        // torn down. This used to be a window where `is_terminal()` and
4537        // `can_reach_server()` were both true, which is why the verdict checks
4538        // terminal first.
4539        client.enable_auto_reconnect.store(false, Ordering::Relaxed);
4540        client.expected_disconnect.store(true, Ordering::Relaxed);
4541
4542        assert!(client.is_terminal());
4543        assert_eq!(client.connection_wait_verdict(), Some(false));
4544
4545        // The window is now closed at the source rather than ordered around:
4546        // `can_reach_server()` rejects a socket marked for retirement, and every
4547        // route into `is_terminal()` marks one — `expected_disconnect` here,
4548        // `is_running` cleared by shutdown, a dead socket for the run loop's
4549        // exit. The ordering stays as belt and braces; this is what makes it
4550        // moot, and what fails if the retirement check is dropped.
4551        assert!(
4552            !client.can_reach_server(),
4553            "a finished client is never a reachable one"
4554        );
4555
4556        client.expected_disconnect.store(false, Ordering::Relaxed);
4557        client.signal_shutdown_sync();
4558        assert!(client.is_terminal() && !client.can_reach_server());
4559    }
4560
4561    /// `<success>` sets `is_logged_in` one step before it increments the
4562    /// generation, because that store is the duplicate guard. A caller that
4563    /// binds a scope in between binds a generation the next instruction retires,
4564    /// and every attempt it then makes is rejected.
4565    #[tokio::test]
4566    async fn the_gap_inside_success_is_not_an_authenticated_connection() {
4567        let client = crate::test_utils::create_test_client_with_name("auth-window").await;
4568        client.is_running.store(true, Ordering::Relaxed);
4569
4570        client.set_connected_for_test(true);
4571
4572        // First half of the window: `handle_success` has set `is_logged_in` and
4573        // has not yet incremented the generation. The marker here is whatever
4574        // the constructor left, unretouched — and a marker that starts at a real
4575        // generation equals the one a fresh client is on, so equality alone
4576        // admits the window on the very first connection.
4577        client.is_logged_in.store(true, Ordering::Relaxed);
4578        assert!(
4579            client.is_logged_in() && client.is_connected(),
4580            "which is why the flags alone said yes"
4581        );
4582        assert!(
4583            !client.can_reach_server(),
4584            "this connection has not authenticated anything yet"
4585        );
4586
4587        // Second half: generation incremented, marker not yet stored.
4588        let current = client.connection_generation.fetch_add(1, Ordering::SeqCst) + 1;
4589        assert!(!client.can_reach_server());
4590        assert_eq!(
4591            client.connection_wait_verdict(),
4592            None,
4593            "the wait carries on"
4594        );
4595
4596        // And once `<success>` finishes publishing, it is a real connection.
4597        client
4598            .authenticated_generation
4599            .store(current, Ordering::SeqCst);
4600        assert_eq!(client.connection_wait_verdict(), Some(true));
4601    }
4602}
4603
4604#[cfg(test)]
4605mod sync_owed_tests {
4606    use super::*;
4607
4608    /// Only a completed sync discharges the request.
4609    ///
4610    /// The first version of this seam listed the ways to fail and requeued for
4611    /// those — which put the deferral next to an `Err` branch that still only
4612    /// logged, so a connection lost while the collection IQ was in flight was
4613    /// reported by `send_iq` as an error and dropped there. Every list of
4614    /// failure modes written so far has been one short; this asks the other
4615    /// question, which has one answer.
4616    #[test]
4617    fn everything_that_is_not_a_completed_sync_is_still_owed() {
4618        assert!(!sync_still_owed(&Ok(SyncOutcome::Completed)));
4619        assert!(sync_still_owed(&Ok(SyncOutcome::Deferred)));
4620        assert!(sync_still_owed(&Err(anyhow::anyhow!(
4621            "the socket died under the collection IQ"
4622        ))));
4623    }
4624}
4625
4626#[cfg(test)]
4627mod terminal_wake_tests {
4628    use super::*;
4629
4630    /// A fatal stream error — conflict, 516, 401, 409 — makes the client
4631    /// terminal by setting two flags and then firing only the per-connection
4632    /// shutdown. Nothing else on that path announces anything.
4633    ///
4634    /// The wait must end there, not when the run loop eventually unwinds far
4635    /// enough to notice. The invariant on `is_terminal` is that every transition
4636    /// into it announces itself; "some other loop gets there first" is not that,
4637    /// and if that loop is what is wedged, it never gets there at all.
4638    #[tokio::test]
4639    async fn a_fatal_stream_error_ends_the_wait() {
4640        let client = crate::test_utils::create_test_client_with_name("await-fatal").await;
4641        client.is_running.store(true, Ordering::Relaxed);
4642
4643        let waiter = {
4644            let client = Arc::clone(&client);
4645            tokio::spawn(async move { client.await_connection().await })
4646        };
4647
4648        crate::test_utils::poll_until("the waiter to park on the notifier", || {
4649            client.socket_ready_notifier.total_listeners() >= 1
4650        })
4651        .await;
4652
4653        // Exactly what `handle_stream_error` does, in its order.
4654        client.expected_disconnect.store(true, Ordering::Relaxed);
4655        client.enable_auto_reconnect.store(false, Ordering::Relaxed);
4656        client.notify_connection_shutdown();
4657
4658        assert!(
4659            !tokio::time::timeout(Duration::from_secs(5), waiter)
4660                .await
4661                .expect("a fatal stream error must end the wait where it happens")
4662                .expect("the waiter should not panic"),
4663            "and it reports that no connection arrived"
4664        );
4665    }
4666}
4667
4668#[cfg(test)]
4669mod reconnect_wake_tests {
4670    use super::*;
4671
4672    /// A teardown that a reconnect follows wakes the wait and must not end it.
4673    ///
4674    /// The wake carries no verdict: it only says the state is worth re-reading.
4675    /// A parked waiter is parked *because* `can_reach_server()` was false, and
4676    /// nothing about a teardown makes it true — so the re-read parks it again.
4677    ///
4678    /// The reverse case cannot arise either. For the wake to release a waiter
4679    /// onto a dying socket, the state would have to go unusable → usable while
4680    /// it was parked; every transition that does that announces itself, and the
4681    /// waiter would have left on that announcement, when the connection really
4682    /// was usable.
4683    #[tokio::test]
4684    async fn a_planned_reconnect_teardown_does_not_end_the_wait() {
4685        let client = crate::test_utils::create_test_client_with_name("await-replan").await;
4686        client.is_running.store(true, Ordering::Relaxed);
4687
4688        let waiter = {
4689            let client = Arc::clone(&client);
4690            tokio::spawn(async move { client.await_connection().await })
4691        };
4692
4693        crate::test_utils::poll_until("the waiter to park on the notifier", || {
4694            client.session_state_notifier.total_listeners() >= 1
4695        })
4696        .await;
4697
4698        // What `reconnect_immediately()` does: a planned teardown, auto-reconnect
4699        // still on, so the client is not finished and a socket is coming back.
4700        client.expected_disconnect.store(true, Ordering::Relaxed);
4701        client.notify_connection_shutdown();
4702        for _ in 0..8 {
4703            tokio::task::yield_now().await;
4704        }
4705        assert!(
4706            !waiter.is_finished(),
4707            "a teardown is not a connection, however loudly it is announced"
4708        );
4709
4710        // And the replacement still ends it.
4711        client.expected_disconnect.store(false, Ordering::Relaxed);
4712        client.set_connected_for_test(true);
4713        client.is_logged_in.store(true, Ordering::Relaxed);
4714        client.authenticated_generation.store(
4715            client.connection_generation.load(Ordering::SeqCst),
4716            Ordering::SeqCst,
4717        );
4718        client.notify_session_state();
4719        assert!(
4720            tokio::time::timeout(Duration::from_secs(5), waiter)
4721                .await
4722                .expect("the replacement connection must end the wait")
4723                .expect("the waiter should not panic")
4724        );
4725    }
4726}
4727
4728#[cfg(test)]
4729mod retiring_socket_tests {
4730    use super::*;
4731
4732    /// A socket already marked for retirement is not one work can be sent on.
4733    ///
4734    /// `reconnect_immediately()` sets `expected_disconnect` before its bounded
4735    /// flushes and closes the transport only afterwards. Every other signal
4736    /// still reads healthy through that window — socket up, session
4737    /// authenticated, generation final — so a wait released there hands its IQ
4738    /// to a connection the run loop has already decided to retire. The server
4739    /// answers, `handle_success` on the replacement retires the scope, the
4740    /// answer is dropped and the attempt is charged anyway.
4741    #[tokio::test]
4742    async fn a_socket_marked_for_reconnect_cannot_carry_work() {
4743        let client = crate::test_utils::create_test_client_with_name("retiring").await;
4744        client.is_running.store(true, Ordering::Relaxed);
4745        client.set_connected_for_test(true);
4746        client.is_logged_in.store(true, Ordering::Relaxed);
4747        client.authenticated_generation.store(
4748            client.connection_generation.load(Ordering::SeqCst),
4749            Ordering::SeqCst,
4750        );
4751        assert!(client.can_reach_server(), "healthy to begin with");
4752
4753        // The first thing `reconnect_immediately()` does, before any teardown.
4754        client.expected_disconnect.store(true, Ordering::Relaxed);
4755
4756        assert!(
4757            client.is_connected() && client.is_logged_in(),
4758            "and every other signal still says the socket is fine"
4759        );
4760        assert!(
4761            !client.can_reach_server(),
4762            "but it is going away, so nothing sent on it comes back"
4763        );
4764        assert!(
4765            !client.is_terminal(),
4766            "which is not the same as the client being finished"
4767        );
4768        assert_eq!(
4769            client.connection_wait_verdict(),
4770            None,
4771            "so the wait carries on to the replacement"
4772        );
4773    }
4774}
4775
4776#[cfg(test)]
4777mod batched_attempt_tests {
4778    use super::*;
4779
4780    /// Only a round that sent a collection IQ may spend an attempt.
4781    ///
4782    /// The first version of this asked the outcome buckets — "anything in
4783    /// `synced`, `fatal` or `retryable` means the wire was reached" — and that
4784    /// is false. `retryable` also collects the collections a scope loss or a
4785    /// `ReservationSkip::WaitTimedOut` dropped *before* the send, and the
4786    /// timeout is exactly the case this predicate exists for: a long patch send
4787    /// holding the collection. The bucket test called that a real attempt and
4788    /// burned the budget anyway.
4789    ///
4790    /// So the flag is recorded at the send and nowhere else. Buckets describe
4791    /// what happened to each collection; only the send knows whether anything
4792    /// was asked.
4793    #[test]
4794    fn only_a_sent_iq_counts_as_reaching_the_server() {
4795        // What a batch whose every reservation timed out looks like. Under the
4796        // bucket test this said true.
4797        let timed_out = BatchedSyncOutcome {
4798            retryable: vec![WAPatchName::Regular, WAPatchName::RegularHigh],
4799            ..Default::default()
4800        };
4801        assert!(
4802            !timed_out.reached_server(),
4803            "a reservation timeout never reached the wire, whatever bucket it lands in"
4804        );
4805
4806        // Nor does a scope lost before reserving, which lands in the same one.
4807        let scope_lost = BatchedSyncOutcome {
4808            retryable: vec![WAPatchName::Regular],
4809            ..Default::default()
4810        };
4811        assert!(!scope_lost.reached_server());
4812
4813        // And an equivalent sync holding it, which lands in `skipped`.
4814        let held = BatchedSyncOutcome {
4815            skipped: vec![WAPatchName::Regular],
4816            ..Default::default()
4817        };
4818        assert!(!held.reached_server());
4819        assert!(!BatchedSyncOutcome::default().reached_server());
4820
4821        // The send is the only thing that sets it, and it survives whatever the
4822        // response turns out to be — including an error, which spent the
4823        // attempt just as much as an answer did.
4824        let mut sent = BatchedSyncOutcome {
4825            retryable: vec![WAPatchName::Regular],
4826            ..Default::default()
4827        };
4828        sent.note_reached_server();
4829        assert!(sent.reached_server());
4830    }
4831}