offline_protocol/protocol/session.rs
1//! Session confirmation, welcome lifecycle, and pending session reconciliation.
2
3use super::{
4 classify_transport_send_error, internal_prefixes, lock_shared_state, send_failure_token,
5 OfflineProtocol, PresenceRescueThrottle, PruneAllowance, RestorableRecord, SessionState,
6 WelcomeDeliveryState, WelcomeLifecycleRecord, CONFIRMATION_PROBE_INTERVAL_SECS,
7 CONFIRMATION_RETRY_INTERVAL_SECS, MAX_REKEY_TRACKED_PEERS, RECONCILIATION_THROTTLE_MS,
8 REKEY_INTERVAL_SECS, SEND_FAIL_REASON_CONFIRM_TIMEOUT, WELCOME_INTERNET_CONFIRM_TIMEOUT_SECS,
9 WELCOME_LIFECYCLE_TTL_SECS, WELCOME_MESH_CONFIRM_TIMEOUT_SECS, WELCOME_NO_CARRIER_RETRY_SECS,
10 WELCOME_PRESENCE_RESCUE_BASE_SECS, WELCOME_PRESENCE_RESCUE_MAX_SECS, WELCOME_RETRY_BATCH_SIZE,
11 WELCOME_RETRY_JITTER_RATIO, WELCOME_UNREACHABLE_RETRY_CAP_SECS, WELCOME_WATCHLIST_MAX_AGE_SECS,
12};
13use crate::events::SecurityWarningCode;
14use crate::mls_observability::MlsOperationContext;
15use crate::protocol::reachability::{Claim, FactSource};
16use crate::{Error, EstablishmentState, Event, Result, SessionStateError};
17use chrono::{Duration as ChronoDuration, Utc};
18use offline_protocol_core::{Message, MessagePriority};
19use offline_protocol_mls::{MlsManager, WelcomeMessage};
20use offline_protocol_transport::TransportType;
21use std::collections::HashSet;
22use std::hash::{Hash, Hasher};
23use std::sync::{Arc, RwLock};
24use std::time::{Duration as StdDuration, Instant};
25use tracing::{debug, info, warn};
26
27impl OfflineProtocol {
28 // ========================================================================
29 // SESSION STATE
30 // ========================================================================
31
32 /// Returns true when persisted state marks this peer session as confirmed.
33 ///
34 /// Uses the in-memory `confirmed_sessions` cache as a fast path to avoid
35 /// hitting persistent storage on every message send. The cache is populated
36 /// when sessions are confirmed and cleared when sessions are invalidated.
37 pub(super) fn is_session_confirmed(&mut self, peer_id: &str) -> Result<bool> {
38 // Fast path: check in-memory cache only — no I/O. The cache is kept in
39 // sync by confirm_session_state() and all invalidation paths (blocking,
40 // stale cleanup, etc.). We deliberately skip the has_mls_session() guard
41 // here because it calls load_group() which hits MlsStorage (10-100ms on
42 // mobile Keychain/Keystore). If the session was wiped externally, the
43 // encrypt call will fail and we handle it there.
44 if self.confirmed_sessions.contains(peer_id) {
45 return Ok(true);
46 }
47
48 let persisted = self
49 .load_session_state_entry(peer_id)?
50 .unwrap_or(SessionState::Pending);
51 if matches!(persisted, SessionState::Confirmed) {
52 if !self.has_mls_session(peer_id)? {
53 warn!(
54 peer_id = %peer_id,
55 "Persisted confirmed state has no matching MLS session; clearing stale state"
56 );
57 self.confirmed_sessions.remove(peer_id);
58 self.clear_confirmation_recovery_tracking(peer_id);
59 self.welcome_lifecycles.remove(peer_id);
60 // The MLS session is gone, so any both-create owner gate for this
61 // peer is stale; clear it (memory + storage) or it would block
62 // `welcome_received` confirmation on the next re-pairing.
63 self.clear_both_create_awaiting_decrypt(peer_id);
64 if let Err(err) = self.clear_session_state_entry(peer_id) {
65 warn!(
66 peer_id = %peer_id,
67 error = %err,
68 "Failed to clear stale persisted session state"
69 );
70 }
71 if let Err(err) = self.clear_welcome_lifecycle_entry(peer_id) {
72 warn!(
73 peer_id = %peer_id,
74 error = %err,
75 "Failed to clear stale persisted welcome lifecycle"
76 );
77 }
78 return Ok(false);
79 }
80 self.confirmed_sessions.insert(peer_id.to_string());
81 return Ok(true);
82 }
83 Ok(false)
84 }
85
86 pub(super) fn has_mls_session(&self, peer_id: &str) -> Result<bool> {
87 let Some(mls) = self.mls_manager.clone() else {
88 return Ok(false);
89 };
90
91 let manager = mls
92 .read()
93 .map_err(|_| Error::Other("MLS lock poisoned".to_string()))?;
94 manager.has_session(peer_id).map_err(Error::Mls)
95 }
96
97 /// Returns the current establishment state for a peer (for API and error reporting).
98 pub(super) fn establishment_state(&self, peer_id: &str) -> Result<EstablishmentState> {
99 if let Some(mls) = &self.mls_manager {
100 let has_session = {
101 let manager = mls
102 .read()
103 .map_err(|_| Error::Other("MLS lock poisoned".to_string()))?;
104 manager.has_session(peer_id).map_err(Error::Mls)?
105 };
106 if has_session {
107 let state = self
108 .load_session_state_entry(peer_id)?
109 .unwrap_or(SessionState::Pending);
110 return Ok(if matches!(state, SessionState::Confirmed) {
111 EstablishmentState::SessionConfirmed
112 } else {
113 EstablishmentState::SessionPending
114 });
115 }
116 }
117
118 let now_ms = Utc::now().timestamp_millis() as u64;
119 if let Some(pkg) = self.pending_key_packages.get(peer_id) {
120 if now_ms < pkg.local_expires_at_ms {
121 return Ok(EstablishmentState::HaveKeyPackage);
122 }
123 }
124 if self.load_peer_key_package_from_storage(peer_id).is_some() {
125 return Ok(EstablishmentState::HaveKeyPackage);
126 }
127 Ok(EstablishmentState::NoKeyPackage)
128 }
129
130 /// Monotonic state transition helper: Pending -> Confirmed only.
131 pub(super) fn confirm_session_state(
132 &mut self,
133 peer_id: &str,
134 source_event: &str,
135 ) -> Result<bool> {
136 if !self.can_confirm_from_source(peer_id, source_event) {
137 warn!(
138 event = "session_confirmation_blocked",
139 session_or_group_id = %peer_id,
140 source_event = %source_event,
141 "session_confirmation_blocked"
142 );
143 return Ok(false);
144 }
145
146 let previous = self.ensure_session_state_entry(peer_id, source_event)?;
147
148 if matches!(previous, SessionState::Confirmed) {
149 self.confirmed_sessions.insert(peer_id.to_string());
150 self.clear_confirmation_recovery_tracking(peer_id);
151 self.clear_both_create_awaiting_decrypt(peer_id);
152 info!(
153 event = "session_state_transition",
154 session_or_group_id = %peer_id,
155 previous_state = "Confirmed",
156 new_state = "Confirmed",
157 source_event = %source_event,
158 "session_state_transition"
159 );
160 return Ok(false);
161 }
162
163 // Persist first, then publish in-memory view.
164 if let Err(err) = self.persist_session_state(peer_id, SessionState::Confirmed, source_event)
165 {
166 self.schedule_confirmation_retry(peer_id, source_event);
167 return Err(err);
168 }
169
170 self.confirmed_sessions.insert(peer_id.to_string());
171 self.clear_confirmation_recovery_tracking(peer_id);
172 // Group-aware proof has arrived (decrypt), so we no longer need to gate
173 // confirmation on decrypt for this both-create peer.
174 self.clear_both_create_awaiting_decrypt(peer_id);
175 // The peer has proved the session, so the outbound Welcome — kept
176 // non-terminal so a lost fragment keeps being retried — is delivered.
177 // Mark it Sent so process_welcome_retry_queue stops re-sending it. Mesh
178 // has no transport-level delivery ack, so this session proof is the mesh
179 // equivalent of on_transport_send_confirmed.
180 self.mark_welcome_confirmed(peer_id, source_event);
181 if source_event != "welcome_received" && source_event != "decrypt_success" {
182 self.maybe_emit_local_session_established(
183 peer_id,
184 Self::session_ready_context_for_source(source_event),
185 );
186 }
187 info!(
188 event = "session_state_transition",
189 session_or_group_id = %peer_id,
190 previous_state = previous.as_str(),
191 new_state = "Confirmed",
192 source_event = %source_event,
193 "session_state_transition"
194 );
195
196 // Replication starts here rather than at the four callers that reach
197 // this transition, because the four callers are exactly the shape of
198 // bug that keeps happening: one of them forgets, and the feature is
199 // silently absent on whichever confirmation path that caller owned.
200 // This is also the only signal that covers all four — the session
201 // lifecycle event does not fire on the probe and ack paths.
202 #[cfg(feature = "data")]
203 self.kick_data_sync(peer_id, source_event);
204
205 Ok(true)
206 }
207
208 /// Marks a non-terminal outbound welcome lifecycle as `Sent` once the peer
209 /// has proved the session (probe / ack / welcome / decrypt). The mesh
210 /// equivalent of [`Self::on_transport_send_confirmed`], which only fires for
211 /// Internet. No-op when there is no welcome lifecycle for `peer_id` (e.g. a
212 /// confirmation keyed by a group id rather than a 1:1 peer) or it is already
213 /// terminal, so it is safe to call from every confirmation path.
214 fn mark_welcome_confirmed(&mut self, peer_id: &str, source_event: &str) {
215 match self.welcome_lifecycles.get(peer_id).map(|r| r.state) {
216 // In flight / retried — fall through and mark Sent below.
217 Some(WelcomeDeliveryState::SendAttempted) | Some(WelcomeDeliveryState::Failed) => {}
218 // Parked, never actually sent (no carrier at send time) for a peer
219 // that converged over some other path. `Created -> Sent` is not a
220 // legal transition and there is nothing in flight to confirm, so drop
221 // the now-moot lifecycle (memory + storage) rather than leaving it to
222 // linger until the session is deleted.
223 Some(WelcomeDeliveryState::Created) => {
224 self.welcome_lifecycles.remove(peer_id);
225 if let Err(err) = self.clear_welcome_lifecycle_entry(peer_id) {
226 warn!(
227 peer_id = %peer_id,
228 error = %err,
229 "Failed to clear parked welcome lifecycle on confirmation"
230 );
231 }
232 return;
233 }
234 // Already terminal (Sent / Expired) or no lifecycle — nothing to do.
235 _ => return,
236 }
237 if let Some(record) = self.welcome_lifecycles.get_mut(peer_id) {
238 record.next_retry_at = None;
239 record.last_reason_code = None;
240 record.last_transport_error = None;
241 }
242 // transition_welcome_state persists the record and logs the transition.
243 if self
244 .transition_welcome_state(peer_id, WelcomeDeliveryState::Sent, source_event)
245 .is_err()
246 {
247 return;
248 }
249 if let Some(snapshot) = self.welcome_lifecycles.get(peer_id).cloned() {
250 if let Ok(state) = lock_shared_state(&self.shared_state) {
251 state.emit_event(Event::welcome_send_succeeded(
252 peer_id.to_string(),
253 snapshot.welcome_message.id.as_str().to_string(),
254 snapshot.group_id,
255 snapshot.attempt,
256 ));
257 }
258 }
259 }
260
261 /// Ensures a session has an explicit persisted state entry.
262 pub(super) fn ensure_session_state_entry(
263 &self,
264 peer_id: &str,
265 source_event: &str,
266 ) -> Result<SessionState> {
267 let existing = self.load_session_state_entry(peer_id)?;
268 if let Some(state) = existing {
269 return Ok(state);
270 }
271
272 self.persist_session_state(peer_id, SessionState::Pending, source_event)?;
273 info!(
274 event = "session_state_transition",
275 session_or_group_id = %peer_id,
276 previous_state = "Absent",
277 new_state = "Pending",
278 source_event = %source_event,
279 "session_state_transition"
280 );
281 Ok(SessionState::Pending)
282 }
283
284 /// Reconstructs runtime confirmation cache from persisted session states.
285 /// `allowance` is the launch's shared advisory pool — this walk's deletes
286 /// are advisory (a dropped record re-bootstraps to `Pending`, which is the
287 /// safe answer), and they add up against the same allowance as the four
288 /// category walks. See `storage::MAX_RESTORE_PRUNE_DELETES`.
289 pub(super) fn restore_session_states_from_manager(
290 &mut self,
291 mls: Arc<RwLock<MlsManager>>,
292 allowance: &mut PruneAllowance,
293 ) -> Result<()> {
294 self.confirmed_sessions.clear();
295
296 let sessions = {
297 let manager = mls
298 .read()
299 .map_err(|_| Error::Other("MLS lock poisoned".to_string()))?;
300 manager.list_sessions()?
301 };
302
303 // The session list itself is deliberately not truncated the way a
304 // container-listed walk is: dropping its tail would silently leave
305 // confirmed peers out of `confirmed_sessions`. What needed bounding was
306 // the *deletes* a bad store makes this walk issue — one device barrier
307 // per peer, with nothing counting them until now.
308 let mut budget = allowance.refusing();
309 for peer_id in sessions {
310 // Before anything that can `continue`: an MLS session with this
311 // peer exists, which is the capability fact, and it holds whether
312 // or not the peer's `session_states` record turns out to be
313 // readable. Deriving it from the record instead would reintroduce
314 // exactly the deletable dependency `encryption_capable_peers`
315 // exists to escape.
316 self.mark_encryption_capable(&peer_id);
317
318 // Restore-path read: a record whose bytes will not decode is
319 // dropped and re-bootstrapped as `Pending` rather than failing
320 // initialization forever. The send path deliberately keeps the
321 // strict loader — see `load_restorable_state_record`.
322 let state = match self.load_session_state_for_restore(&peer_id, Some(&mut budget)) {
323 RestorableRecord::Present(state) => state,
324 RestorableRecord::Absent => self.bootstrap_missing_session_state(&peer_id),
325 // A record is still on disk and could not be read *this*
326 // session. Re-bootstrapping would persist `Pending` straight
327 // over one that may say `Confirmed`, and propagating would fail
328 // `initialize_mls` for as long as that single record stays
329 // unreadable — on an install that then cannot send at all.
330 // Skipping is what leaves the record recoverable: the peer is
331 // simply absent from `confirmed_sessions`, and the send path's
332 // own strict loader (`is_session_confirmed`) still fails closed
333 // for it — and re-reads the record, so a transient failure
334 // heals the moment the store answers again.
335 //
336 // Three consequences worth naming rather than rediscovering,
337 // all of them following from the peer being absent from
338 // `confirmed_sessions` for the run:
339 //
340 // 1. The Welcome machinery gates on
341 // `!confirmed_sessions.contains(peer)` *plus* a live
342 // `welcome_lifecycles` entry (`welcome_pending_peers`,
343 // `resend_unconfirmed_sent_welcome`, the retry ladder). So a
344 // peer whose lifecycle restored as `Sent` while its session
345 // state did not can draw a redundant Welcome re-send.
346 // 2. `flush_restored_confirmed_pending_messages` will not flush
347 // this peer's queued pre-session messages this session. They
348 // stay queued — in memory and on disk — and flush once a
349 // launch can read the record, or once a live confirmation
350 // re-adds the peer.
351 // 3. Tier-2 crypto recovery re-seals a resend only for a peer
352 // in `confirmed_sessions` (`reseal_resend_content`), so a
353 // resend to this peer replays its original ciphertext for
354 // the run rather than re-sealing to the current epoch.
355 //
356 // All three are deferrals on an already-degraded store, and all
357 // three are strictly better than the alternative this replaced
358 // — propagating, which failed `initialize_mls` on every launch
359 // for as long as the one record stayed unreadable, on an
360 // install that could then send nothing at all.
361 RestorableRecord::Unavailable => {
362 warn!(
363 session_or_group_id = %peer_id,
364 "Session state could not be read this session; leaving the record in \
365 place and treating the session as unconfirmed"
366 );
367 continue;
368 }
369 };
370 if matches!(state, SessionState::Confirmed) {
371 self.confirmed_sessions.insert(peer_id.clone());
372 }
373 info!(
374 event = "session_state_restored",
375 session_or_group_id = %peer_id,
376 previous_state = "Pending",
377 new_state = %state.as_str(),
378 source_event = "initialize_mls",
379 "session_state_restored"
380 );
381 }
382
383 if budget.exhausted {
384 warn!(
385 deleted = budget.spent,
386 budget = super::storage::MAX_RESTORE_PRUNE_DELETES,
387 "Session state restore hit its share of the launch delete budget; the \
388 unreadable records left behind are re-bootstrapped as Pending anyway and \
389 dropped on a later launch"
390 );
391 }
392
393 Ok(())
394 }
395
396 /// The state a session with no readable record starts from.
397 ///
398 /// Infallible: the migration write is best effort, like every other
399 /// persistence call on the restore path. It used to propagate, so one
400 /// `StoreFailed` — a full disk, a container the OS has locked — failed
401 /// `initialize_mls` outright and, with `require_encryption` on by default,
402 /// left the install unable to send. `Pending` is the safe answer in memory
403 /// either way, and a launch that can write re-derives the record.
404 pub(super) fn bootstrap_missing_session_state(&self, peer_id: &str) -> SessionState {
405 // Legacy session records without explicit state are treated as Pending.
406 // Recovery is driven by probe/ack reconciliation, never by implicit inference.
407 let restored_state = SessionState::Pending;
408 if let Err(e) = self.persist_session_state(
409 peer_id,
410 restored_state,
411 "initialize_mls_missing_state_migration",
412 ) {
413 warn!(
414 session_or_group_id = %peer_id,
415 error = %e,
416 "Failed to persist the bootstrapped session state; the session is Pending for \
417 this run and the record is re-derived on the next launch"
418 );
419 }
420 info!(
421 event = "session_state_transition",
422 session_or_group_id = %peer_id,
423 previous_state = "Absent",
424 new_state = %restored_state.as_str(),
425 source_event = "initialize_mls_missing_state_migration",
426 "session_state_transition"
427 );
428
429 restored_state
430 }
431
432 // ========================================================================
433 // CONFIRMATION RECOVERY
434 // ========================================================================
435
436 pub(super) fn schedule_confirmation_retry(&mut self, peer_id: &str, source_event: &str) {
437 self.confirmation_retry_due_at
438 .insert(peer_id.to_string(), Utc::now());
439 warn!(
440 event = "session_confirmation_retry_scheduled",
441 session_or_group_id = %peer_id,
442 source_event = %source_event,
443 "session_confirmation_retry_scheduled"
444 );
445 }
446
447 pub(super) fn clear_confirmation_recovery_tracking(&mut self, peer_id: &str) {
448 self.confirmation_retry_due_at.remove(peer_id);
449 self.confirmation_probe_due_at.remove(peer_id);
450 }
451
452 pub(super) fn collect_pending_session_peers(&mut self) -> Result<Vec<String>> {
453 let Some(mls) = self.mls_manager.clone() else {
454 return Ok(Vec::new());
455 };
456
457 let sessions = {
458 let manager = mls
459 .read()
460 .map_err(|_| Error::Other("MLS lock poisoned".to_string()))?;
461 manager.list_sessions()?
462 };
463
464 let mut pending = Vec::new();
465 for peer_id in sessions {
466 if !self.is_session_confirmed(&peer_id)? {
467 pending.push(peer_id);
468 }
469 }
470
471 Ok(pending)
472 }
473
474 pub(super) fn send_session_confirmation_probe(&mut self, peer_id: &str, source_event: &str) {
475 match self.send_internal_message(
476 peer_id,
477 internal_prefixes::SESSION_CONFIRM_PROBE.to_string(),
478 MessagePriority::High,
479 ) {
480 Ok(_) => {
481 info!(
482 event = "session_confirmation_probe_sent",
483 session_or_group_id = %peer_id,
484 source_event = %source_event,
485 "session_confirmation_probe_sent"
486 );
487 }
488 Err(err) => {
489 warn!(
490 event = "session_confirmation_probe_failed",
491 session_or_group_id = %peer_id,
492 source_event = %source_event,
493 error = %err,
494 "session_confirmation_probe_failed"
495 );
496 }
497 }
498 }
499
500 /// Throttled wrapper around session reconciliation. Skips the expensive
501 /// `list_sessions()` storage I/O when there is no pending work or when the
502 /// last scan ran recently (within `RECONCILIATION_THROTTLE_MS`).
503 ///
504 /// This is the entry-point called from `process()` and `receive_message()`
505 /// on every tick/poll. Without throttling, the storage call holds the
506 /// protocol Mutex and blocks `sendMessage()` on mobile platforms.
507 pub(super) fn run_throttled_reconciliation(&mut self, source_event: &str) {
508 // Fast path: nothing pending → skip entirely (no storage I/O)
509 let has_pending_work = !self.pending_encrypted_messages.is_empty()
510 || !self.confirmation_probe_due_at.is_empty()
511 || !self.confirmation_retry_due_at.is_empty();
512
513 if !has_pending_work {
514 return;
515 }
516
517 // Throttle: only run the full scan every RECONCILIATION_THROTTLE_MS
518 let now = Instant::now();
519 let throttle = StdDuration::from_millis(RECONCILIATION_THROTTLE_MS);
520 if let Some(last) = self.last_reconciliation_at {
521 if now.duration_since(last) < throttle {
522 return;
523 }
524 }
525
526 self.last_reconciliation_at = Some(now);
527 self.retry_pending_session_confirmations();
528 self.kick_pending_session_reconciliation(source_event);
529 }
530
531 pub(super) fn kick_pending_session_reconciliation(&mut self, source_event: &str) {
532 let now = Utc::now();
533 let pending_peers = match self.collect_pending_session_peers() {
534 Ok(peers) => peers,
535 Err(err) => {
536 warn!(
537 event = "session_confirmation_probe_scan_failed",
538 source_event = %source_event,
539 error = %err,
540 "session_confirmation_probe_scan_failed"
541 );
542 return;
543 }
544 };
545
546 let pending_set: HashSet<String> = pending_peers.iter().cloned().collect();
547 self.confirmation_probe_due_at
548 .retain(|peer, _| pending_set.contains(peer));
549
550 for peer_id in pending_peers {
551 let due_at = self
552 .confirmation_probe_due_at
553 .get(&peer_id)
554 .copied()
555 .unwrap_or(now);
556 if due_at > now {
557 continue;
558 }
559
560 self.send_session_confirmation_probe(&peer_id, source_event);
561 self.confirmation_probe_due_at.insert(
562 peer_id,
563 now + ChronoDuration::seconds(CONFIRMATION_PROBE_INTERVAL_SECS),
564 );
565 }
566 }
567
568 /// Triggers a rate-limited re-key of the 1:1 session with `peer_id` after an
569 /// epoch-desync decrypt failure.
570 ///
571 /// The self-healing primitive is a two-part move, matching the unblock
572 /// `session_reset` flow: tear down our *own* stale session **and** advertise
573 /// a `session_reset` key package. The peer, on receiving it, drops its stale
574 /// session, auto-establishes a fresh one from our enclosed key package, and
575 /// Welcomes us back — which we, now session-less, simply *join*.
576 ///
577 /// Tearing down our session alone (without the reset key package) would
578 /// deadlock: the peer's original key package was consumed at first
579 /// establishment, so nothing could rebuild the channel, and the peer (still
580 /// believing its session is healthy) would never re-advertise one. Sending
581 /// the reset key package is what breaks that deadlock.
582 ///
583 /// Deleting the local session is also what makes convergence **symmetric and
584 /// single-round regardless of user-id ordering**. Because we keep no local
585 /// session, the peer's returning Welcome is joined via the fresh-session
586 /// path rather than the greater-id-adopts tiebreaker in
587 /// `handle_welcome_message` — so the lexicographically-smaller detector is no
588 /// longer stranded (the failure mode when the stale session was kept). Both
589 /// orderings are covered end-to-end by
590 /// `test_desync_dm_heals_end_to_end_when_detector_id_is_greater` and
591 /// `test_desync_dm_heals_end_to_end_when_detector_id_is_smaller`.
592 ///
593 /// Rate-limited via `rekey_due_at` to at most one re-key per
594 /// [`REKEY_INTERVAL_SECS`] per peer, so a peer replaying stale-epoch
595 /// ciphertext (or an injected wrong-epoch frame) cannot drive a re-key storm.
596 /// The interval is stamped before the send so a send error still counts
597 /// against the limit, and it is **never reset early** — not even by a
598 /// successful decrypt on the healed session. That is deliberate: a genuine
599 /// re-fork and a replayed old-epoch frame are indistinguishable at this layer
600 /// (both surface as `WrongEpoch`), so clearing the floor on heal would let an
601 /// attacker who lands one legit decrypt between replays defeat the limit and
602 /// force ~one teardown per inbound message. The floor lapses naturally after
603 /// [`REKEY_INTERVAL_SECS`]; during that window Tier 1 (un-ACK + sender
604 /// retries) keeps delivery honest, so the only cost of not resetting is that a
605 /// genuine second desync within the window heals up to one interval later —
606 /// an acceptable trade for a rare event against a bounded-churn guarantee.
607 ///
608 /// **SECURITY — this trigger is unauthenticated, and cannot be made
609 /// otherwise.** `peer_id` is the *wire-claimed* sender. Three facts compose:
610 /// `__MLS_ENC__` is a data-plane prefix, deliberately exempt from the
611 /// Ed25519 + derivation control-message gate (see `DATA_PLANE_PREFIXES`);
612 /// OpenMLS
613 /// validates the framing header — group id, then epoch — *before* any AEAD,
614 /// sender-data decryption or signature check, so `WrongEpoch` /
615 /// `NoPastEpochData` is produced with the sender still entirely unverified;
616 /// and a 1:1 slot id is `session:<a>:<b>` over two public user ids. So the
617 /// classification that lands here is reachable by **anyone who can inject a
618 /// frame** — no key material, no captured ciphertext, no session, no replay
619 /// needed. `test_forged_frame_reaches_session_desync_without_any_key_material`
620 /// builds such a frame from scratch to keep this statement honest. It is
621 /// inherent to MLS framing, not an OpenMLS defect, so no upgrade changes it.
622 ///
623 /// The `SenderIdentityMismatch` gate cannot help: it compares the MLS
624 /// credential to the claimed sender, and that credential only exists once
625 /// `process_message` **succeeds**. Every pre-authentication verdict is
626 /// structurally outside its reach.
627 ///
628 /// Because the trigger cannot be authenticated, the mitigation is that
629 /// acting on it is **harmless**, not that it is trusted:
630 /// - `SessionManager::decrypt_message` requires the envelope to name the
631 /// slot shared with the claimed sender, so one derivable session id
632 /// cannot be aimed at arbitrary peers, and `rekey_due_at` cannot be grown
633 /// with attacker-chosen keys.
634 /// - [`REKEY_INTERVAL_SECS`] bounds this to one re-key per peer per window.
635 /// - The heal destroys nothing: queued plaintext survives a reset (it seals
636 /// at flush time against the rebuilt session), and Tier 2 re-seals
637 /// in-flight resends.
638 /// - Each re-key emits `SecurityWarningCode::SessionRekeyTriggered`, so a
639 /// sustained rate — the signature of injected frames rather than a real
640 /// fork — is visible to the app.
641 ///
642 /// **Residual:** an injector can still force bounded re-key churn on a pair
643 /// (delayed delivery, never lost). Closing that needs a signed
644 /// epoch-corroboration exchange before teardown; a liveness-only probe does
645 /// not work, since a healthy peer answers and we would tear down anyway.
646 /// Also see `docs/state-machines/session-lifecycle.md` ("Desync and heal")
647 /// and `docs/security/threat-model.md` (residual risk R2).
648 pub(super) fn schedule_session_rekey(&mut self, peer_id: &str) {
649 if !self.claim_rekey_slot(peer_id) {
650 return;
651 }
652
653 // A re-key is remotely triggerable and, until now, entirely silent to
654 // the app. A genuine fork produces these occasionally; a sustained rate
655 // for one peer is the signature of injected frames, which an operator
656 // cannot otherwise distinguish.
657 //
658 // Emitted on this path only. The warning names an epoch desync because
659 // that is what reached it, and an application asking for a re-key on
660 // its own schedule has not seen one: reusing the code there would put a
661 // fixed token in front of two different facts and leave an operator
662 // reading their own maintenance as the attack signature this exists to
663 // surface.
664 self.emit_security_warning(
665 peer_id,
666 SecurityWarningCode::SessionRekeyTriggered,
667 "1:1 session torn down and re-advertised after an epoch desync",
668 );
669 if self
670 .drive_session_rekey(peer_id, "after epoch desync")
671 .is_err()
672 {
673 // The advertisement did not leave, and this path discards the
674 // session anyway. What reached it is a fork: every later frame on
675 // that session decrypts to nothing, so keeping it buys a pair that
676 // stays silently broken rather than one that rebuilds when the
677 // peer is next heard from. The application-driven entry makes the
678 // opposite choice, and for the opposite reason: the session it was
679 // asked to rotate is healthy.
680 self.discard_local_session(peer_id);
681 }
682 }
683
684 /// Takes this peer's re-key slot, or reports that the floor has not lapsed.
685 ///
686 /// The floor is what bounds a peer replaying stale-epoch ciphertext to one
687 /// teardown per window rather than one per frame, and it is shared with the
688 /// application-driven entry deliberately: a re-key costs the same teardown
689 /// and re-exchange whoever asked for it, and a caller looping on it would
690 /// otherwise be able to do to a pair exactly what the bound exists to stop
691 /// an attacker doing.
692 pub(super) fn claim_rekey_slot(&mut self, peer_id: &str) -> bool {
693 let now = Utc::now();
694 if let Some(due_at) = self.rekey_due_at.get(peer_id) {
695 if *due_at > now {
696 return false;
697 }
698 }
699 // Bounded like every other map keyed by a wire-claimed id: the desync
700 // that reaches the caller is classified before MLS authenticates
701 // anything.
702 if !self.rekey_due_at.contains_key(peer_id)
703 && self.rekey_due_at.len() >= MAX_REKEY_TRACKED_PEERS
704 {
705 self.rekey_due_at.clear();
706 }
707 self.rekey_due_at.insert(
708 peer_id.to_string(),
709 now + ChronoDuration::seconds(REKEY_INTERVAL_SECS),
710 );
711 true
712 }
713
714 /// Advertises a reset key package and tears the local session down.
715 ///
716 /// This is the whole of what a re-key does on the wire, shared by the
717 /// desync path and by [`OfflineProtocol::rekey_session`] so the two cannot
718 /// drift into meaning different things to a peer. `reason` appears in the
719 /// log line only.
720 ///
721 /// # Why the advertisement goes first
722 ///
723 /// An `Err` here means **nothing was sent and nothing was torn down**.
724 /// [`TransportManager::send`](crate::TransportManager::send) reports an
725 /// error only once no transport has accepted the frame, so a failure is a
726 /// peer that has been told nothing, and a caller rotating a *healthy*
727 /// session must be able to leave it alone: discarding it on the strength
728 /// of a frame that never left strands the pair, with this side holding no
729 /// session, the peer holding one it will never be told to drop, and
730 /// nothing in either that re-opens the exchange.
731 ///
732 /// The order is invisible to a peer. Both steps run under one `&mut self`,
733 /// so no inbound frame can be processed between them, and what a peer sees
734 /// on success is what this path has always sent. The property the old
735 /// order was written for still holds for the same reason: by the time the
736 /// peer's returning Welcome is processed there is no local session left,
737 /// so it is *joined* rather than gated by the greater-id-adopts tiebreaker
738 /// in `handle_welcome_message`, and both sides rebuild in a single round.
739 pub(super) fn drive_session_rekey(&mut self, peer_id: &str, reason: &str) -> Result<()> {
740 if let Err(err) = self.send_key_package_to(peer_id, true) {
741 warn!(
742 event = "session_rekey_send_failed",
743 peer_id = %peer_id,
744 reason = %reason,
745 error = %err,
746 "session_rekey_send_failed"
747 );
748 return Err(err);
749 }
750 self.discard_local_session(peer_id);
751 info!(
752 event = "session_rekey_triggered",
753 peer_id = %peer_id,
754 reason = %reason,
755 "Triggered 1:1 session re-key"
756 );
757 Ok(())
758 }
759
760 /// Drops this side's 1:1 session with `peer_id`, best effort.
761 ///
762 /// Mirrors the unblock `session_reset` flow. A missing session is a no-op,
763 /// and a delete that fails is logged rather than raised: the peer has
764 /// already been sent the reset by every caller that reaches here, and the
765 /// rebuild plus the auto key-package exchange re-arm establishment either
766 /// way.
767 fn discard_local_session(&mut self, peer_id: &str) {
768 if self.has_mls_session(peer_id).unwrap_or(false) {
769 if let Err(err) = self.manual_mls_delete_session(peer_id) {
770 debug!(
771 peer_id = %peer_id,
772 error = %err,
773 "rekey: failed to delete local stale session"
774 );
775 }
776 }
777 }
778
779 /// Gives back a re-key slot claimed for work that did not happen.
780 ///
781 /// Only the application-driven entry may call this, and only when the
782 /// advertisement never left: no teardown happened, so nothing was spent
783 /// and a caller back in range a second later should not be made to wait
784 /// out a window that bounds teardowns. The desync path must **not**, since
785 /// it discards the session whether or not the reset was sent, and that is
786 /// exactly the churn the floor exists to bound.
787 pub(super) fn release_rekey_slot(&mut self, peer_id: &str) {
788 self.rekey_due_at.remove(peer_id);
789 }
790
791 pub(super) fn retry_pending_session_confirmations(&mut self) {
792 let now = Utc::now();
793 let due_peers: Vec<String> = self
794 .confirmation_retry_due_at
795 .iter()
796 .filter_map(|(peer_id, due_at)| {
797 if *due_at <= now {
798 Some(peer_id.clone())
799 } else {
800 None
801 }
802 })
803 .collect();
804
805 for peer_id in due_peers {
806 match self.has_mls_session(&peer_id) {
807 Ok(true) => {}
808 Ok(false) => {
809 self.confirmation_retry_due_at.remove(&peer_id);
810 continue;
811 }
812 Err(err) => {
813 warn!(
814 event = "session_confirmation_retry_scan_failed",
815 session_or_group_id = %peer_id,
816 error = %err,
817 "session_confirmation_retry_scan_failed"
818 );
819 self.confirmation_retry_due_at.insert(
820 peer_id,
821 now + ChronoDuration::seconds(CONFIRMATION_RETRY_INTERVAL_SECS),
822 );
823 continue;
824 }
825 }
826
827 if !self.can_confirm_from_source(&peer_id, "confirmation_retry") {
828 debug!(
829 peer_id = %peer_id,
830 "Skipping confirmation retry until welcome send is at least attempted"
831 );
832 continue;
833 }
834
835 match self.confirm_session_state(&peer_id, "confirmation_retry") {
836 Ok(_) => {
837 let _ = self.flush_pending_messages(&peer_id);
838 self.process_pending_decryption(&peer_id);
839 }
840 Err(err) => {
841 warn!(
842 event = "session_confirmation_retry_failed",
843 session_or_group_id = %peer_id,
844 error = %err,
845 "session_confirmation_retry_failed"
846 );
847 self.confirmation_retry_due_at.insert(
848 peer_id,
849 now + ChronoDuration::seconds(CONFIRMATION_RETRY_INTERVAL_SECS),
850 );
851 }
852 }
853 }
854 }
855
856 /// Marks `peer_id` as a both-create owner gate — we kept our own group and
857 /// must observe a group-aware decrypt before confirming — and persists it so
858 /// an owner restart mid-convergence cannot let a stale plaintext probe/ack
859 /// confirm prematurely. Only writes storage on a genuine insert.
860 pub(super) fn mark_both_create_awaiting_decrypt(&mut self, peer_id: &str) {
861 if self
862 .both_create_awaiting_decrypt
863 .insert(peer_id.to_string())
864 {
865 self.persist_both_create_awaiting_decrypt(peer_id);
866 }
867 }
868
869 /// Clears the both-create owner gate for `peer_id` (it has converged) from
870 /// both memory and storage. Only writes storage on a genuine removal.
871 pub(super) fn clear_both_create_awaiting_decrypt(&mut self, peer_id: &str) {
872 if self.both_create_awaiting_decrypt.remove(peer_id) {
873 self.delete_both_create_awaiting_decrypt(peer_id);
874 }
875 }
876
877 pub(super) fn can_confirm_from_source(&self, peer_id: &str, source_event: &str) -> bool {
878 // Both-create owner: we kept our own group and are waiting for the peer to
879 // prove it adopted *our* group. Only a successful decrypt is group-aware
880 // proof of that; a plaintext probe/ack proves only that the peer holds
881 // some session (possibly its own, pre-adoption group), so it must not
882 // confirm us or we would stop retransmitting and strand the peer.
883 if self.both_create_awaiting_decrypt.contains(peer_id) && source_event != "decrypt_success"
884 {
885 return false;
886 }
887
888 // A successful group decrypt is definitive, group-aware proof that the
889 // peer adopted our 1:1 group. `decrypt_success` is raised only inside
890 // `DecryptResult::Success` (see message_dispatch.rs): ciphertext on the
891 // single stored `session:<a>:<b>` group actually decrypted, which is
892 // possible only if the peer is a member. It must therefore confirm the
893 // session regardless of our *local* Welcome's delivery state. Gating it on
894 // a still-active local Welcome (the lifecycle match below) strands a
895 // both-create owner whose own Welcome timed out to `Failed`/`Expired` on a
896 // lossy or asymmetric BLE link (the Android↔iOS case): the owner decrypts
897 // the peer's messages but never confirms, so every outbound send fails
898 // `SessionNotReady` — it can receive but never reply. The both-create gate
899 // above is preserved, so a both-create owner is still restricted to *only*
900 // decrypt_success (a plaintext probe/ack can never confirm it).
901 if source_event == "decrypt_success" {
902 return true;
903 }
904
905 if !matches!(
906 source_event,
907 "confirmation_ack_received" | "confirmation_probe_received" | "confirmation_retry"
908 ) {
909 return true;
910 }
911
912 match self.welcome_lifecycles.get(peer_id) {
913 Some(record) => matches!(
914 record.state,
915 WelcomeDeliveryState::Sent | WelcomeDeliveryState::SendAttempted
916 ),
917 None => matches!(
918 source_event,
919 // Compatibility path for sessions created before welcome lifecycle
920 // persistence existed. A plaintext probe/ack/retry can confirm here
921 // (it is the only evidence such a legacy session has); a group-aware
922 // decrypt already returned true above and never reaches this arm.
923 "confirmation_ack_received" | "confirmation_probe_received" | "confirmation_retry"
924 ),
925 }
926 }
927
928 pub(super) fn abort_pending_session_for_peer(
929 &mut self,
930 peer_id: &str,
931 reason: crate::events::WelcomeReasonCode,
932 ) {
933 self.drop_pending_queue_for_peer(
934 peer_id,
935 &format!("Welcome delivery failed: {}", reason.as_str()),
936 );
937 if let Ok(state) = lock_shared_state(&self.shared_state) {
938 state.emit_event(Event::secure_session_failed(
939 peer_id.to_string(),
940 format!("Welcome delivery failed: {}", reason.as_str()),
941 ));
942 }
943 }
944
945 pub(super) fn maybe_emit_local_session_established(
946 &self,
947 peer_id: &str,
948 context: MlsOperationContext,
949 ) {
950 let Some(record) = self.welcome_lifecycles.get(peer_id) else {
951 return;
952 };
953 if !matches!(record.state, WelcomeDeliveryState::Sent) {
954 return;
955 }
956 self.emit_mls_session_ready(peer_id, &record.group_id, context);
957 if let Ok(state) = lock_shared_state(&self.shared_state) {
958 state.emit_event(Event::secure_session_established(
959 peer_id.to_string(),
960 record.group_id.clone(),
961 record.group_id.starts_with("session:"),
962 true,
963 ));
964 }
965 }
966
967 // ========================================================================
968 // WELCOME LIFECYCLE
969 // ========================================================================
970
971 pub(super) fn send_welcome_message(
972 &mut self,
973 recipient: &str,
974 welcome: &WelcomeMessage,
975 ) -> Result<bool> {
976 let serialized =
977 serde_json::to_string(welcome).map_err(|e| Error::Serialization(e.to_string()))?;
978 let content = format!("{}{}", internal_prefixes::WELCOME, serialized);
979 let mut message =
980 self.create_message(recipient, content, Some(MessagePriority::High), None)?;
981 self.sign_control_message(&mut message)?;
982 let group_id = welcome.group_id.as_str().to_string();
983
984 self.upsert_welcome_lifecycle(recipient, &group_id, message, "welcome_created")?;
985 self.try_send_welcome(recipient, "welcome_initial_send")
986 }
987
988 /// Parks a Welcome that currently has no transport carrier: refresh the
989 /// carrier-relative TTL and schedule a slow re-check, WITHOUT consuming a
990 /// retry attempt, transitioning state, or emitting a `WelcomeSendAttempted`
991 /// event. A no-carrier send is a guaranteed failure, so re-attempting it at
992 /// the data-plane retry rate only churns storage I/O and app events while a
993 /// device is offline. The lifecycle stays in its current non-terminal state
994 /// (Created on the first send, Failed on a later retry) so the retry queue
995 /// re-polls it on [`WELCOME_NO_CARRIER_RETRY_SECS`]; `on_neighbor_discovered`
996 /// re-arms it immediately when a carrier surfaces the peer. Returns
997 /// `Ok(false)` (not sent).
998 ///
999 /// **Invariant: the refreshed TTL always outlives the retry being
1000 /// scheduled.** [`Self::try_send_welcome`] checks `expires_at` *before*
1001 /// running a due retry, so a record whose TTL lapsed while it waited gets
1002 /// expired terminally (`welcome_send_expired` + `secure_session_failed`) by
1003 /// the very timer meant to recover it. That is not hypothetical at the far
1004 /// end of the unreachable ladder in
1005 /// [`Self::apply_recipient_unreachable_failure`]: it reaches 480s at six
1006 /// consecutive parks and [`WELCOME_UNREACHABLE_RETRY_CAP_SECS`] (600s)
1007 /// beyond that, both above the 300s [`WELCOME_LIFECYCLE_TTL_SECS`]. A full
1008 /// extra interval of slack keeps the probe's own outcome — a verdict
1009 /// re-park or a confirm timeout — inside the window too. No-op for the 15s
1010 /// no-carrier park, whose interval is far below the TTL.
1011 fn park_welcome_no_carrier(
1012 &mut self,
1013 peer_id: &str,
1014 reason: crate::events::WelcomeReasonCode,
1015 transport_error: Option<&'static str>,
1016 retry_in_secs: i64,
1017 ) -> Result<bool> {
1018 let snapshot = {
1019 let Some(record) = self.welcome_lifecycles.get_mut(peer_id) else {
1020 return Ok(false);
1021 };
1022 let now = Utc::now();
1023 record.next_retry_at = Some(now + ChronoDuration::seconds(retry_in_secs));
1024 record.expires_at = now
1025 + ChronoDuration::seconds(
1026 WELCOME_LIFECYCLE_TTL_SECS.max(retry_in_secs.saturating_mul(2)),
1027 );
1028 record.last_reason_code = Some(reason);
1029 record.last_transport_error = transport_error.map(str::to_string);
1030 record.clone()
1031 };
1032 self.persist_welcome_lifecycle_entry(&snapshot)?;
1033 debug!(
1034 peer_id = %peer_id,
1035 state = snapshot.state.as_str(),
1036 retry_in_secs,
1037 "Welcome parked: no transport carrier, re-checking on slow interval"
1038 );
1039 Ok(false)
1040 }
1041
1042 pub(super) fn try_send_welcome(&mut self, peer_id: &str, source_event: &str) -> Result<bool> {
1043 let now = Utc::now();
1044 let mut record = self
1045 .welcome_lifecycles
1046 .get(peer_id)
1047 .cloned()
1048 .ok_or_else(|| Error::Other(format!("Missing welcome lifecycle for {}", peer_id)))?;
1049
1050 if matches!(record.state, WelcomeDeliveryState::Sent) {
1051 return Ok(true);
1052 }
1053 if matches!(record.state, WelcomeDeliveryState::Expired) {
1054 return Ok(false);
1055 }
1056
1057 // Only honor the TTL when a carrier actually exists: a Welcome that has
1058 // never had a deliverable transport must not age out.
1059 let carrier_available = !self.transport_manager.get_available_transports().is_empty();
1060 if carrier_available && record.expires_at <= now {
1061 self.transition_welcome_state(peer_id, WelcomeDeliveryState::Expired, source_event)?;
1062 if let Ok(state) = lock_shared_state(&self.shared_state) {
1063 state.emit_event(Event::welcome_send_expired(
1064 peer_id.to_string(),
1065 record.welcome_message.id.as_str().to_string(),
1066 record.attempt,
1067 crate::events::WelcomeReasonCode::RetryExhausted,
1068 ));
1069 }
1070 self.abort_pending_session_for_peer(
1071 peer_id,
1072 crate::events::WelcomeReasonCode::RetryExhausted,
1073 );
1074 return Ok(false);
1075 }
1076
1077 // No carrier at all: a send is guaranteed to fail, so do NOT burn the
1078 // speculative attempt increment, the SendAttempted/Failed transitions
1079 // (two persisted state changes + two info logs), and a
1080 // `WelcomeSendAttempted` event on it every retry tick. Park the
1081 // lifecycle on a slow carrier-relative interval instead and re-check
1082 // later; `on_neighbor_discovered` re-arms it immediately when a carrier
1083 // surfaces the peer. This keeps an offline device quiet rather than
1084 // spinning storage I/O + app events at the data-plane retry rate.
1085 if !carrier_available {
1086 return self.park_welcome_no_carrier(
1087 peer_id,
1088 crate::events::WelcomeReasonCode::TransportUnavailable,
1089 None,
1090 WELCOME_NO_CARRIER_RETRY_SECS,
1091 );
1092 }
1093
1094 record.attempt = record.attempt.saturating_add(1);
1095 self.welcome_lifecycles
1096 .insert(peer_id.to_string(), record.clone());
1097 self.persist_welcome_lifecycle_entry(&record)?;
1098 self.transition_welcome_state(peer_id, WelcomeDeliveryState::SendAttempted, source_event)?;
1099
1100 if let Ok(state) = lock_shared_state(&self.shared_state) {
1101 state.emit_event(Event::welcome_send_attempted(
1102 peer_id.to_string(),
1103 record.welcome_message.id.as_str().to_string(),
1104 record.group_id.clone(),
1105 record.attempt,
1106 ));
1107 }
1108
1109 match self.transport_manager.send(&record.welcome_message) {
1110 Ok(()) => {
1111 let transport_used = self.transport_manager.current_transport();
1112 let mut updated =
1113 self.welcome_lifecycles
1114 .get(peer_id)
1115 .cloned()
1116 .ok_or_else(|| {
1117 Error::Other(format!("Missing welcome lifecycle for {}", peer_id))
1118 })?;
1119
1120 // No transport's send() proves the peer reassembled and joined
1121 // the group, so keep the lifecycle NON-TERMINAL until an explicit
1122 // confirmation arrives and let the retry queue re-send on timeout.
1123 //
1124 // - Internet send() only enqueues for platform polling; the
1125 // confirmation is on_transport_send_confirmed.
1126 // - A BLE / WiFi-Direct GATT WRITE_TYPE_NO_RESPONSE returning Ok
1127 // only means the local stack accepted the bytes. A single lost
1128 // fragment of the multi-fragment Welcome would otherwise leave
1129 // the sender believing the session exists while the peer holds
1130 // undecryptable ciphertext, with NO retry — permanently
1131 // bricking the pair. The mesh confirmation is the peer proving
1132 // the session (its probe / ack / welcome / decrypt), which marks
1133 // the welcome Sent via confirm_session_state.
1134 let confirm_timeout_secs =
1135 if matches!(transport_used, Some(TransportType::Internet)) {
1136 WELCOME_INTERNET_CONFIRM_TIMEOUT_SECS
1137 } else {
1138 WELCOME_MESH_CONFIRM_TIMEOUT_SECS
1139 };
1140 updated.next_retry_at =
1141 Some(Utc::now() + ChronoDuration::seconds(confirm_timeout_secs));
1142 updated.last_reason_code = None;
1143 updated.last_transport_error = None;
1144 self.welcome_lifecycles
1145 .insert(peer_id.to_string(), updated.clone());
1146 self.persist_welcome_lifecycle_entry(&updated)?;
1147
1148 // Seed the confirmation-probe scheduler for this pending peer so
1149 // the SENDER actively reconciles instead of waiting passively.
1150 //
1151 // The Welcome is now in flight but unconfirmed. The sender's only
1152 // built-in convergence path would otherwise be the receiver's
1153 // single proactive encrypted confirm on first Welcome receipt — a
1154 // single point of failure with no retry (a lost fragment, a
1155 // not-yet-ready encryptor, or the retransmit/owner_keep path on
1156 // the receiver all strand it). By marking a probe due now,
1157 // `run_throttled_reconciliation` (whose `has_pending_work` gate is
1158 // otherwise false on the sender, since it has no pending encrypted
1159 // messages) starts emitting `SESSION_CONFIRM_PROBE`s. The peer —
1160 // which holds the session — replies with `SESSION_CONFIRM_ACK`,
1161 // confirming us and marking this Welcome `Sent`. The entry is
1162 // dropped once confirmed (`clear_confirmation_recovery_tracking`)
1163 // or once the peer is no longer pending (`kick`'s retain), so this
1164 // adds no steady-state work after convergence.
1165 self.confirmation_probe_due_at
1166 .insert(peer_id.to_string(), Utc::now());
1167
1168 Ok(false)
1169 }
1170 Err(err) => {
1171 let no_carrier = Self::is_no_carrier_error(&err);
1172 let reason = Self::map_welcome_reason_code(&err);
1173 // Classified, not rendered: the transport layer interpolates
1174 // the peer into `PeerNotReachable`, and this value is both
1175 // persisted with the lifecycle record and emitted on
1176 // `WelcomeSendFailed.transport_error`, where the scrubber
1177 // hashes `peer_id` beside it. The full error is logged above.
1178 self.apply_welcome_send_failure(
1179 peer_id,
1180 reason,
1181 Some(send_failure_token(&err)),
1182 no_carrier,
1183 source_event,
1184 )
1185 }
1186 }
1187 }
1188
1189 /// Re-arms a stalled or expired outbound Welcome when a peer becomes
1190 /// reachable again, so a session that never converged while the peer was
1191 /// offline gets a fresh delivery attempt over the now-available carrier.
1192 ///
1193 /// No-op when the peer's session is already confirmed, when there is no
1194 /// Welcome lifecycle, or when the Welcome is already `Sent` or still
1195 /// in-flight (`SendAttempted`). An `Expired` lifecycle is rebuilt from its
1196 /// retained Welcome message — the MLS group is not torn down on expiry, so
1197 /// the stored Welcome is still valid to re-send — while a `Created`/`Failed`
1198 /// lifecycle has its budget and TTL reset and is retried immediately.
1199 pub(super) fn rearm_welcome_for_peer(&mut self, peer_id: &str, source_event: &str) {
1200 if self.confirmed_sessions.contains(peer_id) {
1201 return;
1202 }
1203 let Some(record) = self.welcome_lifecycles.get(peer_id).cloned() else {
1204 return;
1205 };
1206 match record.state {
1207 WelcomeDeliveryState::Sent | WelcomeDeliveryState::SendAttempted => {}
1208 WelcomeDeliveryState::Created | WelcomeDeliveryState::Failed => {
1209 if let Some(entry) = self.welcome_lifecycles.get_mut(peer_id) {
1210 entry.attempt = 0;
1211 // A reachability edge resets the unreachable-park
1212 // escalation: the next relay verdict starts the interval
1213 // ladder over from its base.
1214 entry.unreachable_parks = 0;
1215 entry.next_retry_at = Some(Utc::now());
1216 entry.last_reason_code = None;
1217 entry.last_transport_error = None;
1218 entry.expires_at =
1219 Utc::now() + ChronoDuration::seconds(WELCOME_LIFECYCLE_TTL_SECS);
1220 }
1221 if let Err(err) = self.try_send_welcome(peer_id, source_event) {
1222 warn!(
1223 peer_id = %peer_id,
1224 error = %err,
1225 "Failed to re-arm welcome on peer reachability"
1226 );
1227 }
1228 }
1229 WelcomeDeliveryState::Expired => {
1230 // Expired is terminal in the lifecycle state machine, so rebuild
1231 // from the retained Welcome message (upsert permits overwrite
1232 // from Expired) before re-sending.
1233 let welcome_message = record.welcome_message.clone();
1234 let group_id = record.group_id.clone();
1235 if let Err(err) =
1236 self.upsert_welcome_lifecycle(peer_id, &group_id, welcome_message, source_event)
1237 {
1238 warn!(
1239 peer_id = %peer_id,
1240 error = %err,
1241 "Failed to rebuild expired welcome on peer reachability"
1242 );
1243 return;
1244 }
1245 if let Err(err) = self.try_send_welcome(peer_id, source_event) {
1246 warn!(
1247 peer_id = %peer_id,
1248 error = %err,
1249 "Failed to re-arm expired welcome on peer reachability"
1250 );
1251 }
1252 }
1253 }
1254 }
1255
1256 /// Ingests an authoritative per-peer presence signal (the internet relay's
1257 /// `PresenceStatusWithLastSeen` answer, bridged by the platform layer).
1258 ///
1259 /// `online` drives the same reachability machinery as a transport-level
1260 /// discovery — outbox flush, welcome re-arm, auto key exchange — plus a
1261 /// rescue for the store-less-relay hole: a welcome whose wire send was
1262 /// confirmed (`Sent`) while the peer was actually offline was dropped by
1263 /// the relay (it forwards or pushes, never stores), and
1264 /// `Self::rearm_welcome_for_peer` deliberately never touches `Sent`.
1265 /// The peer being provably online is the safe moment to rebuild and
1266 /// re-send it: if the original did land, the receiver dedups by message
1267 /// id and the confirmation probe resolves the lifecycle.
1268 ///
1269 /// `offline` parks retry-pending welcomes pending a reachability edge,
1270 /// without burning budget. In-flight records (`SendAttempted`) are left
1271 /// to their confirm deadline — a `DeliveryError`-driven failure or the
1272 /// confirm timeout resolves them, after which the next offline tick parks.
1273 ///
1274 /// The welcome re-arm/re-send on the online edge is throttled per peer
1275 /// with exponential backoff (`WELCOME_PRESENCE_RESCUE_BASE_SECS` ..
1276 /// `WELCOME_PRESENCE_RESCUE_MAX_SECS`): presence answers arrive on the
1277 /// platform's ~20s watch tick, and a peer that is online but can never
1278 /// prove the session (stale key package after a reinstall, incompatible
1279 /// version) must not be re-sent the multi-frame MLS welcome on every
1280 /// tick forever. A throttled online tick still flushes queued data-plane
1281 /// traffic for the peer.
1282 ///
1283 /// Emits `presence_updated` with `source: Internet` — this function is
1284 /// the relay-sourced half of the unified stream and must only be fed
1285 /// relay-observed signals (bridged via `internet_peer_presence`);
1286 /// peer-sent `__PRESENCE__` self-reports are handled in
1287 /// `message_dispatch` and emit `source: Peer`. Self, blocked, or empty
1288 /// peer ids are dropped entirely: an app waiting on `presence_updated`
1289 /// for a blocked peer will never see it.
1290 pub fn on_peer_presence(&mut self, peer_id: &str, online: bool, last_seen_ms: Option<i64>) {
1291 if peer_id.is_empty() || peer_id == self.local_id || self.is_user_blocked(peer_id) {
1292 return;
1293 }
1294 // A presence answer is a claim about this recipient on the carrier
1295 // that answered, so it is recorded as one. It supersedes an earlier
1296 // verdict for that carrier: both answer the same question and this is
1297 // the later answer. Presence facts decay faster than verdicts, since
1298 // this is a third party's report about someone else's connection.
1299 self.reachability.record(
1300 peer_id,
1301 TransportType::Internet,
1302 if online {
1303 Claim::Reachable
1304 } else {
1305 Claim::Unreachable
1306 },
1307 FactSource::GatewayPresence,
1308 std::time::Instant::now(),
1309 );
1310 if online {
1311 // Relay-scoped reachability proof: un-park and re-drive the
1312 // peer's DMs over the internet transport before anything else
1313 // (see `flush_outbox_for_peer_via` — this also cancels
1314 // unanswerable mesh-probe ACKs that would otherwise hold the
1315 // messages hostage past this edge). The rescue branch's inner
1316 // flush then finds successfully re-driven entries awaiting
1317 // fresh ACKs and leaves them alone; entries whose forced send
1318 // *failed* are picked up again, so the inner flush keeps the
1319 // internet override — DORS must not re-route them into the
1320 // mesh void with the park counter already cleared.
1321 self.flush_outbox_for_peer_via(peer_id, Some(TransportType::Internet));
1322 if self.welcome_rescue_permitted(peer_id) {
1323 self.on_neighbor_discovered_via(peer_id, Some(TransportType::Internet));
1324 self.resend_unconfirmed_sent_welcome(peer_id, "peer_presence_online");
1325 self.note_welcome_rescue_attempt(peer_id);
1326 }
1327 } else {
1328 self.park_welcome_peer_unreachable(peer_id);
1329 }
1330
1331 let status = if online {
1332 crate::events::PresenceStatus::Online
1333 } else {
1334 crate::events::PresenceStatus::Offline
1335 };
1336 self.emit_event(Event::presence_updated_with_last_seen(
1337 peer_id.to_string(),
1338 status,
1339 Utc::now().timestamp_millis(),
1340 last_seen_ms,
1341 ));
1342 }
1343
1344 /// Peers with a welcome lifecycle whose session has not been proven —
1345 /// the internet presence watch set. Includes `Sent` records (wire-
1346 /// confirmed but never session-confirmed): only a presence signal can
1347 /// rescue those over a store-less relay.
1348 /// Lifecycles older than `WELCOME_WATCHLIST_MAX_AGE_SECS` are excluded:
1349 /// a permanently-dead peer must not occupy watch-rotation slots (and keep
1350 /// its parked record alive via `expires_at` pushes) forever.
1351 pub fn welcome_pending_peers(&self) -> Vec<String> {
1352 let watch_cutoff = Utc::now() - ChronoDuration::seconds(WELCOME_WATCHLIST_MAX_AGE_SECS);
1353 self.welcome_lifecycles
1354 .iter()
1355 .filter(|(peer_id, record)| {
1356 !self.confirmed_sessions.contains(*peer_id) && record.created_at > watch_cutoff
1357 })
1358 .map(|(peer_id, _)| peer_id.clone())
1359 .collect()
1360 }
1361
1362 /// Rebuilds and re-sends a `Sent`-but-session-unconfirmed welcome (see
1363 /// [`Self::on_peer_presence`]). No-op for any other state.
1364 fn resend_unconfirmed_sent_welcome(&mut self, peer_id: &str, source_event: &str) {
1365 if self.confirmed_sessions.contains(peer_id) {
1366 return;
1367 }
1368 let Some(record) = self.welcome_lifecycles.get(peer_id) else {
1369 return;
1370 };
1371 if !matches!(record.state, WelcomeDeliveryState::Sent) {
1372 return;
1373 }
1374 let welcome_message = record.welcome_message.clone();
1375 let group_id = record.group_id.clone();
1376 if let Err(err) =
1377 self.upsert_welcome_lifecycle(peer_id, &group_id, welcome_message, source_event)
1378 {
1379 warn!(
1380 peer_id = %peer_id,
1381 error = %err,
1382 "Failed to rebuild unconfirmed sent welcome on presence"
1383 );
1384 return;
1385 }
1386 if let Err(err) = self.try_send_welcome(peer_id, source_event) {
1387 warn!(
1388 peer_id = %peer_id,
1389 error = %err,
1390 "Failed to re-send unconfirmed welcome on presence"
1391 );
1392 }
1393 }
1394
1395 /// True when a presence-online rescue for this peer is currently allowed
1396 /// by the per-peer backoff. Trivially true (and the throttle entry is
1397 /// dropped) once the peer has no unconfirmed welcome — there is nothing
1398 /// left to throttle.
1399 fn welcome_rescue_permitted(&mut self, peer_id: &str) -> bool {
1400 let pending = self.welcome_lifecycles.contains_key(peer_id)
1401 && !self.confirmed_sessions.contains(peer_id);
1402 if !pending {
1403 self.welcome_presence_rescue.remove(peer_id);
1404 return true;
1405 }
1406 self.welcome_presence_rescue
1407 .get(peer_id)
1408 .is_none_or(|throttle| throttle.next_allowed_at <= Utc::now())
1409 }
1410
1411 /// Records that a presence-online tick ran the rescue actions for a peer
1412 /// that still has an unconfirmed welcome, doubling the wait before the
1413 /// next one (base 40s, capped at 10 minutes). The counter is per
1414 /// convergence attempt: the entry is dropped (via
1415 /// [`Self::welcome_rescue_permitted`]) as soon as the session confirms
1416 /// or the lifecycle goes away.
1417 fn note_welcome_rescue_attempt(&mut self, peer_id: &str) {
1418 let pending = self.welcome_lifecycles.contains_key(peer_id)
1419 && !self.confirmed_sessions.contains(peer_id);
1420 if !pending {
1421 self.welcome_presence_rescue.remove(peer_id);
1422 return;
1423 }
1424 let throttle = self
1425 .welcome_presence_rescue
1426 .entry(peer_id.to_string())
1427 .or_insert(PresenceRescueThrottle {
1428 next_allowed_at: Utc::now(),
1429 rescues: 0,
1430 });
1431 // Shift is clamped to 8, so 40 << 8 = 10240 max before the cap —
1432 // no overflow risk.
1433 let backoff_secs = (WELCOME_PRESENCE_RESCUE_BASE_SECS << throttle.rescues.min(8))
1434 .min(WELCOME_PRESENCE_RESCUE_MAX_SECS);
1435 throttle.next_allowed_at = Utc::now() + ChronoDuration::seconds(backoff_secs);
1436 throttle.rescues = throttle.rescues.saturating_add(1);
1437 }
1438
1439 /// Parks a welcome pending a peer-reachability edge: reason
1440 /// `PeerUnreachable`, **no timed retry**. The carrier (relay socket) is
1441 /// healthy — only this peer is unreachable on it — so the *data-plane*
1442 /// retry this cancels would re-send the welcome into another
1443 /// `DeliveryError` while burning the real-delivery budget that a
1444 /// carrier-backed failure already charged. Recovery is edge-driven
1445 /// instead: `on_peer_presence(online)` and `on_neighbor_discovered` re-arm
1446 /// via `rearm_welcome_for_peer`, and the peer stays on the presence
1447 /// watchlist (`welcome_pending_peers`) so the platform keeps polling for
1448 /// that edge. The TTL is pushed like a no-carrier park: an unreachable
1449 /// peer must not age the welcome out.
1450 ///
1451 /// Its caller is responsible for not applying this to a record already
1452 /// holding an escalating unreachable probe — see
1453 /// [`Self::park_welcome_peer_unreachable`], which is what distinguishes
1454 /// the budget-burning data-plane retry cancelled here from the
1455 /// budget-refunded probe that must survive.
1456 fn park_welcome_awaiting_peer(&mut self, peer_id: &str) -> Result<()> {
1457 let snapshot = {
1458 let Some(record) = self.welcome_lifecycles.get_mut(peer_id) else {
1459 return Ok(());
1460 };
1461 record.next_retry_at = None;
1462 record.expires_at = Utc::now() + ChronoDuration::seconds(WELCOME_LIFECYCLE_TTL_SECS);
1463 record.last_reason_code = Some(crate::events::WelcomeReasonCode::PeerUnreachable);
1464 record.clone()
1465 };
1466 self.persist_welcome_lifecycle_entry(&snapshot)?;
1467 debug!(
1468 peer_id = %peer_id,
1469 state = snapshot.state.as_str(),
1470 "Welcome parked: peer unreachable on carrier, awaiting reachability edge"
1471 );
1472 Ok(())
1473 }
1474
1475 /// Parks a retry-pending welcome for a peer the relay reports offline.
1476 /// Only `Created`/`Failed` records park: `SendAttempted` keeps its
1477 /// confirm deadline, and `Sent`/`Expired` are handled on the online edge
1478 /// (or corrected by an attributed `DeliveryError`). Skipped entirely when
1479 /// a mesh carrier is also available — relay presence is only
1480 /// authoritative for the internet path, and the data-plane retry track
1481 /// may still deliver over BLE / WiFi-Direct.
1482 ///
1483 /// Never cancels an escalating unreachable probe: a record that already
1484 /// holds one was parked by [`Self::apply_recipient_unreachable_failure`],
1485 /// which already set the same reason code and pushed the same TTL — so this
1486 /// adds nothing and would only downgrade the probe to edge-only. That
1487 /// matters because on an internet-only device this runs once per presence
1488 /// rotation for as long as the peer is down, which would otherwise
1489 /// re-strand the welcome after every single verdict. The distinction is
1490 /// budget: the data-plane retry this *does* cancel was charged an attempt
1491 /// by a carrier-backed failure, while the probe refunds its attempt on
1492 /// every verdict.
1493 ///
1494 /// `last_reason_code` — not `unreachable_parks` alone — is what encodes
1495 /// that distinction. The counter is sticky (only `rearm_welcome_for_peer`
1496 /// clears it), so gating on it by itself would keep shielding the record
1497 /// after the probe track has stopped protecting it: past
1498 /// [`WELCOME_WATCHLIST_MAX_AGE_SECS`] a confirm timeout no longer re-parks
1499 /// (see [`Self::welcome_probe_repark_permitted`]) and instead charges an
1500 /// attempt and arms a plain data-plane retry — precisely the budget-burner
1501 /// this park exists to cancel, and left running it expires the welcome at
1502 /// `max_retries`. Note `park_welcome_awaiting_peer` also stamps
1503 /// `PeerUnreachable` but leaves `next_retry_at = None`, so the
1504 /// `next_retry_at.is_some()` conjunct below is load-bearing, not
1505 /// redundant: the reason code *and* a scheduled retry together are what
1506 /// identify a live probe. A carrier-backed failure stamps
1507 /// `Timeout`/`SendFailed` and a successful send clears the code entirely.
1508 fn park_welcome_peer_unreachable(&mut self, peer_id: &str) {
1509 if self.confirmed_sessions.contains(peer_id) {
1510 return;
1511 }
1512 let Some(record) = self.welcome_lifecycles.get(peer_id) else {
1513 return;
1514 };
1515 if !matches!(
1516 record.state,
1517 WelcomeDeliveryState::Created | WelcomeDeliveryState::Failed
1518 ) {
1519 return;
1520 }
1521 if record.last_reason_code == Some(crate::events::WelcomeReasonCode::PeerUnreachable)
1522 && record.unreachable_parks > 0
1523 && record.next_retry_at.is_some()
1524 {
1525 return;
1526 }
1527 // Local mesh carriers only (BLE / WiFi-Direct): Nostr and Reticulum
1528 // are internet-dependent, so their availability must not veto the
1529 // park — mirrors the carrier guard in
1530 // `apply_recipient_unreachable_failure`.
1531 if self
1532 .transport_manager
1533 .get_available_transports()
1534 .keys()
1535 .any(|transport| matches!(transport, TransportType::BLE | TransportType::WiFiDirect))
1536 {
1537 return;
1538 }
1539 let _ = self.park_welcome_awaiting_peer(peer_id);
1540 }
1541
1542 /// True when an unconfirmed welcome send should resolve as another
1543 /// unreachable verdict ([`Self::apply_recipient_unreachable_failure`])
1544 /// rather than as a carrier-backed failure that ages the record.
1545 ///
1546 /// A live `unreachable_parks` counter means the relay declared the peer
1547 /// offline and no reachability edge has fired since (every edge clears it
1548 /// via `rearm_welcome_for_peer`), so the send that just went unconfirmed
1549 /// was the escalating probe, not a delivery attempt. Without this the
1550 /// probe destroys the very welcome it exists to recover: the relay
1551 /// accepts the frame whenever its push fallback succeeds, which returns
1552 /// *no* `DeliveryError` at all, so the probe resolves at the 10s confirm
1553 /// timeout instead. Treated as carrier-backed that charges the attempt
1554 /// and arms a plain data-plane retry, and since nothing in that ladder
1555 /// pushes `expires_at`, the record walks into `should_expire` within one
1556 /// TTL window — terminal `welcome_send_expired` + `secure_session_failed`
1557 /// for a peer that is merely offline. Re-parking instead refunds the
1558 /// attempt, escalates the interval and pushes the TTL, mirroring the DM
1559 /// path's [`Self::try_repark_exhausted_dm`]: settlement stays reserved
1560 /// for delivery or the record genuinely ageing out below.
1561 ///
1562 /// Only the *presence-offline* answer cancelled that retry before, which
1563 /// is not a defense at all for the case this whole probe exists to serve
1564 /// — a headless consumer that never polls presence.
1565 ///
1566 /// Bounded by [`WELCOME_WATCHLIST_MAX_AGE_SECS`] measured from
1567 /// `created_at`, the same threshold that stops watching a peer as
1568 /// permanently dead: past it the counter no longer shields the record and
1569 /// normal ageing expires it. This is the welcome's twin of the DM probe's
1570 /// absolute bound (`first_sent_at` vs the outbox absolute lifetime) —
1571 /// without it a peer that never returns keeps a welcome probing at the
1572 /// 600s cap forever.
1573 ///
1574 /// Deliberately narrow to `Timeout`: a `SendFailed`/`TransportUnavailable`
1575 /// failure is evidence about the *carrier*, not about this peer, and
1576 /// keeps ageing the record normally.
1577 fn welcome_probe_repark_permitted(&self, peer_id: &str) -> bool {
1578 let Some(record) = self.welcome_lifecycles.get(peer_id) else {
1579 return false;
1580 };
1581 record.unreachable_parks > 0
1582 && record.created_at
1583 > Utc::now() - ChronoDuration::seconds(WELCOME_WATCHLIST_MAX_AGE_SECS)
1584 }
1585
1586 /// Applies the relay's authoritative "recipient offline" verdict
1587 /// (a `recipient_unreachable`-tagged transport failure) to a welcome
1588 /// lifecycle. Also reached *without* a relay verdict when a probe's
1589 /// confirm timeout resolves while the park counter is still live — the
1590 /// same conclusion inferred rather than relayed, see
1591 /// [`Self::welcome_probe_repark_permitted`].
1592 ///
1593 /// The bridge wire-confirms on socket-write success, so by the time the
1594 /// relay's `DeliveryError` arrives the record is normally already `Sent`
1595 /// — a *false* Sent: the store-less relay dropped the content. Correct
1596 /// it back to `Failed` (emitting `welcome_send_failed` so the app's
1597 /// earlier `welcome_send_succeeded` is superseded), refund the attempt
1598 /// (the peer never saw the frame), and park pending a reachability edge.
1599 /// `SendAttempted` (the failure raced ahead of the wire confirm) gets
1600 /// the same treatment; `Created`/`Failed` records just park. Terminal
1601 /// `Expired` records are left to the online edge, which rebuilds them.
1602 ///
1603 /// The record always keeps a timed retry whose interval escalates per
1604 /// consecutive park. Relay presence is authoritative only for the
1605 /// internet path — with a mesh carrier live the peer may still be
1606 /// reachable over BLE / WiFi-Direct (DORS can pick internet even with
1607 /// mesh present) — but an internet-only device needs the timed track just
1608 /// as much: parking it edge-only left the welcome dependent solely on a
1609 /// presence-online answer, i.e. on the platform's presence-polling
1610 /// cadence, which is the same dead end that stalled parked DMs (see
1611 /// [`Self::park_unreachable_dm`]). The escalation (15s → 600s cap) is
1612 /// what bounds the retry: DORS may keep routing to the internet path,
1613 /// where each round trips another budget-refunded `DeliveryError`, and
1614 /// without escalation that would be an unbounded resend loop into the
1615 /// relay. Every verdict refunds the attempt and pushes the TTL — past the
1616 /// scheduled probe, see the invariant on [`Self::park_welcome_no_carrier`]
1617 /// — so the probes never age the welcome out or burn its real-delivery
1618 /// budget, and the counter resets on any reachability edge
1619 /// (`rearm_welcome_for_peer`).
1620 pub(super) fn apply_recipient_unreachable_failure(
1621 &mut self,
1622 peer_id: &str,
1623 transport_error: Option<&'static str>,
1624 ) -> Result<()> {
1625 // A late relay verdict for a session that has since been proven (the
1626 // welcome was rescued over another path and the peer confirmed) must
1627 // not corrupt the converged lifecycle: without this guard the stale
1628 // DeliveryError flips the record back to Failed, refunds an attempt,
1629 // persists the stale state, and emits welcome_send_failed AFTER the
1630 // app already saw secure_session_established.
1631 if self.confirmed_sessions.contains(peer_id) {
1632 return Ok(());
1633 }
1634 let Some(state) = self.welcome_lifecycles.get(peer_id).map(|r| r.state) else {
1635 return Ok(());
1636 };
1637 match state {
1638 WelcomeDeliveryState::Sent | WelcomeDeliveryState::SendAttempted => {
1639 self.transition_welcome_state(
1640 peer_id,
1641 WelcomeDeliveryState::Failed,
1642 "recipient_unreachable",
1643 )?;
1644 if let Some(record) = self.welcome_lifecycles.get_mut(peer_id) {
1645 record.attempt = record.attempt.saturating_sub(1);
1646 record.last_transport_error = transport_error.map(str::to_string);
1647 }
1648 }
1649 WelcomeDeliveryState::Created | WelcomeDeliveryState::Failed => {
1650 if let Some(record) = self.welcome_lifecycles.get_mut(peer_id) {
1651 record.last_transport_error = transport_error.map(str::to_string);
1652 }
1653 }
1654 WelcomeDeliveryState::Expired => return Ok(()),
1655 }
1656 // Escalating timed retry on every carrier (see the doc comment): the
1657 // ladder, not a carrier guard, is what bounds resends into the relay.
1658 let parks = {
1659 let Some(record) = self.welcome_lifecycles.get_mut(peer_id) else {
1660 return Ok(());
1661 };
1662 record.unreachable_parks = record.unreachable_parks.saturating_add(1);
1663 record.unreachable_parks
1664 };
1665 // parks >= 1 here; shift clamped so 15 << 6 = 960 is the largest
1666 // pre-cap value — no overflow risk.
1667 let retry_in_secs = (WELCOME_NO_CARRIER_RETRY_SECS << (parks - 1).min(6))
1668 .min(WELCOME_UNREACHABLE_RETRY_CAP_SECS);
1669 self.park_welcome_no_carrier(
1670 peer_id,
1671 crate::events::WelcomeReasonCode::PeerUnreachable,
1672 transport_error,
1673 retry_in_secs,
1674 )?;
1675
1676 let Some(snapshot) = self.welcome_lifecycles.get(peer_id).cloned() else {
1677 return Ok(());
1678 };
1679 if let Ok(shared) = lock_shared_state(&self.shared_state) {
1680 shared.emit_event(Event::welcome_send_failed(
1681 peer_id.to_string(),
1682 snapshot.welcome_message.id.as_str().to_string(),
1683 snapshot.group_id.clone(),
1684 snapshot.attempt,
1685 crate::events::WelcomeReasonCode::PeerUnreachable,
1686 snapshot
1687 .last_transport_error
1688 .as_deref()
1689 .map(classify_transport_send_error),
1690 // Retryable, and now always with a scheduled time: the park
1691 // keeps an escalating timed probe on every carrier. The
1692 // reachability edges (presence online / peer discovery) still
1693 // re-arm it sooner when they fire.
1694 true,
1695 snapshot.next_retry_at.map(|at| at.timestamp_millis()),
1696 ));
1697 }
1698 Ok(())
1699 }
1700
1701 /// Records a Welcome send failure and decides whether to retry or expire.
1702 ///
1703 /// `no_carrier` is `true` when the failure was simply that no transport
1704 /// carrier exists yet (vs. a present carrier that failed mid-send). A
1705 /// no-carrier failure must NOT age the Welcome: it neither counts toward
1706 /// the retry budget nor lets the TTL expire it, because the peer is merely
1707 /// unreachable for now and will be retried once a carrier appears. The
1708 /// speculative attempt increment from [`Self::try_send_welcome`] is rolled
1709 /// back so a long offline period leaves the real-delivery budget intact.
1710 pub(super) fn apply_welcome_send_failure(
1711 &mut self,
1712 peer_id: &str,
1713 reason: crate::events::WelcomeReasonCode,
1714 transport_error: Option<&'static str>,
1715 no_carrier: bool,
1716 source_event: &str,
1717 ) -> Result<bool> {
1718 let mut updated = self
1719 .welcome_lifecycles
1720 .get(peer_id)
1721 .cloned()
1722 .ok_or_else(|| Error::Other(format!("Missing welcome lifecycle for {}", peer_id)))?;
1723
1724 if matches!(
1725 updated.state,
1726 WelcomeDeliveryState::Sent | WelcomeDeliveryState::Expired
1727 ) {
1728 return Ok(matches!(updated.state, WelcomeDeliveryState::Sent));
1729 }
1730
1731 // A confirm timeout on a welcome that still holds a live
1732 // unreachable-park counter is a *probe* resolving, not a delivery
1733 // failing — re-park it as another unreachable verdict instead of
1734 // ageing it (see [`Self::welcome_probe_repark_permitted`]).
1735 if !no_carrier
1736 && matches!(reason, crate::events::WelcomeReasonCode::Timeout)
1737 && self.welcome_probe_repark_permitted(peer_id)
1738 {
1739 self.apply_recipient_unreachable_failure(peer_id, transport_error)?;
1740 return Ok(false);
1741 }
1742
1743 let max_attempts = self.config.reliability.retry.max_retries.max(1);
1744 // A no-carrier failure never expires the Welcome: neither the retry
1745 // budget nor the TTL applies while the peer is simply unreachable. Only
1746 // a carrier-backed failure (SendFailed / Timeout) ages it toward expiry.
1747 let should_expire =
1748 !no_carrier && (updated.attempt >= max_attempts || updated.expires_at <= Utc::now());
1749 if should_expire {
1750 let terminal_reason = crate::events::WelcomeReasonCode::RetryExhausted;
1751 {
1752 let record = self.welcome_lifecycles.get_mut(peer_id).ok_or_else(|| {
1753 Error::Other(format!("Missing welcome lifecycle for {}", peer_id))
1754 })?;
1755 record.last_reason_code = Some(terminal_reason);
1756 record.last_transport_error = transport_error.map(str::to_string);
1757 record.next_retry_at = None;
1758 }
1759
1760 if !matches!(updated.state, WelcomeDeliveryState::Failed) {
1761 self.transition_welcome_state(peer_id, WelcomeDeliveryState::Failed, source_event)?;
1762 }
1763 self.transition_welcome_state(
1764 peer_id,
1765 WelcomeDeliveryState::Expired,
1766 "welcome_retry_exhausted",
1767 )?;
1768
1769 let expired_snapshot =
1770 self.welcome_lifecycles
1771 .get(peer_id)
1772 .cloned()
1773 .ok_or_else(|| {
1774 Error::Other(format!("Missing welcome lifecycle for {}", peer_id))
1775 })?;
1776 if let Ok(state) = lock_shared_state(&self.shared_state) {
1777 state.emit_event(Event::welcome_send_failed(
1778 peer_id.to_string(),
1779 expired_snapshot.welcome_message.id.as_str().to_string(),
1780 expired_snapshot.group_id.clone(),
1781 expired_snapshot.attempt,
1782 terminal_reason,
1783 expired_snapshot
1784 .last_transport_error
1785 .as_deref()
1786 .map(classify_transport_send_error),
1787 false,
1788 None,
1789 ));
1790 state.emit_event(Event::welcome_send_expired(
1791 peer_id.to_string(),
1792 expired_snapshot.welcome_message.id.as_str().to_string(),
1793 expired_snapshot.attempt,
1794 terminal_reason,
1795 ));
1796 }
1797 self.abort_pending_session_for_peer(peer_id, terminal_reason);
1798 return Ok(false);
1799 }
1800
1801 // A no-carrier failure (the carrier raced away after the pre-send check,
1802 // or an async transport-failed callback with no carrier) re-checks on the
1803 // slow no-carrier interval rather than the data-plane backoff: there is
1804 // nothing to deliver over until a carrier returns, and `try_send_welcome`
1805 // parks subsequent no-carrier ticks cheaply anyway.
1806 let retry_at = if no_carrier {
1807 Utc::now() + ChronoDuration::seconds(WELCOME_NO_CARRIER_RETRY_SECS)
1808 } else {
1809 let delay_ms = self.compute_welcome_retry_delay_ms(peer_id, updated.attempt);
1810 Utc::now() + ChronoDuration::milliseconds(delay_ms as i64)
1811 };
1812
1813 {
1814 let record = self.welcome_lifecycles.get_mut(peer_id).ok_or_else(|| {
1815 Error::Other(format!("Missing welcome lifecycle for {}", peer_id))
1816 })?;
1817 // Roll back the speculative attempt increment from try_send_welcome
1818 // for a no-carrier failure so the retry budget stays reserved for
1819 // real delivery attempts over an available carrier, and push the TTL
1820 // forward so a never-deliverable Welcome does not age out — the TTL
1821 // clock is carrier-relative, not creation-relative.
1822 if no_carrier {
1823 record.attempt = record.attempt.saturating_sub(1);
1824 record.expires_at =
1825 Utc::now() + ChronoDuration::seconds(WELCOME_LIFECYCLE_TTL_SECS);
1826 }
1827 record.last_reason_code = Some(reason);
1828 record.last_transport_error = transport_error.map(str::to_string);
1829 record.next_retry_at = Some(retry_at);
1830 }
1831
1832 if !matches!(updated.state, WelcomeDeliveryState::Failed) {
1833 self.transition_welcome_state(peer_id, WelcomeDeliveryState::Failed, source_event)?;
1834 } else if let Some(record) = self.welcome_lifecycles.get(peer_id).cloned() {
1835 self.persist_welcome_lifecycle_entry(&record)?;
1836 }
1837
1838 updated = self
1839 .welcome_lifecycles
1840 .get(peer_id)
1841 .cloned()
1842 .ok_or_else(|| Error::Other(format!("Missing welcome lifecycle for {}", peer_id)))?;
1843 if let Ok(state) = lock_shared_state(&self.shared_state) {
1844 state.emit_event(Event::welcome_send_failed(
1845 peer_id.to_string(),
1846 updated.welcome_message.id.as_str().to_string(),
1847 updated.group_id,
1848 updated.attempt,
1849 reason,
1850 updated
1851 .last_transport_error
1852 .as_deref()
1853 .map(classify_transport_send_error),
1854 true,
1855 Some(retry_at.timestamp_millis()),
1856 ));
1857 }
1858 Ok(false)
1859 }
1860
1861 pub(super) fn compute_welcome_retry_delay_ms(&self, peer_id: &str, attempt: u32) -> u64 {
1862 let config = &self.config.reliability.retry;
1863 let capped_attempt = attempt.saturating_sub(1);
1864 let base_ms = if capped_attempt == 0 {
1865 config.initial_delay_ms
1866 } else {
1867 let multiplier = config.backoff_multiplier.powi(capped_attempt as i32);
1868 (config.initial_delay_ms as f64 * multiplier as f64) as u64
1869 }
1870 .min(config.max_delay_ms);
1871
1872 let mut hasher = std::collections::hash_map::DefaultHasher::new();
1873 peer_id.hash(&mut hasher);
1874 attempt.hash(&mut hasher);
1875 Utc::now().timestamp_millis().hash(&mut hasher);
1876 let bucket = (hasher.finish() % 10_000) as f64 / 10_000.0;
1877 let jitter_factor = 1.0 + ((bucket * 2.0 - 1.0) * WELCOME_RETRY_JITTER_RATIO);
1878 let jittered = (base_ms as f64 * jitter_factor).round() as i64;
1879 jittered.max(1) as u64
1880 }
1881
1882 pub(super) fn can_transition_welcome_state(
1883 current: WelcomeDeliveryState,
1884 next: WelcomeDeliveryState,
1885 ) -> bool {
1886 matches!(
1887 (current, next),
1888 (
1889 WelcomeDeliveryState::Created,
1890 WelcomeDeliveryState::SendAttempted
1891 ) | (WelcomeDeliveryState::Created, WelcomeDeliveryState::Expired)
1892 | (
1893 WelcomeDeliveryState::SendAttempted,
1894 WelcomeDeliveryState::Sent
1895 )
1896 | (
1897 WelcomeDeliveryState::SendAttempted,
1898 WelcomeDeliveryState::Failed
1899 )
1900 | (
1901 WelcomeDeliveryState::Failed,
1902 WelcomeDeliveryState::SendAttempted
1903 )
1904 | (WelcomeDeliveryState::Failed, WelcomeDeliveryState::Sent)
1905 | (WelcomeDeliveryState::Failed, WelcomeDeliveryState::Expired)
1906 // Sent is normally a sink, but the relay's DeliveryError is
1907 // authoritative proof that a wire-confirmed frame was dropped
1908 // (store-less relay, recipient offline). That correction is
1909 // the one legal way back out of Sent — see
1910 // `on_transport_send_failed`.
1911 | (WelcomeDeliveryState::Sent, WelcomeDeliveryState::Failed)
1912 )
1913 }
1914
1915 pub(super) fn transition_welcome_state(
1916 &mut self,
1917 peer_id: &str,
1918 next_state: WelcomeDeliveryState,
1919 source_event: &str,
1920 ) -> Result<()> {
1921 let (previous_state, record_snapshot) = {
1922 let record = self.welcome_lifecycles.get_mut(peer_id).ok_or_else(|| {
1923 Error::Other(format!(
1924 "Missing welcome lifecycle for transition: {}",
1925 peer_id
1926 ))
1927 })?;
1928
1929 if record.state == next_state {
1930 return Ok(());
1931 }
1932
1933 if !Self::can_transition_welcome_state(record.state, next_state) {
1934 return Err(Error::Other(format!(
1935 "Illegal welcome lifecycle transition for {}: {} -> {}",
1936 peer_id,
1937 record.state.as_str(),
1938 next_state.as_str()
1939 )));
1940 }
1941
1942 let previous = record.state;
1943 record.state = next_state;
1944 if matches!(
1945 next_state,
1946 WelcomeDeliveryState::Sent | WelcomeDeliveryState::Expired
1947 ) {
1948 record.next_retry_at = None;
1949 }
1950 (previous, record.clone())
1951 };
1952
1953 self.persist_welcome_lifecycle_entry(&record_snapshot)?;
1954 info!(
1955 event = "welcome_lifecycle_transition",
1956 session_or_group_id = %peer_id,
1957 previous_state = previous_state.as_str(),
1958 new_state = next_state.as_str(),
1959 source_event = %source_event,
1960 attempt = record_snapshot.attempt,
1961 "welcome_lifecycle_transition"
1962 );
1963 Ok(())
1964 }
1965
1966 pub(super) fn upsert_welcome_lifecycle(
1967 &mut self,
1968 peer_id: &str,
1969 group_id: &str,
1970 welcome_message: Message,
1971 source_event: &str,
1972 ) -> Result<()> {
1973 if let Some(existing) = self.welcome_lifecycles.get(peer_id) {
1974 if !matches!(
1975 existing.state,
1976 WelcomeDeliveryState::Sent | WelcomeDeliveryState::Expired
1977 ) {
1978 return Err(Error::Other(format!(
1979 "Refusing to overwrite active welcome lifecycle for {} in state {}",
1980 peer_id,
1981 existing.state.as_str()
1982 )));
1983 }
1984 }
1985
1986 let now = Utc::now();
1987 let record = WelcomeLifecycleRecord {
1988 peer_id: peer_id.to_string(),
1989 group_id: group_id.to_string(),
1990 state: WelcomeDeliveryState::Created,
1991 attempt: 0,
1992 unreachable_parks: 0,
1993 welcome_message,
1994 next_retry_at: None,
1995 last_reason_code: None,
1996 last_transport_error: None,
1997 created_at: now,
1998 expires_at: now + ChronoDuration::seconds(WELCOME_LIFECYCLE_TTL_SECS),
1999 };
2000 self.welcome_lifecycles
2001 .insert(peer_id.to_string(), record.clone());
2002 self.persist_welcome_lifecycle_entry(&record)?;
2003 info!(
2004 event = "welcome_lifecycle_transition",
2005 session_or_group_id = %peer_id,
2006 previous_state = "Absent",
2007 new_state = WelcomeDeliveryState::Created.as_str(),
2008 source_event = %source_event,
2009 attempt = 0,
2010 "welcome_lifecycle_transition"
2011 );
2012 Ok(())
2013 }
2014
2015 pub(super) fn map_welcome_reason_code(error: &Error) -> crate::events::WelcomeReasonCode {
2016 SessionStateError::classify(error).to_welcome_reason_code()
2017 }
2018
2019 /// True when a send failed because no transport carrier exists yet, as
2020 /// opposed to a present-but-unhealthy carrier (`SendFailed`) or a
2021 /// sent-but-unconfirmed timeout. A no-carrier failure means the peer is
2022 /// simply unreachable right now, so the Welcome must be kept alive and
2023 /// retried rather than counted against its budget — see
2024 /// [`Self::apply_welcome_send_failure`].
2025 pub(super) fn is_no_carrier_error(error: &Error) -> bool {
2026 matches!(
2027 error,
2028 Error::Transport(offline_protocol_transport::Error::TransportNotAvailable(_))
2029 )
2030 }
2031
2032 pub(super) fn session_ready_context_for_source(source_event: &str) -> MlsOperationContext {
2033 match source_event {
2034 "confirmation_ack_received" | "confirmation_probe_received" | "decrypt_success" => {
2035 MlsOperationContext::Receive
2036 }
2037 "welcome_received" => MlsOperationContext::Welcome,
2038 _ => MlsOperationContext::Send,
2039 }
2040 }
2041
2042 pub(super) fn find_welcome_peer_by_message_id(&self, message_id: &str) -> Option<String> {
2043 self.welcome_lifecycles
2044 .iter()
2045 .find_map(|(peer_id, record)| {
2046 if record.welcome_message.id.as_str() == message_id {
2047 return Some(peer_id.clone());
2048 }
2049 None
2050 })
2051 }
2052
2053 pub(super) fn process_welcome_retry_queue(&mut self) -> Result<()> {
2054 let now = Utc::now();
2055 let timed_out_attempts: Vec<String> = self
2056 .welcome_lifecycles
2057 .iter()
2058 .filter_map(|(peer_id, record)| {
2059 // Skip peers whose session is already confirmed: the welcome is
2060 // delivered, mark_welcome_confirmed should have marked it Sent,
2061 // and re-sending would be wasted bandwidth (defensive against any
2062 // keying mismatch that left the lifecycle non-terminal).
2063 if self.confirmed_sessions.contains(peer_id) {
2064 return None;
2065 }
2066 if matches!(record.state, WelcomeDeliveryState::SendAttempted)
2067 && record.next_retry_at.is_some_and(|retry_at| retry_at <= now)
2068 {
2069 return Some(peer_id.clone());
2070 }
2071 None
2072 })
2073 .take(WELCOME_RETRY_BATCH_SIZE)
2074 .collect();
2075
2076 for peer_id in timed_out_attempts {
2077 let _ = self.apply_welcome_send_failure(
2078 &peer_id,
2079 crate::events::WelcomeReasonCode::Timeout,
2080 Some(SEND_FAIL_REASON_CONFIRM_TIMEOUT),
2081 // A confirm timeout means the Welcome was sent over a carrier;
2082 // this is not a no-carrier failure, so it ages normally.
2083 false,
2084 "welcome_confirm_timeout",
2085 )?;
2086 }
2087
2088 let due_peers: Vec<String> = self
2089 .welcome_lifecycles
2090 .iter()
2091 .filter_map(|(peer_id, record)| {
2092 if self.confirmed_sessions.contains(peer_id) {
2093 return None;
2094 }
2095 // Failed = a normal retry-due Welcome. Created with a due
2096 // next_retry_at = a no-carrier-parked Welcome (see
2097 // `park_welcome_no_carrier`): a fresh Created entry has
2098 // next_retry_at == None, so only parked ones match here.
2099 if matches!(
2100 record.state,
2101 WelcomeDeliveryState::Failed | WelcomeDeliveryState::Created
2102 ) && record.next_retry_at.is_some_and(|retry_at| retry_at <= now)
2103 {
2104 return Some(peer_id.clone());
2105 }
2106 None
2107 })
2108 .take(WELCOME_RETRY_BATCH_SIZE)
2109 .collect();
2110
2111 for peer_id in due_peers {
2112 if let Err(err) = self.try_send_welcome(&peer_id, "welcome_retry") {
2113 warn!(
2114 peer_id = %peer_id,
2115 error = %err,
2116 "Failed to process welcome retry"
2117 );
2118 }
2119 }
2120
2121 Ok(())
2122 }
2123
2124 pub(super) fn has_terminal_welcome_failure(&self, peer_id: &str) -> bool {
2125 self.welcome_lifecycles
2126 .get(peer_id)
2127 .is_some_and(|record| matches!(record.state, WelcomeDeliveryState::Expired))
2128 }
2129}