offline_protocol/protocol/session.rs
1//! Session confirmation, welcome lifecycle, and pending session reconciliation.
2
3use super::{
4 classify_transport_send_error, internal_prefixes, lock_shared_state, send_failure_token,
5 GatewayCarrier, OfflineProtocol, PresenceRescueThrottle, PruneAllowance, RestorableRecord,
6 SessionState, WelcomeDeliveryState, WelcomeLifecycleRecord, CONFIRMATION_PROBE_INTERVAL_SECS,
7 CONFIRMATION_RETRY_INTERVAL_SECS, MAX_REKEY_TRACKED_PEERS, RECONCILIATION_THROTTLE_MS,
8 REKEY_INTERVAL_SECS, SEND_FAIL_REASON_CONFIRM_TIMEOUT, WELCOME_INTERNET_CONFIRM_TIMEOUT_SECS,
9 WELCOME_LIFECYCLE_TTL_SECS, WELCOME_MESH_CONFIRM_TIMEOUT_SECS, WELCOME_NO_CARRIER_RETRY_SECS,
10 WELCOME_PRESENCE_RESCUE_BASE_SECS, WELCOME_PRESENCE_RESCUE_MAX_SECS, WELCOME_RETRY_BATCH_SIZE,
11 WELCOME_RETRY_JITTER_RATIO, WELCOME_UNREACHABLE_RETRY_CAP_SECS, WELCOME_WATCHLIST_MAX_AGE_SECS,
12};
13use crate::events::SecurityWarningCode;
14use crate::mls_observability::MlsOperationContext;
15use crate::protocol::reachability::{Claim, FactSource};
16use crate::{Error, EstablishmentState, Event, Result, SessionStateError};
17use chrono::{Duration as ChronoDuration, Utc};
18use offline_protocol_core::{Message, MessagePriority};
19use offline_protocol_mls::{MlsManager, WelcomeMessage};
20use offline_protocol_transport::TransportType;
21use std::collections::HashSet;
22use std::hash::{Hash, Hasher};
23use std::sync::{Arc, RwLock};
24use std::time::{Duration as StdDuration, Instant};
25use tracing::{debug, info, warn};
26
27impl OfflineProtocol {
28 // ========================================================================
29 // SESSION STATE
30 // ========================================================================
31
32 /// Returns true when persisted state marks this peer session as confirmed.
33 ///
34 /// Uses the in-memory `confirmed_sessions` cache as a fast path to avoid
35 /// hitting persistent storage on every message send. The cache is populated
36 /// when sessions are confirmed and cleared when sessions are invalidated.
37 pub(super) fn is_session_confirmed(&mut self, peer_id: &str) -> Result<bool> {
38 // Fast path: check in-memory cache only — no I/O. The cache is kept in
39 // sync by confirm_session_state() and all invalidation paths (blocking,
40 // stale cleanup, etc.). We deliberately skip the has_mls_session() guard
41 // here because it calls load_group() which hits MlsStorage (10-100ms on
42 // mobile Keychain/Keystore). If the session was wiped externally, the
43 // encrypt call will fail and we handle it there.
44 if self.confirmed_sessions.contains(peer_id) {
45 return Ok(true);
46 }
47
48 let persisted = self
49 .load_session_state_entry(peer_id)?
50 .unwrap_or(SessionState::Pending);
51 if matches!(persisted, SessionState::Confirmed) {
52 if !self.has_mls_session(peer_id)? {
53 warn!(
54 peer_id = %peer_id,
55 "Persisted confirmed state has no matching MLS session; clearing stale state"
56 );
57 self.confirmed_sessions.remove(peer_id);
58 self.clear_confirmation_recovery_tracking(peer_id);
59 self.welcome_lifecycles.remove(peer_id);
60 // The MLS session is gone, so any both-create owner gate for this
61 // peer is stale; clear it (memory + storage) or it would block
62 // `welcome_received` confirmation on the next re-pairing.
63 self.clear_both_create_awaiting_decrypt(peer_id);
64 if let Err(err) = self.clear_session_state_entry(peer_id) {
65 warn!(
66 peer_id = %peer_id,
67 error = %err,
68 "Failed to clear stale persisted session state"
69 );
70 }
71 if let Err(err) = self.clear_welcome_lifecycle_entry(peer_id) {
72 warn!(
73 peer_id = %peer_id,
74 error = %err,
75 "Failed to clear stale persisted welcome lifecycle"
76 );
77 }
78 return Ok(false);
79 }
80 self.confirmed_sessions.insert(peer_id.to_string());
81 return Ok(true);
82 }
83 Ok(false)
84 }
85
86 pub(super) fn has_mls_session(&self, peer_id: &str) -> Result<bool> {
87 let Some(mls) = self.mls_manager.clone() else {
88 return Ok(false);
89 };
90
91 let manager = mls
92 .read()
93 .map_err(|_| Error::Other("MLS lock poisoned".to_string()))?;
94 manager.has_session(peer_id).map_err(Error::Mls)
95 }
96
97 /// Returns the current establishment state for a peer (for API and error reporting).
98 pub(super) fn establishment_state(&self, peer_id: &str) -> Result<EstablishmentState> {
99 if let Some(mls) = &self.mls_manager {
100 let has_session = {
101 let manager = mls
102 .read()
103 .map_err(|_| Error::Other("MLS lock poisoned".to_string()))?;
104 manager.has_session(peer_id).map_err(Error::Mls)?
105 };
106 if has_session {
107 let state = self
108 .load_session_state_entry(peer_id)?
109 .unwrap_or(SessionState::Pending);
110 return Ok(if matches!(state, SessionState::Confirmed) {
111 EstablishmentState::SessionConfirmed
112 } else {
113 EstablishmentState::SessionPending
114 });
115 }
116 }
117
118 let now_ms = Utc::now().timestamp_millis() as u64;
119 if let Some(pkg) = self.pending_key_packages.get(peer_id) {
120 if now_ms < pkg.local_expires_at_ms {
121 return Ok(EstablishmentState::HaveKeyPackage);
122 }
123 }
124 if self.load_peer_key_package_from_storage(peer_id).is_some() {
125 return Ok(EstablishmentState::HaveKeyPackage);
126 }
127 Ok(EstablishmentState::NoKeyPackage)
128 }
129
130 /// Monotonic state transition helper: Pending -> Confirmed only.
131 pub(super) fn confirm_session_state(
132 &mut self,
133 peer_id: &str,
134 source_event: &str,
135 ) -> Result<bool> {
136 if !self.can_confirm_from_source(peer_id, source_event) {
137 warn!(
138 event = "session_confirmation_blocked",
139 session_or_group_id = %peer_id,
140 source_event = %source_event,
141 "session_confirmation_blocked"
142 );
143 return Ok(false);
144 }
145
146 let previous = self.ensure_session_state_entry(peer_id, source_event)?;
147
148 if matches!(previous, SessionState::Confirmed) {
149 self.confirmed_sessions.insert(peer_id.to_string());
150 self.clear_confirmation_recovery_tracking(peer_id);
151 self.clear_both_create_awaiting_decrypt(peer_id);
152 info!(
153 event = "session_state_transition",
154 session_or_group_id = %peer_id,
155 previous_state = "Confirmed",
156 new_state = "Confirmed",
157 source_event = %source_event,
158 "session_state_transition"
159 );
160 return Ok(false);
161 }
162
163 // Persist first, then publish in-memory view.
164 if let Err(err) = self.persist_session_state(peer_id, SessionState::Confirmed, source_event)
165 {
166 self.schedule_confirmation_retry(peer_id, source_event);
167 return Err(err);
168 }
169
170 self.confirmed_sessions.insert(peer_id.to_string());
171 self.clear_confirmation_recovery_tracking(peer_id);
172 // Group-aware proof has arrived (decrypt), so we no longer need to gate
173 // confirmation on decrypt for this both-create peer.
174 self.clear_both_create_awaiting_decrypt(peer_id);
175 // The peer has proved the session, so the outbound Welcome — kept
176 // non-terminal so a lost fragment keeps being retried — is delivered.
177 // Mark it Sent so process_welcome_retry_queue stops re-sending it. Mesh
178 // has no transport-level delivery ack, so this session proof is the mesh
179 // equivalent of on_transport_send_confirmed.
180 self.mark_welcome_confirmed(peer_id, source_event);
181 if source_event != "welcome_received" && source_event != "decrypt_success" {
182 self.maybe_emit_local_session_established(
183 peer_id,
184 Self::session_ready_context_for_source(source_event),
185 );
186 }
187 info!(
188 event = "session_state_transition",
189 session_or_group_id = %peer_id,
190 previous_state = previous.as_str(),
191 new_state = "Confirmed",
192 source_event = %source_event,
193 "session_state_transition"
194 );
195
196 // Replication starts here rather than at the four callers that reach
197 // this transition, because the four callers are exactly the shape of
198 // bug that keeps happening: one of them forgets, and the feature is
199 // silently absent on whichever confirmation path that caller owned.
200 // This is also the only signal that covers all four — the session
201 // lifecycle event does not fire on the probe and ack paths.
202 #[cfg(feature = "data")]
203 self.kick_data_sync(peer_id, source_event);
204
205 Ok(true)
206 }
207
208 /// Marks a non-terminal outbound welcome lifecycle as `Sent` once the peer
209 /// has proved the session (probe / ack / welcome / decrypt). The mesh
210 /// equivalent of [`Self::on_transport_send_confirmed`], which only fires for
211 /// Internet. No-op when there is no welcome lifecycle for `peer_id` (e.g. a
212 /// confirmation keyed by a group id rather than a 1:1 peer) or it is already
213 /// terminal, so it is safe to call from every confirmation path.
214 fn mark_welcome_confirmed(&mut self, peer_id: &str, source_event: &str) {
215 match self.welcome_lifecycles.get(peer_id).map(|r| r.state) {
216 // In flight / retried — fall through and mark Sent below.
217 Some(WelcomeDeliveryState::SendAttempted) | Some(WelcomeDeliveryState::Failed) => {}
218 // Parked, never actually sent (no carrier at send time) for a peer
219 // that converged over some other path. `Created -> Sent` is not a
220 // legal transition and there is nothing in flight to confirm, so drop
221 // the now-moot lifecycle (memory + storage) rather than leaving it to
222 // linger until the session is deleted.
223 Some(WelcomeDeliveryState::Created) => {
224 self.welcome_lifecycles.remove(peer_id);
225 if let Err(err) = self.clear_welcome_lifecycle_entry(peer_id) {
226 warn!(
227 peer_id = %peer_id,
228 error = %err,
229 "Failed to clear parked welcome lifecycle on confirmation"
230 );
231 }
232 return;
233 }
234 // Already terminal (Sent / Expired) or no lifecycle — nothing to do.
235 _ => return,
236 }
237 if let Some(record) = self.welcome_lifecycles.get_mut(peer_id) {
238 record.next_retry_at = None;
239 record.last_reason_code = None;
240 record.last_transport_error = None;
241 }
242 // transition_welcome_state persists the record and logs the transition.
243 if self
244 .transition_welcome_state(peer_id, WelcomeDeliveryState::Sent, source_event)
245 .is_err()
246 {
247 return;
248 }
249 if let Some(snapshot) = self.welcome_lifecycles.get(peer_id).cloned() {
250 if let Ok(state) = lock_shared_state(&self.shared_state) {
251 state.emit_event(Event::welcome_send_succeeded(
252 peer_id.to_string(),
253 snapshot.welcome_message.id.as_str().to_string(),
254 snapshot.group_id,
255 snapshot.attempt,
256 ));
257 }
258 }
259 }
260
261 /// Ensures a session has an explicit persisted state entry.
262 pub(super) fn ensure_session_state_entry(
263 &self,
264 peer_id: &str,
265 source_event: &str,
266 ) -> Result<SessionState> {
267 let existing = self.load_session_state_entry(peer_id)?;
268 if let Some(state) = existing {
269 return Ok(state);
270 }
271
272 self.persist_session_state(peer_id, SessionState::Pending, source_event)?;
273 info!(
274 event = "session_state_transition",
275 session_or_group_id = %peer_id,
276 previous_state = "Absent",
277 new_state = "Pending",
278 source_event = %source_event,
279 "session_state_transition"
280 );
281 Ok(SessionState::Pending)
282 }
283
284 /// Reconstructs runtime confirmation cache from persisted session states.
285 /// `allowance` is the launch's shared advisory pool — this walk's deletes
286 /// are advisory (a dropped record re-bootstraps to `Pending`, which is the
287 /// safe answer), and they add up against the same allowance as the four
288 /// category walks. See `storage::MAX_RESTORE_PRUNE_DELETES`.
289 pub(super) fn restore_session_states_from_manager(
290 &mut self,
291 mls: Arc<RwLock<MlsManager>>,
292 allowance: &mut PruneAllowance,
293 ) -> Result<()> {
294 self.confirmed_sessions.clear();
295
296 let sessions = {
297 let manager = mls
298 .read()
299 .map_err(|_| Error::Other("MLS lock poisoned".to_string()))?;
300 manager.list_sessions()?
301 };
302
303 // The session list itself is deliberately not truncated the way a
304 // container-listed walk is: dropping its tail would silently leave
305 // confirmed peers out of `confirmed_sessions`. What needed bounding was
306 // the *deletes* a bad store makes this walk issue — one device barrier
307 // per peer, with nothing counting them until now.
308 let mut budget = allowance.refusing();
309 for peer_id in sessions {
310 // Before anything that can `continue`: an MLS session with this
311 // peer exists, which is the capability fact, and it holds whether
312 // or not the peer's `session_states` record turns out to be
313 // readable. Deriving it from the record instead would reintroduce
314 // exactly the deletable dependency `encryption_capable_peers`
315 // exists to escape.
316 self.mark_encryption_capable(&peer_id);
317
318 // Restore-path read: a record whose bytes will not decode is
319 // dropped and re-bootstrapped as `Pending` rather than failing
320 // initialization forever. The send path deliberately keeps the
321 // strict loader — see `load_restorable_state_record`.
322 let state = match self.load_session_state_for_restore(&peer_id, Some(&mut budget)) {
323 RestorableRecord::Present(state) => state,
324 RestorableRecord::Absent => self.bootstrap_missing_session_state(&peer_id),
325 // A record is still on disk and could not be read *this*
326 // session. Re-bootstrapping would persist `Pending` straight
327 // over one that may say `Confirmed`, and propagating would fail
328 // `initialize_mls` for as long as that single record stays
329 // unreadable — on an install that then cannot send at all.
330 // Skipping is what leaves the record recoverable: the peer is
331 // simply absent from `confirmed_sessions`, and the send path's
332 // own strict loader (`is_session_confirmed`) still fails closed
333 // for it — and re-reads the record, so a transient failure
334 // heals the moment the store answers again.
335 //
336 // Three consequences worth naming rather than rediscovering,
337 // all of them following from the peer being absent from
338 // `confirmed_sessions` for the run:
339 //
340 // 1. The Welcome machinery gates on
341 // `!confirmed_sessions.contains(peer)` *plus* a live
342 // `welcome_lifecycles` entry (`welcome_pending_peers`,
343 // `resend_unconfirmed_sent_welcome`, the retry ladder). So a
344 // peer whose lifecycle restored as `Sent` while its session
345 // state did not can draw a redundant Welcome re-send.
346 // 2. `flush_restored_confirmed_pending_messages` will not flush
347 // this peer's queued pre-session messages this session. They
348 // stay queued — in memory and on disk — and flush once a
349 // launch can read the record, or once a live confirmation
350 // re-adds the peer.
351 // 3. Tier-2 crypto recovery re-seals a resend only for a peer
352 // in `confirmed_sessions` (`reseal_resend_content`), so a
353 // resend to this peer replays its original ciphertext for
354 // the run rather than re-sealing to the current epoch.
355 //
356 // All three are deferrals on an already-degraded store, and all
357 // three are strictly better than the alternative this replaced
358 // — propagating, which failed `initialize_mls` on every launch
359 // for as long as the one record stayed unreadable, on an
360 // install that could then send nothing at all.
361 RestorableRecord::Unavailable => {
362 warn!(
363 session_or_group_id = %peer_id,
364 "Session state could not be read this session; leaving the record in \
365 place and treating the session as unconfirmed"
366 );
367 continue;
368 }
369 };
370 if matches!(state, SessionState::Confirmed) {
371 self.confirmed_sessions.insert(peer_id.clone());
372 }
373 info!(
374 event = "session_state_restored",
375 session_or_group_id = %peer_id,
376 previous_state = "Pending",
377 new_state = %state.as_str(),
378 source_event = "initialize_mls",
379 "session_state_restored"
380 );
381 }
382
383 if budget.exhausted {
384 warn!(
385 deleted = budget.spent,
386 budget = super::storage::MAX_RESTORE_PRUNE_DELETES,
387 "Session state restore hit its share of the launch delete budget; the \
388 unreadable records left behind are re-bootstrapped as Pending anyway and \
389 dropped on a later launch"
390 );
391 }
392
393 Ok(())
394 }
395
396 /// The state a session with no readable record starts from.
397 ///
398 /// Infallible: the migration write is best effort, like every other
399 /// persistence call on the restore path. It used to propagate, so one
400 /// `StoreFailed` — a full disk, a container the OS has locked — failed
401 /// `initialize_mls` outright and, with `require_encryption` on by default,
402 /// left the install unable to send. `Pending` is the safe answer in memory
403 /// either way, and a launch that can write re-derives the record.
404 pub(super) fn bootstrap_missing_session_state(&self, peer_id: &str) -> SessionState {
405 // Legacy session records without explicit state are treated as Pending.
406 // Recovery is driven by probe/ack reconciliation, never by implicit inference.
407 let restored_state = SessionState::Pending;
408 if let Err(e) = self.persist_session_state(
409 peer_id,
410 restored_state,
411 "initialize_mls_missing_state_migration",
412 ) {
413 warn!(
414 session_or_group_id = %peer_id,
415 error = %e,
416 "Failed to persist the bootstrapped session state; the session is Pending for \
417 this run and the record is re-derived on the next launch"
418 );
419 }
420 info!(
421 event = "session_state_transition",
422 session_or_group_id = %peer_id,
423 previous_state = "Absent",
424 new_state = %restored_state.as_str(),
425 source_event = "initialize_mls_missing_state_migration",
426 "session_state_transition"
427 );
428
429 restored_state
430 }
431
432 // ========================================================================
433 // CONFIRMATION RECOVERY
434 // ========================================================================
435
436 pub(super) fn schedule_confirmation_retry(&mut self, peer_id: &str, source_event: &str) {
437 self.confirmation_retry_due_at
438 .insert(peer_id.to_string(), Utc::now());
439 warn!(
440 event = "session_confirmation_retry_scheduled",
441 session_or_group_id = %peer_id,
442 source_event = %source_event,
443 "session_confirmation_retry_scheduled"
444 );
445 }
446
447 pub(super) fn clear_confirmation_recovery_tracking(&mut self, peer_id: &str) {
448 self.confirmation_retry_due_at.remove(peer_id);
449 self.confirmation_probe_due_at.remove(peer_id);
450 self.confirmation_probe_unreachable_parks.remove(peer_id);
451 }
452
453 /// Fold a relay "recipient unreachable" verdict into the session-confirmation
454 /// probe schedule: escalate this peer's next probe on the same 15s -> 600s
455 /// ladder the welcome lifecycle uses (see
456 /// [`Self::apply_recipient_unreachable_failure`]). Without this, a peer that
457 /// established an MLS session then vanished before confirming is re-probed
458 /// every `CONFIRMATION_PROBE_INTERVAL_SECS` (5s) indefinitely, because
459 /// `kick_pending_session_reconciliation` re-arms it every scan and
460 /// `on_transport_send_failed` otherwise never consults this scheduler. A
461 /// handful of such peers pushes aggregate relay traffic past the
462 /// per-connection rate limit, which disconnects the socket on a loop.
463 ///
464 /// No-op unless the peer is currently a probe target (a `confirmation_probe`
465 /// entry exists, so a probe was just attempted), so ordinary DM failures to
466 /// confirmed peers do not touch the probe schedule. The counter is reset on
467 /// a reachability edge in [`Self::on_peer_presence`] so a returning peer is
468 /// re-probed immediately and its session still converges.
469 pub(super) fn note_confirmation_probe_unreachable(&mut self, peer_id: &str) {
470 if !self.confirmation_probe_due_at.contains_key(peer_id) {
471 return;
472 }
473 let parks = {
474 let counter = self
475 .confirmation_probe_unreachable_parks
476 .entry(peer_id.to_string())
477 .or_insert(0);
478 *counter = counter.saturating_add(1);
479 *counter
480 };
481 // parks >= 1; shift clamped so 15 << 6 = 960 is the largest pre-cap
482 // value (no overflow), then capped — identical to the welcome ladder.
483 let backoff_secs = (WELCOME_NO_CARRIER_RETRY_SECS << (parks - 1).min(6))
484 .min(WELCOME_UNREACHABLE_RETRY_CAP_SECS);
485 self.confirmation_probe_due_at.insert(
486 peer_id.to_string(),
487 Utc::now() + ChronoDuration::seconds(backoff_secs),
488 );
489 }
490
491 pub(super) fn collect_pending_session_peers(&mut self) -> Result<Vec<String>> {
492 let Some(mls) = self.mls_manager.clone() else {
493 return Ok(Vec::new());
494 };
495
496 let sessions = {
497 let manager = mls
498 .read()
499 .map_err(|_| Error::Other("MLS lock poisoned".to_string()))?;
500 manager.list_sessions()?
501 };
502
503 let mut pending = Vec::new();
504 for peer_id in sessions {
505 if !self.is_session_confirmed(&peer_id)? {
506 pending.push(peer_id);
507 }
508 }
509
510 Ok(pending)
511 }
512
513 pub(super) fn send_session_confirmation_probe(&mut self, peer_id: &str, source_event: &str) {
514 match self.send_internal_message(
515 peer_id,
516 internal_prefixes::SESSION_CONFIRM_PROBE.to_string(),
517 MessagePriority::High,
518 ) {
519 Ok(_) => {
520 info!(
521 event = "session_confirmation_probe_sent",
522 session_or_group_id = %peer_id,
523 source_event = %source_event,
524 "session_confirmation_probe_sent"
525 );
526 }
527 Err(err) => {
528 warn!(
529 event = "session_confirmation_probe_failed",
530 session_or_group_id = %peer_id,
531 source_event = %source_event,
532 error = %err,
533 "session_confirmation_probe_failed"
534 );
535 }
536 }
537 }
538
539 /// Throttled wrapper around session reconciliation. Skips the expensive
540 /// `list_sessions()` storage I/O when there is no pending work or when the
541 /// last scan ran recently (within `RECONCILIATION_THROTTLE_MS`).
542 ///
543 /// This is the entry-point called from `process()` and `receive_message()`
544 /// on every tick/poll. Without throttling, the storage call holds the
545 /// protocol Mutex and blocks `sendMessage()` on mobile platforms.
546 pub(super) fn run_throttled_reconciliation(&mut self, source_event: &str) {
547 // Fast path: nothing pending → skip entirely (no storage I/O)
548 let has_pending_work = !self.pending_encrypted_messages.is_empty()
549 || !self.confirmation_probe_due_at.is_empty()
550 || !self.confirmation_retry_due_at.is_empty();
551
552 if !has_pending_work {
553 return;
554 }
555
556 // Throttle: only run the full scan every RECONCILIATION_THROTTLE_MS
557 let now = Instant::now();
558 let throttle = StdDuration::from_millis(RECONCILIATION_THROTTLE_MS);
559 if let Some(last) = self.last_reconciliation_at {
560 if now.duration_since(last) < throttle {
561 return;
562 }
563 }
564
565 self.last_reconciliation_at = Some(now);
566 self.retry_pending_session_confirmations();
567 self.kick_pending_session_reconciliation(source_event);
568 }
569
570 pub(super) fn kick_pending_session_reconciliation(&mut self, source_event: &str) {
571 let now = Utc::now();
572 let pending_peers = match self.collect_pending_session_peers() {
573 Ok(peers) => peers,
574 Err(err) => {
575 warn!(
576 event = "session_confirmation_probe_scan_failed",
577 source_event = %source_event,
578 error = %err,
579 "session_confirmation_probe_scan_failed"
580 );
581 return;
582 }
583 };
584
585 let pending_set: HashSet<String> = pending_peers.iter().cloned().collect();
586 self.confirmation_probe_due_at
587 .retain(|peer, _| pending_set.contains(peer));
588 self.confirmation_probe_unreachable_parks
589 .retain(|peer, _| pending_set.contains(peer));
590
591 for peer_id in pending_peers {
592 let due_at = self
593 .confirmation_probe_due_at
594 .get(&peer_id)
595 .copied()
596 .unwrap_or(now);
597 if due_at > now {
598 continue;
599 }
600
601 self.send_session_confirmation_probe(&peer_id, source_event);
602 self.confirmation_probe_due_at.insert(
603 peer_id,
604 now + ChronoDuration::seconds(CONFIRMATION_PROBE_INTERVAL_SECS),
605 );
606 }
607 }
608
609 /// Triggers a rate-limited re-key of the 1:1 session with `peer_id` after an
610 /// epoch-desync decrypt failure.
611 ///
612 /// The self-healing primitive is a two-part move, matching the unblock
613 /// `session_reset` flow: tear down our *own* stale session **and** advertise
614 /// a `session_reset` key package. The peer, on receiving it, drops its stale
615 /// session, auto-establishes a fresh one from our enclosed key package, and
616 /// Welcomes us back — which we, now session-less, simply *join*.
617 ///
618 /// Tearing down our session alone (without the reset key package) would
619 /// deadlock: the peer's original key package was consumed at first
620 /// establishment, so nothing could rebuild the channel, and the peer (still
621 /// believing its session is healthy) would never re-advertise one. Sending
622 /// the reset key package is what breaks that deadlock.
623 ///
624 /// Deleting the local session is also what makes convergence **symmetric and
625 /// single-round regardless of user-id ordering**. Because we keep no local
626 /// session, the peer's returning Welcome is joined via the fresh-session
627 /// path rather than the greater-id-adopts tiebreaker in
628 /// `handle_welcome_message` — so the lexicographically-smaller detector is no
629 /// longer stranded (the failure mode when the stale session was kept). Both
630 /// orderings are covered end-to-end by
631 /// `test_desync_dm_heals_end_to_end_when_detector_id_is_greater` and
632 /// `test_desync_dm_heals_end_to_end_when_detector_id_is_smaller`.
633 ///
634 /// Rate-limited via `rekey_due_at` to at most one re-key per
635 /// [`REKEY_INTERVAL_SECS`] per peer, so a peer replaying stale-epoch
636 /// ciphertext (or an injected wrong-epoch frame) cannot drive a re-key storm.
637 /// The interval is stamped before the send so a send error still counts
638 /// against the limit, and it is **never reset early** — not even by a
639 /// successful decrypt on the healed session. That is deliberate: a genuine
640 /// re-fork and a replayed old-epoch frame are indistinguishable at this layer
641 /// (both surface as `WrongEpoch`), so clearing the floor on heal would let an
642 /// attacker who lands one legit decrypt between replays defeat the limit and
643 /// force ~one teardown per inbound message. The floor lapses naturally after
644 /// [`REKEY_INTERVAL_SECS`]; during that window Tier 1 (un-ACK + sender
645 /// retries) keeps delivery honest, so the only cost of not resetting is that a
646 /// genuine second desync within the window heals up to one interval later —
647 /// an acceptable trade for a rare event against a bounded-churn guarantee.
648 ///
649 /// **SECURITY — this trigger is unauthenticated, and cannot be made
650 /// otherwise.** `peer_id` is the *wire-claimed* sender. Three facts compose:
651 /// `__MLS_ENC__` is a data-plane prefix, deliberately exempt from the
652 /// Ed25519 + derivation control-message gate (see `DATA_PLANE_PREFIXES`);
653 /// OpenMLS
654 /// validates the framing header — group id, then epoch — *before* any AEAD,
655 /// sender-data decryption or signature check, so `WrongEpoch` /
656 /// `NoPastEpochData` is produced with the sender still entirely unverified;
657 /// and a 1:1 slot id is `session:<a>:<b>` over two public user ids. So the
658 /// classification that lands here is reachable by **anyone who can inject a
659 /// frame** — no key material, no captured ciphertext, no session, no replay
660 /// needed. `test_forged_frame_reaches_session_desync_without_any_key_material`
661 /// builds such a frame from scratch to keep this statement honest. It is
662 /// inherent to MLS framing, not an OpenMLS defect, so no upgrade changes it.
663 ///
664 /// The `SenderIdentityMismatch` gate cannot help: it compares the MLS
665 /// credential to the claimed sender, and that credential only exists once
666 /// `process_message` **succeeds**. Every pre-authentication verdict is
667 /// structurally outside its reach.
668 ///
669 /// Because the trigger cannot be authenticated, the mitigation is that
670 /// acting on it is **harmless**, not that it is trusted:
671 /// - `SessionManager::decrypt_message` requires the envelope to name the
672 /// slot shared with the claimed sender, so one derivable session id
673 /// cannot be aimed at arbitrary peers, and `rekey_due_at` cannot be grown
674 /// with attacker-chosen keys.
675 /// - [`REKEY_INTERVAL_SECS`] bounds this to one re-key per peer per window.
676 /// - The heal destroys nothing: queued plaintext survives a reset (it seals
677 /// at flush time against the rebuilt session), and Tier 2 re-seals
678 /// in-flight resends.
679 /// - Each re-key emits `SecurityWarningCode::SessionRekeyTriggered`, so a
680 /// sustained rate — the signature of injected frames rather than a real
681 /// fork — is visible to the app.
682 ///
683 /// **Residual:** an injector can still force bounded re-key churn on a pair
684 /// (delayed delivery, never lost). Closing that needs a signed
685 /// epoch-corroboration exchange before teardown; a liveness-only probe does
686 /// not work, since a healthy peer answers and we would tear down anyway.
687 /// Also see `docs/state-machines/session-lifecycle.md` ("Desync and heal")
688 /// and `docs/security/threat-model.md` (residual risk R2).
689 pub(super) fn schedule_session_rekey(&mut self, peer_id: &str) {
690 if !self.claim_rekey_slot(peer_id) {
691 return;
692 }
693
694 // A re-key is remotely triggerable and, until now, entirely silent to
695 // the app. A genuine fork produces these occasionally; a sustained rate
696 // for one peer is the signature of injected frames, which an operator
697 // cannot otherwise distinguish.
698 //
699 // Emitted on this path only. The warning names an epoch desync because
700 // that is what reached it, and an application asking for a re-key on
701 // its own schedule has not seen one: reusing the code there would put a
702 // fixed token in front of two different facts and leave an operator
703 // reading their own maintenance as the attack signature this exists to
704 // surface.
705 self.emit_security_warning(
706 peer_id,
707 SecurityWarningCode::SessionRekeyTriggered,
708 "1:1 session torn down and re-advertised after an epoch desync",
709 );
710 if self
711 .drive_session_rekey(peer_id, "after epoch desync")
712 .is_err()
713 {
714 // The advertisement did not leave, and this path discards the
715 // session anyway. What reached it is a fork: every later frame on
716 // that session decrypts to nothing, so keeping it buys a pair that
717 // stays silently broken rather than one that rebuilds when the
718 // peer is next heard from. The application-driven entry makes the
719 // opposite choice, and for the opposite reason: the session it was
720 // asked to rotate is healthy.
721 self.discard_local_session(peer_id);
722 }
723 }
724
725 /// Takes this peer's re-key slot, or reports that the floor has not lapsed.
726 ///
727 /// The floor is what bounds a peer replaying stale-epoch ciphertext to one
728 /// teardown per window rather than one per frame, and it is shared with the
729 /// application-driven entry deliberately: a re-key costs the same teardown
730 /// and re-exchange whoever asked for it, and a caller looping on it would
731 /// otherwise be able to do to a pair exactly what the bound exists to stop
732 /// an attacker doing.
733 pub(super) fn claim_rekey_slot(&mut self, peer_id: &str) -> bool {
734 let now = Utc::now();
735 if let Some(due_at) = self.rekey_due_at.get(peer_id) {
736 if *due_at > now {
737 return false;
738 }
739 }
740 // Bounded like every other map keyed by a wire-claimed id: the desync
741 // that reaches the caller is classified before MLS authenticates
742 // anything.
743 if !self.rekey_due_at.contains_key(peer_id)
744 && self.rekey_due_at.len() >= MAX_REKEY_TRACKED_PEERS
745 {
746 self.rekey_due_at.clear();
747 }
748 self.rekey_due_at.insert(
749 peer_id.to_string(),
750 now + ChronoDuration::seconds(REKEY_INTERVAL_SECS),
751 );
752 true
753 }
754
755 /// Advertises a reset key package and tears the local session down.
756 ///
757 /// This is the whole of what a re-key does on the wire, shared by the
758 /// desync path and by [`OfflineProtocol::rekey_session`] so the two cannot
759 /// drift into meaning different things to a peer. `reason` appears in the
760 /// log line only.
761 ///
762 /// # Why the advertisement goes first
763 ///
764 /// An `Err` here means **nothing was sent and nothing was torn down**.
765 /// [`TransportManager::send`](crate::TransportManager::send) reports an
766 /// error only once no transport has accepted the frame, so a failure is a
767 /// peer that has been told nothing, and a caller rotating a *healthy*
768 /// session must be able to leave it alone: discarding it on the strength
769 /// of a frame that never left strands the pair, with this side holding no
770 /// session, the peer holding one it will never be told to drop, and
771 /// nothing in either that re-opens the exchange.
772 ///
773 /// The order is invisible to a peer. Both steps run under one `&mut self`,
774 /// so no inbound frame can be processed between them, and what a peer sees
775 /// on success is what this path has always sent. The property the old
776 /// order was written for still holds for the same reason: by the time the
777 /// peer's returning Welcome is processed there is no local session left,
778 /// so it is *joined* rather than gated by the greater-id-adopts tiebreaker
779 /// in `handle_welcome_message`, and both sides rebuild in a single round.
780 pub(super) fn drive_session_rekey(&mut self, peer_id: &str, reason: &str) -> Result<()> {
781 if let Err(err) = self.send_key_package_to(peer_id, true) {
782 warn!(
783 event = "session_rekey_send_failed",
784 peer_id = %peer_id,
785 reason = %reason,
786 error = %err,
787 "session_rekey_send_failed"
788 );
789 return Err(err);
790 }
791 self.discard_local_session(peer_id);
792 info!(
793 event = "session_rekey_triggered",
794 peer_id = %peer_id,
795 reason = %reason,
796 "Triggered 1:1 session re-key"
797 );
798 Ok(())
799 }
800
801 /// Drops this side's 1:1 session with `peer_id`, best effort.
802 ///
803 /// Mirrors the unblock `session_reset` flow. A missing session is a no-op,
804 /// and a delete that fails is logged rather than raised: the peer has
805 /// already been sent the reset by every caller that reaches here, and the
806 /// rebuild plus the auto key-package exchange re-arm establishment either
807 /// way.
808 fn discard_local_session(&mut self, peer_id: &str) {
809 if self.has_mls_session(peer_id).unwrap_or(false) {
810 if let Err(err) = self.manual_mls_delete_session(peer_id) {
811 debug!(
812 peer_id = %peer_id,
813 error = %err,
814 "rekey: failed to delete local stale session"
815 );
816 }
817 }
818 }
819
820 /// Gives back a re-key slot claimed for work that did not happen.
821 ///
822 /// Only the application-driven entry may call this, and only when the
823 /// advertisement never left: no teardown happened, so nothing was spent
824 /// and a caller back in range a second later should not be made to wait
825 /// out a window that bounds teardowns. The desync path must **not**, since
826 /// it discards the session whether or not the reset was sent, and that is
827 /// exactly the churn the floor exists to bound.
828 pub(super) fn release_rekey_slot(&mut self, peer_id: &str) {
829 self.rekey_due_at.remove(peer_id);
830 }
831
832 pub(super) fn retry_pending_session_confirmations(&mut self) {
833 let now = Utc::now();
834 let due_peers: Vec<String> = self
835 .confirmation_retry_due_at
836 .iter()
837 .filter_map(|(peer_id, due_at)| {
838 if *due_at <= now {
839 Some(peer_id.clone())
840 } else {
841 None
842 }
843 })
844 .collect();
845
846 for peer_id in due_peers {
847 match self.has_mls_session(&peer_id) {
848 Ok(true) => {}
849 Ok(false) => {
850 self.confirmation_retry_due_at.remove(&peer_id);
851 continue;
852 }
853 Err(err) => {
854 warn!(
855 event = "session_confirmation_retry_scan_failed",
856 session_or_group_id = %peer_id,
857 error = %err,
858 "session_confirmation_retry_scan_failed"
859 );
860 self.confirmation_retry_due_at.insert(
861 peer_id,
862 now + ChronoDuration::seconds(CONFIRMATION_RETRY_INTERVAL_SECS),
863 );
864 continue;
865 }
866 }
867
868 if !self.can_confirm_from_source(&peer_id, "confirmation_retry") {
869 debug!(
870 peer_id = %peer_id,
871 "Skipping confirmation retry until welcome send is at least attempted"
872 );
873 continue;
874 }
875
876 match self.confirm_session_state(&peer_id, "confirmation_retry") {
877 Ok(_) => {
878 let _ = self.flush_pending_messages(&peer_id);
879 self.process_pending_decryption(&peer_id);
880 }
881 Err(err) => {
882 warn!(
883 event = "session_confirmation_retry_failed",
884 session_or_group_id = %peer_id,
885 error = %err,
886 "session_confirmation_retry_failed"
887 );
888 self.confirmation_retry_due_at.insert(
889 peer_id,
890 now + ChronoDuration::seconds(CONFIRMATION_RETRY_INTERVAL_SECS),
891 );
892 }
893 }
894 }
895 }
896
897 /// Marks `peer_id` as a both-create owner gate — we kept our own group and
898 /// must observe a group-aware decrypt before confirming — and persists it so
899 /// an owner restart mid-convergence cannot let a stale plaintext probe/ack
900 /// confirm prematurely. Only writes storage on a genuine insert.
901 pub(super) fn mark_both_create_awaiting_decrypt(&mut self, peer_id: &str) {
902 if self
903 .both_create_awaiting_decrypt
904 .insert(peer_id.to_string())
905 {
906 self.persist_both_create_awaiting_decrypt(peer_id);
907 }
908 }
909
910 /// Clears the both-create owner gate for `peer_id` (it has converged) from
911 /// both memory and storage. Only writes storage on a genuine removal.
912 pub(super) fn clear_both_create_awaiting_decrypt(&mut self, peer_id: &str) {
913 if self.both_create_awaiting_decrypt.remove(peer_id) {
914 self.delete_both_create_awaiting_decrypt(peer_id);
915 }
916 }
917
918 pub(super) fn can_confirm_from_source(&self, peer_id: &str, source_event: &str) -> bool {
919 // Both-create owner: we kept our own group and are waiting for the peer to
920 // prove it adopted *our* group. Only a successful decrypt is group-aware
921 // proof of that; a plaintext probe/ack proves only that the peer holds
922 // some session (possibly its own, pre-adoption group), so it must not
923 // confirm us or we would stop retransmitting and strand the peer.
924 if self.both_create_awaiting_decrypt.contains(peer_id) && source_event != "decrypt_success"
925 {
926 return false;
927 }
928
929 // A successful group decrypt is definitive, group-aware proof that the
930 // peer adopted our 1:1 group. `decrypt_success` is raised only inside
931 // `DecryptResult::Success` (see message_dispatch.rs): ciphertext on the
932 // single stored `session:<a>:<b>` group actually decrypted, which is
933 // possible only if the peer is a member. It must therefore confirm the
934 // session regardless of our *local* Welcome's delivery state. Gating it on
935 // a still-active local Welcome (the lifecycle match below) strands a
936 // both-create owner whose own Welcome timed out to `Failed`/`Expired` on a
937 // lossy or asymmetric BLE link (the Android↔iOS case): the owner decrypts
938 // the peer's messages but never confirms, so every outbound send fails
939 // `SessionNotReady` — it can receive but never reply. The both-create gate
940 // above is preserved, so a both-create owner is still restricted to *only*
941 // decrypt_success (a plaintext probe/ack can never confirm it).
942 if source_event == "decrypt_success" {
943 return true;
944 }
945
946 if !matches!(
947 source_event,
948 "confirmation_ack_received" | "confirmation_probe_received" | "confirmation_retry"
949 ) {
950 return true;
951 }
952
953 match self.welcome_lifecycles.get(peer_id) {
954 Some(record) => matches!(
955 record.state,
956 WelcomeDeliveryState::Sent | WelcomeDeliveryState::SendAttempted
957 ),
958 None => matches!(
959 source_event,
960 // Compatibility path for sessions created before welcome lifecycle
961 // persistence existed. A plaintext probe/ack/retry can confirm here
962 // (it is the only evidence such a legacy session has); a group-aware
963 // decrypt already returned true above and never reaches this arm.
964 "confirmation_ack_received" | "confirmation_probe_received" | "confirmation_retry"
965 ),
966 }
967 }
968
969 pub(super) fn abort_pending_session_for_peer(
970 &mut self,
971 peer_id: &str,
972 reason: crate::events::WelcomeReasonCode,
973 ) {
974 self.drop_pending_queue_for_peer(
975 peer_id,
976 &format!("Welcome delivery failed: {}", reason.as_str()),
977 );
978 if let Ok(state) = lock_shared_state(&self.shared_state) {
979 state.emit_event(Event::secure_session_failed(
980 peer_id.to_string(),
981 format!("Welcome delivery failed: {}", reason.as_str()),
982 ));
983 }
984 }
985
986 pub(super) fn maybe_emit_local_session_established(
987 &self,
988 peer_id: &str,
989 context: MlsOperationContext,
990 ) {
991 let Some(record) = self.welcome_lifecycles.get(peer_id) else {
992 return;
993 };
994 if !matches!(record.state, WelcomeDeliveryState::Sent) {
995 return;
996 }
997 self.emit_mls_session_ready(peer_id, &record.group_id, context);
998 if let Ok(state) = lock_shared_state(&self.shared_state) {
999 state.emit_event(Event::secure_session_established(
1000 peer_id.to_string(),
1001 record.group_id.clone(),
1002 record.group_id.starts_with("session:"),
1003 true,
1004 ));
1005 }
1006 }
1007
1008 // ========================================================================
1009 // WELCOME LIFECYCLE
1010 // ========================================================================
1011
1012 pub(super) fn send_welcome_message(
1013 &mut self,
1014 recipient: &str,
1015 welcome: &WelcomeMessage,
1016 ) -> Result<bool> {
1017 let serialized =
1018 serde_json::to_string(welcome).map_err(|e| Error::Serialization(e.to_string()))?;
1019 let content = format!("{}{}", internal_prefixes::WELCOME, serialized);
1020 let mut message =
1021 self.create_message(recipient, content, Some(MessagePriority::High), None)?;
1022 self.sign_control_message(&mut message)?;
1023 let group_id = welcome.group_id.as_str().to_string();
1024
1025 self.upsert_welcome_lifecycle(recipient, &group_id, message, "welcome_created")?;
1026 self.try_send_welcome(recipient, "welcome_initial_send")
1027 }
1028
1029 /// Parks a Welcome that currently has no transport carrier: refresh the
1030 /// carrier-relative TTL and schedule a slow re-check, WITHOUT consuming a
1031 /// retry attempt, transitioning state, or emitting a `WelcomeSendAttempted`
1032 /// event. A no-carrier send is a guaranteed failure, so re-attempting it at
1033 /// the data-plane retry rate only churns storage I/O and app events while a
1034 /// device is offline. The lifecycle stays in its current non-terminal state
1035 /// (Created on the first send, Failed on a later retry) so the retry queue
1036 /// re-polls it on [`WELCOME_NO_CARRIER_RETRY_SECS`]; `on_neighbor_discovered`
1037 /// re-arms it immediately when a carrier surfaces the peer. Returns
1038 /// `Ok(false)` (not sent).
1039 ///
1040 /// **Invariant: the refreshed TTL always outlives the retry being
1041 /// scheduled.** [`Self::try_send_welcome`] checks `expires_at` *before*
1042 /// running a due retry, so a record whose TTL lapsed while it waited gets
1043 /// expired terminally (`welcome_send_expired` + `secure_session_failed`) by
1044 /// the very timer meant to recover it. That is not hypothetical at the far
1045 /// end of the unreachable ladder in
1046 /// [`Self::apply_recipient_unreachable_failure`]: it reaches 480s at six
1047 /// consecutive parks and [`WELCOME_UNREACHABLE_RETRY_CAP_SECS`] (600s)
1048 /// beyond that, both above the 300s [`WELCOME_LIFECYCLE_TTL_SECS`]. A full
1049 /// extra interval of slack keeps the probe's own outcome — a verdict
1050 /// re-park or a confirm timeout — inside the window too. No-op for the 15s
1051 /// no-carrier park, whose interval is far below the TTL.
1052 fn park_welcome_no_carrier(
1053 &mut self,
1054 peer_id: &str,
1055 reason: crate::events::WelcomeReasonCode,
1056 transport_error: Option<&'static str>,
1057 retry_in_secs: i64,
1058 ) -> Result<bool> {
1059 let snapshot = {
1060 let Some(record) = self.welcome_lifecycles.get_mut(peer_id) else {
1061 return Ok(false);
1062 };
1063 let now = Utc::now();
1064 record.next_retry_at = Some(now + ChronoDuration::seconds(retry_in_secs));
1065 record.expires_at = now
1066 + ChronoDuration::seconds(
1067 WELCOME_LIFECYCLE_TTL_SECS.max(retry_in_secs.saturating_mul(2)),
1068 );
1069 record.last_reason_code = Some(reason);
1070 record.last_transport_error = transport_error.map(str::to_string);
1071 record.clone()
1072 };
1073 self.persist_welcome_lifecycle_entry(&snapshot)?;
1074 debug!(
1075 peer_id = %peer_id,
1076 state = snapshot.state.as_str(),
1077 retry_in_secs,
1078 "Welcome parked: no transport carrier, re-checking on slow interval"
1079 );
1080 Ok(false)
1081 }
1082
1083 pub(super) fn try_send_welcome(&mut self, peer_id: &str, source_event: &str) -> Result<bool> {
1084 let now = Utc::now();
1085 let mut record = self
1086 .welcome_lifecycles
1087 .get(peer_id)
1088 .cloned()
1089 .ok_or_else(|| Error::Other(format!("Missing welcome lifecycle for {}", peer_id)))?;
1090
1091 if matches!(record.state, WelcomeDeliveryState::Sent) {
1092 return Ok(true);
1093 }
1094 if matches!(record.state, WelcomeDeliveryState::Expired) {
1095 return Ok(false);
1096 }
1097
1098 // Only honor the TTL when a carrier actually exists: a Welcome that has
1099 // never had a deliverable transport must not age out.
1100 let carrier_available = !self.transport_manager.get_available_transports().is_empty();
1101 if carrier_available && record.expires_at <= now {
1102 self.transition_welcome_state(peer_id, WelcomeDeliveryState::Expired, source_event)?;
1103 if let Ok(state) = lock_shared_state(&self.shared_state) {
1104 state.emit_event(Event::welcome_send_expired(
1105 peer_id.to_string(),
1106 record.welcome_message.id.as_str().to_string(),
1107 record.attempt,
1108 crate::events::WelcomeReasonCode::RetryExhausted,
1109 ));
1110 }
1111 self.abort_pending_session_for_peer(
1112 peer_id,
1113 crate::events::WelcomeReasonCode::RetryExhausted,
1114 );
1115 return Ok(false);
1116 }
1117
1118 // No carrier at all: a send is guaranteed to fail, so do NOT burn the
1119 // speculative attempt increment, the SendAttempted/Failed transitions
1120 // (two persisted state changes + two info logs), and a
1121 // `WelcomeSendAttempted` event on it every retry tick. Park the
1122 // lifecycle on a slow carrier-relative interval instead and re-check
1123 // later; `on_neighbor_discovered` re-arms it immediately when a carrier
1124 // surfaces the peer. This keeps an offline device quiet rather than
1125 // spinning storage I/O + app events at the data-plane retry rate.
1126 if !carrier_available {
1127 return self.park_welcome_no_carrier(
1128 peer_id,
1129 crate::events::WelcomeReasonCode::TransportUnavailable,
1130 None,
1131 WELCOME_NO_CARRIER_RETRY_SECS,
1132 );
1133 }
1134
1135 record.attempt = record.attempt.saturating_add(1);
1136 self.welcome_lifecycles
1137 .insert(peer_id.to_string(), record.clone());
1138 self.persist_welcome_lifecycle_entry(&record)?;
1139 self.transition_welcome_state(peer_id, WelcomeDeliveryState::SendAttempted, source_event)?;
1140
1141 if let Ok(state) = lock_shared_state(&self.shared_state) {
1142 state.emit_event(Event::welcome_send_attempted(
1143 peer_id.to_string(),
1144 record.welcome_message.id.as_str().to_string(),
1145 record.group_id.clone(),
1146 record.attempt,
1147 ));
1148 }
1149
1150 match self.transport_manager.send(&record.welcome_message) {
1151 Ok(()) => {
1152 let transport_used = self.transport_manager.current_transport();
1153 let mut updated =
1154 self.welcome_lifecycles
1155 .get(peer_id)
1156 .cloned()
1157 .ok_or_else(|| {
1158 Error::Other(format!("Missing welcome lifecycle for {}", peer_id))
1159 })?;
1160
1161 // No transport's send() proves the peer reassembled and joined
1162 // the group, so keep the lifecycle NON-TERMINAL until an explicit
1163 // confirmation arrives and let the retry queue re-send on timeout.
1164 //
1165 // - Internet send() only enqueues for platform polling; the
1166 // confirmation is on_transport_send_confirmed.
1167 // - A BLE / WiFi-Direct GATT WRITE_TYPE_NO_RESPONSE returning Ok
1168 // only means the local stack accepted the bytes. A single lost
1169 // fragment of the multi-fragment Welcome would otherwise leave
1170 // the sender believing the session exists while the peer holds
1171 // undecryptable ciphertext, with NO retry — permanently
1172 // bricking the pair. The mesh confirmation is the peer proving
1173 // the session (its probe / ack / welcome / decrypt), which marks
1174 // the welcome Sent via confirm_session_state.
1175 let confirm_timeout_secs =
1176 if matches!(transport_used, Some(TransportType::Internet)) {
1177 WELCOME_INTERNET_CONFIRM_TIMEOUT_SECS
1178 } else {
1179 WELCOME_MESH_CONFIRM_TIMEOUT_SECS
1180 };
1181 updated.next_retry_at =
1182 Some(Utc::now() + ChronoDuration::seconds(confirm_timeout_secs));
1183 updated.last_reason_code = None;
1184 updated.last_transport_error = None;
1185 self.welcome_lifecycles
1186 .insert(peer_id.to_string(), updated.clone());
1187 self.persist_welcome_lifecycle_entry(&updated)?;
1188
1189 // Seed the confirmation-probe scheduler for this pending peer so
1190 // the SENDER actively reconciles instead of waiting passively.
1191 //
1192 // The Welcome is now in flight but unconfirmed. The sender's only
1193 // built-in convergence path would otherwise be the receiver's
1194 // single proactive encrypted confirm on first Welcome receipt — a
1195 // single point of failure with no retry (a lost fragment, a
1196 // not-yet-ready encryptor, or the retransmit/owner_keep path on
1197 // the receiver all strand it). By marking a probe due now,
1198 // `run_throttled_reconciliation` (whose `has_pending_work` gate is
1199 // otherwise false on the sender, since it has no pending encrypted
1200 // messages) starts emitting `SESSION_CONFIRM_PROBE`s. The peer —
1201 // which holds the session — replies with `SESSION_CONFIRM_ACK`,
1202 // confirming us and marking this Welcome `Sent`. The entry is
1203 // dropped once confirmed (`clear_confirmation_recovery_tracking`)
1204 // or once the peer is no longer pending (`kick`'s retain), so this
1205 // adds no steady-state work after convergence.
1206 self.confirmation_probe_due_at
1207 .insert(peer_id.to_string(), Utc::now());
1208
1209 Ok(false)
1210 }
1211 Err(err) => {
1212 let no_carrier = Self::is_no_carrier_error(&err);
1213 let reason = Self::map_welcome_reason_code(&err);
1214 // Classified, not rendered: the transport layer interpolates
1215 // the peer into `PeerNotReachable`, and this value is both
1216 // persisted with the lifecycle record and emitted on
1217 // `WelcomeSendFailed.transport_error`, where the scrubber
1218 // hashes `peer_id` beside it. The full error is logged above.
1219 self.apply_welcome_send_failure(
1220 peer_id,
1221 reason,
1222 Some(send_failure_token(&err)),
1223 no_carrier,
1224 source_event,
1225 )
1226 }
1227 }
1228 }
1229
1230 /// Re-arms a stalled or expired outbound Welcome when a peer becomes
1231 /// reachable again, so a session that never converged while the peer was
1232 /// offline gets a fresh delivery attempt over the now-available carrier.
1233 ///
1234 /// No-op when the peer's session is already confirmed, when there is no
1235 /// Welcome lifecycle, or when the Welcome is already `Sent` or still
1236 /// in-flight (`SendAttempted`). An `Expired` lifecycle is rebuilt from its
1237 /// retained Welcome message — the MLS group is not torn down on expiry, so
1238 /// the stored Welcome is still valid to re-send — while a `Created`/`Failed`
1239 /// lifecycle has its budget and TTL reset and is retried immediately.
1240 pub(super) fn rearm_welcome_for_peer(&mut self, peer_id: &str, source_event: &str) {
1241 if self.confirmed_sessions.contains(peer_id) {
1242 return;
1243 }
1244 let Some(record) = self.welcome_lifecycles.get(peer_id).cloned() else {
1245 return;
1246 };
1247 match record.state {
1248 WelcomeDeliveryState::Sent | WelcomeDeliveryState::SendAttempted => {}
1249 WelcomeDeliveryState::Created | WelcomeDeliveryState::Failed => {
1250 if let Some(entry) = self.welcome_lifecycles.get_mut(peer_id) {
1251 entry.attempt = 0;
1252 // A reachability edge resets the unreachable-park
1253 // escalation: the next relay verdict starts the interval
1254 // ladder over from its base.
1255 entry.unreachable_parks = 0;
1256 entry.next_retry_at = Some(Utc::now());
1257 entry.last_reason_code = None;
1258 entry.last_transport_error = None;
1259 entry.expires_at =
1260 Utc::now() + ChronoDuration::seconds(WELCOME_LIFECYCLE_TTL_SECS);
1261 }
1262 if let Err(err) = self.try_send_welcome(peer_id, source_event) {
1263 warn!(
1264 peer_id = %peer_id,
1265 error = %err,
1266 "Failed to re-arm welcome on peer reachability"
1267 );
1268 }
1269 }
1270 WelcomeDeliveryState::Expired => {
1271 // Expired is terminal in the lifecycle state machine, so rebuild
1272 // from the retained Welcome message (upsert permits overwrite
1273 // from Expired) before re-sending.
1274 let welcome_message = record.welcome_message.clone();
1275 let group_id = record.group_id.clone();
1276 if let Err(err) =
1277 self.upsert_welcome_lifecycle(peer_id, &group_id, welcome_message, source_event)
1278 {
1279 warn!(
1280 peer_id = %peer_id,
1281 error = %err,
1282 "Failed to rebuild expired welcome on peer reachability"
1283 );
1284 return;
1285 }
1286 if let Err(err) = self.try_send_welcome(peer_id, source_event) {
1287 warn!(
1288 peer_id = %peer_id,
1289 error = %err,
1290 "Failed to re-arm expired welcome on peer reachability"
1291 );
1292 }
1293 }
1294 }
1295 }
1296
1297 /// Ingests an authoritative per-peer presence signal (the internet relay's
1298 /// `PresenceStatusWithLastSeen` answer, bridged by the platform layer).
1299 ///
1300 /// `online` drives the same reachability machinery as a transport-level
1301 /// discovery — outbox flush, welcome re-arm, auto key exchange — plus a
1302 /// rescue for the store-less-relay hole: a welcome whose wire send was
1303 /// confirmed (`Sent`) while the peer was actually offline was dropped by
1304 /// the relay (it forwards or pushes, never stores), and
1305 /// `Self::rearm_welcome_for_peer` deliberately never touches `Sent`.
1306 /// The peer being provably online is the safe moment to rebuild and
1307 /// re-send it: if the original did land, the receiver dedups by message
1308 /// id and the confirmation probe resolves the lifecycle.
1309 ///
1310 /// `offline` parks retry-pending welcomes pending a reachability edge,
1311 /// without burning budget. In-flight records (`SendAttempted`) are left
1312 /// to their confirm deadline — a `DeliveryError`-driven failure or the
1313 /// confirm timeout resolves them, after which the next offline tick parks.
1314 ///
1315 /// The welcome re-arm/re-send on the online edge is throttled per peer
1316 /// with exponential backoff (`WELCOME_PRESENCE_RESCUE_BASE_SECS` ..
1317 /// `WELCOME_PRESENCE_RESCUE_MAX_SECS`): presence answers arrive on the
1318 /// platform's ~20s watch tick, and a peer that is online but can never
1319 /// prove the session (stale key package after a reinstall, incompatible
1320 /// version) must not be re-sent the multi-frame MLS welcome on every
1321 /// tick forever. A throttled online tick still flushes queued data-plane
1322 /// traffic for the peer.
1323 ///
1324 /// Emits `presence_updated` naming the carrier that answered — this
1325 /// function is the gateway-sourced half of the unified stream and must
1326 /// only be fed a gateway's own observation (bridged via
1327 /// `internet_peer_presence` or `reticulum_peer_presence`); peer-sent
1328 /// `__PRESENCE__` self-reports are handled in `message_dispatch` and emit
1329 /// `source: Peer`. Self, blocked, or empty peer ids are dropped entirely:
1330 /// an app waiting on `presence_updated` for a blocked peer will never see
1331 /// it.
1332 pub fn on_peer_presence(&mut self, peer_id: &str, online: bool, last_seen_ms: Option<i64>) {
1333 self.on_peer_presence_via(peer_id, online, last_seen_ms, GatewayCarrier::Internet);
1334 }
1335
1336 /// [`Self::on_peer_presence`], for whichever gateway answered.
1337 ///
1338 /// The carrier is threaded rather than assumed because every effect below
1339 /// is carrier-scoped and silently wrong under the wrong one: the fact is
1340 /// recorded against the carrier that made the claim, the un-park flush is
1341 /// pinned to the carrier that just proved it can reach the peer, and the
1342 /// event names the carrier so an app filtering on relay-observed presence
1343 /// still can. A Reticulum gateway's answer forcing a send over the
1344 /// Internet transport would be a reachability proof about one carrier
1345 /// spent on another.
1346 pub fn on_peer_presence_via(
1347 &mut self,
1348 peer_id: &str,
1349 online: bool,
1350 last_seen_ms: Option<i64>,
1351 carrier: GatewayCarrier,
1352 ) {
1353 if peer_id.is_empty() || peer_id == self.local_id || self.is_user_blocked(peer_id) {
1354 return;
1355 }
1356 // A presence answer is a claim about this recipient on the carrier
1357 // that answered, so it is recorded as one. It supersedes an earlier
1358 // verdict for that carrier: both answer the same question and this is
1359 // the later answer. Presence facts decay faster than verdicts, since
1360 // this is a third party's report about someone else's connection.
1361 self.reachability.record(
1362 peer_id,
1363 carrier.transport(),
1364 if online {
1365 Claim::Reachable
1366 } else {
1367 Claim::Unreachable
1368 },
1369 FactSource::GatewayPresence,
1370 std::time::Instant::now(),
1371 );
1372 if online {
1373 // Carrier-scoped reachability proof: un-park and re-drive the
1374 // peer's DMs over that carrier before anything else
1375 // (see `flush_outbox_for_peer_via` — this also cancels
1376 // unanswerable mesh-probe ACKs that would otherwise hold the
1377 // messages hostage past this edge). The rescue branch's inner
1378 // flush then finds successfully re-driven entries awaiting
1379 // fresh ACKs and leaves them alone; entries whose forced send
1380 // *failed* are picked up again, so the inner flush keeps the
1381 // carrier override — DORS must not re-route them into the
1382 // mesh void with the park counter already cleared.
1383 self.flush_outbox_for_peer_via(peer_id, Some(carrier.transport()));
1384 if self.welcome_rescue_permitted(peer_id) {
1385 self.on_neighbor_discovered_via(peer_id, Some(carrier.transport()));
1386 self.resend_unconfirmed_sent_welcome(peer_id, "peer_presence_online");
1387 self.note_welcome_rescue_attempt(peer_id);
1388 }
1389 // A reachability edge resets the session-confirmation probe's
1390 // unreachable backoff: clear the escalation counter and mark a probe
1391 // due now, so a peer that went unreachable while unconfirmed is
1392 // re-probed immediately and its session converges rather than
1393 // waiting out the (up to 600s) backoff.
1394 if self.confirmation_probe_due_at.contains_key(peer_id) {
1395 self.confirmation_probe_unreachable_parks.remove(peer_id);
1396 self.confirmation_probe_due_at
1397 .insert(peer_id.to_string(), Utc::now());
1398 }
1399 } else {
1400 self.park_welcome_peer_unreachable(peer_id);
1401 }
1402
1403 let status = if online {
1404 crate::events::PresenceStatus::Online
1405 } else {
1406 crate::events::PresenceStatus::Offline
1407 };
1408 self.emit_event(Event::presence_updated_from(
1409 peer_id.to_string(),
1410 status,
1411 Utc::now().timestamp_millis(),
1412 last_seen_ms,
1413 carrier.presence_source(),
1414 ));
1415 }
1416
1417 /// Peers with a welcome lifecycle whose session has not been proven —
1418 /// the internet presence watch set. Includes `Sent` records (wire-
1419 /// confirmed but never session-confirmed): only a presence signal can
1420 /// rescue those over a store-less relay.
1421 /// Lifecycles older than `WELCOME_WATCHLIST_MAX_AGE_SECS` are excluded:
1422 /// a permanently-dead peer must not occupy watch-rotation slots (and keep
1423 /// its parked record alive via `expires_at` pushes) forever.
1424 pub fn welcome_pending_peers(&self) -> Vec<String> {
1425 let watch_cutoff = Utc::now() - ChronoDuration::seconds(WELCOME_WATCHLIST_MAX_AGE_SECS);
1426 self.welcome_lifecycles
1427 .iter()
1428 .filter(|(peer_id, record)| {
1429 !self.confirmed_sessions.contains(*peer_id) && record.created_at > watch_cutoff
1430 })
1431 .map(|(peer_id, _)| peer_id.clone())
1432 .collect()
1433 }
1434
1435 /// Rebuilds and re-sends a `Sent`-but-session-unconfirmed welcome (see
1436 /// [`Self::on_peer_presence`]). No-op for any other state.
1437 fn resend_unconfirmed_sent_welcome(&mut self, peer_id: &str, source_event: &str) {
1438 if self.confirmed_sessions.contains(peer_id) {
1439 return;
1440 }
1441 let Some(record) = self.welcome_lifecycles.get(peer_id) else {
1442 return;
1443 };
1444 if !matches!(record.state, WelcomeDeliveryState::Sent) {
1445 return;
1446 }
1447 let welcome_message = record.welcome_message.clone();
1448 let group_id = record.group_id.clone();
1449 if let Err(err) =
1450 self.upsert_welcome_lifecycle(peer_id, &group_id, welcome_message, source_event)
1451 {
1452 warn!(
1453 peer_id = %peer_id,
1454 error = %err,
1455 "Failed to rebuild unconfirmed sent welcome on presence"
1456 );
1457 return;
1458 }
1459 if let Err(err) = self.try_send_welcome(peer_id, source_event) {
1460 warn!(
1461 peer_id = %peer_id,
1462 error = %err,
1463 "Failed to re-send unconfirmed welcome on presence"
1464 );
1465 }
1466 }
1467
1468 /// True when a presence-online rescue for this peer is currently allowed
1469 /// by the per-peer backoff. Trivially true (and the throttle entry is
1470 /// dropped) once the peer has no unconfirmed welcome — there is nothing
1471 /// left to throttle.
1472 fn welcome_rescue_permitted(&mut self, peer_id: &str) -> bool {
1473 let pending = self.welcome_lifecycles.contains_key(peer_id)
1474 && !self.confirmed_sessions.contains(peer_id);
1475 if !pending {
1476 self.welcome_presence_rescue.remove(peer_id);
1477 return true;
1478 }
1479 self.welcome_presence_rescue
1480 .get(peer_id)
1481 .is_none_or(|throttle| throttle.next_allowed_at <= Utc::now())
1482 }
1483
1484 /// Records that a presence-online tick ran the rescue actions for a peer
1485 /// that still has an unconfirmed welcome, doubling the wait before the
1486 /// next one (base 40s, capped at 10 minutes). The counter is per
1487 /// convergence attempt: the entry is dropped (via
1488 /// [`Self::welcome_rescue_permitted`]) as soon as the session confirms
1489 /// or the lifecycle goes away.
1490 fn note_welcome_rescue_attempt(&mut self, peer_id: &str) {
1491 let pending = self.welcome_lifecycles.contains_key(peer_id)
1492 && !self.confirmed_sessions.contains(peer_id);
1493 if !pending {
1494 self.welcome_presence_rescue.remove(peer_id);
1495 return;
1496 }
1497 let throttle = self
1498 .welcome_presence_rescue
1499 .entry(peer_id.to_string())
1500 .or_insert(PresenceRescueThrottle {
1501 next_allowed_at: Utc::now(),
1502 rescues: 0,
1503 });
1504 // Shift is clamped to 8, so 40 << 8 = 10240 max before the cap —
1505 // no overflow risk.
1506 let backoff_secs = (WELCOME_PRESENCE_RESCUE_BASE_SECS << throttle.rescues.min(8))
1507 .min(WELCOME_PRESENCE_RESCUE_MAX_SECS);
1508 throttle.next_allowed_at = Utc::now() + ChronoDuration::seconds(backoff_secs);
1509 throttle.rescues = throttle.rescues.saturating_add(1);
1510 }
1511
1512 /// Parks a welcome pending a peer-reachability edge: reason
1513 /// `PeerUnreachable`, **no timed retry**. The carrier (relay socket) is
1514 /// healthy — only this peer is unreachable on it — so the *data-plane*
1515 /// retry this cancels would re-send the welcome into another
1516 /// `DeliveryError` while burning the real-delivery budget that a
1517 /// carrier-backed failure already charged. Recovery is edge-driven
1518 /// instead: `on_peer_presence(online)` and `on_neighbor_discovered` re-arm
1519 /// via `rearm_welcome_for_peer`, and the peer stays on the presence
1520 /// watchlist (`welcome_pending_peers`) so the platform keeps polling for
1521 /// that edge. The TTL is pushed like a no-carrier park: an unreachable
1522 /// peer must not age the welcome out.
1523 ///
1524 /// Its caller is responsible for not applying this to a record already
1525 /// holding an escalating unreachable probe — see
1526 /// [`Self::park_welcome_peer_unreachable`], which is what distinguishes
1527 /// the budget-burning data-plane retry cancelled here from the
1528 /// budget-refunded probe that must survive.
1529 fn park_welcome_awaiting_peer(&mut self, peer_id: &str) -> Result<()> {
1530 let snapshot = {
1531 let Some(record) = self.welcome_lifecycles.get_mut(peer_id) else {
1532 return Ok(());
1533 };
1534 record.next_retry_at = None;
1535 record.expires_at = Utc::now() + ChronoDuration::seconds(WELCOME_LIFECYCLE_TTL_SECS);
1536 record.last_reason_code = Some(crate::events::WelcomeReasonCode::PeerUnreachable);
1537 record.clone()
1538 };
1539 self.persist_welcome_lifecycle_entry(&snapshot)?;
1540 debug!(
1541 peer_id = %peer_id,
1542 state = snapshot.state.as_str(),
1543 "Welcome parked: peer unreachable on carrier, awaiting reachability edge"
1544 );
1545 Ok(())
1546 }
1547
1548 /// Parks a retry-pending welcome for a peer the relay reports offline.
1549 /// Only `Created`/`Failed` records park: `SendAttempted` keeps its
1550 /// confirm deadline, and `Sent`/`Expired` are handled on the online edge
1551 /// (or corrected by an attributed `DeliveryError`). Skipped entirely when
1552 /// a mesh carrier is also available — relay presence is only
1553 /// authoritative for the internet path, and the data-plane retry track
1554 /// may still deliver over BLE / WiFi-Direct.
1555 ///
1556 /// Never cancels an escalating unreachable probe: a record that already
1557 /// holds one was parked by [`Self::apply_recipient_unreachable_failure`],
1558 /// which already set the same reason code and pushed the same TTL — so this
1559 /// adds nothing and would only downgrade the probe to edge-only. That
1560 /// matters because on an internet-only device this runs once per presence
1561 /// rotation for as long as the peer is down, which would otherwise
1562 /// re-strand the welcome after every single verdict. The distinction is
1563 /// budget: the data-plane retry this *does* cancel was charged an attempt
1564 /// by a carrier-backed failure, while the probe refunds its attempt on
1565 /// every verdict.
1566 ///
1567 /// `last_reason_code` — not `unreachable_parks` alone — is what encodes
1568 /// that distinction. The counter is sticky (only `rearm_welcome_for_peer`
1569 /// clears it), so gating on it by itself would keep shielding the record
1570 /// after the probe track has stopped protecting it: past
1571 /// [`WELCOME_WATCHLIST_MAX_AGE_SECS`] a confirm timeout no longer re-parks
1572 /// (see [`Self::welcome_probe_repark_permitted`]) and instead charges an
1573 /// attempt and arms a plain data-plane retry — precisely the budget-burner
1574 /// this park exists to cancel, and left running it expires the welcome at
1575 /// `max_retries`. Note `park_welcome_awaiting_peer` also stamps
1576 /// `PeerUnreachable` but leaves `next_retry_at = None`, so the
1577 /// `next_retry_at.is_some()` conjunct below is load-bearing, not
1578 /// redundant: the reason code *and* a scheduled retry together are what
1579 /// identify a live probe. A carrier-backed failure stamps
1580 /// `Timeout`/`SendFailed` and a successful send clears the code entirely.
1581 fn park_welcome_peer_unreachable(&mut self, peer_id: &str) {
1582 if self.confirmed_sessions.contains(peer_id) {
1583 return;
1584 }
1585 let Some(record) = self.welcome_lifecycles.get(peer_id) else {
1586 return;
1587 };
1588 if !matches!(
1589 record.state,
1590 WelcomeDeliveryState::Created | WelcomeDeliveryState::Failed
1591 ) {
1592 return;
1593 }
1594 if record.last_reason_code == Some(crate::events::WelcomeReasonCode::PeerUnreachable)
1595 && record.unreachable_parks > 0
1596 && record.next_retry_at.is_some()
1597 {
1598 return;
1599 }
1600 // Local mesh carriers only (BLE / WiFi-Direct): Nostr and Reticulum
1601 // are internet-dependent, so their availability must not veto the
1602 // park — mirrors the carrier guard in
1603 // `apply_recipient_unreachable_failure`.
1604 if self
1605 .transport_manager
1606 .get_available_transports()
1607 .keys()
1608 .any(|transport| matches!(transport, TransportType::BLE | TransportType::WiFiDirect))
1609 {
1610 return;
1611 }
1612 let _ = self.park_welcome_awaiting_peer(peer_id);
1613 }
1614
1615 /// True when an unconfirmed welcome send should resolve as another
1616 /// unreachable verdict ([`Self::apply_recipient_unreachable_failure`])
1617 /// rather than as a carrier-backed failure that ages the record.
1618 ///
1619 /// A live `unreachable_parks` counter means the relay declared the peer
1620 /// offline and no reachability edge has fired since (every edge clears it
1621 /// via `rearm_welcome_for_peer`), so the send that just went unconfirmed
1622 /// was the escalating probe, not a delivery attempt. Without this the
1623 /// probe destroys the very welcome it exists to recover: the relay
1624 /// accepts the frame whenever its push fallback succeeds, which returns
1625 /// *no* `DeliveryError` at all, so the probe resolves at the 10s confirm
1626 /// timeout instead. Treated as carrier-backed that charges the attempt
1627 /// and arms a plain data-plane retry, and since nothing in that ladder
1628 /// pushes `expires_at`, the record walks into `should_expire` within one
1629 /// TTL window — terminal `welcome_send_expired` + `secure_session_failed`
1630 /// for a peer that is merely offline. Re-parking instead refunds the
1631 /// attempt, escalates the interval and pushes the TTL, mirroring the DM
1632 /// path's [`Self::try_repark_exhausted_dm`]: settlement stays reserved
1633 /// for delivery or the record genuinely ageing out below.
1634 ///
1635 /// Only the *presence-offline* answer cancelled that retry before, which
1636 /// is not a defense at all for the case this whole probe exists to serve
1637 /// — a headless consumer that never polls presence.
1638 ///
1639 /// Bounded by [`WELCOME_WATCHLIST_MAX_AGE_SECS`] measured from
1640 /// `created_at`, the same threshold that stops watching a peer as
1641 /// permanently dead: past it the counter no longer shields the record and
1642 /// normal ageing expires it. This is the welcome's twin of the DM probe's
1643 /// absolute bound (`first_sent_at` vs the outbox absolute lifetime) —
1644 /// without it a peer that never returns keeps a welcome probing at the
1645 /// 600s cap forever.
1646 ///
1647 /// Deliberately narrow to `Timeout`: a `SendFailed`/`TransportUnavailable`
1648 /// failure is evidence about the *carrier*, not about this peer, and
1649 /// keeps ageing the record normally.
1650 fn welcome_probe_repark_permitted(&self, peer_id: &str) -> bool {
1651 let Some(record) = self.welcome_lifecycles.get(peer_id) else {
1652 return false;
1653 };
1654 record.unreachable_parks > 0
1655 && record.created_at
1656 > Utc::now() - ChronoDuration::seconds(WELCOME_WATCHLIST_MAX_AGE_SECS)
1657 }
1658
1659 /// Applies the relay's authoritative "recipient offline" verdict
1660 /// (a `recipient_unreachable`-tagged transport failure) to a welcome
1661 /// lifecycle. Also reached *without* a relay verdict when a probe's
1662 /// confirm timeout resolves while the park counter is still live — the
1663 /// same conclusion inferred rather than relayed, see
1664 /// [`Self::welcome_probe_repark_permitted`].
1665 ///
1666 /// The bridge wire-confirms on socket-write success, so by the time the
1667 /// relay's `DeliveryError` arrives the record is normally already `Sent`
1668 /// — a *false* Sent: the store-less relay dropped the content. Correct
1669 /// it back to `Failed` (emitting `welcome_send_failed` so the app's
1670 /// earlier `welcome_send_succeeded` is superseded), refund the attempt
1671 /// (the peer never saw the frame), and park pending a reachability edge.
1672 /// `SendAttempted` (the failure raced ahead of the wire confirm) gets
1673 /// the same treatment; `Created`/`Failed` records just park. Terminal
1674 /// `Expired` records are left to the online edge, which rebuilds them.
1675 ///
1676 /// The record always keeps a timed retry whose interval escalates per
1677 /// consecutive park. Relay presence is authoritative only for the
1678 /// internet path — with a mesh carrier live the peer may still be
1679 /// reachable over BLE / WiFi-Direct (DORS can pick internet even with
1680 /// mesh present) — but an internet-only device needs the timed track just
1681 /// as much: parking it edge-only left the welcome dependent solely on a
1682 /// presence-online answer, i.e. on the platform's presence-polling
1683 /// cadence, which is the same dead end that stalled parked DMs (see
1684 /// [`Self::park_unreachable_dm`]). The escalation (15s → 600s cap) is
1685 /// what bounds the retry: DORS may keep routing to the internet path,
1686 /// where each round trips another budget-refunded `DeliveryError`, and
1687 /// without escalation that would be an unbounded resend loop into the
1688 /// relay. Every verdict refunds the attempt and pushes the TTL — past the
1689 /// scheduled probe, see the invariant on [`Self::park_welcome_no_carrier`]
1690 /// — so the probes never age the welcome out or burn its real-delivery
1691 /// budget, and the counter resets on any reachability edge
1692 /// (`rearm_welcome_for_peer`).
1693 pub(super) fn apply_recipient_unreachable_failure(
1694 &mut self,
1695 peer_id: &str,
1696 transport_error: Option<&'static str>,
1697 ) -> Result<()> {
1698 // A late relay verdict for a session that has since been proven (the
1699 // welcome was rescued over another path and the peer confirmed) must
1700 // not corrupt the converged lifecycle: without this guard the stale
1701 // DeliveryError flips the record back to Failed, refunds an attempt,
1702 // persists the stale state, and emits welcome_send_failed AFTER the
1703 // app already saw secure_session_established.
1704 if self.confirmed_sessions.contains(peer_id) {
1705 return Ok(());
1706 }
1707 let Some(state) = self.welcome_lifecycles.get(peer_id).map(|r| r.state) else {
1708 return Ok(());
1709 };
1710 match state {
1711 WelcomeDeliveryState::Sent | WelcomeDeliveryState::SendAttempted => {
1712 self.transition_welcome_state(
1713 peer_id,
1714 WelcomeDeliveryState::Failed,
1715 "recipient_unreachable",
1716 )?;
1717 if let Some(record) = self.welcome_lifecycles.get_mut(peer_id) {
1718 record.attempt = record.attempt.saturating_sub(1);
1719 record.last_transport_error = transport_error.map(str::to_string);
1720 }
1721 }
1722 WelcomeDeliveryState::Created | WelcomeDeliveryState::Failed => {
1723 if let Some(record) = self.welcome_lifecycles.get_mut(peer_id) {
1724 record.last_transport_error = transport_error.map(str::to_string);
1725 }
1726 }
1727 WelcomeDeliveryState::Expired => return Ok(()),
1728 }
1729 // Escalating timed retry on every carrier (see the doc comment): the
1730 // ladder, not a carrier guard, is what bounds resends into the relay.
1731 let parks = {
1732 let Some(record) = self.welcome_lifecycles.get_mut(peer_id) else {
1733 return Ok(());
1734 };
1735 record.unreachable_parks = record.unreachable_parks.saturating_add(1);
1736 record.unreachable_parks
1737 };
1738 // parks >= 1 here; shift clamped so 15 << 6 = 960 is the largest
1739 // pre-cap value — no overflow risk.
1740 let retry_in_secs = (WELCOME_NO_CARRIER_RETRY_SECS << (parks - 1).min(6))
1741 .min(WELCOME_UNREACHABLE_RETRY_CAP_SECS);
1742 self.park_welcome_no_carrier(
1743 peer_id,
1744 crate::events::WelcomeReasonCode::PeerUnreachable,
1745 transport_error,
1746 retry_in_secs,
1747 )?;
1748
1749 let Some(snapshot) = self.welcome_lifecycles.get(peer_id).cloned() else {
1750 return Ok(());
1751 };
1752 if let Ok(shared) = lock_shared_state(&self.shared_state) {
1753 shared.emit_event(Event::welcome_send_failed(
1754 peer_id.to_string(),
1755 snapshot.welcome_message.id.as_str().to_string(),
1756 snapshot.group_id.clone(),
1757 snapshot.attempt,
1758 crate::events::WelcomeReasonCode::PeerUnreachable,
1759 snapshot
1760 .last_transport_error
1761 .as_deref()
1762 .map(classify_transport_send_error),
1763 // Retryable, and now always with a scheduled time: the park
1764 // keeps an escalating timed probe on every carrier. The
1765 // reachability edges (presence online / peer discovery) still
1766 // re-arm it sooner when they fire.
1767 true,
1768 snapshot.next_retry_at.map(|at| at.timestamp_millis()),
1769 ));
1770 }
1771 Ok(())
1772 }
1773
1774 /// Records a Welcome send failure and decides whether to retry or expire.
1775 ///
1776 /// `no_carrier` is `true` when the failure was simply that no transport
1777 /// carrier exists yet (vs. a present carrier that failed mid-send). A
1778 /// no-carrier failure must NOT age the Welcome: it neither counts toward
1779 /// the retry budget nor lets the TTL expire it, because the peer is merely
1780 /// unreachable for now and will be retried once a carrier appears. The
1781 /// speculative attempt increment from [`Self::try_send_welcome`] is rolled
1782 /// back so a long offline period leaves the real-delivery budget intact.
1783 pub(super) fn apply_welcome_send_failure(
1784 &mut self,
1785 peer_id: &str,
1786 reason: crate::events::WelcomeReasonCode,
1787 transport_error: Option<&'static str>,
1788 no_carrier: bool,
1789 source_event: &str,
1790 ) -> Result<bool> {
1791 let mut updated = self
1792 .welcome_lifecycles
1793 .get(peer_id)
1794 .cloned()
1795 .ok_or_else(|| Error::Other(format!("Missing welcome lifecycle for {}", peer_id)))?;
1796
1797 if matches!(
1798 updated.state,
1799 WelcomeDeliveryState::Sent | WelcomeDeliveryState::Expired
1800 ) {
1801 return Ok(matches!(updated.state, WelcomeDeliveryState::Sent));
1802 }
1803
1804 // A confirm timeout on a welcome that still holds a live
1805 // unreachable-park counter is a *probe* resolving, not a delivery
1806 // failing — re-park it as another unreachable verdict instead of
1807 // ageing it (see [`Self::welcome_probe_repark_permitted`]).
1808 if !no_carrier
1809 && matches!(reason, crate::events::WelcomeReasonCode::Timeout)
1810 && self.welcome_probe_repark_permitted(peer_id)
1811 {
1812 self.apply_recipient_unreachable_failure(peer_id, transport_error)?;
1813 return Ok(false);
1814 }
1815
1816 let max_attempts = self.config.reliability.retry.max_retries.max(1);
1817 // A no-carrier failure never expires the Welcome: neither the retry
1818 // budget nor the TTL applies while the peer is simply unreachable. Only
1819 // a carrier-backed failure (SendFailed / Timeout) ages it toward expiry.
1820 let should_expire =
1821 !no_carrier && (updated.attempt >= max_attempts || updated.expires_at <= Utc::now());
1822 if should_expire {
1823 let terminal_reason = crate::events::WelcomeReasonCode::RetryExhausted;
1824 {
1825 let record = self.welcome_lifecycles.get_mut(peer_id).ok_or_else(|| {
1826 Error::Other(format!("Missing welcome lifecycle for {}", peer_id))
1827 })?;
1828 record.last_reason_code = Some(terminal_reason);
1829 record.last_transport_error = transport_error.map(str::to_string);
1830 record.next_retry_at = None;
1831 }
1832
1833 if !matches!(updated.state, WelcomeDeliveryState::Failed) {
1834 self.transition_welcome_state(peer_id, WelcomeDeliveryState::Failed, source_event)?;
1835 }
1836 self.transition_welcome_state(
1837 peer_id,
1838 WelcomeDeliveryState::Expired,
1839 "welcome_retry_exhausted",
1840 )?;
1841
1842 let expired_snapshot =
1843 self.welcome_lifecycles
1844 .get(peer_id)
1845 .cloned()
1846 .ok_or_else(|| {
1847 Error::Other(format!("Missing welcome lifecycle for {}", peer_id))
1848 })?;
1849 if let Ok(state) = lock_shared_state(&self.shared_state) {
1850 state.emit_event(Event::welcome_send_failed(
1851 peer_id.to_string(),
1852 expired_snapshot.welcome_message.id.as_str().to_string(),
1853 expired_snapshot.group_id.clone(),
1854 expired_snapshot.attempt,
1855 terminal_reason,
1856 expired_snapshot
1857 .last_transport_error
1858 .as_deref()
1859 .map(classify_transport_send_error),
1860 false,
1861 None,
1862 ));
1863 state.emit_event(Event::welcome_send_expired(
1864 peer_id.to_string(),
1865 expired_snapshot.welcome_message.id.as_str().to_string(),
1866 expired_snapshot.attempt,
1867 terminal_reason,
1868 ));
1869 }
1870 self.abort_pending_session_for_peer(peer_id, terminal_reason);
1871 return Ok(false);
1872 }
1873
1874 // A no-carrier failure (the carrier raced away after the pre-send check,
1875 // or an async transport-failed callback with no carrier) re-checks on the
1876 // slow no-carrier interval rather than the data-plane backoff: there is
1877 // nothing to deliver over until a carrier returns, and `try_send_welcome`
1878 // parks subsequent no-carrier ticks cheaply anyway.
1879 let retry_at = if no_carrier {
1880 Utc::now() + ChronoDuration::seconds(WELCOME_NO_CARRIER_RETRY_SECS)
1881 } else {
1882 let delay_ms = self.compute_welcome_retry_delay_ms(peer_id, updated.attempt);
1883 Utc::now() + ChronoDuration::milliseconds(delay_ms as i64)
1884 };
1885
1886 {
1887 let record = self.welcome_lifecycles.get_mut(peer_id).ok_or_else(|| {
1888 Error::Other(format!("Missing welcome lifecycle for {}", peer_id))
1889 })?;
1890 // Roll back the speculative attempt increment from try_send_welcome
1891 // for a no-carrier failure so the retry budget stays reserved for
1892 // real delivery attempts over an available carrier, and push the TTL
1893 // forward so a never-deliverable Welcome does not age out — the TTL
1894 // clock is carrier-relative, not creation-relative.
1895 if no_carrier {
1896 record.attempt = record.attempt.saturating_sub(1);
1897 record.expires_at =
1898 Utc::now() + ChronoDuration::seconds(WELCOME_LIFECYCLE_TTL_SECS);
1899 }
1900 record.last_reason_code = Some(reason);
1901 record.last_transport_error = transport_error.map(str::to_string);
1902 record.next_retry_at = Some(retry_at);
1903 }
1904
1905 if !matches!(updated.state, WelcomeDeliveryState::Failed) {
1906 self.transition_welcome_state(peer_id, WelcomeDeliveryState::Failed, source_event)?;
1907 } else if let Some(record) = self.welcome_lifecycles.get(peer_id).cloned() {
1908 self.persist_welcome_lifecycle_entry(&record)?;
1909 }
1910
1911 updated = self
1912 .welcome_lifecycles
1913 .get(peer_id)
1914 .cloned()
1915 .ok_or_else(|| Error::Other(format!("Missing welcome lifecycle for {}", peer_id)))?;
1916 if let Ok(state) = lock_shared_state(&self.shared_state) {
1917 state.emit_event(Event::welcome_send_failed(
1918 peer_id.to_string(),
1919 updated.welcome_message.id.as_str().to_string(),
1920 updated.group_id,
1921 updated.attempt,
1922 reason,
1923 updated
1924 .last_transport_error
1925 .as_deref()
1926 .map(classify_transport_send_error),
1927 true,
1928 Some(retry_at.timestamp_millis()),
1929 ));
1930 }
1931 Ok(false)
1932 }
1933
1934 pub(super) fn compute_welcome_retry_delay_ms(&self, peer_id: &str, attempt: u32) -> u64 {
1935 let config = &self.config.reliability.retry;
1936 let capped_attempt = attempt.saturating_sub(1);
1937 let base_ms = if capped_attempt == 0 {
1938 config.initial_delay_ms
1939 } else {
1940 let multiplier = config.backoff_multiplier.powi(capped_attempt as i32);
1941 (config.initial_delay_ms as f64 * multiplier as f64) as u64
1942 }
1943 .min(config.max_delay_ms);
1944
1945 let mut hasher = std::collections::hash_map::DefaultHasher::new();
1946 peer_id.hash(&mut hasher);
1947 attempt.hash(&mut hasher);
1948 Utc::now().timestamp_millis().hash(&mut hasher);
1949 let bucket = (hasher.finish() % 10_000) as f64 / 10_000.0;
1950 let jitter_factor = 1.0 + ((bucket * 2.0 - 1.0) * WELCOME_RETRY_JITTER_RATIO);
1951 let jittered = (base_ms as f64 * jitter_factor).round() as i64;
1952 jittered.max(1) as u64
1953 }
1954
1955 pub(super) fn can_transition_welcome_state(
1956 current: WelcomeDeliveryState,
1957 next: WelcomeDeliveryState,
1958 ) -> bool {
1959 matches!(
1960 (current, next),
1961 (
1962 WelcomeDeliveryState::Created,
1963 WelcomeDeliveryState::SendAttempted
1964 ) | (WelcomeDeliveryState::Created, WelcomeDeliveryState::Expired)
1965 | (
1966 WelcomeDeliveryState::SendAttempted,
1967 WelcomeDeliveryState::Sent
1968 )
1969 | (
1970 WelcomeDeliveryState::SendAttempted,
1971 WelcomeDeliveryState::Failed
1972 )
1973 | (
1974 WelcomeDeliveryState::Failed,
1975 WelcomeDeliveryState::SendAttempted
1976 )
1977 | (WelcomeDeliveryState::Failed, WelcomeDeliveryState::Sent)
1978 | (WelcomeDeliveryState::Failed, WelcomeDeliveryState::Expired)
1979 // Sent is normally a sink, but the relay's DeliveryError is
1980 // authoritative proof that a wire-confirmed frame was dropped
1981 // (store-less relay, recipient offline). That correction is
1982 // the one legal way back out of Sent — see
1983 // `on_transport_send_failed`.
1984 | (WelcomeDeliveryState::Sent, WelcomeDeliveryState::Failed)
1985 )
1986 }
1987
1988 pub(super) fn transition_welcome_state(
1989 &mut self,
1990 peer_id: &str,
1991 next_state: WelcomeDeliveryState,
1992 source_event: &str,
1993 ) -> Result<()> {
1994 let (previous_state, record_snapshot) = {
1995 let record = self.welcome_lifecycles.get_mut(peer_id).ok_or_else(|| {
1996 Error::Other(format!(
1997 "Missing welcome lifecycle for transition: {}",
1998 peer_id
1999 ))
2000 })?;
2001
2002 if record.state == next_state {
2003 return Ok(());
2004 }
2005
2006 if !Self::can_transition_welcome_state(record.state, next_state) {
2007 return Err(Error::Other(format!(
2008 "Illegal welcome lifecycle transition for {}: {} -> {}",
2009 peer_id,
2010 record.state.as_str(),
2011 next_state.as_str()
2012 )));
2013 }
2014
2015 let previous = record.state;
2016 record.state = next_state;
2017 if matches!(
2018 next_state,
2019 WelcomeDeliveryState::Sent | WelcomeDeliveryState::Expired
2020 ) {
2021 record.next_retry_at = None;
2022 }
2023 (previous, record.clone())
2024 };
2025
2026 self.persist_welcome_lifecycle_entry(&record_snapshot)?;
2027 info!(
2028 event = "welcome_lifecycle_transition",
2029 session_or_group_id = %peer_id,
2030 previous_state = previous_state.as_str(),
2031 new_state = next_state.as_str(),
2032 source_event = %source_event,
2033 attempt = record_snapshot.attempt,
2034 "welcome_lifecycle_transition"
2035 );
2036 Ok(())
2037 }
2038
2039 pub(super) fn upsert_welcome_lifecycle(
2040 &mut self,
2041 peer_id: &str,
2042 group_id: &str,
2043 welcome_message: Message,
2044 source_event: &str,
2045 ) -> Result<()> {
2046 if let Some(existing) = self.welcome_lifecycles.get(peer_id) {
2047 if !matches!(
2048 existing.state,
2049 WelcomeDeliveryState::Sent | WelcomeDeliveryState::Expired
2050 ) {
2051 return Err(Error::Other(format!(
2052 "Refusing to overwrite active welcome lifecycle for {} in state {}",
2053 peer_id,
2054 existing.state.as_str()
2055 )));
2056 }
2057 }
2058
2059 let now = Utc::now();
2060 let record = WelcomeLifecycleRecord {
2061 peer_id: peer_id.to_string(),
2062 group_id: group_id.to_string(),
2063 state: WelcomeDeliveryState::Created,
2064 attempt: 0,
2065 unreachable_parks: 0,
2066 welcome_message,
2067 next_retry_at: None,
2068 last_reason_code: None,
2069 last_transport_error: None,
2070 created_at: now,
2071 expires_at: now + ChronoDuration::seconds(WELCOME_LIFECYCLE_TTL_SECS),
2072 };
2073 self.welcome_lifecycles
2074 .insert(peer_id.to_string(), record.clone());
2075 self.persist_welcome_lifecycle_entry(&record)?;
2076 info!(
2077 event = "welcome_lifecycle_transition",
2078 session_or_group_id = %peer_id,
2079 previous_state = "Absent",
2080 new_state = WelcomeDeliveryState::Created.as_str(),
2081 source_event = %source_event,
2082 attempt = 0,
2083 "welcome_lifecycle_transition"
2084 );
2085 Ok(())
2086 }
2087
2088 pub(super) fn map_welcome_reason_code(error: &Error) -> crate::events::WelcomeReasonCode {
2089 SessionStateError::classify(error).to_welcome_reason_code()
2090 }
2091
2092 /// True when a send failed because no transport carrier exists yet, as
2093 /// opposed to a present-but-unhealthy carrier (`SendFailed`) or a
2094 /// sent-but-unconfirmed timeout. A no-carrier failure means the peer is
2095 /// simply unreachable right now, so the Welcome must be kept alive and
2096 /// retried rather than counted against its budget — see
2097 /// [`Self::apply_welcome_send_failure`].
2098 pub(super) fn is_no_carrier_error(error: &Error) -> bool {
2099 matches!(
2100 error,
2101 Error::Transport(offline_protocol_transport::Error::TransportNotAvailable(_))
2102 )
2103 }
2104
2105 pub(super) fn session_ready_context_for_source(source_event: &str) -> MlsOperationContext {
2106 match source_event {
2107 "confirmation_ack_received" | "confirmation_probe_received" | "decrypt_success" => {
2108 MlsOperationContext::Receive
2109 }
2110 "welcome_received" => MlsOperationContext::Welcome,
2111 _ => MlsOperationContext::Send,
2112 }
2113 }
2114
2115 pub(super) fn find_welcome_peer_by_message_id(&self, message_id: &str) -> Option<String> {
2116 self.welcome_lifecycles
2117 .iter()
2118 .find_map(|(peer_id, record)| {
2119 if record.welcome_message.id.as_str() == message_id {
2120 return Some(peer_id.clone());
2121 }
2122 None
2123 })
2124 }
2125
2126 pub(super) fn process_welcome_retry_queue(&mut self) -> Result<()> {
2127 let now = Utc::now();
2128 let timed_out_attempts: Vec<String> = self
2129 .welcome_lifecycles
2130 .iter()
2131 .filter_map(|(peer_id, record)| {
2132 // Skip peers whose session is already confirmed: the welcome is
2133 // delivered, mark_welcome_confirmed should have marked it Sent,
2134 // and re-sending would be wasted bandwidth (defensive against any
2135 // keying mismatch that left the lifecycle non-terminal).
2136 if self.confirmed_sessions.contains(peer_id) {
2137 return None;
2138 }
2139 if matches!(record.state, WelcomeDeliveryState::SendAttempted)
2140 && record.next_retry_at.is_some_and(|retry_at| retry_at <= now)
2141 {
2142 return Some(peer_id.clone());
2143 }
2144 None
2145 })
2146 .take(WELCOME_RETRY_BATCH_SIZE)
2147 .collect();
2148
2149 for peer_id in timed_out_attempts {
2150 let _ = self.apply_welcome_send_failure(
2151 &peer_id,
2152 crate::events::WelcomeReasonCode::Timeout,
2153 Some(SEND_FAIL_REASON_CONFIRM_TIMEOUT),
2154 // A confirm timeout means the Welcome was sent over a carrier;
2155 // this is not a no-carrier failure, so it ages normally.
2156 false,
2157 "welcome_confirm_timeout",
2158 )?;
2159 }
2160
2161 let due_peers: Vec<String> = self
2162 .welcome_lifecycles
2163 .iter()
2164 .filter_map(|(peer_id, record)| {
2165 if self.confirmed_sessions.contains(peer_id) {
2166 return None;
2167 }
2168 // Failed = a normal retry-due Welcome. Created with a due
2169 // next_retry_at = a no-carrier-parked Welcome (see
2170 // `park_welcome_no_carrier`): a fresh Created entry has
2171 // next_retry_at == None, so only parked ones match here.
2172 if matches!(
2173 record.state,
2174 WelcomeDeliveryState::Failed | WelcomeDeliveryState::Created
2175 ) && record.next_retry_at.is_some_and(|retry_at| retry_at <= now)
2176 {
2177 return Some(peer_id.clone());
2178 }
2179 None
2180 })
2181 .take(WELCOME_RETRY_BATCH_SIZE)
2182 .collect();
2183
2184 for peer_id in due_peers {
2185 if let Err(err) = self.try_send_welcome(&peer_id, "welcome_retry") {
2186 warn!(
2187 peer_id = %peer_id,
2188 error = %err,
2189 "Failed to process welcome retry"
2190 );
2191 }
2192 }
2193
2194 Ok(())
2195 }
2196
2197 pub(super) fn has_terminal_welcome_failure(&self, peer_id: &str) -> bool {
2198 self.welcome_lifecycles
2199 .get(peer_id)
2200 .is_some_and(|record| matches!(record.state, WelcomeDeliveryState::Expired))
2201 }
2202}