offline_protocol/protocol/session.rs
1//! Session confirmation, welcome lifecycle, and pending session reconciliation.
2
3use super::{
4 classify_transport_send_error, internal_prefixes, key_package_resend_wait_secs,
5 lock_shared_state, send_failure_token, GatewayCarrier, OfflineProtocol, PresenceRescueThrottle,
6 PruneAllowance, RestorableRecord, SessionState, WelcomeDeliveryState, WelcomeLifecycleRecord,
7 CONFIRMATION_PROBE_INTERVAL_SECS, CONFIRMATION_RETRY_INTERVAL_SECS, MAX_REKEY_TRACKED_PEERS,
8 RECONCILIATION_THROTTLE_MS, REKEY_INTERVAL_SECS, SEND_FAIL_REASON_CONFIRM_TIMEOUT,
9 WELCOME_INTERNET_CONFIRM_TIMEOUT_SECS, WELCOME_LIFECYCLE_TTL_SECS,
10 WELCOME_MESH_CONFIRM_TIMEOUT_SECS, WELCOME_NO_CARRIER_RETRY_SECS,
11 WELCOME_PRESENCE_RESCUE_BASE_SECS, WELCOME_PRESENCE_RESCUE_MAX_SECS, WELCOME_RETRY_BATCH_SIZE,
12 WELCOME_RETRY_JITTER_RATIO, WELCOME_UNREACHABLE_RETRY_CAP_SECS, WELCOME_WATCHLIST_MAX_AGE_SECS,
13};
14use crate::events::SecurityWarningCode;
15use crate::mls_observability::MlsOperationContext;
16use crate::protocol::reachability::{Claim, FactSource};
17use crate::{Error, EstablishmentState, Event, Result, SessionStateError};
18use chrono::{Duration as ChronoDuration, Utc};
19use offline_protocol_core::{Message, MessageId, MessagePriority};
20use offline_protocol_mls::{MlsManager, WelcomeMessage};
21use offline_protocol_transport::TransportType;
22use std::collections::HashSet;
23use std::hash::{Hash, Hasher};
24use std::sync::{Arc, RwLock};
25use std::time::{Duration as StdDuration, Instant};
26use tracing::{debug, info, warn};
27
28impl OfflineProtocol {
29 // ========================================================================
30 // SESSION STATE
31 // ========================================================================
32
33 /// Returns true when persisted state marks this peer session as confirmed.
34 ///
35 /// Uses the in-memory `confirmed_sessions` cache as a fast path to avoid
36 /// hitting persistent storage on every message send. The cache is populated
37 /// when sessions are confirmed and cleared when sessions are invalidated.
38 pub(super) fn is_session_confirmed(&mut self, peer_id: &str) -> Result<bool> {
39 // Fast path: check in-memory cache only — no I/O. The cache is kept in
40 // sync by confirm_session_state() and all invalidation paths (blocking,
41 // stale cleanup, etc.). We deliberately skip the has_mls_session() guard
42 // here because it calls load_group() which hits MlsStorage (10-100ms on
43 // mobile Keychain/Keystore). If the session was wiped externally, the
44 // encrypt call will fail and we handle it there.
45 if self.confirmed_sessions.contains(peer_id) {
46 return Ok(true);
47 }
48
49 let persisted = self
50 .load_session_state_entry(peer_id)?
51 .unwrap_or(SessionState::Pending);
52 if matches!(persisted, SessionState::Confirmed) {
53 if !self.has_mls_session(peer_id)? {
54 warn!(
55 peer_id = %peer_id,
56 "Persisted confirmed state has no matching MLS session; clearing stale state"
57 );
58 self.confirmed_sessions.remove(peer_id);
59 self.clear_confirmation_recovery_tracking(peer_id);
60 self.welcome_lifecycles.remove(peer_id);
61 // The MLS session is gone, so any both-create owner gate for this
62 // peer is stale; clear it (memory + storage) or it would block
63 // `welcome_received` confirmation on the next re-pairing.
64 self.clear_both_create_awaiting_decrypt(peer_id);
65 if let Err(err) = self.clear_session_state_entry(peer_id) {
66 warn!(
67 peer_id = %peer_id,
68 error = %err,
69 "Failed to clear stale persisted session state"
70 );
71 }
72 if let Err(err) = self.clear_welcome_lifecycle_entry(peer_id) {
73 warn!(
74 peer_id = %peer_id,
75 error = %err,
76 "Failed to clear stale persisted welcome lifecycle"
77 );
78 }
79 return Ok(false);
80 }
81 self.confirmed_sessions.insert(peer_id.to_string());
82 return Ok(true);
83 }
84 Ok(false)
85 }
86
87 pub(super) fn has_mls_session(&self, peer_id: &str) -> Result<bool> {
88 let Some(mls) = self.mls_manager.clone() else {
89 return Ok(false);
90 };
91
92 let manager = mls
93 .read()
94 .map_err(|_| Error::Other("MLS lock poisoned".to_string()))?;
95 manager.has_session(peer_id).map_err(Error::Mls)
96 }
97
98 /// Returns the current establishment state for a peer (for API and error reporting).
99 pub(super) fn establishment_state(&self, peer_id: &str) -> Result<EstablishmentState> {
100 if let Some(mls) = &self.mls_manager {
101 let has_session = {
102 let manager = mls
103 .read()
104 .map_err(|_| Error::Other("MLS lock poisoned".to_string()))?;
105 manager.has_session(peer_id).map_err(Error::Mls)?
106 };
107 if has_session {
108 let state = self
109 .load_session_state_entry(peer_id)?
110 .unwrap_or(SessionState::Pending);
111 return Ok(if matches!(state, SessionState::Confirmed) {
112 EstablishmentState::SessionConfirmed
113 } else {
114 EstablishmentState::SessionPending
115 });
116 }
117 }
118
119 let now_ms = Utc::now().timestamp_millis() as u64;
120 if let Some(pkg) = self.pending_key_packages.get(peer_id) {
121 if now_ms < pkg.local_expires_at_ms {
122 return Ok(EstablishmentState::HaveKeyPackage);
123 }
124 }
125 if self.load_peer_key_package_from_storage(peer_id).is_some() {
126 return Ok(EstablishmentState::HaveKeyPackage);
127 }
128 Ok(EstablishmentState::NoKeyPackage)
129 }
130
131 /// Monotonic state transition helper: Pending -> Confirmed only.
132 pub(super) fn confirm_session_state(
133 &mut self,
134 peer_id: &str,
135 source_event: &str,
136 ) -> Result<bool> {
137 if !self.can_confirm_from_source(peer_id, source_event) {
138 warn!(
139 event = "session_confirmation_blocked",
140 session_or_group_id = %peer_id,
141 source_event = %source_event,
142 "session_confirmation_blocked"
143 );
144 return Ok(false);
145 }
146
147 let previous = self.ensure_session_state_entry(peer_id, source_event)?;
148
149 if matches!(previous, SessionState::Confirmed) {
150 self.confirmed_sessions.insert(peer_id.to_string());
151 self.clear_confirmation_recovery_tracking(peer_id);
152 self.clear_both_create_awaiting_decrypt(peer_id);
153 info!(
154 event = "session_state_transition",
155 session_or_group_id = %peer_id,
156 previous_state = "Confirmed",
157 new_state = "Confirmed",
158 source_event = %source_event,
159 "session_state_transition"
160 );
161 return Ok(false);
162 }
163
164 // Persist first, then publish in-memory view.
165 if let Err(err) = self.persist_session_state(peer_id, SessionState::Confirmed, source_event)
166 {
167 self.schedule_confirmation_retry(peer_id, source_event);
168 return Err(err);
169 }
170
171 self.confirmed_sessions.insert(peer_id.to_string());
172 self.clear_confirmation_recovery_tracking(peer_id);
173 // Group-aware proof has arrived (decrypt), so we no longer need to gate
174 // confirmation on decrypt for this both-create peer.
175 self.clear_both_create_awaiting_decrypt(peer_id);
176 // The peer has proved the session, so the outbound Welcome — kept
177 // non-terminal so a lost fragment keeps being retried — is delivered.
178 // Mark it Sent so process_welcome_retry_queue stops re-sending it. Mesh
179 // has no transport-level delivery ack, so this session proof is the mesh
180 // equivalent of on_transport_send_confirmed.
181 self.mark_welcome_confirmed(peer_id, source_event);
182 if source_event != "welcome_received" && source_event != "decrypt_success" {
183 self.maybe_emit_local_session_established(
184 peer_id,
185 Self::session_ready_context_for_source(source_event),
186 );
187 }
188 info!(
189 event = "session_state_transition",
190 session_or_group_id = %peer_id,
191 previous_state = previous.as_str(),
192 new_state = "Confirmed",
193 source_event = %source_event,
194 "session_state_transition"
195 );
196
197 // Replication starts here rather than at the four callers that reach
198 // this transition, because the four callers are exactly the shape of
199 // bug that keeps happening: one of them forgets, and the feature is
200 // silently absent on whichever confirmation path that caller owned.
201 // This is also the only signal that covers all four — the session
202 // lifecycle event does not fire on the probe and ack paths.
203 #[cfg(feature = "data")]
204 self.kick_data_sync(peer_id, source_event);
205
206 Ok(true)
207 }
208
209 /// Marks a non-terminal outbound welcome lifecycle as `Sent` once the peer
210 /// has proved the session (probe / ack / welcome / decrypt). The mesh
211 /// equivalent of [`Self::on_transport_send_confirmed`], which only fires for
212 /// Internet. No-op when there is no welcome lifecycle for `peer_id` (e.g. a
213 /// confirmation keyed by a group id rather than a 1:1 peer) or it is already
214 /// terminal, so it is safe to call from every confirmation path.
215 fn mark_welcome_confirmed(&mut self, peer_id: &str, source_event: &str) {
216 match self.welcome_lifecycles.get(peer_id).map(|r| r.state) {
217 // In flight / retried — fall through and mark Sent below.
218 Some(WelcomeDeliveryState::SendAttempted) | Some(WelcomeDeliveryState::Failed) => {}
219 // Parked, never actually sent (no carrier at send time) for a peer
220 // that converged over some other path. `Created -> Sent` is not a
221 // legal transition and there is nothing in flight to confirm, so drop
222 // the now-moot lifecycle (memory + storage) rather than leaving it to
223 // linger until the session is deleted.
224 Some(WelcomeDeliveryState::Created) => {
225 self.welcome_lifecycles.remove(peer_id);
226 if let Err(err) = self.clear_welcome_lifecycle_entry(peer_id) {
227 warn!(
228 peer_id = %peer_id,
229 error = %err,
230 "Failed to clear parked welcome lifecycle on confirmation"
231 );
232 }
233 return;
234 }
235 // Already terminal (Sent / Expired) or no lifecycle — nothing to do.
236 _ => return,
237 }
238 if let Some(record) = self.welcome_lifecycles.get_mut(peer_id) {
239 record.next_retry_at = None;
240 record.last_reason_code = None;
241 record.last_transport_error = None;
242 }
243 // transition_welcome_state persists the record and logs the transition.
244 if self
245 .transition_welcome_state(peer_id, WelcomeDeliveryState::Sent, source_event)
246 .is_err()
247 {
248 return;
249 }
250 if let Some(snapshot) = self.welcome_lifecycles.get(peer_id).cloned() {
251 if let Ok(state) = lock_shared_state(&self.shared_state) {
252 state.emit_event(Event::welcome_send_succeeded(
253 peer_id.to_string(),
254 snapshot.welcome_message.id.as_str().to_string(),
255 snapshot.group_id,
256 snapshot.attempt,
257 ));
258 }
259 }
260 }
261
262 /// Ensures a session has an explicit persisted state entry.
263 pub(super) fn ensure_session_state_entry(
264 &self,
265 peer_id: &str,
266 source_event: &str,
267 ) -> Result<SessionState> {
268 let existing = self.load_session_state_entry(peer_id)?;
269 if let Some(state) = existing {
270 return Ok(state);
271 }
272
273 self.persist_session_state(peer_id, SessionState::Pending, source_event)?;
274 info!(
275 event = "session_state_transition",
276 session_or_group_id = %peer_id,
277 previous_state = "Absent",
278 new_state = "Pending",
279 source_event = %source_event,
280 "session_state_transition"
281 );
282 Ok(SessionState::Pending)
283 }
284
285 /// Reconstructs runtime confirmation cache from persisted session states.
286 /// `allowance` is the launch's shared advisory pool — this walk's deletes
287 /// are advisory (a dropped record re-bootstraps to `Pending`, which is the
288 /// safe answer), and they add up against the same allowance as the four
289 /// category walks. See `storage::MAX_RESTORE_PRUNE_DELETES`.
290 pub(super) fn restore_session_states_from_manager(
291 &mut self,
292 mls: Arc<RwLock<MlsManager>>,
293 allowance: &mut PruneAllowance,
294 ) -> Result<()> {
295 self.confirmed_sessions.clear();
296
297 let sessions = {
298 let manager = mls
299 .read()
300 .map_err(|_| Error::Other("MLS lock poisoned".to_string()))?;
301 manager.list_sessions()?
302 };
303
304 // The session list itself is deliberately not truncated the way a
305 // container-listed walk is: dropping its tail would silently leave
306 // confirmed peers out of `confirmed_sessions`. What needed bounding was
307 // the *deletes* a bad store makes this walk issue — one device barrier
308 // per peer, with nothing counting them until now.
309 let mut budget = allowance.refusing();
310 for peer_id in sessions {
311 // Before anything that can `continue`: an MLS session with this
312 // peer exists, which is the capability fact, and it holds whether
313 // or not the peer's `session_states` record turns out to be
314 // readable. Deriving it from the record instead would reintroduce
315 // exactly the deletable dependency `encryption_capable_peers`
316 // exists to escape.
317 self.mark_encryption_capable(&peer_id);
318
319 // Restore-path read: a record whose bytes will not decode is
320 // dropped and re-bootstrapped as `Pending` rather than failing
321 // initialization forever. The send path deliberately keeps the
322 // strict loader — see `load_restorable_state_record`.
323 let state = match self.load_session_state_for_restore(&peer_id, Some(&mut budget)) {
324 RestorableRecord::Present(state) => state,
325 RestorableRecord::Absent => self.bootstrap_missing_session_state(&peer_id),
326 // A record is still on disk and could not be read *this*
327 // session. Re-bootstrapping would persist `Pending` straight
328 // over one that may say `Confirmed`, and propagating would fail
329 // `initialize_mls` for as long as that single record stays
330 // unreadable — on an install that then cannot send at all.
331 // Skipping is what leaves the record recoverable: the peer is
332 // simply absent from `confirmed_sessions`, and the send path's
333 // own strict loader (`is_session_confirmed`) still fails closed
334 // for it — and re-reads the record, so a transient failure
335 // heals the moment the store answers again.
336 //
337 // Three consequences worth naming rather than rediscovering,
338 // all of them following from the peer being absent from
339 // `confirmed_sessions` for the run:
340 //
341 // 1. The Welcome machinery gates on
342 // `!confirmed_sessions.contains(peer)` *plus* a live
343 // `welcome_lifecycles` entry (`welcome_pending_peers`,
344 // `resend_unconfirmed_sent_welcome`, the retry ladder). So a
345 // peer whose lifecycle restored as `Sent` while its session
346 // state did not can draw a redundant Welcome re-send.
347 // 2. `flush_restored_confirmed_pending_messages` will not flush
348 // this peer's queued pre-session messages this session. They
349 // stay queued — in memory and on disk — and flush once a
350 // launch can read the record, or once a live confirmation
351 // re-adds the peer.
352 // 3. Tier-2 crypto recovery re-seals a resend only for a peer
353 // in `confirmed_sessions` (`reseal_resend_content`), so a
354 // resend to this peer replays its original ciphertext for
355 // the run rather than re-sealing to the current epoch.
356 //
357 // All three are deferrals on an already-degraded store, and all
358 // three are strictly better than the alternative this replaced
359 // — propagating, which failed `initialize_mls` on every launch
360 // for as long as the one record stayed unreadable, on an
361 // install that could then send nothing at all.
362 RestorableRecord::Unavailable => {
363 warn!(
364 session_or_group_id = %peer_id,
365 "Session state could not be read this session; leaving the record in \
366 place and treating the session as unconfirmed"
367 );
368 continue;
369 }
370 };
371 if matches!(state, SessionState::Confirmed) {
372 self.confirmed_sessions.insert(peer_id.clone());
373 }
374 info!(
375 event = "session_state_restored",
376 session_or_group_id = %peer_id,
377 previous_state = "Pending",
378 new_state = %state.as_str(),
379 source_event = "initialize_mls",
380 "session_state_restored"
381 );
382 }
383
384 if budget.exhausted {
385 warn!(
386 deleted = budget.spent,
387 budget = super::storage::MAX_RESTORE_PRUNE_DELETES,
388 "Session state restore hit its share of the launch delete budget; the \
389 unreadable records left behind are re-bootstrapped as Pending anyway and \
390 dropped on a later launch"
391 );
392 }
393
394 Ok(())
395 }
396
397 /// The state a session with no readable record starts from.
398 ///
399 /// Infallible: the migration write is best effort, like every other
400 /// persistence call on the restore path. It used to propagate, so one
401 /// `StoreFailed` — a full disk, a container the OS has locked — failed
402 /// `initialize_mls` outright and, with `require_encryption` on by default,
403 /// left the install unable to send. `Pending` is the safe answer in memory
404 /// either way, and a launch that can write re-derives the record.
405 pub(super) fn bootstrap_missing_session_state(&self, peer_id: &str) -> SessionState {
406 // Legacy session records without explicit state are treated as Pending.
407 // Recovery is driven by probe/ack reconciliation, never by implicit inference.
408 let restored_state = SessionState::Pending;
409 if let Err(e) = self.persist_session_state(
410 peer_id,
411 restored_state,
412 "initialize_mls_missing_state_migration",
413 ) {
414 warn!(
415 session_or_group_id = %peer_id,
416 error = %e,
417 "Failed to persist the bootstrapped session state; the session is Pending for \
418 this run and the record is re-derived on the next launch"
419 );
420 }
421 info!(
422 event = "session_state_transition",
423 session_or_group_id = %peer_id,
424 previous_state = "Absent",
425 new_state = %restored_state.as_str(),
426 source_event = "initialize_mls_missing_state_migration",
427 "session_state_transition"
428 );
429
430 restored_state
431 }
432
433 // ========================================================================
434 // CONFIRMATION RECOVERY
435 // ========================================================================
436
437 pub(super) fn schedule_confirmation_retry(&mut self, peer_id: &str, source_event: &str) {
438 self.confirmation_retry_due_at
439 .insert(peer_id.to_string(), Utc::now());
440 warn!(
441 event = "session_confirmation_retry_scheduled",
442 session_or_group_id = %peer_id,
443 source_event = %source_event,
444 "session_confirmation_retry_scheduled"
445 );
446 }
447
448 pub(super) fn clear_confirmation_recovery_tracking(&mut self, peer_id: &str) {
449 self.confirmation_retry_due_at.remove(peer_id);
450 self.confirmation_probe_due_at.remove(peer_id);
451 self.confirmation_probe_unreachable_parks.remove(peer_id);
452 // Withdrawn with the tracking: once the map forgets it, no later
453 // probe can supersede it, and it would keep its full retry ladder.
454 if let Some(probe) = self.confirmation_probe_outstanding.remove(peer_id) {
455 self.forget_outbound_message(&probe);
456 }
457 self.confirmation_probe_superseded.remove(peer_id);
458 }
459
460 /// Applies a relay verdict on a probe that has since been superseded
461 /// (`confirmation_probe_superseded`) the way the verdict handlers apply
462 /// one on a queued probe: the reachability fact, then the probe back-off.
463 /// Consumed, so one probe's verdict escalates once, as it did while the
464 /// probe was still queued. A no-op for any other id; returns whether
465 /// the verdict was applied.
466 pub(super) fn note_superseded_probe_verdict(
467 &mut self,
468 message_id: &MessageId,
469 carrier: Option<TransportType>,
470 ) -> bool {
471 let Some(peer) = self
472 .confirmation_probe_superseded
473 .iter()
474 .find(|(_, probe)| *probe == message_id)
475 .map(|(peer, _)| peer.clone())
476 else {
477 return false;
478 };
479 self.confirmation_probe_superseded.remove(&peer);
480 if let Some(carrier) = carrier {
481 self.reachability.record(
482 &peer,
483 carrier,
484 Claim::Unreachable,
485 FactSource::GatewayVerdict,
486 Instant::now(),
487 );
488 }
489 self.note_confirmation_probe_unreachable(&peer);
490 true
491 }
492
493 /// Fold a relay "recipient unreachable" verdict into the session-confirmation
494 /// probe schedule: escalate this peer's next probe on the same 15s -> 600s
495 /// ladder the welcome lifecycle uses (see
496 /// [`Self::apply_recipient_unreachable_failure`]). Without this, a peer that
497 /// established an MLS session then vanished before confirming is re-probed
498 /// every `CONFIRMATION_PROBE_INTERVAL_SECS` (5s) indefinitely, because
499 /// `send_session_confirmation_probe` re-arms it on every probe and
500 /// `on_transport_send_failed` otherwise never consults this scheduler. A
501 /// handful of such peers pushes aggregate relay traffic past the
502 /// per-connection rate limit, which disconnects the socket on a loop.
503 ///
504 /// No-op unless the peer is currently a probe target (a `confirmation_probe`
505 /// entry exists, so a probe was just attempted), so ordinary DM failures to
506 /// confirmed peers do not touch the probe schedule. The counter is reset on
507 /// a reachability edge in [`Self::on_peer_presence`] so a returning peer is
508 /// re-probed immediately and its session still converges.
509 pub(super) fn note_confirmation_probe_unreachable(&mut self, peer_id: &str) {
510 if !self.confirmation_probe_due_at.contains_key(peer_id) {
511 return;
512 }
513 let parks = {
514 let counter = self
515 .confirmation_probe_unreachable_parks
516 .entry(peer_id.to_string())
517 .or_insert(0);
518 *counter = counter.saturating_add(1);
519 *counter
520 };
521 // parks >= 1; shift clamped so 15 << 6 = 960 is the largest pre-cap
522 // value (no overflow), then capped — identical to the welcome ladder.
523 let backoff_secs = (WELCOME_NO_CARRIER_RETRY_SECS << (parks - 1).min(6))
524 .min(WELCOME_UNREACHABLE_RETRY_CAP_SECS);
525 self.confirmation_probe_due_at.insert(
526 peer_id.to_string(),
527 Utc::now() + ChronoDuration::seconds(backoff_secs),
528 );
529 }
530
531 pub(super) fn collect_pending_session_peers(&mut self) -> Result<Vec<String>> {
532 let Some(mls) = self.mls_manager.clone() else {
533 return Ok(Vec::new());
534 };
535
536 let sessions = {
537 let manager = mls
538 .read()
539 .map_err(|_| Error::Other("MLS lock poisoned".to_string()))?;
540 manager.list_sessions()?
541 };
542
543 let mut pending = Vec::new();
544 for peer_id in sessions {
545 if !self.is_session_confirmed(&peer_id)? {
546 pending.push(peer_id);
547 }
548 }
549
550 Ok(pending)
551 }
552
553 pub(super) fn send_session_confirmation_probe(&mut self, peer_id: &str, source_event: &str) {
554 // Superseded, not failed: the new probe asks the same question, so
555 // the old one's retry ladder would only stack onto it. Here rather
556 // than in the scan because the Welcome fast path sends one too, and
557 // a probe it replaced would keep its full ladder. Silent, and no
558 // delivery failure is recorded against the carrier. Before the send,
559 // not after: if the send errors the peer has no probe in flight
560 // until the next scan, but a new probe can never evict a message at
561 // outbox capacity to make room beside the one it replaces. The
562 // superseded id is kept so a relay verdict that arrives for it late
563 // still backs the schedule off (`note_superseded_probe_verdict`).
564 if let Some(previous) = self.confirmation_probe_outstanding.remove(peer_id) {
565 self.forget_outbound_message(&previous);
566 self.confirmation_probe_superseded
567 .insert(peer_id.to_string(), previous);
568 }
569 // The sender owns the cadence, so a fast-path probe is not superseded
570 // by the next scan before its ACK can arrive, and a relay verdict for
571 // it can back the schedule off (`note_confirmation_probe_unreachable`
572 // only escalates a peer that has a due time). Stamped before the send
573 // so a verdict delivered during it is not overwritten.
574 //
575 // A later due time is kept: the scan only sends once the due time has
576 // passed, but the fast path can fire while the schedule is backed off,
577 // and resetting it would undo the ladder a relay verdict built.
578 let next = Utc::now() + ChronoDuration::seconds(CONFIRMATION_PROBE_INTERVAL_SECS);
579 let due_at = self
580 .confirmation_probe_due_at
581 .entry(peer_id.to_string())
582 .or_insert(next);
583 *due_at = (*due_at).max(next);
584 match self.send_internal_message(
585 peer_id,
586 internal_prefixes::SESSION_CONFIRM_PROBE.to_string(),
587 MessagePriority::High,
588 ) {
589 Ok(message_id) => {
590 self.confirmation_probe_outstanding
591 .insert(peer_id.to_string(), message_id);
592 info!(
593 event = "session_confirmation_probe_sent",
594 session_or_group_id = %peer_id,
595 source_event = %source_event,
596 "session_confirmation_probe_sent"
597 );
598 }
599 Err(err) => {
600 // The send can fail after the probe entered the outbox, and
601 // then no id comes back to track. Any probe still queued for
602 // this peer is such an orphan, since the tracked one was
603 // withdrawn above. (A full ACK tracker no longer lands here:
604 // it holds the probe and returns its id.)
605 let orphans: Vec<_> = self
606 .outbox
607 .values()
608 .filter(|entry| {
609 entry.message.recipient.as_str() == peer_id
610 && Self::is_confirmation_probe(&entry.message)
611 })
612 .map(|entry| entry.message.id.clone())
613 .collect();
614 for orphan in orphans {
615 self.forget_outbound_message(&orphan);
616 }
617 warn!(
618 event = "session_confirmation_probe_failed",
619 session_or_group_id = %peer_id,
620 source_event = %source_event,
621 error = %err,
622 "session_confirmation_probe_failed"
623 );
624 }
625 }
626 }
627
628 /// Throttled wrapper around session reconciliation. Skips the expensive
629 /// `list_sessions()` storage I/O when there is no pending work or when the
630 /// last scan ran recently (within `RECONCILIATION_THROTTLE_MS`).
631 ///
632 /// This is the entry-point called from `process()` and `receive_message()`
633 /// on every tick/poll. Without throttling, the storage call holds the
634 /// protocol Mutex and blocks `sendMessage()` on mobile platforms.
635 pub(super) fn run_throttled_reconciliation(&mut self, source_event: &str) {
636 // Fast path: nothing pending → skip entirely (no storage I/O)
637 let has_pending_work = !self.pending_encrypted_messages.is_empty()
638 || !self.confirmation_probe_due_at.is_empty()
639 || !self.confirmation_retry_due_at.is_empty();
640
641 if !has_pending_work {
642 return;
643 }
644
645 // Throttle: only run the full scan every RECONCILIATION_THROTTLE_MS
646 let now = Instant::now();
647 let throttle = StdDuration::from_millis(RECONCILIATION_THROTTLE_MS);
648 if let Some(last) = self.last_reconciliation_at {
649 if now.duration_since(last) < throttle {
650 return;
651 }
652 }
653
654 self.last_reconciliation_at = Some(now);
655 self.retry_pending_session_confirmations();
656 self.kick_pending_session_reconciliation(source_event);
657
658 // A message queued for a session is the standing reason to want one,
659 // so a lost key package is re-armed here too, not only on a discovery
660 // that a quiet pair may never produce.
661 let waiting: Vec<String> = self.pending_encrypted_messages.keys().cloned().collect();
662 for peer_id in waiting {
663 self.rearm_key_package_for_peer(&peer_id);
664 }
665 }
666
667 pub(super) fn kick_pending_session_reconciliation(&mut self, source_event: &str) {
668 let now = Utc::now();
669 let pending_peers = match self.collect_pending_session_peers() {
670 Ok(peers) => peers,
671 Err(err) => {
672 warn!(
673 event = "session_confirmation_probe_scan_failed",
674 source_event = %source_event,
675 error = %err,
676 "session_confirmation_probe_scan_failed"
677 );
678 return;
679 }
680 };
681
682 let pending_set: HashSet<String> = pending_peers.iter().cloned().collect();
683 self.confirmation_probe_due_at
684 .retain(|peer, _| pending_set.contains(peer));
685 self.confirmation_probe_unreachable_parks
686 .retain(|peer, _| pending_set.contains(peer));
687 self.confirmation_probe_superseded
688 .retain(|peer, _| pending_set.contains(peer));
689 // Not a plain `retain`: a probe whose peer left the pending set
690 // without passing through `clear_confirmation_recovery_tracking`
691 // would be forgotten while still queued.
692 let settled: Vec<String> = self
693 .confirmation_probe_outstanding
694 .keys()
695 .filter(|peer| !pending_set.contains(*peer))
696 .cloned()
697 .collect();
698 for peer in settled {
699 if let Some(probe) = self.confirmation_probe_outstanding.remove(&peer) {
700 self.forget_outbound_message(&probe);
701 }
702 }
703
704 for peer_id in pending_peers {
705 let due_at = self
706 .confirmation_probe_due_at
707 .get(&peer_id)
708 .copied()
709 .unwrap_or(now);
710 if due_at > now {
711 continue;
712 }
713 self.send_session_confirmation_probe(&peer_id, source_event);
714 }
715 }
716
717 /// Triggers a rate-limited re-key of the 1:1 session with `peer_id` after an
718 /// epoch-desync decrypt failure.
719 ///
720 /// The self-healing primitive is a two-part move, matching the unblock
721 /// `session_reset` flow: tear down our *own* stale session **and** advertise
722 /// a `session_reset` key package. The peer, on receiving it, drops its stale
723 /// session, auto-establishes a fresh one from our enclosed key package, and
724 /// Welcomes us back — which we, now session-less, simply *join*.
725 ///
726 /// Tearing down our session alone (without the reset key package) would
727 /// deadlock: the peer's original key package was consumed at first
728 /// establishment, so nothing could rebuild the channel, and the peer (still
729 /// believing its session is healthy) would never re-advertise one. Sending
730 /// the reset key package is what breaks that deadlock.
731 ///
732 /// Deleting the local session is also what makes convergence **symmetric and
733 /// single-round regardless of user-id ordering**. Because we keep no local
734 /// session, the peer's returning Welcome is joined via the fresh-session
735 /// path rather than the greater-id-adopts tiebreaker in
736 /// `handle_welcome_message` — so the lexicographically-smaller detector is no
737 /// longer stranded (the failure mode when the stale session was kept). Both
738 /// orderings are covered end-to-end by
739 /// `test_desync_dm_heals_end_to_end_when_detector_id_is_greater` and
740 /// `test_desync_dm_heals_end_to_end_when_detector_id_is_smaller`.
741 ///
742 /// Rate-limited via `rekey_due_at` to at most one re-key per
743 /// [`REKEY_INTERVAL_SECS`] per peer, so a peer replaying stale-epoch
744 /// ciphertext (or an injected wrong-epoch frame) cannot drive a re-key storm.
745 /// The interval is stamped before the send so a send error still counts
746 /// against the limit, and it is **never reset early** — not even by a
747 /// successful decrypt on the healed session. That is deliberate: a genuine
748 /// re-fork and a replayed old-epoch frame are indistinguishable at this layer
749 /// (both surface as `WrongEpoch`), so clearing the floor on heal would let an
750 /// attacker who lands one legit decrypt between replays defeat the limit and
751 /// force ~one teardown per inbound message. The floor lapses naturally after
752 /// [`REKEY_INTERVAL_SECS`]; during that window Tier 1 (un-ACK + sender
753 /// retries) keeps delivery honest, so the only cost of not resetting is that a
754 /// genuine second desync within the window heals up to one interval later —
755 /// an acceptable trade for a rare event against a bounded-churn guarantee.
756 ///
757 /// **SECURITY — this trigger is unauthenticated, and cannot be made
758 /// otherwise.** `peer_id` is the *wire-claimed* sender. Three facts compose:
759 /// `__MLS_ENC__` is a data-plane prefix, deliberately exempt from the
760 /// Ed25519 + derivation control-message gate (see `DATA_PLANE_PREFIXES`);
761 /// OpenMLS
762 /// validates the framing header — group id, then epoch — *before* any AEAD,
763 /// sender-data decryption or signature check, so `WrongEpoch` /
764 /// `NoPastEpochData` is produced with the sender still entirely unverified;
765 /// and a 1:1 slot id is `session:<a>:<b>` over two public user ids. So the
766 /// classification that lands here is reachable by **anyone who can inject a
767 /// frame** — no key material, no captured ciphertext, no session, no replay
768 /// needed. `test_forged_frame_reaches_session_desync_without_any_key_material`
769 /// builds such a frame from scratch to keep this statement honest. It is
770 /// inherent to MLS framing, not an OpenMLS defect, so no upgrade changes it.
771 ///
772 /// The `SenderIdentityMismatch` gate cannot help: it compares the MLS
773 /// credential to the claimed sender, and that credential only exists once
774 /// `process_message` **succeeds**. Every pre-authentication verdict is
775 /// structurally outside its reach.
776 ///
777 /// Because the trigger cannot be authenticated, the mitigation is that
778 /// acting on it is **harmless**, not that it is trusted:
779 /// - `SessionManager::decrypt_message` requires the envelope to name the
780 /// slot shared with the claimed sender, so one derivable session id
781 /// cannot be aimed at arbitrary peers, and `rekey_due_at` cannot be grown
782 /// with attacker-chosen keys.
783 /// - [`REKEY_INTERVAL_SECS`] bounds this to one re-key per peer per window.
784 /// - The heal destroys nothing: queued plaintext survives a reset (it seals
785 /// at flush time against the rebuilt session), and Tier 2 re-seals
786 /// in-flight resends.
787 /// - Each re-key emits `SecurityWarningCode::SessionRekeyTriggered`, so a
788 /// sustained rate — the signature of injected frames rather than a real
789 /// fork — is visible to the app.
790 ///
791 /// **Residual:** an injector can still force bounded re-key churn on a pair
792 /// (delayed delivery, never lost). Closing that needs a signed
793 /// epoch-corroboration exchange before teardown; a liveness-only probe does
794 /// not work, since a healthy peer answers and we would tear down anyway.
795 /// Also see `docs/state-machines/session-lifecycle.md` ("Desync and heal")
796 /// and `docs/security/threat-model.md` (residual risk R2).
797 pub(super) fn schedule_session_rekey(&mut self, peer_id: &str) {
798 if !self.claim_rekey_slot(peer_id) {
799 return;
800 }
801
802 // A re-key is remotely triggerable and, until now, entirely silent to
803 // the app. A genuine fork produces these occasionally; a sustained rate
804 // for one peer is the signature of injected frames, which an operator
805 // cannot otherwise distinguish.
806 //
807 // Emitted on this path only. The warning names an epoch desync because
808 // that is what reached it, and an application asking for a re-key on
809 // its own schedule has not seen one: reusing the code there would put a
810 // fixed token in front of two different facts and leave an operator
811 // reading their own maintenance as the attack signature this exists to
812 // surface.
813 self.emit_security_warning(
814 peer_id,
815 SecurityWarningCode::SessionRekeyTriggered,
816 "1:1 session torn down and re-advertised after an epoch desync",
817 );
818 if self
819 .drive_session_rekey(peer_id, "after epoch desync")
820 .is_err()
821 {
822 // The advertisement did not leave, and this path discards the
823 // session anyway. What reached it is a fork: every later frame on
824 // that session decrypts to nothing, so keeping it buys a pair that
825 // stays silently broken rather than one that rebuilds when the
826 // peer is next heard from. The application-driven entry makes the
827 // opposite choice, and for the opposite reason: the session it was
828 // asked to rotate is healthy.
829 self.discard_local_session(peer_id);
830 }
831 }
832
833 /// Takes this peer's re-key slot, or reports that the floor has not lapsed.
834 ///
835 /// The floor is what bounds a peer replaying stale-epoch ciphertext to one
836 /// teardown per window rather than one per frame, and it is shared with the
837 /// application-driven entry deliberately: a re-key costs the same teardown
838 /// and re-exchange whoever asked for it, and a caller looping on it would
839 /// otherwise be able to do to a pair exactly what the bound exists to stop
840 /// an attacker doing.
841 pub(super) fn claim_rekey_slot(&mut self, peer_id: &str) -> bool {
842 let now = Utc::now();
843 if let Some(due_at) = self.rekey_due_at.get(peer_id) {
844 if *due_at > now {
845 return false;
846 }
847 }
848 // Bounded like every other map keyed by a wire-claimed id: the desync
849 // that reaches the caller is classified before MLS authenticates
850 // anything.
851 if !self.rekey_due_at.contains_key(peer_id)
852 && self.rekey_due_at.len() >= MAX_REKEY_TRACKED_PEERS
853 {
854 self.rekey_due_at.clear();
855 }
856 self.rekey_due_at.insert(
857 peer_id.to_string(),
858 now + ChronoDuration::seconds(REKEY_INTERVAL_SECS),
859 );
860 true
861 }
862
863 /// Advertises a reset key package and tears the local session down.
864 ///
865 /// This is the whole of what a re-key does on the wire, shared by the
866 /// desync path and by [`OfflineProtocol::rekey_session`] so the two cannot
867 /// drift into meaning different things to a peer. `reason` appears in the
868 /// log line only.
869 ///
870 /// # Why the advertisement goes first
871 ///
872 /// An `Err` here means **nothing was sent and nothing was torn down**.
873 /// [`TransportManager::send`](crate::TransportManager::send) reports an
874 /// error only once no transport has accepted the frame, so a failure is a
875 /// peer that has been told nothing, and a caller rotating a *healthy*
876 /// session must be able to leave it alone: discarding it on the strength
877 /// of a frame that never left strands the pair, with this side holding no
878 /// session, the peer holding one it will never be told to drop, and
879 /// nothing in either that re-opens the exchange.
880 ///
881 /// The order is invisible to a peer. Both steps run under one `&mut self`,
882 /// so no inbound frame can be processed between them, and what a peer sees
883 /// on success is what this path has always sent. The property the old
884 /// order was written for still holds for the same reason: by the time the
885 /// peer's returning Welcome is processed there is no local session left,
886 /// so it is *joined* rather than gated by the greater-id-adopts tiebreaker
887 /// in `handle_welcome_message`, and both sides rebuild in a single round.
888 pub(super) fn drive_session_rekey(&mut self, peer_id: &str, reason: &str) -> Result<()> {
889 if let Err(err) = self.send_key_package_to(peer_id, true) {
890 warn!(
891 event = "session_rekey_send_failed",
892 peer_id = %peer_id,
893 reason = %reason,
894 error = %err,
895 "session_rekey_send_failed"
896 );
897 return Err(err);
898 }
899 self.discard_local_session(peer_id);
900 info!(
901 event = "session_rekey_triggered",
902 peer_id = %peer_id,
903 reason = %reason,
904 "Triggered 1:1 session re-key"
905 );
906 Ok(())
907 }
908
909 /// Drops this side's 1:1 session with `peer_id`, best effort.
910 ///
911 /// Mirrors the unblock `session_reset` flow. A missing session is a no-op,
912 /// and a delete that fails is logged rather than raised: the peer has
913 /// already been sent the reset by every caller that reaches here, and the
914 /// rebuild plus the auto key-package exchange re-arm establishment either
915 /// way.
916 fn discard_local_session(&mut self, peer_id: &str) {
917 if self.has_mls_session(peer_id).unwrap_or(false) {
918 if let Err(err) = self.manual_mls_delete_session(peer_id) {
919 debug!(
920 peer_id = %peer_id,
921 error = %err,
922 "rekey: failed to delete local stale session"
923 );
924 }
925 }
926 }
927
928 /// Gives back a re-key slot claimed for work that did not happen.
929 ///
930 /// Only the application-driven entry may call this, and only when the
931 /// advertisement never left: no teardown happened, so nothing was spent
932 /// and a caller back in range a second later should not be made to wait
933 /// out a window that bounds teardowns. The desync path must **not**, since
934 /// it discards the session whether or not the reset was sent, and that is
935 /// exactly the churn the floor exists to bound.
936 pub(super) fn release_rekey_slot(&mut self, peer_id: &str) {
937 self.rekey_due_at.remove(peer_id);
938 }
939
940 pub(super) fn retry_pending_session_confirmations(&mut self) {
941 let now = Utc::now();
942 let due_peers: Vec<String> = self
943 .confirmation_retry_due_at
944 .iter()
945 .filter_map(|(peer_id, due_at)| {
946 if *due_at <= now {
947 Some(peer_id.clone())
948 } else {
949 None
950 }
951 })
952 .collect();
953
954 for peer_id in due_peers {
955 match self.has_mls_session(&peer_id) {
956 Ok(true) => {}
957 Ok(false) => {
958 self.confirmation_retry_due_at.remove(&peer_id);
959 continue;
960 }
961 Err(err) => {
962 warn!(
963 event = "session_confirmation_retry_scan_failed",
964 session_or_group_id = %peer_id,
965 error = %err,
966 "session_confirmation_retry_scan_failed"
967 );
968 self.confirmation_retry_due_at.insert(
969 peer_id,
970 now + ChronoDuration::seconds(CONFIRMATION_RETRY_INTERVAL_SECS),
971 );
972 continue;
973 }
974 }
975
976 if !self.can_confirm_from_source(&peer_id, "confirmation_retry") {
977 debug!(
978 peer_id = %peer_id,
979 "Skipping confirmation retry until welcome send is at least attempted"
980 );
981 continue;
982 }
983
984 match self.confirm_session_state(&peer_id, "confirmation_retry") {
985 Ok(_) => {
986 let _ = self.flush_pending_messages(&peer_id);
987 self.process_pending_decryption(&peer_id);
988 }
989 Err(err) => {
990 warn!(
991 event = "session_confirmation_retry_failed",
992 session_or_group_id = %peer_id,
993 error = %err,
994 "session_confirmation_retry_failed"
995 );
996 self.confirmation_retry_due_at.insert(
997 peer_id,
998 now + ChronoDuration::seconds(CONFIRMATION_RETRY_INTERVAL_SECS),
999 );
1000 }
1001 }
1002 }
1003 }
1004
1005 /// Marks `peer_id` as a both-create owner gate — we kept our own group and
1006 /// must observe a group-aware decrypt before confirming — and persists it so
1007 /// an owner restart mid-convergence cannot let a stale plaintext probe/ack
1008 /// confirm prematurely. Only writes storage on a genuine insert.
1009 pub(super) fn mark_both_create_awaiting_decrypt(&mut self, peer_id: &str) {
1010 if self
1011 .both_create_awaiting_decrypt
1012 .insert(peer_id.to_string())
1013 {
1014 self.persist_both_create_awaiting_decrypt(peer_id);
1015 }
1016 }
1017
1018 /// Clears the both-create owner gate for `peer_id` (it has converged) from
1019 /// both memory and storage. Only writes storage on a genuine removal.
1020 pub(super) fn clear_both_create_awaiting_decrypt(&mut self, peer_id: &str) {
1021 if self.both_create_awaiting_decrypt.remove(peer_id) {
1022 self.delete_both_create_awaiting_decrypt(peer_id);
1023 }
1024 }
1025
1026 pub(super) fn can_confirm_from_source(&self, peer_id: &str, source_event: &str) -> bool {
1027 // Both-create owner: we kept our own group and are waiting for the peer to
1028 // prove it adopted *our* group. Only a successful decrypt is group-aware
1029 // proof of that; a plaintext probe/ack proves only that the peer holds
1030 // some session (possibly its own, pre-adoption group), so it must not
1031 // confirm us or we would stop retransmitting and strand the peer.
1032 if self.both_create_awaiting_decrypt.contains(peer_id) && source_event != "decrypt_success"
1033 {
1034 return false;
1035 }
1036
1037 // A successful group decrypt is definitive, group-aware proof that the
1038 // peer adopted our 1:1 group. `decrypt_success` is raised only inside
1039 // `DecryptResult::Success` (see message_dispatch.rs): ciphertext on the
1040 // single stored `session:<a>:<b>` group actually decrypted, which is
1041 // possible only if the peer is a member. It must therefore confirm the
1042 // session regardless of our *local* Welcome's delivery state. Gating it on
1043 // a still-active local Welcome (the lifecycle match below) strands a
1044 // both-create owner whose own Welcome timed out to `Failed`/`Expired` on a
1045 // lossy or asymmetric BLE link (the Android↔iOS case): the owner decrypts
1046 // the peer's messages but never confirms, so every outbound send fails
1047 // `SessionNotReady` — it can receive but never reply. The both-create gate
1048 // above is preserved, so a both-create owner is still restricted to *only*
1049 // decrypt_success (a plaintext probe/ack can never confirm it).
1050 if source_event == "decrypt_success" {
1051 return true;
1052 }
1053
1054 if !matches!(
1055 source_event,
1056 "confirmation_ack_received" | "confirmation_probe_received" | "confirmation_retry"
1057 ) {
1058 return true;
1059 }
1060
1061 match self.welcome_lifecycles.get(peer_id) {
1062 Some(record) => matches!(
1063 record.state,
1064 WelcomeDeliveryState::Sent | WelcomeDeliveryState::SendAttempted
1065 ),
1066 None => matches!(
1067 source_event,
1068 // Compatibility path for sessions created before welcome lifecycle
1069 // persistence existed. A plaintext probe/ack/retry can confirm here
1070 // (it is the only evidence such a legacy session has); a group-aware
1071 // decrypt already returned true above and never reaches this arm.
1072 "confirmation_ack_received" | "confirmation_probe_received" | "confirmation_retry"
1073 ),
1074 }
1075 }
1076
1077 pub(super) fn abort_pending_session_for_peer(
1078 &mut self,
1079 peer_id: &str,
1080 reason: crate::events::WelcomeReasonCode,
1081 ) {
1082 self.drop_pending_queue_for_peer(peer_id, reason.welcome_failure_reason());
1083 if let Ok(state) = lock_shared_state(&self.shared_state) {
1084 state.emit_event(Event::secure_session_failed(
1085 peer_id.to_string(),
1086 format!("Welcome delivery failed: {}", reason.as_str()),
1087 ));
1088 }
1089 }
1090
1091 pub(super) fn maybe_emit_local_session_established(
1092 &self,
1093 peer_id: &str,
1094 context: MlsOperationContext,
1095 ) {
1096 let Some(record) = self.welcome_lifecycles.get(peer_id) else {
1097 return;
1098 };
1099 if !matches!(record.state, WelcomeDeliveryState::Sent) {
1100 return;
1101 }
1102 self.emit_mls_session_ready(peer_id, &record.group_id, context);
1103 if let Ok(state) = lock_shared_state(&self.shared_state) {
1104 state.emit_event(Event::secure_session_established(
1105 peer_id.to_string(),
1106 record.group_id.clone(),
1107 record.group_id.starts_with("session:"),
1108 true,
1109 ));
1110 }
1111 }
1112
1113 // ========================================================================
1114 // WELCOME LIFECYCLE
1115 // ========================================================================
1116
1117 /// `user_send` is true when a user's own send created the session: their
1118 /// message waits on this Welcome, so its first send must alert (see
1119 /// [`Self::try_send_welcome`]).
1120 pub(super) fn send_welcome_message(
1121 &mut self,
1122 recipient: &str,
1123 welcome: &WelcomeMessage,
1124 user_send: bool,
1125 ) -> Result<bool> {
1126 let serialized =
1127 serde_json::to_string(welcome).map_err(|e| Error::Serialization(e.to_string()))?;
1128 let content = format!("{}{}", internal_prefixes::WELCOME, serialized);
1129 let mut message =
1130 self.create_message(recipient, content, Some(MessagePriority::High), None)?;
1131 self.sign_control_message(&mut message)?;
1132 let group_id = welcome.group_id.as_str().to_string();
1133
1134 self.upsert_welcome_lifecycle(recipient, &group_id, message, "welcome_created")?;
1135 self.try_send_welcome(recipient, "welcome_initial_send", user_send)
1136 }
1137
1138 /// Parks a Welcome that currently has no transport carrier: refresh the
1139 /// carrier-relative TTL and schedule a slow re-check, WITHOUT consuming a
1140 /// retry attempt, transitioning state, or emitting a `WelcomeSendAttempted`
1141 /// event. A no-carrier send is a guaranteed failure, so re-attempting it at
1142 /// the data-plane retry rate only churns storage I/O and app events while a
1143 /// device is offline. The lifecycle stays in its current non-terminal state
1144 /// (Created on the first send, Failed on a later retry) so the retry queue
1145 /// re-polls it on [`WELCOME_NO_CARRIER_RETRY_SECS`]; `on_neighbor_discovered`
1146 /// re-arms it immediately when a carrier surfaces the peer. Returns
1147 /// `Ok(false)` (not sent).
1148 ///
1149 /// **Invariant: the refreshed TTL always outlives the retry being
1150 /// scheduled.** [`Self::try_send_welcome`] checks `expires_at` *before*
1151 /// running a due retry, so a record whose TTL lapsed while it waited gets
1152 /// expired terminally (`welcome_send_expired` + `secure_session_failed`) by
1153 /// the very timer meant to recover it. That is not hypothetical at the far
1154 /// end of the unreachable ladder in
1155 /// [`Self::apply_recipient_unreachable_failure`]: it reaches 480s at six
1156 /// consecutive parks and [`WELCOME_UNREACHABLE_RETRY_CAP_SECS`] (600s)
1157 /// beyond that, both above the 300s [`WELCOME_LIFECYCLE_TTL_SECS`]. A full
1158 /// extra interval of slack keeps the probe's own outcome — a verdict
1159 /// re-park or a confirm timeout — inside the window too. No-op for the 15s
1160 /// no-carrier park, whose interval is far below the TTL.
1161 fn park_welcome_no_carrier(
1162 &mut self,
1163 peer_id: &str,
1164 reason: crate::events::WelcomeReasonCode,
1165 transport_error: Option<&'static str>,
1166 retry_in_secs: i64,
1167 ) -> Result<bool> {
1168 let snapshot = {
1169 let Some(record) = self.welcome_lifecycles.get_mut(peer_id) else {
1170 return Ok(false);
1171 };
1172 let now = Utc::now();
1173 record.next_retry_at = Some(now + ChronoDuration::seconds(retry_in_secs));
1174 record.expires_at = now
1175 + ChronoDuration::seconds(
1176 WELCOME_LIFECYCLE_TTL_SECS.max(retry_in_secs.saturating_mul(2)),
1177 );
1178 record.last_reason_code = Some(reason);
1179 record.last_transport_error = transport_error.map(str::to_string);
1180 record.clone()
1181 };
1182 self.persist_welcome_lifecycle_entry(&snapshot)?;
1183 debug!(
1184 peer_id = %peer_id,
1185 state = snapshot.state.as_str(),
1186 retry_in_secs,
1187 "Welcome parked: no transport carrier, re-checking on slow interval"
1188 );
1189 Ok(false)
1190 }
1191
1192 /// The Welcome is the frame that wakes an offline peer for a first
1193 /// message queued behind it, so the relay push is decided per send rather
1194 /// than baked into the stored frame: it alerts when `user_send` (the
1195 /// message is being queued right now) or when user content is already
1196 /// waiting for the peer, and is otherwise
1197 /// [`crate::constants::NO_ALERT_KEY`]-marked. Metadata is outside the
1198 /// control signature, so changing it here leaves the frame verifiable.
1199 pub(super) fn try_send_welcome(
1200 &mut self,
1201 peer_id: &str,
1202 source_event: &str,
1203 user_send: bool,
1204 ) -> Result<bool> {
1205 let now = Utc::now();
1206 let mut record = self
1207 .welcome_lifecycles
1208 .get(peer_id)
1209 .cloned()
1210 .ok_or_else(|| Error::Other(format!("Missing welcome lifecycle for {}", peer_id)))?;
1211
1212 if matches!(record.state, WelcomeDeliveryState::Sent) {
1213 return Ok(true);
1214 }
1215 if matches!(record.state, WelcomeDeliveryState::Expired) {
1216 return Ok(false);
1217 }
1218
1219 // Only honor the TTL when a carrier actually exists: a Welcome that has
1220 // never had a deliverable transport must not age out.
1221 let carrier_available = !self.transport_manager.get_available_transports().is_empty();
1222 if carrier_available && record.expires_at <= now {
1223 self.transition_welcome_state(peer_id, WelcomeDeliveryState::Expired, source_event)?;
1224 if let Ok(state) = lock_shared_state(&self.shared_state) {
1225 state.emit_event(Event::welcome_send_expired(
1226 peer_id.to_string(),
1227 record.welcome_message.id.as_str().to_string(),
1228 record.attempt,
1229 crate::events::WelcomeReasonCode::RetryExhausted,
1230 ));
1231 }
1232 self.abort_pending_session_for_peer(
1233 peer_id,
1234 crate::events::WelcomeReasonCode::RetryExhausted,
1235 );
1236 return Ok(false);
1237 }
1238
1239 // No carrier at all: a send is guaranteed to fail, so do NOT burn the
1240 // speculative attempt increment, the SendAttempted/Failed transitions
1241 // (two persisted state changes + two info logs), and a
1242 // `WelcomeSendAttempted` event on it every retry tick. Park the
1243 // lifecycle on a slow carrier-relative interval instead and re-check
1244 // later; `on_neighbor_discovered` re-arms it immediately when a carrier
1245 // surfaces the peer. This keeps an offline device quiet rather than
1246 // spinning storage I/O + app events at the data-plane retry rate.
1247 if !carrier_available {
1248 return self.park_welcome_no_carrier(
1249 peer_id,
1250 crate::events::WelcomeReasonCode::TransportUnavailable,
1251 None,
1252 WELCOME_NO_CARRIER_RETRY_SECS,
1253 );
1254 }
1255
1256 record.attempt = record.attempt.saturating_add(1);
1257 self.welcome_lifecycles
1258 .insert(peer_id.to_string(), record.clone());
1259 self.persist_welcome_lifecycle_entry(&record)?;
1260 self.transition_welcome_state(peer_id, WelcomeDeliveryState::SendAttempted, source_event)?;
1261
1262 if let Ok(state) = lock_shared_state(&self.shared_state) {
1263 state.emit_event(Event::welcome_send_attempted(
1264 peer_id.to_string(),
1265 record.welcome_message.id.as_str().to_string(),
1266 record.group_id.clone(),
1267 record.attempt,
1268 ));
1269 }
1270
1271 // A Welcome for a peer no carrier is known to reach is handed to
1272 // neighbours to carry, as the key package that produced it was: a
1273 // session between two devices that only reach each other through a
1274 // third is built from exactly these two frames crossing it. Carried
1275 // or sent, it is equally unproved, so it takes the same in-flight
1276 // path below and the peer's probe or decrypt is still what settles
1277 // it. A carried one waits the mesh confirmation timeout, since a
1278 // neighbour is a mesh link whatever the selector last picked.
1279 let mut welcome = record.welcome_message.clone();
1280 if user_send || self.has_user_content_waiting_for(peer_id) {
1281 welcome.metadata.remove(crate::constants::NO_ALERT_KEY);
1282 } else {
1283 welcome
1284 .metadata
1285 .insert(crate::constants::NO_ALERT_KEY.to_string(), "1".to_string());
1286 }
1287 let sent = self
1288 .send_or_carry_handshake(&welcome)
1289 .map(|handed_to_mesh| {
1290 if handed_to_mesh > 0 {
1291 None
1292 } else {
1293 self.transport_manager.current_transport()
1294 }
1295 });
1296
1297 match sent {
1298 Ok(transport_used) => {
1299 let mut updated =
1300 self.welcome_lifecycles
1301 .get(peer_id)
1302 .cloned()
1303 .ok_or_else(|| {
1304 Error::Other(format!("Missing welcome lifecycle for {}", peer_id))
1305 })?;
1306
1307 // No transport's send() proves the peer reassembled and joined
1308 // the group, so keep the lifecycle NON-TERMINAL until an explicit
1309 // confirmation arrives and let the retry queue re-send on timeout.
1310 //
1311 // - Internet send() only enqueues for platform polling; the
1312 // confirmation is on_transport_send_confirmed.
1313 // - A BLE / WiFi-Direct GATT WRITE_TYPE_NO_RESPONSE returning Ok
1314 // only means the local stack accepted the bytes. A single lost
1315 // fragment of the multi-fragment Welcome would otherwise leave
1316 // the sender believing the session exists while the peer holds
1317 // undecryptable ciphertext, with NO retry — permanently
1318 // bricking the pair. The mesh confirmation is the peer proving
1319 // the session (its probe / ack / welcome / decrypt), which marks
1320 // the welcome Sent via confirm_session_state.
1321 let confirm_timeout_secs =
1322 if matches!(transport_used, Some(TransportType::Internet)) {
1323 WELCOME_INTERNET_CONFIRM_TIMEOUT_SECS
1324 } else {
1325 WELCOME_MESH_CONFIRM_TIMEOUT_SECS
1326 };
1327 updated.next_retry_at =
1328 Some(Utc::now() + ChronoDuration::seconds(confirm_timeout_secs));
1329 updated.last_reason_code = None;
1330 updated.last_transport_error = None;
1331 self.welcome_lifecycles
1332 .insert(peer_id.to_string(), updated.clone());
1333 self.persist_welcome_lifecycle_entry(&updated)?;
1334
1335 // Seed the confirmation-probe scheduler for this pending peer so
1336 // the SENDER actively reconciles instead of waiting passively.
1337 //
1338 // The Welcome is now in flight but unconfirmed. The sender's only
1339 // built-in convergence path would otherwise be the receiver's
1340 // single proactive encrypted confirm on first Welcome receipt — a
1341 // single point of failure with no retry (a lost fragment, a
1342 // not-yet-ready encryptor, or the retransmit/owner_keep path on
1343 // the receiver all strand it). By marking a probe due now,
1344 // `run_throttled_reconciliation` (whose `has_pending_work` gate is
1345 // otherwise false on the sender, since it has no pending encrypted
1346 // messages) starts emitting `SESSION_CONFIRM_PROBE`s. The peer —
1347 // which holds the session — replies with `SESSION_CONFIRM_ACK`,
1348 // confirming us and marking this Welcome `Sent`. The entry is
1349 // dropped once confirmed (`clear_confirmation_recovery_tracking`)
1350 // or once the peer is no longer pending (`kick`'s retain), so this
1351 // adds no steady-state work after convergence.
1352 self.confirmation_probe_due_at
1353 .insert(peer_id.to_string(), Utc::now());
1354
1355 Ok(false)
1356 }
1357 Err(err) => {
1358 let no_carrier = Self::is_no_carrier_error(&err);
1359 let reason = Self::map_welcome_reason_code(&err);
1360 // Classified, not rendered: the transport layer interpolates
1361 // the peer into `PeerNotReachable`, and this value is both
1362 // persisted with the lifecycle record and emitted on
1363 // `WelcomeSendFailed.transport_error`, where the scrubber
1364 // hashes `peer_id` beside it. The full error is logged above.
1365 self.apply_welcome_send_failure(
1366 peer_id,
1367 reason,
1368 Some(send_failure_token(&err)),
1369 no_carrier,
1370 source_event,
1371 )
1372 }
1373 }
1374 }
1375
1376 /// Re-arms a stalled or expired outbound Welcome when a peer becomes
1377 /// reachable again, so a session that never converged while the peer was
1378 /// offline gets a fresh delivery attempt over the now-available carrier.
1379 ///
1380 /// No-op when the peer's session is already confirmed, when there is no
1381 /// Welcome lifecycle, or when the Welcome is already `Sent` or still
1382 /// in-flight (`SendAttempted`). An `Expired` lifecycle is rebuilt from its
1383 /// retained Welcome message — the MLS group is not torn down on expiry, so
1384 /// the stored Welcome is still valid to re-send — while a `Created`/`Failed`
1385 /// lifecycle has its budget and TTL reset and is retried immediately.
1386 pub(super) fn rearm_welcome_for_peer(&mut self, peer_id: &str, source_event: &str) {
1387 if self.confirmed_sessions.contains(peer_id) {
1388 return;
1389 }
1390 let Some(record) = self.welcome_lifecycles.get(peer_id).cloned() else {
1391 return;
1392 };
1393 match record.state {
1394 WelcomeDeliveryState::Sent | WelcomeDeliveryState::SendAttempted => {}
1395 WelcomeDeliveryState::Created | WelcomeDeliveryState::Failed => {
1396 if let Some(entry) = self.welcome_lifecycles.get_mut(peer_id) {
1397 entry.attempt = 0;
1398 // A reachability edge resets the unreachable-park
1399 // escalation: the next relay verdict starts the interval
1400 // ladder over from its base.
1401 entry.unreachable_parks = 0;
1402 entry.next_retry_at = Some(Utc::now());
1403 entry.last_reason_code = None;
1404 entry.last_transport_error = None;
1405 entry.expires_at =
1406 Utc::now() + ChronoDuration::seconds(WELCOME_LIFECYCLE_TTL_SECS);
1407 }
1408 if let Err(err) = self.try_send_welcome(peer_id, source_event, false) {
1409 warn!(
1410 peer_id = %peer_id,
1411 error = %err,
1412 "Failed to re-arm welcome on peer reachability"
1413 );
1414 }
1415 }
1416 WelcomeDeliveryState::Expired => {
1417 // Expired is terminal in the lifecycle state machine, so rebuild
1418 // from the retained Welcome message (upsert permits overwrite
1419 // from Expired) before re-sending.
1420 let welcome_message = record.welcome_message.clone();
1421 let group_id = record.group_id.clone();
1422 if let Err(err) =
1423 self.upsert_welcome_lifecycle(peer_id, &group_id, welcome_message, source_event)
1424 {
1425 warn!(
1426 peer_id = %peer_id,
1427 error = %err,
1428 "Failed to rebuild expired welcome on peer reachability"
1429 );
1430 return;
1431 }
1432 if let Err(err) = self.try_send_welcome(peer_id, source_event, false) {
1433 warn!(
1434 peer_id = %peer_id,
1435 error = %err,
1436 "Failed to re-arm expired welcome on peer reachability"
1437 );
1438 }
1439 }
1440 }
1441 }
1442
1443 /// Pushes our key package to `peer_id` again when an earlier push produced
1444 /// no session.
1445 ///
1446 /// Invariant: a key package we pushed stands in for an exchange only until
1447 /// the interval lapses; past it, no session with the peer means the push
1448 /// is treated as lost. Without this the push is fire-once: a carrier can
1449 /// drop that one frame without reporting it (a superseded peer stream
1450 /// drops its late frames, `docs/spec/stream-framing.md`), the peer is already marked
1451 /// as sent to, and when both sides' pushes are lost neither ever sends
1452 /// again, so the pair never forms a session.
1453 ///
1454 /// The re-push is the same package (ADR 0012): the pool hands a peer its
1455 /// own live package until a Welcome consumes it, so no key material is
1456 /// minted. A session that exists but is unconfirmed is left to the Welcome
1457 /// lifecycle and the confirmation probes, which own that half.
1458 ///
1459 /// Only a peer already pushed to qualifies; the first push belongs to
1460 /// discovery, except for a peer only the mesh reaches, which discovery
1461 /// never names (see [`Self::push_key_package_across_mesh`]). The stamp is taken before the send, so a send that fails
1462 /// still waits out the interval instead of being retried on every tick.
1463 /// A send that fails does not count as a repeat, though: the backoff is
1464 /// for a peer that was asked and did not answer, and a peer whose frame
1465 /// never left was not asked. Counting it would walk an unreachable peer
1466 /// up to the cap while it is away, and nothing resets the count when it
1467 /// comes back over a carrier with no neighbour-lost event.
1468 ///
1469 /// A blocked peer is never pushed to. Blocking is bidirectional, and the
1470 /// tick reaches this for any peer with a queued message, which a block
1471 /// leaves in place; without the check the tick would advertise our
1472 /// presence and key package to the blocked peer every window.
1473 pub(super) fn rearm_key_package_for_peer(&mut self, peer_id: &str) {
1474 if !self.config.encryption.enabled
1475 || !self.config.encryption.auto_key_exchange
1476 || self.confirmed_sessions.contains(peer_id)
1477 || self.is_user_blocked(peer_id)
1478 {
1479 return;
1480 }
1481 let Some((sent_at, repeats)) = self.key_package_sent_to.get_mut(peer_id) else {
1482 self.push_key_package_across_mesh(peer_id);
1483 return;
1484 };
1485 if sent_at.elapsed() < StdDuration::from_secs(key_package_resend_wait_secs(*repeats)) {
1486 return;
1487 }
1488 // Stamped whatever the storage check below says: it is storage I/O,
1489 // and discovery runs on every inbound body, so an unconfirmed session
1490 // (or a failing read) must cost one check per wait, not one per
1491 // frame. A check that finds a session, and a push that leaves, each
1492 // double the wait, so a peer that never answers settles at one push
1493 // per cap.
1494 *sent_at = Instant::now();
1495 let backed_off = match self.has_mls_session(peer_id) {
1496 Ok(false) => {
1497 debug!(peer_id = %peer_id, "Key package produced no session, pushing it again");
1498 match self.send_key_package_to(peer_id, false) {
1499 Ok(()) => true,
1500 Err(e) => {
1501 debug!(peer_id = %peer_id, error = %e, "Key package re-push deferred");
1502 false
1503 }
1504 }
1505 }
1506 Ok(true) | Err(_) => true,
1507 };
1508 if backed_off {
1509 if let Some((_, repeats)) = self.key_package_sent_to.get_mut(peer_id) {
1510 *repeats = repeats.saturating_add(1);
1511 }
1512 }
1513 }
1514
1515 /// Makes the first key-package push to a peer that only the mesh can
1516 /// reach, for whom a message is waiting.
1517 ///
1518 /// Invariant: a message queued for a session is enough to start that
1519 /// session over any path that can carry a frame to the peer. The first
1520 /// push is otherwise discovery's, and discovery names only the device a
1521 /// frame arrived from: on a peer two hops away every frame arrives
1522 /// attributed to the device in the middle, so the pair is never
1523 /// discovered and, before this, never started a session. A message to a
1524 /// device the sender had never been next to waited out its seven days.
1525 ///
1526 /// Three things keep it narrow:
1527 ///
1528 /// - **Only with a message waiting.** The caller is the reconciliation
1529 /// tick, for peers holding a queued message, so the package goes to
1530 /// addresses the application asked to reach, never to whoever is heard.
1531 /// - **Only when nothing else reaches the peer.** A direct link means
1532 /// discovery has already pushed, and an infrastructure carrier has its
1533 /// own exchange; both are left alone.
1534 /// - **Only once by this route.** A push that a neighbour took is recorded
1535 /// like any other, so every repeat after it is the re-push ladder's, on
1536 /// its doubling wait. A push nobody took is tried again on the next
1537 /// tick, and costs nothing new: the pool hands the peer the same
1538 /// package until a Welcome consumes it (ADR 0012).
1539 fn push_key_package_across_mesh(&mut self, peer_id: &str) {
1540 if self.can_reach_recipient(peer_id) || self.transport_manager.mesh_neighbors().is_empty() {
1541 return;
1542 }
1543 if !matches!(self.has_mls_session(peer_id), Ok(false)) {
1544 return;
1545 }
1546 debug!(peer_id = %peer_id, "Message waiting for a peer only the mesh reaches, pushing our key package");
1547 if let Err(e) = self.send_key_package_to(peer_id, false) {
1548 debug!(peer_id = %peer_id, error = %e, "Key package push across the mesh deferred");
1549 }
1550 }
1551
1552 /// Ingests an authoritative per-peer presence signal (the internet relay's
1553 /// `PresenceStatusWithLastSeen` answer, bridged by the platform layer).
1554 ///
1555 /// `online` drives the same reachability machinery as a transport-level
1556 /// discovery — outbox flush, welcome re-arm, auto key exchange — plus a
1557 /// rescue for the store-less-relay hole: a welcome whose wire send was
1558 /// confirmed (`Sent`) while the peer was actually offline was dropped by
1559 /// the relay (it forwards or pushes, never stores), and
1560 /// `Self::rearm_welcome_for_peer` deliberately never touches `Sent`.
1561 /// The peer being provably online is the safe moment to rebuild and
1562 /// re-send it: if the original did land, the receiver dedups by message
1563 /// id and the confirmation probe resolves the lifecycle.
1564 ///
1565 /// `offline` parks retry-pending welcomes pending a reachability edge,
1566 /// without burning budget. In-flight records (`SendAttempted`) are left
1567 /// to their confirm deadline — a `DeliveryError`-driven failure or the
1568 /// confirm timeout resolves them, after which the next offline tick parks.
1569 ///
1570 /// The welcome re-arm/re-send on the online edge is throttled per peer
1571 /// with exponential backoff (`WELCOME_PRESENCE_RESCUE_BASE_SECS` ..
1572 /// `WELCOME_PRESENCE_RESCUE_MAX_SECS`): presence answers arrive on the
1573 /// platform's ~20s watch tick, and a peer that is online but can never
1574 /// prove the session (stale key package after a reinstall, incompatible
1575 /// version) must not be re-sent the multi-frame MLS welcome on every
1576 /// tick forever. A throttled online tick still flushes queued data-plane
1577 /// traffic for the peer.
1578 ///
1579 /// Emits `presence_updated` naming the carrier that answered — this
1580 /// function is the gateway-sourced half of the unified stream and must
1581 /// only be fed a gateway's own observation (bridged via
1582 /// `internet_peer_presence` or `reticulum_peer_presence`); peer-sent
1583 /// `__PRESENCE__` self-reports are handled in `message_dispatch` and emit
1584 /// `source: Peer`. Self, blocked, or empty peer ids are dropped entirely:
1585 /// an app waiting on `presence_updated` for a blocked peer will never see
1586 /// it.
1587 pub fn on_peer_presence(&mut self, peer_id: &str, online: bool, last_seen_ms: Option<i64>) {
1588 self.on_peer_presence_via(peer_id, online, last_seen_ms, GatewayCarrier::Internet);
1589 }
1590
1591 /// [`Self::on_peer_presence`], for whichever gateway answered.
1592 ///
1593 /// The carrier is threaded rather than assumed because every effect below
1594 /// is carrier-scoped and silently wrong under the wrong one: the fact is
1595 /// recorded against the carrier that made the claim, the un-park flush is
1596 /// pinned to the carrier that just proved it can reach the peer, and the
1597 /// event names the carrier so an app filtering on relay-observed presence
1598 /// still can. A Reticulum gateway's answer forcing a send over the
1599 /// Internet transport would be a reachability proof about one carrier
1600 /// spent on another.
1601 pub fn on_peer_presence_via(
1602 &mut self,
1603 peer_id: &str,
1604 online: bool,
1605 last_seen_ms: Option<i64>,
1606 carrier: GatewayCarrier,
1607 ) {
1608 if peer_id.is_empty() || peer_id == self.local_id || self.is_user_blocked(peer_id) {
1609 return;
1610 }
1611 // A presence answer is a claim about this recipient on the carrier
1612 // that answered, so it is recorded as one. It supersedes an earlier
1613 // verdict for that carrier: both answer the same question and this is
1614 // the later answer. Presence facts decay faster than verdicts, since
1615 // this is a third party's report about someone else's connection.
1616 self.reachability.record(
1617 peer_id,
1618 carrier.transport(),
1619 if online {
1620 Claim::Reachable
1621 } else {
1622 Claim::Unreachable
1623 },
1624 FactSource::GatewayPresence,
1625 std::time::Instant::now(),
1626 );
1627 if online {
1628 // Carrier-scoped reachability proof: un-park and re-drive the
1629 // peer's DMs over that carrier before anything else
1630 // (see `flush_outbox_for_peer_via` — this also cancels
1631 // unanswerable mesh-probe ACKs that would otherwise hold the
1632 // messages hostage past this edge). The rescue branch's inner
1633 // flush then finds successfully re-driven entries awaiting
1634 // fresh ACKs and leaves them alone; entries whose forced send
1635 // *failed* are picked up again, so the inner flush keeps the
1636 // carrier override — DORS must not re-route them into the
1637 // mesh void with the park counter already cleared.
1638 self.flush_outbox_for_peer_via(peer_id, Some(carrier.transport()));
1639 if self.welcome_rescue_permitted(peer_id) {
1640 self.on_neighbor_discovered_via(peer_id, Some(carrier.transport()));
1641 self.resend_unconfirmed_sent_welcome(peer_id, "peer_presence_online");
1642 self.note_welcome_rescue_attempt(peer_id);
1643 }
1644 // A reachability edge resets the session-confirmation probe's
1645 // unreachable backoff: clear the escalation counter and mark a probe
1646 // due now, so a peer that went unreachable while unconfirmed is
1647 // re-probed immediately and its session converges rather than
1648 // waiting out the (up to 600s) backoff.
1649 if self.confirmation_probe_due_at.contains_key(peer_id) {
1650 self.confirmation_probe_unreachable_parks.remove(peer_id);
1651 self.confirmation_probe_due_at
1652 .insert(peer_id.to_string(), Utc::now());
1653 }
1654 } else {
1655 self.park_welcome_peer_unreachable(peer_id);
1656 }
1657
1658 let status = if online {
1659 crate::events::PresenceStatus::Online
1660 } else {
1661 crate::events::PresenceStatus::Offline
1662 };
1663 self.emit_event(Event::presence_updated_from(
1664 peer_id.to_string(),
1665 status,
1666 Utc::now().timestamp_millis(),
1667 last_seen_ms,
1668 carrier.presence_source(),
1669 ));
1670 }
1671
1672 /// Peers with a welcome lifecycle whose session has not been proven —
1673 /// the internet presence watch set. Includes `Sent` records (wire-
1674 /// confirmed but never session-confirmed): only a presence signal can
1675 /// rescue those over a store-less relay.
1676 /// Lifecycles older than `WELCOME_WATCHLIST_MAX_AGE_SECS` are excluded:
1677 /// a permanently-dead peer must not occupy watch-rotation slots (and keep
1678 /// its parked record alive via `expires_at` pushes) forever.
1679 pub fn welcome_pending_peers(&self) -> Vec<String> {
1680 let watch_cutoff = Utc::now() - ChronoDuration::seconds(WELCOME_WATCHLIST_MAX_AGE_SECS);
1681 self.welcome_lifecycles
1682 .iter()
1683 .filter(|(peer_id, record)| {
1684 !self.confirmed_sessions.contains(*peer_id) && record.created_at > watch_cutoff
1685 })
1686 .map(|(peer_id, _)| peer_id.clone())
1687 .collect()
1688 }
1689
1690 /// Rebuilds and re-sends a `Sent`-but-session-unconfirmed welcome (see
1691 /// [`Self::on_peer_presence`]). No-op for any other state.
1692 fn resend_unconfirmed_sent_welcome(&mut self, peer_id: &str, source_event: &str) {
1693 if self.confirmed_sessions.contains(peer_id) {
1694 return;
1695 }
1696 let Some(record) = self.welcome_lifecycles.get(peer_id) else {
1697 return;
1698 };
1699 if !matches!(record.state, WelcomeDeliveryState::Sent) {
1700 return;
1701 }
1702 let welcome_message = record.welcome_message.clone();
1703 let group_id = record.group_id.clone();
1704 if let Err(err) =
1705 self.upsert_welcome_lifecycle(peer_id, &group_id, welcome_message, source_event)
1706 {
1707 warn!(
1708 peer_id = %peer_id,
1709 error = %err,
1710 "Failed to rebuild unconfirmed sent welcome on presence"
1711 );
1712 return;
1713 }
1714 if let Err(err) = self.try_send_welcome(peer_id, source_event, false) {
1715 warn!(
1716 peer_id = %peer_id,
1717 error = %err,
1718 "Failed to re-send unconfirmed welcome on presence"
1719 );
1720 }
1721 }
1722
1723 /// True when a presence-online rescue for this peer is currently allowed
1724 /// by the per-peer backoff. Trivially true (and the throttle entry is
1725 /// dropped) once the peer has no unconfirmed welcome — there is nothing
1726 /// left to throttle.
1727 fn welcome_rescue_permitted(&mut self, peer_id: &str) -> bool {
1728 let pending = self.welcome_lifecycles.contains_key(peer_id)
1729 && !self.confirmed_sessions.contains(peer_id);
1730 if !pending {
1731 self.welcome_presence_rescue.remove(peer_id);
1732 return true;
1733 }
1734 self.welcome_presence_rescue
1735 .get(peer_id)
1736 .is_none_or(|throttle| throttle.next_allowed_at <= Utc::now())
1737 }
1738
1739 /// Records that a presence-online tick ran the rescue actions for a peer
1740 /// that still has an unconfirmed welcome, doubling the wait before the
1741 /// next one (base 40s, capped at 10 minutes). The counter is per
1742 /// convergence attempt: the entry is dropped (via
1743 /// [`Self::welcome_rescue_permitted`]) as soon as the session confirms
1744 /// or the lifecycle goes away.
1745 fn note_welcome_rescue_attempt(&mut self, peer_id: &str) {
1746 let pending = self.welcome_lifecycles.contains_key(peer_id)
1747 && !self.confirmed_sessions.contains(peer_id);
1748 if !pending {
1749 self.welcome_presence_rescue.remove(peer_id);
1750 return;
1751 }
1752 let throttle = self
1753 .welcome_presence_rescue
1754 .entry(peer_id.to_string())
1755 .or_insert(PresenceRescueThrottle {
1756 next_allowed_at: Utc::now(),
1757 rescues: 0,
1758 });
1759 // Shift is clamped to 8, so 40 << 8 = 10240 max before the cap —
1760 // no overflow risk.
1761 let backoff_secs = (WELCOME_PRESENCE_RESCUE_BASE_SECS << throttle.rescues.min(8))
1762 .min(WELCOME_PRESENCE_RESCUE_MAX_SECS);
1763 throttle.next_allowed_at = Utc::now() + ChronoDuration::seconds(backoff_secs);
1764 throttle.rescues = throttle.rescues.saturating_add(1);
1765 }
1766
1767 /// Parks a welcome pending a peer-reachability edge: reason
1768 /// `PeerUnreachable`, **no timed retry**. The carrier (relay socket) is
1769 /// healthy — only this peer is unreachable on it — so the *data-plane*
1770 /// retry this cancels would re-send the welcome into another
1771 /// `DeliveryError` while burning the real-delivery budget that a
1772 /// carrier-backed failure already charged. Recovery is edge-driven
1773 /// instead: `on_peer_presence(online)` and `on_neighbor_discovered` re-arm
1774 /// via `rearm_welcome_for_peer`, and the peer stays on the presence
1775 /// watchlist (`welcome_pending_peers`) so the platform keeps polling for
1776 /// that edge. The TTL is pushed like a no-carrier park: an unreachable
1777 /// peer must not age the welcome out.
1778 ///
1779 /// Its caller is responsible for not applying this to a record already
1780 /// holding an escalating unreachable probe — see
1781 /// [`Self::park_welcome_peer_unreachable`], which is what distinguishes
1782 /// the budget-burning data-plane retry cancelled here from the
1783 /// budget-refunded probe that must survive.
1784 fn park_welcome_awaiting_peer(&mut self, peer_id: &str) -> Result<()> {
1785 let snapshot = {
1786 let Some(record) = self.welcome_lifecycles.get_mut(peer_id) else {
1787 return Ok(());
1788 };
1789 record.next_retry_at = None;
1790 record.expires_at = Utc::now() + ChronoDuration::seconds(WELCOME_LIFECYCLE_TTL_SECS);
1791 record.last_reason_code = Some(crate::events::WelcomeReasonCode::PeerUnreachable);
1792 record.clone()
1793 };
1794 self.persist_welcome_lifecycle_entry(&snapshot)?;
1795 debug!(
1796 peer_id = %peer_id,
1797 state = snapshot.state.as_str(),
1798 "Welcome parked: peer unreachable on carrier, awaiting reachability edge"
1799 );
1800 Ok(())
1801 }
1802
1803 /// Parks a retry-pending welcome for a peer the relay reports offline.
1804 /// Only `Created`/`Failed` records park: `SendAttempted` keeps its
1805 /// confirm deadline, and `Sent`/`Expired` are handled on the online edge
1806 /// (or corrected by an attributed `DeliveryError`). Skipped entirely when
1807 /// a mesh carrier is also available — relay presence is only
1808 /// authoritative for the internet path, and the data-plane retry track
1809 /// may still deliver over BLE / WiFi-Direct.
1810 ///
1811 /// Never cancels an escalating unreachable probe: a record that already
1812 /// holds one was parked by [`Self::apply_recipient_unreachable_failure`],
1813 /// which already set the same reason code and pushed the same TTL — so this
1814 /// adds nothing and would only downgrade the probe to edge-only. That
1815 /// matters because on an internet-only device this runs once per presence
1816 /// rotation for as long as the peer is down, which would otherwise
1817 /// re-strand the welcome after every single verdict. The distinction is
1818 /// budget: the data-plane retry this *does* cancel was charged an attempt
1819 /// by a carrier-backed failure, while the probe refunds its attempt on
1820 /// every verdict.
1821 ///
1822 /// `last_reason_code` — not `unreachable_parks` alone — is what encodes
1823 /// that distinction. The counter is sticky (only `rearm_welcome_for_peer`
1824 /// clears it), so gating on it by itself would keep shielding the record
1825 /// after the probe track has stopped protecting it: past
1826 /// [`WELCOME_WATCHLIST_MAX_AGE_SECS`] a confirm timeout no longer re-parks
1827 /// (see [`Self::welcome_probe_repark_permitted`]) and instead charges an
1828 /// attempt and arms a plain data-plane retry — precisely the budget-burner
1829 /// this park exists to cancel, and left running it expires the welcome at
1830 /// `max_retries`. Note `park_welcome_awaiting_peer` also stamps
1831 /// `PeerUnreachable` but leaves `next_retry_at = None`, so the
1832 /// `next_retry_at.is_some()` conjunct below is load-bearing, not
1833 /// redundant: the reason code *and* a scheduled retry together are what
1834 /// identify a live probe. A carrier-backed failure stamps
1835 /// `Timeout`/`SendFailed` and a successful send clears the code entirely.
1836 fn park_welcome_peer_unreachable(&mut self, peer_id: &str) {
1837 if self.confirmed_sessions.contains(peer_id) {
1838 return;
1839 }
1840 let Some(record) = self.welcome_lifecycles.get(peer_id) else {
1841 return;
1842 };
1843 if !matches!(
1844 record.state,
1845 WelcomeDeliveryState::Created | WelcomeDeliveryState::Failed
1846 ) {
1847 return;
1848 }
1849 if record.last_reason_code == Some(crate::events::WelcomeReasonCode::PeerUnreachable)
1850 && record.unreachable_parks > 0
1851 && record.next_retry_at.is_some()
1852 {
1853 return;
1854 }
1855 // Local mesh carriers only (BLE / WiFi-Direct): Nostr and Reticulum
1856 // are internet-dependent, so their availability must not veto the
1857 // park — mirrors the carrier guard in
1858 // `apply_recipient_unreachable_failure`.
1859 if self
1860 .transport_manager
1861 .get_available_transports()
1862 .keys()
1863 .any(|transport| matches!(transport, TransportType::BLE | TransportType::WiFiDirect))
1864 {
1865 return;
1866 }
1867 let _ = self.park_welcome_awaiting_peer(peer_id);
1868 }
1869
1870 /// True when an unconfirmed welcome send should resolve as another
1871 /// unreachable verdict ([`Self::apply_recipient_unreachable_failure`])
1872 /// rather than as a carrier-backed failure that ages the record.
1873 ///
1874 /// A live `unreachable_parks` counter means the relay declared the peer
1875 /// offline and no reachability edge has fired since (every edge clears it
1876 /// via `rearm_welcome_for_peer`), so the send that just went unconfirmed
1877 /// was the escalating probe, not a delivery attempt. Without this the
1878 /// probe destroys the very welcome it exists to recover: the relay
1879 /// accepts the frame whenever its push fallback succeeds, which returns
1880 /// *no* `DeliveryError` at all, so the probe resolves at the 10s confirm
1881 /// timeout instead. Treated as carrier-backed that charges the attempt
1882 /// and arms a plain data-plane retry, and since nothing in that ladder
1883 /// pushes `expires_at`, the record walks into `should_expire` within one
1884 /// TTL window — terminal `welcome_send_expired` + `secure_session_failed`
1885 /// for a peer that is merely offline. Re-parking instead refunds the
1886 /// attempt, escalates the interval and pushes the TTL, mirroring the DM
1887 /// path's [`Self::try_repark_exhausted_dm`]: settlement stays reserved
1888 /// for delivery or the record genuinely ageing out below.
1889 ///
1890 /// Only the *presence-offline* answer cancelled that retry before, which
1891 /// is not a defense at all for the case this whole probe exists to serve
1892 /// — a headless consumer that never polls presence.
1893 ///
1894 /// Bounded by [`WELCOME_WATCHLIST_MAX_AGE_SECS`] measured from
1895 /// `created_at`, the same threshold that stops watching a peer as
1896 /// permanently dead: past it the counter no longer shields the record and
1897 /// normal ageing expires it. This is the welcome's twin of the DM probe's
1898 /// absolute bound (`first_sent_at` vs the outbox absolute lifetime) —
1899 /// without it a peer that never returns keeps a welcome probing at the
1900 /// 600s cap forever.
1901 ///
1902 /// Deliberately narrow to `Timeout`: a `SendFailed`/`TransportUnavailable`
1903 /// failure is evidence about the *carrier*, not about this peer, and
1904 /// keeps ageing the record normally.
1905 fn welcome_probe_repark_permitted(&self, peer_id: &str) -> bool {
1906 let Some(record) = self.welcome_lifecycles.get(peer_id) else {
1907 return false;
1908 };
1909 record.unreachable_parks > 0
1910 && record.created_at
1911 > Utc::now() - ChronoDuration::seconds(WELCOME_WATCHLIST_MAX_AGE_SECS)
1912 }
1913
1914 /// Applies the relay's authoritative "recipient offline" verdict
1915 /// (a `recipient_unreachable`-tagged transport failure) to a welcome
1916 /// lifecycle. Also reached *without* a relay verdict when a probe's
1917 /// confirm timeout resolves while the park counter is still live — the
1918 /// same conclusion inferred rather than relayed, see
1919 /// [`Self::welcome_probe_repark_permitted`].
1920 ///
1921 /// The bridge wire-confirms on socket-write success, so by the time the
1922 /// relay's `DeliveryError` arrives the record is normally already `Sent`
1923 /// — a *false* Sent: the store-less relay dropped the content. Correct
1924 /// it back to `Failed` (emitting `welcome_send_failed` so the app's
1925 /// earlier `welcome_send_succeeded` is superseded), refund the attempt
1926 /// (the peer never saw the frame), and park pending a reachability edge.
1927 /// `SendAttempted` (the failure raced ahead of the wire confirm) gets
1928 /// the same treatment; `Created`/`Failed` records just park. Terminal
1929 /// `Expired` records are left to the online edge, which rebuilds them.
1930 ///
1931 /// The record always keeps a timed retry whose interval escalates per
1932 /// consecutive park. Relay presence is authoritative only for the
1933 /// internet path — with a mesh carrier live the peer may still be
1934 /// reachable over BLE / WiFi-Direct (DORS can pick internet even with
1935 /// mesh present) — but an internet-only device needs the timed track just
1936 /// as much: parking it edge-only left the welcome dependent solely on a
1937 /// presence-online answer, i.e. on the platform's presence-polling
1938 /// cadence, which is the same dead end that stalled parked DMs (see
1939 /// [`Self::park_unreachable_dm`]). The escalation (15s → 600s cap) is
1940 /// what bounds the retry: DORS may keep routing to the internet path,
1941 /// where each round trips another budget-refunded `DeliveryError`, and
1942 /// without escalation that would be an unbounded resend loop into the
1943 /// relay. Every verdict refunds the attempt and pushes the TTL — past the
1944 /// scheduled probe, see the invariant on [`Self::park_welcome_no_carrier`]
1945 /// — so the probes never age the welcome out or burn its real-delivery
1946 /// budget, and the counter resets on any reachability edge
1947 /// (`rearm_welcome_for_peer`).
1948 pub(super) fn apply_recipient_unreachable_failure(
1949 &mut self,
1950 peer_id: &str,
1951 transport_error: Option<&'static str>,
1952 ) -> Result<()> {
1953 // A late relay verdict for a session that has since been proven (the
1954 // welcome was rescued over another path and the peer confirmed) must
1955 // not corrupt the converged lifecycle: without this guard the stale
1956 // DeliveryError flips the record back to Failed, refunds an attempt,
1957 // persists the stale state, and emits welcome_send_failed AFTER the
1958 // app already saw secure_session_established.
1959 if self.confirmed_sessions.contains(peer_id) {
1960 return Ok(());
1961 }
1962 let Some(state) = self.welcome_lifecycles.get(peer_id).map(|r| r.state) else {
1963 return Ok(());
1964 };
1965 match state {
1966 WelcomeDeliveryState::Sent | WelcomeDeliveryState::SendAttempted => {
1967 self.transition_welcome_state(
1968 peer_id,
1969 WelcomeDeliveryState::Failed,
1970 "recipient_unreachable",
1971 )?;
1972 if let Some(record) = self.welcome_lifecycles.get_mut(peer_id) {
1973 record.attempt = record.attempt.saturating_sub(1);
1974 record.last_transport_error = transport_error.map(str::to_string);
1975 }
1976 }
1977 WelcomeDeliveryState::Created | WelcomeDeliveryState::Failed => {
1978 if let Some(record) = self.welcome_lifecycles.get_mut(peer_id) {
1979 record.last_transport_error = transport_error.map(str::to_string);
1980 }
1981 }
1982 WelcomeDeliveryState::Expired => return Ok(()),
1983 }
1984 // Escalating timed retry on every carrier (see the doc comment): the
1985 // ladder, not a carrier guard, is what bounds resends into the relay.
1986 let parks = {
1987 let Some(record) = self.welcome_lifecycles.get_mut(peer_id) else {
1988 return Ok(());
1989 };
1990 record.unreachable_parks = record.unreachable_parks.saturating_add(1);
1991 record.unreachable_parks
1992 };
1993 // parks >= 1 here; shift clamped so 15 << 6 = 960 is the largest
1994 // pre-cap value — no overflow risk.
1995 let retry_in_secs = (WELCOME_NO_CARRIER_RETRY_SECS << (parks - 1).min(6))
1996 .min(WELCOME_UNREACHABLE_RETRY_CAP_SECS);
1997 self.park_welcome_no_carrier(
1998 peer_id,
1999 crate::events::WelcomeReasonCode::PeerUnreachable,
2000 transport_error,
2001 retry_in_secs,
2002 )?;
2003
2004 let Some(snapshot) = self.welcome_lifecycles.get(peer_id).cloned() else {
2005 return Ok(());
2006 };
2007 if let Ok(shared) = lock_shared_state(&self.shared_state) {
2008 shared.emit_event(Event::welcome_send_failed(
2009 peer_id.to_string(),
2010 snapshot.welcome_message.id.as_str().to_string(),
2011 snapshot.group_id.clone(),
2012 snapshot.attempt,
2013 crate::events::WelcomeReasonCode::PeerUnreachable,
2014 snapshot
2015 .last_transport_error
2016 .as_deref()
2017 .map(classify_transport_send_error),
2018 // Retryable, and now always with a scheduled time: the park
2019 // keeps an escalating timed probe on every carrier. The
2020 // reachability edges (presence online / peer discovery) still
2021 // re-arm it sooner when they fire.
2022 true,
2023 snapshot.next_retry_at.map(|at| at.timestamp_millis()),
2024 ));
2025 }
2026 Ok(())
2027 }
2028
2029 /// Records a Welcome send failure and decides whether to retry or expire.
2030 ///
2031 /// `no_carrier` is `true` when the failure was simply that no transport
2032 /// carrier exists yet (vs. a present carrier that failed mid-send). A
2033 /// no-carrier failure must NOT age the Welcome: it neither counts toward
2034 /// the retry budget nor lets the TTL expire it, because the peer is merely
2035 /// unreachable for now and will be retried once a carrier appears. The
2036 /// speculative attempt increment from [`Self::try_send_welcome`] is rolled
2037 /// back so a long offline period leaves the real-delivery budget intact.
2038 pub(super) fn apply_welcome_send_failure(
2039 &mut self,
2040 peer_id: &str,
2041 reason: crate::events::WelcomeReasonCode,
2042 transport_error: Option<&'static str>,
2043 no_carrier: bool,
2044 source_event: &str,
2045 ) -> Result<bool> {
2046 let mut updated = self
2047 .welcome_lifecycles
2048 .get(peer_id)
2049 .cloned()
2050 .ok_or_else(|| Error::Other(format!("Missing welcome lifecycle for {}", peer_id)))?;
2051
2052 if matches!(
2053 updated.state,
2054 WelcomeDeliveryState::Sent | WelcomeDeliveryState::Expired
2055 ) {
2056 return Ok(matches!(updated.state, WelcomeDeliveryState::Sent));
2057 }
2058
2059 // A confirm timeout on a welcome that still holds a live
2060 // unreachable-park counter is a *probe* resolving, not a delivery
2061 // failing — re-park it as another unreachable verdict instead of
2062 // ageing it (see [`Self::welcome_probe_repark_permitted`]).
2063 if !no_carrier
2064 && matches!(reason, crate::events::WelcomeReasonCode::Timeout)
2065 && self.welcome_probe_repark_permitted(peer_id)
2066 {
2067 self.apply_recipient_unreachable_failure(peer_id, transport_error)?;
2068 return Ok(false);
2069 }
2070
2071 let max_attempts = self.config.reliability.retry.max_retries.max(1);
2072 // A no-carrier failure never expires the Welcome: neither the retry
2073 // budget nor the TTL applies while the peer is simply unreachable. Only
2074 // a carrier-backed failure (SendFailed / Timeout) ages it toward expiry.
2075 let should_expire =
2076 !no_carrier && (updated.attempt >= max_attempts || updated.expires_at <= Utc::now());
2077 if should_expire {
2078 let terminal_reason = crate::events::WelcomeReasonCode::RetryExhausted;
2079 {
2080 let record = self.welcome_lifecycles.get_mut(peer_id).ok_or_else(|| {
2081 Error::Other(format!("Missing welcome lifecycle for {}", peer_id))
2082 })?;
2083 record.last_reason_code = Some(terminal_reason);
2084 record.last_transport_error = transport_error.map(str::to_string);
2085 record.next_retry_at = None;
2086 }
2087
2088 if !matches!(updated.state, WelcomeDeliveryState::Failed) {
2089 self.transition_welcome_state(peer_id, WelcomeDeliveryState::Failed, source_event)?;
2090 }
2091 self.transition_welcome_state(
2092 peer_id,
2093 WelcomeDeliveryState::Expired,
2094 "welcome_retry_exhausted",
2095 )?;
2096
2097 let expired_snapshot =
2098 self.welcome_lifecycles
2099 .get(peer_id)
2100 .cloned()
2101 .ok_or_else(|| {
2102 Error::Other(format!("Missing welcome lifecycle for {}", peer_id))
2103 })?;
2104 if let Ok(state) = lock_shared_state(&self.shared_state) {
2105 state.emit_event(Event::welcome_send_failed(
2106 peer_id.to_string(),
2107 expired_snapshot.welcome_message.id.as_str().to_string(),
2108 expired_snapshot.group_id.clone(),
2109 expired_snapshot.attempt,
2110 terminal_reason,
2111 expired_snapshot
2112 .last_transport_error
2113 .as_deref()
2114 .map(classify_transport_send_error),
2115 false,
2116 None,
2117 ));
2118 state.emit_event(Event::welcome_send_expired(
2119 peer_id.to_string(),
2120 expired_snapshot.welcome_message.id.as_str().to_string(),
2121 expired_snapshot.attempt,
2122 terminal_reason,
2123 ));
2124 }
2125 self.abort_pending_session_for_peer(peer_id, terminal_reason);
2126 return Ok(false);
2127 }
2128
2129 // A no-carrier failure (the carrier raced away after the pre-send check,
2130 // or an async transport-failed callback with no carrier) re-checks on the
2131 // slow no-carrier interval rather than the data-plane backoff: there is
2132 // nothing to deliver over until a carrier returns, and `try_send_welcome`
2133 // parks subsequent no-carrier ticks cheaply anyway.
2134 let retry_at = if no_carrier {
2135 Utc::now() + ChronoDuration::seconds(WELCOME_NO_CARRIER_RETRY_SECS)
2136 } else {
2137 let delay_ms = self.compute_welcome_retry_delay_ms(peer_id, updated.attempt);
2138 Utc::now() + ChronoDuration::milliseconds(delay_ms as i64)
2139 };
2140
2141 {
2142 let record = self.welcome_lifecycles.get_mut(peer_id).ok_or_else(|| {
2143 Error::Other(format!("Missing welcome lifecycle for {}", peer_id))
2144 })?;
2145 // Roll back the speculative attempt increment from try_send_welcome
2146 // for a no-carrier failure so the retry budget stays reserved for
2147 // real delivery attempts over an available carrier, and push the TTL
2148 // forward so a never-deliverable Welcome does not age out — the TTL
2149 // clock is carrier-relative, not creation-relative.
2150 if no_carrier {
2151 record.attempt = record.attempt.saturating_sub(1);
2152 record.expires_at =
2153 Utc::now() + ChronoDuration::seconds(WELCOME_LIFECYCLE_TTL_SECS);
2154 }
2155 record.last_reason_code = Some(reason);
2156 record.last_transport_error = transport_error.map(str::to_string);
2157 record.next_retry_at = Some(retry_at);
2158 }
2159
2160 if !matches!(updated.state, WelcomeDeliveryState::Failed) {
2161 self.transition_welcome_state(peer_id, WelcomeDeliveryState::Failed, source_event)?;
2162 } else if let Some(record) = self.welcome_lifecycles.get(peer_id).cloned() {
2163 self.persist_welcome_lifecycle_entry(&record)?;
2164 }
2165
2166 updated = self
2167 .welcome_lifecycles
2168 .get(peer_id)
2169 .cloned()
2170 .ok_or_else(|| Error::Other(format!("Missing welcome lifecycle for {}", peer_id)))?;
2171 if let Ok(state) = lock_shared_state(&self.shared_state) {
2172 state.emit_event(Event::welcome_send_failed(
2173 peer_id.to_string(),
2174 updated.welcome_message.id.as_str().to_string(),
2175 updated.group_id,
2176 updated.attempt,
2177 reason,
2178 updated
2179 .last_transport_error
2180 .as_deref()
2181 .map(classify_transport_send_error),
2182 true,
2183 Some(retry_at.timestamp_millis()),
2184 ));
2185 }
2186 Ok(false)
2187 }
2188
2189 pub(super) fn compute_welcome_retry_delay_ms(&self, peer_id: &str, attempt: u32) -> u64 {
2190 let config = &self.config.reliability.retry;
2191 let capped_attempt = attempt.saturating_sub(1);
2192 let base_ms = if capped_attempt == 0 {
2193 config.initial_delay_ms
2194 } else {
2195 let multiplier = config.backoff_multiplier.powi(capped_attempt as i32);
2196 (config.initial_delay_ms as f64 * multiplier as f64) as u64
2197 }
2198 .min(config.max_delay_ms);
2199
2200 let mut hasher = std::collections::hash_map::DefaultHasher::new();
2201 peer_id.hash(&mut hasher);
2202 attempt.hash(&mut hasher);
2203 Utc::now().timestamp_millis().hash(&mut hasher);
2204 let bucket = (hasher.finish() % 10_000) as f64 / 10_000.0;
2205 let jitter_factor = 1.0 + ((bucket * 2.0 - 1.0) * WELCOME_RETRY_JITTER_RATIO);
2206 let jittered = (base_ms as f64 * jitter_factor).round() as i64;
2207 jittered.max(1) as u64
2208 }
2209
2210 pub(super) fn can_transition_welcome_state(
2211 current: WelcomeDeliveryState,
2212 next: WelcomeDeliveryState,
2213 ) -> bool {
2214 matches!(
2215 (current, next),
2216 (
2217 WelcomeDeliveryState::Created,
2218 WelcomeDeliveryState::SendAttempted
2219 ) | (WelcomeDeliveryState::Created, WelcomeDeliveryState::Expired)
2220 | (
2221 WelcomeDeliveryState::SendAttempted,
2222 WelcomeDeliveryState::Sent
2223 )
2224 | (
2225 WelcomeDeliveryState::SendAttempted,
2226 WelcomeDeliveryState::Failed
2227 )
2228 | (
2229 WelcomeDeliveryState::Failed,
2230 WelcomeDeliveryState::SendAttempted
2231 )
2232 | (WelcomeDeliveryState::Failed, WelcomeDeliveryState::Sent)
2233 | (WelcomeDeliveryState::Failed, WelcomeDeliveryState::Expired)
2234 // Sent is normally a sink, but the relay's DeliveryError is
2235 // authoritative proof that a wire-confirmed frame was dropped
2236 // (store-less relay, recipient offline). That correction is
2237 // the one legal way back out of Sent — see
2238 // `on_transport_send_failed`.
2239 | (WelcomeDeliveryState::Sent, WelcomeDeliveryState::Failed)
2240 )
2241 }
2242
2243 pub(super) fn transition_welcome_state(
2244 &mut self,
2245 peer_id: &str,
2246 next_state: WelcomeDeliveryState,
2247 source_event: &str,
2248 ) -> Result<()> {
2249 let (previous_state, record_snapshot) = {
2250 let record = self.welcome_lifecycles.get_mut(peer_id).ok_or_else(|| {
2251 Error::Other(format!(
2252 "Missing welcome lifecycle for transition: {}",
2253 peer_id
2254 ))
2255 })?;
2256
2257 if record.state == next_state {
2258 return Ok(());
2259 }
2260
2261 if !Self::can_transition_welcome_state(record.state, next_state) {
2262 return Err(Error::Other(format!(
2263 "Illegal welcome lifecycle transition for {}: {} -> {}",
2264 peer_id,
2265 record.state.as_str(),
2266 next_state.as_str()
2267 )));
2268 }
2269
2270 let previous = record.state;
2271 record.state = next_state;
2272 if matches!(
2273 next_state,
2274 WelcomeDeliveryState::Sent | WelcomeDeliveryState::Expired
2275 ) {
2276 record.next_retry_at = None;
2277 }
2278 (previous, record.clone())
2279 };
2280
2281 self.persist_welcome_lifecycle_entry(&record_snapshot)?;
2282 info!(
2283 event = "welcome_lifecycle_transition",
2284 session_or_group_id = %peer_id,
2285 previous_state = previous_state.as_str(),
2286 new_state = next_state.as_str(),
2287 source_event = %source_event,
2288 attempt = record_snapshot.attempt,
2289 "welcome_lifecycle_transition"
2290 );
2291 Ok(())
2292 }
2293
2294 pub(super) fn upsert_welcome_lifecycle(
2295 &mut self,
2296 peer_id: &str,
2297 group_id: &str,
2298 welcome_message: Message,
2299 source_event: &str,
2300 ) -> Result<()> {
2301 if let Some(existing) = self.welcome_lifecycles.get(peer_id) {
2302 if !matches!(
2303 existing.state,
2304 WelcomeDeliveryState::Sent | WelcomeDeliveryState::Expired
2305 ) {
2306 return Err(Error::Other(format!(
2307 "Refusing to overwrite active welcome lifecycle for {} in state {}",
2308 peer_id,
2309 existing.state.as_str()
2310 )));
2311 }
2312 }
2313
2314 let now = Utc::now();
2315 let record = WelcomeLifecycleRecord {
2316 peer_id: peer_id.to_string(),
2317 group_id: group_id.to_string(),
2318 state: WelcomeDeliveryState::Created,
2319 attempt: 0,
2320 unreachable_parks: 0,
2321 welcome_message,
2322 next_retry_at: None,
2323 last_reason_code: None,
2324 last_transport_error: None,
2325 created_at: now,
2326 expires_at: now + ChronoDuration::seconds(WELCOME_LIFECYCLE_TTL_SECS),
2327 };
2328 self.welcome_lifecycles
2329 .insert(peer_id.to_string(), record.clone());
2330 self.persist_welcome_lifecycle_entry(&record)?;
2331 info!(
2332 event = "welcome_lifecycle_transition",
2333 session_or_group_id = %peer_id,
2334 previous_state = "Absent",
2335 new_state = WelcomeDeliveryState::Created.as_str(),
2336 source_event = %source_event,
2337 attempt = 0,
2338 "welcome_lifecycle_transition"
2339 );
2340 Ok(())
2341 }
2342
2343 pub(super) fn map_welcome_reason_code(error: &Error) -> crate::events::WelcomeReasonCode {
2344 SessionStateError::classify(error).to_welcome_reason_code()
2345 }
2346
2347 /// True when a send failed because no transport carrier exists yet, as
2348 /// opposed to a present-but-unhealthy carrier (`SendFailed`) or a
2349 /// sent-but-unconfirmed timeout. A no-carrier failure means the peer is
2350 /// simply unreachable right now, so the Welcome must be kept alive and
2351 /// retried rather than counted against its budget — see
2352 /// [`Self::apply_welcome_send_failure`].
2353 pub(super) fn is_no_carrier_error(error: &Error) -> bool {
2354 matches!(
2355 error,
2356 Error::Transport(offline_protocol_transport::Error::TransportNotAvailable(_))
2357 )
2358 }
2359
2360 pub(super) fn session_ready_context_for_source(source_event: &str) -> MlsOperationContext {
2361 match source_event {
2362 "confirmation_ack_received" | "confirmation_probe_received" | "decrypt_success" => {
2363 MlsOperationContext::Receive
2364 }
2365 "welcome_received" => MlsOperationContext::Welcome,
2366 _ => MlsOperationContext::Send,
2367 }
2368 }
2369
2370 pub(super) fn find_welcome_peer_by_message_id(&self, message_id: &str) -> Option<String> {
2371 self.welcome_lifecycles
2372 .iter()
2373 .find_map(|(peer_id, record)| {
2374 if record.welcome_message.id.as_str() == message_id {
2375 return Some(peer_id.clone());
2376 }
2377 None
2378 })
2379 }
2380
2381 pub(super) fn process_welcome_retry_queue(&mut self) -> Result<()> {
2382 let now = Utc::now();
2383 let timed_out_attempts: Vec<String> = self
2384 .welcome_lifecycles
2385 .iter()
2386 .filter_map(|(peer_id, record)| {
2387 // Skip peers whose session is already confirmed: the welcome is
2388 // delivered, mark_welcome_confirmed should have marked it Sent,
2389 // and re-sending would be wasted bandwidth (defensive against any
2390 // keying mismatch that left the lifecycle non-terminal).
2391 if self.confirmed_sessions.contains(peer_id) {
2392 return None;
2393 }
2394 if matches!(record.state, WelcomeDeliveryState::SendAttempted)
2395 && record.next_retry_at.is_some_and(|retry_at| retry_at <= now)
2396 {
2397 return Some(peer_id.clone());
2398 }
2399 None
2400 })
2401 .take(WELCOME_RETRY_BATCH_SIZE)
2402 .collect();
2403
2404 for peer_id in timed_out_attempts {
2405 let _ = self.apply_welcome_send_failure(
2406 &peer_id,
2407 crate::events::WelcomeReasonCode::Timeout,
2408 Some(SEND_FAIL_REASON_CONFIRM_TIMEOUT),
2409 // A confirm timeout means the Welcome was sent over a carrier;
2410 // this is not a no-carrier failure, so it ages normally.
2411 false,
2412 "welcome_confirm_timeout",
2413 )?;
2414 }
2415
2416 let due_peers: Vec<String> = self
2417 .welcome_lifecycles
2418 .iter()
2419 .filter_map(|(peer_id, record)| {
2420 if self.confirmed_sessions.contains(peer_id) {
2421 return None;
2422 }
2423 // Failed = a normal retry-due Welcome. Created with a due
2424 // next_retry_at = a no-carrier-parked Welcome (see
2425 // `park_welcome_no_carrier`): a fresh Created entry has
2426 // next_retry_at == None, so only parked ones match here.
2427 if matches!(
2428 record.state,
2429 WelcomeDeliveryState::Failed | WelcomeDeliveryState::Created
2430 ) && record.next_retry_at.is_some_and(|retry_at| retry_at <= now)
2431 {
2432 return Some(peer_id.clone());
2433 }
2434 None
2435 })
2436 .take(WELCOME_RETRY_BATCH_SIZE)
2437 .collect();
2438
2439 for peer_id in due_peers {
2440 if let Err(err) = self.try_send_welcome(&peer_id, "welcome_retry", false) {
2441 warn!(
2442 peer_id = %peer_id,
2443 error = %err,
2444 "Failed to process welcome retry"
2445 );
2446 }
2447 }
2448
2449 Ok(())
2450 }
2451
2452 pub(super) fn has_terminal_welcome_failure(&self, peer_id: &str) -> bool {
2453 self.welcome_lifecycles
2454 .get(peer_id)
2455 .is_some_and(|record| matches!(record.state, WelcomeDeliveryState::Expired))
2456 }
2457
2458 /// Whether a message the user sent is waiting for `peer_id`: queued
2459 /// behind session establishment, or on the outbox unmarked (an entry
2460 /// restored from before the marker existed counts, the safe direction).
2461 fn has_user_content_waiting_for(&self, peer_id: &str) -> bool {
2462 self.pending_encrypted_messages
2463 .get(peer_id)
2464 .is_some_and(|queued| !queued.is_empty())
2465 || self.outbox.values().any(|entry| {
2466 entry.message.recipient.as_str() == peer_id
2467 && !entry
2468 .message
2469 .metadata
2470 .contains_key(crate::constants::NO_ALERT_KEY)
2471 })
2472 }
2473}