Skip to main content

hashtree_network/
mesh_store_core.rs

1//! Shared routed mesh store core.
2//!
3//! This module provides a concrete store wrapper that works with any local storage
4//! backend plus any signaling transport and peer-link factory. Both production
5//! and simulation (mocks) use this same code.
6
7use async_trait::async_trait;
8use futures::{stream::FuturesUnordered, FutureExt, StreamExt};
9use std::collections::hash_map::DefaultHasher;
10use std::collections::{HashMap, HashSet, VecDeque};
11use std::future::Future;
12use std::hash::{Hash as _, Hasher};
13use std::ops::Range;
14use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
15use std::sync::Arc;
16use std::time::Duration;
17use tokio::sync::{oneshot, Mutex, Notify, RwLock};
18use tokio::time::Instant;
19
20use hashtree_core::{Hash, Store, StoreError};
21
22use crate::peer_selector::{PeerMetadataSnapshot, PeerSelector, SelectionStrategy};
23use crate::protocol::{
24    create_pubsub_frame, create_pubsub_interest, create_pubsub_inventory, create_pubsub_want,
25    create_quote_request, create_quote_response_available, create_quote_response_unavailable,
26    create_request, create_request_with_quote, create_response, encode_pubsub_frame,
27    encode_pubsub_interest, encode_pubsub_inventory, encode_pubsub_want, encode_quote_request,
28    encode_quote_response, encode_request, encode_response, hash_to_key, parse_message,
29    DataMessage, DataQuoteRequest, DataQuoteResponse, PubsubFrame, PubsubInterest, PubsubInventory,
30    PubsubWant,
31};
32use crate::pubsub_strategy::{
33    reciprocal_virtual_finish, select_reciprocal_outbound_job, OutboundJobCandidate,
34    PeerTrafficSnapshot, PubsubCandidate, PubsubSchedulerConfig,
35};
36use crate::signaling::MeshRouter;
37use crate::transport::{PeerLinkFactory, SignalingTransport, TransportError};
38use crate::types::{
39    should_forward_htl, PeerHTLConfig, SignalingMessage, TimedSeenSet, MAX_HTL, MESH_EVENT_POLICY,
40};
41
42// Keep the on-disk namespace stable across the crate rename so existing peer
43// metadata does not disappear for users upgrading from the old package name.
44const PEER_METADATA_POINTER_SLOT_KEY: &[u8] = b"hashtree-mesh/peer-metadata/latest/v1";
45const RECENT_FORWARD_MISS_CAPACITY: usize = 4096;
46const MIN_RECENT_FORWARD_MISS_TTL_MS: u64 = 250;
47const PUBSUB_SEEN_CAPACITY: usize = 16_384;
48const PUBSUB_INBOX_CAPACITY: usize = 4_096;
49const PUBSUB_FRAME_CACHE_CAPACITY: usize = 4_096;
50const VERIFIED_BLOCK_DELIVERY_CAPACITY: usize = 4_096;
51const PUBSUB_SEEN_TTL: Duration = Duration::from_secs(120);
52
53/// Pending request awaiting response
54struct PendingRequest {
55    response_tx: oneshot::Sender<Option<Vec<u8>>>,
56    started_at: Instant,
57    queried_peers: Vec<String>,
58}
59
60struct PendingQuoteRequest {
61    response_tx: oneshot::Sender<Option<NegotiatedQuote>>,
62    preferred_mint_url: Option<String>,
63    offered_payment_sat: u64,
64}
65
66struct PendingForwardRequest {
67    requester_ids: HashSet<String>,
68}
69
70type PeerWireStats = PeerTrafficSnapshot;
71
72struct PendingResponseSend {
73    job_id: u64,
74    peer_id: String,
75    bytes: Vec<u8>,
76    ready_at: Instant,
77    queue_sequence: u64,
78}
79
80#[async_trait]
81pub trait MeshReadSource: Send + Sync {
82    fn id(&self) -> &str;
83
84    fn is_available(&self) -> bool {
85        true
86    }
87
88    async fn get(&self, hash: &Hash) -> Option<Vec<u8>>;
89}
90
91#[derive(Debug, Clone)]
92struct NegotiatedQuote {
93    peer_id: String,
94    quote_id: u64,
95    #[allow(dead_code)]
96    mint_url: Option<String>,
97}
98
99struct IssuedQuote {
100    expires_at: Instant,
101    #[allow(dead_code)]
102    payment_sat: u64,
103    #[allow(dead_code)]
104    mint_url: Option<String>,
105}
106
107#[derive(Debug, Clone, Default)]
108struct AdaptiveSourceStats {
109    requests: u64,
110    successes: u64,
111    misses: u64,
112    failures: u64,
113    timeouts: u64,
114    srtt_ms: f64,
115    rttvar_ms: f64,
116    backoff_level: u32,
117    backed_off_until: Option<Instant>,
118    last_success_at: Option<Instant>,
119    last_failure_at: Option<Instant>,
120}
121
122#[derive(Debug, Clone)]
123enum RouteFetchOutcome {
124    Hit(Vec<u8>),
125    Miss,
126    Timeout,
127}
128
129struct InflightSourceFetch {
130    waiters: Vec<oneshot::Sender<RouteFetchOutcome>>,
131}
132
133enum SourceFetchOutcome {
134    Hit {
135        source_id: String,
136        data: Vec<u8>,
137        elapsed_ms: u64,
138    },
139    Miss {
140        source_id: String,
141    },
142    Failure {
143        source_id: String,
144    },
145}
146
147const INITIAL_SOURCE_BACKOFF_MS: u64 = 250;
148const MAX_SOURCE_BACKOFF_MS: u64 = 10_000;
149const SOURCE_SCORE_TIE_DELTA: f64 = 0.15;
150const RECENT_SOURCE_SUCCESS_WINDOW: Duration = Duration::from_secs(60);
151const ACTIVE_PEER_REQUEST_RANK_PENALTY: usize = 3;
152
153fn source_reliability_score(stats: &AdaptiveSourceStats) -> f64 {
154    (stats.successes as f64 + 1.0) / (stats.requests as f64 + 2.0)
155}
156
157fn source_latency_score(stats: &AdaptiveSourceStats) -> f64 {
158    if stats.srtt_ms <= 0.0 {
159        return 0.5;
160    }
161    (500.0 / (stats.srtt_ms + 50.0)).min(1.0)
162}
163
164fn source_has_history(stats: &AdaptiveSourceStats) -> bool {
165    stats.requests > 0
166        || stats.successes > 0
167        || stats.misses > 0
168        || stats.failures > 0
169        || stats.timeouts > 0
170}
171
172fn adaptive_source_score(stats: &AdaptiveSourceStats, now: Instant) -> f64 {
173    if let Some(backed_off_until) = stats.backed_off_until {
174        if backed_off_until > now {
175            return f64::NEG_INFINITY;
176        }
177    }
178
179    let miss_penalty = if stats.requests > 0 {
180        (stats.misses as f64 / stats.requests as f64) * 0.15
181    } else {
182        0.0
183    };
184    let failure_penalty = if stats.requests > 0 {
185        ((stats.failures + stats.timeouts) as f64 / stats.requests as f64) * 0.3
186    } else {
187        0.0
188    };
189    let recency_bonus = if stats
190        .last_success_at
191        .is_some_and(|last| now.duration_since(last) < RECENT_SOURCE_SUCCESS_WINDOW)
192    {
193        0.1
194    } else {
195        0.0
196    };
197
198    0.6 * source_reliability_score(stats) + 0.3 * source_latency_score(stats) + recency_bonus
199        - miss_penalty
200        - failure_penalty
201}
202
203fn peer_endpoint_has_history(stats: &crate::peer_selector::PeerStats) -> bool {
204    stats.requests_sent > 0 || stats.successes > 0 || stats.failures > 0 || stats.timeouts > 0
205}
206
207fn peer_endpoint_score(stats: &crate::peer_selector::PeerStats, now: Instant) -> f64 {
208    if stats.backed_off_until.is_some_and(|until| until > now) {
209        return f64::NEG_INFINITY;
210    }
211
212    let miss_penalty = 0.0;
213    let failure_penalty = if stats.requests_sent > 0 {
214        ((stats.failures + stats.timeouts) as f64 / stats.requests_sent as f64) * 0.3
215    } else {
216        0.0
217    };
218    let recency_bonus = if stats
219        .last_success
220        .is_some_and(|last| now.duration_since(last) < RECENT_SOURCE_SUCCESS_WINDOW)
221    {
222        0.1
223    } else {
224        0.0
225    };
226
227    0.6 * stats.success_rate()
228        + 0.3
229            * source_latency_score(&AdaptiveSourceStats {
230                srtt_ms: stats.srtt_ms,
231                ..AdaptiveSourceStats::default()
232            })
233        + recency_bonus
234        - miss_penalty
235        - failure_penalty
236}
237
238#[derive(Clone)]
239enum ReadRoute {
240    Peers(Vec<String>),
241    Sources,
242}
243
244impl ReadRoute {
245    fn id(&self) -> &'static str {
246        match self {
247            Self::Peers(_) => "peers",
248            Self::Sources => "sources",
249        }
250    }
251}
252
253struct RankedReadRoute {
254    route: ReadRoute,
255    best_endpoint_id: String,
256    score: f64,
257    has_history: bool,
258}
259
260fn ranked_route_kind(route: &ReadRoute) -> u8 {
261    match route {
262        ReadRoute::Sources => 0,
263        ReadRoute::Peers(_) => 1,
264    }
265}
266
267#[derive(Debug, Clone)]
268struct MeshReadContext {
269    exclude_peer_id: Option<String>,
270    request_htl: u8,
271}
272
273impl Default for MeshReadContext {
274    fn default() -> Self {
275        Self {
276            exclude_peer_id: None,
277            request_htl: MAX_HTL,
278        }
279    }
280}
281
282/// Aggregate stats from draining currently available peer-link messages.
283#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
284pub struct DataPumpStats {
285    pub processed: usize,
286    pub request_messages: usize,
287    pub response_messages: usize,
288    pub quote_request_messages: u64,
289    pub quote_response_messages: u64,
290    pub pubsub_interest_messages: u64,
291    pub pubsub_frame_messages: u64,
292    pub pubsub_inventory_messages: u64,
293    pub pubsub_want_messages: u64,
294    pub processed_bytes: u64,
295}
296
297/// Pubsub data delivered to a local subscription.
298#[derive(Debug, Clone, PartialEq, Eq)]
299pub struct PubsubEvent {
300    pub stream_id: String,
301    pub seq: u64,
302    pub origin_peer_id: String,
303    pub from_peer_id: String,
304    pub payload: Vec<u8>,
305}
306
307/// Evidence that this peer won an outstanding, hash-verified block request.
308#[derive(Debug, Clone, PartialEq, Eq)]
309pub struct VerifiedBlockDelivery {
310    pub hash: Hash,
311    pub provider_peer_id: String,
312    pub payload_bytes: u64,
313}
314
315/// One atomic drain of verified delivery evidence and any overflow since the prior drain.
316///
317/// Dropped evidence is intentionally not recoverable or billable through this API. An
318/// application adapter must surface a non-zero count and must not infer a payment claim.
319#[derive(Debug, Clone, Default, PartialEq, Eq)]
320pub struct VerifiedBlockDeliveryBatch {
321    pub deliveries: Vec<VerifiedBlockDelivery>,
322    pub dropped_since_last_drain: u64,
323}
324
325#[derive(Default)]
326struct VerifiedBlockDeliveryBuffer {
327    deliveries: VecDeque<VerifiedBlockDelivery>,
328    dropped_since_last_drain: u64,
329}
330
331/// Send-side accounting from a pubsub publish or forwarded pubsub message.
332#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
333pub struct PubsubPublishStats {
334    pub selected_peers: usize,
335    pub sent_peers: usize,
336    pub sent_bytes: u64,
337    pub deferred_peers: usize,
338}
339
340/// Production pubsub delivery strategy.
341#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
342pub enum PubsubDeliveryMode {
343    /// Push full frames only along advertised interest routes.
344    InterestPush,
345    /// Route small inventories along advertised interest paths and pull payloads back along want paths.
346    #[default]
347    HtlInvWant,
348}
349
350/// Request dispatch strategy for peer queries.
351///
352/// `MeshStoreCore` supports two practical retrieval modes:
353/// - Flood (`usize::MAX` fanout): maximize success/latency at bandwidth cost.
354/// - Staged hedging: probe a subset first, then expand.
355#[derive(Debug, Clone, Copy)]
356pub struct RequestDispatchConfig {
357    /// Number of peers queried immediately.
358    pub initial_fanout: usize,
359    /// Number of additional peers to query on each hedge step.
360    pub hedge_fanout: usize,
361    /// Total peers allowed for this request.
362    pub max_fanout: usize,
363    /// Delay between hedge waves (ms). `0` means send all waves immediately.
364    pub hedge_interval_ms: u64,
365}
366
367impl Default for RequestDispatchConfig {
368    fn default() -> Self {
369        Self {
370            initial_fanout: usize::MAX,
371            hedge_fanout: usize::MAX,
372            max_fanout: usize::MAX,
373            hedge_interval_ms: 0,
374        }
375    }
376}
377
378/// Normalize fanout config against current peer availability.
379pub fn normalize_dispatch_config(
380    dispatch: RequestDispatchConfig,
381    available_peers: usize,
382) -> RequestDispatchConfig {
383    let mut cfg = dispatch;
384    let cap = if cfg.max_fanout == 0 {
385        available_peers
386    } else {
387        cfg.max_fanout.min(available_peers)
388    };
389    cfg.max_fanout = cap;
390    cfg.initial_fanout = if cfg.initial_fanout == 0 {
391        1
392    } else {
393        cfg.initial_fanout.min(cap.max(1))
394    };
395    cfg.hedge_fanout = if cfg.hedge_fanout == 0 {
396        1
397    } else {
398        cfg.hedge_fanout.min(cap.max(1))
399    };
400    cfg
401}
402
403/// Build wave sizes for staged hedged dispatch.
404pub fn build_hedged_wave_plan(peer_count: usize, dispatch: RequestDispatchConfig) -> Vec<usize> {
405    if peer_count == 0 {
406        return Vec::new();
407    }
408    let cap = dispatch.max_fanout.min(peer_count);
409    if cap == 0 {
410        return Vec::new();
411    }
412
413    let mut plan = Vec::new();
414    let mut sent = 0usize;
415    let first = dispatch.initial_fanout.min(cap).max(1);
416    plan.push(first);
417    sent += first;
418
419    while sent < cap {
420        let next = dispatch.hedge_fanout.min(cap - sent).max(1);
421        plan.push(next);
422        sent += next;
423    }
424    plan
425}
426
427/// Outcome returned after waiting on a hedged dispatch wave.
428#[derive(Debug)]
429pub enum HedgedWaveAction<T> {
430    Continue,
431    Success(T),
432    Abort,
433}
434
435/// Run a staged hedged dispatch over peer index ranges.
436///
437/// This scheduler is shared by the reusable `MeshStoreCore` and the native
438/// `hashtree-cli` mesh path so tests and production use the same wave timing.
439pub async fn run_hedged_waves<T, SendWave, SendWaveFut, WaitWave, WaitWaveFut>(
440    peer_count: usize,
441    dispatch: RequestDispatchConfig,
442    request_timeout: Duration,
443    mut send_wave: SendWave,
444    mut wait_wave: WaitWave,
445) -> Option<T>
446where
447    SendWave: FnMut(Range<usize>) -> SendWaveFut,
448    SendWaveFut: Future<Output = usize>,
449    WaitWave: FnMut(Duration) -> WaitWaveFut,
450    WaitWaveFut: Future<Output = HedgedWaveAction<T>>,
451{
452    let dispatch = normalize_dispatch_config(dispatch, peer_count);
453    let wave_plan = build_hedged_wave_plan(peer_count, dispatch);
454    if wave_plan.is_empty() {
455        return None;
456    }
457
458    let deadline = Instant::now() + request_timeout;
459    let mut sent_total = 0usize;
460    let mut next_peer_idx = 0usize;
461
462    for (wave_idx, wave_size) in wave_plan.iter().copied().enumerate() {
463        let from = next_peer_idx;
464        let to = (next_peer_idx + wave_size).min(peer_count);
465        next_peer_idx = to;
466
467        if from == to {
468            continue;
469        }
470
471        sent_total += send_wave(from..to).await;
472        if sent_total == 0 {
473            if next_peer_idx >= peer_count {
474                break;
475            }
476            continue;
477        }
478
479        let now = Instant::now();
480        if now >= deadline {
481            break;
482        }
483        let remaining = deadline.saturating_duration_since(now);
484        let is_last_wave = wave_idx + 1 == wave_plan.len() || next_peer_idx >= peer_count;
485        let wait = if is_last_wave {
486            remaining
487        } else if dispatch.hedge_interval_ms == 0 {
488            Duration::ZERO
489        } else {
490            Duration::from_millis(dispatch.hedge_interval_ms).min(remaining)
491        };
492
493        if wait.is_zero() {
494            continue;
495        }
496
497        match wait_wave(wait).await {
498            HedgedWaveAction::Continue => {}
499            HedgedWaveAction::Success(value) => return Some(value),
500            HedgedWaveAction::Abort => break,
501        }
502    }
503
504    None
505}
506
507/// Keep selector membership aligned with currently connected peer IDs.
508pub async fn sync_selector_peers(selector: &RwLock<PeerSelector>, current_peer_ids: &[String]) {
509    let mut selector = selector.write().await;
510    let current: HashSet<&str> = current_peer_ids.iter().map(String::as_str).collect();
511    let known: Vec<String> = selector.all_stats().map(|s| s.peer_id.clone()).collect();
512    for peer_id in known {
513        if !current.contains(peer_id.as_str()) {
514            selector.remove_peer(&peer_id);
515        }
516    }
517    for peer_id in current_peer_ids {
518        selector.add_peer(peer_id.clone());
519    }
520}
521
522/// Response behavior profile for simulation/game-theory actors.
523///
524/// Defaults to honest behavior (always respond correctly, no extra delay).
525#[derive(Debug, Clone, Copy)]
526pub struct ResponseBehaviorConfig {
527    /// Probability that a node drops a response even when it has data.
528    pub drop_response_prob: f64,
529    /// Probability that a node responds with corrupted payload.
530    pub corrupt_response_prob: f64,
531    /// Baseline response delay before a peer starts sending any data.
532    pub extra_delay_ms: u64,
533    /// Additional delay before the first response byte becomes available.
534    pub first_byte_delay_ms: u64,
535    /// Sustained throughput for delivering large payloads. `0` disables size-based slowdown.
536    pub bytes_per_second: u64,
537    /// Probability that an otherwise honest response experiences an extra stall.
538    pub stall_response_prob: f64,
539    /// Extra delay injected when a stall event happens.
540    pub stall_delay_ms: u64,
541}
542
543impl Default for ResponseBehaviorConfig {
544    fn default() -> Self {
545        Self {
546            drop_response_prob: 0.0,
547            corrupt_response_prob: 0.0,
548            extra_delay_ms: 0,
549            first_byte_delay_ms: 0,
550            bytes_per_second: 0,
551            stall_response_prob: 0.0,
552            stall_delay_ms: 0,
553        }
554    }
555}
556
557impl ResponseBehaviorConfig {
558    fn normalized(self) -> Self {
559        Self {
560            drop_response_prob: self.drop_response_prob.clamp(0.0, 1.0),
561            corrupt_response_prob: self.corrupt_response_prob.clamp(0.0, 1.0),
562            extra_delay_ms: self.extra_delay_ms,
563            first_byte_delay_ms: self.first_byte_delay_ms,
564            bytes_per_second: self.bytes_per_second,
565            stall_response_prob: self.stall_response_prob.clamp(0.0, 1.0),
566            stall_delay_ms: self.stall_delay_ms,
567        }
568    }
569}
570
571/// Routing policy for request ordering + dispatch fanout.
572#[derive(Debug, Clone)]
573pub struct MeshRoutingConfig {
574    pub selection_strategy: SelectionStrategy,
575    pub fairness_enabled: bool,
576    /// Blend weight for payment-priority ranking in selector (`0.0` disables).
577    pub cashu_payment_weight: f64,
578    /// Refuse serving peers that have reached this many unpaid post-delivery settlements.
579    /// `0` disables refusal and only keeps metadata/downranking.
580    pub cashu_payment_default_block_threshold: u64,
581    /// Cashu mint URLs this node is willing to use for settlement.
582    pub cashu_accepted_mints: Vec<String>,
583    /// Preferred Cashu mint URL when initiating paid retrieval.
584    pub cashu_default_mint: Option<String>,
585    /// Baseline cap for accepting a peer-suggested mint outside the trusted set.
586    pub cashu_peer_suggested_mint_base_cap_sat: u64,
587    /// Additional sats allowed per successful delivery from that peer.
588    pub cashu_peer_suggested_mint_success_step_sat: u64,
589    /// Additional sats allowed per successful post-delivery payment received from that peer.
590    pub cashu_peer_suggested_mint_receipt_step_sat: u64,
591    /// Hard upper bound for any single peer-suggested mint quote we accept.
592    pub cashu_peer_suggested_mint_max_cap_sat: u64,
593    pub dispatch: RequestDispatchConfig,
594    pub response_behavior: ResponseBehaviorConfig,
595    pub pubsub_scheduler: PubsubSchedulerConfig,
596    pub pubsub_delivery_mode: PubsubDeliveryMode,
597    /// Forward peer pubsub interests, inventories, and payloads for downstream peers.
598    pub pubsub_forwarding: bool,
599    /// Initial hops-to-live for locally originated pubsub interest/inventory frames.
600    pub pubsub_max_htl: u8,
601}
602
603impl Default for MeshRoutingConfig {
604    fn default() -> Self {
605        Self {
606            selection_strategy: SelectionStrategy::Weighted,
607            fairness_enabled: true,
608            cashu_payment_weight: 0.0,
609            cashu_payment_default_block_threshold: 0,
610            cashu_accepted_mints: Vec::new(),
611            cashu_default_mint: None,
612            cashu_peer_suggested_mint_base_cap_sat: 0,
613            cashu_peer_suggested_mint_success_step_sat: 0,
614            cashu_peer_suggested_mint_receipt_step_sat: 0,
615            cashu_peer_suggested_mint_max_cap_sat: 0,
616            dispatch: RequestDispatchConfig::default(),
617            response_behavior: ResponseBehaviorConfig::default(),
618            pubsub_scheduler: PubsubSchedulerConfig::default(),
619            pubsub_delivery_mode: PubsubDeliveryMode::HtlInvWant,
620            pubsub_forwarding: true,
621            pubsub_max_htl: MESH_EVENT_POLICY.max_htl,
622        }
623    }
624}
625
626impl MeshRoutingConfig {
627    fn pubsub_initial_htl(&self) -> u8 {
628        self.pubsub_max_htl.clamp(1, MAX_HTL)
629    }
630}
631
632/// Routed mesh store core that works with any storage backend and transport
633/// implementation.
634///
635/// This is the shared code between production and simulation.
636/// - Production: transport-specific crates compose `MeshStoreCore` with their links
637/// - Simulation: `MeshStoreCore<MemoryStore, MockRelayTransport, MockConnectionFactory>`
638pub struct MeshStoreCore<S, R, F>
639where
640    S: Store + Send + Sync + 'static,
641    R: SignalingTransport + Send + Sync + 'static,
642    F: PeerLinkFactory + Send + Sync + 'static,
643{
644    /// Local backing store
645    local_store: Arc<S>,
646    /// Mesh router (handles peer discovery and connection)
647    signaling: Arc<MeshRouter<R, F>>,
648    /// Per-peer HTL config
649    htl_configs: RwLock<HashMap<String, PeerHTLConfig>>,
650    /// Pending requests we sent
651    pending_requests: RwLock<HashMap<String, PendingRequest>>,
652    /// Pending quote negotiations keyed by requested hash.
653    pending_quotes: RwLock<HashMap<String, PendingQuoteRequest>>,
654    /// Forwarded peer requests currently being resolved through the mesh/upstream.
655    pending_forward_requests: RwLock<HashMap<String, PendingForwardRequest>>,
656    /// Bounded negative cache for recently forwarded misses/timeouts.
657    recent_forward_misses: Mutex<TimedSeenSet>,
658    /// Quotes we issued to peers and will accept exactly once until expiry.
659    issued_quotes: RwLock<HashMap<(String, String, u64), IssuedQuote>>,
660    /// Monotonic quote identifier generator.
661    next_quote_id: RwLock<u64>,
662    /// Non-peer read sources such as upstream Blossom servers.
663    read_sources: RwLock<HashMap<String, Arc<dyn MeshReadSource>>>,
664    /// Adaptive health stats for non-peer read sources.
665    read_source_stats: RwLock<HashMap<String, AdaptiveSourceStats>>,
666    /// Shared in-flight upstream reads keyed by hash.
667    inflight_source_fetches: Mutex<HashMap<String, InflightSourceFetch>>,
668    /// Adaptive selector for peer ordering.
669    peer_selector: RwLock<PeerSelector>,
670    /// Active per-peer in-flight reads so concurrent block fetches spread across peers.
671    peer_active_requests: RwLock<HashMap<String, usize>>,
672    /// Actual wire traffic stats used for upload-side reciprocity scheduling.
673    peer_wire_stats: RwLock<HashMap<String, PeerWireStats>>,
674    /// Streams this node wants delivered locally.
675    pubsub_local_interests: RwLock<HashSet<String>>,
676    /// Current sequence per local stream interest.
677    pubsub_local_interest_versions: RwLock<HashMap<String, u64>>,
678    /// Reverse pubsub routes: stream id -> peers with local/downstream interest.
679    pubsub_peer_interests: RwLock<HashMap<String, HashSet<String>>>,
680    /// Route owner for each downstream subscriber interest.
681    pubsub_interest_routes: RwLock<HashMap<(String, String), String>>,
682    /// Latest interest sequence observed per subscriber/stream.
683    pubsub_interest_versions: RwLock<HashMap<(String, String), u64>>,
684    /// Bounded dedupe for pubsub interest floods.
685    pubsub_seen_interests: Mutex<TimedSeenSet>,
686    /// Bounded dedupe for pubsub data frames.
687    pubsub_seen_frames: Mutex<TimedSeenSet>,
688    /// Bounded dedupe for pubsub inventory floods.
689    pubsub_seen_inventories: Mutex<TimedSeenSet>,
690    /// Bounded dedupe for pubsub wants by requesting peer.
691    pubsub_seen_wants: Mutex<TimedSeenSet>,
692    /// First upstream peer that announced each inventory key.
693    pubsub_inventory_routes: RwLock<HashMap<String, String>>,
694    /// Downstream peers waiting for a payload after sending a want.
695    pubsub_want_routes: RwLock<HashMap<String, HashSet<String>>>,
696    /// Dedupe for wants this node already sent upstream.
697    pubsub_upstream_wants: Mutex<TimedSeenSet>,
698    /// Small payload cache for serving wants after inventory-first announcements.
699    pubsub_frame_cache: Mutex<VecDeque<(String, PubsubFrame)>>,
700    /// Local pubsub delivery inbox.
701    pubsub_inbox: Mutex<VecDeque<PubsubEvent>>,
702    /// Bounded application-facing evidence for first-winner block deliveries.
703    verified_block_deliveries: Mutex<VerifiedBlockDeliveryBuffer>,
704    /// Wakes consumers waiting for local pubsub deliveries.
705    pubsub_notify: Notify,
706    /// Per stream/peer deferred counts for aging pubsub strategies.
707    pubsub_deferred_counts: RwLock<HashMap<(String, String), u64>>,
708    /// Monotonic sequence for locally originated pubsub interest updates.
709    next_pubsub_interest_seq: AtomicU64,
710    /// Pending content responses waiting for upload arbitration.
711    pending_response_sends: Mutex<Vec<PendingResponseSend>>,
712    /// Upload response scheduler state.
713    response_scheduler_running: AtomicBool,
714    /// Monotonic id for queued response sends.
715    next_response_job_id: AtomicU64,
716    /// Routing/dispatch configuration.
717    routing: MeshRoutingConfig,
718    /// Request timeout
719    request_timeout: Duration,
720    /// Debug mode
721    debug: bool,
722    /// Running flag
723    running: RwLock<bool>,
724}
725
726impl<S, R, F> MeshStoreCore<S, R, F>
727where
728    S: Store + Send + Sync + 'static,
729    R: SignalingTransport + Send + Sync + 'static,
730    F: PeerLinkFactory + Send + Sync + 'static,
731{
732    /// Create a new routed mesh store core.
733    pub fn new(
734        local_store: Arc<S>,
735        signaling: Arc<MeshRouter<R, F>>,
736        request_timeout: Duration,
737        debug: bool,
738    ) -> Self {
739        Self::new_with_routing(
740            local_store,
741            signaling,
742            request_timeout,
743            debug,
744            Default::default(),
745        )
746    }
747
748    /// Create a new routed mesh store core with explicit routing configuration.
749    pub fn new_with_routing(
750        local_store: Arc<S>,
751        signaling: Arc<MeshRouter<R, F>>,
752        request_timeout: Duration,
753        debug: bool,
754        routing: MeshRoutingConfig,
755    ) -> Self {
756        let mut selector = PeerSelector::with_strategy(routing.selection_strategy);
757        selector.set_fairness(routing.fairness_enabled);
758        selector.set_cashu_payment_weight(routing.cashu_payment_weight);
759        Self {
760            local_store,
761            signaling,
762            htl_configs: RwLock::new(HashMap::new()),
763            pending_requests: RwLock::new(HashMap::new()),
764            pending_quotes: RwLock::new(HashMap::new()),
765            pending_forward_requests: RwLock::new(HashMap::new()),
766            recent_forward_misses: Mutex::new(TimedSeenSet::new(
767                RECENT_FORWARD_MISS_CAPACITY,
768                Self::recent_forward_miss_ttl(request_timeout),
769            )),
770            issued_quotes: RwLock::new(HashMap::new()),
771            next_quote_id: RwLock::new(1),
772            read_sources: RwLock::new(HashMap::new()),
773            read_source_stats: RwLock::new(HashMap::new()),
774            inflight_source_fetches: Mutex::new(HashMap::new()),
775            peer_selector: RwLock::new(selector),
776            peer_active_requests: RwLock::new(HashMap::new()),
777            peer_wire_stats: RwLock::new(HashMap::new()),
778            pubsub_local_interests: RwLock::new(HashSet::new()),
779            pubsub_local_interest_versions: RwLock::new(HashMap::new()),
780            pubsub_peer_interests: RwLock::new(HashMap::new()),
781            pubsub_interest_routes: RwLock::new(HashMap::new()),
782            pubsub_interest_versions: RwLock::new(HashMap::new()),
783            pubsub_seen_interests: Mutex::new(TimedSeenSet::new(
784                PUBSUB_SEEN_CAPACITY,
785                PUBSUB_SEEN_TTL,
786            )),
787            pubsub_seen_frames: Mutex::new(TimedSeenSet::new(
788                PUBSUB_SEEN_CAPACITY,
789                PUBSUB_SEEN_TTL,
790            )),
791            pubsub_seen_inventories: Mutex::new(TimedSeenSet::new(
792                PUBSUB_SEEN_CAPACITY,
793                PUBSUB_SEEN_TTL,
794            )),
795            pubsub_seen_wants: Mutex::new(TimedSeenSet::new(PUBSUB_SEEN_CAPACITY, PUBSUB_SEEN_TTL)),
796            pubsub_inventory_routes: RwLock::new(HashMap::new()),
797            pubsub_want_routes: RwLock::new(HashMap::new()),
798            pubsub_upstream_wants: Mutex::new(TimedSeenSet::new(
799                PUBSUB_SEEN_CAPACITY,
800                PUBSUB_SEEN_TTL,
801            )),
802            pubsub_frame_cache: Mutex::new(VecDeque::new()),
803            pubsub_inbox: Mutex::new(VecDeque::new()),
804            verified_block_deliveries: Mutex::new(VerifiedBlockDeliveryBuffer::default()),
805            pubsub_notify: Notify::new(),
806            pubsub_deferred_counts: RwLock::new(HashMap::new()),
807            next_pubsub_interest_seq: AtomicU64::new(1),
808            pending_response_sends: Mutex::new(Vec::new()),
809            response_scheduler_running: AtomicBool::new(false),
810            next_response_job_id: AtomicU64::new(1),
811            routing,
812            request_timeout,
813            debug,
814            running: RwLock::new(false),
815        }
816    }
817
818    fn recent_forward_miss_ttl(request_timeout: Duration) -> Duration {
819        let ttl_ms = request_timeout
820            .as_millis()
821            .saturating_mul(2)
822            .max(MIN_RECENT_FORWARD_MISS_TTL_MS as u128)
823            .min(u64::MAX as u128) as u64;
824        Duration::from_millis(ttl_ms)
825    }
826
827    /// Start the store (begin listening for messages)
828    pub async fn start(&self) -> Result<(), TransportError> {
829        *self.running.write().await = true;
830
831        // Send initial hello
832        self.signaling.send_hello(vec![]).await?;
833
834        Ok(())
835    }
836
837    /// Stop the store
838    pub async fn stop(&self) {
839        *self.running.write().await = false;
840    }
841
842    /// Process incoming signaling message
843    pub async fn process_signaling(&self, msg: SignalingMessage) -> Result<(), TransportError> {
844        // When a new peer connects, initialize their HTL config
845        let peer_id = msg.peer_id().to_string();
846        {
847            let mut configs = self.htl_configs.write().await;
848            if !configs.contains_key(&peer_id) {
849                configs.insert(peer_id.clone(), PeerHTLConfig::random());
850            }
851        }
852        self.peer_selector.write().await.add_peer(peer_id.clone());
853
854        let result = self.signaling.handle_message(msg).await;
855        if result.is_ok() {
856            self.announce_pubsub_interests_to_peer(&peer_id).await;
857        }
858        result
859    }
860
861    /// Get signaling manager reference
862    pub fn signaling(&self) -> &Arc<MeshRouter<R, F>> {
863        &self.signaling
864    }
865
866    fn response_behavior(&self) -> ResponseBehaviorConfig {
867        self.routing.response_behavior.normalized()
868    }
869
870    async fn record_peer_wire_sent(&self, peer_id: &str, bytes: u64) {
871        if bytes == 0 {
872            return;
873        }
874        let mut stats = self.peer_wire_stats.write().await;
875        let entry = stats.entry(peer_id.to_string()).or_default();
876        entry.bytes_sent = entry.bytes_sent.saturating_add(bytes);
877    }
878
879    async fn record_peer_wire_received(&self, peer_id: &str, bytes: u64) {
880        if bytes == 0 {
881            return;
882        }
883        let mut stats = self.peer_wire_stats.write().await;
884        let entry = stats.entry(peer_id.to_string()).or_default();
885        entry.bytes_received = entry.bytes_received.saturating_add(bytes);
886    }
887
888    /// Record ingress from a peer that matched local or downstream demand.
889    ///
890    /// Raw bytes are tracked separately in `record_peer_wire_received`; this
891    /// counter is the reciprocity signal used by shared outbound scheduling.
892    pub async fn record_useful_bytes_received_from_peer(&self, peer_id: &str, bytes: u64) {
893        if bytes == 0 {
894            return;
895        }
896        let mut stats = self.peer_wire_stats.write().await;
897        let entry = stats.entry(peer_id.to_string()).or_default();
898        entry.useful_bytes_received = entry.useful_bytes_received.saturating_add(bytes);
899    }
900
901    /// Snapshot peer traffic for production pubsub scheduling or diagnostics.
902    pub async fn peer_traffic_snapshot(&self, peer_id: &str) -> PeerTrafficSnapshot {
903        self.peer_wire_stats
904            .read()
905            .await
906            .get(peer_id)
907            .copied()
908            .unwrap_or_default()
909    }
910
911    /// Snapshot all known peer traffic for production pubsub scheduling.
912    pub async fn peer_traffic_snapshots(&self) -> HashMap<String, PeerTrafficSnapshot> {
913        self.peer_wire_stats.read().await.clone()
914    }
915
916    fn pubsub_key(origin_peer_id: &str, stream_id: &str, seq: u64) -> String {
917        format!("{origin_peer_id}:{stream_id}:{seq}")
918    }
919
920    fn pubsub_frame_key(frame: &PubsubFrame) -> String {
921        Self::pubsub_key(&frame.origin_peer_id, &frame.stream_id, frame.seq)
922    }
923
924    fn pubsub_interest_key(interest: &PubsubInterest) -> String {
925        format!(
926            "{}:{}:{}:{}",
927            interest.subscriber_peer_id, interest.stream_id, interest.seq, interest.active
928        )
929    }
930
931    fn next_pubsub_interest_seq(&self) -> u64 {
932        self.next_pubsub_interest_seq
933            .fetch_add(1, Ordering::Relaxed)
934    }
935
936    async fn record_peer_pubsub_wire_sent(&self, peer_id: &str, bytes: u64, bandwidth_debt: f64) {
937        if bytes == 0 {
938            return;
939        }
940        let mut stats = self.peer_wire_stats.write().await;
941        let entry = stats.entry(peer_id.to_string()).or_default();
942        entry.bytes_sent = entry.bytes_sent.saturating_add(bytes);
943        entry.bandwidth_debt = bandwidth_debt;
944    }
945
946    async fn send_pubsub_interest_to_peers(
947        &self,
948        interest: &PubsubInterest,
949        exclude_peer_id: Option<&str>,
950    ) -> PubsubPublishStats {
951        if !should_forward_htl(interest.htl) {
952            return PubsubPublishStats::default();
953        }
954
955        let mut peer_ids = self.signaling.peer_ids().await;
956        peer_ids.sort();
957        peer_ids.retain(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude));
958
959        let bytes = encode_pubsub_interest(interest);
960        let mut stats = PubsubPublishStats {
961            selected_peers: peer_ids.len(),
962            ..Default::default()
963        };
964        for peer_id in peer_ids {
965            let Some(channel) = self.signaling.get_channel(&peer_id).await else {
966                continue;
967            };
968            if channel.send(bytes.clone()).await.is_ok() {
969                stats.sent_peers += 1;
970                stats.sent_bytes = stats.sent_bytes.saturating_add(bytes.len() as u64);
971                self.record_peer_wire_sent(&peer_id, bytes.len() as u64)
972                    .await;
973            }
974        }
975        stats
976    }
977
978    async fn announce_pubsub_interests_to_peer(&self, peer_id: &str) {
979        let mut interests = self
980            .pubsub_local_interests
981            .read()
982            .await
983            .iter()
984            .cloned()
985            .collect::<Vec<_>>();
986        interests.sort();
987        if interests.is_empty() {
988            return;
989        }
990
991        let interests = {
992            let versions = self.pubsub_local_interest_versions.read().await;
993            interests
994                .into_iter()
995                .filter_map(|stream_id| {
996                    versions
997                        .get(&stream_id)
998                        .copied()
999                        .map(|seq| (stream_id, seq))
1000                })
1001                .collect::<Vec<_>>()
1002        };
1003
1004        for (stream_id, seq) in interests {
1005            let interest = create_pubsub_interest(
1006                stream_id,
1007                self.signaling.peer_id().to_string(),
1008                seq,
1009                true,
1010                MAX_HTL,
1011            );
1012            let Some(channel) = self.signaling.get_channel(peer_id).await else {
1013                continue;
1014            };
1015            let bytes = encode_pubsub_interest(&interest);
1016            if channel.send(bytes.clone()).await.is_ok() {
1017                self.record_peer_wire_sent(peer_id, bytes.len() as u64)
1018                    .await;
1019            }
1020        }
1021    }
1022
1023    fn remove_pubsub_peer_interest(
1024        peer_interests: &mut HashMap<String, HashSet<String>>,
1025        routes: &HashMap<(String, String), String>,
1026        stream_id: &str,
1027        peer_id: &str,
1028    ) {
1029        let still_has_route = routes
1030            .iter()
1031            .any(|((stream, _subscriber), peer)| stream == stream_id && peer == peer_id);
1032        if still_has_route {
1033            return;
1034        }
1035        if let Some(peers) = peer_interests.get_mut(stream_id) {
1036            peers.remove(peer_id);
1037            if peers.is_empty() {
1038                peer_interests.remove(stream_id);
1039            }
1040        }
1041    }
1042
1043    async fn apply_pubsub_interest_route(
1044        &self,
1045        from_peer: &str,
1046        interest: &PubsubInterest,
1047    ) -> bool {
1048        if interest.stream_id.is_empty() || interest.subscriber_peer_id.is_empty() {
1049            return false;
1050        }
1051        if interest.subscriber_peer_id == self.signaling.peer_id() {
1052            return false;
1053        }
1054
1055        let interest_key = Self::pubsub_interest_key(interest);
1056        if !self
1057            .pubsub_seen_interests
1058            .lock()
1059            .await
1060            .insert_if_new(interest_key)
1061        {
1062            return false;
1063        }
1064
1065        let route_key = (
1066            interest.stream_id.clone(),
1067            interest.subscriber_peer_id.clone(),
1068        );
1069        {
1070            let mut versions = self.pubsub_interest_versions.write().await;
1071            if versions
1072                .get(&route_key)
1073                .is_some_and(|latest| *latest >= interest.seq)
1074            {
1075                return false;
1076            }
1077            versions.insert(route_key.clone(), interest.seq);
1078        }
1079
1080        let mut peer_interests = self.pubsub_peer_interests.write().await;
1081        let mut routes = self.pubsub_interest_routes.write().await;
1082        if interest.active {
1083            if let Some(previous_peer) = routes.insert(route_key, from_peer.to_string()) {
1084                if previous_peer != from_peer {
1085                    Self::remove_pubsub_peer_interest(
1086                        &mut peer_interests,
1087                        &routes,
1088                        &interest.stream_id,
1089                        &previous_peer,
1090                    );
1091                }
1092            }
1093            peer_interests
1094                .entry(interest.stream_id.clone())
1095                .or_default()
1096                .insert(from_peer.to_string());
1097        } else if let Some(previous_peer) = routes.remove(&route_key) {
1098            Self::remove_pubsub_peer_interest(
1099                &mut peer_interests,
1100                &routes,
1101                &interest.stream_id,
1102                &previous_peer,
1103            );
1104        } else {
1105            Self::remove_pubsub_peer_interest(
1106                &mut peer_interests,
1107                &routes,
1108                &interest.stream_id,
1109                from_peer,
1110            );
1111        }
1112
1113        true
1114    }
1115
1116    async fn interested_pubsub_peers(
1117        &self,
1118        stream_id: &str,
1119        exclude_peer_id: Option<&str>,
1120    ) -> Vec<String> {
1121        let connected = self
1122            .signaling
1123            .peer_ids()
1124            .await
1125            .into_iter()
1126            .collect::<HashSet<_>>();
1127        let mut peers = self
1128            .pubsub_peer_interests
1129            .read()
1130            .await
1131            .get(stream_id)
1132            .map(|peers| peers.iter().cloned().collect::<Vec<_>>())
1133            .unwrap_or_default();
1134        peers.retain(|peer_id| {
1135            connected.contains(peer_id) && exclude_peer_id.is_none_or(|exclude| peer_id != exclude)
1136        });
1137        peers.sort();
1138        peers
1139    }
1140
1141    async fn decrement_pubsub_htl_for_peer(&self, peer_id: &str, htl: u8) -> u8 {
1142        let htl_config = {
1143            let configs = self.htl_configs.read().await;
1144            configs
1145                .get(peer_id)
1146                .cloned()
1147                .unwrap_or_else(PeerHTLConfig::random)
1148        };
1149        htl_config.decrement_with_policy(htl, &MESH_EVENT_POLICY)
1150    }
1151
1152    async fn send_pubsub_inventory_to_peers(
1153        &self,
1154        inv: &PubsubInventory,
1155        peer_ids: &[String],
1156    ) -> PubsubPublishStats {
1157        if peer_ids.is_empty() || !should_forward_htl(inv.htl) {
1158            return PubsubPublishStats::default();
1159        }
1160
1161        let mut stats = PubsubPublishStats {
1162            selected_peers: peer_ids.len(),
1163            ..Default::default()
1164        };
1165        for peer_id in peer_ids {
1166            let send_htl = self.decrement_pubsub_htl_for_peer(peer_id, inv.htl).await;
1167            if !should_forward_htl(send_htl) {
1168                continue;
1169            }
1170            let Some(channel) = self.signaling.get_channel(peer_id).await else {
1171                continue;
1172            };
1173            let mut outgoing = inv.clone();
1174            outgoing.htl = send_htl;
1175            let bytes = encode_pubsub_inventory(&outgoing);
1176            let message_bytes = bytes.len() as u64;
1177            if channel.send(bytes).await.is_ok() {
1178                stats.sent_peers += 1;
1179                stats.sent_bytes = stats.sent_bytes.saturating_add(message_bytes);
1180                self.record_peer_wire_sent(peer_id, message_bytes).await;
1181            }
1182        }
1183        stats
1184    }
1185
1186    async fn send_pubsub_want_to_peer(&self, want: &PubsubWant, peer_id: &str) -> bool {
1187        let Some(channel) = self.signaling.get_channel(peer_id).await else {
1188            return false;
1189        };
1190        let bytes = encode_pubsub_want(want);
1191        let message_bytes = bytes.len() as u64;
1192        match channel.send(bytes).await {
1193            Ok(()) => {
1194                self.record_peer_wire_sent(peer_id, message_bytes).await;
1195                true
1196            }
1197            Err(_) => false,
1198        }
1199    }
1200
1201    async fn send_pubsub_want_upstream(
1202        &self,
1203        key: &str,
1204        want: &PubsubWant,
1205        exclude_peer_id: Option<&str>,
1206    ) -> bool {
1207        let upstream = {
1208            let routes = self.pubsub_inventory_routes.read().await;
1209            routes.get(key).cloned()
1210        };
1211        let Some(upstream) = upstream else {
1212            return false;
1213        };
1214        if exclude_peer_id.is_some_and(|exclude| exclude == upstream) {
1215            return false;
1216        }
1217        let want_key = format!("{key}:{upstream}");
1218        if !self
1219            .pubsub_upstream_wants
1220            .lock()
1221            .await
1222            .insert_if_new(want_key)
1223        {
1224            return false;
1225        }
1226        self.send_pubsub_want_to_peer(want, &upstream).await
1227    }
1228
1229    async fn cache_pubsub_frame(&self, key: String, frame: PubsubFrame) {
1230        let mut cache = self.pubsub_frame_cache.lock().await;
1231        if let Some(index) = cache.iter().position(|(cached_key, _)| cached_key == &key) {
1232            cache.remove(index);
1233        }
1234        cache.push_back((key, frame));
1235        while cache.len() > PUBSUB_FRAME_CACHE_CAPACITY {
1236            cache.pop_front();
1237        }
1238    }
1239
1240    async fn cached_pubsub_frame(&self, key: &str) -> Option<PubsubFrame> {
1241        self.pubsub_frame_cache
1242            .lock()
1243            .await
1244            .iter()
1245            .find_map(|(cached_key, frame)| {
1246                if cached_key == key {
1247                    Some(frame.clone())
1248                } else {
1249                    None
1250                }
1251            })
1252    }
1253
1254    async fn remember_pubsub_want_peer(&self, key: String, from_peer: &str) -> bool {
1255        let mut routes = self.pubsub_want_routes.write().await;
1256        routes.entry(key).or_default().insert(from_peer.to_string())
1257    }
1258
1259    async fn take_pubsub_want_peers(
1260        &self,
1261        key: &str,
1262        exclude_peer_id: Option<&str>,
1263    ) -> Vec<String> {
1264        let connected = self
1265            .signaling
1266            .peer_ids()
1267            .await
1268            .into_iter()
1269            .collect::<HashSet<_>>();
1270        let mut peers = self
1271            .pubsub_want_routes
1272            .write()
1273            .await
1274            .remove(key)
1275            .map(|peers| peers.into_iter().collect::<Vec<_>>())
1276            .unwrap_or_default();
1277        peers.retain(|peer_id| {
1278            connected.contains(peer_id) && exclude_peer_id.is_none_or(|exclude| peer_id != exclude)
1279        });
1280        peers.sort();
1281        peers
1282    }
1283
1284    async fn select_pubsub_peers(
1285        &self,
1286        stream_id: &str,
1287        seq: u64,
1288        message_bytes: u64,
1289        peer_ids: &[String],
1290    ) -> (Vec<String>, Vec<String>) {
1291        let traffic = self.peer_wire_stats.read().await;
1292        let deferred_counts = self.pubsub_deferred_counts.read().await;
1293        let candidates = peer_ids
1294            .iter()
1295            .map(|peer_id| PubsubCandidate {
1296                peer_id: peer_id.clone(),
1297                traffic: traffic.get(peer_id).copied().unwrap_or_default(),
1298                deferred_count: deferred_counts
1299                    .get(&(stream_id.to_string(), peer_id.clone()))
1300                    .copied()
1301                    .unwrap_or_default(),
1302            })
1303            .collect::<Vec<_>>();
1304        drop(deferred_counts);
1305        drop(traffic);
1306
1307        let selection = self.routing.pubsub_scheduler.select(
1308            stream_id,
1309            seq,
1310            self.signaling.peer_id(),
1311            message_bytes,
1312            &candidates,
1313        );
1314
1315        {
1316            let mut deferred_counts = self.pubsub_deferred_counts.write().await;
1317            for peer_id in &selection.deferred {
1318                *deferred_counts
1319                    .entry((stream_id.to_string(), peer_id.clone()))
1320                    .or_insert(0) += 1;
1321            }
1322            for peer_id in &selection.selected {
1323                deferred_counts.remove(&(stream_id.to_string(), peer_id.clone()));
1324            }
1325        }
1326
1327        (selection.selected, selection.deferred)
1328    }
1329
1330    async fn send_pubsub_frame_to_peers(
1331        &self,
1332        frame: &PubsubFrame,
1333        peer_ids: &[String],
1334    ) -> PubsubPublishStats {
1335        if peer_ids.is_empty() || !should_forward_htl(frame.htl) {
1336            return PubsubPublishStats::default();
1337        }
1338
1339        let bytes = encode_pubsub_frame(frame);
1340        let message_bytes = bytes.len() as u64;
1341        let (selected, deferred) = self
1342            .select_pubsub_peers(&frame.stream_id, frame.seq, message_bytes, peer_ids)
1343            .await;
1344        let mut stats = PubsubPublishStats {
1345            selected_peers: selected.len(),
1346            deferred_peers: deferred.len(),
1347            ..Default::default()
1348        };
1349
1350        for peer_id in selected {
1351            let Some(channel) = self.signaling.get_channel(&peer_id).await else {
1352                continue;
1353            };
1354            let snapshot = self.peer_traffic_snapshot(&peer_id).await;
1355            let bandwidth_debt = reciprocal_virtual_finish(snapshot, message_bytes);
1356            if channel.send(bytes.clone()).await.is_ok() {
1357                stats.sent_peers += 1;
1358                stats.sent_bytes = stats.sent_bytes.saturating_add(message_bytes);
1359                self.record_peer_pubsub_wire_sent(&peer_id, message_bytes, bandwidth_debt)
1360                    .await;
1361            }
1362        }
1363
1364        stats
1365    }
1366
1367    async fn enqueue_pubsub_event(&self, event: PubsubEvent) {
1368        let mut inbox = self.pubsub_inbox.lock().await;
1369        inbox.push_back(event);
1370        while inbox.len() > PUBSUB_INBOX_CAPACITY {
1371            inbox.pop_front();
1372        }
1373        self.pubsub_notify.notify_one();
1374    }
1375
1376    /// Subscribe this node to a pubsub stream and advertise that interest.
1377    pub async fn subscribe_pubsub(
1378        self: &Arc<Self>,
1379        stream_id: impl Into<String>,
1380    ) -> PubsubPublishStats {
1381        let stream_id = stream_id.into();
1382        if stream_id.is_empty() {
1383            return PubsubPublishStats::default();
1384        }
1385        self.pubsub_local_interests
1386            .write()
1387            .await
1388            .insert(stream_id.clone());
1389        let seq = {
1390            let mut versions = self.pubsub_local_interest_versions.write().await;
1391            match versions.get(&stream_id).copied() {
1392                Some(seq) => seq,
1393                None => {
1394                    let seq = self.next_pubsub_interest_seq();
1395                    versions.insert(stream_id.clone(), seq);
1396                    seq
1397                }
1398            }
1399        };
1400        let interest = create_pubsub_interest(
1401            stream_id,
1402            self.signaling.peer_id().to_string(),
1403            seq,
1404            true,
1405            self.routing.pubsub_initial_htl(),
1406        );
1407        self.send_pubsub_interest_to_peers(&interest, None).await
1408    }
1409
1410    /// Stop local delivery for a pubsub stream and advertise the withdrawn interest.
1411    pub async fn unsubscribe_pubsub(
1412        self: &Arc<Self>,
1413        stream_id: impl Into<String>,
1414    ) -> PubsubPublishStats {
1415        let stream_id = stream_id.into();
1416        if stream_id.is_empty() {
1417            return PubsubPublishStats::default();
1418        }
1419        self.pubsub_local_interests.write().await.remove(&stream_id);
1420        self.pubsub_local_interest_versions
1421            .write()
1422            .await
1423            .remove(&stream_id);
1424        let interest = create_pubsub_interest(
1425            stream_id,
1426            self.signaling.peer_id().to_string(),
1427            self.next_pubsub_interest_seq(),
1428            false,
1429            self.routing.pubsub_initial_htl(),
1430        );
1431        self.send_pubsub_interest_to_peers(&interest, None).await
1432    }
1433
1434    /// Publish bytes on a pubsub stream through the configured mesh delivery mode.
1435    pub async fn publish_pubsub(
1436        self: &Arc<Self>,
1437        stream_id: impl Into<String>,
1438        seq: u64,
1439        payload: Vec<u8>,
1440    ) -> PubsubPublishStats {
1441        let stream_id = stream_id.into();
1442        if stream_id.is_empty() {
1443            return PubsubPublishStats::default();
1444        }
1445        let payload_bytes = payload.len() as u64;
1446        let frame = create_pubsub_frame(
1447            stream_id.clone(),
1448            seq,
1449            self.signaling.peer_id().to_string(),
1450            payload.clone(),
1451            self.routing.pubsub_initial_htl(),
1452        );
1453        let frame_key = Self::pubsub_frame_key(&frame);
1454        self.pubsub_seen_frames
1455            .lock()
1456            .await
1457            .insert_if_new(frame_key.clone());
1458        self.cache_pubsub_frame(frame_key, frame.clone()).await;
1459
1460        if self
1461            .pubsub_local_interests
1462            .read()
1463            .await
1464            .contains(&stream_id)
1465        {
1466            self.enqueue_pubsub_event(PubsubEvent {
1467                stream_id: stream_id.clone(),
1468                seq,
1469                origin_peer_id: self.signaling.peer_id().to_string(),
1470                from_peer_id: self.signaling.peer_id().to_string(),
1471                payload,
1472            })
1473            .await;
1474        }
1475
1476        match self.routing.pubsub_delivery_mode {
1477            PubsubDeliveryMode::InterestPush => {
1478                let peers = self.interested_pubsub_peers(&stream_id, None).await;
1479                self.send_pubsub_frame_to_peers(&frame, &peers).await
1480            }
1481            PubsubDeliveryMode::HtlInvWant => {
1482                let inv = create_pubsub_inventory(
1483                    stream_id,
1484                    seq,
1485                    self.signaling.peer_id().to_string(),
1486                    payload_bytes,
1487                    self.routing.pubsub_initial_htl(),
1488                );
1489                let peers = self.interested_pubsub_peers(&inv.stream_id, None).await;
1490                self.send_pubsub_inventory_to_peers(&inv, &peers).await
1491            }
1492        }
1493    }
1494
1495    /// Drain locally delivered pubsub events.
1496    pub async fn drain_pubsub_events(&self) -> Vec<PubsubEvent> {
1497        self.pubsub_inbox.lock().await.drain(..).collect()
1498    }
1499
1500    /// Drain verified first-winner block deliveries for an application adapter.
1501    pub async fn drain_verified_block_deliveries(&self) -> VerifiedBlockDeliveryBatch {
1502        let mut buffer = self.verified_block_deliveries.lock().await;
1503        VerifiedBlockDeliveryBatch {
1504            deliveries: buffer.deliveries.drain(..).collect(),
1505            dropped_since_last_drain: std::mem::take(&mut buffer.dropped_since_last_drain),
1506        }
1507    }
1508
1509    /// Wait until a locally delivered pubsub event is available, then return it.
1510    pub async fn recv_pubsub_event(&self) -> PubsubEvent {
1511        loop {
1512            if let Some(event) = self.pubsub_inbox.lock().await.pop_front() {
1513                return event;
1514            }
1515            self.pubsub_notify.notified().await;
1516        }
1517    }
1518
1519    /// Connected peers that currently have local or downstream interest in a stream.
1520    pub async fn pubsub_interest_peers(&self, stream_id: &str) -> Vec<String> {
1521        self.interested_pubsub_peers(stream_id, None).await
1522    }
1523
1524    fn choose_ready_response_job(
1525        ready_jobs: &[(u64, String, usize, Instant, u64)],
1526        stats: &HashMap<String, PeerWireStats>,
1527    ) -> Option<(u64, f64)> {
1528        let jobs = ready_jobs
1529            .iter()
1530            .map(|job| OutboundJobCandidate {
1531                job_id: job.0,
1532                peer_id: job.1.clone(),
1533                message_bytes: job.2 as u64,
1534                queue_sequence: job.4,
1535            })
1536            .collect::<Vec<_>>();
1537        select_reciprocal_outbound_job(&jobs, |peer_id| {
1538            stats.get(peer_id).copied().unwrap_or_default()
1539        })
1540        .map(|choice| (choice.job_id, choice.virtual_finish))
1541    }
1542
1543    async fn enqueue_response_send(
1544        self: &Arc<Self>,
1545        peer_id: String,
1546        bytes: Vec<u8>,
1547        ready_at: Instant,
1548    ) {
1549        let job_id = self.next_response_job_id.fetch_add(1, Ordering::Relaxed);
1550        {
1551            let mut queue = self.pending_response_sends.lock().await;
1552            queue.push(PendingResponseSend {
1553                job_id,
1554                peer_id,
1555                bytes,
1556                ready_at,
1557                queue_sequence: job_id,
1558            });
1559        }
1560
1561        if self
1562            .response_scheduler_running
1563            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
1564            .is_ok()
1565        {
1566            let this = Arc::clone(self);
1567            tokio::spawn(async move {
1568                this.run_response_scheduler().await;
1569            });
1570        }
1571    }
1572
1573    async fn run_response_scheduler(self: Arc<Self>) {
1574        loop {
1575            let snapshot = {
1576                let queue = self.pending_response_sends.lock().await;
1577                if queue.is_empty() {
1578                    self.response_scheduler_running
1579                        .store(false, Ordering::Release);
1580                    return;
1581                }
1582                queue
1583                    .iter()
1584                    .map(|job| {
1585                        (
1586                            job.job_id,
1587                            job.peer_id.clone(),
1588                            job.bytes.len(),
1589                            job.ready_at,
1590                            job.queue_sequence,
1591                        )
1592                    })
1593                    .collect::<Vec<_>>()
1594            };
1595
1596            let now = Instant::now();
1597            let mut earliest_ready_at: Option<Instant> = None;
1598            let mut ready_jobs = Vec::new();
1599            for job in &snapshot {
1600                if job.3 <= now {
1601                    ready_jobs.push(job.clone());
1602                } else {
1603                    earliest_ready_at = Some(match earliest_ready_at {
1604                        Some(current) => current.min(job.3),
1605                        None => job.3,
1606                    });
1607                }
1608            }
1609
1610            if ready_jobs.is_empty() {
1611                if let Some(ready_at) = earliest_ready_at {
1612                    tokio::time::sleep(ready_at.saturating_duration_since(Instant::now())).await;
1613                    continue;
1614                }
1615                self.response_scheduler_running
1616                    .store(false, Ordering::Release);
1617                return;
1618            }
1619
1620            let (selected_job_id, selected_finish) = {
1621                let stats = self.peer_wire_stats.read().await;
1622                Self::choose_ready_response_job(&ready_jobs, &stats).expect("ready response job")
1623            };
1624
1625            let selected = {
1626                let mut queue = self.pending_response_sends.lock().await;
1627                let Some(index) = queue.iter().position(|job| job.job_id == selected_job_id) else {
1628                    continue;
1629                };
1630                queue.swap_remove(index)
1631            };
1632
1633            let sent = if let Some(channel) = self.signaling.get_channel(&selected.peer_id).await {
1634                channel.send(selected.bytes.clone()).await.is_ok()
1635            } else {
1636                false
1637            };
1638
1639            let queued_peers = {
1640                let queue = self.pending_response_sends.lock().await;
1641                queue
1642                    .iter()
1643                    .map(|job| job.peer_id.clone())
1644                    .collect::<HashSet<_>>()
1645            };
1646            let mut stats = self.peer_wire_stats.write().await;
1647            let entry = stats.entry(selected.peer_id.clone()).or_default();
1648            if sent {
1649                entry.bytes_sent = entry.bytes_sent.saturating_add(selected.bytes.len() as u64);
1650                entry.bandwidth_debt = selected_finish;
1651            }
1652            if queued_peers.is_empty() {
1653                for peer_stats in stats.values_mut() {
1654                    peer_stats.bandwidth_debt = 0.0;
1655                }
1656            } else {
1657                let floor = queued_peers
1658                    .iter()
1659                    .filter_map(|peer_id| stats.get(peer_id).map(|peer| peer.bandwidth_debt))
1660                    .fold(f64::INFINITY, f64::min);
1661                if floor.is_finite() && floor > 0.0 {
1662                    for peer_id in queued_peers {
1663                        if let Some(peer_stats) = stats.get_mut(&peer_id) {
1664                            peer_stats.bandwidth_debt =
1665                                (peer_stats.bandwidth_debt - floor).max(0.0);
1666                        }
1667                    }
1668                }
1669            }
1670        }
1671    }
1672
1673    fn deterministic_actor_draw_for(peer_id: &str, hash: &Hash, salt: u64) -> f64 {
1674        let mut hasher = DefaultHasher::new();
1675        peer_id.hash(&mut hasher);
1676        hash.hash(&mut hasher);
1677        salt.hash(&mut hasher);
1678        let v = hasher.finish();
1679        (v as f64) / (u64::MAX as f64)
1680    }
1681
1682    fn deterministic_actor_draw(&self, hash: &Hash, salt: u64) -> f64 {
1683        Self::deterministic_actor_draw_for(self.signaling.peer_id(), hash, salt)
1684    }
1685
1686    fn peer_metadata_pointer_slot_hash() -> Hash {
1687        hashtree_core::sha256(PEER_METADATA_POINTER_SLOT_KEY)
1688    }
1689
1690    fn decode_hash_hex(hash_hex: &str) -> Result<Hash, StoreError> {
1691        let bytes = hex::decode(hash_hex)
1692            .map_err(|e| StoreError::Other(format!("Invalid hash hex: {e}")))?;
1693        if bytes.len() != 32 {
1694            return Err(StoreError::Other(format!(
1695                "Invalid hash length {}, expected 32 bytes",
1696                bytes.len()
1697            )));
1698        }
1699        let mut hash = [0u8; 32];
1700        hash.copy_from_slice(&bytes);
1701        Ok(hash)
1702    }
1703
1704    fn should_drop_response(&self, hash: &Hash) -> bool {
1705        let p = self.response_behavior().drop_response_prob;
1706        if p <= 0.0 {
1707            return false;
1708        }
1709        self.deterministic_actor_draw(hash, 0xD0_D0_D0_D0_D0_D0_D0_D0) < p
1710    }
1711
1712    fn should_corrupt_response(&self, hash: &Hash) -> bool {
1713        let p = self.response_behavior().corrupt_response_prob;
1714        if p <= 0.0 {
1715            return false;
1716        }
1717        self.deterministic_actor_draw(hash, 0xC0_C0_C0_C0_C0_C0_C0_C0) < p
1718    }
1719
1720    fn should_stall_response(&self, hash: &Hash) -> bool {
1721        let p = self.response_behavior().stall_response_prob;
1722        if p <= 0.0 {
1723            return false;
1724        }
1725        self.deterministic_actor_draw(hash, 0x5A_11_5A_11_5A_11_5A_11) < p
1726    }
1727
1728    fn response_send_delay(&self, hash: &Hash, payload_len: usize) -> Duration {
1729        let behavior = self.response_behavior();
1730        let mut total_ms = behavior
1731            .extra_delay_ms
1732            .saturating_add(behavior.first_byte_delay_ms);
1733
1734        if behavior.bytes_per_second > 0 && payload_len > 0 {
1735            let throughput_ms = ((payload_len as u128) * 1000)
1736                .div_ceil(behavior.bytes_per_second as u128)
1737                .min(u64::MAX as u128) as u64;
1738            total_ms = total_ms.saturating_add(throughput_ms);
1739        }
1740
1741        if behavior.stall_delay_ms > 0 && self.should_stall_response(hash) {
1742            total_ms = total_ms.saturating_add(behavior.stall_delay_ms);
1743        }
1744
1745        Duration::from_millis(total_ms)
1746    }
1747
1748    async fn ordered_connected_peers(&self, exclude_peer_id: Option<&str>) -> Vec<String> {
1749        let current_peer_ids = self.signaling.peer_ids().await;
1750        if current_peer_ids.is_empty() {
1751            return Vec::new();
1752        }
1753
1754        sync_selector_peers(&self.peer_selector, &current_peer_ids).await;
1755        let hash_get_peer_ids: HashSet<String> = self
1756            .signaling
1757            .hash_get_peer_ids()
1758            .await
1759            .into_iter()
1760            .collect();
1761        let mut candidate_peer_ids: Vec<String> = current_peer_ids
1762            .into_iter()
1763            .filter(|peer_id| hash_get_peer_ids.contains(peer_id))
1764            .filter(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude))
1765            .collect();
1766        if candidate_peer_ids.is_empty() {
1767            return Vec::new();
1768        }
1769
1770        let current_set: HashSet<&str> = candidate_peer_ids.iter().map(String::as_str).collect();
1771        let mut selector = self.peer_selector.write().await;
1772        let mut selector_order = selector.select_peers();
1773        selector_order.retain(|peer_id| current_set.contains(peer_id.as_str()));
1774        if selector_order.is_empty() {
1775            let mut fallback = candidate_peer_ids;
1776            fallback.sort();
1777            return fallback;
1778        }
1779        let backed_off: HashMap<String, bool> = candidate_peer_ids
1780            .iter()
1781            .map(|peer_id| (peer_id.clone(), selector.is_peer_backed_off(peer_id)))
1782            .collect();
1783        drop(selector);
1784
1785        let rank: HashMap<&str, usize> = selector_order
1786            .iter()
1787            .enumerate()
1788            .map(|(idx, peer_id)| (peer_id.as_str(), idx))
1789            .collect();
1790        let active = self.peer_active_requests.read().await;
1791        candidate_peer_ids.sort_by(|left, right| {
1792            let left_backed_off = backed_off.get(left).copied().unwrap_or(false);
1793            let right_backed_off = backed_off.get(right).copied().unwrap_or(false);
1794            if left_backed_off != right_backed_off {
1795                return if left_backed_off {
1796                    std::cmp::Ordering::Greater
1797                } else {
1798                    std::cmp::Ordering::Less
1799                };
1800            }
1801            let left_rank = rank.get(left.as_str()).copied().unwrap_or(usize::MAX / 2);
1802            let right_rank = rank.get(right.as_str()).copied().unwrap_or(usize::MAX / 2);
1803            let left_load = active.get(left).copied().unwrap_or(0);
1804            let right_load = active.get(right).copied().unwrap_or(0);
1805            (left_rank + left_load.saturating_mul(ACTIVE_PEER_REQUEST_RANK_PENALTY))
1806                .cmp(&(right_rank + right_load.saturating_mul(ACTIVE_PEER_REQUEST_RANK_PENALTY)))
1807                .then_with(|| left.cmp(right))
1808        });
1809        candidate_peer_ids
1810    }
1811
1812    async fn reserve_peer_request(&self, peer_id: &str) {
1813        let mut active = self.peer_active_requests.write().await;
1814        *active.entry(peer_id.to_string()).or_insert(0) += 1;
1815    }
1816
1817    async fn release_peer_request(&self, peer_id: &str) {
1818        let mut active = self.peer_active_requests.write().await;
1819        let Some(count) = active.get_mut(peer_id) else {
1820            return;
1821        };
1822        if *count <= 1 {
1823            active.remove(peer_id);
1824        } else {
1825            *count -= 1;
1826        }
1827    }
1828
1829    async fn release_queried_peer_requests(&self, peer_ids: &[String]) {
1830        for peer_id in peer_ids {
1831            self.release_peer_request(peer_id).await;
1832        }
1833    }
1834
1835    fn requested_quote_mint(&self) -> Option<&str> {
1836        if let Some(default_mint) = self.routing.cashu_default_mint.as_deref() {
1837            if self.routing.cashu_accepted_mints.is_empty()
1838                || self
1839                    .routing
1840                    .cashu_accepted_mints
1841                    .iter()
1842                    .any(|mint| mint == default_mint)
1843            {
1844                return Some(default_mint);
1845            }
1846        }
1847
1848        self.routing
1849            .cashu_accepted_mints
1850            .first()
1851            .map(String::as_str)
1852    }
1853
1854    fn choose_quote_mint(&self, requested_mint: Option<&str>) -> Option<String> {
1855        if let Some(requested_mint) = requested_mint {
1856            if self.accepts_quote_mint(Some(requested_mint)) {
1857                return Some(requested_mint.to_string());
1858            }
1859        }
1860        if let Some(default_mint) = self.routing.cashu_default_mint.as_ref() {
1861            return Some(default_mint.clone());
1862        }
1863        if let Some(first_mint) = self.routing.cashu_accepted_mints.first() {
1864            return Some(first_mint.clone());
1865        }
1866        requested_mint.map(str::to_string)
1867    }
1868
1869    fn accepts_quote_mint(&self, mint_url: Option<&str>) -> bool {
1870        if self.routing.cashu_accepted_mints.is_empty() {
1871            return true;
1872        }
1873
1874        let Some(mint_url) = mint_url else {
1875            return false;
1876        };
1877        self.routing
1878            .cashu_accepted_mints
1879            .iter()
1880            .any(|mint| mint == mint_url)
1881    }
1882
1883    fn trusts_quote_mint(&self, mint_url: Option<&str>) -> bool {
1884        let Some(mint_url) = mint_url else {
1885            return self.routing.cashu_default_mint.is_none()
1886                && self.routing.cashu_accepted_mints.is_empty();
1887        };
1888        self.routing.cashu_default_mint.as_deref() == Some(mint_url)
1889            || self
1890                .routing
1891                .cashu_accepted_mints
1892                .iter()
1893                .any(|mint| mint == mint_url)
1894    }
1895
1896    async fn peer_suggested_mint_cap_sat(&self, peer_id: &str) -> u64 {
1897        let base = self.routing.cashu_peer_suggested_mint_base_cap_sat;
1898        if base == 0 {
1899            return 0;
1900        }
1901
1902        let selector = self.peer_selector.read().await;
1903        let Some(stats) = selector.get_stats(peer_id) else {
1904            let max_cap = self.routing.cashu_peer_suggested_mint_max_cap_sat;
1905            return if max_cap > 0 { base.min(max_cap) } else { base };
1906        };
1907
1908        if stats.cashu_payment_defaults > 0
1909            && stats.cashu_payment_defaults >= stats.cashu_payment_receipts
1910        {
1911            return 0;
1912        }
1913
1914        let success_bonus = stats
1915            .successes
1916            .saturating_mul(self.routing.cashu_peer_suggested_mint_success_step_sat);
1917        let receipt_bonus = stats
1918            .cashu_payment_receipts
1919            .saturating_mul(self.routing.cashu_peer_suggested_mint_receipt_step_sat);
1920        let mut cap = base
1921            .saturating_add(success_bonus)
1922            .saturating_add(receipt_bonus);
1923        let max_cap = self.routing.cashu_peer_suggested_mint_max_cap_sat;
1924        if max_cap > 0 {
1925            cap = cap.min(max_cap);
1926        }
1927        cap
1928    }
1929
1930    async fn should_accept_quote_response(
1931        &self,
1932        from_peer: &str,
1933        preferred_mint_url: Option<&str>,
1934        offered_payment_sat: u64,
1935        res: &DataQuoteResponse,
1936    ) -> bool {
1937        let Some(payment_sat) = res.p else {
1938            return false;
1939        };
1940        if payment_sat > offered_payment_sat {
1941            return false;
1942        }
1943
1944        let response_mint = res.m.as_deref();
1945        if response_mint == preferred_mint_url {
1946            return true;
1947        }
1948        if self.trusts_quote_mint(response_mint) {
1949            return true;
1950        }
1951        if response_mint.is_none() {
1952            return false;
1953        }
1954
1955        payment_sat <= self.peer_suggested_mint_cap_sat(from_peer).await
1956    }
1957
1958    async fn issue_quote(
1959        &self,
1960        peer_id: &str,
1961        hash_key: &str,
1962        payment_sat: u64,
1963        ttl_ms: u32,
1964        mint_url: Option<&str>,
1965    ) -> u64 {
1966        let quote_id = {
1967            let mut next = self.next_quote_id.write().await;
1968            let quote_id = *next;
1969            *next = next.saturating_add(1);
1970            quote_id
1971        };
1972
1973        let expires_at = Instant::now() + Duration::from_millis(ttl_ms as u64);
1974        self.issued_quotes.write().await.insert(
1975            (peer_id.to_string(), hash_key.to_string(), quote_id),
1976            IssuedQuote {
1977                expires_at,
1978                payment_sat,
1979                mint_url: mint_url.map(str::to_string),
1980            },
1981        );
1982        quote_id
1983    }
1984
1985    async fn take_valid_quote(&self, peer_id: &str, hash_key: &str, quote_id: u64) -> bool {
1986        let key = (peer_id.to_string(), hash_key.to_string(), quote_id);
1987        let Some(quote) = self.issued_quotes.write().await.remove(&key) else {
1988            return false;
1989        };
1990        quote.expires_at > Instant::now()
1991    }
1992
1993    async fn send_request_to_peer(
1994        &self,
1995        peer_id: &str,
1996        hash: &Hash,
1997        request_htl: u8,
1998        quote_id: Option<u64>,
1999    ) -> bool {
2000        if !should_forward_htl(request_htl) {
2001            return false;
2002        }
2003
2004        let channel = match self.signaling.get_channel(peer_id).await {
2005            Some(c) => c,
2006            None => return false,
2007        };
2008
2009        let htl_config = {
2010            let configs = self.htl_configs.read().await;
2011            configs
2012                .get(peer_id)
2013                .cloned()
2014                .unwrap_or_else(PeerHTLConfig::random)
2015        };
2016
2017        let send_htl = htl_config.decrement(request_htl);
2018        let req = match quote_id {
2019            Some(quote_id) => create_request_with_quote(hash, send_htl, quote_id),
2020            None => create_request(hash, send_htl),
2021        };
2022        let request_bytes = encode_request(&req);
2023        let request_len = request_bytes.len() as u64;
2024
2025        {
2026            let mut selector = self.peer_selector.write().await;
2027            selector.record_request(peer_id, request_len);
2028        }
2029
2030        match channel.send(request_bytes).await {
2031            Ok(()) => {
2032                self.record_peer_wire_sent(peer_id, request_len).await;
2033                true
2034            }
2035            Err(_) => {
2036                self.peer_selector.write().await.record_failure(peer_id);
2037                false
2038            }
2039        }
2040    }
2041
2042    async fn send_quote_request_to_peer(
2043        &self,
2044        peer_id: &str,
2045        hash: &Hash,
2046        payment_sat: u64,
2047        ttl_ms: u32,
2048        mint_url: Option<&str>,
2049    ) -> bool {
2050        let channel = match self.signaling.get_channel(peer_id).await {
2051            Some(c) => c,
2052            None => return false,
2053        };
2054
2055        let req = create_quote_request(hash, ttl_ms, payment_sat, mint_url);
2056        let request_bytes = encode_quote_request(&req);
2057        let request_len = request_bytes.len() as u64;
2058
2059        match channel.send(request_bytes).await {
2060            Ok(()) => {
2061                self.record_peer_wire_sent(peer_id, request_len).await;
2062                true
2063            }
2064            Err(_) => false,
2065        }
2066    }
2067
2068    pub async fn set_read_sources(&self, sources: Vec<Arc<dyn MeshReadSource>>) {
2069        let mut by_id = HashMap::new();
2070        let mut stats = self.read_source_stats.write().await;
2071        for source in sources {
2072            let source_id = source.id().to_string();
2073            by_id.insert(source_id.clone(), source);
2074            stats
2075                .entry(source_id)
2076                .or_insert_with(AdaptiveSourceStats::default);
2077        }
2078        *self.read_sources.write().await = by_id;
2079    }
2080
2081    async fn record_read_source_request(&self, source_id: &str) {
2082        let mut stats = self.read_source_stats.write().await;
2083        stats
2084            .entry(source_id.to_string())
2085            .or_insert_with(AdaptiveSourceStats::default)
2086            .requests += 1;
2087    }
2088
2089    async fn record_read_source_miss(&self, source_id: &str) {
2090        let mut stats = self.read_source_stats.write().await;
2091        stats
2092            .entry(source_id.to_string())
2093            .or_insert_with(AdaptiveSourceStats::default)
2094            .misses += 1;
2095    }
2096
2097    async fn record_read_source_success(&self, source_id: &str, elapsed_ms: u64) {
2098        let now = Instant::now();
2099        let mut stats = self.read_source_stats.write().await;
2100        let stats = stats
2101            .entry(source_id.to_string())
2102            .or_insert_with(AdaptiveSourceStats::default);
2103        stats.successes += 1;
2104        stats.last_success_at = Some(now);
2105        stats.backoff_level = 0;
2106        stats.backed_off_until = None;
2107        if stats.srtt_ms <= 0.0 {
2108            stats.srtt_ms = elapsed_ms as f64;
2109            stats.rttvar_ms = elapsed_ms as f64 / 2.0;
2110            return;
2111        }
2112        let elapsed = elapsed_ms as f64;
2113        stats.rttvar_ms = 0.75 * stats.rttvar_ms + 0.25 * (stats.srtt_ms - elapsed).abs();
2114        stats.srtt_ms = 0.875 * stats.srtt_ms + 0.125 * elapsed;
2115    }
2116
2117    async fn record_read_source_failure(&self, source_id: &str) {
2118        let now = Instant::now();
2119        let mut stats = self.read_source_stats.write().await;
2120        let stats = stats
2121            .entry(source_id.to_string())
2122            .or_insert_with(AdaptiveSourceStats::default);
2123        stats.failures += 1;
2124        stats.last_failure_at = Some(now);
2125        Self::apply_source_backoff(stats, now);
2126    }
2127
2128    async fn record_read_source_timeout(&self, source_id: &str) {
2129        let now = Instant::now();
2130        let mut stats = self.read_source_stats.write().await;
2131        let stats = stats
2132            .entry(source_id.to_string())
2133            .or_insert_with(AdaptiveSourceStats::default);
2134        stats.timeouts += 1;
2135        stats.last_failure_at = Some(now);
2136        Self::apply_source_backoff(stats, now);
2137    }
2138
2139    fn apply_source_backoff(stats: &mut AdaptiveSourceStats, now: Instant) {
2140        stats.backoff_level = stats.backoff_level.saturating_add(1);
2141        let backoff_ms = (INITIAL_SOURCE_BACKOFF_MS
2142            .saturating_mul(2u64.saturating_pow(stats.backoff_level.saturating_sub(1))))
2143        .min(MAX_SOURCE_BACKOFF_MS);
2144        stats.backed_off_until = Some(now + Duration::from_millis(backoff_ms));
2145    }
2146
2147    async fn ordered_read_sources(&self) -> Vec<Arc<dyn MeshReadSource>> {
2148        let sources = self.read_sources.read().await;
2149        if sources.is_empty() {
2150            return Vec::new();
2151        }
2152
2153        let mut available: Vec<Arc<dyn MeshReadSource>> = sources
2154            .values()
2155            .filter(|source| source.is_available())
2156            .cloned()
2157            .collect();
2158        if available.is_empty() {
2159            return Vec::new();
2160        }
2161
2162        let now = Instant::now();
2163        let stats = self.read_source_stats.read().await;
2164        let mut healthy: Vec<Arc<dyn MeshReadSource>> = available
2165            .iter()
2166            .filter(|source| {
2167                stats
2168                    .get(source.id())
2169                    .and_then(|s| s.backed_off_until)
2170                    .is_none_or(|until| until <= now)
2171            })
2172            .cloned()
2173            .collect();
2174        if !healthy.is_empty() {
2175            available = std::mem::take(&mut healthy);
2176        }
2177
2178        available.sort_by(|left, right| {
2179            let left_stats = stats.get(left.id()).cloned().unwrap_or_default();
2180            let right_stats = stats.get(right.id()).cloned().unwrap_or_default();
2181            adaptive_source_score(&right_stats, now)
2182                .partial_cmp(&adaptive_source_score(&left_stats, now))
2183                .unwrap_or(std::cmp::Ordering::Equal)
2184                .then_with(|| left.id().cmp(right.id()))
2185        });
2186        available
2187    }
2188
2189    async fn should_probe_multiple_read_sources(
2190        &self,
2191        ordered_sources: &[Arc<dyn MeshReadSource>],
2192    ) -> bool {
2193        if ordered_sources.len() <= 1 {
2194            return false;
2195        }
2196        let stats = self.read_source_stats.read().await;
2197        let best = stats
2198            .get(ordered_sources[0].id())
2199            .cloned()
2200            .unwrap_or_default();
2201        let second = stats
2202            .get(ordered_sources[1].id())
2203            .cloned()
2204            .unwrap_or_default();
2205        if !source_has_history(&best) || !source_has_history(&second) {
2206            return false;
2207        }
2208        let now = Instant::now();
2209        adaptive_source_score(&best, now) - adaptive_source_score(&second, now)
2210            < SOURCE_SCORE_TIE_DELTA
2211    }
2212
2213    async fn source_dispatch_for(&self, source_count: usize) -> RequestDispatchConfig {
2214        if source_count == 0 {
2215            return self.routing.dispatch;
2216        }
2217        let ordered_sources = self.ordered_read_sources().await;
2218        let probe_multiple = self
2219            .should_probe_multiple_read_sources(&ordered_sources)
2220            .await;
2221        let initial_fanout = if probe_multiple {
2222            source_count.min(2)
2223        } else {
2224            1
2225        };
2226        RequestDispatchConfig {
2227            initial_fanout,
2228            hedge_fanout: self.routing.dispatch.hedge_fanout,
2229            max_fanout: self.routing.dispatch.max_fanout.min(source_count),
2230            hedge_interval_ms: self.routing.dispatch.hedge_interval_ms,
2231        }
2232    }
2233
2234    /// Get peer count
2235    pub async fn peer_count(&self) -> usize {
2236        self.signaling.peer_count().await
2237    }
2238
2239    /// Get connected mesh peer IDs.
2240    pub async fn peer_ids(&self) -> Vec<String> {
2241        self.signaling.peer_ids().await
2242    }
2243
2244    /// Check if we need more peers
2245    pub async fn needs_peers(&self) -> bool {
2246        self.signaling.needs_peers().await
2247    }
2248
2249    /// Re-broadcast hello to refresh discovery as topology changes.
2250    pub async fn send_hello(&self) -> Result<(), TransportError> {
2251        self.signaling.send_hello(vec![]).await
2252    }
2253
2254    /// Drain all currently available peer-link messages and handle them.
2255    ///
2256    /// This keeps the message pump logic shared between simulation and the
2257    /// default production wrapper instead of duplicating per-channel loops.
2258    pub async fn drain_available_data_messages(self: &Arc<Self>) -> DataPumpStats {
2259        let mut stats = DataPumpStats::default();
2260        let peer_ids = self.signaling.peer_ids().await;
2261        for peer_id in peer_ids {
2262            let Some(channel) = self.signaling.get_channel(&peer_id).await else {
2263                continue;
2264            };
2265
2266            while let Some(data) = channel.try_recv() {
2267                stats.processed += 1;
2268                stats.processed_bytes += data.len() as u64;
2269                if let Some(msg) = parse_message(&data) {
2270                    match msg {
2271                        DataMessage::Request(_) => stats.request_messages += 1,
2272                        DataMessage::Response(_) => stats.response_messages += 1,
2273                        DataMessage::QuoteRequest(_) => stats.quote_request_messages += 1,
2274                        DataMessage::QuoteResponse(_) => stats.quote_response_messages += 1,
2275                        DataMessage::PubsubInterest(_) => stats.pubsub_interest_messages += 1,
2276                        DataMessage::PubsubFrame(_) => stats.pubsub_frame_messages += 1,
2277                        DataMessage::PubsubInventory(_) => stats.pubsub_inventory_messages += 1,
2278                        DataMessage::PubsubWant(_) => stats.pubsub_want_messages += 1,
2279                        DataMessage::Payment(_)
2280                        | DataMessage::PaymentAck(_)
2281                        | DataMessage::Chunk(_)
2282                        | DataMessage::PeerHints(_) => {}
2283                    }
2284                }
2285                self.handle_data_message(&peer_id, &data).await;
2286            }
2287        }
2288        stats
2289    }
2290
2291    /// Apply an out-of-band payment credit to a peer's routing priority.
2292    pub async fn record_cashu_payment_for_peer(&self, peer_id: &str, amount_sat: u64) {
2293        self.peer_selector
2294            .write()
2295            .await
2296            .record_cashu_payment(peer_id, amount_sat);
2297    }
2298
2299    /// Record a post-delivery payment we received from a peer.
2300    pub async fn record_cashu_receipt_from_peer(&self, peer_id: &str, amount_sat: u64) {
2301        self.peer_selector
2302            .write()
2303            .await
2304            .record_cashu_receipt(peer_id, amount_sat);
2305    }
2306
2307    /// Record that a peer failed to pay after we delivered successfully.
2308    pub async fn record_cashu_payment_default_from_peer(&self, peer_id: &str) {
2309        self.peer_selector
2310            .write()
2311            .await
2312            .record_cashu_payment_default(peer_id);
2313    }
2314
2315    /// Snapshot routing/selection summary for inspection/debugging.
2316    pub async fn selector_summary(&self) -> crate::peer_selector::SelectorSummary {
2317        self.peer_selector.read().await.summary()
2318    }
2319
2320    fn should_refuse_requests_from_peer(&self, selector: &PeerSelector, peer_id: &str) -> bool {
2321        selector.is_peer_blocked_for_payment_defaults(
2322            peer_id,
2323            self.routing.cashu_payment_default_block_threshold,
2324        )
2325    }
2326
2327    /// Export live peer metadata for inspection/debugging.
2328    pub async fn peer_metadata_snapshot(&self) -> PeerMetadataSnapshot {
2329        self.peer_selector
2330            .read()
2331            .await
2332            .export_peer_metadata_snapshot()
2333    }
2334
2335    /// Snapshot current peer metadata and persist it into `local_store`.
2336    ///
2337    /// Uses content-addressed storage for the snapshot body and a reserved
2338    /// mutable pointer slot for the "latest snapshot hash".
2339    pub async fn persist_peer_metadata(&self) -> Result<Hash, StoreError> {
2340        let snapshot = self
2341            .peer_selector
2342            .read()
2343            .await
2344            .export_peer_metadata_snapshot();
2345        let bytes = serde_json::to_vec(&snapshot).map_err(|e| {
2346            StoreError::Other(format!("Failed to encode peer metadata snapshot: {e}"))
2347        })?;
2348        let snapshot_hash = hashtree_core::sha256(&bytes);
2349        let _ = self.local_store.put(snapshot_hash, bytes).await?;
2350
2351        let pointer_slot = Self::peer_metadata_pointer_slot_hash();
2352        let pointer_bytes = hex::encode(snapshot_hash).into_bytes();
2353        let _ = self.local_store.delete(&pointer_slot).await?;
2354        let _ = self.local_store.put(pointer_slot, pointer_bytes).await?;
2355
2356        Ok(snapshot_hash)
2357    }
2358
2359    /// Load persisted peer metadata from `local_store` if available.
2360    pub async fn load_peer_metadata(&self) -> Result<bool, StoreError> {
2361        let pointer_slot = Self::peer_metadata_pointer_slot_hash();
2362        let Some(pointer_bytes) = self.local_store.get(&pointer_slot).await? else {
2363            return Ok(false);
2364        };
2365        let pointer_hex = std::str::from_utf8(&pointer_bytes).map_err(|e| {
2366            StoreError::Other(format!("Peer metadata pointer is not valid UTF-8: {e}"))
2367        })?;
2368        let snapshot_hash = Self::decode_hash_hex(pointer_hex.trim())?;
2369
2370        let Some(snapshot_bytes) = self.local_store.get(&snapshot_hash).await? else {
2371            return Ok(false);
2372        };
2373        let snapshot: PeerMetadataSnapshot =
2374            serde_json::from_slice(&snapshot_bytes).map_err(|e| {
2375                StoreError::Other(format!("Failed to decode peer metadata snapshot: {e}"))
2376            })?;
2377        self.peer_selector
2378            .write()
2379            .await
2380            .import_peer_metadata_snapshot(&snapshot);
2381        Ok(true)
2382    }
2383
2384    /// Request data from peers after negotiating a paid quote.
2385    ///
2386    /// If quote negotiation fails or the quoted peer does not deliver, the store
2387    /// falls back to the normal unpaid retrieval path to preserve liveness.
2388    pub async fn get_with_quote(
2389        &self,
2390        hash: &Hash,
2391        payment_sat: u64,
2392        quote_ttl: Duration,
2393    ) -> Result<Option<Vec<u8>>, StoreError> {
2394        if let Some(data) = self.local_store.get(hash).await? {
2395            return Ok(Some(data));
2396        }
2397        Ok(self
2398            .request_from_peers_with_quote(hash, payment_sat, quote_ttl)
2399            .await)
2400    }
2401
2402    async fn request_from_peers_with_quote(
2403        &self,
2404        hash: &Hash,
2405        payment_sat: u64,
2406        quote_ttl: Duration,
2407    ) -> Option<Vec<u8>> {
2408        let ordered_peer_ids = self.ordered_connected_peers(None).await;
2409        if ordered_peer_ids.is_empty() {
2410            return None;
2411        }
2412
2413        if let Some(quote) = self
2414            .request_quote_from_peers(hash, payment_sat, quote_ttl, &ordered_peer_ids)
2415            .await
2416        {
2417            if let Some(data) = self
2418                .request_from_single_peer(hash, &quote.peer_id, MAX_HTL, Some(quote.quote_id))
2419                .await
2420            {
2421                return Some(data);
2422            }
2423        }
2424
2425        self.request_from_mesh(hash).await
2426    }
2427
2428    async fn request_quote_from_peers(
2429        &self,
2430        hash: &Hash,
2431        payment_sat: u64,
2432        quote_ttl: Duration,
2433        ordered_peer_ids: &[String],
2434    ) -> Option<NegotiatedQuote> {
2435        if ordered_peer_ids.is_empty() {
2436            return None;
2437        }
2438        let ttl_ms = quote_ttl.as_millis().min(u32::MAX as u128) as u32;
2439        if ttl_ms == 0 {
2440            return None;
2441        }
2442        let requested_mint = self.requested_quote_mint().map(str::to_string);
2443
2444        let hash_key = hash_to_key(hash);
2445        let (tx, rx) = oneshot::channel();
2446        self.pending_quotes.write().await.insert(
2447            hash_key.clone(),
2448            PendingQuoteRequest {
2449                response_tx: tx,
2450                preferred_mint_url: requested_mint.clone(),
2451                offered_payment_sat: payment_sat,
2452            },
2453        );
2454
2455        let rx = Arc::new(Mutex::new(rx));
2456        let result = run_hedged_waves(
2457            ordered_peer_ids.len(),
2458            self.routing.dispatch,
2459            self.request_timeout,
2460            |range| {
2461                let wave_peer_ids = ordered_peer_ids[range].to_vec();
2462                let requested_mint = requested_mint.clone();
2463                let hash = *hash;
2464                async move {
2465                    let mut sent = 0usize;
2466                    for peer_id in wave_peer_ids {
2467                        if self
2468                            .send_quote_request_to_peer(
2469                                &peer_id,
2470                                &hash,
2471                                payment_sat,
2472                                ttl_ms,
2473                                requested_mint.as_deref(),
2474                            )
2475                            .await
2476                        {
2477                            sent += 1;
2478                        }
2479                    }
2480                    sent
2481                }
2482            },
2483            |wait| {
2484                let rx = rx.clone();
2485                async move {
2486                    let mut rx = rx.lock().await;
2487                    match tokio::time::timeout(wait, &mut *rx).await {
2488                        Ok(Ok(Some(quote))) => HedgedWaveAction::Success(quote),
2489                        Ok(Ok(None)) | Ok(Err(_)) => HedgedWaveAction::Abort,
2490                        Err(_) => HedgedWaveAction::Continue,
2491                    }
2492                }
2493            },
2494        )
2495        .await;
2496        let _ = self.pending_quotes.write().await.remove(&hash_key);
2497        result
2498    }
2499
2500    async fn request_from_single_peer(
2501        &self,
2502        hash: &Hash,
2503        peer_id: &str,
2504        request_htl: u8,
2505        quote_id: Option<u64>,
2506    ) -> Option<Vec<u8>> {
2507        let hash_key = hash_to_key(hash);
2508        let (tx, rx) = oneshot::channel();
2509        self.pending_requests.write().await.insert(
2510            hash_key.clone(),
2511            PendingRequest {
2512                response_tx: tx,
2513                started_at: Instant::now(),
2514                queried_peers: vec![peer_id.to_string()],
2515            },
2516        );
2517
2518        let mut rx = rx;
2519        if !self
2520            .send_request_to_peer(peer_id, hash, request_htl, quote_id)
2521            .await
2522        {
2523            let _ = self.pending_requests.write().await.remove(&hash_key);
2524            return None;
2525        }
2526        self.reserve_peer_request(peer_id).await;
2527
2528        if let Ok(Ok(Some(data))) = tokio::time::timeout(self.request_timeout, &mut rx).await {
2529            if hashtree_core::sha256(&data) == *hash {
2530                let _ = self.local_store.put(*hash, data.clone()).await;
2531                return Some(data);
2532            }
2533        }
2534
2535        if let Some(pending) = self.pending_requests.write().await.remove(&hash_key) {
2536            self.release_queried_peer_requests(&pending.queried_peers)
2537                .await;
2538            for peer_id in pending.queried_peers {
2539                self.peer_selector.write().await.record_timeout(&peer_id);
2540            }
2541        }
2542        let _ = self.take_forward_requesters(&hash_key).await;
2543        None
2544    }
2545
2546    async fn request_from_ordered_peers(
2547        &self,
2548        hash: &Hash,
2549        ordered_peer_ids: &[String],
2550        request_htl: u8,
2551    ) -> RouteFetchOutcome {
2552        let hash_key = hash_to_key(hash);
2553        let (tx, rx) = oneshot::channel();
2554        self.pending_requests.write().await.insert(
2555            hash_key.clone(),
2556            PendingRequest {
2557                response_tx: tx,
2558                started_at: Instant::now(),
2559                queried_peers: Vec::new(),
2560            },
2561        );
2562
2563        let rx = Arc::new(Mutex::new(rx));
2564        let result = run_hedged_waves(
2565            ordered_peer_ids.len(),
2566            self.routing.dispatch,
2567            self.request_timeout,
2568            |range| {
2569                let wave_peer_ids = ordered_peer_ids[range].to_vec();
2570                let hash = *hash;
2571                let hash_key = hash_key.clone();
2572                async move {
2573                    let mut sent = 0usize;
2574                    for peer_id in wave_peer_ids {
2575                        if self
2576                            .send_request_to_peer(&peer_id, &hash, request_htl, None)
2577                            .await
2578                        {
2579                            sent += 1;
2580                            self.reserve_peer_request(&peer_id).await;
2581                            if let Some(pending) =
2582                                self.pending_requests.write().await.get_mut(&hash_key)
2583                            {
2584                                pending.queried_peers.push(peer_id);
2585                            }
2586                        }
2587                    }
2588                    sent
2589                }
2590            },
2591            |wait| {
2592                let rx = rx.clone();
2593                async move {
2594                    let mut rx = rx.lock().await;
2595                    match tokio::time::timeout(wait, &mut *rx).await {
2596                        Ok(Ok(Some(data))) if hashtree_core::sha256(&data) == *hash => {
2597                            HedgedWaveAction::Success(data)
2598                        }
2599                        Ok(Ok(Some(_))) => HedgedWaveAction::Continue,
2600                        Ok(Ok(None)) | Ok(Err(_)) => HedgedWaveAction::Abort,
2601                        Err(_) => HedgedWaveAction::Continue,
2602                    }
2603                }
2604            },
2605        )
2606        .await;
2607
2608        let Some(data) = result else {
2609            if let Some(pending) = self.pending_requests.write().await.remove(&hash_key) {
2610                self.release_queried_peer_requests(&pending.queried_peers)
2611                    .await;
2612                for peer_id in pending.queried_peers {
2613                    self.peer_selector.write().await.record_timeout(&peer_id);
2614                }
2615            }
2616            let _ = self.take_forward_requesters(&hash_key).await;
2617            return RouteFetchOutcome::Timeout;
2618        };
2619
2620        let _ = self.local_store.put(*hash, data.clone()).await;
2621        RouteFetchOutcome::Hit(data)
2622    }
2623
2624    async fn request_from_read_sources_inner(&self, hash: &Hash) -> RouteFetchOutcome {
2625        let ordered_sources = self.ordered_read_sources().await;
2626        if ordered_sources.is_empty() {
2627            return RouteFetchOutcome::Miss;
2628        }
2629
2630        let dispatch = normalize_dispatch_config(
2631            self.source_dispatch_for(ordered_sources.len()).await,
2632            ordered_sources.len(),
2633        );
2634        let wave_plan = build_hedged_wave_plan(ordered_sources.len(), dispatch);
2635        if wave_plan.is_empty() {
2636            return RouteFetchOutcome::Miss;
2637        }
2638
2639        let deadline = Instant::now() + self.request_timeout;
2640        let mut pending = FuturesUnordered::new();
2641        let mut pending_source_ids = HashSet::new();
2642        let mut saw_timeout = false;
2643        let mut next_source_idx = 0usize;
2644
2645        for (wave_idx, wave_size) in wave_plan.iter().copied().enumerate() {
2646            let from = next_source_idx;
2647            let to = (next_source_idx + wave_size).min(ordered_sources.len());
2648            next_source_idx = to;
2649
2650            for source in &ordered_sources[from..to] {
2651                let source = Arc::clone(source);
2652                let source_id = source.id().to_string();
2653                self.record_read_source_request(&source_id).await;
2654                pending_source_ids.insert(source_id.clone());
2655                let hash = *hash;
2656                pending.push(tokio::spawn(async move {
2657                    let started_at = Instant::now();
2658                    let result = std::panic::AssertUnwindSafe(source.get(&hash))
2659                        .catch_unwind()
2660                        .await;
2661                    match result {
2662                        Ok(Some(data)) => SourceFetchOutcome::Hit {
2663                            source_id,
2664                            data,
2665                            elapsed_ms: started_at.elapsed().as_millis().max(1) as u64,
2666                        },
2667                        Ok(None) => SourceFetchOutcome::Miss { source_id },
2668                        Err(_) => SourceFetchOutcome::Failure { source_id },
2669                    }
2670                }));
2671            }
2672
2673            let is_last_wave =
2674                wave_idx + 1 == wave_plan.len() || next_source_idx >= ordered_sources.len();
2675            let window_end = if is_last_wave {
2676                deadline
2677            } else {
2678                (Instant::now() + Duration::from_millis(dispatch.hedge_interval_ms)).min(deadline)
2679            };
2680
2681            while Instant::now() < window_end {
2682                let remaining = window_end.saturating_duration_since(Instant::now());
2683                let Some(result) = tokio::time::timeout(remaining, pending.next())
2684                    .await
2685                    .ok()
2686                    .flatten()
2687                else {
2688                    break;
2689                };
2690                let Ok(outcome) = result else {
2691                    continue;
2692                };
2693                match outcome {
2694                    SourceFetchOutcome::Hit {
2695                        source_id,
2696                        data,
2697                        elapsed_ms,
2698                    } => {
2699                        pending_source_ids.remove(&source_id);
2700                        self.record_read_source_success(&source_id, elapsed_ms)
2701                            .await;
2702                        return RouteFetchOutcome::Hit(data);
2703                    }
2704                    SourceFetchOutcome::Miss { source_id } => {
2705                        pending_source_ids.remove(&source_id);
2706                        self.record_read_source_miss(&source_id).await;
2707                    }
2708                    SourceFetchOutcome::Failure { source_id } => {
2709                        pending_source_ids.remove(&source_id);
2710                        self.record_read_source_failure(&source_id).await;
2711                    }
2712                }
2713            }
2714
2715            if Instant::now() >= deadline {
2716                break;
2717            }
2718        }
2719
2720        for source_id in pending_source_ids {
2721            saw_timeout = true;
2722            self.record_read_source_timeout(&source_id).await;
2723        }
2724        if saw_timeout {
2725            RouteFetchOutcome::Timeout
2726        } else {
2727            RouteFetchOutcome::Miss
2728        }
2729    }
2730
2731    async fn request_from_read_sources(&self, hash: &Hash) -> RouteFetchOutcome {
2732        let hash_key = hash_to_key(hash);
2733        let existing_wait = {
2734            let mut inflight = self.inflight_source_fetches.lock().await;
2735            if let Some(existing) = inflight.get_mut(&hash_key) {
2736                let (tx, rx) = oneshot::channel();
2737                existing.waiters.push(tx);
2738                Some(rx)
2739            } else {
2740                inflight.insert(
2741                    hash_key.clone(),
2742                    InflightSourceFetch {
2743                        waiters: Vec::new(),
2744                    },
2745                );
2746                None
2747            }
2748        };
2749
2750        if let Some(wait) = existing_wait {
2751            return wait.await.unwrap_or(RouteFetchOutcome::Timeout);
2752        }
2753
2754        let result = self.request_from_read_sources_inner(hash).await;
2755        if let RouteFetchOutcome::Hit(hit) = &result {
2756            let _ = self.local_store.put(*hash, hit.clone()).await;
2757        }
2758        self.complete_inflight_source_fetch(&hash_key, result.clone())
2759            .await;
2760
2761        result
2762    }
2763
2764    async fn complete_inflight_source_fetch(&self, hash_key: &str, result: RouteFetchOutcome) {
2765        let waiters = self
2766            .inflight_source_fetches
2767            .lock()
2768            .await
2769            .remove(hash_key)
2770            .map(|inflight| inflight.waiters)
2771            .unwrap_or_default();
2772        for waiter in waiters {
2773            let _ = waiter.send(result.clone());
2774        }
2775    }
2776
2777    async fn cancel_pending_peer_route(&self, hash: &Hash) {
2778        let hash_key = hash_to_key(hash);
2779        if let Some(pending) = self.pending_requests.write().await.remove(&hash_key) {
2780            self.release_queried_peer_requests(&pending.queried_peers)
2781                .await;
2782        }
2783    }
2784
2785    async fn cancel_losing_route(&self, hash: &Hash, route: &ReadRoute, winner_data: &[u8]) {
2786        match route {
2787            ReadRoute::Peers(_) => self.cancel_pending_peer_route(hash).await,
2788            ReadRoute::Sources => {
2789                let hash_key = hash_to_key(hash);
2790                self.complete_inflight_source_fetch(
2791                    &hash_key,
2792                    RouteFetchOutcome::Hit(winner_data.to_vec()),
2793                )
2794                .await;
2795            }
2796        }
2797    }
2798
2799    async fn ranked_read_routes(&self, context: &MeshReadContext) -> Vec<RankedReadRoute> {
2800        let mut routes = Vec::new();
2801        let ordered_peers = if should_forward_htl(context.request_htl) {
2802            self.ordered_connected_peers(context.exclude_peer_id.as_deref())
2803                .await
2804        } else {
2805            Vec::new()
2806        };
2807        if !ordered_peers.is_empty() {
2808            let best_peer_id = ordered_peers[0].clone();
2809            let selector = self.peer_selector.read().await;
2810            let best_peer = selector.get_stats(&best_peer_id).cloned();
2811            let now = Instant::now();
2812            let (score, has_history) = match best_peer.as_ref() {
2813                Some(stats) => (
2814                    peer_endpoint_score(stats, now),
2815                    peer_endpoint_has_history(stats),
2816                ),
2817                None => (0.0, false),
2818            };
2819            routes.push(RankedReadRoute {
2820                route: ReadRoute::Peers(ordered_peers),
2821                best_endpoint_id: format!("peer:{best_peer_id}"),
2822                score,
2823                has_history,
2824            });
2825        }
2826        let ordered_sources = self.ordered_read_sources().await;
2827        if let Some(best_source) = ordered_sources.first() {
2828            let stats = self.read_source_stats.read().await;
2829            let best_source_stats = stats.get(best_source.id()).cloned().unwrap_or_default();
2830            let now = Instant::now();
2831            routes.push(RankedReadRoute {
2832                route: ReadRoute::Sources,
2833                best_endpoint_id: format!("source:{}", best_source.id()),
2834                score: adaptive_source_score(&best_source_stats, now),
2835                has_history: source_has_history(&best_source_stats),
2836            });
2837        }
2838        if routes.len() <= 1 {
2839            return routes;
2840        }
2841
2842        routes.sort_by(|left, right| {
2843            right
2844                .score
2845                .partial_cmp(&left.score)
2846                .unwrap_or(std::cmp::Ordering::Equal)
2847                .then_with(|| ranked_route_kind(&left.route).cmp(&ranked_route_kind(&right.route)))
2848                .then_with(|| left.best_endpoint_id.cmp(&right.best_endpoint_id))
2849                .then_with(|| left.route.id().cmp(right.route.id()))
2850        });
2851        routes
2852    }
2853
2854    fn should_probe_multiple_routes(&self, routes: &[RankedReadRoute]) -> bool {
2855        if routes.len() <= 1 {
2856            return false;
2857        }
2858        if !routes[0].has_history || !routes[1].has_history {
2859            return false;
2860        }
2861        (routes[0].score - routes[1].score) < SOURCE_SCORE_TIE_DELTA
2862    }
2863
2864    async fn run_read_route(
2865        &self,
2866        hash: &Hash,
2867        route: &ReadRoute,
2868        context: &MeshReadContext,
2869    ) -> RouteFetchOutcome {
2870        match route {
2871            ReadRoute::Peers(peer_ids) => {
2872                self.request_from_ordered_peers(hash, peer_ids, context.request_htl)
2873                    .await
2874            }
2875            ReadRoute::Sources => self.request_from_read_sources(hash).await,
2876        }
2877    }
2878
2879    async fn request_from_mesh_with_context(
2880        &self,
2881        hash: &Hash,
2882        context: &MeshReadContext,
2883    ) -> Option<Vec<u8>> {
2884        let routes = self.ranked_read_routes(context).await;
2885        match routes.as_slice() {
2886            [] => None,
2887            [ranked] => match self.run_read_route(hash, &ranked.route, context).await {
2888                RouteFetchOutcome::Hit(data) => Some(data),
2889                RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => None,
2890            },
2891            [first, second, ..] => {
2892                if self.should_probe_multiple_routes(&routes) {
2893                    let first_fut = self.run_read_route(hash, &first.route, context);
2894                    let second_fut = self.run_read_route(hash, &second.route, context);
2895                    tokio::pin!(first_fut);
2896                    tokio::pin!(second_fut);
2897                    let mut first_done = false;
2898                    let mut second_done = false;
2899                    loop {
2900                        tokio::select! {
2901                            result = &mut first_fut, if !first_done => {
2902                                first_done = true;
2903                                if let RouteFetchOutcome::Hit(data) = result {
2904                                    if !second_done {
2905                                        self.cancel_losing_route(hash, &second.route, &data).await;
2906                                    }
2907                                    return Some(data);
2908                                }
2909                            }
2910                            result = &mut second_fut, if !second_done => {
2911                                second_done = true;
2912                                if let RouteFetchOutcome::Hit(data) = result {
2913                                    if !first_done {
2914                                        self.cancel_losing_route(hash, &first.route, &data).await;
2915                                    }
2916                                    return Some(data);
2917                                }
2918                            }
2919                            else => break,
2920                        }
2921                        if first_done && second_done {
2922                            break;
2923                        }
2924                    }
2925                    None
2926                } else {
2927                    match self.run_read_route(hash, &first.route, context).await {
2928                        RouteFetchOutcome::Hit(data) => return Some(data),
2929                        RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => {}
2930                    }
2931                    for ranked in routes.iter().skip(1) {
2932                        match self.run_read_route(hash, &ranked.route, context).await {
2933                            RouteFetchOutcome::Hit(data) => return Some(data),
2934                            RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => {}
2935                        }
2936                    }
2937                    None
2938                }
2939            }
2940        }
2941    }
2942
2943    async fn request_from_mesh(&self, hash: &Hash) -> Option<Vec<u8>> {
2944        self.request_from_mesh_with_context(hash, &MeshReadContext::default())
2945            .await
2946    }
2947
2948    async fn begin_forward_request(&self, hash_key: &str, requester_id: &str) -> bool {
2949        let mut pending = self.pending_forward_requests.write().await;
2950        if let Some(existing) = pending.get_mut(hash_key) {
2951            existing.requester_ids.insert(requester_id.to_string());
2952            return false;
2953        }
2954
2955        let mut requester_ids = HashSet::new();
2956        requester_ids.insert(requester_id.to_string());
2957        pending.insert(
2958            hash_key.to_string(),
2959            PendingForwardRequest { requester_ids },
2960        );
2961        true
2962    }
2963
2964    async fn was_recent_forward_miss(&self, hash_key: &str) -> bool {
2965        self.recent_forward_misses.lock().await.contains(hash_key)
2966    }
2967
2968    async fn mark_recent_forward_miss(&self, hash_key: &str) {
2969        let _ = self
2970            .recent_forward_misses
2971            .lock()
2972            .await
2973            .insert_if_new(hash_key.to_string());
2974    }
2975
2976    async fn take_forward_requesters(&self, hash_key: &str) -> Vec<String> {
2977        self.pending_forward_requests
2978            .write()
2979            .await
2980            .remove(hash_key)
2981            .map(|pending| pending.requester_ids.into_iter().collect())
2982            .unwrap_or_default()
2983    }
2984
2985    async fn complete_pending_response(
2986        self: &Arc<Self>,
2987        from_peer: &str,
2988        hash: &Hash,
2989        hash_key: String,
2990        payload: Vec<u8>,
2991    ) {
2992        let pending = self.pending_requests.write().await.remove(&hash_key);
2993        if let Some(pending) = pending {
2994            let payload_bytes = payload.len() as u64;
2995            self.record_useful_bytes_received_from_peer(from_peer, payload_bytes)
2996                .await;
2997            {
2998                let mut deliveries = self.verified_block_deliveries.lock().await;
2999                deliveries.deliveries.push_back(VerifiedBlockDelivery {
3000                    hash: *hash,
3001                    provider_peer_id: from_peer.to_string(),
3002                    payload_bytes,
3003                });
3004                while deliveries.deliveries.len() > VERIFIED_BLOCK_DELIVERY_CAPACITY {
3005                    deliveries.deliveries.pop_front();
3006                    deliveries.dropped_since_last_drain =
3007                        deliveries.dropped_since_last_drain.saturating_add(1);
3008                }
3009            }
3010            self.release_queried_peer_requests(&pending.queried_peers)
3011                .await;
3012            let rtt_ms = pending.started_at.elapsed().as_millis() as u64;
3013            self.peer_selector.write().await.record_success(
3014                from_peer,
3015                rtt_ms,
3016                payload.len() as u64,
3017            );
3018            let forward_requesters = self.take_forward_requesters(&hash_key).await;
3019            let response_bytes = if forward_requesters.is_empty() {
3020                None
3021            } else {
3022                Some(encode_response(&create_response(hash, payload.clone())))
3023            };
3024            let _ = pending.response_tx.send(Some(payload));
3025            if let Some(response_bytes) = response_bytes {
3026                for requester_id in forward_requesters {
3027                    Arc::clone(self)
3028                        .enqueue_response_send(requester_id, response_bytes.clone(), Instant::now())
3029                        .await;
3030                }
3031            }
3032        }
3033    }
3034
3035    async fn handle_quote_response_message(&self, from_peer: &str, res: DataQuoteResponse) {
3036        if !res.a {
3037            return;
3038        }
3039
3040        let Some(quote_id) = res.q else {
3041            return;
3042        };
3043
3044        let hash_key = hash_to_key(&res.h);
3045        let (preferred_mint_url, offered_payment_sat) = {
3046            let pending_quotes = self.pending_quotes.read().await;
3047            let Some(pending) = pending_quotes.get(&hash_key) else {
3048                return;
3049            };
3050            (
3051                pending.preferred_mint_url.clone(),
3052                pending.offered_payment_sat,
3053            )
3054        };
3055        if !self
3056            .should_accept_quote_response(
3057                from_peer,
3058                preferred_mint_url.as_deref(),
3059                offered_payment_sat,
3060                &res,
3061            )
3062            .await
3063        {
3064            return;
3065        }
3066        let mut pending_quotes = self.pending_quotes.write().await;
3067        if let Some(pending) = pending_quotes.remove(&hash_key) {
3068            let _ = pending.response_tx.send(Some(NegotiatedQuote {
3069                peer_id: from_peer.to_string(),
3070                quote_id,
3071                mint_url: res.m,
3072            }));
3073        }
3074    }
3075
3076    async fn handle_response_message(
3077        self: &Arc<Self>,
3078        from_peer: &str,
3079        res: crate::protocol::DataResponse,
3080    ) {
3081        let hash_key = hash_to_key(&res.h);
3082        let hash = match crate::protocol::bytes_to_hash(&res.h) {
3083            Some(h) => h,
3084            None => return,
3085        };
3086
3087        // Ignore malformed/corrupt payload and keep waiting for a valid response.
3088        if hashtree_core::sha256(&res.d) != hash {
3089            self.peer_selector.write().await.record_failure(from_peer);
3090            if self.debug {
3091                println!("[MeshStoreCore] Ignoring invalid response payload for {hash_key}");
3092            }
3093            return;
3094        }
3095
3096        self.complete_pending_response(from_peer, &hash, hash_key, res.d)
3097            .await;
3098    }
3099
3100    async fn handle_quote_request_message(&self, from_peer: &str, req: DataQuoteRequest) {
3101        let hash = match crate::protocol::bytes_to_hash(&req.h) {
3102            Some(h) => h,
3103            None => return,
3104        };
3105        let hash_key = hash_to_key(&hash);
3106
3107        {
3108            let selector = self.peer_selector.read().await;
3109            if self.should_refuse_requests_from_peer(&selector, from_peer) {
3110                if self.debug {
3111                    println!(
3112                        "[MeshStoreCore] Refusing quote request from delinquent peer {}",
3113                        from_peer
3114                    );
3115                }
3116                return;
3117            }
3118        }
3119
3120        let chosen_mint = self.choose_quote_mint(req.m.as_deref());
3121        let can_serve = self.local_store.has(&hash).await.ok().unwrap_or(false)
3122            && !self.should_drop_response(&hash)
3123            && !self.should_corrupt_response(&hash);
3124
3125        let res = if can_serve {
3126            let quote_id = self
3127                .issue_quote(from_peer, &hash_key, req.p, req.t, chosen_mint.as_deref())
3128                .await;
3129            create_quote_response_available(&hash, quote_id, req.p, req.t, chosen_mint.as_deref())
3130        } else {
3131            create_quote_response_unavailable(&hash)
3132        };
3133        let response_bytes = encode_quote_response(&res);
3134        if let Some(channel) = self.signaling.get_channel(from_peer).await {
3135            if channel.send(response_bytes.clone()).await.is_ok() {
3136                self.record_peer_wire_sent(from_peer, response_bytes.len() as u64)
3137                    .await;
3138            }
3139        }
3140    }
3141
3142    async fn handle_request_message(
3143        self: &Arc<Self>,
3144        from_peer: &str,
3145        req: crate::protocol::DataRequest,
3146    ) {
3147        let hash = match crate::protocol::bytes_to_hash(&req.h) {
3148            Some(h) => h,
3149            None => return,
3150        };
3151        let hash_key = hash_to_key(&hash);
3152
3153        if let Some(quote_id) = req.q {
3154            if !self.take_valid_quote(from_peer, &hash_key, quote_id).await {
3155                if self.debug {
3156                    println!(
3157                        "[MeshStoreCore] Refusing request with invalid or expired quote {} from {}",
3158                        quote_id, from_peer
3159                    );
3160                }
3161                return;
3162            }
3163        }
3164
3165        let allow_peer_forwarding = {
3166            let selector = self.peer_selector.read().await;
3167            !self.should_refuse_requests_from_peer(&selector, from_peer)
3168        };
3169
3170        // Check local store
3171        if let Ok(Some(mut data)) = self.local_store.get(&hash).await {
3172            if self.should_drop_response(&hash) {
3173                if self.debug {
3174                    println!(
3175                        "[MeshStoreCore] Dropping response for {} due to actor profile",
3176                        hash_to_key(&hash)
3177                    );
3178                }
3179                return;
3180            }
3181
3182            let response_delay = self.response_send_delay(&hash, data.len());
3183            if self.should_corrupt_response(&hash) {
3184                if data.is_empty() {
3185                    data.push(0x80);
3186                } else {
3187                    data[0] ^= 0x80;
3188                }
3189            }
3190
3191            // Send response
3192            let res = create_response(&hash, data);
3193            let response_bytes = encode_response(&res);
3194            let ready_at = Instant::now() + response_delay;
3195            Arc::clone(self)
3196                .enqueue_response_send(from_peer.to_string(), response_bytes, ready_at)
3197                .await;
3198            return;
3199        }
3200
3201        if self.pending_requests.read().await.contains_key(&hash_key) {
3202            let _ = self.begin_forward_request(&hash_key, from_peer).await;
3203            return;
3204        }
3205
3206        if self.was_recent_forward_miss(&hash_key).await {
3207            if self.debug {
3208                println!(
3209                    "[MeshStoreCore] Suppressing recently missed forwarded request for {}",
3210                    hash_key
3211                );
3212            }
3213            return;
3214        }
3215
3216        if !self.begin_forward_request(&hash_key, from_peer).await {
3217            return;
3218        }
3219
3220        let from_peer = from_peer.to_string();
3221        let this = Arc::clone(self);
3222        let request_htl = req.htl;
3223        tokio::spawn(async move {
3224            let result = if allow_peer_forwarding {
3225                let context = MeshReadContext {
3226                    exclude_peer_id: Some(from_peer.clone()),
3227                    request_htl,
3228                };
3229                this.request_from_mesh_with_context(&hash, &context).await
3230            } else {
3231                if this.debug {
3232                    println!(
3233                        "[MeshStoreCore] Serving request from delinquent peer {} via read sources only",
3234                        from_peer
3235                    );
3236                }
3237                match this.request_from_read_sources(&hash).await {
3238                    RouteFetchOutcome::Hit(data) => Some(data),
3239                    RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => None,
3240                }
3241            };
3242            let requester_ids = this.take_forward_requesters(&hash_key).await;
3243            if let Some(data) = result {
3244                let ready_at = Instant::now() + this.response_send_delay(&hash, data.len());
3245                let res = create_response(&hash, data);
3246                let response_bytes = encode_response(&res);
3247                for requester_id in requester_ids {
3248                    Arc::clone(&this)
3249                        .enqueue_response_send(requester_id, response_bytes.clone(), ready_at)
3250                        .await;
3251                }
3252            } else {
3253                this.mark_recent_forward_miss(&hash_key).await;
3254            }
3255        });
3256    }
3257
3258    async fn handle_pubsub_interest_message(
3259        self: &Arc<Self>,
3260        from_peer: &str,
3261        mut interest: PubsubInterest,
3262    ) {
3263        if !self.apply_pubsub_interest_route(from_peer, &interest).await {
3264            return;
3265        }
3266
3267        if !self.routing.pubsub_forwarding || interest.htl <= 1 {
3268            return;
3269        }
3270        interest.htl = interest.htl.saturating_sub(1);
3271        let _ = self
3272            .send_pubsub_interest_to_peers(&interest, Some(from_peer))
3273            .await;
3274    }
3275
3276    async fn handle_pubsub_frame_message(
3277        self: &Arc<Self>,
3278        from_peer: &str,
3279        mut frame: PubsubFrame,
3280        wire_bytes: usize,
3281    ) {
3282        if frame.stream_id.is_empty() || frame.origin_peer_id.is_empty() {
3283            return;
3284        }
3285        if frame.origin_peer_id == self.signaling.peer_id() {
3286            return;
3287        }
3288
3289        let frame_key = Self::pubsub_frame_key(&frame);
3290        if !self
3291            .pubsub_seen_frames
3292            .lock()
3293            .await
3294            .insert_if_new(frame_key.clone())
3295        {
3296            return;
3297        }
3298        self.cache_pubsub_frame(frame_key.clone(), frame.clone())
3299            .await;
3300
3301        let local_interested = self
3302            .pubsub_local_interests
3303            .read()
3304            .await
3305            .contains(&frame.stream_id);
3306        let mut downstream_peers = if self.routing.pubsub_forwarding && frame.htl > 1 {
3307            match self.routing.pubsub_delivery_mode {
3308                PubsubDeliveryMode::InterestPush => {
3309                    let mut peers = self
3310                        .interested_pubsub_peers(&frame.stream_id, Some(from_peer))
3311                        .await;
3312                    peers.extend(
3313                        self.take_pubsub_want_peers(&frame_key, Some(from_peer))
3314                            .await,
3315                    );
3316                    peers.sort();
3317                    peers.dedup();
3318                    peers
3319                }
3320                PubsubDeliveryMode::HtlInvWant => {
3321                    self.take_pubsub_want_peers(&frame_key, Some(from_peer))
3322                        .await
3323                }
3324            }
3325        } else {
3326            Vec::new()
3327        };
3328        downstream_peers.retain(|peer_id| peer_id != from_peer);
3329
3330        if local_interested || !downstream_peers.is_empty() {
3331            self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3332                .await;
3333        }
3334
3335        if local_interested {
3336            self.enqueue_pubsub_event(PubsubEvent {
3337                stream_id: frame.stream_id.clone(),
3338                seq: frame.seq,
3339                origin_peer_id: frame.origin_peer_id.clone(),
3340                from_peer_id: from_peer.to_string(),
3341                payload: frame.payload.clone(),
3342            })
3343            .await;
3344        }
3345
3346        if downstream_peers.is_empty() {
3347            return;
3348        }
3349
3350        frame.htl = frame.htl.saturating_sub(1);
3351        let _ = self
3352            .send_pubsub_frame_to_peers(&frame, &downstream_peers)
3353            .await;
3354    }
3355
3356    async fn handle_pubsub_inventory_message(
3357        self: &Arc<Self>,
3358        from_peer: &str,
3359        inv: PubsubInventory,
3360        wire_bytes: usize,
3361    ) {
3362        if inv.stream_id.is_empty() || inv.origin_peer_id.is_empty() {
3363            return;
3364        }
3365        if inv.origin_peer_id == self.signaling.peer_id() {
3366            return;
3367        }
3368
3369        let key = Self::pubsub_key(&inv.origin_peer_id, &inv.stream_id, inv.seq);
3370        if !self
3371            .pubsub_seen_inventories
3372            .lock()
3373            .await
3374            .insert_if_new(key.clone())
3375        {
3376            return;
3377        }
3378        {
3379            let mut routes = self.pubsub_inventory_routes.write().await;
3380            routes
3381                .entry(key.clone())
3382                .or_insert_with(|| from_peer.to_string());
3383        }
3384
3385        let local_interested = self
3386            .pubsub_local_interests
3387            .read()
3388            .await
3389            .contains(&inv.stream_id);
3390        let downstream_peers = if self.routing.pubsub_forwarding {
3391            self.interested_pubsub_peers(&inv.stream_id, Some(from_peer))
3392                .await
3393        } else {
3394            Vec::new()
3395        };
3396        if local_interested || !downstream_peers.is_empty() {
3397            self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3398                .await;
3399            let want =
3400                create_pubsub_want(inv.stream_id.clone(), inv.seq, inv.origin_peer_id.clone());
3401            let _ = self.send_pubsub_want_upstream(&key, &want, None).await;
3402        }
3403
3404        if !self.routing.pubsub_forwarding
3405            || downstream_peers.is_empty()
3406            || !should_forward_htl(inv.htl)
3407        {
3408            return;
3409        }
3410        let _ = self
3411            .send_pubsub_inventory_to_peers(&inv, &downstream_peers)
3412            .await;
3413    }
3414
3415    async fn handle_pubsub_want_message(
3416        self: &Arc<Self>,
3417        from_peer: &str,
3418        want: PubsubWant,
3419        wire_bytes: usize,
3420    ) {
3421        if want.stream_id.is_empty() || want.origin_peer_id.is_empty() {
3422            return;
3423        }
3424        if want.origin_peer_id == from_peer {
3425            return;
3426        }
3427
3428        let key = Self::pubsub_key(&want.origin_peer_id, &want.stream_id, want.seq);
3429        let want_key = format!("{from_peer}:{key}");
3430        if !self.pubsub_seen_wants.lock().await.insert_if_new(want_key) {
3431            return;
3432        }
3433
3434        if let Some(frame) = self.cached_pubsub_frame(&key).await {
3435            self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3436                .await;
3437            let peers = vec![from_peer.to_string()];
3438            let _ = self.send_pubsub_frame_to_peers(&frame, &peers).await;
3439            return;
3440        }
3441
3442        let has_upstream_route = self.pubsub_inventory_routes.read().await.contains_key(&key);
3443        if !has_upstream_route {
3444            return;
3445        }
3446
3447        if self.remember_pubsub_want_peer(key.clone(), from_peer).await {
3448            self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3449                .await;
3450        }
3451        let _ = self
3452            .send_pubsub_want_upstream(&key, &want, Some(from_peer))
3453            .await;
3454    }
3455
3456    /// Handle incoming data message
3457    pub async fn handle_data_message(self: &Arc<Self>, from_peer: &str, data: &[u8]) {
3458        self.record_peer_wire_received(from_peer, data.len() as u64)
3459            .await;
3460        let parsed = match parse_message(data) {
3461            Some(m) => m,
3462            None => return,
3463        };
3464
3465        match parsed {
3466            DataMessage::Request(req) => {
3467                self.handle_request_message(from_peer, req).await;
3468            }
3469            DataMessage::Response(res) => {
3470                self.handle_response_message(from_peer, res).await;
3471            }
3472            DataMessage::QuoteRequest(req) => {
3473                self.handle_quote_request_message(from_peer, req).await;
3474            }
3475            DataMessage::QuoteResponse(res) => {
3476                self.handle_quote_response_message(from_peer, res).await;
3477            }
3478            DataMessage::PubsubInterest(interest) => {
3479                self.handle_pubsub_interest_message(from_peer, interest)
3480                    .await;
3481            }
3482            DataMessage::PubsubFrame(frame) => {
3483                self.handle_pubsub_frame_message(from_peer, frame, data.len())
3484                    .await;
3485            }
3486            DataMessage::PubsubInventory(inv) => {
3487                self.handle_pubsub_inventory_message(from_peer, inv, data.len())
3488                    .await;
3489            }
3490            DataMessage::PubsubWant(want) => {
3491                self.handle_pubsub_want_message(from_peer, want, data.len())
3492                    .await;
3493            }
3494            DataMessage::Payment(_)
3495            | DataMessage::PaymentAck(_)
3496            | DataMessage::Chunk(_)
3497            | DataMessage::PeerHints(_) => {}
3498        }
3499    }
3500}
3501
3502#[async_trait]
3503impl<S, R, F> Store for MeshStoreCore<S, R, F>
3504where
3505    S: Store + Send + Sync + 'static,
3506    R: SignalingTransport + Send + Sync + 'static,
3507    F: PeerLinkFactory + Send + Sync + 'static,
3508{
3509    async fn put(&self, hash: Hash, data: Vec<u8>) -> Result<bool, StoreError> {
3510        self.local_store.put(hash, data).await
3511    }
3512
3513    async fn get(&self, hash: &Hash) -> Result<Option<Vec<u8>>, StoreError> {
3514        // Try local first
3515        if let Some(data) = self.local_store.get(hash).await? {
3516            return Ok(Some(data));
3517        }
3518
3519        // Try peers
3520        Ok(self.request_from_mesh(hash).await)
3521    }
3522
3523    async fn has(&self, hash: &Hash) -> Result<bool, StoreError> {
3524        self.local_store.has(hash).await
3525    }
3526
3527    async fn delete(&self, hash: &Hash) -> Result<bool, StoreError> {
3528        self.local_store.delete(hash).await
3529    }
3530}
3531
3532#[cfg(test)]
3533mod delivery_tests;
3534
3535#[cfg(test)]
3536mod tests;
3537
3538/// Type alias for simulation store.
3539pub type SimMeshStore<S> =
3540    MeshStoreCore<S, crate::mock::MockRelayTransport, crate::mock::MockConnectionFactory>;