Skip to main content

hashtree_network/
mesh_store_core.rs

1//! Shared routed mesh store core.
2//!
3//! This module provides a concrete store wrapper that works with any local storage
4//! backend plus any signaling transport and peer-link factory. Both production
5//! and simulation (mocks) use this same code.
6
7use async_trait::async_trait;
8use std::collections::hash_map::DefaultHasher;
9use std::collections::{HashMap, HashSet, VecDeque};
10use std::future::Future;
11use std::hash::{Hash as _, Hasher};
12use std::ops::Range;
13use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
14use std::sync::Arc;
15use std::time::Duration;
16use tokio::sync::{oneshot, Mutex, Notify, RwLock};
17use tokio::time::Instant;
18
19use hashtree_core::{
20    BlobReply, BlobRequest, BlobRoute, BlobRouteContext, Hash, Store, StoreError, BLOB_MAX_BYTES,
21};
22
23use crate::peer_selector::{PeerMetadataSnapshot, PeerSelector, SelectionStrategy};
24use crate::protocol::{
25    create_pubsub_frame, create_pubsub_interest, create_pubsub_inventory, create_pubsub_want,
26    create_quote_request, create_quote_response_available, create_quote_response_unavailable,
27    create_request, create_request_with_quote, create_response, encode_pubsub_frame,
28    encode_pubsub_interest, encode_pubsub_inventory, encode_pubsub_want, encode_quote_request,
29    encode_quote_response, encode_request, encode_response, hash_to_key, parse_message,
30    DataMessage, DataQuoteRequest, DataQuoteResponse, PubsubFrame, PubsubInterest, PubsubInventory,
31    PubsubWant,
32};
33use crate::pubsub_strategy::{
34    reciprocal_virtual_finish, select_reciprocal_outbound_job, OutboundJobCandidate,
35    PeerTrafficSnapshot, PubsubCandidate, PubsubSchedulerConfig,
36};
37use crate::signaling::MeshRouter;
38use crate::transport::{PeerLinkFactory, SignalingTransport, TransportError};
39use crate::types::{
40    should_forward_htl, PeerHTLConfig, SignalingMessage, TimedSeenSet, MAX_HTL, MESH_EVENT_POLICY,
41};
42
43// Keep the on-disk namespace stable across the crate rename so existing peer
44// metadata does not disappear for users upgrading from the old package name.
45const PEER_METADATA_POINTER_SLOT_KEY: &[u8] = b"hashtree-mesh/peer-metadata/latest/v1";
46const PUBSUB_SEEN_CAPACITY: usize = 16_384;
47const PUBSUB_INBOX_CAPACITY: usize = 4_096;
48const PUBSUB_FRAME_CACHE_CAPACITY: usize = 4_096;
49const VERIFIED_BLOCK_DELIVERY_CAPACITY: usize = 4_096;
50const PUBSUB_SEEN_TTL: Duration = Duration::from_secs(120);
51
52/// Pending request awaiting response
53struct PendingRequest {
54    owner: Arc<()>,
55    response_tx: oneshot::Sender<Option<Vec<u8>>>,
56    started_at: Instant,
57    queried_peers: Vec<String>,
58}
59
60#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
61struct PendingRequestKey {
62    hash: Hash,
63    htl: u8,
64}
65
66impl PendingRequestKey {
67    fn new(hash: Hash, htl: u8) -> Self {
68        Self { hash, htl }
69    }
70}
71
72struct PendingQuoteRequest {
73    response_tx: oneshot::Sender<Option<NegotiatedQuote>>,
74    preferred_mint_url: Option<String>,
75    offered_payment_sat: u64,
76}
77
78struct PendingForwardRequest {
79    requester_ids: HashSet<String>,
80}
81
82type PeerWireStats = PeerTrafficSnapshot;
83
84struct PendingResponseSend {
85    job_id: u64,
86    peer_id: String,
87    bytes: Vec<u8>,
88    ready_at: Instant,
89    queue_sequence: u64,
90}
91
92#[derive(Debug, Clone)]
93struct NegotiatedQuote {
94    peer_id: String,
95    quote_id: u64,
96    #[allow(dead_code)]
97    mint_url: Option<String>,
98}
99
100struct IssuedQuote {
101    expires_at: Instant,
102    #[allow(dead_code)]
103    payment_sat: u64,
104    #[allow(dead_code)]
105    mint_url: Option<String>,
106}
107
108#[derive(Debug, Clone)]
109enum RouteFetchOutcome {
110    Hit(Vec<u8>),
111    Miss,
112    Timeout,
113}
114
115const ACTIVE_PEER_REQUEST_RANK_PENALTY: usize = 3;
116
117#[derive(Debug, Clone)]
118struct MeshReadContext {
119    exclude_peer_id: Option<String>,
120    request_htl: u8,
121    deadline: Option<Instant>,
122    attempt_budget: Option<usize>,
123}
124
125impl Default for MeshReadContext {
126    fn default() -> Self {
127        Self {
128            exclude_peer_id: None,
129            request_htl: MAX_HTL,
130            deadline: None,
131            attempt_budget: None,
132        }
133    }
134}
135
136/// Aggregate stats from draining currently available peer-link messages.
137#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
138pub struct DataPumpStats {
139    pub processed: usize,
140    pub request_messages: usize,
141    pub response_messages: usize,
142    pub quote_request_messages: u64,
143    pub quote_response_messages: u64,
144    pub pubsub_interest_messages: u64,
145    pub pubsub_frame_messages: u64,
146    pub pubsub_inventory_messages: u64,
147    pub pubsub_want_messages: u64,
148    pub processed_bytes: u64,
149}
150
151/// Pubsub data delivered to a local subscription.
152#[derive(Debug, Clone, PartialEq, Eq)]
153pub struct PubsubEvent {
154    pub stream_id: String,
155    pub seq: u64,
156    pub origin_peer_id: String,
157    pub from_peer_id: String,
158    pub payload: Vec<u8>,
159}
160
161/// Evidence that this peer won an outstanding, hash-verified block request.
162#[derive(Debug, Clone, PartialEq, Eq)]
163pub struct VerifiedBlockDelivery {
164    pub hash: Hash,
165    pub provider_peer_id: String,
166    pub payload_bytes: u64,
167}
168
169/// One atomic drain of verified delivery evidence and any overflow since the prior drain.
170///
171/// Dropped evidence is intentionally not recoverable or billable through this API. An
172/// application adapter must surface a non-zero count and must not infer a payment claim.
173#[derive(Debug, Clone, Default, PartialEq, Eq)]
174pub struct VerifiedBlockDeliveryBatch {
175    pub deliveries: Vec<VerifiedBlockDelivery>,
176    pub dropped_since_last_drain: u64,
177}
178
179#[derive(Default)]
180struct VerifiedBlockDeliveryBuffer {
181    deliveries: VecDeque<VerifiedBlockDelivery>,
182    dropped_since_last_drain: u64,
183}
184
185/// Send-side accounting from a pubsub publish or forwarded pubsub message.
186#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
187pub struct PubsubPublishStats {
188    pub selected_peers: usize,
189    pub sent_peers: usize,
190    pub sent_bytes: u64,
191    pub deferred_peers: usize,
192}
193
194/// Production pubsub delivery strategy.
195#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
196pub enum PubsubDeliveryMode {
197    /// Push full frames only along advertised interest routes.
198    InterestPush,
199    /// Route small inventories along advertised interest paths and pull payloads back along want paths.
200    #[default]
201    HtlInvWant,
202}
203
204/// Request dispatch strategy for peer queries.
205///
206/// Requests use bounded staged hedging by default. Callers may explicitly
207/// raise the cap for controlled simulations, but production never needs an
208/// unbounded sentinel.
209#[derive(Debug, Clone, Copy)]
210pub struct RequestDispatchConfig {
211    /// Number of peers queried immediately.
212    pub initial_fanout: usize,
213    /// Number of additional peers to query on each hedge step.
214    pub hedge_fanout: usize,
215    /// Total peers allowed for this request.
216    pub max_fanout: usize,
217    /// Delay between hedge waves (ms). `0` means send all waves immediately.
218    pub hedge_interval_ms: u64,
219}
220
221impl Default for RequestDispatchConfig {
222    fn default() -> Self {
223        Self {
224            initial_fanout: 2,
225            hedge_fanout: 1,
226            max_fanout: 4,
227            hedge_interval_ms: 50,
228        }
229    }
230}
231
232/// Normalize fanout config against current peer availability.
233pub fn normalize_dispatch_config(
234    dispatch: RequestDispatchConfig,
235    available_peers: usize,
236) -> RequestDispatchConfig {
237    let mut cfg = dispatch;
238    let cap = if cfg.max_fanout == 0 {
239        available_peers
240    } else {
241        cfg.max_fanout.min(available_peers)
242    };
243    cfg.max_fanout = cap;
244    cfg.initial_fanout = if cfg.initial_fanout == 0 {
245        1
246    } else {
247        cfg.initial_fanout.min(cap.max(1))
248    };
249    cfg.hedge_fanout = if cfg.hedge_fanout == 0 {
250        1
251    } else {
252        cfg.hedge_fanout.min(cap.max(1))
253    };
254    cfg
255}
256
257/// Build wave sizes for staged hedged dispatch.
258pub fn build_hedged_wave_plan(peer_count: usize, dispatch: RequestDispatchConfig) -> Vec<usize> {
259    if peer_count == 0 {
260        return Vec::new();
261    }
262    let cap = dispatch.max_fanout.min(peer_count);
263    if cap == 0 {
264        return Vec::new();
265    }
266
267    let mut plan = Vec::new();
268    let mut sent = 0usize;
269    let first = dispatch.initial_fanout.min(cap).max(1);
270    plan.push(first);
271    sent += first;
272
273    while sent < cap {
274        let next = dispatch.hedge_fanout.min(cap - sent).max(1);
275        plan.push(next);
276        sent += next;
277    }
278    plan
279}
280
281/// Outcome returned after waiting on a hedged dispatch wave.
282#[derive(Debug)]
283pub enum HedgedWaveAction<T> {
284    Continue,
285    Success(T),
286    Abort,
287}
288
289/// Run a staged hedged dispatch over peer index ranges.
290///
291/// This scheduler is shared by the reusable `MeshStoreCore` and the native
292/// `hashtree-cli` mesh path so tests and production use the same wave timing.
293pub async fn run_hedged_waves<T, SendWave, SendWaveFut, WaitWave, WaitWaveFut>(
294    peer_count: usize,
295    dispatch: RequestDispatchConfig,
296    request_timeout: Duration,
297    mut send_wave: SendWave,
298    mut wait_wave: WaitWave,
299) -> Option<T>
300where
301    SendWave: FnMut(Range<usize>) -> SendWaveFut,
302    SendWaveFut: Future<Output = usize>,
303    WaitWave: FnMut(Duration) -> WaitWaveFut,
304    WaitWaveFut: Future<Output = HedgedWaveAction<T>>,
305{
306    let dispatch = normalize_dispatch_config(dispatch, peer_count);
307    let wave_plan = build_hedged_wave_plan(peer_count, dispatch);
308    if wave_plan.is_empty() {
309        return None;
310    }
311
312    let deadline = Instant::now() + request_timeout;
313    let mut sent_total = 0usize;
314    let mut next_peer_idx = 0usize;
315
316    for (wave_idx, wave_size) in wave_plan.iter().copied().enumerate() {
317        let from = next_peer_idx;
318        let to = (next_peer_idx + wave_size).min(peer_count);
319        next_peer_idx = to;
320
321        if from == to {
322            continue;
323        }
324
325        sent_total += send_wave(from..to).await;
326        if sent_total == 0 {
327            if next_peer_idx >= peer_count {
328                break;
329            }
330            continue;
331        }
332
333        let now = Instant::now();
334        if now >= deadline {
335            break;
336        }
337        let remaining = deadline.saturating_duration_since(now);
338        let is_last_wave = wave_idx + 1 == wave_plan.len() || next_peer_idx >= peer_count;
339        let wait = if is_last_wave {
340            remaining
341        } else if dispatch.hedge_interval_ms == 0 {
342            Duration::ZERO
343        } else {
344            Duration::from_millis(dispatch.hedge_interval_ms).min(remaining)
345        };
346
347        if wait.is_zero() {
348            continue;
349        }
350
351        match wait_wave(wait).await {
352            HedgedWaveAction::Continue => {}
353            HedgedWaveAction::Success(value) => return Some(value),
354            HedgedWaveAction::Abort => break,
355        }
356    }
357
358    None
359}
360
361/// Keep selector membership aligned with currently connected peer IDs.
362pub async fn sync_selector_peers(selector: &RwLock<PeerSelector>, current_peer_ids: &[String]) {
363    let mut selector = selector.write().await;
364    let current: HashSet<&str> = current_peer_ids.iter().map(String::as_str).collect();
365    let known: Vec<String> = selector.all_stats().map(|s| s.peer_id.clone()).collect();
366    for peer_id in known {
367        if !current.contains(peer_id.as_str()) {
368            selector.remove_peer(&peer_id);
369        }
370    }
371    for peer_id in current_peer_ids {
372        selector.add_peer(peer_id.clone());
373    }
374}
375
376/// Response behavior profile for simulation/game-theory actors.
377///
378/// Defaults to honest behavior (always respond correctly, no extra delay).
379#[derive(Debug, Clone, Copy)]
380pub struct ResponseBehaviorConfig {
381    /// Probability that a node drops a response even when it has data.
382    pub drop_response_prob: f64,
383    /// Probability that a node responds with corrupted payload.
384    pub corrupt_response_prob: f64,
385    /// Baseline response delay before a peer starts sending any data.
386    pub extra_delay_ms: u64,
387    /// Additional delay before the first response byte becomes available.
388    pub first_byte_delay_ms: u64,
389    /// Sustained throughput for delivering large payloads. `0` disables size-based slowdown.
390    pub bytes_per_second: u64,
391    /// Probability that an otherwise honest response experiences an extra stall.
392    pub stall_response_prob: f64,
393    /// Extra delay injected when a stall event happens.
394    pub stall_delay_ms: u64,
395}
396
397impl Default for ResponseBehaviorConfig {
398    fn default() -> Self {
399        Self {
400            drop_response_prob: 0.0,
401            corrupt_response_prob: 0.0,
402            extra_delay_ms: 0,
403            first_byte_delay_ms: 0,
404            bytes_per_second: 0,
405            stall_response_prob: 0.0,
406            stall_delay_ms: 0,
407        }
408    }
409}
410
411impl ResponseBehaviorConfig {
412    fn normalized(self) -> Self {
413        Self {
414            drop_response_prob: self.drop_response_prob.clamp(0.0, 1.0),
415            corrupt_response_prob: self.corrupt_response_prob.clamp(0.0, 1.0),
416            extra_delay_ms: self.extra_delay_ms,
417            first_byte_delay_ms: self.first_byte_delay_ms,
418            bytes_per_second: self.bytes_per_second,
419            stall_response_prob: self.stall_response_prob.clamp(0.0, 1.0),
420            stall_delay_ms: self.stall_delay_ms,
421        }
422    }
423}
424
425/// Routing policy for request ordering + dispatch fanout.
426#[derive(Debug, Clone)]
427pub struct MeshRoutingConfig {
428    pub selection_strategy: SelectionStrategy,
429    pub fairness_enabled: bool,
430    /// Blend weight for payment-priority ranking in selector (`0.0` disables).
431    pub cashu_payment_weight: f64,
432    /// Refuse serving peers that have reached this many unpaid post-delivery settlements.
433    /// `0` disables refusal and only keeps metadata/downranking.
434    pub cashu_payment_default_block_threshold: u64,
435    /// Cashu mint URLs this node is willing to use for settlement.
436    pub cashu_accepted_mints: Vec<String>,
437    /// Preferred Cashu mint URL when initiating paid retrieval.
438    pub cashu_default_mint: Option<String>,
439    /// Baseline cap for accepting a peer-suggested mint outside the trusted set.
440    pub cashu_peer_suggested_mint_base_cap_sat: u64,
441    /// Additional sats allowed per successful delivery from that peer.
442    pub cashu_peer_suggested_mint_success_step_sat: u64,
443    /// Additional sats allowed per successful post-delivery payment received from that peer.
444    pub cashu_peer_suggested_mint_receipt_step_sat: u64,
445    /// Hard upper bound for any single peer-suggested mint quote we accept.
446    pub cashu_peer_suggested_mint_max_cap_sat: u64,
447    pub dispatch: RequestDispatchConfig,
448    pub response_behavior: ResponseBehaviorConfig,
449    pub pubsub_scheduler: PubsubSchedulerConfig,
450    pub pubsub_delivery_mode: PubsubDeliveryMode,
451    /// Forward peer pubsub interests, inventories, and payloads for downstream peers.
452    pub pubsub_forwarding: bool,
453    /// Initial hops-to-live for locally originated pubsub interest/inventory frames.
454    pub pubsub_max_htl: u8,
455}
456
457impl Default for MeshRoutingConfig {
458    fn default() -> Self {
459        Self {
460            selection_strategy: SelectionStrategy::Weighted,
461            fairness_enabled: true,
462            cashu_payment_weight: 0.0,
463            cashu_payment_default_block_threshold: 0,
464            cashu_accepted_mints: Vec::new(),
465            cashu_default_mint: None,
466            cashu_peer_suggested_mint_base_cap_sat: 0,
467            cashu_peer_suggested_mint_success_step_sat: 0,
468            cashu_peer_suggested_mint_receipt_step_sat: 0,
469            cashu_peer_suggested_mint_max_cap_sat: 0,
470            dispatch: RequestDispatchConfig::default(),
471            response_behavior: ResponseBehaviorConfig::default(),
472            pubsub_scheduler: PubsubSchedulerConfig::default(),
473            pubsub_delivery_mode: PubsubDeliveryMode::HtlInvWant,
474            pubsub_forwarding: true,
475            pubsub_max_htl: MESH_EVENT_POLICY.max_htl,
476        }
477    }
478}
479
480impl MeshRoutingConfig {
481    fn pubsub_initial_htl(&self) -> u8 {
482        self.pubsub_max_htl.clamp(1, MAX_HTL)
483    }
484}
485
486/// Routed mesh store core that works with any storage backend and transport
487/// implementation.
488///
489/// This is the shared code between production and simulation.
490/// - Production: transport-specific crates compose `MeshStoreCore` with their links
491/// - Simulation: `MeshStoreCore<MemoryStore, MockRelayTransport, MockConnectionFactory>`
492pub struct MeshStoreCore<S, R, F>
493where
494    S: Store + Send + Sync + 'static,
495    R: SignalingTransport + Send + Sync + 'static,
496    F: PeerLinkFactory + Send + Sync + 'static,
497{
498    /// Local backing store
499    local_store: Arc<S>,
500    /// Mesh router (handles peer discovery and connection)
501    signaling: Arc<MeshRouter<R, F>>,
502    /// Per-peer HTL config
503    htl_configs: RwLock<HashMap<String, PeerHTLConfig>>,
504    /// Pending requests we sent
505    pending_requests: RwLock<HashMap<PendingRequestKey, Vec<PendingRequest>>>,
506    /// Pending quote negotiations keyed by requested hash.
507    pending_quotes: RwLock<HashMap<String, PendingQuoteRequest>>,
508    /// Forwarded peer requests currently being resolved through the mesh/upstream.
509    pending_forward_requests: RwLock<HashMap<PendingRequestKey, PendingForwardRequest>>,
510    /// Bounded negative cache for recently forwarded misses/timeouts.
511    /// Quotes we issued to peers and will accept exactly once until expiry.
512    issued_quotes: RwLock<HashMap<(String, String, u64), IssuedQuote>>,
513    /// Monotonic quote identifier generator.
514    next_quote_id: RwLock<u64>,
515    /// Adaptive selector for peer ordering.
516    peer_selector: RwLock<PeerSelector>,
517    /// Active per-peer in-flight reads so concurrent block fetches spread across peers.
518    peer_active_requests: RwLock<HashMap<String, usize>>,
519    /// Actual wire traffic stats used for upload-side reciprocity scheduling.
520    peer_wire_stats: RwLock<HashMap<String, PeerWireStats>>,
521    /// Streams this node wants delivered locally.
522    pubsub_local_interests: RwLock<HashSet<String>>,
523    /// Current sequence per local stream interest.
524    pubsub_local_interest_versions: RwLock<HashMap<String, u64>>,
525    /// Reverse pubsub routes: stream id -> peers with local/downstream interest.
526    pubsub_peer_interests: RwLock<HashMap<String, HashSet<String>>>,
527    /// Route owner for each downstream subscriber interest.
528    pubsub_interest_routes: RwLock<HashMap<(String, String), String>>,
529    /// Latest interest sequence observed per subscriber/stream.
530    pubsub_interest_versions: RwLock<HashMap<(String, String), u64>>,
531    /// Bounded dedupe for pubsub interest floods.
532    pubsub_seen_interests: Mutex<TimedSeenSet>,
533    /// Bounded dedupe for pubsub data frames.
534    pubsub_seen_frames: Mutex<TimedSeenSet>,
535    /// Bounded dedupe for pubsub inventory floods.
536    pubsub_seen_inventories: Mutex<TimedSeenSet>,
537    /// Bounded dedupe for pubsub wants by requesting peer.
538    pubsub_seen_wants: Mutex<TimedSeenSet>,
539    /// First upstream peer that announced each inventory key.
540    pubsub_inventory_routes: RwLock<HashMap<String, String>>,
541    /// Downstream peers waiting for a payload after sending a want.
542    pubsub_want_routes: RwLock<HashMap<String, HashSet<String>>>,
543    /// Dedupe for wants this node already sent upstream.
544    pubsub_upstream_wants: Mutex<TimedSeenSet>,
545    /// Small payload cache for serving wants after inventory-first announcements.
546    pubsub_frame_cache: Mutex<VecDeque<(String, PubsubFrame)>>,
547    /// Local pubsub delivery inbox.
548    pubsub_inbox: Mutex<VecDeque<PubsubEvent>>,
549    /// Bounded application-facing evidence for first-winner block deliveries.
550    verified_block_deliveries: Mutex<VerifiedBlockDeliveryBuffer>,
551    /// Wakes consumers waiting for local pubsub deliveries.
552    pubsub_notify: Notify,
553    /// Per stream/peer deferred counts for aging pubsub strategies.
554    pubsub_deferred_counts: RwLock<HashMap<(String, String), u64>>,
555    /// Monotonic sequence for locally originated pubsub interest updates.
556    next_pubsub_interest_seq: AtomicU64,
557    /// Pending content responses waiting for upload arbitration.
558    pending_response_sends: Mutex<Vec<PendingResponseSend>>,
559    /// Upload response scheduler state.
560    response_scheduler_running: AtomicBool,
561    /// Monotonic id for queued response sends.
562    next_response_job_id: AtomicU64,
563    /// Routing/dispatch configuration.
564    routing: MeshRoutingConfig,
565    /// Request timeout
566    request_timeout: Duration,
567    /// Debug mode
568    debug: bool,
569    /// Running flag
570    running: RwLock<bool>,
571}
572
573impl<S, R, F> MeshStoreCore<S, R, F>
574where
575    S: Store + Send + Sync + 'static,
576    R: SignalingTransport + Send + Sync + 'static,
577    F: PeerLinkFactory + Send + Sync + 'static,
578{
579    /// Create a new routed mesh store core.
580    pub fn new(
581        local_store: Arc<S>,
582        signaling: Arc<MeshRouter<R, F>>,
583        request_timeout: Duration,
584        debug: bool,
585    ) -> Self {
586        Self::new_with_routing(
587            local_store,
588            signaling,
589            request_timeout,
590            debug,
591            Default::default(),
592        )
593    }
594
595    /// Create a new routed mesh store core with explicit routing configuration.
596    pub fn new_with_routing(
597        local_store: Arc<S>,
598        signaling: Arc<MeshRouter<R, F>>,
599        request_timeout: Duration,
600        debug: bool,
601        routing: MeshRoutingConfig,
602    ) -> Self {
603        let mut selector = PeerSelector::with_strategy(routing.selection_strategy);
604        selector.set_fairness(routing.fairness_enabled);
605        selector.set_cashu_payment_weight(routing.cashu_payment_weight);
606        Self {
607            local_store,
608            signaling,
609            htl_configs: RwLock::new(HashMap::new()),
610            pending_requests: RwLock::new(HashMap::new()),
611            pending_quotes: RwLock::new(HashMap::new()),
612            pending_forward_requests: RwLock::new(HashMap::new()),
613            issued_quotes: RwLock::new(HashMap::new()),
614            next_quote_id: RwLock::new(1),
615            peer_selector: RwLock::new(selector),
616            peer_active_requests: RwLock::new(HashMap::new()),
617            peer_wire_stats: RwLock::new(HashMap::new()),
618            pubsub_local_interests: RwLock::new(HashSet::new()),
619            pubsub_local_interest_versions: RwLock::new(HashMap::new()),
620            pubsub_peer_interests: RwLock::new(HashMap::new()),
621            pubsub_interest_routes: RwLock::new(HashMap::new()),
622            pubsub_interest_versions: RwLock::new(HashMap::new()),
623            pubsub_seen_interests: Mutex::new(TimedSeenSet::new(
624                PUBSUB_SEEN_CAPACITY,
625                PUBSUB_SEEN_TTL,
626            )),
627            pubsub_seen_frames: Mutex::new(TimedSeenSet::new(
628                PUBSUB_SEEN_CAPACITY,
629                PUBSUB_SEEN_TTL,
630            )),
631            pubsub_seen_inventories: Mutex::new(TimedSeenSet::new(
632                PUBSUB_SEEN_CAPACITY,
633                PUBSUB_SEEN_TTL,
634            )),
635            pubsub_seen_wants: Mutex::new(TimedSeenSet::new(PUBSUB_SEEN_CAPACITY, PUBSUB_SEEN_TTL)),
636            pubsub_inventory_routes: RwLock::new(HashMap::new()),
637            pubsub_want_routes: RwLock::new(HashMap::new()),
638            pubsub_upstream_wants: Mutex::new(TimedSeenSet::new(
639                PUBSUB_SEEN_CAPACITY,
640                PUBSUB_SEEN_TTL,
641            )),
642            pubsub_frame_cache: Mutex::new(VecDeque::new()),
643            pubsub_inbox: Mutex::new(VecDeque::new()),
644            verified_block_deliveries: Mutex::new(VerifiedBlockDeliveryBuffer::default()),
645            pubsub_notify: Notify::new(),
646            pubsub_deferred_counts: RwLock::new(HashMap::new()),
647            next_pubsub_interest_seq: AtomicU64::new(1),
648            pending_response_sends: Mutex::new(Vec::new()),
649            response_scheduler_running: AtomicBool::new(false),
650            next_response_job_id: AtomicU64::new(1),
651            routing,
652            request_timeout,
653            debug,
654            running: RwLock::new(false),
655        }
656    }
657
658    /// Start the store (begin listening for messages)
659    pub async fn start(&self) -> Result<(), TransportError> {
660        *self.running.write().await = true;
661
662        // Send initial hello
663        self.signaling.send_hello(vec![]).await?;
664
665        Ok(())
666    }
667
668    /// Stop the store
669    pub async fn stop(&self) {
670        *self.running.write().await = false;
671    }
672
673    /// Process incoming signaling message
674    pub async fn process_signaling(&self, msg: SignalingMessage) -> Result<(), TransportError> {
675        // When a new peer connects, initialize their HTL config
676        let peer_id = msg.peer_id().to_string();
677        {
678            let mut configs = self.htl_configs.write().await;
679            if !configs.contains_key(&peer_id) {
680                configs.insert(peer_id.clone(), PeerHTLConfig::random());
681            }
682        }
683        self.peer_selector.write().await.add_peer(peer_id.clone());
684
685        let result = self.signaling.handle_message(msg).await;
686        if result.is_ok() {
687            self.announce_pubsub_interests_to_peer(&peer_id).await;
688        }
689        result
690    }
691
692    /// Get signaling manager reference
693    pub fn signaling(&self) -> &Arc<MeshRouter<R, F>> {
694        &self.signaling
695    }
696
697    fn response_behavior(&self) -> ResponseBehaviorConfig {
698        self.routing.response_behavior.normalized()
699    }
700
701    async fn record_peer_wire_sent(&self, peer_id: &str, bytes: u64) {
702        if bytes == 0 {
703            return;
704        }
705        let mut stats = self.peer_wire_stats.write().await;
706        let entry = stats.entry(peer_id.to_string()).or_default();
707        entry.bytes_sent = entry.bytes_sent.saturating_add(bytes);
708    }
709
710    async fn record_peer_wire_received(&self, peer_id: &str, bytes: u64) {
711        if bytes == 0 {
712            return;
713        }
714        let mut stats = self.peer_wire_stats.write().await;
715        let entry = stats.entry(peer_id.to_string()).or_default();
716        entry.bytes_received = entry.bytes_received.saturating_add(bytes);
717    }
718
719    /// Record ingress from a peer that matched local or downstream demand.
720    ///
721    /// Raw bytes are tracked separately in `record_peer_wire_received`; this
722    /// counter is the reciprocity signal used by shared outbound scheduling.
723    pub async fn record_useful_bytes_received_from_peer(&self, peer_id: &str, bytes: u64) {
724        if bytes == 0 {
725            return;
726        }
727        let mut stats = self.peer_wire_stats.write().await;
728        let entry = stats.entry(peer_id.to_string()).or_default();
729        entry.useful_bytes_received = entry.useful_bytes_received.saturating_add(bytes);
730    }
731
732    /// Snapshot peer traffic for production pubsub scheduling or diagnostics.
733    pub async fn peer_traffic_snapshot(&self, peer_id: &str) -> PeerTrafficSnapshot {
734        self.peer_wire_stats
735            .read()
736            .await
737            .get(peer_id)
738            .copied()
739            .unwrap_or_default()
740    }
741
742    /// Snapshot all known peer traffic for production pubsub scheduling.
743    pub async fn peer_traffic_snapshots(&self) -> HashMap<String, PeerTrafficSnapshot> {
744        self.peer_wire_stats.read().await.clone()
745    }
746
747    fn pubsub_key(origin_peer_id: &str, stream_id: &str, seq: u64) -> String {
748        format!("{origin_peer_id}:{stream_id}:{seq}")
749    }
750
751    fn pubsub_frame_key(frame: &PubsubFrame) -> String {
752        Self::pubsub_key(&frame.origin_peer_id, &frame.stream_id, frame.seq)
753    }
754
755    fn pubsub_interest_key(interest: &PubsubInterest) -> String {
756        format!(
757            "{}:{}:{}:{}",
758            interest.subscriber_peer_id, interest.stream_id, interest.seq, interest.active
759        )
760    }
761
762    fn next_pubsub_interest_seq(&self) -> u64 {
763        self.next_pubsub_interest_seq
764            .fetch_add(1, Ordering::Relaxed)
765    }
766
767    async fn record_peer_pubsub_wire_sent(&self, peer_id: &str, bytes: u64, bandwidth_debt: f64) {
768        if bytes == 0 {
769            return;
770        }
771        let mut stats = self.peer_wire_stats.write().await;
772        let entry = stats.entry(peer_id.to_string()).or_default();
773        entry.bytes_sent = entry.bytes_sent.saturating_add(bytes);
774        entry.bandwidth_debt = bandwidth_debt;
775    }
776
777    async fn send_pubsub_interest_to_peers(
778        &self,
779        interest: &PubsubInterest,
780        exclude_peer_id: Option<&str>,
781    ) -> PubsubPublishStats {
782        if !should_forward_htl(interest.htl) {
783            return PubsubPublishStats::default();
784        }
785
786        let mut peer_ids = self.signaling.peer_ids().await;
787        peer_ids.sort();
788        peer_ids.retain(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude));
789
790        let bytes = encode_pubsub_interest(interest);
791        let mut stats = PubsubPublishStats {
792            selected_peers: peer_ids.len(),
793            ..Default::default()
794        };
795        for peer_id in peer_ids {
796            let Some(channel) = self.signaling.get_channel(&peer_id).await else {
797                continue;
798            };
799            if channel.send(bytes.clone()).await.is_ok() {
800                stats.sent_peers += 1;
801                stats.sent_bytes = stats.sent_bytes.saturating_add(bytes.len() as u64);
802                self.record_peer_wire_sent(&peer_id, bytes.len() as u64)
803                    .await;
804            }
805        }
806        stats
807    }
808
809    async fn announce_pubsub_interests_to_peer(&self, peer_id: &str) {
810        let mut interests = self
811            .pubsub_local_interests
812            .read()
813            .await
814            .iter()
815            .cloned()
816            .collect::<Vec<_>>();
817        interests.sort();
818        if interests.is_empty() {
819            return;
820        }
821
822        let interests = {
823            let versions = self.pubsub_local_interest_versions.read().await;
824            interests
825                .into_iter()
826                .filter_map(|stream_id| {
827                    versions
828                        .get(&stream_id)
829                        .copied()
830                        .map(|seq| (stream_id, seq))
831                })
832                .collect::<Vec<_>>()
833        };
834
835        for (stream_id, seq) in interests {
836            let interest = create_pubsub_interest(
837                stream_id,
838                self.signaling.peer_id().to_string(),
839                seq,
840                true,
841                MAX_HTL,
842            );
843            let Some(channel) = self.signaling.get_channel(peer_id).await else {
844                continue;
845            };
846            let bytes = encode_pubsub_interest(&interest);
847            if channel.send(bytes.clone()).await.is_ok() {
848                self.record_peer_wire_sent(peer_id, bytes.len() as u64)
849                    .await;
850            }
851        }
852    }
853
854    fn remove_pubsub_peer_interest(
855        peer_interests: &mut HashMap<String, HashSet<String>>,
856        routes: &HashMap<(String, String), String>,
857        stream_id: &str,
858        peer_id: &str,
859    ) {
860        let still_has_route = routes
861            .iter()
862            .any(|((stream, _subscriber), peer)| stream == stream_id && peer == peer_id);
863        if still_has_route {
864            return;
865        }
866        if let Some(peers) = peer_interests.get_mut(stream_id) {
867            peers.remove(peer_id);
868            if peers.is_empty() {
869                peer_interests.remove(stream_id);
870            }
871        }
872    }
873
874    async fn apply_pubsub_interest_route(
875        &self,
876        from_peer: &str,
877        interest: &PubsubInterest,
878    ) -> bool {
879        if interest.stream_id.is_empty() || interest.subscriber_peer_id.is_empty() {
880            return false;
881        }
882        if interest.subscriber_peer_id == self.signaling.peer_id() {
883            return false;
884        }
885
886        let interest_key = Self::pubsub_interest_key(interest);
887        if !self
888            .pubsub_seen_interests
889            .lock()
890            .await
891            .insert_if_new(interest_key)
892        {
893            return false;
894        }
895
896        let route_key = (
897            interest.stream_id.clone(),
898            interest.subscriber_peer_id.clone(),
899        );
900        {
901            let mut versions = self.pubsub_interest_versions.write().await;
902            if versions
903                .get(&route_key)
904                .is_some_and(|latest| *latest >= interest.seq)
905            {
906                return false;
907            }
908            versions.insert(route_key.clone(), interest.seq);
909        }
910
911        let mut peer_interests = self.pubsub_peer_interests.write().await;
912        let mut routes = self.pubsub_interest_routes.write().await;
913        if interest.active {
914            if let Some(previous_peer) = routes.insert(route_key, from_peer.to_string()) {
915                if previous_peer != from_peer {
916                    Self::remove_pubsub_peer_interest(
917                        &mut peer_interests,
918                        &routes,
919                        &interest.stream_id,
920                        &previous_peer,
921                    );
922                }
923            }
924            peer_interests
925                .entry(interest.stream_id.clone())
926                .or_default()
927                .insert(from_peer.to_string());
928        } else if let Some(previous_peer) = routes.remove(&route_key) {
929            Self::remove_pubsub_peer_interest(
930                &mut peer_interests,
931                &routes,
932                &interest.stream_id,
933                &previous_peer,
934            );
935        } else {
936            Self::remove_pubsub_peer_interest(
937                &mut peer_interests,
938                &routes,
939                &interest.stream_id,
940                from_peer,
941            );
942        }
943
944        true
945    }
946
947    async fn interested_pubsub_peers(
948        &self,
949        stream_id: &str,
950        exclude_peer_id: Option<&str>,
951    ) -> Vec<String> {
952        let connected = self
953            .signaling
954            .peer_ids()
955            .await
956            .into_iter()
957            .collect::<HashSet<_>>();
958        let mut peers = self
959            .pubsub_peer_interests
960            .read()
961            .await
962            .get(stream_id)
963            .map(|peers| peers.iter().cloned().collect::<Vec<_>>())
964            .unwrap_or_default();
965        peers.retain(|peer_id| {
966            connected.contains(peer_id) && exclude_peer_id.is_none_or(|exclude| peer_id != exclude)
967        });
968        peers.sort();
969        peers
970    }
971
972    async fn decrement_pubsub_htl_for_peer(&self, peer_id: &str, htl: u8) -> u8 {
973        let htl_config = {
974            let configs = self.htl_configs.read().await;
975            configs
976                .get(peer_id)
977                .cloned()
978                .unwrap_or_else(PeerHTLConfig::random)
979        };
980        htl_config.decrement_with_policy(htl, &MESH_EVENT_POLICY)
981    }
982
983    async fn send_pubsub_inventory_to_peers(
984        &self,
985        inv: &PubsubInventory,
986        peer_ids: &[String],
987        consume_forwarding_hop: bool,
988    ) -> PubsubPublishStats {
989        if peer_ids.is_empty() || !should_forward_htl(inv.htl) {
990            return PubsubPublishStats::default();
991        }
992
993        let mut stats = PubsubPublishStats {
994            selected_peers: peer_ids.len(),
995            ..Default::default()
996        };
997        for peer_id in peer_ids {
998            let send_htl = if consume_forwarding_hop {
999                self.decrement_pubsub_htl_for_peer(peer_id, inv.htl).await
1000            } else {
1001                inv.htl
1002            };
1003            if !should_forward_htl(send_htl) {
1004                continue;
1005            }
1006            let Some(channel) = self.signaling.get_channel(peer_id).await else {
1007                continue;
1008            };
1009            let mut outgoing = inv.clone();
1010            outgoing.htl = send_htl;
1011            let bytes = encode_pubsub_inventory(&outgoing);
1012            let message_bytes = bytes.len() as u64;
1013            if channel.send(bytes).await.is_ok() {
1014                stats.sent_peers += 1;
1015                stats.sent_bytes = stats.sent_bytes.saturating_add(message_bytes);
1016                self.record_peer_wire_sent(peer_id, message_bytes).await;
1017            }
1018        }
1019        stats
1020    }
1021
1022    async fn send_pubsub_want_to_peer(&self, want: &PubsubWant, peer_id: &str) -> bool {
1023        let Some(channel) = self.signaling.get_channel(peer_id).await else {
1024            return false;
1025        };
1026        let bytes = encode_pubsub_want(want);
1027        let message_bytes = bytes.len() as u64;
1028        match channel.send(bytes).await {
1029            Ok(()) => {
1030                self.record_peer_wire_sent(peer_id, message_bytes).await;
1031                true
1032            }
1033            Err(_) => false,
1034        }
1035    }
1036
1037    async fn send_pubsub_want_upstream(
1038        &self,
1039        key: &str,
1040        want: &PubsubWant,
1041        exclude_peer_id: Option<&str>,
1042    ) -> bool {
1043        let upstream = {
1044            let routes = self.pubsub_inventory_routes.read().await;
1045            routes.get(key).cloned()
1046        };
1047        let Some(upstream) = upstream else {
1048            return false;
1049        };
1050        if exclude_peer_id.is_some_and(|exclude| exclude == upstream) {
1051            return false;
1052        }
1053        let want_key = format!("{key}:{upstream}");
1054        if !self
1055            .pubsub_upstream_wants
1056            .lock()
1057            .await
1058            .insert_if_new(want_key)
1059        {
1060            return false;
1061        }
1062        self.send_pubsub_want_to_peer(want, &upstream).await
1063    }
1064
1065    async fn cache_pubsub_frame(&self, key: String, frame: PubsubFrame) {
1066        let mut cache = self.pubsub_frame_cache.lock().await;
1067        if let Some(index) = cache.iter().position(|(cached_key, _)| cached_key == &key) {
1068            cache.remove(index);
1069        }
1070        cache.push_back((key, frame));
1071        while cache.len() > PUBSUB_FRAME_CACHE_CAPACITY {
1072            cache.pop_front();
1073        }
1074    }
1075
1076    async fn cached_pubsub_frame(&self, key: &str) -> Option<PubsubFrame> {
1077        self.pubsub_frame_cache
1078            .lock()
1079            .await
1080            .iter()
1081            .find_map(|(cached_key, frame)| {
1082                if cached_key == key {
1083                    Some(frame.clone())
1084                } else {
1085                    None
1086                }
1087            })
1088    }
1089
1090    async fn remember_pubsub_want_peer(&self, key: String, from_peer: &str) -> bool {
1091        let mut routes = self.pubsub_want_routes.write().await;
1092        routes.entry(key).or_default().insert(from_peer.to_string())
1093    }
1094
1095    async fn take_pubsub_want_peers(
1096        &self,
1097        key: &str,
1098        exclude_peer_id: Option<&str>,
1099    ) -> Vec<String> {
1100        let connected = self
1101            .signaling
1102            .peer_ids()
1103            .await
1104            .into_iter()
1105            .collect::<HashSet<_>>();
1106        let mut peers = self
1107            .pubsub_want_routes
1108            .write()
1109            .await
1110            .remove(key)
1111            .map(|peers| peers.into_iter().collect::<Vec<_>>())
1112            .unwrap_or_default();
1113        peers.retain(|peer_id| {
1114            connected.contains(peer_id) && exclude_peer_id.is_none_or(|exclude| peer_id != exclude)
1115        });
1116        peers.sort();
1117        peers
1118    }
1119
1120    async fn select_pubsub_peers(
1121        &self,
1122        stream_id: &str,
1123        seq: u64,
1124        message_bytes: u64,
1125        peer_ids: &[String],
1126    ) -> (Vec<String>, Vec<String>) {
1127        let traffic = self.peer_wire_stats.read().await;
1128        let deferred_counts = self.pubsub_deferred_counts.read().await;
1129        let candidates = peer_ids
1130            .iter()
1131            .map(|peer_id| PubsubCandidate {
1132                peer_id: peer_id.clone(),
1133                traffic: traffic.get(peer_id).copied().unwrap_or_default(),
1134                deferred_count: deferred_counts
1135                    .get(&(stream_id.to_string(), peer_id.clone()))
1136                    .copied()
1137                    .unwrap_or_default(),
1138            })
1139            .collect::<Vec<_>>();
1140        drop(deferred_counts);
1141        drop(traffic);
1142
1143        let selection = self.routing.pubsub_scheduler.select(
1144            stream_id,
1145            seq,
1146            self.signaling.peer_id(),
1147            message_bytes,
1148            &candidates,
1149        );
1150
1151        {
1152            let mut deferred_counts = self.pubsub_deferred_counts.write().await;
1153            for peer_id in &selection.deferred {
1154                *deferred_counts
1155                    .entry((stream_id.to_string(), peer_id.clone()))
1156                    .or_insert(0) += 1;
1157            }
1158            for peer_id in &selection.selected {
1159                deferred_counts.remove(&(stream_id.to_string(), peer_id.clone()));
1160            }
1161        }
1162
1163        (selection.selected, selection.deferred)
1164    }
1165
1166    async fn send_pubsub_frame_to_peers(
1167        &self,
1168        frame: &PubsubFrame,
1169        peer_ids: &[String],
1170    ) -> PubsubPublishStats {
1171        if peer_ids.is_empty() || !should_forward_htl(frame.htl) {
1172            return PubsubPublishStats::default();
1173        }
1174
1175        let bytes = encode_pubsub_frame(frame);
1176        let message_bytes = bytes.len() as u64;
1177        let (selected, deferred) = self
1178            .select_pubsub_peers(&frame.stream_id, frame.seq, message_bytes, peer_ids)
1179            .await;
1180        let mut stats = PubsubPublishStats {
1181            selected_peers: selected.len(),
1182            deferred_peers: deferred.len(),
1183            ..Default::default()
1184        };
1185
1186        for peer_id in selected {
1187            let Some(channel) = self.signaling.get_channel(&peer_id).await else {
1188                continue;
1189            };
1190            let snapshot = self.peer_traffic_snapshot(&peer_id).await;
1191            let bandwidth_debt = reciprocal_virtual_finish(snapshot, message_bytes);
1192            if channel.send(bytes.clone()).await.is_ok() {
1193                stats.sent_peers += 1;
1194                stats.sent_bytes = stats.sent_bytes.saturating_add(message_bytes);
1195                self.record_peer_pubsub_wire_sent(&peer_id, message_bytes, bandwidth_debt)
1196                    .await;
1197            }
1198        }
1199
1200        stats
1201    }
1202
1203    async fn enqueue_pubsub_event(&self, event: PubsubEvent) {
1204        let mut inbox = self.pubsub_inbox.lock().await;
1205        inbox.push_back(event);
1206        while inbox.len() > PUBSUB_INBOX_CAPACITY {
1207            inbox.pop_front();
1208        }
1209        self.pubsub_notify.notify_one();
1210    }
1211
1212    /// Subscribe this node to a pubsub stream and advertise that interest.
1213    pub async fn subscribe_pubsub(
1214        self: &Arc<Self>,
1215        stream_id: impl Into<String>,
1216    ) -> PubsubPublishStats {
1217        let stream_id = stream_id.into();
1218        if stream_id.is_empty() {
1219            return PubsubPublishStats::default();
1220        }
1221        self.pubsub_local_interests
1222            .write()
1223            .await
1224            .insert(stream_id.clone());
1225        let seq = {
1226            let mut versions = self.pubsub_local_interest_versions.write().await;
1227            match versions.get(&stream_id).copied() {
1228                Some(seq) => seq,
1229                None => {
1230                    let seq = self.next_pubsub_interest_seq();
1231                    versions.insert(stream_id.clone(), seq);
1232                    seq
1233                }
1234            }
1235        };
1236        let interest = create_pubsub_interest(
1237            stream_id,
1238            self.signaling.peer_id().to_string(),
1239            seq,
1240            true,
1241            self.routing.pubsub_initial_htl(),
1242        );
1243        self.send_pubsub_interest_to_peers(&interest, None).await
1244    }
1245
1246    /// Stop local delivery for a pubsub stream and advertise the withdrawn interest.
1247    pub async fn unsubscribe_pubsub(
1248        self: &Arc<Self>,
1249        stream_id: impl Into<String>,
1250    ) -> PubsubPublishStats {
1251        let stream_id = stream_id.into();
1252        if stream_id.is_empty() {
1253            return PubsubPublishStats::default();
1254        }
1255        self.pubsub_local_interests.write().await.remove(&stream_id);
1256        self.pubsub_local_interest_versions
1257            .write()
1258            .await
1259            .remove(&stream_id);
1260        let interest = create_pubsub_interest(
1261            stream_id,
1262            self.signaling.peer_id().to_string(),
1263            self.next_pubsub_interest_seq(),
1264            false,
1265            self.routing.pubsub_initial_htl(),
1266        );
1267        self.send_pubsub_interest_to_peers(&interest, None).await
1268    }
1269
1270    /// Publish bytes on a pubsub stream through the configured mesh delivery mode.
1271    pub async fn publish_pubsub(
1272        self: &Arc<Self>,
1273        stream_id: impl Into<String>,
1274        seq: u64,
1275        payload: Vec<u8>,
1276    ) -> PubsubPublishStats {
1277        let stream_id = stream_id.into();
1278        if stream_id.is_empty() {
1279            return PubsubPublishStats::default();
1280        }
1281        let payload_bytes = payload.len() as u64;
1282        let frame = create_pubsub_frame(
1283            stream_id.clone(),
1284            seq,
1285            self.signaling.peer_id().to_string(),
1286            payload.clone(),
1287            self.routing.pubsub_initial_htl(),
1288        );
1289        let frame_key = Self::pubsub_frame_key(&frame);
1290        self.pubsub_seen_frames
1291            .lock()
1292            .await
1293            .insert_if_new(frame_key.clone());
1294        self.cache_pubsub_frame(frame_key, frame.clone()).await;
1295
1296        if self
1297            .pubsub_local_interests
1298            .read()
1299            .await
1300            .contains(&stream_id)
1301        {
1302            self.enqueue_pubsub_event(PubsubEvent {
1303                stream_id: stream_id.clone(),
1304                seq,
1305                origin_peer_id: self.signaling.peer_id().to_string(),
1306                from_peer_id: self.signaling.peer_id().to_string(),
1307                payload,
1308            })
1309            .await;
1310        }
1311
1312        match self.routing.pubsub_delivery_mode {
1313            PubsubDeliveryMode::InterestPush => {
1314                let peers = self.interested_pubsub_peers(&stream_id, None).await;
1315                self.send_pubsub_frame_to_peers(&frame, &peers).await
1316            }
1317            PubsubDeliveryMode::HtlInvWant => {
1318                let inv = create_pubsub_inventory(
1319                    stream_id,
1320                    seq,
1321                    self.signaling.peer_id().to_string(),
1322                    payload_bytes,
1323                    self.routing.pubsub_initial_htl(),
1324                );
1325                let peers = self.interested_pubsub_peers(&inv.stream_id, None).await;
1326                self.send_pubsub_inventory_to_peers(&inv, &peers, false)
1327                    .await
1328            }
1329        }
1330    }
1331
1332    /// Drain locally delivered pubsub events.
1333    pub async fn drain_pubsub_events(&self) -> Vec<PubsubEvent> {
1334        self.pubsub_inbox.lock().await.drain(..).collect()
1335    }
1336
1337    /// Drain verified first-winner block deliveries for an application adapter.
1338    pub async fn drain_verified_block_deliveries(&self) -> VerifiedBlockDeliveryBatch {
1339        let mut buffer = self.verified_block_deliveries.lock().await;
1340        VerifiedBlockDeliveryBatch {
1341            deliveries: buffer.deliveries.drain(..).collect(),
1342            dropped_since_last_drain: std::mem::take(&mut buffer.dropped_since_last_drain),
1343        }
1344    }
1345
1346    /// Wait until a locally delivered pubsub event is available, then return it.
1347    pub async fn recv_pubsub_event(&self) -> PubsubEvent {
1348        loop {
1349            if let Some(event) = self.pubsub_inbox.lock().await.pop_front() {
1350                return event;
1351            }
1352            self.pubsub_notify.notified().await;
1353        }
1354    }
1355
1356    /// Connected peers that currently have local or downstream interest in a stream.
1357    pub async fn pubsub_interest_peers(&self, stream_id: &str) -> Vec<String> {
1358        self.interested_pubsub_peers(stream_id, None).await
1359    }
1360
1361    fn choose_ready_response_job(
1362        ready_jobs: &[(u64, String, usize, Instant, u64)],
1363        stats: &HashMap<String, PeerWireStats>,
1364    ) -> Option<(u64, f64)> {
1365        let jobs = ready_jobs
1366            .iter()
1367            .map(|job| OutboundJobCandidate {
1368                job_id: job.0,
1369                peer_id: job.1.clone(),
1370                message_bytes: job.2 as u64,
1371                queue_sequence: job.4,
1372            })
1373            .collect::<Vec<_>>();
1374        select_reciprocal_outbound_job(&jobs, |peer_id| {
1375            stats.get(peer_id).copied().unwrap_or_default()
1376        })
1377        .map(|choice| (choice.job_id, choice.virtual_finish))
1378    }
1379
1380    async fn enqueue_response_send(
1381        self: &Arc<Self>,
1382        peer_id: String,
1383        bytes: Vec<u8>,
1384        ready_at: Instant,
1385    ) {
1386        let job_id = self.next_response_job_id.fetch_add(1, Ordering::Relaxed);
1387        {
1388            let mut queue = self.pending_response_sends.lock().await;
1389            queue.push(PendingResponseSend {
1390                job_id,
1391                peer_id,
1392                bytes,
1393                ready_at,
1394                queue_sequence: job_id,
1395            });
1396        }
1397
1398        if self
1399            .response_scheduler_running
1400            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
1401            .is_ok()
1402        {
1403            let this = Arc::clone(self);
1404            tokio::spawn(async move {
1405                this.run_response_scheduler().await;
1406            });
1407        }
1408    }
1409
1410    async fn run_response_scheduler(self: Arc<Self>) {
1411        loop {
1412            let snapshot = {
1413                let queue = self.pending_response_sends.lock().await;
1414                if queue.is_empty() {
1415                    self.response_scheduler_running
1416                        .store(false, Ordering::Release);
1417                    return;
1418                }
1419                queue
1420                    .iter()
1421                    .map(|job| {
1422                        (
1423                            job.job_id,
1424                            job.peer_id.clone(),
1425                            job.bytes.len(),
1426                            job.ready_at,
1427                            job.queue_sequence,
1428                        )
1429                    })
1430                    .collect::<Vec<_>>()
1431            };
1432
1433            let now = Instant::now();
1434            let mut earliest_ready_at: Option<Instant> = None;
1435            let mut ready_jobs = Vec::new();
1436            for job in &snapshot {
1437                if job.3 <= now {
1438                    ready_jobs.push(job.clone());
1439                } else {
1440                    earliest_ready_at = Some(match earliest_ready_at {
1441                        Some(current) => current.min(job.3),
1442                        None => job.3,
1443                    });
1444                }
1445            }
1446
1447            if ready_jobs.is_empty() {
1448                if let Some(ready_at) = earliest_ready_at {
1449                    tokio::time::sleep(ready_at.saturating_duration_since(Instant::now())).await;
1450                    continue;
1451                }
1452                self.response_scheduler_running
1453                    .store(false, Ordering::Release);
1454                return;
1455            }
1456
1457            let (selected_job_id, selected_finish) = {
1458                let stats = self.peer_wire_stats.read().await;
1459                Self::choose_ready_response_job(&ready_jobs, &stats).expect("ready response job")
1460            };
1461
1462            let selected = {
1463                let mut queue = self.pending_response_sends.lock().await;
1464                let Some(index) = queue.iter().position(|job| job.job_id == selected_job_id) else {
1465                    continue;
1466                };
1467                queue.swap_remove(index)
1468            };
1469
1470            let sent = if let Some(channel) = self.signaling.get_channel(&selected.peer_id).await {
1471                channel.send(selected.bytes.clone()).await.is_ok()
1472            } else {
1473                false
1474            };
1475
1476            let queued_peers = {
1477                let queue = self.pending_response_sends.lock().await;
1478                queue
1479                    .iter()
1480                    .map(|job| job.peer_id.clone())
1481                    .collect::<HashSet<_>>()
1482            };
1483            let mut stats = self.peer_wire_stats.write().await;
1484            let entry = stats.entry(selected.peer_id.clone()).or_default();
1485            if sent {
1486                entry.bytes_sent = entry.bytes_sent.saturating_add(selected.bytes.len() as u64);
1487                entry.bandwidth_debt = selected_finish;
1488            }
1489            if queued_peers.is_empty() {
1490                for peer_stats in stats.values_mut() {
1491                    peer_stats.bandwidth_debt = 0.0;
1492                }
1493            } else {
1494                let floor = queued_peers
1495                    .iter()
1496                    .filter_map(|peer_id| stats.get(peer_id).map(|peer| peer.bandwidth_debt))
1497                    .fold(f64::INFINITY, f64::min);
1498                if floor.is_finite() && floor > 0.0 {
1499                    for peer_id in queued_peers {
1500                        if let Some(peer_stats) = stats.get_mut(&peer_id) {
1501                            peer_stats.bandwidth_debt =
1502                                (peer_stats.bandwidth_debt - floor).max(0.0);
1503                        }
1504                    }
1505                }
1506            }
1507        }
1508    }
1509
1510    fn deterministic_actor_draw_for(peer_id: &str, hash: &Hash, salt: u64) -> f64 {
1511        let mut hasher = DefaultHasher::new();
1512        peer_id.hash(&mut hasher);
1513        hash.hash(&mut hasher);
1514        salt.hash(&mut hasher);
1515        let v = hasher.finish();
1516        (v as f64) / (u64::MAX as f64)
1517    }
1518
1519    fn deterministic_actor_draw(&self, hash: &Hash, salt: u64) -> f64 {
1520        Self::deterministic_actor_draw_for(self.signaling.peer_id(), hash, salt)
1521    }
1522
1523    fn peer_metadata_pointer_slot_hash() -> Hash {
1524        hashtree_core::sha256(PEER_METADATA_POINTER_SLOT_KEY)
1525    }
1526
1527    fn decode_hash_hex(hash_hex: &str) -> Result<Hash, StoreError> {
1528        let bytes = hex::decode(hash_hex)
1529            .map_err(|e| StoreError::Other(format!("Invalid hash hex: {e}")))?;
1530        if bytes.len() != 32 {
1531            return Err(StoreError::Other(format!(
1532                "Invalid hash length {}, expected 32 bytes",
1533                bytes.len()
1534            )));
1535        }
1536        let mut hash = [0u8; 32];
1537        hash.copy_from_slice(&bytes);
1538        Ok(hash)
1539    }
1540
1541    fn should_drop_response(&self, hash: &Hash) -> bool {
1542        let p = self.response_behavior().drop_response_prob;
1543        if p <= 0.0 {
1544            return false;
1545        }
1546        self.deterministic_actor_draw(hash, 0xD0_D0_D0_D0_D0_D0_D0_D0) < p
1547    }
1548
1549    fn should_corrupt_response(&self, hash: &Hash) -> bool {
1550        let p = self.response_behavior().corrupt_response_prob;
1551        if p <= 0.0 {
1552            return false;
1553        }
1554        self.deterministic_actor_draw(hash, 0xC0_C0_C0_C0_C0_C0_C0_C0) < p
1555    }
1556
1557    fn should_stall_response(&self, hash: &Hash) -> bool {
1558        let p = self.response_behavior().stall_response_prob;
1559        if p <= 0.0 {
1560            return false;
1561        }
1562        self.deterministic_actor_draw(hash, 0x5A_11_5A_11_5A_11_5A_11) < p
1563    }
1564
1565    fn response_send_delay(&self, hash: &Hash, payload_len: usize) -> Duration {
1566        let behavior = self.response_behavior();
1567        let mut total_ms = behavior
1568            .extra_delay_ms
1569            .saturating_add(behavior.first_byte_delay_ms);
1570
1571        if behavior.bytes_per_second > 0 && payload_len > 0 {
1572            let throughput_ms = ((payload_len as u128) * 1000)
1573                .div_ceil(behavior.bytes_per_second as u128)
1574                .min(u64::MAX as u128) as u64;
1575            total_ms = total_ms.saturating_add(throughput_ms);
1576        }
1577
1578        if behavior.stall_delay_ms > 0 && self.should_stall_response(hash) {
1579            total_ms = total_ms.saturating_add(behavior.stall_delay_ms);
1580        }
1581
1582        Duration::from_millis(total_ms)
1583    }
1584
1585    async fn ordered_connected_peers(&self, exclude_peer_id: Option<&str>) -> Vec<String> {
1586        let current_peer_ids = self.signaling.peer_ids().await;
1587        if current_peer_ids.is_empty() {
1588            return Vec::new();
1589        }
1590
1591        sync_selector_peers(&self.peer_selector, &current_peer_ids).await;
1592        let hash_get_peer_ids: HashSet<String> = self
1593            .signaling
1594            .hash_get_peer_ids()
1595            .await
1596            .into_iter()
1597            .collect();
1598        let mut candidate_peer_ids: Vec<String> = current_peer_ids
1599            .into_iter()
1600            .filter(|peer_id| hash_get_peer_ids.contains(peer_id))
1601            .filter(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude))
1602            .collect();
1603        if candidate_peer_ids.is_empty() {
1604            return Vec::new();
1605        }
1606
1607        let current_set: HashSet<&str> = candidate_peer_ids.iter().map(String::as_str).collect();
1608        let mut selector = self.peer_selector.write().await;
1609        let mut selector_order = selector.select_peers();
1610        selector_order.retain(|peer_id| current_set.contains(peer_id.as_str()));
1611        if selector_order.is_empty() {
1612            let mut fallback = candidate_peer_ids;
1613            fallback.sort();
1614            return fallback;
1615        }
1616        let backed_off: HashMap<String, bool> = candidate_peer_ids
1617            .iter()
1618            .map(|peer_id| (peer_id.clone(), selector.is_peer_backed_off(peer_id)))
1619            .collect();
1620        drop(selector);
1621
1622        let rank: HashMap<&str, usize> = selector_order
1623            .iter()
1624            .enumerate()
1625            .map(|(idx, peer_id)| (peer_id.as_str(), idx))
1626            .collect();
1627        let active = self.peer_active_requests.read().await;
1628        candidate_peer_ids.sort_by(|left, right| {
1629            let left_backed_off = backed_off.get(left).copied().unwrap_or(false);
1630            let right_backed_off = backed_off.get(right).copied().unwrap_or(false);
1631            if left_backed_off != right_backed_off {
1632                return if left_backed_off {
1633                    std::cmp::Ordering::Greater
1634                } else {
1635                    std::cmp::Ordering::Less
1636                };
1637            }
1638            let left_rank = rank.get(left.as_str()).copied().unwrap_or(usize::MAX / 2);
1639            let right_rank = rank.get(right.as_str()).copied().unwrap_or(usize::MAX / 2);
1640            let left_load = active.get(left).copied().unwrap_or(0);
1641            let right_load = active.get(right).copied().unwrap_or(0);
1642            (left_rank + left_load.saturating_mul(ACTIVE_PEER_REQUEST_RANK_PENALTY))
1643                .cmp(&(right_rank + right_load.saturating_mul(ACTIVE_PEER_REQUEST_RANK_PENALTY)))
1644                .then_with(|| left.cmp(right))
1645        });
1646        candidate_peer_ids
1647    }
1648
1649    async fn reserve_peer_request(&self, peer_id: &str) {
1650        let mut active = self.peer_active_requests.write().await;
1651        *active.entry(peer_id.to_string()).or_insert(0) += 1;
1652    }
1653
1654    async fn release_peer_request(&self, peer_id: &str) {
1655        let mut active = self.peer_active_requests.write().await;
1656        let Some(count) = active.get_mut(peer_id) else {
1657            return;
1658        };
1659        if *count <= 1 {
1660            active.remove(peer_id);
1661        } else {
1662            *count -= 1;
1663        }
1664    }
1665
1666    async fn release_queried_peer_requests(&self, peer_ids: &[String]) {
1667        for peer_id in peer_ids {
1668            self.release_peer_request(peer_id).await;
1669        }
1670    }
1671
1672    fn requested_quote_mint(&self) -> Option<&str> {
1673        if let Some(default_mint) = self.routing.cashu_default_mint.as_deref() {
1674            if self.routing.cashu_accepted_mints.is_empty()
1675                || self
1676                    .routing
1677                    .cashu_accepted_mints
1678                    .iter()
1679                    .any(|mint| mint == default_mint)
1680            {
1681                return Some(default_mint);
1682            }
1683        }
1684
1685        self.routing
1686            .cashu_accepted_mints
1687            .first()
1688            .map(String::as_str)
1689    }
1690
1691    fn choose_quote_mint(&self, requested_mint: Option<&str>) -> Option<String> {
1692        if let Some(requested_mint) = requested_mint {
1693            if self.accepts_quote_mint(Some(requested_mint)) {
1694                return Some(requested_mint.to_string());
1695            }
1696        }
1697        if let Some(default_mint) = self.routing.cashu_default_mint.as_ref() {
1698            return Some(default_mint.clone());
1699        }
1700        if let Some(first_mint) = self.routing.cashu_accepted_mints.first() {
1701            return Some(first_mint.clone());
1702        }
1703        requested_mint.map(str::to_string)
1704    }
1705
1706    fn accepts_quote_mint(&self, mint_url: Option<&str>) -> bool {
1707        if self.routing.cashu_accepted_mints.is_empty() {
1708            return true;
1709        }
1710
1711        let Some(mint_url) = mint_url else {
1712            return false;
1713        };
1714        self.routing
1715            .cashu_accepted_mints
1716            .iter()
1717            .any(|mint| mint == mint_url)
1718    }
1719
1720    fn trusts_quote_mint(&self, mint_url: Option<&str>) -> bool {
1721        let Some(mint_url) = mint_url else {
1722            return self.routing.cashu_default_mint.is_none()
1723                && self.routing.cashu_accepted_mints.is_empty();
1724        };
1725        self.routing.cashu_default_mint.as_deref() == Some(mint_url)
1726            || self
1727                .routing
1728                .cashu_accepted_mints
1729                .iter()
1730                .any(|mint| mint == mint_url)
1731    }
1732
1733    async fn peer_suggested_mint_cap_sat(&self, peer_id: &str) -> u64 {
1734        let base = self.routing.cashu_peer_suggested_mint_base_cap_sat;
1735        if base == 0 {
1736            return 0;
1737        }
1738
1739        let selector = self.peer_selector.read().await;
1740        let Some(stats) = selector.get_stats(peer_id) else {
1741            let max_cap = self.routing.cashu_peer_suggested_mint_max_cap_sat;
1742            return if max_cap > 0 { base.min(max_cap) } else { base };
1743        };
1744
1745        if stats.cashu_payment_defaults > 0
1746            && stats.cashu_payment_defaults >= stats.cashu_payment_receipts
1747        {
1748            return 0;
1749        }
1750
1751        let success_bonus = stats
1752            .successes
1753            .saturating_mul(self.routing.cashu_peer_suggested_mint_success_step_sat);
1754        let receipt_bonus = stats
1755            .cashu_payment_receipts
1756            .saturating_mul(self.routing.cashu_peer_suggested_mint_receipt_step_sat);
1757        let mut cap = base
1758            .saturating_add(success_bonus)
1759            .saturating_add(receipt_bonus);
1760        let max_cap = self.routing.cashu_peer_suggested_mint_max_cap_sat;
1761        if max_cap > 0 {
1762            cap = cap.min(max_cap);
1763        }
1764        cap
1765    }
1766
1767    async fn should_accept_quote_response(
1768        &self,
1769        from_peer: &str,
1770        preferred_mint_url: Option<&str>,
1771        offered_payment_sat: u64,
1772        res: &DataQuoteResponse,
1773    ) -> bool {
1774        let Some(payment_sat) = res.p else {
1775            return false;
1776        };
1777        if payment_sat > offered_payment_sat {
1778            return false;
1779        }
1780
1781        let response_mint = res.m.as_deref();
1782        if response_mint == preferred_mint_url {
1783            return true;
1784        }
1785        if self.trusts_quote_mint(response_mint) {
1786            return true;
1787        }
1788        if response_mint.is_none() {
1789            return false;
1790        }
1791
1792        payment_sat <= self.peer_suggested_mint_cap_sat(from_peer).await
1793    }
1794
1795    async fn issue_quote(
1796        &self,
1797        peer_id: &str,
1798        hash_key: &str,
1799        payment_sat: u64,
1800        ttl_ms: u32,
1801        mint_url: Option<&str>,
1802    ) -> u64 {
1803        let quote_id = {
1804            let mut next = self.next_quote_id.write().await;
1805            let quote_id = *next;
1806            *next = next.saturating_add(1);
1807            quote_id
1808        };
1809
1810        let expires_at = Instant::now() + Duration::from_millis(ttl_ms as u64);
1811        self.issued_quotes.write().await.insert(
1812            (peer_id.to_string(), hash_key.to_string(), quote_id),
1813            IssuedQuote {
1814                expires_at,
1815                payment_sat,
1816                mint_url: mint_url.map(str::to_string),
1817            },
1818        );
1819        quote_id
1820    }
1821
1822    async fn take_valid_quote(&self, peer_id: &str, hash_key: &str, quote_id: u64) -> bool {
1823        let key = (peer_id.to_string(), hash_key.to_string(), quote_id);
1824        let Some(quote) = self.issued_quotes.write().await.remove(&key) else {
1825            return false;
1826        };
1827        quote.expires_at > Instant::now()
1828    }
1829
1830    async fn send_request_to_peer(
1831        &self,
1832        peer_id: &str,
1833        hash: &Hash,
1834        request_htl: u8,
1835        quote_id: Option<u64>,
1836    ) -> bool {
1837        if !should_forward_htl(request_htl) {
1838            return false;
1839        }
1840
1841        let channel = match self.signaling.get_channel(peer_id).await {
1842            Some(c) => c,
1843            None => return false,
1844        };
1845
1846        // Hashtree owns HTL and consumes exactly one unit when forwarding a
1847        // blob request to another mesh peer. Transport/routing hops below this
1848        // layer must not alter it.
1849        let send_htl = request_htl.saturating_sub(1);
1850        let req = match quote_id {
1851            Some(quote_id) => create_request_with_quote(hash, send_htl, quote_id),
1852            None => create_request(hash, send_htl),
1853        };
1854        let request_bytes = encode_request(&req);
1855        let request_len = request_bytes.len() as u64;
1856
1857        {
1858            let mut selector = self.peer_selector.write().await;
1859            selector.record_request(peer_id, request_len);
1860        }
1861
1862        match channel.send(request_bytes).await {
1863            Ok(()) => {
1864                self.record_peer_wire_sent(peer_id, request_len).await;
1865                true
1866            }
1867            Err(_) => {
1868                self.peer_selector.write().await.record_failure(peer_id);
1869                false
1870            }
1871        }
1872    }
1873
1874    async fn send_quote_request_to_peer(
1875        &self,
1876        peer_id: &str,
1877        hash: &Hash,
1878        payment_sat: u64,
1879        ttl_ms: u32,
1880        mint_url: Option<&str>,
1881    ) -> bool {
1882        let channel = match self.signaling.get_channel(peer_id).await {
1883            Some(c) => c,
1884            None => return false,
1885        };
1886
1887        let req = create_quote_request(hash, ttl_ms, payment_sat, mint_url);
1888        let request_bytes = encode_quote_request(&req);
1889        let request_len = request_bytes.len() as u64;
1890
1891        match channel.send(request_bytes).await {
1892            Ok(()) => {
1893                self.record_peer_wire_sent(peer_id, request_len).await;
1894                true
1895            }
1896            Err(_) => false,
1897        }
1898    }
1899
1900    /// Get peer count
1901    pub async fn peer_count(&self) -> usize {
1902        self.signaling.peer_count().await
1903    }
1904
1905    /// Get connected mesh peer IDs.
1906    pub async fn peer_ids(&self) -> Vec<String> {
1907        self.signaling.peer_ids().await
1908    }
1909
1910    /// Check if we need more peers
1911    pub async fn needs_peers(&self) -> bool {
1912        self.signaling.needs_peers().await
1913    }
1914
1915    /// Re-broadcast hello to refresh discovery as topology changes.
1916    pub async fn send_hello(&self) -> Result<(), TransportError> {
1917        self.signaling.send_hello(vec![]).await
1918    }
1919
1920    /// Drain all currently available peer-link messages and handle them.
1921    ///
1922    /// This keeps the message pump logic shared between simulation and the
1923    /// default production wrapper instead of duplicating per-channel loops.
1924    pub async fn drain_available_data_messages(self: &Arc<Self>) -> DataPumpStats {
1925        let mut stats = DataPumpStats::default();
1926        let peer_ids = self.signaling.peer_ids().await;
1927        for peer_id in peer_ids {
1928            let Some(channel) = self.signaling.get_channel(&peer_id).await else {
1929                continue;
1930            };
1931
1932            while let Some(data) = channel.try_recv() {
1933                stats.processed += 1;
1934                stats.processed_bytes += data.len() as u64;
1935                if let Some(msg) = parse_message(&data) {
1936                    match msg {
1937                        DataMessage::Request(_) => stats.request_messages += 1,
1938                        DataMessage::Response(_) => stats.response_messages += 1,
1939                        DataMessage::QuoteRequest(_) => stats.quote_request_messages += 1,
1940                        DataMessage::QuoteResponse(_) => stats.quote_response_messages += 1,
1941                        DataMessage::PubsubInterest(_) => stats.pubsub_interest_messages += 1,
1942                        DataMessage::PubsubFrame(_) => stats.pubsub_frame_messages += 1,
1943                        DataMessage::PubsubInventory(_) => stats.pubsub_inventory_messages += 1,
1944                        DataMessage::PubsubWant(_) => stats.pubsub_want_messages += 1,
1945                        DataMessage::Payment(_)
1946                        | DataMessage::PaymentAck(_)
1947                        | DataMessage::Chunk(_)
1948                        | DataMessage::PeerHints(_) => {}
1949                    }
1950                }
1951                self.handle_data_message(&peer_id, &data).await;
1952            }
1953        }
1954        stats
1955    }
1956
1957    /// Apply an out-of-band payment credit to a peer's routing priority.
1958    pub async fn record_cashu_payment_for_peer(&self, peer_id: &str, amount_sat: u64) {
1959        self.peer_selector
1960            .write()
1961            .await
1962            .record_cashu_payment(peer_id, amount_sat);
1963    }
1964
1965    /// Record a post-delivery payment we received from a peer.
1966    pub async fn record_cashu_receipt_from_peer(&self, peer_id: &str, amount_sat: u64) {
1967        self.peer_selector
1968            .write()
1969            .await
1970            .record_cashu_receipt(peer_id, amount_sat);
1971    }
1972
1973    /// Record that a peer failed to pay after we delivered successfully.
1974    pub async fn record_cashu_payment_default_from_peer(&self, peer_id: &str) {
1975        self.peer_selector
1976            .write()
1977            .await
1978            .record_cashu_payment_default(peer_id);
1979    }
1980
1981    /// Snapshot routing/selection summary for inspection/debugging.
1982    pub async fn selector_summary(&self) -> crate::peer_selector::SelectorSummary {
1983        self.peer_selector.read().await.summary()
1984    }
1985
1986    fn should_refuse_requests_from_peer(&self, selector: &PeerSelector, peer_id: &str) -> bool {
1987        selector.is_peer_blocked_for_payment_defaults(
1988            peer_id,
1989            self.routing.cashu_payment_default_block_threshold,
1990        )
1991    }
1992
1993    /// Export live peer metadata for inspection/debugging.
1994    pub async fn peer_metadata_snapshot(&self) -> PeerMetadataSnapshot {
1995        self.peer_selector
1996            .read()
1997            .await
1998            .export_peer_metadata_snapshot()
1999    }
2000
2001    /// Snapshot current peer metadata and persist it into `local_store`.
2002    ///
2003    /// Uses content-addressed storage for the snapshot body and a reserved
2004    /// mutable pointer slot for the "latest snapshot hash".
2005    pub async fn persist_peer_metadata(&self) -> Result<Hash, StoreError> {
2006        let snapshot = self
2007            .peer_selector
2008            .read()
2009            .await
2010            .export_peer_metadata_snapshot();
2011        let bytes = serde_json::to_vec(&snapshot).map_err(|e| {
2012            StoreError::Other(format!("Failed to encode peer metadata snapshot: {e}"))
2013        })?;
2014        let snapshot_hash = hashtree_core::sha256(&bytes);
2015        let _ = self.local_store.put(snapshot_hash, bytes).await?;
2016
2017        let pointer_slot = Self::peer_metadata_pointer_slot_hash();
2018        let pointer_bytes = hex::encode(snapshot_hash).into_bytes();
2019        let _ = self.local_store.delete(&pointer_slot).await?;
2020        let _ = self.local_store.put(pointer_slot, pointer_bytes).await?;
2021
2022        Ok(snapshot_hash)
2023    }
2024
2025    /// Load persisted peer metadata from `local_store` if available.
2026    pub async fn load_peer_metadata(&self) -> Result<bool, StoreError> {
2027        let pointer_slot = Self::peer_metadata_pointer_slot_hash();
2028        let Some(pointer_bytes) = self.local_store.get(&pointer_slot).await? else {
2029            return Ok(false);
2030        };
2031        let pointer_hex = std::str::from_utf8(&pointer_bytes).map_err(|e| {
2032            StoreError::Other(format!("Peer metadata pointer is not valid UTF-8: {e}"))
2033        })?;
2034        let snapshot_hash = Self::decode_hash_hex(pointer_hex.trim())?;
2035
2036        let Some(snapshot_bytes) = self.local_store.get(&snapshot_hash).await? else {
2037            return Ok(false);
2038        };
2039        let snapshot: PeerMetadataSnapshot =
2040            serde_json::from_slice(&snapshot_bytes).map_err(|e| {
2041                StoreError::Other(format!("Failed to decode peer metadata snapshot: {e}"))
2042            })?;
2043        self.peer_selector
2044            .write()
2045            .await
2046            .import_peer_metadata_snapshot(&snapshot);
2047        Ok(true)
2048    }
2049
2050    /// Request data from peers after negotiating a paid quote.
2051    ///
2052    /// If quote negotiation fails or the quoted peer does not deliver, the store
2053    /// falls back to the normal unpaid retrieval path to preserve liveness.
2054    pub async fn get_with_quote(
2055        &self,
2056        hash: &Hash,
2057        payment_sat: u64,
2058        quote_ttl: Duration,
2059    ) -> Result<Option<Vec<u8>>, StoreError> {
2060        if let Some(data) = self.local_store.get(hash).await? {
2061            if hashtree_core::sha256(&data) != *hash {
2062                return Err(StoreError::Other(
2063                    "local store returned corrupt content".to_string(),
2064                ));
2065            }
2066            return Ok(Some(data));
2067        }
2068        self.request_from_peers_with_quote(hash, payment_sat, quote_ttl)
2069            .await
2070    }
2071
2072    async fn request_from_peers_with_quote(
2073        &self,
2074        hash: &Hash,
2075        payment_sat: u64,
2076        quote_ttl: Duration,
2077    ) -> Result<Option<Vec<u8>>, StoreError> {
2078        let ordered_peer_ids = self.ordered_connected_peers(None).await;
2079        if ordered_peer_ids.is_empty() {
2080            return Ok(None);
2081        }
2082
2083        if let Some(quote) = self
2084            .request_quote_from_peers(hash, payment_sat, quote_ttl, &ordered_peer_ids)
2085            .await
2086        {
2087            if let Some(data) = self
2088                .request_from_single_peer(hash, &quote.peer_id, MAX_HTL, Some(quote.quote_id))
2089                .await
2090            {
2091                return Ok(Some(data));
2092            }
2093        }
2094
2095        match self
2096            .request_from_mesh_with_context(hash, &MeshReadContext::default())
2097            .await
2098        {
2099            RouteFetchOutcome::Hit(data) => Ok(Some(data)),
2100            RouteFetchOutcome::Miss => Ok(None),
2101            RouteFetchOutcome::Timeout => Err(StoreError::Other(
2102                "blob retrieval deadline expired before the search completed".to_string(),
2103            )),
2104        }
2105    }
2106
2107    async fn request_quote_from_peers(
2108        &self,
2109        hash: &Hash,
2110        payment_sat: u64,
2111        quote_ttl: Duration,
2112        ordered_peer_ids: &[String],
2113    ) -> Option<NegotiatedQuote> {
2114        if ordered_peer_ids.is_empty() {
2115            return None;
2116        }
2117        let ttl_ms = quote_ttl.as_millis().min(u32::MAX as u128) as u32;
2118        if ttl_ms == 0 {
2119            return None;
2120        }
2121        let requested_mint = self.requested_quote_mint().map(str::to_string);
2122
2123        let hash_key = hash_to_key(hash);
2124        let (tx, rx) = oneshot::channel();
2125        self.pending_quotes.write().await.insert(
2126            hash_key.clone(),
2127            PendingQuoteRequest {
2128                response_tx: tx,
2129                preferred_mint_url: requested_mint.clone(),
2130                offered_payment_sat: payment_sat,
2131            },
2132        );
2133
2134        let rx = Arc::new(Mutex::new(rx));
2135        let result = run_hedged_waves(
2136            ordered_peer_ids.len(),
2137            self.routing.dispatch,
2138            self.request_timeout,
2139            |range| {
2140                let wave_peer_ids = ordered_peer_ids[range].to_vec();
2141                let requested_mint = requested_mint.clone();
2142                let hash = *hash;
2143                async move {
2144                    let mut sent = 0usize;
2145                    for peer_id in wave_peer_ids {
2146                        if self
2147                            .send_quote_request_to_peer(
2148                                &peer_id,
2149                                &hash,
2150                                payment_sat,
2151                                ttl_ms,
2152                                requested_mint.as_deref(),
2153                            )
2154                            .await
2155                        {
2156                            sent += 1;
2157                        }
2158                    }
2159                    sent
2160                }
2161            },
2162            |wait| {
2163                let rx = rx.clone();
2164                async move {
2165                    let mut rx = rx.lock().await;
2166                    match tokio::time::timeout(wait, &mut *rx).await {
2167                        Ok(Ok(Some(quote))) => HedgedWaveAction::Success(quote),
2168                        Ok(Ok(None)) | Ok(Err(_)) => HedgedWaveAction::Abort,
2169                        Err(_) => HedgedWaveAction::Continue,
2170                    }
2171                }
2172            },
2173        )
2174        .await;
2175        let _ = self.pending_quotes.write().await.remove(&hash_key);
2176        result
2177    }
2178
2179    async fn register_pending_request(
2180        &self,
2181        request_key: PendingRequestKey,
2182        queried_peers: Vec<String>,
2183    ) -> (Arc<()>, oneshot::Receiver<Option<Vec<u8>>>) {
2184        let owner = Arc::new(());
2185        let (response_tx, response_rx) = oneshot::channel();
2186        self.pending_requests
2187            .write()
2188            .await
2189            .entry(request_key)
2190            .or_default()
2191            .push(PendingRequest {
2192                owner: owner.clone(),
2193                response_tx,
2194                started_at: Instant::now(),
2195                queried_peers,
2196            });
2197        (owner, response_rx)
2198    }
2199
2200    async fn take_pending_request(
2201        &self,
2202        request_key: PendingRequestKey,
2203        owner: &Arc<()>,
2204    ) -> Option<(PendingRequest, bool)> {
2205        let mut pending = self.pending_requests.write().await;
2206        let (request, remove_key) = {
2207            let requests = pending.get_mut(&request_key)?;
2208            let index = requests
2209                .iter()
2210                .position(|request| Arc::ptr_eq(&request.owner, owner))?;
2211            let request = requests.swap_remove(index);
2212            (request, requests.is_empty())
2213        };
2214        if remove_key {
2215            pending.remove(&request_key);
2216        }
2217        Some((request, remove_key))
2218    }
2219
2220    async fn request_from_single_peer(
2221        &self,
2222        hash: &Hash,
2223        peer_id: &str,
2224        request_htl: u8,
2225        quote_id: Option<u64>,
2226    ) -> Option<Vec<u8>> {
2227        let request_key = PendingRequestKey::new(*hash, request_htl);
2228        let (owner, rx) = self
2229            .register_pending_request(request_key, vec![peer_id.to_string()])
2230            .await;
2231
2232        let mut rx = rx;
2233        if !self
2234            .send_request_to_peer(peer_id, hash, request_htl, quote_id)
2235            .await
2236        {
2237            if self
2238                .take_pending_request(request_key, &owner)
2239                .await
2240                .is_some_and(|(_, last)| last)
2241            {
2242                let _ = self.take_forward_requesters(request_key).await;
2243            }
2244            return None;
2245        }
2246        self.reserve_peer_request(peer_id).await;
2247
2248        if let Ok(Ok(Some(data))) = tokio::time::timeout(self.request_timeout, &mut rx).await {
2249            if data.len() <= BLOB_MAX_BYTES && hashtree_core::sha256(&data) == *hash {
2250                let _ = self.local_store.put(*hash, data.clone()).await;
2251                return Some(data);
2252            }
2253        }
2254
2255        if let Some((pending, last)) = self.take_pending_request(request_key, &owner).await {
2256            self.release_queried_peer_requests(&pending.queried_peers)
2257                .await;
2258            for peer_id in pending.queried_peers {
2259                self.peer_selector.write().await.record_timeout(&peer_id);
2260            }
2261            if last {
2262                let _ = self.take_forward_requesters(request_key).await;
2263            }
2264        }
2265        None
2266    }
2267
2268    async fn request_from_ordered_peers(
2269        &self,
2270        hash: &Hash,
2271        ordered_peer_ids: &[String],
2272        request_htl: u8,
2273        timeout: Duration,
2274    ) -> RouteFetchOutcome {
2275        let request_key = PendingRequestKey::new(*hash, request_htl);
2276        let (owner, rx) = self.register_pending_request(request_key, Vec::new()).await;
2277
2278        let rx = Arc::new(Mutex::new(rx));
2279        let result = run_hedged_waves(
2280            ordered_peer_ids.len(),
2281            normalize_dispatch_config(self.routing.dispatch, ordered_peer_ids.len()),
2282            timeout,
2283            |range| {
2284                let wave_peer_ids = ordered_peer_ids[range].to_vec();
2285                let hash = *hash;
2286                let owner = owner.clone();
2287                async move {
2288                    let mut sent = 0usize;
2289                    for peer_id in wave_peer_ids {
2290                        if self
2291                            .send_request_to_peer(&peer_id, &hash, request_htl, None)
2292                            .await
2293                        {
2294                            sent += 1;
2295                            self.reserve_peer_request(&peer_id).await;
2296                            if let Some(pending) = self
2297                                .pending_requests
2298                                .write()
2299                                .await
2300                                .get_mut(&request_key)
2301                                .and_then(|requests| {
2302                                    requests
2303                                        .iter_mut()
2304                                        .find(|request| Arc::ptr_eq(&request.owner, &owner))
2305                                })
2306                            {
2307                                pending.queried_peers.push(peer_id);
2308                            }
2309                        }
2310                    }
2311                    sent
2312                }
2313            },
2314            |wait| {
2315                let rx = rx.clone();
2316                async move {
2317                    let mut rx = rx.lock().await;
2318                    match tokio::time::timeout(wait, &mut *rx).await {
2319                        Ok(Ok(Some(data)))
2320                            if data.len() <= BLOB_MAX_BYTES
2321                                && hashtree_core::sha256(&data) == *hash =>
2322                        {
2323                            HedgedWaveAction::Success(data)
2324                        }
2325                        Ok(Ok(Some(_))) => HedgedWaveAction::Continue,
2326                        Ok(Ok(None)) | Ok(Err(_)) => HedgedWaveAction::Abort,
2327                        Err(_) => HedgedWaveAction::Continue,
2328                    }
2329                }
2330            },
2331        )
2332        .await;
2333
2334        let Some(data) = result else {
2335            if let Some((pending, last)) = self.take_pending_request(request_key, &owner).await {
2336                self.release_queried_peer_requests(&pending.queried_peers)
2337                    .await;
2338                for peer_id in pending.queried_peers {
2339                    self.peer_selector.write().await.record_timeout(&peer_id);
2340                }
2341                if last {
2342                    let _ = self.take_forward_requesters(request_key).await;
2343                }
2344            }
2345            return RouteFetchOutcome::Timeout;
2346        };
2347
2348        let _ = self.local_store.put(*hash, data.clone()).await;
2349        RouteFetchOutcome::Hit(data)
2350    }
2351
2352    async fn request_from_mesh_with_context(
2353        &self,
2354        hash: &Hash,
2355        context: &MeshReadContext,
2356    ) -> RouteFetchOutcome {
2357        if !should_forward_htl(context.request_htl) {
2358            return RouteFetchOutcome::Miss;
2359        }
2360        let mut peers = self
2361            .ordered_connected_peers(context.exclude_peer_id.as_deref())
2362            .await;
2363        if let Some(attempt_budget) = context.attempt_budget {
2364            peers.truncate(attempt_budget);
2365        }
2366        if peers.is_empty() {
2367            return RouteFetchOutcome::Miss;
2368        }
2369        let timeout = context
2370            .deadline
2371            .map(|deadline| deadline.saturating_duration_since(Instant::now()))
2372            .unwrap_or(self.request_timeout)
2373            .min(self.request_timeout);
2374        if timeout.is_zero() {
2375            return RouteFetchOutcome::Timeout;
2376        }
2377        self.request_from_ordered_peers(hash, &peers, context.request_htl, timeout)
2378            .await
2379    }
2380
2381    async fn begin_forward_request(
2382        &self,
2383        request_key: PendingRequestKey,
2384        requester_id: &str,
2385    ) -> bool {
2386        let mut pending = self.pending_forward_requests.write().await;
2387        if let Some(existing) = pending.get_mut(&request_key) {
2388            existing.requester_ids.insert(requester_id.to_string());
2389            return false;
2390        }
2391
2392        let mut requester_ids = HashSet::new();
2393        requester_ids.insert(requester_id.to_string());
2394        pending.insert(request_key, PendingForwardRequest { requester_ids });
2395        true
2396    }
2397
2398    async fn take_forward_requesters(&self, request_key: PendingRequestKey) -> Vec<String> {
2399        self.pending_forward_requests
2400            .write()
2401            .await
2402            .remove(&request_key)
2403            .map(|pending| pending.requester_ids.into_iter().collect())
2404            .unwrap_or_default()
2405    }
2406
2407    async fn complete_pending_response(
2408        self: &Arc<Self>,
2409        from_peer: &str,
2410        hash: &Hash,
2411        payload: Vec<u8>,
2412    ) {
2413        let pending = {
2414            let mut requests = self.pending_requests.write().await;
2415            let matching_keys: Vec<_> = requests
2416                .keys()
2417                .filter(|key| key.hash == *hash)
2418                .copied()
2419                .collect();
2420            matching_keys
2421                .into_iter()
2422                .flat_map(|key| {
2423                    requests
2424                        .remove(&key)
2425                        .into_iter()
2426                        .flatten()
2427                        .map(move |request| (key, request))
2428                })
2429                .collect::<Vec<_>>()
2430        };
2431        if pending.is_empty() {
2432            return;
2433        }
2434
2435        let payload_bytes = payload.len() as u64;
2436        self.record_useful_bytes_received_from_peer(from_peer, payload_bytes)
2437            .await;
2438        {
2439            let mut deliveries = self.verified_block_deliveries.lock().await;
2440            deliveries.deliveries.push_back(VerifiedBlockDelivery {
2441                hash: *hash,
2442                provider_peer_id: from_peer.to_string(),
2443                payload_bytes,
2444            });
2445            while deliveries.deliveries.len() > VERIFIED_BLOCK_DELIVERY_CAPACITY {
2446                deliveries.deliveries.pop_front();
2447                deliveries.dropped_since_last_drain =
2448                    deliveries.dropped_since_last_drain.saturating_add(1);
2449            }
2450        }
2451
2452        let rtt_ms = pending
2453            .iter()
2454            .map(|(_, request)| request.started_at.elapsed().as_millis() as u64)
2455            .max()
2456            .unwrap_or_default();
2457        let queried_peers: Vec<_> = pending
2458            .iter()
2459            .flat_map(|(_, request)| request.queried_peers.iter().cloned())
2460            .collect();
2461        self.release_queried_peer_requests(&queried_peers).await;
2462        self.peer_selector
2463            .write()
2464            .await
2465            .record_success(from_peer, rtt_ms, payload_bytes);
2466
2467        let mut forward_requesters = HashSet::new();
2468        for (request_key, pending) in pending {
2469            forward_requesters.extend(self.take_forward_requesters(request_key).await);
2470            let _ = pending.response_tx.send(Some(payload.clone()));
2471        }
2472        if !forward_requesters.is_empty() {
2473            let response_bytes = encode_response(&create_response(hash, payload));
2474            for requester_id in forward_requesters {
2475                Arc::clone(self)
2476                    .enqueue_response_send(requester_id, response_bytes.clone(), Instant::now())
2477                    .await;
2478            }
2479        }
2480    }
2481
2482    async fn handle_quote_response_message(&self, from_peer: &str, res: DataQuoteResponse) {
2483        if !res.a {
2484            return;
2485        }
2486
2487        let Some(quote_id) = res.q else {
2488            return;
2489        };
2490
2491        let hash_key = hash_to_key(&res.h);
2492        let (preferred_mint_url, offered_payment_sat) = {
2493            let pending_quotes = self.pending_quotes.read().await;
2494            let Some(pending) = pending_quotes.get(&hash_key) else {
2495                return;
2496            };
2497            (
2498                pending.preferred_mint_url.clone(),
2499                pending.offered_payment_sat,
2500            )
2501        };
2502        if !self
2503            .should_accept_quote_response(
2504                from_peer,
2505                preferred_mint_url.as_deref(),
2506                offered_payment_sat,
2507                &res,
2508            )
2509            .await
2510        {
2511            return;
2512        }
2513        let mut pending_quotes = self.pending_quotes.write().await;
2514        if let Some(pending) = pending_quotes.remove(&hash_key) {
2515            let _ = pending.response_tx.send(Some(NegotiatedQuote {
2516                peer_id: from_peer.to_string(),
2517                quote_id,
2518                mint_url: res.m,
2519            }));
2520        }
2521    }
2522
2523    async fn handle_response_message(
2524        self: &Arc<Self>,
2525        from_peer: &str,
2526        res: crate::protocol::DataResponse,
2527    ) {
2528        let hash_key = hash_to_key(&res.h);
2529        let hash = match crate::protocol::bytes_to_hash(&res.h) {
2530            Some(h) => h,
2531            None => return,
2532        };
2533
2534        // Ignore malformed/corrupt payload and keep waiting for a valid response.
2535        if hashtree_core::sha256(&res.d) != hash {
2536            self.peer_selector.write().await.record_failure(from_peer);
2537            if self.debug {
2538                println!("[MeshStoreCore] Ignoring invalid response payload for {hash_key}");
2539            }
2540            return;
2541        }
2542
2543        self.complete_pending_response(from_peer, &hash, res.d)
2544            .await;
2545    }
2546
2547    async fn handle_quote_request_message(&self, from_peer: &str, req: DataQuoteRequest) {
2548        let hash = match crate::protocol::bytes_to_hash(&req.h) {
2549            Some(h) => h,
2550            None => return,
2551        };
2552        let hash_key = hash_to_key(&hash);
2553
2554        {
2555            let selector = self.peer_selector.read().await;
2556            if self.should_refuse_requests_from_peer(&selector, from_peer) {
2557                if self.debug {
2558                    println!(
2559                        "[MeshStoreCore] Refusing quote request from delinquent peer {}",
2560                        from_peer
2561                    );
2562                }
2563                return;
2564            }
2565        }
2566
2567        let chosen_mint = self.choose_quote_mint(req.m.as_deref());
2568        let can_serve = self.local_store.has(&hash).await.ok().unwrap_or(false)
2569            && !self.should_drop_response(&hash)
2570            && !self.should_corrupt_response(&hash);
2571
2572        let res = if can_serve {
2573            let quote_id = self
2574                .issue_quote(from_peer, &hash_key, req.p, req.t, chosen_mint.as_deref())
2575                .await;
2576            create_quote_response_available(&hash, quote_id, req.p, req.t, chosen_mint.as_deref())
2577        } else {
2578            create_quote_response_unavailable(&hash)
2579        };
2580        let response_bytes = encode_quote_response(&res);
2581        if let Some(channel) = self.signaling.get_channel(from_peer).await {
2582            if channel.send(response_bytes.clone()).await.is_ok() {
2583                self.record_peer_wire_sent(from_peer, response_bytes.len() as u64)
2584                    .await;
2585            }
2586        }
2587    }
2588
2589    async fn handle_request_message(
2590        self: &Arc<Self>,
2591        from_peer: &str,
2592        req: crate::protocol::DataRequest,
2593    ) {
2594        if req.htl > MAX_HTL {
2595            return;
2596        }
2597        let hash = match crate::protocol::bytes_to_hash(&req.h) {
2598            Some(h) => h,
2599            None => return,
2600        };
2601        let hash_key = hash_to_key(&hash);
2602        let request_key = PendingRequestKey::new(hash, req.htl);
2603
2604        if let Some(quote_id) = req.q {
2605            if !self.take_valid_quote(from_peer, &hash_key, quote_id).await {
2606                if self.debug {
2607                    println!(
2608                        "[MeshStoreCore] Refusing request with invalid or expired quote {} from {}",
2609                        quote_id, from_peer
2610                    );
2611                }
2612                return;
2613            }
2614        }
2615
2616        let allow_peer_forwarding = {
2617            let selector = self.peer_selector.read().await;
2618            !self.should_refuse_requests_from_peer(&selector, from_peer)
2619        };
2620
2621        // Check local store
2622        if let Ok(Some(mut data)) = self.local_store.get(&hash).await {
2623            if data.len() <= BLOB_MAX_BYTES && hashtree_core::sha256(&data) == hash {
2624                if self.should_drop_response(&hash) {
2625                    if self.debug {
2626                        println!(
2627                            "[MeshStoreCore] Dropping response for {} due to actor profile",
2628                            hash_to_key(&hash)
2629                        );
2630                    }
2631                    return;
2632                }
2633
2634                let response_delay = self.response_send_delay(&hash, data.len());
2635                if self.should_corrupt_response(&hash) {
2636                    if data.is_empty() {
2637                        data.push(0x80);
2638                    } else {
2639                        data[0] ^= 0x80;
2640                    }
2641                }
2642
2643                let res = create_response(&hash, data);
2644                let response_bytes = encode_response(&res);
2645                let ready_at = Instant::now() + response_delay;
2646                Arc::clone(self)
2647                    .enqueue_response_send(from_peer.to_string(), response_bytes, ready_at)
2648                    .await;
2649                return;
2650            }
2651        }
2652
2653        if self
2654            .pending_requests
2655            .read()
2656            .await
2657            .contains_key(&request_key)
2658        {
2659            let _ = self.begin_forward_request(request_key, from_peer).await;
2660            return;
2661        }
2662
2663        if !self.begin_forward_request(request_key, from_peer).await {
2664            return;
2665        }
2666
2667        let from_peer = from_peer.to_string();
2668        let this = Arc::clone(self);
2669        let request_htl = req.htl;
2670        tokio::spawn(async move {
2671            let result = if allow_peer_forwarding {
2672                let context = MeshReadContext {
2673                    exclude_peer_id: Some(from_peer.clone()),
2674                    request_htl,
2675                    deadline: None,
2676                    attempt_budget: None,
2677                };
2678                this.request_from_mesh_with_context(&hash, &context).await
2679            } else {
2680                if this.debug {
2681                    println!(
2682                        "[MeshStoreCore] Refusing to forward request from delinquent peer {}",
2683                        from_peer
2684                    );
2685                }
2686                RouteFetchOutcome::Miss
2687            };
2688            let requester_ids = this.take_forward_requesters(request_key).await;
2689            match result {
2690                RouteFetchOutcome::Hit(data) => {
2691                    let ready_at = Instant::now() + this.response_send_delay(&hash, data.len());
2692                    let res = create_response(&hash, data);
2693                    let response_bytes = encode_response(&res);
2694                    for requester_id in requester_ids {
2695                        Arc::clone(&this)
2696                            .enqueue_response_send(requester_id, response_bytes.clone(), ready_at)
2697                            .await;
2698                    }
2699                }
2700                RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => {}
2701            }
2702        });
2703    }
2704
2705    async fn handle_pubsub_interest_message(
2706        self: &Arc<Self>,
2707        from_peer: &str,
2708        mut interest: PubsubInterest,
2709    ) {
2710        if !self.apply_pubsub_interest_route(from_peer, &interest).await {
2711            return;
2712        }
2713
2714        if !self.routing.pubsub_forwarding || interest.htl <= 1 {
2715            return;
2716        }
2717        interest.htl = interest.htl.saturating_sub(1);
2718        let _ = self
2719            .send_pubsub_interest_to_peers(&interest, Some(from_peer))
2720            .await;
2721    }
2722
2723    async fn handle_pubsub_frame_message(
2724        self: &Arc<Self>,
2725        from_peer: &str,
2726        mut frame: PubsubFrame,
2727        wire_bytes: usize,
2728    ) {
2729        if frame.stream_id.is_empty() || frame.origin_peer_id.is_empty() {
2730            return;
2731        }
2732        if frame.origin_peer_id == self.signaling.peer_id() {
2733            return;
2734        }
2735
2736        let frame_key = Self::pubsub_frame_key(&frame);
2737        if !self
2738            .pubsub_seen_frames
2739            .lock()
2740            .await
2741            .insert_if_new(frame_key.clone())
2742        {
2743            return;
2744        }
2745        self.cache_pubsub_frame(frame_key.clone(), frame.clone())
2746            .await;
2747
2748        let local_interested = self
2749            .pubsub_local_interests
2750            .read()
2751            .await
2752            .contains(&frame.stream_id);
2753        let mut downstream_peers = if self.routing.pubsub_forwarding && frame.htl > 1 {
2754            match self.routing.pubsub_delivery_mode {
2755                PubsubDeliveryMode::InterestPush => {
2756                    let mut peers = self
2757                        .interested_pubsub_peers(&frame.stream_id, Some(from_peer))
2758                        .await;
2759                    peers.extend(
2760                        self.take_pubsub_want_peers(&frame_key, Some(from_peer))
2761                            .await,
2762                    );
2763                    peers.sort();
2764                    peers.dedup();
2765                    peers
2766                }
2767                PubsubDeliveryMode::HtlInvWant => {
2768                    self.take_pubsub_want_peers(&frame_key, Some(from_peer))
2769                        .await
2770                }
2771            }
2772        } else {
2773            Vec::new()
2774        };
2775        downstream_peers.retain(|peer_id| peer_id != from_peer);
2776
2777        if local_interested || !downstream_peers.is_empty() {
2778            self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
2779                .await;
2780        }
2781
2782        if local_interested {
2783            self.enqueue_pubsub_event(PubsubEvent {
2784                stream_id: frame.stream_id.clone(),
2785                seq: frame.seq,
2786                origin_peer_id: frame.origin_peer_id.clone(),
2787                from_peer_id: from_peer.to_string(),
2788                payload: frame.payload.clone(),
2789            })
2790            .await;
2791        }
2792
2793        if downstream_peers.is_empty() {
2794            return;
2795        }
2796
2797        frame.htl = frame.htl.saturating_sub(1);
2798        let _ = self
2799            .send_pubsub_frame_to_peers(&frame, &downstream_peers)
2800            .await;
2801    }
2802
2803    async fn handle_pubsub_inventory_message(
2804        self: &Arc<Self>,
2805        from_peer: &str,
2806        inv: PubsubInventory,
2807        wire_bytes: usize,
2808    ) {
2809        if inv.stream_id.is_empty() || inv.origin_peer_id.is_empty() {
2810            return;
2811        }
2812        if inv.origin_peer_id == self.signaling.peer_id() {
2813            return;
2814        }
2815
2816        let key = Self::pubsub_key(&inv.origin_peer_id, &inv.stream_id, inv.seq);
2817        if !self
2818            .pubsub_seen_inventories
2819            .lock()
2820            .await
2821            .insert_if_new_or_higher(key.clone(), inv.htl)
2822        {
2823            return;
2824        }
2825        {
2826            let mut routes = self.pubsub_inventory_routes.write().await;
2827            routes.insert(key.clone(), from_peer.to_string());
2828        }
2829
2830        let local_interested = self
2831            .pubsub_local_interests
2832            .read()
2833            .await
2834            .contains(&inv.stream_id);
2835        let downstream_peers = if self.routing.pubsub_forwarding {
2836            self.interested_pubsub_peers(&inv.stream_id, Some(from_peer))
2837                .await
2838        } else {
2839            Vec::new()
2840        };
2841        if local_interested || !downstream_peers.is_empty() {
2842            self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
2843                .await;
2844            let want =
2845                create_pubsub_want(inv.stream_id.clone(), inv.seq, inv.origin_peer_id.clone());
2846            let _ = self.send_pubsub_want_upstream(&key, &want, None).await;
2847        }
2848
2849        if !self.routing.pubsub_forwarding
2850            || downstream_peers.is_empty()
2851            || !should_forward_htl(inv.htl)
2852        {
2853            return;
2854        }
2855        let _ = self
2856            .send_pubsub_inventory_to_peers(&inv, &downstream_peers, true)
2857            .await;
2858    }
2859
2860    async fn handle_pubsub_want_message(
2861        self: &Arc<Self>,
2862        from_peer: &str,
2863        want: PubsubWant,
2864        wire_bytes: usize,
2865    ) {
2866        if want.stream_id.is_empty() || want.origin_peer_id.is_empty() {
2867            return;
2868        }
2869        if want.origin_peer_id == from_peer {
2870            return;
2871        }
2872
2873        let key = Self::pubsub_key(&want.origin_peer_id, &want.stream_id, want.seq);
2874        let want_key = format!("{from_peer}:{key}");
2875        if !self.pubsub_seen_wants.lock().await.insert_if_new(want_key) {
2876            return;
2877        }
2878
2879        if let Some(frame) = self.cached_pubsub_frame(&key).await {
2880            self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
2881                .await;
2882            let peers = vec![from_peer.to_string()];
2883            let _ = self.send_pubsub_frame_to_peers(&frame, &peers).await;
2884            return;
2885        }
2886
2887        let has_upstream_route = self.pubsub_inventory_routes.read().await.contains_key(&key);
2888        if !has_upstream_route {
2889            return;
2890        }
2891
2892        if self.remember_pubsub_want_peer(key.clone(), from_peer).await {
2893            self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
2894                .await;
2895        }
2896        let _ = self
2897            .send_pubsub_want_upstream(&key, &want, Some(from_peer))
2898            .await;
2899    }
2900
2901    /// Handle incoming data message
2902    pub async fn handle_data_message(self: &Arc<Self>, from_peer: &str, data: &[u8]) {
2903        self.record_peer_wire_received(from_peer, data.len() as u64)
2904            .await;
2905        let parsed = match parse_message(data) {
2906            Some(m) => m,
2907            None => return,
2908        };
2909
2910        match parsed {
2911            DataMessage::Request(req) => {
2912                self.handle_request_message(from_peer, req).await;
2913            }
2914            DataMessage::Response(res) => {
2915                self.handle_response_message(from_peer, res).await;
2916            }
2917            DataMessage::QuoteRequest(req) => {
2918                self.handle_quote_request_message(from_peer, req).await;
2919            }
2920            DataMessage::QuoteResponse(res) => {
2921                self.handle_quote_response_message(from_peer, res).await;
2922            }
2923            DataMessage::PubsubInterest(interest) => {
2924                self.handle_pubsub_interest_message(from_peer, interest)
2925                    .await;
2926            }
2927            DataMessage::PubsubFrame(frame) => {
2928                self.handle_pubsub_frame_message(from_peer, frame, data.len())
2929                    .await;
2930            }
2931            DataMessage::PubsubInventory(inv) => {
2932                self.handle_pubsub_inventory_message(from_peer, inv, data.len())
2933                    .await;
2934            }
2935            DataMessage::PubsubWant(want) => {
2936                self.handle_pubsub_want_message(from_peer, want, data.len())
2937                    .await;
2938            }
2939            DataMessage::Payment(_)
2940            | DataMessage::PaymentAck(_)
2941            | DataMessage::Chunk(_)
2942            | DataMessage::PeerHints(_) => {}
2943        }
2944    }
2945}
2946
2947#[async_trait]
2948impl<S, R, F> BlobRoute for MeshStoreCore<S, R, F>
2949where
2950    S: Store + Send + Sync + 'static,
2951    R: SignalingTransport + Send + Sync + 'static,
2952    F: PeerLinkFactory + Send + Sync + 'static,
2953{
2954    async fn route(&self, request: BlobRequest) -> Result<BlobReply, StoreError> {
2955        self.route_with_context(
2956            request,
2957            BlobRouteContext {
2958                deadline: (Instant::now() + self.request_timeout).into(),
2959                attempt_budget: self.routing.dispatch.max_fanout.max(1),
2960            },
2961        )
2962        .await
2963    }
2964
2965    async fn route_with_context(
2966        &self,
2967        request: BlobRequest,
2968        route_context: BlobRouteContext,
2969    ) -> Result<BlobReply, StoreError> {
2970        if request.htl > MAX_HTL {
2971            return Err(StoreError::Other(format!(
2972                "Hashtree blob HTL {} exceeds the maximum of {MAX_HTL}",
2973                request.htl
2974            )));
2975        }
2976        if let Some(data) = self.local_store.get(&request.hash).await? {
2977            if data.len() > BLOB_MAX_BYTES {
2978                return Err(StoreError::Other(format!(
2979                    "local store returned {} bytes, exceeding the {BLOB_MAX_BYTES}-byte limit",
2980                    data.len()
2981                )));
2982            }
2983            if hashtree_core::sha256(&data) != request.hash {
2984                return Err(StoreError::Other(
2985                    "local store returned corrupt content".to_string(),
2986                ));
2987            }
2988            return Ok(BlobReply::Data(data));
2989        }
2990
2991        if request.htl == 0 {
2992            return Ok(BlobReply::NoResult);
2993        }
2994
2995        let context = MeshReadContext {
2996            exclude_peer_id: None,
2997            request_htl: request.htl,
2998            deadline: Some(route_context.deadline.into()),
2999            attempt_budget: Some(route_context.attempt_budget),
3000        };
3001        match self
3002            .request_from_mesh_with_context(&request.hash, &context)
3003            .await
3004        {
3005            RouteFetchOutcome::Hit(data) => {
3006                if data.len() > BLOB_MAX_BYTES {
3007                    return Err(StoreError::Other(format!(
3008                        "blob route returned {} bytes, exceeding the {BLOB_MAX_BYTES}-byte limit",
3009                        data.len()
3010                    )));
3011                }
3012                if hashtree_core::sha256(&data) != request.hash {
3013                    return Err(StoreError::Other(
3014                        "blob route returned corrupt content".to_string(),
3015                    ));
3016                }
3017                Ok(BlobReply::Data(data))
3018            }
3019            RouteFetchOutcome::Miss => Ok(BlobReply::NoResult),
3020            RouteFetchOutcome::Timeout => Err(StoreError::Other(
3021                "blob retrieval deadline expired before the search completed".to_string(),
3022            )),
3023        }
3024    }
3025}
3026
3027#[async_trait]
3028impl<S, R, F> Store for MeshStoreCore<S, R, F>
3029where
3030    S: Store + Send + Sync + 'static,
3031    R: SignalingTransport + Send + Sync + 'static,
3032    F: PeerLinkFactory + Send + Sync + 'static,
3033{
3034    async fn put(&self, hash: Hash, data: Vec<u8>) -> Result<bool, StoreError> {
3035        self.local_store.put(hash, data).await
3036    }
3037
3038    async fn get(&self, hash: &Hash) -> Result<Option<Vec<u8>>, StoreError> {
3039        Ok(
3040            match self
3041                .route(BlobRequest {
3042                    hash: *hash,
3043                    htl: MAX_HTL,
3044                })
3045                .await?
3046            {
3047                BlobReply::Data(data) => Some(data),
3048                BlobReply::NoResult => None,
3049            },
3050        )
3051    }
3052
3053    async fn has(&self, hash: &Hash) -> Result<bool, StoreError> {
3054        self.local_store.has(hash).await
3055    }
3056
3057    async fn delete(&self, hash: &Hash) -> Result<bool, StoreError> {
3058        self.local_store.delete(hash).await
3059    }
3060}
3061
3062#[cfg(test)]
3063mod delivery_tests;
3064
3065#[cfg(test)]
3066mod tests;
3067
3068/// Type alias for simulation store.
3069pub type SimMeshStore<S> =
3070    MeshStoreCore<S, crate::mock::MockRelayTransport, crate::mock::MockConnectionFactory>;