Skip to main content

hashtree_network/
mesh_store_core.rs

1//! Shared routed mesh store core.
2//!
3//! This module provides a concrete store wrapper that works with any local storage
4//! backend plus any signaling transport and peer-link factory. Both production
5//! and simulation (mocks) use this same code.
6
7use async_trait::async_trait;
8use std::collections::hash_map::DefaultHasher;
9use std::collections::{HashMap, HashSet, VecDeque};
10use std::future::Future;
11use std::hash::{Hash as _, Hasher};
12use std::ops::Range;
13use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
14use std::sync::Arc;
15use std::time::Duration;
16use tokio::sync::{oneshot, Mutex, Notify, RwLock};
17use tokio::time::Instant;
18
19use hashtree_core::{
20    BlobReply, BlobRequest, BlobRoute, BlobRouteContext, Hash, Store, StoreError, BLOB_MAX_BYTES,
21};
22
23use crate::peer_selector::{PeerMetadataSnapshot, PeerSelector, SelectionStrategy};
24use crate::protocol::{
25    create_pubsub_frame, create_pubsub_interest, create_pubsub_inventory, create_pubsub_want,
26    create_quote_request, create_quote_response_available, create_quote_response_unavailable,
27    create_request, create_request_with_quote, create_response, encode_pubsub_frame,
28    encode_pubsub_interest, encode_pubsub_inventory, encode_pubsub_want, encode_quote_request,
29    encode_quote_response, encode_request, encode_response, hash_to_key, parse_message,
30    DataMessage, DataQuoteRequest, DataQuoteResponse, PubsubFrame, PubsubInterest, PubsubInventory,
31    PubsubWant,
32};
33use crate::pubsub_strategy::{
34    reciprocal_virtual_finish, select_reciprocal_outbound_job, OutboundJobCandidate,
35    PeerTrafficSnapshot, PubsubCandidate, PubsubSchedulerConfig,
36};
37use crate::signaling::MeshRouter;
38use crate::transport::{PeerLinkFactory, SignalingTransport, TransportError};
39use crate::types::{
40    should_forward_htl, PeerHTLConfig, SignalingMessage, TimedSeenSet, MAX_HTL, MESH_EVENT_POLICY,
41};
42
43// Keep the on-disk namespace stable across the crate rename so existing peer
44// metadata does not disappear for users upgrading from the old package name.
45const PEER_METADATA_POINTER_SLOT_KEY: &[u8] = b"hashtree-mesh/peer-metadata/latest/v1";
46const PUBSUB_SEEN_CAPACITY: usize = 16_384;
47const PUBSUB_INBOX_CAPACITY: usize = 4_096;
48const PUBSUB_FRAME_CACHE_CAPACITY: usize = 4_096;
49const VERIFIED_BLOCK_DELIVERY_CAPACITY: usize = 4_096;
50const PUBSUB_SEEN_TTL: Duration = Duration::from_secs(120);
51
52/// Pending request awaiting response
53struct PendingRequest {
54    owner: Arc<()>,
55    response_tx: oneshot::Sender<Option<Vec<u8>>>,
56    started_at: Instant,
57    queried_peers: Vec<String>,
58}
59
60#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
61struct PendingRequestKey {
62    hash: Hash,
63    htl: u8,
64}
65
66impl PendingRequestKey {
67    fn new(hash: Hash, htl: u8) -> Self {
68        Self { hash, htl }
69    }
70}
71
72struct PendingQuoteRequest {
73    response_tx: oneshot::Sender<Option<NegotiatedQuote>>,
74    preferred_mint_url: Option<String>,
75    offered_payment_sat: u64,
76}
77
78struct PendingForwardRequest {
79    requester_ids: HashSet<String>,
80}
81
82type PeerWireStats = PeerTrafficSnapshot;
83
84struct PendingResponseSend {
85    job_id: u64,
86    peer_id: String,
87    bytes: Vec<u8>,
88    ready_at: Instant,
89    queue_sequence: u64,
90}
91
92#[derive(Debug, Clone)]
93struct NegotiatedQuote {
94    peer_id: String,
95    quote_id: u64,
96    #[allow(dead_code)]
97    mint_url: Option<String>,
98}
99
100struct IssuedQuote {
101    expires_at: Instant,
102    #[allow(dead_code)]
103    payment_sat: u64,
104    #[allow(dead_code)]
105    mint_url: Option<String>,
106}
107
108#[derive(Debug, Clone)]
109enum RouteFetchOutcome {
110    Hit(Vec<u8>),
111    Miss,
112    Timeout,
113}
114
115const ACTIVE_PEER_REQUEST_RANK_PENALTY: usize = 3;
116
117#[derive(Debug, Clone)]
118struct MeshReadContext {
119    exclude_peer_id: Option<String>,
120    request_htl: u8,
121    deadline: Option<Instant>,
122    attempt_budget: Option<usize>,
123}
124
125impl Default for MeshReadContext {
126    fn default() -> Self {
127        Self {
128            exclude_peer_id: None,
129            request_htl: MAX_HTL,
130            deadline: None,
131            attempt_budget: None,
132        }
133    }
134}
135
136/// Aggregate stats from draining currently available peer-link messages.
137#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
138pub struct DataPumpStats {
139    pub processed: usize,
140    pub request_messages: usize,
141    pub response_messages: usize,
142    pub quote_request_messages: u64,
143    pub quote_response_messages: u64,
144    pub pubsub_interest_messages: u64,
145    pub pubsub_frame_messages: u64,
146    pub pubsub_inventory_messages: u64,
147    pub pubsub_want_messages: u64,
148    pub processed_bytes: u64,
149}
150
151/// Pubsub data delivered to a local subscription.
152#[derive(Debug, Clone, PartialEq, Eq)]
153pub struct PubsubEvent {
154    pub stream_id: String,
155    pub seq: u64,
156    pub origin_peer_id: String,
157    pub from_peer_id: String,
158    pub payload: Vec<u8>,
159}
160
161/// Evidence that this peer won an outstanding, hash-verified block request.
162#[derive(Debug, Clone, PartialEq, Eq)]
163pub struct VerifiedBlockDelivery {
164    pub hash: Hash,
165    pub provider_peer_id: String,
166    pub payload_bytes: u64,
167}
168
169/// One atomic drain of verified delivery evidence and any overflow since the prior drain.
170///
171/// Dropped evidence is intentionally not recoverable or billable through this API. An
172/// application adapter must surface a non-zero count and must not infer a payment claim.
173#[derive(Debug, Clone, Default, PartialEq, Eq)]
174pub struct VerifiedBlockDeliveryBatch {
175    pub deliveries: Vec<VerifiedBlockDelivery>,
176    pub dropped_since_last_drain: u64,
177}
178
179#[derive(Default)]
180struct VerifiedBlockDeliveryBuffer {
181    deliveries: VecDeque<VerifiedBlockDelivery>,
182    dropped_since_last_drain: u64,
183}
184
185/// Send-side accounting from a pubsub publish or forwarded pubsub message.
186#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
187pub struct PubsubPublishStats {
188    pub selected_peers: usize,
189    pub sent_peers: usize,
190    pub sent_bytes: u64,
191    pub deferred_peers: usize,
192}
193
194/// Production pubsub delivery strategy.
195#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
196pub enum PubsubDeliveryMode {
197    /// Push full frames only along advertised interest routes.
198    InterestPush,
199    /// Route small inventories along advertised interest paths and pull payloads back along want paths.
200    #[default]
201    HtlInvWant,
202}
203
204/// Request dispatch strategy for peer queries.
205///
206/// Requests use bounded staged hedging by default. Callers may explicitly
207/// raise the cap for controlled simulations, but production never needs an
208/// unbounded sentinel.
209#[derive(Debug, Clone, Copy)]
210pub struct RequestDispatchConfig {
211    /// Number of peers queried immediately.
212    pub initial_fanout: usize,
213    /// Number of additional peers to query on each hedge step.
214    pub hedge_fanout: usize,
215    /// Total peers allowed for this request.
216    pub max_fanout: usize,
217    /// Delay between hedge waves (ms). `0` means send all waves immediately.
218    pub hedge_interval_ms: u64,
219}
220
221impl Default for RequestDispatchConfig {
222    fn default() -> Self {
223        Self {
224            initial_fanout: 2,
225            hedge_fanout: 1,
226            max_fanout: 4,
227            hedge_interval_ms: 50,
228        }
229    }
230}
231
232/// Normalize fanout config against current peer availability.
233pub fn normalize_dispatch_config(
234    dispatch: RequestDispatchConfig,
235    available_peers: usize,
236) -> RequestDispatchConfig {
237    let mut cfg = dispatch;
238    let cap = if cfg.max_fanout == 0 {
239        available_peers
240    } else {
241        cfg.max_fanout.min(available_peers)
242    };
243    cfg.max_fanout = cap;
244    cfg.initial_fanout = if cfg.initial_fanout == 0 {
245        1
246    } else {
247        cfg.initial_fanout.min(cap.max(1))
248    };
249    cfg.hedge_fanout = if cfg.hedge_fanout == 0 {
250        1
251    } else {
252        cfg.hedge_fanout.min(cap.max(1))
253    };
254    cfg
255}
256
257/// Build wave sizes for staged hedged dispatch.
258pub fn build_hedged_wave_plan(peer_count: usize, dispatch: RequestDispatchConfig) -> Vec<usize> {
259    if peer_count == 0 {
260        return Vec::new();
261    }
262    let cap = dispatch.max_fanout.min(peer_count);
263    if cap == 0 {
264        return Vec::new();
265    }
266
267    let mut plan = Vec::new();
268    let mut sent = 0usize;
269    let first = dispatch.initial_fanout.min(cap).max(1);
270    plan.push(first);
271    sent += first;
272
273    while sent < cap {
274        let next = dispatch.hedge_fanout.min(cap - sent).max(1);
275        plan.push(next);
276        sent += next;
277    }
278    plan
279}
280
281/// Outcome returned after waiting on a hedged dispatch wave.
282#[derive(Debug)]
283pub enum HedgedWaveAction<T> {
284    Continue,
285    Success(T),
286    Abort,
287}
288
289/// Run a staged hedged dispatch over peer index ranges.
290///
291/// This scheduler is shared by the reusable `MeshStoreCore` and the native
292/// `hashtree-cli` mesh path so tests and production use the same wave timing.
293pub async fn run_hedged_waves<T, SendWave, SendWaveFut, WaitWave, WaitWaveFut>(
294    peer_count: usize,
295    dispatch: RequestDispatchConfig,
296    request_timeout: Duration,
297    mut send_wave: SendWave,
298    mut wait_wave: WaitWave,
299) -> Option<T>
300where
301    SendWave: FnMut(Range<usize>) -> SendWaveFut,
302    SendWaveFut: Future<Output = usize>,
303    WaitWave: FnMut(Duration) -> WaitWaveFut,
304    WaitWaveFut: Future<Output = HedgedWaveAction<T>>,
305{
306    let dispatch = normalize_dispatch_config(dispatch, peer_count);
307    let wave_plan = build_hedged_wave_plan(peer_count, dispatch);
308    if wave_plan.is_empty() {
309        return None;
310    }
311
312    let deadline = Instant::now() + request_timeout;
313    let mut sent_total = 0usize;
314    let mut next_peer_idx = 0usize;
315
316    for (wave_idx, wave_size) in wave_plan.iter().copied().enumerate() {
317        let from = next_peer_idx;
318        let to = (next_peer_idx + wave_size).min(peer_count);
319        next_peer_idx = to;
320
321        if from == to {
322            continue;
323        }
324
325        sent_total += send_wave(from..to).await;
326        if sent_total == 0 {
327            if next_peer_idx >= peer_count {
328                break;
329            }
330            continue;
331        }
332
333        let now = Instant::now();
334        if now >= deadline {
335            break;
336        }
337        let remaining = deadline.saturating_duration_since(now);
338        let is_last_wave = wave_idx + 1 == wave_plan.len() || next_peer_idx >= peer_count;
339        let wait = if is_last_wave {
340            remaining
341        } else if dispatch.hedge_interval_ms == 0 {
342            Duration::ZERO
343        } else {
344            Duration::from_millis(dispatch.hedge_interval_ms).min(remaining)
345        };
346
347        if wait.is_zero() {
348            continue;
349        }
350
351        match wait_wave(wait).await {
352            HedgedWaveAction::Continue => {}
353            HedgedWaveAction::Success(value) => return Some(value),
354            HedgedWaveAction::Abort => break,
355        }
356    }
357
358    None
359}
360
361/// Keep selector membership aligned with currently connected peer IDs.
362pub async fn sync_selector_peers(selector: &RwLock<PeerSelector>, current_peer_ids: &[String]) {
363    let mut selector = selector.write().await;
364    let current: HashSet<&str> = current_peer_ids.iter().map(String::as_str).collect();
365    let known: Vec<String> = selector.all_stats().map(|s| s.peer_id.clone()).collect();
366    for peer_id in known {
367        if !current.contains(peer_id.as_str()) {
368            selector.remove_peer(&peer_id);
369        }
370    }
371    for peer_id in current_peer_ids {
372        selector.add_peer(peer_id.clone());
373    }
374}
375
376/// Response behavior profile for simulation/game-theory actors.
377///
378/// Defaults to honest behavior (always respond correctly, no extra delay).
379#[derive(Debug, Clone, Copy)]
380pub struct ResponseBehaviorConfig {
381    /// Probability that a node drops a response even when it has data.
382    pub drop_response_prob: f64,
383    /// Probability that a node responds with corrupted payload.
384    pub corrupt_response_prob: f64,
385    /// Baseline response delay before a peer starts sending any data.
386    pub extra_delay_ms: u64,
387    /// Additional delay before the first response byte becomes available.
388    pub first_byte_delay_ms: u64,
389    /// Sustained throughput for delivering large payloads. `0` disables size-based slowdown.
390    pub bytes_per_second: u64,
391    /// Probability that an otherwise honest response experiences an extra stall.
392    pub stall_response_prob: f64,
393    /// Extra delay injected when a stall event happens.
394    pub stall_delay_ms: u64,
395}
396
397impl Default for ResponseBehaviorConfig {
398    fn default() -> Self {
399        Self {
400            drop_response_prob: 0.0,
401            corrupt_response_prob: 0.0,
402            extra_delay_ms: 0,
403            first_byte_delay_ms: 0,
404            bytes_per_second: 0,
405            stall_response_prob: 0.0,
406            stall_delay_ms: 0,
407        }
408    }
409}
410
411impl ResponseBehaviorConfig {
412    fn normalized(self) -> Self {
413        Self {
414            drop_response_prob: self.drop_response_prob.clamp(0.0, 1.0),
415            corrupt_response_prob: self.corrupt_response_prob.clamp(0.0, 1.0),
416            extra_delay_ms: self.extra_delay_ms,
417            first_byte_delay_ms: self.first_byte_delay_ms,
418            bytes_per_second: self.bytes_per_second,
419            stall_response_prob: self.stall_response_prob.clamp(0.0, 1.0),
420            stall_delay_ms: self.stall_delay_ms,
421        }
422    }
423}
424
425/// Routing policy for request ordering + dispatch fanout.
426#[derive(Debug, Clone)]
427pub struct MeshRoutingConfig {
428    pub selection_strategy: SelectionStrategy,
429    pub fairness_enabled: bool,
430    /// Blend weight for payment-priority ranking in selector (`0.0` disables).
431    pub cashu_payment_weight: f64,
432    /// Refuse serving peers that have reached this many unpaid post-delivery settlements.
433    /// `0` disables refusal and only keeps metadata/downranking.
434    pub cashu_payment_default_block_threshold: u64,
435    /// Cashu mint URLs this node is willing to use for settlement.
436    pub cashu_accepted_mints: Vec<String>,
437    /// Preferred Cashu mint URL when initiating paid retrieval.
438    pub cashu_default_mint: Option<String>,
439    /// Baseline cap for accepting a peer-suggested mint outside the trusted set.
440    pub cashu_peer_suggested_mint_base_cap_sat: u64,
441    /// Additional sats allowed per successful delivery from that peer.
442    pub cashu_peer_suggested_mint_success_step_sat: u64,
443    /// Additional sats allowed per successful post-delivery payment received from that peer.
444    pub cashu_peer_suggested_mint_receipt_step_sat: u64,
445    /// Hard upper bound for any single peer-suggested mint quote we accept.
446    pub cashu_peer_suggested_mint_max_cap_sat: u64,
447    pub dispatch: RequestDispatchConfig,
448    pub response_behavior: ResponseBehaviorConfig,
449    pub pubsub_scheduler: PubsubSchedulerConfig,
450    pub pubsub_delivery_mode: PubsubDeliveryMode,
451    /// Forward peer pubsub interests, inventories, and payloads for downstream peers.
452    pub pubsub_forwarding: bool,
453    /// Initial hops-to-live for locally originated pubsub interest/inventory frames.
454    pub pubsub_max_htl: u8,
455}
456
457impl Default for MeshRoutingConfig {
458    fn default() -> Self {
459        Self {
460            selection_strategy: SelectionStrategy::Weighted,
461            fairness_enabled: true,
462            cashu_payment_weight: 0.0,
463            cashu_payment_default_block_threshold: 0,
464            cashu_accepted_mints: Vec::new(),
465            cashu_default_mint: None,
466            cashu_peer_suggested_mint_base_cap_sat: 0,
467            cashu_peer_suggested_mint_success_step_sat: 0,
468            cashu_peer_suggested_mint_receipt_step_sat: 0,
469            cashu_peer_suggested_mint_max_cap_sat: 0,
470            dispatch: RequestDispatchConfig::default(),
471            response_behavior: ResponseBehaviorConfig::default(),
472            pubsub_scheduler: PubsubSchedulerConfig::default(),
473            pubsub_delivery_mode: PubsubDeliveryMode::HtlInvWant,
474            pubsub_forwarding: true,
475            pubsub_max_htl: MESH_EVENT_POLICY.max_htl,
476        }
477    }
478}
479
480impl MeshRoutingConfig {
481    fn pubsub_initial_htl(&self) -> u8 {
482        self.pubsub_max_htl.clamp(1, MAX_HTL)
483    }
484}
485
486/// Routed mesh store core that works with any storage backend and transport
487/// implementation.
488///
489/// This is the shared code between production and simulation.
490/// - Production: transport-specific crates compose `MeshStoreCore` with their links
491/// - Simulation: `MeshStoreCore<MemoryStore, MockRelayTransport, MockConnectionFactory>`
492pub struct MeshStoreCore<S, R, F>
493where
494    S: Store + Send + Sync + 'static,
495    R: SignalingTransport + Send + Sync + 'static,
496    F: PeerLinkFactory + Send + Sync + 'static,
497{
498    /// Local backing store
499    local_store: Arc<S>,
500    /// Mesh router (handles peer discovery and connection)
501    signaling: Arc<MeshRouter<R, F>>,
502    /// Per-peer HTL config
503    htl_configs: RwLock<HashMap<String, PeerHTLConfig>>,
504    /// Pending requests we sent
505    pending_requests: RwLock<HashMap<PendingRequestKey, Vec<PendingRequest>>>,
506    /// Pending quote negotiations keyed by requested hash.
507    pending_quotes: RwLock<HashMap<String, PendingQuoteRequest>>,
508    /// Forwarded peer requests currently being resolved through the mesh/upstream.
509    pending_forward_requests: RwLock<HashMap<PendingRequestKey, PendingForwardRequest>>,
510    /// Bounded negative cache for recently forwarded misses/timeouts.
511    /// Quotes we issued to peers and will accept exactly once until expiry.
512    issued_quotes: RwLock<HashMap<(String, String, u64), IssuedQuote>>,
513    /// Monotonic quote identifier generator.
514    next_quote_id: RwLock<u64>,
515    /// Adaptive selector for peer ordering.
516    peer_selector: RwLock<PeerSelector>,
517    /// Active per-peer in-flight reads so concurrent block fetches spread across peers.
518    peer_active_requests: RwLock<HashMap<String, usize>>,
519    /// Actual wire traffic stats used for upload-side reciprocity scheduling.
520    peer_wire_stats: RwLock<HashMap<String, PeerWireStats>>,
521    /// Streams this node wants delivered locally.
522    pubsub_local_interests: RwLock<HashSet<String>>,
523    /// Current sequence per local stream interest.
524    pubsub_local_interest_versions: RwLock<HashMap<String, u64>>,
525    /// Reverse pubsub routes: stream id -> peers with local/downstream interest.
526    pubsub_peer_interests: RwLock<HashMap<String, HashSet<String>>>,
527    /// Route owner for each downstream subscriber interest.
528    pubsub_interest_routes: RwLock<HashMap<(String, String), String>>,
529    /// Latest interest sequence observed per subscriber/stream.
530    pubsub_interest_versions: RwLock<HashMap<(String, String), u64>>,
531    /// Bounded dedupe for pubsub interest floods.
532    pubsub_seen_interests: Mutex<TimedSeenSet>,
533    /// Bounded dedupe for pubsub data frames.
534    pubsub_seen_frames: Mutex<TimedSeenSet>,
535    /// Bounded dedupe for pubsub inventory floods.
536    pubsub_seen_inventories: Mutex<TimedSeenSet>,
537    /// Bounded dedupe for pubsub wants by requesting peer.
538    pubsub_seen_wants: Mutex<TimedSeenSet>,
539    /// First upstream peer that announced each inventory key.
540    pubsub_inventory_routes: RwLock<HashMap<String, String>>,
541    /// Downstream peers waiting for a payload after sending a want.
542    pubsub_want_routes: RwLock<HashMap<String, HashSet<String>>>,
543    /// Dedupe for wants this node already sent upstream.
544    pubsub_upstream_wants: Mutex<TimedSeenSet>,
545    /// Small payload cache for serving wants after inventory-first announcements.
546    pubsub_frame_cache: Mutex<VecDeque<(String, PubsubFrame)>>,
547    /// Local pubsub delivery inbox.
548    pubsub_inbox: Mutex<VecDeque<PubsubEvent>>,
549    /// Bounded application-facing evidence for first-winner block deliveries.
550    verified_block_deliveries: Mutex<VerifiedBlockDeliveryBuffer>,
551    /// Wakes consumers waiting for local pubsub deliveries.
552    pubsub_notify: Notify,
553    /// Per stream/peer deferred counts for aging pubsub strategies.
554    pubsub_deferred_counts: RwLock<HashMap<(String, String), u64>>,
555    /// Monotonic sequence for locally originated pubsub interest updates.
556    next_pubsub_interest_seq: AtomicU64,
557    /// Pending content responses waiting for upload arbitration.
558    pending_response_sends: Mutex<Vec<PendingResponseSend>>,
559    /// Upload response scheduler state.
560    response_scheduler_running: AtomicBool,
561    /// Monotonic id for queued response sends.
562    next_response_job_id: AtomicU64,
563    /// Routing/dispatch configuration.
564    routing: MeshRoutingConfig,
565    /// Request timeout
566    request_timeout: Duration,
567    /// Debug mode
568    debug: bool,
569    /// Running flag
570    running: RwLock<bool>,
571}
572
573impl<S, R, F> MeshStoreCore<S, R, F>
574where
575    S: Store + Send + Sync + 'static,
576    R: SignalingTransport + Send + Sync + 'static,
577    F: PeerLinkFactory + Send + Sync + 'static,
578{
579    /// Create a new routed mesh store core.
580    pub fn new(
581        local_store: Arc<S>,
582        signaling: Arc<MeshRouter<R, F>>,
583        request_timeout: Duration,
584        debug: bool,
585    ) -> Self {
586        Self::new_with_routing(
587            local_store,
588            signaling,
589            request_timeout,
590            debug,
591            Default::default(),
592        )
593    }
594
595    /// Create a new routed mesh store core with explicit routing configuration.
596    pub fn new_with_routing(
597        local_store: Arc<S>,
598        signaling: Arc<MeshRouter<R, F>>,
599        request_timeout: Duration,
600        debug: bool,
601        routing: MeshRoutingConfig,
602    ) -> Self {
603        let mut selector = PeerSelector::with_strategy(routing.selection_strategy);
604        selector.set_fairness(routing.fairness_enabled);
605        selector.set_cashu_payment_weight(routing.cashu_payment_weight);
606        Self {
607            local_store,
608            signaling,
609            htl_configs: RwLock::new(HashMap::new()),
610            pending_requests: RwLock::new(HashMap::new()),
611            pending_quotes: RwLock::new(HashMap::new()),
612            pending_forward_requests: RwLock::new(HashMap::new()),
613            issued_quotes: RwLock::new(HashMap::new()),
614            next_quote_id: RwLock::new(1),
615            peer_selector: RwLock::new(selector),
616            peer_active_requests: RwLock::new(HashMap::new()),
617            peer_wire_stats: RwLock::new(HashMap::new()),
618            pubsub_local_interests: RwLock::new(HashSet::new()),
619            pubsub_local_interest_versions: RwLock::new(HashMap::new()),
620            pubsub_peer_interests: RwLock::new(HashMap::new()),
621            pubsub_interest_routes: RwLock::new(HashMap::new()),
622            pubsub_interest_versions: RwLock::new(HashMap::new()),
623            pubsub_seen_interests: Mutex::new(TimedSeenSet::new(
624                PUBSUB_SEEN_CAPACITY,
625                PUBSUB_SEEN_TTL,
626            )),
627            pubsub_seen_frames: Mutex::new(TimedSeenSet::new(
628                PUBSUB_SEEN_CAPACITY,
629                PUBSUB_SEEN_TTL,
630            )),
631            pubsub_seen_inventories: Mutex::new(TimedSeenSet::new(
632                PUBSUB_SEEN_CAPACITY,
633                PUBSUB_SEEN_TTL,
634            )),
635            pubsub_seen_wants: Mutex::new(TimedSeenSet::new(PUBSUB_SEEN_CAPACITY, PUBSUB_SEEN_TTL)),
636            pubsub_inventory_routes: RwLock::new(HashMap::new()),
637            pubsub_want_routes: RwLock::new(HashMap::new()),
638            pubsub_upstream_wants: Mutex::new(TimedSeenSet::new(
639                PUBSUB_SEEN_CAPACITY,
640                PUBSUB_SEEN_TTL,
641            )),
642            pubsub_frame_cache: Mutex::new(VecDeque::new()),
643            pubsub_inbox: Mutex::new(VecDeque::new()),
644            verified_block_deliveries: Mutex::new(VerifiedBlockDeliveryBuffer::default()),
645            pubsub_notify: Notify::new(),
646            pubsub_deferred_counts: RwLock::new(HashMap::new()),
647            next_pubsub_interest_seq: AtomicU64::new(1),
648            pending_response_sends: Mutex::new(Vec::new()),
649            response_scheduler_running: AtomicBool::new(false),
650            next_response_job_id: AtomicU64::new(1),
651            routing,
652            request_timeout,
653            debug,
654            running: RwLock::new(false),
655        }
656    }
657
658    /// Start the store (begin listening for messages)
659    pub async fn start(&self) -> Result<(), TransportError> {
660        *self.running.write().await = true;
661
662        // Send initial hello
663        self.signaling.send_hello(vec![]).await?;
664
665        Ok(())
666    }
667
668    /// Stop the store
669    pub async fn stop(&self) {
670        *self.running.write().await = false;
671    }
672
673    /// Process incoming signaling message
674    pub async fn process_signaling(&self, msg: SignalingMessage) -> Result<(), TransportError> {
675        // When a new peer connects, initialize their HTL config
676        let peer_id = msg.peer_id().to_string();
677        {
678            let mut configs = self.htl_configs.write().await;
679            if !configs.contains_key(&peer_id) {
680                configs.insert(peer_id.clone(), PeerHTLConfig::random());
681            }
682        }
683        self.peer_selector.write().await.add_peer(peer_id.clone());
684
685        let result = self.signaling.handle_message(msg).await;
686        if result.is_ok() {
687            self.announce_pubsub_interests_to_peer(&peer_id).await;
688        }
689        result
690    }
691
692    /// Get signaling manager reference
693    pub fn signaling(&self) -> &Arc<MeshRouter<R, F>> {
694        &self.signaling
695    }
696
697    fn response_behavior(&self) -> ResponseBehaviorConfig {
698        self.routing.response_behavior.normalized()
699    }
700
701    async fn record_peer_wire_sent(&self, peer_id: &str, bytes: u64) {
702        if bytes == 0 {
703            return;
704        }
705        let mut stats = self.peer_wire_stats.write().await;
706        let entry = stats.entry(peer_id.to_string()).or_default();
707        entry.bytes_sent = entry.bytes_sent.saturating_add(bytes);
708    }
709
710    async fn record_peer_wire_received(&self, peer_id: &str, bytes: u64) {
711        if bytes == 0 {
712            return;
713        }
714        let mut stats = self.peer_wire_stats.write().await;
715        let entry = stats.entry(peer_id.to_string()).or_default();
716        entry.bytes_received = entry.bytes_received.saturating_add(bytes);
717    }
718
719    /// Record ingress from a peer that matched local or downstream demand.
720    ///
721    /// Raw bytes are tracked separately in `record_peer_wire_received`; this
722    /// counter is the reciprocity signal used by shared outbound scheduling.
723    pub async fn record_useful_bytes_received_from_peer(&self, peer_id: &str, bytes: u64) {
724        if bytes == 0 {
725            return;
726        }
727        let mut stats = self.peer_wire_stats.write().await;
728        let entry = stats.entry(peer_id.to_string()).or_default();
729        entry.useful_bytes_received = entry.useful_bytes_received.saturating_add(bytes);
730    }
731
732    /// Snapshot peer traffic for production pubsub scheduling or diagnostics.
733    pub async fn peer_traffic_snapshot(&self, peer_id: &str) -> PeerTrafficSnapshot {
734        self.peer_wire_stats
735            .read()
736            .await
737            .get(peer_id)
738            .copied()
739            .unwrap_or_default()
740    }
741
742    /// Snapshot all known peer traffic for production pubsub scheduling.
743    pub async fn peer_traffic_snapshots(&self) -> HashMap<String, PeerTrafficSnapshot> {
744        self.peer_wire_stats.read().await.clone()
745    }
746
747    fn pubsub_key(origin_peer_id: &str, stream_id: &str, seq: u64) -> String {
748        format!("{origin_peer_id}:{stream_id}:{seq}")
749    }
750
751    fn pubsub_frame_key(frame: &PubsubFrame) -> String {
752        Self::pubsub_key(&frame.origin_peer_id, &frame.stream_id, frame.seq)
753    }
754
755    fn pubsub_interest_key(interest: &PubsubInterest) -> String {
756        format!(
757            "{}:{}:{}:{}",
758            interest.subscriber_peer_id, interest.stream_id, interest.seq, interest.active
759        )
760    }
761
762    fn next_pubsub_interest_seq(&self) -> u64 {
763        self.next_pubsub_interest_seq
764            .fetch_add(1, Ordering::Relaxed)
765    }
766
767    async fn record_peer_pubsub_wire_sent(&self, peer_id: &str, bytes: u64, bandwidth_debt: f64) {
768        if bytes == 0 {
769            return;
770        }
771        let mut stats = self.peer_wire_stats.write().await;
772        let entry = stats.entry(peer_id.to_string()).or_default();
773        entry.bytes_sent = entry.bytes_sent.saturating_add(bytes);
774        entry.bandwidth_debt = bandwidth_debt;
775    }
776
777    async fn send_pubsub_interest_to_peers(
778        &self,
779        interest: &PubsubInterest,
780        exclude_peer_id: Option<&str>,
781    ) -> PubsubPublishStats {
782        if !should_forward_htl(interest.htl) {
783            return PubsubPublishStats::default();
784        }
785
786        let mut peer_ids = self.signaling.peer_ids().await;
787        peer_ids.sort();
788        peer_ids.retain(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude));
789
790        let bytes = encode_pubsub_interest(interest);
791        let mut stats = PubsubPublishStats {
792            selected_peers: peer_ids.len(),
793            ..Default::default()
794        };
795        for peer_id in peer_ids {
796            let Some(channel) = self.signaling.get_channel(&peer_id).await else {
797                continue;
798            };
799            if channel.send(bytes.clone()).await.is_ok() {
800                stats.sent_peers += 1;
801                stats.sent_bytes = stats.sent_bytes.saturating_add(bytes.len() as u64);
802                self.record_peer_wire_sent(&peer_id, bytes.len() as u64)
803                    .await;
804            }
805        }
806        stats
807    }
808
809    async fn announce_pubsub_interests_to_peer(&self, peer_id: &str) {
810        let mut interests = self
811            .pubsub_local_interests
812            .read()
813            .await
814            .iter()
815            .cloned()
816            .collect::<Vec<_>>();
817        interests.sort();
818        if interests.is_empty() {
819            return;
820        }
821
822        let interests = {
823            let versions = self.pubsub_local_interest_versions.read().await;
824            interests
825                .into_iter()
826                .filter_map(|stream_id| {
827                    versions
828                        .get(&stream_id)
829                        .copied()
830                        .map(|seq| (stream_id, seq))
831                })
832                .collect::<Vec<_>>()
833        };
834
835        for (stream_id, seq) in interests {
836            let interest = create_pubsub_interest(
837                stream_id,
838                self.signaling.peer_id().to_string(),
839                seq,
840                true,
841                MAX_HTL,
842            );
843            let Some(channel) = self.signaling.get_channel(peer_id).await else {
844                continue;
845            };
846            let bytes = encode_pubsub_interest(&interest);
847            if channel.send(bytes.clone()).await.is_ok() {
848                self.record_peer_wire_sent(peer_id, bytes.len() as u64)
849                    .await;
850            }
851        }
852    }
853
854    fn remove_pubsub_peer_interest(
855        peer_interests: &mut HashMap<String, HashSet<String>>,
856        routes: &HashMap<(String, String), String>,
857        stream_id: &str,
858        peer_id: &str,
859    ) {
860        let still_has_route = routes
861            .iter()
862            .any(|((stream, _subscriber), peer)| stream == stream_id && peer == peer_id);
863        if still_has_route {
864            return;
865        }
866        if let Some(peers) = peer_interests.get_mut(stream_id) {
867            peers.remove(peer_id);
868            if peers.is_empty() {
869                peer_interests.remove(stream_id);
870            }
871        }
872    }
873
874    async fn apply_pubsub_interest_route(
875        &self,
876        from_peer: &str,
877        interest: &PubsubInterest,
878    ) -> bool {
879        if interest.stream_id.is_empty() || interest.subscriber_peer_id.is_empty() {
880            return false;
881        }
882        if interest.subscriber_peer_id == self.signaling.peer_id() {
883            return false;
884        }
885
886        let interest_key = Self::pubsub_interest_key(interest);
887        if !self
888            .pubsub_seen_interests
889            .lock()
890            .await
891            .insert_if_new(interest_key)
892        {
893            return false;
894        }
895
896        let route_key = (
897            interest.stream_id.clone(),
898            interest.subscriber_peer_id.clone(),
899        );
900        {
901            let mut versions = self.pubsub_interest_versions.write().await;
902            if versions
903                .get(&route_key)
904                .is_some_and(|latest| *latest >= interest.seq)
905            {
906                return false;
907            }
908            versions.insert(route_key.clone(), interest.seq);
909        }
910
911        let mut peer_interests = self.pubsub_peer_interests.write().await;
912        let mut routes = self.pubsub_interest_routes.write().await;
913        if interest.active {
914            if let Some(previous_peer) = routes.insert(route_key, from_peer.to_string()) {
915                if previous_peer != from_peer {
916                    Self::remove_pubsub_peer_interest(
917                        &mut peer_interests,
918                        &routes,
919                        &interest.stream_id,
920                        &previous_peer,
921                    );
922                }
923            }
924            peer_interests
925                .entry(interest.stream_id.clone())
926                .or_default()
927                .insert(from_peer.to_string());
928        } else if let Some(previous_peer) = routes.remove(&route_key) {
929            Self::remove_pubsub_peer_interest(
930                &mut peer_interests,
931                &routes,
932                &interest.stream_id,
933                &previous_peer,
934            );
935        } else {
936            Self::remove_pubsub_peer_interest(
937                &mut peer_interests,
938                &routes,
939                &interest.stream_id,
940                from_peer,
941            );
942        }
943
944        true
945    }
946
947    async fn interested_pubsub_peers(
948        &self,
949        stream_id: &str,
950        exclude_peer_id: Option<&str>,
951    ) -> Vec<String> {
952        let connected = self
953            .signaling
954            .peer_ids()
955            .await
956            .into_iter()
957            .collect::<HashSet<_>>();
958        let mut peers = self
959            .pubsub_peer_interests
960            .read()
961            .await
962            .get(stream_id)
963            .map(|peers| peers.iter().cloned().collect::<Vec<_>>())
964            .unwrap_or_default();
965        peers.retain(|peer_id| {
966            connected.contains(peer_id) && exclude_peer_id.is_none_or(|exclude| peer_id != exclude)
967        });
968        peers.sort();
969        peers
970    }
971
972    async fn decrement_pubsub_htl_for_peer(&self, peer_id: &str, htl: u8) -> u8 {
973        let htl_config = {
974            let configs = self.htl_configs.read().await;
975            configs
976                .get(peer_id)
977                .cloned()
978                .unwrap_or_else(PeerHTLConfig::random)
979        };
980        htl_config.decrement_with_policy(htl, &MESH_EVENT_POLICY)
981    }
982
983    async fn send_pubsub_inventory_to_peers(
984        &self,
985        inv: &PubsubInventory,
986        peer_ids: &[String],
987    ) -> PubsubPublishStats {
988        if peer_ids.is_empty() || !should_forward_htl(inv.htl) {
989            return PubsubPublishStats::default();
990        }
991
992        let mut stats = PubsubPublishStats {
993            selected_peers: peer_ids.len(),
994            ..Default::default()
995        };
996        for peer_id in peer_ids {
997            let send_htl = self.decrement_pubsub_htl_for_peer(peer_id, inv.htl).await;
998            if !should_forward_htl(send_htl) {
999                continue;
1000            }
1001            let Some(channel) = self.signaling.get_channel(peer_id).await else {
1002                continue;
1003            };
1004            let mut outgoing = inv.clone();
1005            outgoing.htl = send_htl;
1006            let bytes = encode_pubsub_inventory(&outgoing);
1007            let message_bytes = bytes.len() as u64;
1008            if channel.send(bytes).await.is_ok() {
1009                stats.sent_peers += 1;
1010                stats.sent_bytes = stats.sent_bytes.saturating_add(message_bytes);
1011                self.record_peer_wire_sent(peer_id, message_bytes).await;
1012            }
1013        }
1014        stats
1015    }
1016
1017    async fn send_pubsub_want_to_peer(&self, want: &PubsubWant, peer_id: &str) -> bool {
1018        let Some(channel) = self.signaling.get_channel(peer_id).await else {
1019            return false;
1020        };
1021        let bytes = encode_pubsub_want(want);
1022        let message_bytes = bytes.len() as u64;
1023        match channel.send(bytes).await {
1024            Ok(()) => {
1025                self.record_peer_wire_sent(peer_id, message_bytes).await;
1026                true
1027            }
1028            Err(_) => false,
1029        }
1030    }
1031
1032    async fn send_pubsub_want_upstream(
1033        &self,
1034        key: &str,
1035        want: &PubsubWant,
1036        exclude_peer_id: Option<&str>,
1037    ) -> bool {
1038        let upstream = {
1039            let routes = self.pubsub_inventory_routes.read().await;
1040            routes.get(key).cloned()
1041        };
1042        let Some(upstream) = upstream else {
1043            return false;
1044        };
1045        if exclude_peer_id.is_some_and(|exclude| exclude == upstream) {
1046            return false;
1047        }
1048        let want_key = format!("{key}:{upstream}");
1049        if !self
1050            .pubsub_upstream_wants
1051            .lock()
1052            .await
1053            .insert_if_new(want_key)
1054        {
1055            return false;
1056        }
1057        self.send_pubsub_want_to_peer(want, &upstream).await
1058    }
1059
1060    async fn cache_pubsub_frame(&self, key: String, frame: PubsubFrame) {
1061        let mut cache = self.pubsub_frame_cache.lock().await;
1062        if let Some(index) = cache.iter().position(|(cached_key, _)| cached_key == &key) {
1063            cache.remove(index);
1064        }
1065        cache.push_back((key, frame));
1066        while cache.len() > PUBSUB_FRAME_CACHE_CAPACITY {
1067            cache.pop_front();
1068        }
1069    }
1070
1071    async fn cached_pubsub_frame(&self, key: &str) -> Option<PubsubFrame> {
1072        self.pubsub_frame_cache
1073            .lock()
1074            .await
1075            .iter()
1076            .find_map(|(cached_key, frame)| {
1077                if cached_key == key {
1078                    Some(frame.clone())
1079                } else {
1080                    None
1081                }
1082            })
1083    }
1084
1085    async fn remember_pubsub_want_peer(&self, key: String, from_peer: &str) -> bool {
1086        let mut routes = self.pubsub_want_routes.write().await;
1087        routes.entry(key).or_default().insert(from_peer.to_string())
1088    }
1089
1090    async fn take_pubsub_want_peers(
1091        &self,
1092        key: &str,
1093        exclude_peer_id: Option<&str>,
1094    ) -> Vec<String> {
1095        let connected = self
1096            .signaling
1097            .peer_ids()
1098            .await
1099            .into_iter()
1100            .collect::<HashSet<_>>();
1101        let mut peers = self
1102            .pubsub_want_routes
1103            .write()
1104            .await
1105            .remove(key)
1106            .map(|peers| peers.into_iter().collect::<Vec<_>>())
1107            .unwrap_or_default();
1108        peers.retain(|peer_id| {
1109            connected.contains(peer_id) && exclude_peer_id.is_none_or(|exclude| peer_id != exclude)
1110        });
1111        peers.sort();
1112        peers
1113    }
1114
1115    async fn select_pubsub_peers(
1116        &self,
1117        stream_id: &str,
1118        seq: u64,
1119        message_bytes: u64,
1120        peer_ids: &[String],
1121    ) -> (Vec<String>, Vec<String>) {
1122        let traffic = self.peer_wire_stats.read().await;
1123        let deferred_counts = self.pubsub_deferred_counts.read().await;
1124        let candidates = peer_ids
1125            .iter()
1126            .map(|peer_id| PubsubCandidate {
1127                peer_id: peer_id.clone(),
1128                traffic: traffic.get(peer_id).copied().unwrap_or_default(),
1129                deferred_count: deferred_counts
1130                    .get(&(stream_id.to_string(), peer_id.clone()))
1131                    .copied()
1132                    .unwrap_or_default(),
1133            })
1134            .collect::<Vec<_>>();
1135        drop(deferred_counts);
1136        drop(traffic);
1137
1138        let selection = self.routing.pubsub_scheduler.select(
1139            stream_id,
1140            seq,
1141            self.signaling.peer_id(),
1142            message_bytes,
1143            &candidates,
1144        );
1145
1146        {
1147            let mut deferred_counts = self.pubsub_deferred_counts.write().await;
1148            for peer_id in &selection.deferred {
1149                *deferred_counts
1150                    .entry((stream_id.to_string(), peer_id.clone()))
1151                    .or_insert(0) += 1;
1152            }
1153            for peer_id in &selection.selected {
1154                deferred_counts.remove(&(stream_id.to_string(), peer_id.clone()));
1155            }
1156        }
1157
1158        (selection.selected, selection.deferred)
1159    }
1160
1161    async fn send_pubsub_frame_to_peers(
1162        &self,
1163        frame: &PubsubFrame,
1164        peer_ids: &[String],
1165    ) -> PubsubPublishStats {
1166        if peer_ids.is_empty() || !should_forward_htl(frame.htl) {
1167            return PubsubPublishStats::default();
1168        }
1169
1170        let bytes = encode_pubsub_frame(frame);
1171        let message_bytes = bytes.len() as u64;
1172        let (selected, deferred) = self
1173            .select_pubsub_peers(&frame.stream_id, frame.seq, message_bytes, peer_ids)
1174            .await;
1175        let mut stats = PubsubPublishStats {
1176            selected_peers: selected.len(),
1177            deferred_peers: deferred.len(),
1178            ..Default::default()
1179        };
1180
1181        for peer_id in selected {
1182            let Some(channel) = self.signaling.get_channel(&peer_id).await else {
1183                continue;
1184            };
1185            let snapshot = self.peer_traffic_snapshot(&peer_id).await;
1186            let bandwidth_debt = reciprocal_virtual_finish(snapshot, message_bytes);
1187            if channel.send(bytes.clone()).await.is_ok() {
1188                stats.sent_peers += 1;
1189                stats.sent_bytes = stats.sent_bytes.saturating_add(message_bytes);
1190                self.record_peer_pubsub_wire_sent(&peer_id, message_bytes, bandwidth_debt)
1191                    .await;
1192            }
1193        }
1194
1195        stats
1196    }
1197
1198    async fn enqueue_pubsub_event(&self, event: PubsubEvent) {
1199        let mut inbox = self.pubsub_inbox.lock().await;
1200        inbox.push_back(event);
1201        while inbox.len() > PUBSUB_INBOX_CAPACITY {
1202            inbox.pop_front();
1203        }
1204        self.pubsub_notify.notify_one();
1205    }
1206
1207    /// Subscribe this node to a pubsub stream and advertise that interest.
1208    pub async fn subscribe_pubsub(
1209        self: &Arc<Self>,
1210        stream_id: impl Into<String>,
1211    ) -> PubsubPublishStats {
1212        let stream_id = stream_id.into();
1213        if stream_id.is_empty() {
1214            return PubsubPublishStats::default();
1215        }
1216        self.pubsub_local_interests
1217            .write()
1218            .await
1219            .insert(stream_id.clone());
1220        let seq = {
1221            let mut versions = self.pubsub_local_interest_versions.write().await;
1222            match versions.get(&stream_id).copied() {
1223                Some(seq) => seq,
1224                None => {
1225                    let seq = self.next_pubsub_interest_seq();
1226                    versions.insert(stream_id.clone(), seq);
1227                    seq
1228                }
1229            }
1230        };
1231        let interest = create_pubsub_interest(
1232            stream_id,
1233            self.signaling.peer_id().to_string(),
1234            seq,
1235            true,
1236            self.routing.pubsub_initial_htl(),
1237        );
1238        self.send_pubsub_interest_to_peers(&interest, None).await
1239    }
1240
1241    /// Stop local delivery for a pubsub stream and advertise the withdrawn interest.
1242    pub async fn unsubscribe_pubsub(
1243        self: &Arc<Self>,
1244        stream_id: impl Into<String>,
1245    ) -> PubsubPublishStats {
1246        let stream_id = stream_id.into();
1247        if stream_id.is_empty() {
1248            return PubsubPublishStats::default();
1249        }
1250        self.pubsub_local_interests.write().await.remove(&stream_id);
1251        self.pubsub_local_interest_versions
1252            .write()
1253            .await
1254            .remove(&stream_id);
1255        let interest = create_pubsub_interest(
1256            stream_id,
1257            self.signaling.peer_id().to_string(),
1258            self.next_pubsub_interest_seq(),
1259            false,
1260            self.routing.pubsub_initial_htl(),
1261        );
1262        self.send_pubsub_interest_to_peers(&interest, None).await
1263    }
1264
1265    /// Publish bytes on a pubsub stream through the configured mesh delivery mode.
1266    pub async fn publish_pubsub(
1267        self: &Arc<Self>,
1268        stream_id: impl Into<String>,
1269        seq: u64,
1270        payload: Vec<u8>,
1271    ) -> PubsubPublishStats {
1272        let stream_id = stream_id.into();
1273        if stream_id.is_empty() {
1274            return PubsubPublishStats::default();
1275        }
1276        let payload_bytes = payload.len() as u64;
1277        let frame = create_pubsub_frame(
1278            stream_id.clone(),
1279            seq,
1280            self.signaling.peer_id().to_string(),
1281            payload.clone(),
1282            self.routing.pubsub_initial_htl(),
1283        );
1284        let frame_key = Self::pubsub_frame_key(&frame);
1285        self.pubsub_seen_frames
1286            .lock()
1287            .await
1288            .insert_if_new(frame_key.clone());
1289        self.cache_pubsub_frame(frame_key, frame.clone()).await;
1290
1291        if self
1292            .pubsub_local_interests
1293            .read()
1294            .await
1295            .contains(&stream_id)
1296        {
1297            self.enqueue_pubsub_event(PubsubEvent {
1298                stream_id: stream_id.clone(),
1299                seq,
1300                origin_peer_id: self.signaling.peer_id().to_string(),
1301                from_peer_id: self.signaling.peer_id().to_string(),
1302                payload,
1303            })
1304            .await;
1305        }
1306
1307        match self.routing.pubsub_delivery_mode {
1308            PubsubDeliveryMode::InterestPush => {
1309                let peers = self.interested_pubsub_peers(&stream_id, None).await;
1310                self.send_pubsub_frame_to_peers(&frame, &peers).await
1311            }
1312            PubsubDeliveryMode::HtlInvWant => {
1313                let inv = create_pubsub_inventory(
1314                    stream_id,
1315                    seq,
1316                    self.signaling.peer_id().to_string(),
1317                    payload_bytes,
1318                    self.routing.pubsub_initial_htl(),
1319                );
1320                let peers = self.interested_pubsub_peers(&inv.stream_id, None).await;
1321                self.send_pubsub_inventory_to_peers(&inv, &peers).await
1322            }
1323        }
1324    }
1325
1326    /// Drain locally delivered pubsub events.
1327    pub async fn drain_pubsub_events(&self) -> Vec<PubsubEvent> {
1328        self.pubsub_inbox.lock().await.drain(..).collect()
1329    }
1330
1331    /// Drain verified first-winner block deliveries for an application adapter.
1332    pub async fn drain_verified_block_deliveries(&self) -> VerifiedBlockDeliveryBatch {
1333        let mut buffer = self.verified_block_deliveries.lock().await;
1334        VerifiedBlockDeliveryBatch {
1335            deliveries: buffer.deliveries.drain(..).collect(),
1336            dropped_since_last_drain: std::mem::take(&mut buffer.dropped_since_last_drain),
1337        }
1338    }
1339
1340    /// Wait until a locally delivered pubsub event is available, then return it.
1341    pub async fn recv_pubsub_event(&self) -> PubsubEvent {
1342        loop {
1343            if let Some(event) = self.pubsub_inbox.lock().await.pop_front() {
1344                return event;
1345            }
1346            self.pubsub_notify.notified().await;
1347        }
1348    }
1349
1350    /// Connected peers that currently have local or downstream interest in a stream.
1351    pub async fn pubsub_interest_peers(&self, stream_id: &str) -> Vec<String> {
1352        self.interested_pubsub_peers(stream_id, None).await
1353    }
1354
1355    fn choose_ready_response_job(
1356        ready_jobs: &[(u64, String, usize, Instant, u64)],
1357        stats: &HashMap<String, PeerWireStats>,
1358    ) -> Option<(u64, f64)> {
1359        let jobs = ready_jobs
1360            .iter()
1361            .map(|job| OutboundJobCandidate {
1362                job_id: job.0,
1363                peer_id: job.1.clone(),
1364                message_bytes: job.2 as u64,
1365                queue_sequence: job.4,
1366            })
1367            .collect::<Vec<_>>();
1368        select_reciprocal_outbound_job(&jobs, |peer_id| {
1369            stats.get(peer_id).copied().unwrap_or_default()
1370        })
1371        .map(|choice| (choice.job_id, choice.virtual_finish))
1372    }
1373
1374    async fn enqueue_response_send(
1375        self: &Arc<Self>,
1376        peer_id: String,
1377        bytes: Vec<u8>,
1378        ready_at: Instant,
1379    ) {
1380        let job_id = self.next_response_job_id.fetch_add(1, Ordering::Relaxed);
1381        {
1382            let mut queue = self.pending_response_sends.lock().await;
1383            queue.push(PendingResponseSend {
1384                job_id,
1385                peer_id,
1386                bytes,
1387                ready_at,
1388                queue_sequence: job_id,
1389            });
1390        }
1391
1392        if self
1393            .response_scheduler_running
1394            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
1395            .is_ok()
1396        {
1397            let this = Arc::clone(self);
1398            tokio::spawn(async move {
1399                this.run_response_scheduler().await;
1400            });
1401        }
1402    }
1403
1404    async fn run_response_scheduler(self: Arc<Self>) {
1405        loop {
1406            let snapshot = {
1407                let queue = self.pending_response_sends.lock().await;
1408                if queue.is_empty() {
1409                    self.response_scheduler_running
1410                        .store(false, Ordering::Release);
1411                    return;
1412                }
1413                queue
1414                    .iter()
1415                    .map(|job| {
1416                        (
1417                            job.job_id,
1418                            job.peer_id.clone(),
1419                            job.bytes.len(),
1420                            job.ready_at,
1421                            job.queue_sequence,
1422                        )
1423                    })
1424                    .collect::<Vec<_>>()
1425            };
1426
1427            let now = Instant::now();
1428            let mut earliest_ready_at: Option<Instant> = None;
1429            let mut ready_jobs = Vec::new();
1430            for job in &snapshot {
1431                if job.3 <= now {
1432                    ready_jobs.push(job.clone());
1433                } else {
1434                    earliest_ready_at = Some(match earliest_ready_at {
1435                        Some(current) => current.min(job.3),
1436                        None => job.3,
1437                    });
1438                }
1439            }
1440
1441            if ready_jobs.is_empty() {
1442                if let Some(ready_at) = earliest_ready_at {
1443                    tokio::time::sleep(ready_at.saturating_duration_since(Instant::now())).await;
1444                    continue;
1445                }
1446                self.response_scheduler_running
1447                    .store(false, Ordering::Release);
1448                return;
1449            }
1450
1451            let (selected_job_id, selected_finish) = {
1452                let stats = self.peer_wire_stats.read().await;
1453                Self::choose_ready_response_job(&ready_jobs, &stats).expect("ready response job")
1454            };
1455
1456            let selected = {
1457                let mut queue = self.pending_response_sends.lock().await;
1458                let Some(index) = queue.iter().position(|job| job.job_id == selected_job_id) else {
1459                    continue;
1460                };
1461                queue.swap_remove(index)
1462            };
1463
1464            let sent = if let Some(channel) = self.signaling.get_channel(&selected.peer_id).await {
1465                channel.send(selected.bytes.clone()).await.is_ok()
1466            } else {
1467                false
1468            };
1469
1470            let queued_peers = {
1471                let queue = self.pending_response_sends.lock().await;
1472                queue
1473                    .iter()
1474                    .map(|job| job.peer_id.clone())
1475                    .collect::<HashSet<_>>()
1476            };
1477            let mut stats = self.peer_wire_stats.write().await;
1478            let entry = stats.entry(selected.peer_id.clone()).or_default();
1479            if sent {
1480                entry.bytes_sent = entry.bytes_sent.saturating_add(selected.bytes.len() as u64);
1481                entry.bandwidth_debt = selected_finish;
1482            }
1483            if queued_peers.is_empty() {
1484                for peer_stats in stats.values_mut() {
1485                    peer_stats.bandwidth_debt = 0.0;
1486                }
1487            } else {
1488                let floor = queued_peers
1489                    .iter()
1490                    .filter_map(|peer_id| stats.get(peer_id).map(|peer| peer.bandwidth_debt))
1491                    .fold(f64::INFINITY, f64::min);
1492                if floor.is_finite() && floor > 0.0 {
1493                    for peer_id in queued_peers {
1494                        if let Some(peer_stats) = stats.get_mut(&peer_id) {
1495                            peer_stats.bandwidth_debt =
1496                                (peer_stats.bandwidth_debt - floor).max(0.0);
1497                        }
1498                    }
1499                }
1500            }
1501        }
1502    }
1503
1504    fn deterministic_actor_draw_for(peer_id: &str, hash: &Hash, salt: u64) -> f64 {
1505        let mut hasher = DefaultHasher::new();
1506        peer_id.hash(&mut hasher);
1507        hash.hash(&mut hasher);
1508        salt.hash(&mut hasher);
1509        let v = hasher.finish();
1510        (v as f64) / (u64::MAX as f64)
1511    }
1512
1513    fn deterministic_actor_draw(&self, hash: &Hash, salt: u64) -> f64 {
1514        Self::deterministic_actor_draw_for(self.signaling.peer_id(), hash, salt)
1515    }
1516
1517    fn peer_metadata_pointer_slot_hash() -> Hash {
1518        hashtree_core::sha256(PEER_METADATA_POINTER_SLOT_KEY)
1519    }
1520
1521    fn decode_hash_hex(hash_hex: &str) -> Result<Hash, StoreError> {
1522        let bytes = hex::decode(hash_hex)
1523            .map_err(|e| StoreError::Other(format!("Invalid hash hex: {e}")))?;
1524        if bytes.len() != 32 {
1525            return Err(StoreError::Other(format!(
1526                "Invalid hash length {}, expected 32 bytes",
1527                bytes.len()
1528            )));
1529        }
1530        let mut hash = [0u8; 32];
1531        hash.copy_from_slice(&bytes);
1532        Ok(hash)
1533    }
1534
1535    fn should_drop_response(&self, hash: &Hash) -> bool {
1536        let p = self.response_behavior().drop_response_prob;
1537        if p <= 0.0 {
1538            return false;
1539        }
1540        self.deterministic_actor_draw(hash, 0xD0_D0_D0_D0_D0_D0_D0_D0) < p
1541    }
1542
1543    fn should_corrupt_response(&self, hash: &Hash) -> bool {
1544        let p = self.response_behavior().corrupt_response_prob;
1545        if p <= 0.0 {
1546            return false;
1547        }
1548        self.deterministic_actor_draw(hash, 0xC0_C0_C0_C0_C0_C0_C0_C0) < p
1549    }
1550
1551    fn should_stall_response(&self, hash: &Hash) -> bool {
1552        let p = self.response_behavior().stall_response_prob;
1553        if p <= 0.0 {
1554            return false;
1555        }
1556        self.deterministic_actor_draw(hash, 0x5A_11_5A_11_5A_11_5A_11) < p
1557    }
1558
1559    fn response_send_delay(&self, hash: &Hash, payload_len: usize) -> Duration {
1560        let behavior = self.response_behavior();
1561        let mut total_ms = behavior
1562            .extra_delay_ms
1563            .saturating_add(behavior.first_byte_delay_ms);
1564
1565        if behavior.bytes_per_second > 0 && payload_len > 0 {
1566            let throughput_ms = ((payload_len as u128) * 1000)
1567                .div_ceil(behavior.bytes_per_second as u128)
1568                .min(u64::MAX as u128) as u64;
1569            total_ms = total_ms.saturating_add(throughput_ms);
1570        }
1571
1572        if behavior.stall_delay_ms > 0 && self.should_stall_response(hash) {
1573            total_ms = total_ms.saturating_add(behavior.stall_delay_ms);
1574        }
1575
1576        Duration::from_millis(total_ms)
1577    }
1578
1579    async fn ordered_connected_peers(&self, exclude_peer_id: Option<&str>) -> Vec<String> {
1580        let current_peer_ids = self.signaling.peer_ids().await;
1581        if current_peer_ids.is_empty() {
1582            return Vec::new();
1583        }
1584
1585        sync_selector_peers(&self.peer_selector, &current_peer_ids).await;
1586        let hash_get_peer_ids: HashSet<String> = self
1587            .signaling
1588            .hash_get_peer_ids()
1589            .await
1590            .into_iter()
1591            .collect();
1592        let mut candidate_peer_ids: Vec<String> = current_peer_ids
1593            .into_iter()
1594            .filter(|peer_id| hash_get_peer_ids.contains(peer_id))
1595            .filter(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude))
1596            .collect();
1597        if candidate_peer_ids.is_empty() {
1598            return Vec::new();
1599        }
1600
1601        let current_set: HashSet<&str> = candidate_peer_ids.iter().map(String::as_str).collect();
1602        let mut selector = self.peer_selector.write().await;
1603        let mut selector_order = selector.select_peers();
1604        selector_order.retain(|peer_id| current_set.contains(peer_id.as_str()));
1605        if selector_order.is_empty() {
1606            let mut fallback = candidate_peer_ids;
1607            fallback.sort();
1608            return fallback;
1609        }
1610        let backed_off: HashMap<String, bool> = candidate_peer_ids
1611            .iter()
1612            .map(|peer_id| (peer_id.clone(), selector.is_peer_backed_off(peer_id)))
1613            .collect();
1614        drop(selector);
1615
1616        let rank: HashMap<&str, usize> = selector_order
1617            .iter()
1618            .enumerate()
1619            .map(|(idx, peer_id)| (peer_id.as_str(), idx))
1620            .collect();
1621        let active = self.peer_active_requests.read().await;
1622        candidate_peer_ids.sort_by(|left, right| {
1623            let left_backed_off = backed_off.get(left).copied().unwrap_or(false);
1624            let right_backed_off = backed_off.get(right).copied().unwrap_or(false);
1625            if left_backed_off != right_backed_off {
1626                return if left_backed_off {
1627                    std::cmp::Ordering::Greater
1628                } else {
1629                    std::cmp::Ordering::Less
1630                };
1631            }
1632            let left_rank = rank.get(left.as_str()).copied().unwrap_or(usize::MAX / 2);
1633            let right_rank = rank.get(right.as_str()).copied().unwrap_or(usize::MAX / 2);
1634            let left_load = active.get(left).copied().unwrap_or(0);
1635            let right_load = active.get(right).copied().unwrap_or(0);
1636            (left_rank + left_load.saturating_mul(ACTIVE_PEER_REQUEST_RANK_PENALTY))
1637                .cmp(&(right_rank + right_load.saturating_mul(ACTIVE_PEER_REQUEST_RANK_PENALTY)))
1638                .then_with(|| left.cmp(right))
1639        });
1640        candidate_peer_ids
1641    }
1642
1643    async fn reserve_peer_request(&self, peer_id: &str) {
1644        let mut active = self.peer_active_requests.write().await;
1645        *active.entry(peer_id.to_string()).or_insert(0) += 1;
1646    }
1647
1648    async fn release_peer_request(&self, peer_id: &str) {
1649        let mut active = self.peer_active_requests.write().await;
1650        let Some(count) = active.get_mut(peer_id) else {
1651            return;
1652        };
1653        if *count <= 1 {
1654            active.remove(peer_id);
1655        } else {
1656            *count -= 1;
1657        }
1658    }
1659
1660    async fn release_queried_peer_requests(&self, peer_ids: &[String]) {
1661        for peer_id in peer_ids {
1662            self.release_peer_request(peer_id).await;
1663        }
1664    }
1665
1666    fn requested_quote_mint(&self) -> Option<&str> {
1667        if let Some(default_mint) = self.routing.cashu_default_mint.as_deref() {
1668            if self.routing.cashu_accepted_mints.is_empty()
1669                || self
1670                    .routing
1671                    .cashu_accepted_mints
1672                    .iter()
1673                    .any(|mint| mint == default_mint)
1674            {
1675                return Some(default_mint);
1676            }
1677        }
1678
1679        self.routing
1680            .cashu_accepted_mints
1681            .first()
1682            .map(String::as_str)
1683    }
1684
1685    fn choose_quote_mint(&self, requested_mint: Option<&str>) -> Option<String> {
1686        if let Some(requested_mint) = requested_mint {
1687            if self.accepts_quote_mint(Some(requested_mint)) {
1688                return Some(requested_mint.to_string());
1689            }
1690        }
1691        if let Some(default_mint) = self.routing.cashu_default_mint.as_ref() {
1692            return Some(default_mint.clone());
1693        }
1694        if let Some(first_mint) = self.routing.cashu_accepted_mints.first() {
1695            return Some(first_mint.clone());
1696        }
1697        requested_mint.map(str::to_string)
1698    }
1699
1700    fn accepts_quote_mint(&self, mint_url: Option<&str>) -> bool {
1701        if self.routing.cashu_accepted_mints.is_empty() {
1702            return true;
1703        }
1704
1705        let Some(mint_url) = mint_url else {
1706            return false;
1707        };
1708        self.routing
1709            .cashu_accepted_mints
1710            .iter()
1711            .any(|mint| mint == mint_url)
1712    }
1713
1714    fn trusts_quote_mint(&self, mint_url: Option<&str>) -> bool {
1715        let Some(mint_url) = mint_url else {
1716            return self.routing.cashu_default_mint.is_none()
1717                && self.routing.cashu_accepted_mints.is_empty();
1718        };
1719        self.routing.cashu_default_mint.as_deref() == Some(mint_url)
1720            || self
1721                .routing
1722                .cashu_accepted_mints
1723                .iter()
1724                .any(|mint| mint == mint_url)
1725    }
1726
1727    async fn peer_suggested_mint_cap_sat(&self, peer_id: &str) -> u64 {
1728        let base = self.routing.cashu_peer_suggested_mint_base_cap_sat;
1729        if base == 0 {
1730            return 0;
1731        }
1732
1733        let selector = self.peer_selector.read().await;
1734        let Some(stats) = selector.get_stats(peer_id) else {
1735            let max_cap = self.routing.cashu_peer_suggested_mint_max_cap_sat;
1736            return if max_cap > 0 { base.min(max_cap) } else { base };
1737        };
1738
1739        if stats.cashu_payment_defaults > 0
1740            && stats.cashu_payment_defaults >= stats.cashu_payment_receipts
1741        {
1742            return 0;
1743        }
1744
1745        let success_bonus = stats
1746            .successes
1747            .saturating_mul(self.routing.cashu_peer_suggested_mint_success_step_sat);
1748        let receipt_bonus = stats
1749            .cashu_payment_receipts
1750            .saturating_mul(self.routing.cashu_peer_suggested_mint_receipt_step_sat);
1751        let mut cap = base
1752            .saturating_add(success_bonus)
1753            .saturating_add(receipt_bonus);
1754        let max_cap = self.routing.cashu_peer_suggested_mint_max_cap_sat;
1755        if max_cap > 0 {
1756            cap = cap.min(max_cap);
1757        }
1758        cap
1759    }
1760
1761    async fn should_accept_quote_response(
1762        &self,
1763        from_peer: &str,
1764        preferred_mint_url: Option<&str>,
1765        offered_payment_sat: u64,
1766        res: &DataQuoteResponse,
1767    ) -> bool {
1768        let Some(payment_sat) = res.p else {
1769            return false;
1770        };
1771        if payment_sat > offered_payment_sat {
1772            return false;
1773        }
1774
1775        let response_mint = res.m.as_deref();
1776        if response_mint == preferred_mint_url {
1777            return true;
1778        }
1779        if self.trusts_quote_mint(response_mint) {
1780            return true;
1781        }
1782        if response_mint.is_none() {
1783            return false;
1784        }
1785
1786        payment_sat <= self.peer_suggested_mint_cap_sat(from_peer).await
1787    }
1788
1789    async fn issue_quote(
1790        &self,
1791        peer_id: &str,
1792        hash_key: &str,
1793        payment_sat: u64,
1794        ttl_ms: u32,
1795        mint_url: Option<&str>,
1796    ) -> u64 {
1797        let quote_id = {
1798            let mut next = self.next_quote_id.write().await;
1799            let quote_id = *next;
1800            *next = next.saturating_add(1);
1801            quote_id
1802        };
1803
1804        let expires_at = Instant::now() + Duration::from_millis(ttl_ms as u64);
1805        self.issued_quotes.write().await.insert(
1806            (peer_id.to_string(), hash_key.to_string(), quote_id),
1807            IssuedQuote {
1808                expires_at,
1809                payment_sat,
1810                mint_url: mint_url.map(str::to_string),
1811            },
1812        );
1813        quote_id
1814    }
1815
1816    async fn take_valid_quote(&self, peer_id: &str, hash_key: &str, quote_id: u64) -> bool {
1817        let key = (peer_id.to_string(), hash_key.to_string(), quote_id);
1818        let Some(quote) = self.issued_quotes.write().await.remove(&key) else {
1819            return false;
1820        };
1821        quote.expires_at > Instant::now()
1822    }
1823
1824    async fn send_request_to_peer(
1825        &self,
1826        peer_id: &str,
1827        hash: &Hash,
1828        request_htl: u8,
1829        quote_id: Option<u64>,
1830    ) -> bool {
1831        if !should_forward_htl(request_htl) {
1832            return false;
1833        }
1834
1835        let channel = match self.signaling.get_channel(peer_id).await {
1836            Some(c) => c,
1837            None => return false,
1838        };
1839
1840        // Hashtree owns HTL and consumes exactly one unit when forwarding a
1841        // blob request to another mesh peer. Transport/routing hops below this
1842        // layer must not alter it.
1843        let send_htl = request_htl.saturating_sub(1);
1844        let req = match quote_id {
1845            Some(quote_id) => create_request_with_quote(hash, send_htl, quote_id),
1846            None => create_request(hash, send_htl),
1847        };
1848        let request_bytes = encode_request(&req);
1849        let request_len = request_bytes.len() as u64;
1850
1851        {
1852            let mut selector = self.peer_selector.write().await;
1853            selector.record_request(peer_id, request_len);
1854        }
1855
1856        match channel.send(request_bytes).await {
1857            Ok(()) => {
1858                self.record_peer_wire_sent(peer_id, request_len).await;
1859                true
1860            }
1861            Err(_) => {
1862                self.peer_selector.write().await.record_failure(peer_id);
1863                false
1864            }
1865        }
1866    }
1867
1868    async fn send_quote_request_to_peer(
1869        &self,
1870        peer_id: &str,
1871        hash: &Hash,
1872        payment_sat: u64,
1873        ttl_ms: u32,
1874        mint_url: Option<&str>,
1875    ) -> bool {
1876        let channel = match self.signaling.get_channel(peer_id).await {
1877            Some(c) => c,
1878            None => return false,
1879        };
1880
1881        let req = create_quote_request(hash, ttl_ms, payment_sat, mint_url);
1882        let request_bytes = encode_quote_request(&req);
1883        let request_len = request_bytes.len() as u64;
1884
1885        match channel.send(request_bytes).await {
1886            Ok(()) => {
1887                self.record_peer_wire_sent(peer_id, request_len).await;
1888                true
1889            }
1890            Err(_) => false,
1891        }
1892    }
1893
1894    /// Get peer count
1895    pub async fn peer_count(&self) -> usize {
1896        self.signaling.peer_count().await
1897    }
1898
1899    /// Get connected mesh peer IDs.
1900    pub async fn peer_ids(&self) -> Vec<String> {
1901        self.signaling.peer_ids().await
1902    }
1903
1904    /// Check if we need more peers
1905    pub async fn needs_peers(&self) -> bool {
1906        self.signaling.needs_peers().await
1907    }
1908
1909    /// Re-broadcast hello to refresh discovery as topology changes.
1910    pub async fn send_hello(&self) -> Result<(), TransportError> {
1911        self.signaling.send_hello(vec![]).await
1912    }
1913
1914    /// Drain all currently available peer-link messages and handle them.
1915    ///
1916    /// This keeps the message pump logic shared between simulation and the
1917    /// default production wrapper instead of duplicating per-channel loops.
1918    pub async fn drain_available_data_messages(self: &Arc<Self>) -> DataPumpStats {
1919        let mut stats = DataPumpStats::default();
1920        let peer_ids = self.signaling.peer_ids().await;
1921        for peer_id in peer_ids {
1922            let Some(channel) = self.signaling.get_channel(&peer_id).await else {
1923                continue;
1924            };
1925
1926            while let Some(data) = channel.try_recv() {
1927                stats.processed += 1;
1928                stats.processed_bytes += data.len() as u64;
1929                if let Some(msg) = parse_message(&data) {
1930                    match msg {
1931                        DataMessage::Request(_) => stats.request_messages += 1,
1932                        DataMessage::Response(_) => stats.response_messages += 1,
1933                        DataMessage::QuoteRequest(_) => stats.quote_request_messages += 1,
1934                        DataMessage::QuoteResponse(_) => stats.quote_response_messages += 1,
1935                        DataMessage::PubsubInterest(_) => stats.pubsub_interest_messages += 1,
1936                        DataMessage::PubsubFrame(_) => stats.pubsub_frame_messages += 1,
1937                        DataMessage::PubsubInventory(_) => stats.pubsub_inventory_messages += 1,
1938                        DataMessage::PubsubWant(_) => stats.pubsub_want_messages += 1,
1939                        DataMessage::Payment(_)
1940                        | DataMessage::PaymentAck(_)
1941                        | DataMessage::Chunk(_)
1942                        | DataMessage::PeerHints(_) => {}
1943                    }
1944                }
1945                self.handle_data_message(&peer_id, &data).await;
1946            }
1947        }
1948        stats
1949    }
1950
1951    /// Apply an out-of-band payment credit to a peer's routing priority.
1952    pub async fn record_cashu_payment_for_peer(&self, peer_id: &str, amount_sat: u64) {
1953        self.peer_selector
1954            .write()
1955            .await
1956            .record_cashu_payment(peer_id, amount_sat);
1957    }
1958
1959    /// Record a post-delivery payment we received from a peer.
1960    pub async fn record_cashu_receipt_from_peer(&self, peer_id: &str, amount_sat: u64) {
1961        self.peer_selector
1962            .write()
1963            .await
1964            .record_cashu_receipt(peer_id, amount_sat);
1965    }
1966
1967    /// Record that a peer failed to pay after we delivered successfully.
1968    pub async fn record_cashu_payment_default_from_peer(&self, peer_id: &str) {
1969        self.peer_selector
1970            .write()
1971            .await
1972            .record_cashu_payment_default(peer_id);
1973    }
1974
1975    /// Snapshot routing/selection summary for inspection/debugging.
1976    pub async fn selector_summary(&self) -> crate::peer_selector::SelectorSummary {
1977        self.peer_selector.read().await.summary()
1978    }
1979
1980    fn should_refuse_requests_from_peer(&self, selector: &PeerSelector, peer_id: &str) -> bool {
1981        selector.is_peer_blocked_for_payment_defaults(
1982            peer_id,
1983            self.routing.cashu_payment_default_block_threshold,
1984        )
1985    }
1986
1987    /// Export live peer metadata for inspection/debugging.
1988    pub async fn peer_metadata_snapshot(&self) -> PeerMetadataSnapshot {
1989        self.peer_selector
1990            .read()
1991            .await
1992            .export_peer_metadata_snapshot()
1993    }
1994
1995    /// Snapshot current peer metadata and persist it into `local_store`.
1996    ///
1997    /// Uses content-addressed storage for the snapshot body and a reserved
1998    /// mutable pointer slot for the "latest snapshot hash".
1999    pub async fn persist_peer_metadata(&self) -> Result<Hash, StoreError> {
2000        let snapshot = self
2001            .peer_selector
2002            .read()
2003            .await
2004            .export_peer_metadata_snapshot();
2005        let bytes = serde_json::to_vec(&snapshot).map_err(|e| {
2006            StoreError::Other(format!("Failed to encode peer metadata snapshot: {e}"))
2007        })?;
2008        let snapshot_hash = hashtree_core::sha256(&bytes);
2009        let _ = self.local_store.put(snapshot_hash, bytes).await?;
2010
2011        let pointer_slot = Self::peer_metadata_pointer_slot_hash();
2012        let pointer_bytes = hex::encode(snapshot_hash).into_bytes();
2013        let _ = self.local_store.delete(&pointer_slot).await?;
2014        let _ = self.local_store.put(pointer_slot, pointer_bytes).await?;
2015
2016        Ok(snapshot_hash)
2017    }
2018
2019    /// Load persisted peer metadata from `local_store` if available.
2020    pub async fn load_peer_metadata(&self) -> Result<bool, StoreError> {
2021        let pointer_slot = Self::peer_metadata_pointer_slot_hash();
2022        let Some(pointer_bytes) = self.local_store.get(&pointer_slot).await? else {
2023            return Ok(false);
2024        };
2025        let pointer_hex = std::str::from_utf8(&pointer_bytes).map_err(|e| {
2026            StoreError::Other(format!("Peer metadata pointer is not valid UTF-8: {e}"))
2027        })?;
2028        let snapshot_hash = Self::decode_hash_hex(pointer_hex.trim())?;
2029
2030        let Some(snapshot_bytes) = self.local_store.get(&snapshot_hash).await? else {
2031            return Ok(false);
2032        };
2033        let snapshot: PeerMetadataSnapshot =
2034            serde_json::from_slice(&snapshot_bytes).map_err(|e| {
2035                StoreError::Other(format!("Failed to decode peer metadata snapshot: {e}"))
2036            })?;
2037        self.peer_selector
2038            .write()
2039            .await
2040            .import_peer_metadata_snapshot(&snapshot);
2041        Ok(true)
2042    }
2043
2044    /// Request data from peers after negotiating a paid quote.
2045    ///
2046    /// If quote negotiation fails or the quoted peer does not deliver, the store
2047    /// falls back to the normal unpaid retrieval path to preserve liveness.
2048    pub async fn get_with_quote(
2049        &self,
2050        hash: &Hash,
2051        payment_sat: u64,
2052        quote_ttl: Duration,
2053    ) -> Result<Option<Vec<u8>>, StoreError> {
2054        if let Some(data) = self.local_store.get(hash).await? {
2055            if hashtree_core::sha256(&data) != *hash {
2056                return Err(StoreError::Other(
2057                    "local store returned corrupt content".to_string(),
2058                ));
2059            }
2060            return Ok(Some(data));
2061        }
2062        self.request_from_peers_with_quote(hash, payment_sat, quote_ttl)
2063            .await
2064    }
2065
2066    async fn request_from_peers_with_quote(
2067        &self,
2068        hash: &Hash,
2069        payment_sat: u64,
2070        quote_ttl: Duration,
2071    ) -> Result<Option<Vec<u8>>, StoreError> {
2072        let ordered_peer_ids = self.ordered_connected_peers(None).await;
2073        if ordered_peer_ids.is_empty() {
2074            return Ok(None);
2075        }
2076
2077        if let Some(quote) = self
2078            .request_quote_from_peers(hash, payment_sat, quote_ttl, &ordered_peer_ids)
2079            .await
2080        {
2081            if let Some(data) = self
2082                .request_from_single_peer(hash, &quote.peer_id, MAX_HTL, Some(quote.quote_id))
2083                .await
2084            {
2085                return Ok(Some(data));
2086            }
2087        }
2088
2089        match self
2090            .request_from_mesh_with_context(hash, &MeshReadContext::default())
2091            .await
2092        {
2093            RouteFetchOutcome::Hit(data) => Ok(Some(data)),
2094            RouteFetchOutcome::Miss => Ok(None),
2095            RouteFetchOutcome::Timeout => Err(StoreError::Other(
2096                "blob retrieval deadline expired before the search completed".to_string(),
2097            )),
2098        }
2099    }
2100
2101    async fn request_quote_from_peers(
2102        &self,
2103        hash: &Hash,
2104        payment_sat: u64,
2105        quote_ttl: Duration,
2106        ordered_peer_ids: &[String],
2107    ) -> Option<NegotiatedQuote> {
2108        if ordered_peer_ids.is_empty() {
2109            return None;
2110        }
2111        let ttl_ms = quote_ttl.as_millis().min(u32::MAX as u128) as u32;
2112        if ttl_ms == 0 {
2113            return None;
2114        }
2115        let requested_mint = self.requested_quote_mint().map(str::to_string);
2116
2117        let hash_key = hash_to_key(hash);
2118        let (tx, rx) = oneshot::channel();
2119        self.pending_quotes.write().await.insert(
2120            hash_key.clone(),
2121            PendingQuoteRequest {
2122                response_tx: tx,
2123                preferred_mint_url: requested_mint.clone(),
2124                offered_payment_sat: payment_sat,
2125            },
2126        );
2127
2128        let rx = Arc::new(Mutex::new(rx));
2129        let result = run_hedged_waves(
2130            ordered_peer_ids.len(),
2131            self.routing.dispatch,
2132            self.request_timeout,
2133            |range| {
2134                let wave_peer_ids = ordered_peer_ids[range].to_vec();
2135                let requested_mint = requested_mint.clone();
2136                let hash = *hash;
2137                async move {
2138                    let mut sent = 0usize;
2139                    for peer_id in wave_peer_ids {
2140                        if self
2141                            .send_quote_request_to_peer(
2142                                &peer_id,
2143                                &hash,
2144                                payment_sat,
2145                                ttl_ms,
2146                                requested_mint.as_deref(),
2147                            )
2148                            .await
2149                        {
2150                            sent += 1;
2151                        }
2152                    }
2153                    sent
2154                }
2155            },
2156            |wait| {
2157                let rx = rx.clone();
2158                async move {
2159                    let mut rx = rx.lock().await;
2160                    match tokio::time::timeout(wait, &mut *rx).await {
2161                        Ok(Ok(Some(quote))) => HedgedWaveAction::Success(quote),
2162                        Ok(Ok(None)) | Ok(Err(_)) => HedgedWaveAction::Abort,
2163                        Err(_) => HedgedWaveAction::Continue,
2164                    }
2165                }
2166            },
2167        )
2168        .await;
2169        let _ = self.pending_quotes.write().await.remove(&hash_key);
2170        result
2171    }
2172
2173    async fn register_pending_request(
2174        &self,
2175        request_key: PendingRequestKey,
2176        queried_peers: Vec<String>,
2177    ) -> (Arc<()>, oneshot::Receiver<Option<Vec<u8>>>) {
2178        let owner = Arc::new(());
2179        let (response_tx, response_rx) = oneshot::channel();
2180        self.pending_requests
2181            .write()
2182            .await
2183            .entry(request_key)
2184            .or_default()
2185            .push(PendingRequest {
2186                owner: owner.clone(),
2187                response_tx,
2188                started_at: Instant::now(),
2189                queried_peers,
2190            });
2191        (owner, response_rx)
2192    }
2193
2194    async fn take_pending_request(
2195        &self,
2196        request_key: PendingRequestKey,
2197        owner: &Arc<()>,
2198    ) -> Option<(PendingRequest, bool)> {
2199        let mut pending = self.pending_requests.write().await;
2200        let (request, remove_key) = {
2201            let requests = pending.get_mut(&request_key)?;
2202            let index = requests
2203                .iter()
2204                .position(|request| Arc::ptr_eq(&request.owner, owner))?;
2205            let request = requests.swap_remove(index);
2206            (request, requests.is_empty())
2207        };
2208        if remove_key {
2209            pending.remove(&request_key);
2210        }
2211        Some((request, remove_key))
2212    }
2213
2214    async fn request_from_single_peer(
2215        &self,
2216        hash: &Hash,
2217        peer_id: &str,
2218        request_htl: u8,
2219        quote_id: Option<u64>,
2220    ) -> Option<Vec<u8>> {
2221        let request_key = PendingRequestKey::new(*hash, request_htl);
2222        let (owner, rx) = self
2223            .register_pending_request(request_key, vec![peer_id.to_string()])
2224            .await;
2225
2226        let mut rx = rx;
2227        if !self
2228            .send_request_to_peer(peer_id, hash, request_htl, quote_id)
2229            .await
2230        {
2231            if self
2232                .take_pending_request(request_key, &owner)
2233                .await
2234                .is_some_and(|(_, last)| last)
2235            {
2236                let _ = self.take_forward_requesters(request_key).await;
2237            }
2238            return None;
2239        }
2240        self.reserve_peer_request(peer_id).await;
2241
2242        if let Ok(Ok(Some(data))) = tokio::time::timeout(self.request_timeout, &mut rx).await {
2243            if data.len() <= BLOB_MAX_BYTES && hashtree_core::sha256(&data) == *hash {
2244                let _ = self.local_store.put(*hash, data.clone()).await;
2245                return Some(data);
2246            }
2247        }
2248
2249        if let Some((pending, last)) = self.take_pending_request(request_key, &owner).await {
2250            self.release_queried_peer_requests(&pending.queried_peers)
2251                .await;
2252            for peer_id in pending.queried_peers {
2253                self.peer_selector.write().await.record_timeout(&peer_id);
2254            }
2255            if last {
2256                let _ = self.take_forward_requesters(request_key).await;
2257            }
2258        }
2259        None
2260    }
2261
2262    async fn request_from_ordered_peers(
2263        &self,
2264        hash: &Hash,
2265        ordered_peer_ids: &[String],
2266        request_htl: u8,
2267        timeout: Duration,
2268    ) -> RouteFetchOutcome {
2269        let request_key = PendingRequestKey::new(*hash, request_htl);
2270        let (owner, rx) = self.register_pending_request(request_key, Vec::new()).await;
2271
2272        let rx = Arc::new(Mutex::new(rx));
2273        let result = run_hedged_waves(
2274            ordered_peer_ids.len(),
2275            normalize_dispatch_config(self.routing.dispatch, ordered_peer_ids.len()),
2276            timeout,
2277            |range| {
2278                let wave_peer_ids = ordered_peer_ids[range].to_vec();
2279                let hash = *hash;
2280                let owner = owner.clone();
2281                async move {
2282                    let mut sent = 0usize;
2283                    for peer_id in wave_peer_ids {
2284                        if self
2285                            .send_request_to_peer(&peer_id, &hash, request_htl, None)
2286                            .await
2287                        {
2288                            sent += 1;
2289                            self.reserve_peer_request(&peer_id).await;
2290                            if let Some(pending) = self
2291                                .pending_requests
2292                                .write()
2293                                .await
2294                                .get_mut(&request_key)
2295                                .and_then(|requests| {
2296                                    requests
2297                                        .iter_mut()
2298                                        .find(|request| Arc::ptr_eq(&request.owner, &owner))
2299                                })
2300                            {
2301                                pending.queried_peers.push(peer_id);
2302                            }
2303                        }
2304                    }
2305                    sent
2306                }
2307            },
2308            |wait| {
2309                let rx = rx.clone();
2310                async move {
2311                    let mut rx = rx.lock().await;
2312                    match tokio::time::timeout(wait, &mut *rx).await {
2313                        Ok(Ok(Some(data)))
2314                            if data.len() <= BLOB_MAX_BYTES
2315                                && hashtree_core::sha256(&data) == *hash =>
2316                        {
2317                            HedgedWaveAction::Success(data)
2318                        }
2319                        Ok(Ok(Some(_))) => HedgedWaveAction::Continue,
2320                        Ok(Ok(None)) | Ok(Err(_)) => HedgedWaveAction::Abort,
2321                        Err(_) => HedgedWaveAction::Continue,
2322                    }
2323                }
2324            },
2325        )
2326        .await;
2327
2328        let Some(data) = result else {
2329            if let Some((pending, last)) = self.take_pending_request(request_key, &owner).await {
2330                self.release_queried_peer_requests(&pending.queried_peers)
2331                    .await;
2332                for peer_id in pending.queried_peers {
2333                    self.peer_selector.write().await.record_timeout(&peer_id);
2334                }
2335                if last {
2336                    let _ = self.take_forward_requesters(request_key).await;
2337                }
2338            }
2339            return RouteFetchOutcome::Timeout;
2340        };
2341
2342        let _ = self.local_store.put(*hash, data.clone()).await;
2343        RouteFetchOutcome::Hit(data)
2344    }
2345
2346    async fn request_from_mesh_with_context(
2347        &self,
2348        hash: &Hash,
2349        context: &MeshReadContext,
2350    ) -> RouteFetchOutcome {
2351        if !should_forward_htl(context.request_htl) {
2352            return RouteFetchOutcome::Miss;
2353        }
2354        let mut peers = self
2355            .ordered_connected_peers(context.exclude_peer_id.as_deref())
2356            .await;
2357        if let Some(attempt_budget) = context.attempt_budget {
2358            peers.truncate(attempt_budget);
2359        }
2360        if peers.is_empty() {
2361            return RouteFetchOutcome::Miss;
2362        }
2363        let timeout = context
2364            .deadline
2365            .map(|deadline| deadline.saturating_duration_since(Instant::now()))
2366            .unwrap_or(self.request_timeout)
2367            .min(self.request_timeout);
2368        if timeout.is_zero() {
2369            return RouteFetchOutcome::Timeout;
2370        }
2371        self.request_from_ordered_peers(hash, &peers, context.request_htl, timeout)
2372            .await
2373    }
2374
2375    async fn begin_forward_request(
2376        &self,
2377        request_key: PendingRequestKey,
2378        requester_id: &str,
2379    ) -> bool {
2380        let mut pending = self.pending_forward_requests.write().await;
2381        if let Some(existing) = pending.get_mut(&request_key) {
2382            existing.requester_ids.insert(requester_id.to_string());
2383            return false;
2384        }
2385
2386        let mut requester_ids = HashSet::new();
2387        requester_ids.insert(requester_id.to_string());
2388        pending.insert(request_key, PendingForwardRequest { requester_ids });
2389        true
2390    }
2391
2392    async fn take_forward_requesters(&self, request_key: PendingRequestKey) -> Vec<String> {
2393        self.pending_forward_requests
2394            .write()
2395            .await
2396            .remove(&request_key)
2397            .map(|pending| pending.requester_ids.into_iter().collect())
2398            .unwrap_or_default()
2399    }
2400
2401    async fn complete_pending_response(
2402        self: &Arc<Self>,
2403        from_peer: &str,
2404        hash: &Hash,
2405        payload: Vec<u8>,
2406    ) {
2407        let pending = {
2408            let mut requests = self.pending_requests.write().await;
2409            let matching_keys: Vec<_> = requests
2410                .keys()
2411                .filter(|key| key.hash == *hash)
2412                .copied()
2413                .collect();
2414            matching_keys
2415                .into_iter()
2416                .flat_map(|key| {
2417                    requests
2418                        .remove(&key)
2419                        .into_iter()
2420                        .flatten()
2421                        .map(move |request| (key, request))
2422                })
2423                .collect::<Vec<_>>()
2424        };
2425        if pending.is_empty() {
2426            return;
2427        }
2428
2429        let payload_bytes = payload.len() as u64;
2430        self.record_useful_bytes_received_from_peer(from_peer, payload_bytes)
2431            .await;
2432        {
2433            let mut deliveries = self.verified_block_deliveries.lock().await;
2434            deliveries.deliveries.push_back(VerifiedBlockDelivery {
2435                hash: *hash,
2436                provider_peer_id: from_peer.to_string(),
2437                payload_bytes,
2438            });
2439            while deliveries.deliveries.len() > VERIFIED_BLOCK_DELIVERY_CAPACITY {
2440                deliveries.deliveries.pop_front();
2441                deliveries.dropped_since_last_drain =
2442                    deliveries.dropped_since_last_drain.saturating_add(1);
2443            }
2444        }
2445
2446        let rtt_ms = pending
2447            .iter()
2448            .map(|(_, request)| request.started_at.elapsed().as_millis() as u64)
2449            .max()
2450            .unwrap_or_default();
2451        let queried_peers: Vec<_> = pending
2452            .iter()
2453            .flat_map(|(_, request)| request.queried_peers.iter().cloned())
2454            .collect();
2455        self.release_queried_peer_requests(&queried_peers).await;
2456        self.peer_selector
2457            .write()
2458            .await
2459            .record_success(from_peer, rtt_ms, payload_bytes);
2460
2461        let mut forward_requesters = HashSet::new();
2462        for (request_key, pending) in pending {
2463            forward_requesters.extend(self.take_forward_requesters(request_key).await);
2464            let _ = pending.response_tx.send(Some(payload.clone()));
2465        }
2466        if !forward_requesters.is_empty() {
2467            let response_bytes = encode_response(&create_response(hash, payload));
2468            for requester_id in forward_requesters {
2469                Arc::clone(self)
2470                    .enqueue_response_send(requester_id, response_bytes.clone(), Instant::now())
2471                    .await;
2472            }
2473        }
2474    }
2475
2476    async fn handle_quote_response_message(&self, from_peer: &str, res: DataQuoteResponse) {
2477        if !res.a {
2478            return;
2479        }
2480
2481        let Some(quote_id) = res.q else {
2482            return;
2483        };
2484
2485        let hash_key = hash_to_key(&res.h);
2486        let (preferred_mint_url, offered_payment_sat) = {
2487            let pending_quotes = self.pending_quotes.read().await;
2488            let Some(pending) = pending_quotes.get(&hash_key) else {
2489                return;
2490            };
2491            (
2492                pending.preferred_mint_url.clone(),
2493                pending.offered_payment_sat,
2494            )
2495        };
2496        if !self
2497            .should_accept_quote_response(
2498                from_peer,
2499                preferred_mint_url.as_deref(),
2500                offered_payment_sat,
2501                &res,
2502            )
2503            .await
2504        {
2505            return;
2506        }
2507        let mut pending_quotes = self.pending_quotes.write().await;
2508        if let Some(pending) = pending_quotes.remove(&hash_key) {
2509            let _ = pending.response_tx.send(Some(NegotiatedQuote {
2510                peer_id: from_peer.to_string(),
2511                quote_id,
2512                mint_url: res.m,
2513            }));
2514        }
2515    }
2516
2517    async fn handle_response_message(
2518        self: &Arc<Self>,
2519        from_peer: &str,
2520        res: crate::protocol::DataResponse,
2521    ) {
2522        let hash_key = hash_to_key(&res.h);
2523        let hash = match crate::protocol::bytes_to_hash(&res.h) {
2524            Some(h) => h,
2525            None => return,
2526        };
2527
2528        // Ignore malformed/corrupt payload and keep waiting for a valid response.
2529        if hashtree_core::sha256(&res.d) != hash {
2530            self.peer_selector.write().await.record_failure(from_peer);
2531            if self.debug {
2532                println!("[MeshStoreCore] Ignoring invalid response payload for {hash_key}");
2533            }
2534            return;
2535        }
2536
2537        self.complete_pending_response(from_peer, &hash, res.d)
2538            .await;
2539    }
2540
2541    async fn handle_quote_request_message(&self, from_peer: &str, req: DataQuoteRequest) {
2542        let hash = match crate::protocol::bytes_to_hash(&req.h) {
2543            Some(h) => h,
2544            None => return,
2545        };
2546        let hash_key = hash_to_key(&hash);
2547
2548        {
2549            let selector = self.peer_selector.read().await;
2550            if self.should_refuse_requests_from_peer(&selector, from_peer) {
2551                if self.debug {
2552                    println!(
2553                        "[MeshStoreCore] Refusing quote request from delinquent peer {}",
2554                        from_peer
2555                    );
2556                }
2557                return;
2558            }
2559        }
2560
2561        let chosen_mint = self.choose_quote_mint(req.m.as_deref());
2562        let can_serve = self.local_store.has(&hash).await.ok().unwrap_or(false)
2563            && !self.should_drop_response(&hash)
2564            && !self.should_corrupt_response(&hash);
2565
2566        let res = if can_serve {
2567            let quote_id = self
2568                .issue_quote(from_peer, &hash_key, req.p, req.t, chosen_mint.as_deref())
2569                .await;
2570            create_quote_response_available(&hash, quote_id, req.p, req.t, chosen_mint.as_deref())
2571        } else {
2572            create_quote_response_unavailable(&hash)
2573        };
2574        let response_bytes = encode_quote_response(&res);
2575        if let Some(channel) = self.signaling.get_channel(from_peer).await {
2576            if channel.send(response_bytes.clone()).await.is_ok() {
2577                self.record_peer_wire_sent(from_peer, response_bytes.len() as u64)
2578                    .await;
2579            }
2580        }
2581    }
2582
2583    async fn handle_request_message(
2584        self: &Arc<Self>,
2585        from_peer: &str,
2586        req: crate::protocol::DataRequest,
2587    ) {
2588        if req.htl > MAX_HTL {
2589            return;
2590        }
2591        let hash = match crate::protocol::bytes_to_hash(&req.h) {
2592            Some(h) => h,
2593            None => return,
2594        };
2595        let hash_key = hash_to_key(&hash);
2596        let request_key = PendingRequestKey::new(hash, req.htl);
2597
2598        if let Some(quote_id) = req.q {
2599            if !self.take_valid_quote(from_peer, &hash_key, quote_id).await {
2600                if self.debug {
2601                    println!(
2602                        "[MeshStoreCore] Refusing request with invalid or expired quote {} from {}",
2603                        quote_id, from_peer
2604                    );
2605                }
2606                return;
2607            }
2608        }
2609
2610        let allow_peer_forwarding = {
2611            let selector = self.peer_selector.read().await;
2612            !self.should_refuse_requests_from_peer(&selector, from_peer)
2613        };
2614
2615        // Check local store
2616        if let Ok(Some(mut data)) = self.local_store.get(&hash).await {
2617            if data.len() <= BLOB_MAX_BYTES && hashtree_core::sha256(&data) == hash {
2618                if self.should_drop_response(&hash) {
2619                    if self.debug {
2620                        println!(
2621                            "[MeshStoreCore] Dropping response for {} due to actor profile",
2622                            hash_to_key(&hash)
2623                        );
2624                    }
2625                    return;
2626                }
2627
2628                let response_delay = self.response_send_delay(&hash, data.len());
2629                if self.should_corrupt_response(&hash) {
2630                    if data.is_empty() {
2631                        data.push(0x80);
2632                    } else {
2633                        data[0] ^= 0x80;
2634                    }
2635                }
2636
2637                let res = create_response(&hash, data);
2638                let response_bytes = encode_response(&res);
2639                let ready_at = Instant::now() + response_delay;
2640                Arc::clone(self)
2641                    .enqueue_response_send(from_peer.to_string(), response_bytes, ready_at)
2642                    .await;
2643                return;
2644            }
2645        }
2646
2647        if self
2648            .pending_requests
2649            .read()
2650            .await
2651            .contains_key(&request_key)
2652        {
2653            let _ = self.begin_forward_request(request_key, from_peer).await;
2654            return;
2655        }
2656
2657        if !self.begin_forward_request(request_key, from_peer).await {
2658            return;
2659        }
2660
2661        let from_peer = from_peer.to_string();
2662        let this = Arc::clone(self);
2663        let request_htl = req.htl;
2664        tokio::spawn(async move {
2665            let result = if allow_peer_forwarding {
2666                let context = MeshReadContext {
2667                    exclude_peer_id: Some(from_peer.clone()),
2668                    request_htl,
2669                    deadline: None,
2670                    attempt_budget: None,
2671                };
2672                this.request_from_mesh_with_context(&hash, &context).await
2673            } else {
2674                if this.debug {
2675                    println!(
2676                        "[MeshStoreCore] Refusing to forward request from delinquent peer {}",
2677                        from_peer
2678                    );
2679                }
2680                RouteFetchOutcome::Miss
2681            };
2682            let requester_ids = this.take_forward_requesters(request_key).await;
2683            match result {
2684                RouteFetchOutcome::Hit(data) => {
2685                    let ready_at = Instant::now() + this.response_send_delay(&hash, data.len());
2686                    let res = create_response(&hash, data);
2687                    let response_bytes = encode_response(&res);
2688                    for requester_id in requester_ids {
2689                        Arc::clone(&this)
2690                            .enqueue_response_send(requester_id, response_bytes.clone(), ready_at)
2691                            .await;
2692                    }
2693                }
2694                RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => {}
2695            }
2696        });
2697    }
2698
2699    async fn handle_pubsub_interest_message(
2700        self: &Arc<Self>,
2701        from_peer: &str,
2702        mut interest: PubsubInterest,
2703    ) {
2704        if !self.apply_pubsub_interest_route(from_peer, &interest).await {
2705            return;
2706        }
2707
2708        if !self.routing.pubsub_forwarding || interest.htl <= 1 {
2709            return;
2710        }
2711        interest.htl = interest.htl.saturating_sub(1);
2712        let _ = self
2713            .send_pubsub_interest_to_peers(&interest, Some(from_peer))
2714            .await;
2715    }
2716
2717    async fn handle_pubsub_frame_message(
2718        self: &Arc<Self>,
2719        from_peer: &str,
2720        mut frame: PubsubFrame,
2721        wire_bytes: usize,
2722    ) {
2723        if frame.stream_id.is_empty() || frame.origin_peer_id.is_empty() {
2724            return;
2725        }
2726        if frame.origin_peer_id == self.signaling.peer_id() {
2727            return;
2728        }
2729
2730        let frame_key = Self::pubsub_frame_key(&frame);
2731        if !self
2732            .pubsub_seen_frames
2733            .lock()
2734            .await
2735            .insert_if_new(frame_key.clone())
2736        {
2737            return;
2738        }
2739        self.cache_pubsub_frame(frame_key.clone(), frame.clone())
2740            .await;
2741
2742        let local_interested = self
2743            .pubsub_local_interests
2744            .read()
2745            .await
2746            .contains(&frame.stream_id);
2747        let mut downstream_peers = if self.routing.pubsub_forwarding && frame.htl > 1 {
2748            match self.routing.pubsub_delivery_mode {
2749                PubsubDeliveryMode::InterestPush => {
2750                    let mut peers = self
2751                        .interested_pubsub_peers(&frame.stream_id, Some(from_peer))
2752                        .await;
2753                    peers.extend(
2754                        self.take_pubsub_want_peers(&frame_key, Some(from_peer))
2755                            .await,
2756                    );
2757                    peers.sort();
2758                    peers.dedup();
2759                    peers
2760                }
2761                PubsubDeliveryMode::HtlInvWant => {
2762                    self.take_pubsub_want_peers(&frame_key, Some(from_peer))
2763                        .await
2764                }
2765            }
2766        } else {
2767            Vec::new()
2768        };
2769        downstream_peers.retain(|peer_id| peer_id != from_peer);
2770
2771        if local_interested || !downstream_peers.is_empty() {
2772            self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
2773                .await;
2774        }
2775
2776        if local_interested {
2777            self.enqueue_pubsub_event(PubsubEvent {
2778                stream_id: frame.stream_id.clone(),
2779                seq: frame.seq,
2780                origin_peer_id: frame.origin_peer_id.clone(),
2781                from_peer_id: from_peer.to_string(),
2782                payload: frame.payload.clone(),
2783            })
2784            .await;
2785        }
2786
2787        if downstream_peers.is_empty() {
2788            return;
2789        }
2790
2791        frame.htl = frame.htl.saturating_sub(1);
2792        let _ = self
2793            .send_pubsub_frame_to_peers(&frame, &downstream_peers)
2794            .await;
2795    }
2796
2797    async fn handle_pubsub_inventory_message(
2798        self: &Arc<Self>,
2799        from_peer: &str,
2800        inv: PubsubInventory,
2801        wire_bytes: usize,
2802    ) {
2803        if inv.stream_id.is_empty() || inv.origin_peer_id.is_empty() {
2804            return;
2805        }
2806        if inv.origin_peer_id == self.signaling.peer_id() {
2807            return;
2808        }
2809
2810        let key = Self::pubsub_key(&inv.origin_peer_id, &inv.stream_id, inv.seq);
2811        if !self
2812            .pubsub_seen_inventories
2813            .lock()
2814            .await
2815            .insert_if_new(key.clone())
2816        {
2817            return;
2818        }
2819        {
2820            let mut routes = self.pubsub_inventory_routes.write().await;
2821            routes
2822                .entry(key.clone())
2823                .or_insert_with(|| from_peer.to_string());
2824        }
2825
2826        let local_interested = self
2827            .pubsub_local_interests
2828            .read()
2829            .await
2830            .contains(&inv.stream_id);
2831        let downstream_peers = if self.routing.pubsub_forwarding {
2832            self.interested_pubsub_peers(&inv.stream_id, Some(from_peer))
2833                .await
2834        } else {
2835            Vec::new()
2836        };
2837        if local_interested || !downstream_peers.is_empty() {
2838            self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
2839                .await;
2840            let want =
2841                create_pubsub_want(inv.stream_id.clone(), inv.seq, inv.origin_peer_id.clone());
2842            let _ = self.send_pubsub_want_upstream(&key, &want, None).await;
2843        }
2844
2845        if !self.routing.pubsub_forwarding
2846            || downstream_peers.is_empty()
2847            || !should_forward_htl(inv.htl)
2848        {
2849            return;
2850        }
2851        let _ = self
2852            .send_pubsub_inventory_to_peers(&inv, &downstream_peers)
2853            .await;
2854    }
2855
2856    async fn handle_pubsub_want_message(
2857        self: &Arc<Self>,
2858        from_peer: &str,
2859        want: PubsubWant,
2860        wire_bytes: usize,
2861    ) {
2862        if want.stream_id.is_empty() || want.origin_peer_id.is_empty() {
2863            return;
2864        }
2865        if want.origin_peer_id == from_peer {
2866            return;
2867        }
2868
2869        let key = Self::pubsub_key(&want.origin_peer_id, &want.stream_id, want.seq);
2870        let want_key = format!("{from_peer}:{key}");
2871        if !self.pubsub_seen_wants.lock().await.insert_if_new(want_key) {
2872            return;
2873        }
2874
2875        if let Some(frame) = self.cached_pubsub_frame(&key).await {
2876            self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
2877                .await;
2878            let peers = vec![from_peer.to_string()];
2879            let _ = self.send_pubsub_frame_to_peers(&frame, &peers).await;
2880            return;
2881        }
2882
2883        let has_upstream_route = self.pubsub_inventory_routes.read().await.contains_key(&key);
2884        if !has_upstream_route {
2885            return;
2886        }
2887
2888        if self.remember_pubsub_want_peer(key.clone(), from_peer).await {
2889            self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
2890                .await;
2891        }
2892        let _ = self
2893            .send_pubsub_want_upstream(&key, &want, Some(from_peer))
2894            .await;
2895    }
2896
2897    /// Handle incoming data message
2898    pub async fn handle_data_message(self: &Arc<Self>, from_peer: &str, data: &[u8]) {
2899        self.record_peer_wire_received(from_peer, data.len() as u64)
2900            .await;
2901        let parsed = match parse_message(data) {
2902            Some(m) => m,
2903            None => return,
2904        };
2905
2906        match parsed {
2907            DataMessage::Request(req) => {
2908                self.handle_request_message(from_peer, req).await;
2909            }
2910            DataMessage::Response(res) => {
2911                self.handle_response_message(from_peer, res).await;
2912            }
2913            DataMessage::QuoteRequest(req) => {
2914                self.handle_quote_request_message(from_peer, req).await;
2915            }
2916            DataMessage::QuoteResponse(res) => {
2917                self.handle_quote_response_message(from_peer, res).await;
2918            }
2919            DataMessage::PubsubInterest(interest) => {
2920                self.handle_pubsub_interest_message(from_peer, interest)
2921                    .await;
2922            }
2923            DataMessage::PubsubFrame(frame) => {
2924                self.handle_pubsub_frame_message(from_peer, frame, data.len())
2925                    .await;
2926            }
2927            DataMessage::PubsubInventory(inv) => {
2928                self.handle_pubsub_inventory_message(from_peer, inv, data.len())
2929                    .await;
2930            }
2931            DataMessage::PubsubWant(want) => {
2932                self.handle_pubsub_want_message(from_peer, want, data.len())
2933                    .await;
2934            }
2935            DataMessage::Payment(_)
2936            | DataMessage::PaymentAck(_)
2937            | DataMessage::Chunk(_)
2938            | DataMessage::PeerHints(_) => {}
2939        }
2940    }
2941}
2942
2943#[async_trait]
2944impl<S, R, F> BlobRoute for MeshStoreCore<S, R, F>
2945where
2946    S: Store + Send + Sync + 'static,
2947    R: SignalingTransport + Send + Sync + 'static,
2948    F: PeerLinkFactory + Send + Sync + 'static,
2949{
2950    async fn route(&self, request: BlobRequest) -> Result<BlobReply, StoreError> {
2951        self.route_with_context(
2952            request,
2953            BlobRouteContext {
2954                deadline: (Instant::now() + self.request_timeout).into(),
2955                attempt_budget: self.routing.dispatch.max_fanout.max(1),
2956            },
2957        )
2958        .await
2959    }
2960
2961    async fn route_with_context(
2962        &self,
2963        request: BlobRequest,
2964        route_context: BlobRouteContext,
2965    ) -> Result<BlobReply, StoreError> {
2966        if request.htl > MAX_HTL {
2967            return Err(StoreError::Other(format!(
2968                "Hashtree blob HTL {} exceeds the maximum of {MAX_HTL}",
2969                request.htl
2970            )));
2971        }
2972        if let Some(data) = self.local_store.get(&request.hash).await? {
2973            if data.len() > BLOB_MAX_BYTES {
2974                return Err(StoreError::Other(format!(
2975                    "local store returned {} bytes, exceeding the {BLOB_MAX_BYTES}-byte limit",
2976                    data.len()
2977                )));
2978            }
2979            if hashtree_core::sha256(&data) != request.hash {
2980                return Err(StoreError::Other(
2981                    "local store returned corrupt content".to_string(),
2982                ));
2983            }
2984            return Ok(BlobReply::Data(data));
2985        }
2986
2987        if request.htl == 0 {
2988            return Ok(BlobReply::NoResult);
2989        }
2990
2991        let context = MeshReadContext {
2992            exclude_peer_id: None,
2993            request_htl: request.htl,
2994            deadline: Some(route_context.deadline.into()),
2995            attempt_budget: Some(route_context.attempt_budget),
2996        };
2997        match self
2998            .request_from_mesh_with_context(&request.hash, &context)
2999            .await
3000        {
3001            RouteFetchOutcome::Hit(data) => {
3002                if data.len() > BLOB_MAX_BYTES {
3003                    return Err(StoreError::Other(format!(
3004                        "blob route returned {} bytes, exceeding the {BLOB_MAX_BYTES}-byte limit",
3005                        data.len()
3006                    )));
3007                }
3008                if hashtree_core::sha256(&data) != request.hash {
3009                    return Err(StoreError::Other(
3010                        "blob route returned corrupt content".to_string(),
3011                    ));
3012                }
3013                Ok(BlobReply::Data(data))
3014            }
3015            RouteFetchOutcome::Miss => Ok(BlobReply::NoResult),
3016            RouteFetchOutcome::Timeout => Err(StoreError::Other(
3017                "blob retrieval deadline expired before the search completed".to_string(),
3018            )),
3019        }
3020    }
3021}
3022
3023#[async_trait]
3024impl<S, R, F> Store for MeshStoreCore<S, R, F>
3025where
3026    S: Store + Send + Sync + 'static,
3027    R: SignalingTransport + Send + Sync + 'static,
3028    F: PeerLinkFactory + Send + Sync + 'static,
3029{
3030    async fn put(&self, hash: Hash, data: Vec<u8>) -> Result<bool, StoreError> {
3031        self.local_store.put(hash, data).await
3032    }
3033
3034    async fn get(&self, hash: &Hash) -> Result<Option<Vec<u8>>, StoreError> {
3035        Ok(
3036            match self
3037                .route(BlobRequest {
3038                    hash: *hash,
3039                    htl: MAX_HTL,
3040                })
3041                .await?
3042            {
3043                BlobReply::Data(data) => Some(data),
3044                BlobReply::NoResult => None,
3045            },
3046        )
3047    }
3048
3049    async fn has(&self, hash: &Hash) -> Result<bool, StoreError> {
3050        self.local_store.has(hash).await
3051    }
3052
3053    async fn delete(&self, hash: &Hash) -> Result<bool, StoreError> {
3054        self.local_store.delete(hash).await
3055    }
3056}
3057
3058#[cfg(test)]
3059mod delivery_tests;
3060
3061#[cfg(test)]
3062mod tests;
3063
3064/// Type alias for simulation store.
3065pub type SimMeshStore<S> =
3066    MeshStoreCore<S, crate::mock::MockRelayTransport, crate::mock::MockConnectionFactory>;