1use async_trait::async_trait;
8use futures::{stream::FuturesUnordered, FutureExt, StreamExt};
9use std::collections::hash_map::DefaultHasher;
10use std::collections::{HashMap, HashSet, VecDeque};
11use std::future::Future;
12use std::hash::{Hash as _, Hasher};
13use std::ops::Range;
14use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
15use std::sync::Arc;
16use std::time::Duration;
17use tokio::sync::{oneshot, Mutex, Notify, RwLock};
18use tokio::time::Instant;
19
20use hashtree_core::{Hash, Store, StoreError};
21
22use crate::peer_selector::{PeerMetadataSnapshot, PeerSelector, SelectionStrategy};
23use crate::protocol::{
24 create_pubsub_frame, create_pubsub_interest, create_pubsub_inventory, create_pubsub_want,
25 create_quote_request, create_quote_response_available, create_quote_response_unavailable,
26 create_request, create_request_with_quote, create_response, encode_pubsub_frame,
27 encode_pubsub_interest, encode_pubsub_inventory, encode_pubsub_want, encode_quote_request,
28 encode_quote_response, encode_request, encode_response, hash_to_key, parse_message,
29 DataMessage, DataQuoteRequest, DataQuoteResponse, PubsubFrame, PubsubInterest, PubsubInventory,
30 PubsubWant,
31};
32use crate::pubsub_strategy::{
33 reciprocal_virtual_finish, select_reciprocal_outbound_job, OutboundJobCandidate,
34 PeerTrafficSnapshot, PubsubCandidate, PubsubSchedulerConfig,
35};
36use crate::signaling::MeshRouter;
37use crate::transport::{PeerLinkFactory, SignalingTransport, TransportError};
38use crate::types::{
39 should_forward_htl, PeerHTLConfig, SignalingMessage, TimedSeenSet, MAX_HTL, MESH_EVENT_POLICY,
40};
41
42const PEER_METADATA_POINTER_SLOT_KEY: &[u8] = b"hashtree-mesh/peer-metadata/latest/v1";
45const RECENT_FORWARD_MISS_CAPACITY: usize = 4096;
46const MIN_RECENT_FORWARD_MISS_TTL_MS: u64 = 250;
47const PUBSUB_SEEN_CAPACITY: usize = 16_384;
48const PUBSUB_INBOX_CAPACITY: usize = 4_096;
49const PUBSUB_FRAME_CACHE_CAPACITY: usize = 4_096;
50const VERIFIED_BLOCK_DELIVERY_CAPACITY: usize = 4_096;
51const PUBSUB_SEEN_TTL: Duration = Duration::from_secs(120);
52
53struct PendingRequest {
55 response_tx: oneshot::Sender<Option<Vec<u8>>>,
56 started_at: Instant,
57 queried_peers: Vec<String>,
58}
59
60struct PendingQuoteRequest {
61 response_tx: oneshot::Sender<Option<NegotiatedQuote>>,
62 preferred_mint_url: Option<String>,
63 offered_payment_sat: u64,
64}
65
66struct PendingForwardRequest {
67 requester_ids: HashSet<String>,
68}
69
70type PeerWireStats = PeerTrafficSnapshot;
71
72struct PendingResponseSend {
73 job_id: u64,
74 peer_id: String,
75 bytes: Vec<u8>,
76 ready_at: Instant,
77 queue_sequence: u64,
78}
79
80#[async_trait]
81pub trait MeshReadSource: Send + Sync {
82 fn id(&self) -> &str;
83
84 fn is_available(&self) -> bool {
85 true
86 }
87
88 async fn get(&self, hash: &Hash) -> Option<Vec<u8>>;
89}
90
91#[derive(Debug, Clone)]
92struct NegotiatedQuote {
93 peer_id: String,
94 quote_id: u64,
95 #[allow(dead_code)]
96 mint_url: Option<String>,
97}
98
99struct IssuedQuote {
100 expires_at: Instant,
101 #[allow(dead_code)]
102 payment_sat: u64,
103 #[allow(dead_code)]
104 mint_url: Option<String>,
105}
106
107#[derive(Debug, Clone, Default)]
108struct AdaptiveSourceStats {
109 requests: u64,
110 successes: u64,
111 misses: u64,
112 failures: u64,
113 timeouts: u64,
114 srtt_ms: f64,
115 rttvar_ms: f64,
116 backoff_level: u32,
117 backed_off_until: Option<Instant>,
118 last_success_at: Option<Instant>,
119 last_failure_at: Option<Instant>,
120}
121
122#[derive(Debug, Clone)]
123enum RouteFetchOutcome {
124 Hit(Vec<u8>),
125 Miss,
126 Timeout,
127}
128
129struct InflightSourceFetch {
130 waiters: Vec<oneshot::Sender<RouteFetchOutcome>>,
131}
132
133enum SourceFetchOutcome {
134 Hit {
135 source_id: String,
136 data: Vec<u8>,
137 elapsed_ms: u64,
138 },
139 Miss {
140 source_id: String,
141 },
142 Failure {
143 source_id: String,
144 },
145}
146
147const INITIAL_SOURCE_BACKOFF_MS: u64 = 250;
148const MAX_SOURCE_BACKOFF_MS: u64 = 10_000;
149const SOURCE_SCORE_TIE_DELTA: f64 = 0.15;
150const RECENT_SOURCE_SUCCESS_WINDOW: Duration = Duration::from_secs(60);
151const ACTIVE_PEER_REQUEST_RANK_PENALTY: usize = 3;
152
153fn source_reliability_score(stats: &AdaptiveSourceStats) -> f64 {
154 (stats.successes as f64 + 1.0) / (stats.requests as f64 + 2.0)
155}
156
157fn source_latency_score(stats: &AdaptiveSourceStats) -> f64 {
158 if stats.srtt_ms <= 0.0 {
159 return 0.5;
160 }
161 (500.0 / (stats.srtt_ms + 50.0)).min(1.0)
162}
163
164fn source_has_history(stats: &AdaptiveSourceStats) -> bool {
165 stats.requests > 0
166 || stats.successes > 0
167 || stats.misses > 0
168 || stats.failures > 0
169 || stats.timeouts > 0
170}
171
172fn adaptive_source_score(stats: &AdaptiveSourceStats, now: Instant) -> f64 {
173 if let Some(backed_off_until) = stats.backed_off_until {
174 if backed_off_until > now {
175 return f64::NEG_INFINITY;
176 }
177 }
178
179 let miss_penalty = if stats.requests > 0 {
180 (stats.misses as f64 / stats.requests as f64) * 0.15
181 } else {
182 0.0
183 };
184 let failure_penalty = if stats.requests > 0 {
185 ((stats.failures + stats.timeouts) as f64 / stats.requests as f64) * 0.3
186 } else {
187 0.0
188 };
189 let recency_bonus = if stats
190 .last_success_at
191 .is_some_and(|last| now.duration_since(last) < RECENT_SOURCE_SUCCESS_WINDOW)
192 {
193 0.1
194 } else {
195 0.0
196 };
197
198 0.6 * source_reliability_score(stats) + 0.3 * source_latency_score(stats) + recency_bonus
199 - miss_penalty
200 - failure_penalty
201}
202
203fn peer_endpoint_has_history(stats: &crate::peer_selector::PeerStats) -> bool {
204 stats.requests_sent > 0 || stats.successes > 0 || stats.failures > 0 || stats.timeouts > 0
205}
206
207fn peer_endpoint_score(stats: &crate::peer_selector::PeerStats, now: Instant) -> f64 {
208 if stats.backed_off_until.is_some_and(|until| until > now) {
209 return f64::NEG_INFINITY;
210 }
211
212 let miss_penalty = 0.0;
213 let failure_penalty = if stats.requests_sent > 0 {
214 ((stats.failures + stats.timeouts) as f64 / stats.requests_sent as f64) * 0.3
215 } else {
216 0.0
217 };
218 let recency_bonus = if stats
219 .last_success
220 .is_some_and(|last| now.duration_since(last) < RECENT_SOURCE_SUCCESS_WINDOW)
221 {
222 0.1
223 } else {
224 0.0
225 };
226
227 0.6 * stats.success_rate()
228 + 0.3
229 * source_latency_score(&AdaptiveSourceStats {
230 srtt_ms: stats.srtt_ms,
231 ..AdaptiveSourceStats::default()
232 })
233 + recency_bonus
234 - miss_penalty
235 - failure_penalty
236}
237
238#[derive(Clone)]
239enum ReadRoute {
240 Peers(Vec<String>),
241 Sources,
242}
243
244impl ReadRoute {
245 fn id(&self) -> &'static str {
246 match self {
247 Self::Peers(_) => "peers",
248 Self::Sources => "sources",
249 }
250 }
251}
252
253struct RankedReadRoute {
254 route: ReadRoute,
255 best_endpoint_id: String,
256 score: f64,
257 has_history: bool,
258}
259
260fn ranked_route_kind(route: &ReadRoute) -> u8 {
261 match route {
262 ReadRoute::Sources => 0,
263 ReadRoute::Peers(_) => 1,
264 }
265}
266
267#[derive(Debug, Clone)]
268struct MeshReadContext {
269 exclude_peer_id: Option<String>,
270 request_htl: u8,
271}
272
273impl Default for MeshReadContext {
274 fn default() -> Self {
275 Self {
276 exclude_peer_id: None,
277 request_htl: MAX_HTL,
278 }
279 }
280}
281
282#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
284pub struct DataPumpStats {
285 pub processed: usize,
286 pub request_messages: usize,
287 pub response_messages: usize,
288 pub quote_request_messages: u64,
289 pub quote_response_messages: u64,
290 pub pubsub_interest_messages: u64,
291 pub pubsub_frame_messages: u64,
292 pub pubsub_inventory_messages: u64,
293 pub pubsub_want_messages: u64,
294 pub processed_bytes: u64,
295}
296
297#[derive(Debug, Clone, PartialEq, Eq)]
299pub struct PubsubEvent {
300 pub stream_id: String,
301 pub seq: u64,
302 pub origin_peer_id: String,
303 pub from_peer_id: String,
304 pub payload: Vec<u8>,
305}
306
307#[derive(Debug, Clone, PartialEq, Eq)]
309pub struct VerifiedBlockDelivery {
310 pub hash: Hash,
311 pub provider_peer_id: String,
312 pub payload_bytes: u64,
313}
314
315#[derive(Debug, Clone, Default, PartialEq, Eq)]
320pub struct VerifiedBlockDeliveryBatch {
321 pub deliveries: Vec<VerifiedBlockDelivery>,
322 pub dropped_since_last_drain: u64,
323}
324
325#[derive(Default)]
326struct VerifiedBlockDeliveryBuffer {
327 deliveries: VecDeque<VerifiedBlockDelivery>,
328 dropped_since_last_drain: u64,
329}
330
331#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
333pub struct PubsubPublishStats {
334 pub selected_peers: usize,
335 pub sent_peers: usize,
336 pub sent_bytes: u64,
337 pub deferred_peers: usize,
338}
339
340#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
342pub enum PubsubDeliveryMode {
343 InterestPush,
345 #[default]
347 HtlInvWant,
348}
349
350#[derive(Debug, Clone, Copy)]
356pub struct RequestDispatchConfig {
357 pub initial_fanout: usize,
359 pub hedge_fanout: usize,
361 pub max_fanout: usize,
363 pub hedge_interval_ms: u64,
365}
366
367impl Default for RequestDispatchConfig {
368 fn default() -> Self {
369 Self {
370 initial_fanout: usize::MAX,
371 hedge_fanout: usize::MAX,
372 max_fanout: usize::MAX,
373 hedge_interval_ms: 0,
374 }
375 }
376}
377
378pub fn normalize_dispatch_config(
380 dispatch: RequestDispatchConfig,
381 available_peers: usize,
382) -> RequestDispatchConfig {
383 let mut cfg = dispatch;
384 let cap = if cfg.max_fanout == 0 {
385 available_peers
386 } else {
387 cfg.max_fanout.min(available_peers)
388 };
389 cfg.max_fanout = cap;
390 cfg.initial_fanout = if cfg.initial_fanout == 0 {
391 1
392 } else {
393 cfg.initial_fanout.min(cap.max(1))
394 };
395 cfg.hedge_fanout = if cfg.hedge_fanout == 0 {
396 1
397 } else {
398 cfg.hedge_fanout.min(cap.max(1))
399 };
400 cfg
401}
402
403pub fn build_hedged_wave_plan(peer_count: usize, dispatch: RequestDispatchConfig) -> Vec<usize> {
405 if peer_count == 0 {
406 return Vec::new();
407 }
408 let cap = dispatch.max_fanout.min(peer_count);
409 if cap == 0 {
410 return Vec::new();
411 }
412
413 let mut plan = Vec::new();
414 let mut sent = 0usize;
415 let first = dispatch.initial_fanout.min(cap).max(1);
416 plan.push(first);
417 sent += first;
418
419 while sent < cap {
420 let next = dispatch.hedge_fanout.min(cap - sent).max(1);
421 plan.push(next);
422 sent += next;
423 }
424 plan
425}
426
427#[derive(Debug)]
429pub enum HedgedWaveAction<T> {
430 Continue,
431 Success(T),
432 Abort,
433}
434
435pub async fn run_hedged_waves<T, SendWave, SendWaveFut, WaitWave, WaitWaveFut>(
440 peer_count: usize,
441 dispatch: RequestDispatchConfig,
442 request_timeout: Duration,
443 mut send_wave: SendWave,
444 mut wait_wave: WaitWave,
445) -> Option<T>
446where
447 SendWave: FnMut(Range<usize>) -> SendWaveFut,
448 SendWaveFut: Future<Output = usize>,
449 WaitWave: FnMut(Duration) -> WaitWaveFut,
450 WaitWaveFut: Future<Output = HedgedWaveAction<T>>,
451{
452 let dispatch = normalize_dispatch_config(dispatch, peer_count);
453 let wave_plan = build_hedged_wave_plan(peer_count, dispatch);
454 if wave_plan.is_empty() {
455 return None;
456 }
457
458 let deadline = Instant::now() + request_timeout;
459 let mut sent_total = 0usize;
460 let mut next_peer_idx = 0usize;
461
462 for (wave_idx, wave_size) in wave_plan.iter().copied().enumerate() {
463 let from = next_peer_idx;
464 let to = (next_peer_idx + wave_size).min(peer_count);
465 next_peer_idx = to;
466
467 if from == to {
468 continue;
469 }
470
471 sent_total += send_wave(from..to).await;
472 if sent_total == 0 {
473 if next_peer_idx >= peer_count {
474 break;
475 }
476 continue;
477 }
478
479 let now = Instant::now();
480 if now >= deadline {
481 break;
482 }
483 let remaining = deadline.saturating_duration_since(now);
484 let is_last_wave = wave_idx + 1 == wave_plan.len() || next_peer_idx >= peer_count;
485 let wait = if is_last_wave {
486 remaining
487 } else if dispatch.hedge_interval_ms == 0 {
488 Duration::ZERO
489 } else {
490 Duration::from_millis(dispatch.hedge_interval_ms).min(remaining)
491 };
492
493 if wait.is_zero() {
494 continue;
495 }
496
497 match wait_wave(wait).await {
498 HedgedWaveAction::Continue => {}
499 HedgedWaveAction::Success(value) => return Some(value),
500 HedgedWaveAction::Abort => break,
501 }
502 }
503
504 None
505}
506
507pub async fn sync_selector_peers(selector: &RwLock<PeerSelector>, current_peer_ids: &[String]) {
509 let mut selector = selector.write().await;
510 let current: HashSet<&str> = current_peer_ids.iter().map(String::as_str).collect();
511 let known: Vec<String> = selector.all_stats().map(|s| s.peer_id.clone()).collect();
512 for peer_id in known {
513 if !current.contains(peer_id.as_str()) {
514 selector.remove_peer(&peer_id);
515 }
516 }
517 for peer_id in current_peer_ids {
518 selector.add_peer(peer_id.clone());
519 }
520}
521
522#[derive(Debug, Clone, Copy)]
526pub struct ResponseBehaviorConfig {
527 pub drop_response_prob: f64,
529 pub corrupt_response_prob: f64,
531 pub extra_delay_ms: u64,
533 pub first_byte_delay_ms: u64,
535 pub bytes_per_second: u64,
537 pub stall_response_prob: f64,
539 pub stall_delay_ms: u64,
541}
542
543impl Default for ResponseBehaviorConfig {
544 fn default() -> Self {
545 Self {
546 drop_response_prob: 0.0,
547 corrupt_response_prob: 0.0,
548 extra_delay_ms: 0,
549 first_byte_delay_ms: 0,
550 bytes_per_second: 0,
551 stall_response_prob: 0.0,
552 stall_delay_ms: 0,
553 }
554 }
555}
556
557impl ResponseBehaviorConfig {
558 fn normalized(self) -> Self {
559 Self {
560 drop_response_prob: self.drop_response_prob.clamp(0.0, 1.0),
561 corrupt_response_prob: self.corrupt_response_prob.clamp(0.0, 1.0),
562 extra_delay_ms: self.extra_delay_ms,
563 first_byte_delay_ms: self.first_byte_delay_ms,
564 bytes_per_second: self.bytes_per_second,
565 stall_response_prob: self.stall_response_prob.clamp(0.0, 1.0),
566 stall_delay_ms: self.stall_delay_ms,
567 }
568 }
569}
570
571#[derive(Debug, Clone)]
573pub struct MeshRoutingConfig {
574 pub selection_strategy: SelectionStrategy,
575 pub fairness_enabled: bool,
576 pub cashu_payment_weight: f64,
578 pub cashu_payment_default_block_threshold: u64,
581 pub cashu_accepted_mints: Vec<String>,
583 pub cashu_default_mint: Option<String>,
585 pub cashu_peer_suggested_mint_base_cap_sat: u64,
587 pub cashu_peer_suggested_mint_success_step_sat: u64,
589 pub cashu_peer_suggested_mint_receipt_step_sat: u64,
591 pub cashu_peer_suggested_mint_max_cap_sat: u64,
593 pub dispatch: RequestDispatchConfig,
594 pub response_behavior: ResponseBehaviorConfig,
595 pub pubsub_scheduler: PubsubSchedulerConfig,
596 pub pubsub_delivery_mode: PubsubDeliveryMode,
597 pub pubsub_forwarding: bool,
599 pub pubsub_max_htl: u8,
601}
602
603impl Default for MeshRoutingConfig {
604 fn default() -> Self {
605 Self {
606 selection_strategy: SelectionStrategy::Weighted,
607 fairness_enabled: true,
608 cashu_payment_weight: 0.0,
609 cashu_payment_default_block_threshold: 0,
610 cashu_accepted_mints: Vec::new(),
611 cashu_default_mint: None,
612 cashu_peer_suggested_mint_base_cap_sat: 0,
613 cashu_peer_suggested_mint_success_step_sat: 0,
614 cashu_peer_suggested_mint_receipt_step_sat: 0,
615 cashu_peer_suggested_mint_max_cap_sat: 0,
616 dispatch: RequestDispatchConfig::default(),
617 response_behavior: ResponseBehaviorConfig::default(),
618 pubsub_scheduler: PubsubSchedulerConfig::default(),
619 pubsub_delivery_mode: PubsubDeliveryMode::HtlInvWant,
620 pubsub_forwarding: true,
621 pubsub_max_htl: MESH_EVENT_POLICY.max_htl,
622 }
623 }
624}
625
626impl MeshRoutingConfig {
627 fn pubsub_initial_htl(&self) -> u8 {
628 self.pubsub_max_htl.clamp(1, MAX_HTL)
629 }
630}
631
632pub struct MeshStoreCore<S, R, F>
639where
640 S: Store + Send + Sync + 'static,
641 R: SignalingTransport + Send + Sync + 'static,
642 F: PeerLinkFactory + Send + Sync + 'static,
643{
644 local_store: Arc<S>,
646 signaling: Arc<MeshRouter<R, F>>,
648 htl_configs: RwLock<HashMap<String, PeerHTLConfig>>,
650 pending_requests: RwLock<HashMap<String, PendingRequest>>,
652 pending_quotes: RwLock<HashMap<String, PendingQuoteRequest>>,
654 pending_forward_requests: RwLock<HashMap<String, PendingForwardRequest>>,
656 recent_forward_misses: Mutex<TimedSeenSet>,
658 issued_quotes: RwLock<HashMap<(String, String, u64), IssuedQuote>>,
660 next_quote_id: RwLock<u64>,
662 read_sources: RwLock<HashMap<String, Arc<dyn MeshReadSource>>>,
664 read_source_stats: RwLock<HashMap<String, AdaptiveSourceStats>>,
666 inflight_source_fetches: Mutex<HashMap<String, InflightSourceFetch>>,
668 peer_selector: RwLock<PeerSelector>,
670 peer_active_requests: RwLock<HashMap<String, usize>>,
672 peer_wire_stats: RwLock<HashMap<String, PeerWireStats>>,
674 pubsub_local_interests: RwLock<HashSet<String>>,
676 pubsub_local_interest_versions: RwLock<HashMap<String, u64>>,
678 pubsub_peer_interests: RwLock<HashMap<String, HashSet<String>>>,
680 pubsub_interest_routes: RwLock<HashMap<(String, String), String>>,
682 pubsub_interest_versions: RwLock<HashMap<(String, String), u64>>,
684 pubsub_seen_interests: Mutex<TimedSeenSet>,
686 pubsub_seen_frames: Mutex<TimedSeenSet>,
688 pubsub_seen_inventories: Mutex<TimedSeenSet>,
690 pubsub_seen_wants: Mutex<TimedSeenSet>,
692 pubsub_inventory_routes: RwLock<HashMap<String, String>>,
694 pubsub_want_routes: RwLock<HashMap<String, HashSet<String>>>,
696 pubsub_upstream_wants: Mutex<TimedSeenSet>,
698 pubsub_frame_cache: Mutex<VecDeque<(String, PubsubFrame)>>,
700 pubsub_inbox: Mutex<VecDeque<PubsubEvent>>,
702 verified_block_deliveries: Mutex<VerifiedBlockDeliveryBuffer>,
704 pubsub_notify: Notify,
706 pubsub_deferred_counts: RwLock<HashMap<(String, String), u64>>,
708 next_pubsub_interest_seq: AtomicU64,
710 pending_response_sends: Mutex<Vec<PendingResponseSend>>,
712 response_scheduler_running: AtomicBool,
714 next_response_job_id: AtomicU64,
716 routing: MeshRoutingConfig,
718 request_timeout: Duration,
720 debug: bool,
722 running: RwLock<bool>,
724}
725
726impl<S, R, F> MeshStoreCore<S, R, F>
727where
728 S: Store + Send + Sync + 'static,
729 R: SignalingTransport + Send + Sync + 'static,
730 F: PeerLinkFactory + Send + Sync + 'static,
731{
732 pub fn new(
734 local_store: Arc<S>,
735 signaling: Arc<MeshRouter<R, F>>,
736 request_timeout: Duration,
737 debug: bool,
738 ) -> Self {
739 Self::new_with_routing(
740 local_store,
741 signaling,
742 request_timeout,
743 debug,
744 Default::default(),
745 )
746 }
747
748 pub fn new_with_routing(
750 local_store: Arc<S>,
751 signaling: Arc<MeshRouter<R, F>>,
752 request_timeout: Duration,
753 debug: bool,
754 routing: MeshRoutingConfig,
755 ) -> Self {
756 let mut selector = PeerSelector::with_strategy(routing.selection_strategy);
757 selector.set_fairness(routing.fairness_enabled);
758 selector.set_cashu_payment_weight(routing.cashu_payment_weight);
759 Self {
760 local_store,
761 signaling,
762 htl_configs: RwLock::new(HashMap::new()),
763 pending_requests: RwLock::new(HashMap::new()),
764 pending_quotes: RwLock::new(HashMap::new()),
765 pending_forward_requests: RwLock::new(HashMap::new()),
766 recent_forward_misses: Mutex::new(TimedSeenSet::new(
767 RECENT_FORWARD_MISS_CAPACITY,
768 Self::recent_forward_miss_ttl(request_timeout),
769 )),
770 issued_quotes: RwLock::new(HashMap::new()),
771 next_quote_id: RwLock::new(1),
772 read_sources: RwLock::new(HashMap::new()),
773 read_source_stats: RwLock::new(HashMap::new()),
774 inflight_source_fetches: Mutex::new(HashMap::new()),
775 peer_selector: RwLock::new(selector),
776 peer_active_requests: RwLock::new(HashMap::new()),
777 peer_wire_stats: RwLock::new(HashMap::new()),
778 pubsub_local_interests: RwLock::new(HashSet::new()),
779 pubsub_local_interest_versions: RwLock::new(HashMap::new()),
780 pubsub_peer_interests: RwLock::new(HashMap::new()),
781 pubsub_interest_routes: RwLock::new(HashMap::new()),
782 pubsub_interest_versions: RwLock::new(HashMap::new()),
783 pubsub_seen_interests: Mutex::new(TimedSeenSet::new(
784 PUBSUB_SEEN_CAPACITY,
785 PUBSUB_SEEN_TTL,
786 )),
787 pubsub_seen_frames: Mutex::new(TimedSeenSet::new(
788 PUBSUB_SEEN_CAPACITY,
789 PUBSUB_SEEN_TTL,
790 )),
791 pubsub_seen_inventories: Mutex::new(TimedSeenSet::new(
792 PUBSUB_SEEN_CAPACITY,
793 PUBSUB_SEEN_TTL,
794 )),
795 pubsub_seen_wants: Mutex::new(TimedSeenSet::new(PUBSUB_SEEN_CAPACITY, PUBSUB_SEEN_TTL)),
796 pubsub_inventory_routes: RwLock::new(HashMap::new()),
797 pubsub_want_routes: RwLock::new(HashMap::new()),
798 pubsub_upstream_wants: Mutex::new(TimedSeenSet::new(
799 PUBSUB_SEEN_CAPACITY,
800 PUBSUB_SEEN_TTL,
801 )),
802 pubsub_frame_cache: Mutex::new(VecDeque::new()),
803 pubsub_inbox: Mutex::new(VecDeque::new()),
804 verified_block_deliveries: Mutex::new(VerifiedBlockDeliveryBuffer::default()),
805 pubsub_notify: Notify::new(),
806 pubsub_deferred_counts: RwLock::new(HashMap::new()),
807 next_pubsub_interest_seq: AtomicU64::new(1),
808 pending_response_sends: Mutex::new(Vec::new()),
809 response_scheduler_running: AtomicBool::new(false),
810 next_response_job_id: AtomicU64::new(1),
811 routing,
812 request_timeout,
813 debug,
814 running: RwLock::new(false),
815 }
816 }
817
818 fn recent_forward_miss_ttl(request_timeout: Duration) -> Duration {
819 let ttl_ms = request_timeout
820 .as_millis()
821 .saturating_mul(2)
822 .max(MIN_RECENT_FORWARD_MISS_TTL_MS as u128)
823 .min(u64::MAX as u128) as u64;
824 Duration::from_millis(ttl_ms)
825 }
826
827 pub async fn start(&self) -> Result<(), TransportError> {
829 *self.running.write().await = true;
830
831 self.signaling.send_hello(vec![]).await?;
833
834 Ok(())
835 }
836
837 pub async fn stop(&self) {
839 *self.running.write().await = false;
840 }
841
842 pub async fn process_signaling(&self, msg: SignalingMessage) -> Result<(), TransportError> {
844 let peer_id = msg.peer_id().to_string();
846 {
847 let mut configs = self.htl_configs.write().await;
848 if !configs.contains_key(&peer_id) {
849 configs.insert(peer_id.clone(), PeerHTLConfig::random());
850 }
851 }
852 self.peer_selector.write().await.add_peer(peer_id.clone());
853
854 let result = self.signaling.handle_message(msg).await;
855 if result.is_ok() {
856 self.announce_pubsub_interests_to_peer(&peer_id).await;
857 }
858 result
859 }
860
861 pub fn signaling(&self) -> &Arc<MeshRouter<R, F>> {
863 &self.signaling
864 }
865
866 fn response_behavior(&self) -> ResponseBehaviorConfig {
867 self.routing.response_behavior.normalized()
868 }
869
870 async fn record_peer_wire_sent(&self, peer_id: &str, bytes: u64) {
871 if bytes == 0 {
872 return;
873 }
874 let mut stats = self.peer_wire_stats.write().await;
875 let entry = stats.entry(peer_id.to_string()).or_default();
876 entry.bytes_sent = entry.bytes_sent.saturating_add(bytes);
877 }
878
879 async fn record_peer_wire_received(&self, peer_id: &str, bytes: u64) {
880 if bytes == 0 {
881 return;
882 }
883 let mut stats = self.peer_wire_stats.write().await;
884 let entry = stats.entry(peer_id.to_string()).or_default();
885 entry.bytes_received = entry.bytes_received.saturating_add(bytes);
886 }
887
888 pub async fn record_useful_bytes_received_from_peer(&self, peer_id: &str, bytes: u64) {
893 if bytes == 0 {
894 return;
895 }
896 let mut stats = self.peer_wire_stats.write().await;
897 let entry = stats.entry(peer_id.to_string()).or_default();
898 entry.useful_bytes_received = entry.useful_bytes_received.saturating_add(bytes);
899 }
900
901 pub async fn peer_traffic_snapshot(&self, peer_id: &str) -> PeerTrafficSnapshot {
903 self.peer_wire_stats
904 .read()
905 .await
906 .get(peer_id)
907 .copied()
908 .unwrap_or_default()
909 }
910
911 pub async fn peer_traffic_snapshots(&self) -> HashMap<String, PeerTrafficSnapshot> {
913 self.peer_wire_stats.read().await.clone()
914 }
915
916 fn pubsub_key(origin_peer_id: &str, stream_id: &str, seq: u64) -> String {
917 format!("{origin_peer_id}:{stream_id}:{seq}")
918 }
919
920 fn pubsub_frame_key(frame: &PubsubFrame) -> String {
921 Self::pubsub_key(&frame.origin_peer_id, &frame.stream_id, frame.seq)
922 }
923
924 fn pubsub_interest_key(interest: &PubsubInterest) -> String {
925 format!(
926 "{}:{}:{}:{}",
927 interest.subscriber_peer_id, interest.stream_id, interest.seq, interest.active
928 )
929 }
930
931 fn next_pubsub_interest_seq(&self) -> u64 {
932 self.next_pubsub_interest_seq
933 .fetch_add(1, Ordering::Relaxed)
934 }
935
936 async fn record_peer_pubsub_wire_sent(&self, peer_id: &str, bytes: u64, bandwidth_debt: f64) {
937 if bytes == 0 {
938 return;
939 }
940 let mut stats = self.peer_wire_stats.write().await;
941 let entry = stats.entry(peer_id.to_string()).or_default();
942 entry.bytes_sent = entry.bytes_sent.saturating_add(bytes);
943 entry.bandwidth_debt = bandwidth_debt;
944 }
945
946 async fn send_pubsub_interest_to_peers(
947 &self,
948 interest: &PubsubInterest,
949 exclude_peer_id: Option<&str>,
950 ) -> PubsubPublishStats {
951 if !should_forward_htl(interest.htl) {
952 return PubsubPublishStats::default();
953 }
954
955 let mut peer_ids = self.signaling.peer_ids().await;
956 peer_ids.sort();
957 peer_ids.retain(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude));
958
959 let bytes = encode_pubsub_interest(interest);
960 let mut stats = PubsubPublishStats {
961 selected_peers: peer_ids.len(),
962 ..Default::default()
963 };
964 for peer_id in peer_ids {
965 let Some(channel) = self.signaling.get_channel(&peer_id).await else {
966 continue;
967 };
968 if channel.send(bytes.clone()).await.is_ok() {
969 stats.sent_peers += 1;
970 stats.sent_bytes = stats.sent_bytes.saturating_add(bytes.len() as u64);
971 self.record_peer_wire_sent(&peer_id, bytes.len() as u64)
972 .await;
973 }
974 }
975 stats
976 }
977
978 async fn announce_pubsub_interests_to_peer(&self, peer_id: &str) {
979 let mut interests = self
980 .pubsub_local_interests
981 .read()
982 .await
983 .iter()
984 .cloned()
985 .collect::<Vec<_>>();
986 interests.sort();
987 if interests.is_empty() {
988 return;
989 }
990
991 let interests = {
992 let versions = self.pubsub_local_interest_versions.read().await;
993 interests
994 .into_iter()
995 .filter_map(|stream_id| {
996 versions
997 .get(&stream_id)
998 .copied()
999 .map(|seq| (stream_id, seq))
1000 })
1001 .collect::<Vec<_>>()
1002 };
1003
1004 for (stream_id, seq) in interests {
1005 let interest = create_pubsub_interest(
1006 stream_id,
1007 self.signaling.peer_id().to_string(),
1008 seq,
1009 true,
1010 MAX_HTL,
1011 );
1012 let Some(channel) = self.signaling.get_channel(peer_id).await else {
1013 continue;
1014 };
1015 let bytes = encode_pubsub_interest(&interest);
1016 if channel.send(bytes.clone()).await.is_ok() {
1017 self.record_peer_wire_sent(peer_id, bytes.len() as u64)
1018 .await;
1019 }
1020 }
1021 }
1022
1023 fn remove_pubsub_peer_interest(
1024 peer_interests: &mut HashMap<String, HashSet<String>>,
1025 routes: &HashMap<(String, String), String>,
1026 stream_id: &str,
1027 peer_id: &str,
1028 ) {
1029 let still_has_route = routes
1030 .iter()
1031 .any(|((stream, _subscriber), peer)| stream == stream_id && peer == peer_id);
1032 if still_has_route {
1033 return;
1034 }
1035 if let Some(peers) = peer_interests.get_mut(stream_id) {
1036 peers.remove(peer_id);
1037 if peers.is_empty() {
1038 peer_interests.remove(stream_id);
1039 }
1040 }
1041 }
1042
1043 async fn apply_pubsub_interest_route(
1044 &self,
1045 from_peer: &str,
1046 interest: &PubsubInterest,
1047 ) -> bool {
1048 if interest.stream_id.is_empty() || interest.subscriber_peer_id.is_empty() {
1049 return false;
1050 }
1051 if interest.subscriber_peer_id == self.signaling.peer_id() {
1052 return false;
1053 }
1054
1055 let interest_key = Self::pubsub_interest_key(interest);
1056 if !self
1057 .pubsub_seen_interests
1058 .lock()
1059 .await
1060 .insert_if_new(interest_key)
1061 {
1062 return false;
1063 }
1064
1065 let route_key = (
1066 interest.stream_id.clone(),
1067 interest.subscriber_peer_id.clone(),
1068 );
1069 {
1070 let mut versions = self.pubsub_interest_versions.write().await;
1071 if versions
1072 .get(&route_key)
1073 .is_some_and(|latest| *latest >= interest.seq)
1074 {
1075 return false;
1076 }
1077 versions.insert(route_key.clone(), interest.seq);
1078 }
1079
1080 let mut peer_interests = self.pubsub_peer_interests.write().await;
1081 let mut routes = self.pubsub_interest_routes.write().await;
1082 if interest.active {
1083 if let Some(previous_peer) = routes.insert(route_key, from_peer.to_string()) {
1084 if previous_peer != from_peer {
1085 Self::remove_pubsub_peer_interest(
1086 &mut peer_interests,
1087 &routes,
1088 &interest.stream_id,
1089 &previous_peer,
1090 );
1091 }
1092 }
1093 peer_interests
1094 .entry(interest.stream_id.clone())
1095 .or_default()
1096 .insert(from_peer.to_string());
1097 } else if let Some(previous_peer) = routes.remove(&route_key) {
1098 Self::remove_pubsub_peer_interest(
1099 &mut peer_interests,
1100 &routes,
1101 &interest.stream_id,
1102 &previous_peer,
1103 );
1104 } else {
1105 Self::remove_pubsub_peer_interest(
1106 &mut peer_interests,
1107 &routes,
1108 &interest.stream_id,
1109 from_peer,
1110 );
1111 }
1112
1113 true
1114 }
1115
1116 async fn interested_pubsub_peers(
1117 &self,
1118 stream_id: &str,
1119 exclude_peer_id: Option<&str>,
1120 ) -> Vec<String> {
1121 let connected = self
1122 .signaling
1123 .peer_ids()
1124 .await
1125 .into_iter()
1126 .collect::<HashSet<_>>();
1127 let mut peers = self
1128 .pubsub_peer_interests
1129 .read()
1130 .await
1131 .get(stream_id)
1132 .map(|peers| peers.iter().cloned().collect::<Vec<_>>())
1133 .unwrap_or_default();
1134 peers.retain(|peer_id| {
1135 connected.contains(peer_id) && exclude_peer_id.is_none_or(|exclude| peer_id != exclude)
1136 });
1137 peers.sort();
1138 peers
1139 }
1140
1141 async fn decrement_pubsub_htl_for_peer(&self, peer_id: &str, htl: u8) -> u8 {
1142 let htl_config = {
1143 let configs = self.htl_configs.read().await;
1144 configs
1145 .get(peer_id)
1146 .cloned()
1147 .unwrap_or_else(PeerHTLConfig::random)
1148 };
1149 htl_config.decrement_with_policy(htl, &MESH_EVENT_POLICY)
1150 }
1151
1152 async fn send_pubsub_inventory_to_peers(
1153 &self,
1154 inv: &PubsubInventory,
1155 peer_ids: &[String],
1156 ) -> PubsubPublishStats {
1157 if peer_ids.is_empty() || !should_forward_htl(inv.htl) {
1158 return PubsubPublishStats::default();
1159 }
1160
1161 let mut stats = PubsubPublishStats {
1162 selected_peers: peer_ids.len(),
1163 ..Default::default()
1164 };
1165 for peer_id in peer_ids {
1166 let send_htl = self.decrement_pubsub_htl_for_peer(peer_id, inv.htl).await;
1167 if !should_forward_htl(send_htl) {
1168 continue;
1169 }
1170 let Some(channel) = self.signaling.get_channel(peer_id).await else {
1171 continue;
1172 };
1173 let mut outgoing = inv.clone();
1174 outgoing.htl = send_htl;
1175 let bytes = encode_pubsub_inventory(&outgoing);
1176 let message_bytes = bytes.len() as u64;
1177 if channel.send(bytes).await.is_ok() {
1178 stats.sent_peers += 1;
1179 stats.sent_bytes = stats.sent_bytes.saturating_add(message_bytes);
1180 self.record_peer_wire_sent(peer_id, message_bytes).await;
1181 }
1182 }
1183 stats
1184 }
1185
1186 async fn send_pubsub_want_to_peer(&self, want: &PubsubWant, peer_id: &str) -> bool {
1187 let Some(channel) = self.signaling.get_channel(peer_id).await else {
1188 return false;
1189 };
1190 let bytes = encode_pubsub_want(want);
1191 let message_bytes = bytes.len() as u64;
1192 match channel.send(bytes).await {
1193 Ok(()) => {
1194 self.record_peer_wire_sent(peer_id, message_bytes).await;
1195 true
1196 }
1197 Err(_) => false,
1198 }
1199 }
1200
1201 async fn send_pubsub_want_upstream(
1202 &self,
1203 key: &str,
1204 want: &PubsubWant,
1205 exclude_peer_id: Option<&str>,
1206 ) -> bool {
1207 let upstream = {
1208 let routes = self.pubsub_inventory_routes.read().await;
1209 routes.get(key).cloned()
1210 };
1211 let Some(upstream) = upstream else {
1212 return false;
1213 };
1214 if exclude_peer_id.is_some_and(|exclude| exclude == upstream) {
1215 return false;
1216 }
1217 let want_key = format!("{key}:{upstream}");
1218 if !self
1219 .pubsub_upstream_wants
1220 .lock()
1221 .await
1222 .insert_if_new(want_key)
1223 {
1224 return false;
1225 }
1226 self.send_pubsub_want_to_peer(want, &upstream).await
1227 }
1228
1229 async fn cache_pubsub_frame(&self, key: String, frame: PubsubFrame) {
1230 let mut cache = self.pubsub_frame_cache.lock().await;
1231 if let Some(index) = cache.iter().position(|(cached_key, _)| cached_key == &key) {
1232 cache.remove(index);
1233 }
1234 cache.push_back((key, frame));
1235 while cache.len() > PUBSUB_FRAME_CACHE_CAPACITY {
1236 cache.pop_front();
1237 }
1238 }
1239
1240 async fn cached_pubsub_frame(&self, key: &str) -> Option<PubsubFrame> {
1241 self.pubsub_frame_cache
1242 .lock()
1243 .await
1244 .iter()
1245 .find_map(|(cached_key, frame)| {
1246 if cached_key == key {
1247 Some(frame.clone())
1248 } else {
1249 None
1250 }
1251 })
1252 }
1253
1254 async fn remember_pubsub_want_peer(&self, key: String, from_peer: &str) -> bool {
1255 let mut routes = self.pubsub_want_routes.write().await;
1256 routes.entry(key).or_default().insert(from_peer.to_string())
1257 }
1258
1259 async fn take_pubsub_want_peers(
1260 &self,
1261 key: &str,
1262 exclude_peer_id: Option<&str>,
1263 ) -> Vec<String> {
1264 let connected = self
1265 .signaling
1266 .peer_ids()
1267 .await
1268 .into_iter()
1269 .collect::<HashSet<_>>();
1270 let mut peers = self
1271 .pubsub_want_routes
1272 .write()
1273 .await
1274 .remove(key)
1275 .map(|peers| peers.into_iter().collect::<Vec<_>>())
1276 .unwrap_or_default();
1277 peers.retain(|peer_id| {
1278 connected.contains(peer_id) && exclude_peer_id.is_none_or(|exclude| peer_id != exclude)
1279 });
1280 peers.sort();
1281 peers
1282 }
1283
1284 async fn select_pubsub_peers(
1285 &self,
1286 stream_id: &str,
1287 seq: u64,
1288 message_bytes: u64,
1289 peer_ids: &[String],
1290 ) -> (Vec<String>, Vec<String>) {
1291 let traffic = self.peer_wire_stats.read().await;
1292 let deferred_counts = self.pubsub_deferred_counts.read().await;
1293 let candidates = peer_ids
1294 .iter()
1295 .map(|peer_id| PubsubCandidate {
1296 peer_id: peer_id.clone(),
1297 traffic: traffic.get(peer_id).copied().unwrap_or_default(),
1298 deferred_count: deferred_counts
1299 .get(&(stream_id.to_string(), peer_id.clone()))
1300 .copied()
1301 .unwrap_or_default(),
1302 })
1303 .collect::<Vec<_>>();
1304 drop(deferred_counts);
1305 drop(traffic);
1306
1307 let selection = self.routing.pubsub_scheduler.select(
1308 stream_id,
1309 seq,
1310 self.signaling.peer_id(),
1311 message_bytes,
1312 &candidates,
1313 );
1314
1315 {
1316 let mut deferred_counts = self.pubsub_deferred_counts.write().await;
1317 for peer_id in &selection.deferred {
1318 *deferred_counts
1319 .entry((stream_id.to_string(), peer_id.clone()))
1320 .or_insert(0) += 1;
1321 }
1322 for peer_id in &selection.selected {
1323 deferred_counts.remove(&(stream_id.to_string(), peer_id.clone()));
1324 }
1325 }
1326
1327 (selection.selected, selection.deferred)
1328 }
1329
1330 async fn send_pubsub_frame_to_peers(
1331 &self,
1332 frame: &PubsubFrame,
1333 peer_ids: &[String],
1334 ) -> PubsubPublishStats {
1335 if peer_ids.is_empty() || !should_forward_htl(frame.htl) {
1336 return PubsubPublishStats::default();
1337 }
1338
1339 let bytes = encode_pubsub_frame(frame);
1340 let message_bytes = bytes.len() as u64;
1341 let (selected, deferred) = self
1342 .select_pubsub_peers(&frame.stream_id, frame.seq, message_bytes, peer_ids)
1343 .await;
1344 let mut stats = PubsubPublishStats {
1345 selected_peers: selected.len(),
1346 deferred_peers: deferred.len(),
1347 ..Default::default()
1348 };
1349
1350 for peer_id in selected {
1351 let Some(channel) = self.signaling.get_channel(&peer_id).await else {
1352 continue;
1353 };
1354 let snapshot = self.peer_traffic_snapshot(&peer_id).await;
1355 let bandwidth_debt = reciprocal_virtual_finish(snapshot, message_bytes);
1356 if channel.send(bytes.clone()).await.is_ok() {
1357 stats.sent_peers += 1;
1358 stats.sent_bytes = stats.sent_bytes.saturating_add(message_bytes);
1359 self.record_peer_pubsub_wire_sent(&peer_id, message_bytes, bandwidth_debt)
1360 .await;
1361 }
1362 }
1363
1364 stats
1365 }
1366
1367 async fn enqueue_pubsub_event(&self, event: PubsubEvent) {
1368 let mut inbox = self.pubsub_inbox.lock().await;
1369 inbox.push_back(event);
1370 while inbox.len() > PUBSUB_INBOX_CAPACITY {
1371 inbox.pop_front();
1372 }
1373 self.pubsub_notify.notify_one();
1374 }
1375
1376 pub async fn subscribe_pubsub(
1378 self: &Arc<Self>,
1379 stream_id: impl Into<String>,
1380 ) -> PubsubPublishStats {
1381 let stream_id = stream_id.into();
1382 if stream_id.is_empty() {
1383 return PubsubPublishStats::default();
1384 }
1385 self.pubsub_local_interests
1386 .write()
1387 .await
1388 .insert(stream_id.clone());
1389 let seq = {
1390 let mut versions = self.pubsub_local_interest_versions.write().await;
1391 match versions.get(&stream_id).copied() {
1392 Some(seq) => seq,
1393 None => {
1394 let seq = self.next_pubsub_interest_seq();
1395 versions.insert(stream_id.clone(), seq);
1396 seq
1397 }
1398 }
1399 };
1400 let interest = create_pubsub_interest(
1401 stream_id,
1402 self.signaling.peer_id().to_string(),
1403 seq,
1404 true,
1405 self.routing.pubsub_initial_htl(),
1406 );
1407 self.send_pubsub_interest_to_peers(&interest, None).await
1408 }
1409
1410 pub async fn unsubscribe_pubsub(
1412 self: &Arc<Self>,
1413 stream_id: impl Into<String>,
1414 ) -> PubsubPublishStats {
1415 let stream_id = stream_id.into();
1416 if stream_id.is_empty() {
1417 return PubsubPublishStats::default();
1418 }
1419 self.pubsub_local_interests.write().await.remove(&stream_id);
1420 self.pubsub_local_interest_versions
1421 .write()
1422 .await
1423 .remove(&stream_id);
1424 let interest = create_pubsub_interest(
1425 stream_id,
1426 self.signaling.peer_id().to_string(),
1427 self.next_pubsub_interest_seq(),
1428 false,
1429 self.routing.pubsub_initial_htl(),
1430 );
1431 self.send_pubsub_interest_to_peers(&interest, None).await
1432 }
1433
1434 pub async fn publish_pubsub(
1436 self: &Arc<Self>,
1437 stream_id: impl Into<String>,
1438 seq: u64,
1439 payload: Vec<u8>,
1440 ) -> PubsubPublishStats {
1441 let stream_id = stream_id.into();
1442 if stream_id.is_empty() {
1443 return PubsubPublishStats::default();
1444 }
1445 let payload_bytes = payload.len() as u64;
1446 let frame = create_pubsub_frame(
1447 stream_id.clone(),
1448 seq,
1449 self.signaling.peer_id().to_string(),
1450 payload.clone(),
1451 self.routing.pubsub_initial_htl(),
1452 );
1453 let frame_key = Self::pubsub_frame_key(&frame);
1454 self.pubsub_seen_frames
1455 .lock()
1456 .await
1457 .insert_if_new(frame_key.clone());
1458 self.cache_pubsub_frame(frame_key, frame.clone()).await;
1459
1460 if self
1461 .pubsub_local_interests
1462 .read()
1463 .await
1464 .contains(&stream_id)
1465 {
1466 self.enqueue_pubsub_event(PubsubEvent {
1467 stream_id: stream_id.clone(),
1468 seq,
1469 origin_peer_id: self.signaling.peer_id().to_string(),
1470 from_peer_id: self.signaling.peer_id().to_string(),
1471 payload,
1472 })
1473 .await;
1474 }
1475
1476 match self.routing.pubsub_delivery_mode {
1477 PubsubDeliveryMode::InterestPush => {
1478 let peers = self.interested_pubsub_peers(&stream_id, None).await;
1479 self.send_pubsub_frame_to_peers(&frame, &peers).await
1480 }
1481 PubsubDeliveryMode::HtlInvWant => {
1482 let inv = create_pubsub_inventory(
1483 stream_id,
1484 seq,
1485 self.signaling.peer_id().to_string(),
1486 payload_bytes,
1487 self.routing.pubsub_initial_htl(),
1488 );
1489 let peers = self.interested_pubsub_peers(&inv.stream_id, None).await;
1490 self.send_pubsub_inventory_to_peers(&inv, &peers).await
1491 }
1492 }
1493 }
1494
1495 pub async fn drain_pubsub_events(&self) -> Vec<PubsubEvent> {
1497 self.pubsub_inbox.lock().await.drain(..).collect()
1498 }
1499
1500 pub async fn drain_verified_block_deliveries(&self) -> VerifiedBlockDeliveryBatch {
1502 let mut buffer = self.verified_block_deliveries.lock().await;
1503 VerifiedBlockDeliveryBatch {
1504 deliveries: buffer.deliveries.drain(..).collect(),
1505 dropped_since_last_drain: std::mem::take(&mut buffer.dropped_since_last_drain),
1506 }
1507 }
1508
1509 pub async fn recv_pubsub_event(&self) -> PubsubEvent {
1511 loop {
1512 if let Some(event) = self.pubsub_inbox.lock().await.pop_front() {
1513 return event;
1514 }
1515 self.pubsub_notify.notified().await;
1516 }
1517 }
1518
1519 pub async fn pubsub_interest_peers(&self, stream_id: &str) -> Vec<String> {
1521 self.interested_pubsub_peers(stream_id, None).await
1522 }
1523
1524 fn choose_ready_response_job(
1525 ready_jobs: &[(u64, String, usize, Instant, u64)],
1526 stats: &HashMap<String, PeerWireStats>,
1527 ) -> Option<(u64, f64)> {
1528 let jobs = ready_jobs
1529 .iter()
1530 .map(|job| OutboundJobCandidate {
1531 job_id: job.0,
1532 peer_id: job.1.clone(),
1533 message_bytes: job.2 as u64,
1534 queue_sequence: job.4,
1535 })
1536 .collect::<Vec<_>>();
1537 select_reciprocal_outbound_job(&jobs, |peer_id| {
1538 stats.get(peer_id).copied().unwrap_or_default()
1539 })
1540 .map(|choice| (choice.job_id, choice.virtual_finish))
1541 }
1542
1543 async fn enqueue_response_send(
1544 self: &Arc<Self>,
1545 peer_id: String,
1546 bytes: Vec<u8>,
1547 ready_at: Instant,
1548 ) {
1549 let job_id = self.next_response_job_id.fetch_add(1, Ordering::Relaxed);
1550 {
1551 let mut queue = self.pending_response_sends.lock().await;
1552 queue.push(PendingResponseSend {
1553 job_id,
1554 peer_id,
1555 bytes,
1556 ready_at,
1557 queue_sequence: job_id,
1558 });
1559 }
1560
1561 if self
1562 .response_scheduler_running
1563 .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
1564 .is_ok()
1565 {
1566 let this = Arc::clone(self);
1567 tokio::spawn(async move {
1568 this.run_response_scheduler().await;
1569 });
1570 }
1571 }
1572
1573 async fn run_response_scheduler(self: Arc<Self>) {
1574 loop {
1575 let snapshot = {
1576 let queue = self.pending_response_sends.lock().await;
1577 if queue.is_empty() {
1578 self.response_scheduler_running
1579 .store(false, Ordering::Release);
1580 return;
1581 }
1582 queue
1583 .iter()
1584 .map(|job| {
1585 (
1586 job.job_id,
1587 job.peer_id.clone(),
1588 job.bytes.len(),
1589 job.ready_at,
1590 job.queue_sequence,
1591 )
1592 })
1593 .collect::<Vec<_>>()
1594 };
1595
1596 let now = Instant::now();
1597 let mut earliest_ready_at: Option<Instant> = None;
1598 let mut ready_jobs = Vec::new();
1599 for job in &snapshot {
1600 if job.3 <= now {
1601 ready_jobs.push(job.clone());
1602 } else {
1603 earliest_ready_at = Some(match earliest_ready_at {
1604 Some(current) => current.min(job.3),
1605 None => job.3,
1606 });
1607 }
1608 }
1609
1610 if ready_jobs.is_empty() {
1611 if let Some(ready_at) = earliest_ready_at {
1612 tokio::time::sleep(ready_at.saturating_duration_since(Instant::now())).await;
1613 continue;
1614 }
1615 self.response_scheduler_running
1616 .store(false, Ordering::Release);
1617 return;
1618 }
1619
1620 let (selected_job_id, selected_finish) = {
1621 let stats = self.peer_wire_stats.read().await;
1622 Self::choose_ready_response_job(&ready_jobs, &stats).expect("ready response job")
1623 };
1624
1625 let selected = {
1626 let mut queue = self.pending_response_sends.lock().await;
1627 let Some(index) = queue.iter().position(|job| job.job_id == selected_job_id) else {
1628 continue;
1629 };
1630 queue.swap_remove(index)
1631 };
1632
1633 let sent = if let Some(channel) = self.signaling.get_channel(&selected.peer_id).await {
1634 channel.send(selected.bytes.clone()).await.is_ok()
1635 } else {
1636 false
1637 };
1638
1639 let queued_peers = {
1640 let queue = self.pending_response_sends.lock().await;
1641 queue
1642 .iter()
1643 .map(|job| job.peer_id.clone())
1644 .collect::<HashSet<_>>()
1645 };
1646 let mut stats = self.peer_wire_stats.write().await;
1647 let entry = stats.entry(selected.peer_id.clone()).or_default();
1648 if sent {
1649 entry.bytes_sent = entry.bytes_sent.saturating_add(selected.bytes.len() as u64);
1650 entry.bandwidth_debt = selected_finish;
1651 }
1652 if queued_peers.is_empty() {
1653 for peer_stats in stats.values_mut() {
1654 peer_stats.bandwidth_debt = 0.0;
1655 }
1656 } else {
1657 let floor = queued_peers
1658 .iter()
1659 .filter_map(|peer_id| stats.get(peer_id).map(|peer| peer.bandwidth_debt))
1660 .fold(f64::INFINITY, f64::min);
1661 if floor.is_finite() && floor > 0.0 {
1662 for peer_id in queued_peers {
1663 if let Some(peer_stats) = stats.get_mut(&peer_id) {
1664 peer_stats.bandwidth_debt =
1665 (peer_stats.bandwidth_debt - floor).max(0.0);
1666 }
1667 }
1668 }
1669 }
1670 }
1671 }
1672
1673 fn deterministic_actor_draw_for(peer_id: &str, hash: &Hash, salt: u64) -> f64 {
1674 let mut hasher = DefaultHasher::new();
1675 peer_id.hash(&mut hasher);
1676 hash.hash(&mut hasher);
1677 salt.hash(&mut hasher);
1678 let v = hasher.finish();
1679 (v as f64) / (u64::MAX as f64)
1680 }
1681
1682 fn deterministic_actor_draw(&self, hash: &Hash, salt: u64) -> f64 {
1683 Self::deterministic_actor_draw_for(self.signaling.peer_id(), hash, salt)
1684 }
1685
1686 fn peer_metadata_pointer_slot_hash() -> Hash {
1687 hashtree_core::sha256(PEER_METADATA_POINTER_SLOT_KEY)
1688 }
1689
1690 fn decode_hash_hex(hash_hex: &str) -> Result<Hash, StoreError> {
1691 let bytes = hex::decode(hash_hex)
1692 .map_err(|e| StoreError::Other(format!("Invalid hash hex: {e}")))?;
1693 if bytes.len() != 32 {
1694 return Err(StoreError::Other(format!(
1695 "Invalid hash length {}, expected 32 bytes",
1696 bytes.len()
1697 )));
1698 }
1699 let mut hash = [0u8; 32];
1700 hash.copy_from_slice(&bytes);
1701 Ok(hash)
1702 }
1703
1704 fn should_drop_response(&self, hash: &Hash) -> bool {
1705 let p = self.response_behavior().drop_response_prob;
1706 if p <= 0.0 {
1707 return false;
1708 }
1709 self.deterministic_actor_draw(hash, 0xD0_D0_D0_D0_D0_D0_D0_D0) < p
1710 }
1711
1712 fn should_corrupt_response(&self, hash: &Hash) -> bool {
1713 let p = self.response_behavior().corrupt_response_prob;
1714 if p <= 0.0 {
1715 return false;
1716 }
1717 self.deterministic_actor_draw(hash, 0xC0_C0_C0_C0_C0_C0_C0_C0) < p
1718 }
1719
1720 fn should_stall_response(&self, hash: &Hash) -> bool {
1721 let p = self.response_behavior().stall_response_prob;
1722 if p <= 0.0 {
1723 return false;
1724 }
1725 self.deterministic_actor_draw(hash, 0x5A_11_5A_11_5A_11_5A_11) < p
1726 }
1727
1728 fn response_send_delay(&self, hash: &Hash, payload_len: usize) -> Duration {
1729 let behavior = self.response_behavior();
1730 let mut total_ms = behavior
1731 .extra_delay_ms
1732 .saturating_add(behavior.first_byte_delay_ms);
1733
1734 if behavior.bytes_per_second > 0 && payload_len > 0 {
1735 let throughput_ms = ((payload_len as u128) * 1000)
1736 .div_ceil(behavior.bytes_per_second as u128)
1737 .min(u64::MAX as u128) as u64;
1738 total_ms = total_ms.saturating_add(throughput_ms);
1739 }
1740
1741 if behavior.stall_delay_ms > 0 && self.should_stall_response(hash) {
1742 total_ms = total_ms.saturating_add(behavior.stall_delay_ms);
1743 }
1744
1745 Duration::from_millis(total_ms)
1746 }
1747
1748 async fn ordered_connected_peers(&self, exclude_peer_id: Option<&str>) -> Vec<String> {
1749 let current_peer_ids = self.signaling.peer_ids().await;
1750 if current_peer_ids.is_empty() {
1751 return Vec::new();
1752 }
1753
1754 sync_selector_peers(&self.peer_selector, ¤t_peer_ids).await;
1755 let hash_get_peer_ids: HashSet<String> = self
1756 .signaling
1757 .hash_get_peer_ids()
1758 .await
1759 .into_iter()
1760 .collect();
1761 let mut candidate_peer_ids: Vec<String> = current_peer_ids
1762 .into_iter()
1763 .filter(|peer_id| hash_get_peer_ids.contains(peer_id))
1764 .filter(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude))
1765 .collect();
1766 if candidate_peer_ids.is_empty() {
1767 return Vec::new();
1768 }
1769
1770 let current_set: HashSet<&str> = candidate_peer_ids.iter().map(String::as_str).collect();
1771 let mut selector = self.peer_selector.write().await;
1772 let mut selector_order = selector.select_peers();
1773 selector_order.retain(|peer_id| current_set.contains(peer_id.as_str()));
1774 if selector_order.is_empty() {
1775 let mut fallback = candidate_peer_ids;
1776 fallback.sort();
1777 return fallback;
1778 }
1779 let backed_off: HashMap<String, bool> = candidate_peer_ids
1780 .iter()
1781 .map(|peer_id| (peer_id.clone(), selector.is_peer_backed_off(peer_id)))
1782 .collect();
1783 drop(selector);
1784
1785 let rank: HashMap<&str, usize> = selector_order
1786 .iter()
1787 .enumerate()
1788 .map(|(idx, peer_id)| (peer_id.as_str(), idx))
1789 .collect();
1790 let active = self.peer_active_requests.read().await;
1791 candidate_peer_ids.sort_by(|left, right| {
1792 let left_backed_off = backed_off.get(left).copied().unwrap_or(false);
1793 let right_backed_off = backed_off.get(right).copied().unwrap_or(false);
1794 if left_backed_off != right_backed_off {
1795 return if left_backed_off {
1796 std::cmp::Ordering::Greater
1797 } else {
1798 std::cmp::Ordering::Less
1799 };
1800 }
1801 let left_rank = rank.get(left.as_str()).copied().unwrap_or(usize::MAX / 2);
1802 let right_rank = rank.get(right.as_str()).copied().unwrap_or(usize::MAX / 2);
1803 let left_load = active.get(left).copied().unwrap_or(0);
1804 let right_load = active.get(right).copied().unwrap_or(0);
1805 (left_rank + left_load.saturating_mul(ACTIVE_PEER_REQUEST_RANK_PENALTY))
1806 .cmp(&(right_rank + right_load.saturating_mul(ACTIVE_PEER_REQUEST_RANK_PENALTY)))
1807 .then_with(|| left.cmp(right))
1808 });
1809 candidate_peer_ids
1810 }
1811
1812 async fn reserve_peer_request(&self, peer_id: &str) {
1813 let mut active = self.peer_active_requests.write().await;
1814 *active.entry(peer_id.to_string()).or_insert(0) += 1;
1815 }
1816
1817 async fn release_peer_request(&self, peer_id: &str) {
1818 let mut active = self.peer_active_requests.write().await;
1819 let Some(count) = active.get_mut(peer_id) else {
1820 return;
1821 };
1822 if *count <= 1 {
1823 active.remove(peer_id);
1824 } else {
1825 *count -= 1;
1826 }
1827 }
1828
1829 async fn release_queried_peer_requests(&self, peer_ids: &[String]) {
1830 for peer_id in peer_ids {
1831 self.release_peer_request(peer_id).await;
1832 }
1833 }
1834
1835 fn requested_quote_mint(&self) -> Option<&str> {
1836 if let Some(default_mint) = self.routing.cashu_default_mint.as_deref() {
1837 if self.routing.cashu_accepted_mints.is_empty()
1838 || self
1839 .routing
1840 .cashu_accepted_mints
1841 .iter()
1842 .any(|mint| mint == default_mint)
1843 {
1844 return Some(default_mint);
1845 }
1846 }
1847
1848 self.routing
1849 .cashu_accepted_mints
1850 .first()
1851 .map(String::as_str)
1852 }
1853
1854 fn choose_quote_mint(&self, requested_mint: Option<&str>) -> Option<String> {
1855 if let Some(requested_mint) = requested_mint {
1856 if self.accepts_quote_mint(Some(requested_mint)) {
1857 return Some(requested_mint.to_string());
1858 }
1859 }
1860 if let Some(default_mint) = self.routing.cashu_default_mint.as_ref() {
1861 return Some(default_mint.clone());
1862 }
1863 if let Some(first_mint) = self.routing.cashu_accepted_mints.first() {
1864 return Some(first_mint.clone());
1865 }
1866 requested_mint.map(str::to_string)
1867 }
1868
1869 fn accepts_quote_mint(&self, mint_url: Option<&str>) -> bool {
1870 if self.routing.cashu_accepted_mints.is_empty() {
1871 return true;
1872 }
1873
1874 let Some(mint_url) = mint_url else {
1875 return false;
1876 };
1877 self.routing
1878 .cashu_accepted_mints
1879 .iter()
1880 .any(|mint| mint == mint_url)
1881 }
1882
1883 fn trusts_quote_mint(&self, mint_url: Option<&str>) -> bool {
1884 let Some(mint_url) = mint_url else {
1885 return self.routing.cashu_default_mint.is_none()
1886 && self.routing.cashu_accepted_mints.is_empty();
1887 };
1888 self.routing.cashu_default_mint.as_deref() == Some(mint_url)
1889 || self
1890 .routing
1891 .cashu_accepted_mints
1892 .iter()
1893 .any(|mint| mint == mint_url)
1894 }
1895
1896 async fn peer_suggested_mint_cap_sat(&self, peer_id: &str) -> u64 {
1897 let base = self.routing.cashu_peer_suggested_mint_base_cap_sat;
1898 if base == 0 {
1899 return 0;
1900 }
1901
1902 let selector = self.peer_selector.read().await;
1903 let Some(stats) = selector.get_stats(peer_id) else {
1904 let max_cap = self.routing.cashu_peer_suggested_mint_max_cap_sat;
1905 return if max_cap > 0 { base.min(max_cap) } else { base };
1906 };
1907
1908 if stats.cashu_payment_defaults > 0
1909 && stats.cashu_payment_defaults >= stats.cashu_payment_receipts
1910 {
1911 return 0;
1912 }
1913
1914 let success_bonus = stats
1915 .successes
1916 .saturating_mul(self.routing.cashu_peer_suggested_mint_success_step_sat);
1917 let receipt_bonus = stats
1918 .cashu_payment_receipts
1919 .saturating_mul(self.routing.cashu_peer_suggested_mint_receipt_step_sat);
1920 let mut cap = base
1921 .saturating_add(success_bonus)
1922 .saturating_add(receipt_bonus);
1923 let max_cap = self.routing.cashu_peer_suggested_mint_max_cap_sat;
1924 if max_cap > 0 {
1925 cap = cap.min(max_cap);
1926 }
1927 cap
1928 }
1929
1930 async fn should_accept_quote_response(
1931 &self,
1932 from_peer: &str,
1933 preferred_mint_url: Option<&str>,
1934 offered_payment_sat: u64,
1935 res: &DataQuoteResponse,
1936 ) -> bool {
1937 let Some(payment_sat) = res.p else {
1938 return false;
1939 };
1940 if payment_sat > offered_payment_sat {
1941 return false;
1942 }
1943
1944 let response_mint = res.m.as_deref();
1945 if response_mint == preferred_mint_url {
1946 return true;
1947 }
1948 if self.trusts_quote_mint(response_mint) {
1949 return true;
1950 }
1951 if response_mint.is_none() {
1952 return false;
1953 }
1954
1955 payment_sat <= self.peer_suggested_mint_cap_sat(from_peer).await
1956 }
1957
1958 async fn issue_quote(
1959 &self,
1960 peer_id: &str,
1961 hash_key: &str,
1962 payment_sat: u64,
1963 ttl_ms: u32,
1964 mint_url: Option<&str>,
1965 ) -> u64 {
1966 let quote_id = {
1967 let mut next = self.next_quote_id.write().await;
1968 let quote_id = *next;
1969 *next = next.saturating_add(1);
1970 quote_id
1971 };
1972
1973 let expires_at = Instant::now() + Duration::from_millis(ttl_ms as u64);
1974 self.issued_quotes.write().await.insert(
1975 (peer_id.to_string(), hash_key.to_string(), quote_id),
1976 IssuedQuote {
1977 expires_at,
1978 payment_sat,
1979 mint_url: mint_url.map(str::to_string),
1980 },
1981 );
1982 quote_id
1983 }
1984
1985 async fn take_valid_quote(&self, peer_id: &str, hash_key: &str, quote_id: u64) -> bool {
1986 let key = (peer_id.to_string(), hash_key.to_string(), quote_id);
1987 let Some(quote) = self.issued_quotes.write().await.remove(&key) else {
1988 return false;
1989 };
1990 quote.expires_at > Instant::now()
1991 }
1992
1993 async fn send_request_to_peer(
1994 &self,
1995 peer_id: &str,
1996 hash: &Hash,
1997 request_htl: u8,
1998 quote_id: Option<u64>,
1999 ) -> bool {
2000 if !should_forward_htl(request_htl) {
2001 return false;
2002 }
2003
2004 let channel = match self.signaling.get_channel(peer_id).await {
2005 Some(c) => c,
2006 None => return false,
2007 };
2008
2009 let htl_config = {
2010 let configs = self.htl_configs.read().await;
2011 configs
2012 .get(peer_id)
2013 .cloned()
2014 .unwrap_or_else(PeerHTLConfig::random)
2015 };
2016
2017 let send_htl = htl_config.decrement(request_htl);
2018 let req = match quote_id {
2019 Some(quote_id) => create_request_with_quote(hash, send_htl, quote_id),
2020 None => create_request(hash, send_htl),
2021 };
2022 let request_bytes = encode_request(&req);
2023 let request_len = request_bytes.len() as u64;
2024
2025 {
2026 let mut selector = self.peer_selector.write().await;
2027 selector.record_request(peer_id, request_len);
2028 }
2029
2030 match channel.send(request_bytes).await {
2031 Ok(()) => {
2032 self.record_peer_wire_sent(peer_id, request_len).await;
2033 true
2034 }
2035 Err(_) => {
2036 self.peer_selector.write().await.record_failure(peer_id);
2037 false
2038 }
2039 }
2040 }
2041
2042 async fn send_quote_request_to_peer(
2043 &self,
2044 peer_id: &str,
2045 hash: &Hash,
2046 payment_sat: u64,
2047 ttl_ms: u32,
2048 mint_url: Option<&str>,
2049 ) -> bool {
2050 let channel = match self.signaling.get_channel(peer_id).await {
2051 Some(c) => c,
2052 None => return false,
2053 };
2054
2055 let req = create_quote_request(hash, ttl_ms, payment_sat, mint_url);
2056 let request_bytes = encode_quote_request(&req);
2057 let request_len = request_bytes.len() as u64;
2058
2059 match channel.send(request_bytes).await {
2060 Ok(()) => {
2061 self.record_peer_wire_sent(peer_id, request_len).await;
2062 true
2063 }
2064 Err(_) => false,
2065 }
2066 }
2067
2068 pub async fn set_read_sources(&self, sources: Vec<Arc<dyn MeshReadSource>>) {
2069 let mut by_id = HashMap::new();
2070 let mut stats = self.read_source_stats.write().await;
2071 for source in sources {
2072 let source_id = source.id().to_string();
2073 by_id.insert(source_id.clone(), source);
2074 stats
2075 .entry(source_id)
2076 .or_insert_with(AdaptiveSourceStats::default);
2077 }
2078 *self.read_sources.write().await = by_id;
2079 }
2080
2081 async fn record_read_source_request(&self, source_id: &str) {
2082 let mut stats = self.read_source_stats.write().await;
2083 stats
2084 .entry(source_id.to_string())
2085 .or_insert_with(AdaptiveSourceStats::default)
2086 .requests += 1;
2087 }
2088
2089 async fn record_read_source_miss(&self, source_id: &str) {
2090 let mut stats = self.read_source_stats.write().await;
2091 stats
2092 .entry(source_id.to_string())
2093 .or_insert_with(AdaptiveSourceStats::default)
2094 .misses += 1;
2095 }
2096
2097 async fn record_read_source_success(&self, source_id: &str, elapsed_ms: u64) {
2098 let now = Instant::now();
2099 let mut stats = self.read_source_stats.write().await;
2100 let stats = stats
2101 .entry(source_id.to_string())
2102 .or_insert_with(AdaptiveSourceStats::default);
2103 stats.successes += 1;
2104 stats.last_success_at = Some(now);
2105 stats.backoff_level = 0;
2106 stats.backed_off_until = None;
2107 if stats.srtt_ms <= 0.0 {
2108 stats.srtt_ms = elapsed_ms as f64;
2109 stats.rttvar_ms = elapsed_ms as f64 / 2.0;
2110 return;
2111 }
2112 let elapsed = elapsed_ms as f64;
2113 stats.rttvar_ms = 0.75 * stats.rttvar_ms + 0.25 * (stats.srtt_ms - elapsed).abs();
2114 stats.srtt_ms = 0.875 * stats.srtt_ms + 0.125 * elapsed;
2115 }
2116
2117 async fn record_read_source_failure(&self, source_id: &str) {
2118 let now = Instant::now();
2119 let mut stats = self.read_source_stats.write().await;
2120 let stats = stats
2121 .entry(source_id.to_string())
2122 .or_insert_with(AdaptiveSourceStats::default);
2123 stats.failures += 1;
2124 stats.last_failure_at = Some(now);
2125 Self::apply_source_backoff(stats, now);
2126 }
2127
2128 async fn record_read_source_timeout(&self, source_id: &str) {
2129 let now = Instant::now();
2130 let mut stats = self.read_source_stats.write().await;
2131 let stats = stats
2132 .entry(source_id.to_string())
2133 .or_insert_with(AdaptiveSourceStats::default);
2134 stats.timeouts += 1;
2135 stats.last_failure_at = Some(now);
2136 Self::apply_source_backoff(stats, now);
2137 }
2138
2139 fn apply_source_backoff(stats: &mut AdaptiveSourceStats, now: Instant) {
2140 stats.backoff_level = stats.backoff_level.saturating_add(1);
2141 let backoff_ms = (INITIAL_SOURCE_BACKOFF_MS
2142 .saturating_mul(2u64.saturating_pow(stats.backoff_level.saturating_sub(1))))
2143 .min(MAX_SOURCE_BACKOFF_MS);
2144 stats.backed_off_until = Some(now + Duration::from_millis(backoff_ms));
2145 }
2146
2147 async fn ordered_read_sources(&self) -> Vec<Arc<dyn MeshReadSource>> {
2148 let sources = self.read_sources.read().await;
2149 if sources.is_empty() {
2150 return Vec::new();
2151 }
2152
2153 let mut available: Vec<Arc<dyn MeshReadSource>> = sources
2154 .values()
2155 .filter(|source| source.is_available())
2156 .cloned()
2157 .collect();
2158 if available.is_empty() {
2159 return Vec::new();
2160 }
2161
2162 let now = Instant::now();
2163 let stats = self.read_source_stats.read().await;
2164 let mut healthy: Vec<Arc<dyn MeshReadSource>> = available
2165 .iter()
2166 .filter(|source| {
2167 stats
2168 .get(source.id())
2169 .and_then(|s| s.backed_off_until)
2170 .is_none_or(|until| until <= now)
2171 })
2172 .cloned()
2173 .collect();
2174 if !healthy.is_empty() {
2175 available = std::mem::take(&mut healthy);
2176 }
2177
2178 available.sort_by(|left, right| {
2179 let left_stats = stats.get(left.id()).cloned().unwrap_or_default();
2180 let right_stats = stats.get(right.id()).cloned().unwrap_or_default();
2181 adaptive_source_score(&right_stats, now)
2182 .partial_cmp(&adaptive_source_score(&left_stats, now))
2183 .unwrap_or(std::cmp::Ordering::Equal)
2184 .then_with(|| left.id().cmp(right.id()))
2185 });
2186 available
2187 }
2188
2189 async fn should_probe_multiple_read_sources(
2190 &self,
2191 ordered_sources: &[Arc<dyn MeshReadSource>],
2192 ) -> bool {
2193 if ordered_sources.len() <= 1 {
2194 return false;
2195 }
2196 let stats = self.read_source_stats.read().await;
2197 let best = stats
2198 .get(ordered_sources[0].id())
2199 .cloned()
2200 .unwrap_or_default();
2201 let second = stats
2202 .get(ordered_sources[1].id())
2203 .cloned()
2204 .unwrap_or_default();
2205 if !source_has_history(&best) || !source_has_history(&second) {
2206 return false;
2207 }
2208 let now = Instant::now();
2209 adaptive_source_score(&best, now) - adaptive_source_score(&second, now)
2210 < SOURCE_SCORE_TIE_DELTA
2211 }
2212
2213 async fn source_dispatch_for(&self, source_count: usize) -> RequestDispatchConfig {
2214 if source_count == 0 {
2215 return self.routing.dispatch;
2216 }
2217 let ordered_sources = self.ordered_read_sources().await;
2218 let probe_multiple = self
2219 .should_probe_multiple_read_sources(&ordered_sources)
2220 .await;
2221 let initial_fanout = if probe_multiple {
2222 source_count.min(2)
2223 } else {
2224 1
2225 };
2226 RequestDispatchConfig {
2227 initial_fanout,
2228 hedge_fanout: self.routing.dispatch.hedge_fanout,
2229 max_fanout: self.routing.dispatch.max_fanout.min(source_count),
2230 hedge_interval_ms: self.routing.dispatch.hedge_interval_ms,
2231 }
2232 }
2233
2234 pub async fn peer_count(&self) -> usize {
2236 self.signaling.peer_count().await
2237 }
2238
2239 pub async fn peer_ids(&self) -> Vec<String> {
2241 self.signaling.peer_ids().await
2242 }
2243
2244 pub async fn needs_peers(&self) -> bool {
2246 self.signaling.needs_peers().await
2247 }
2248
2249 pub async fn send_hello(&self) -> Result<(), TransportError> {
2251 self.signaling.send_hello(vec![]).await
2252 }
2253
2254 pub async fn drain_available_data_messages(self: &Arc<Self>) -> DataPumpStats {
2259 let mut stats = DataPumpStats::default();
2260 let peer_ids = self.signaling.peer_ids().await;
2261 for peer_id in peer_ids {
2262 let Some(channel) = self.signaling.get_channel(&peer_id).await else {
2263 continue;
2264 };
2265
2266 while let Some(data) = channel.try_recv() {
2267 stats.processed += 1;
2268 stats.processed_bytes += data.len() as u64;
2269 if let Some(msg) = parse_message(&data) {
2270 match msg {
2271 DataMessage::Request(_) => stats.request_messages += 1,
2272 DataMessage::Response(_) => stats.response_messages += 1,
2273 DataMessage::QuoteRequest(_) => stats.quote_request_messages += 1,
2274 DataMessage::QuoteResponse(_) => stats.quote_response_messages += 1,
2275 DataMessage::PubsubInterest(_) => stats.pubsub_interest_messages += 1,
2276 DataMessage::PubsubFrame(_) => stats.pubsub_frame_messages += 1,
2277 DataMessage::PubsubInventory(_) => stats.pubsub_inventory_messages += 1,
2278 DataMessage::PubsubWant(_) => stats.pubsub_want_messages += 1,
2279 DataMessage::Payment(_)
2280 | DataMessage::PaymentAck(_)
2281 | DataMessage::Chunk(_)
2282 | DataMessage::PeerHints(_) => {}
2283 }
2284 }
2285 self.handle_data_message(&peer_id, &data).await;
2286 }
2287 }
2288 stats
2289 }
2290
2291 pub async fn record_cashu_payment_for_peer(&self, peer_id: &str, amount_sat: u64) {
2293 self.peer_selector
2294 .write()
2295 .await
2296 .record_cashu_payment(peer_id, amount_sat);
2297 }
2298
2299 pub async fn record_cashu_receipt_from_peer(&self, peer_id: &str, amount_sat: u64) {
2301 self.peer_selector
2302 .write()
2303 .await
2304 .record_cashu_receipt(peer_id, amount_sat);
2305 }
2306
2307 pub async fn record_cashu_payment_default_from_peer(&self, peer_id: &str) {
2309 self.peer_selector
2310 .write()
2311 .await
2312 .record_cashu_payment_default(peer_id);
2313 }
2314
2315 pub async fn selector_summary(&self) -> crate::peer_selector::SelectorSummary {
2317 self.peer_selector.read().await.summary()
2318 }
2319
2320 fn should_refuse_requests_from_peer(&self, selector: &PeerSelector, peer_id: &str) -> bool {
2321 selector.is_peer_blocked_for_payment_defaults(
2322 peer_id,
2323 self.routing.cashu_payment_default_block_threshold,
2324 )
2325 }
2326
2327 pub async fn peer_metadata_snapshot(&self) -> PeerMetadataSnapshot {
2329 self.peer_selector
2330 .read()
2331 .await
2332 .export_peer_metadata_snapshot()
2333 }
2334
2335 pub async fn persist_peer_metadata(&self) -> Result<Hash, StoreError> {
2340 let snapshot = self
2341 .peer_selector
2342 .read()
2343 .await
2344 .export_peer_metadata_snapshot();
2345 let bytes = serde_json::to_vec(&snapshot).map_err(|e| {
2346 StoreError::Other(format!("Failed to encode peer metadata snapshot: {e}"))
2347 })?;
2348 let snapshot_hash = hashtree_core::sha256(&bytes);
2349 let _ = self.local_store.put(snapshot_hash, bytes).await?;
2350
2351 let pointer_slot = Self::peer_metadata_pointer_slot_hash();
2352 let pointer_bytes = hex::encode(snapshot_hash).into_bytes();
2353 let _ = self.local_store.delete(&pointer_slot).await?;
2354 let _ = self.local_store.put(pointer_slot, pointer_bytes).await?;
2355
2356 Ok(snapshot_hash)
2357 }
2358
2359 pub async fn load_peer_metadata(&self) -> Result<bool, StoreError> {
2361 let pointer_slot = Self::peer_metadata_pointer_slot_hash();
2362 let Some(pointer_bytes) = self.local_store.get(&pointer_slot).await? else {
2363 return Ok(false);
2364 };
2365 let pointer_hex = std::str::from_utf8(&pointer_bytes).map_err(|e| {
2366 StoreError::Other(format!("Peer metadata pointer is not valid UTF-8: {e}"))
2367 })?;
2368 let snapshot_hash = Self::decode_hash_hex(pointer_hex.trim())?;
2369
2370 let Some(snapshot_bytes) = self.local_store.get(&snapshot_hash).await? else {
2371 return Ok(false);
2372 };
2373 let snapshot: PeerMetadataSnapshot =
2374 serde_json::from_slice(&snapshot_bytes).map_err(|e| {
2375 StoreError::Other(format!("Failed to decode peer metadata snapshot: {e}"))
2376 })?;
2377 self.peer_selector
2378 .write()
2379 .await
2380 .import_peer_metadata_snapshot(&snapshot);
2381 Ok(true)
2382 }
2383
2384 pub async fn get_with_quote(
2389 &self,
2390 hash: &Hash,
2391 payment_sat: u64,
2392 quote_ttl: Duration,
2393 ) -> Result<Option<Vec<u8>>, StoreError> {
2394 if let Some(data) = self.local_store.get(hash).await? {
2395 return Ok(Some(data));
2396 }
2397 Ok(self
2398 .request_from_peers_with_quote(hash, payment_sat, quote_ttl)
2399 .await)
2400 }
2401
2402 async fn request_from_peers_with_quote(
2403 &self,
2404 hash: &Hash,
2405 payment_sat: u64,
2406 quote_ttl: Duration,
2407 ) -> Option<Vec<u8>> {
2408 let ordered_peer_ids = self.ordered_connected_peers(None).await;
2409 if ordered_peer_ids.is_empty() {
2410 return None;
2411 }
2412
2413 if let Some(quote) = self
2414 .request_quote_from_peers(hash, payment_sat, quote_ttl, &ordered_peer_ids)
2415 .await
2416 {
2417 if let Some(data) = self
2418 .request_from_single_peer(hash, "e.peer_id, MAX_HTL, Some(quote.quote_id))
2419 .await
2420 {
2421 return Some(data);
2422 }
2423 }
2424
2425 self.request_from_mesh(hash).await
2426 }
2427
2428 async fn request_quote_from_peers(
2429 &self,
2430 hash: &Hash,
2431 payment_sat: u64,
2432 quote_ttl: Duration,
2433 ordered_peer_ids: &[String],
2434 ) -> Option<NegotiatedQuote> {
2435 if ordered_peer_ids.is_empty() {
2436 return None;
2437 }
2438 let ttl_ms = quote_ttl.as_millis().min(u32::MAX as u128) as u32;
2439 if ttl_ms == 0 {
2440 return None;
2441 }
2442 let requested_mint = self.requested_quote_mint().map(str::to_string);
2443
2444 let hash_key = hash_to_key(hash);
2445 let (tx, rx) = oneshot::channel();
2446 self.pending_quotes.write().await.insert(
2447 hash_key.clone(),
2448 PendingQuoteRequest {
2449 response_tx: tx,
2450 preferred_mint_url: requested_mint.clone(),
2451 offered_payment_sat: payment_sat,
2452 },
2453 );
2454
2455 let rx = Arc::new(Mutex::new(rx));
2456 let result = run_hedged_waves(
2457 ordered_peer_ids.len(),
2458 self.routing.dispatch,
2459 self.request_timeout,
2460 |range| {
2461 let wave_peer_ids = ordered_peer_ids[range].to_vec();
2462 let requested_mint = requested_mint.clone();
2463 let hash = *hash;
2464 async move {
2465 let mut sent = 0usize;
2466 for peer_id in wave_peer_ids {
2467 if self
2468 .send_quote_request_to_peer(
2469 &peer_id,
2470 &hash,
2471 payment_sat,
2472 ttl_ms,
2473 requested_mint.as_deref(),
2474 )
2475 .await
2476 {
2477 sent += 1;
2478 }
2479 }
2480 sent
2481 }
2482 },
2483 |wait| {
2484 let rx = rx.clone();
2485 async move {
2486 let mut rx = rx.lock().await;
2487 match tokio::time::timeout(wait, &mut *rx).await {
2488 Ok(Ok(Some(quote))) => HedgedWaveAction::Success(quote),
2489 Ok(Ok(None)) | Ok(Err(_)) => HedgedWaveAction::Abort,
2490 Err(_) => HedgedWaveAction::Continue,
2491 }
2492 }
2493 },
2494 )
2495 .await;
2496 let _ = self.pending_quotes.write().await.remove(&hash_key);
2497 result
2498 }
2499
2500 async fn request_from_single_peer(
2501 &self,
2502 hash: &Hash,
2503 peer_id: &str,
2504 request_htl: u8,
2505 quote_id: Option<u64>,
2506 ) -> Option<Vec<u8>> {
2507 let hash_key = hash_to_key(hash);
2508 let (tx, rx) = oneshot::channel();
2509 self.pending_requests.write().await.insert(
2510 hash_key.clone(),
2511 PendingRequest {
2512 response_tx: tx,
2513 started_at: Instant::now(),
2514 queried_peers: vec![peer_id.to_string()],
2515 },
2516 );
2517
2518 let mut rx = rx;
2519 if !self
2520 .send_request_to_peer(peer_id, hash, request_htl, quote_id)
2521 .await
2522 {
2523 let _ = self.pending_requests.write().await.remove(&hash_key);
2524 return None;
2525 }
2526 self.reserve_peer_request(peer_id).await;
2527
2528 if let Ok(Ok(Some(data))) = tokio::time::timeout(self.request_timeout, &mut rx).await {
2529 if hashtree_core::sha256(&data) == *hash {
2530 let _ = self.local_store.put(*hash, data.clone()).await;
2531 return Some(data);
2532 }
2533 }
2534
2535 if let Some(pending) = self.pending_requests.write().await.remove(&hash_key) {
2536 self.release_queried_peer_requests(&pending.queried_peers)
2537 .await;
2538 for peer_id in pending.queried_peers {
2539 self.peer_selector.write().await.record_timeout(&peer_id);
2540 }
2541 }
2542 let _ = self.take_forward_requesters(&hash_key).await;
2543 None
2544 }
2545
2546 async fn request_from_ordered_peers(
2547 &self,
2548 hash: &Hash,
2549 ordered_peer_ids: &[String],
2550 request_htl: u8,
2551 ) -> RouteFetchOutcome {
2552 let hash_key = hash_to_key(hash);
2553 let (tx, rx) = oneshot::channel();
2554 self.pending_requests.write().await.insert(
2555 hash_key.clone(),
2556 PendingRequest {
2557 response_tx: tx,
2558 started_at: Instant::now(),
2559 queried_peers: Vec::new(),
2560 },
2561 );
2562
2563 let rx = Arc::new(Mutex::new(rx));
2564 let result = run_hedged_waves(
2565 ordered_peer_ids.len(),
2566 self.routing.dispatch,
2567 self.request_timeout,
2568 |range| {
2569 let wave_peer_ids = ordered_peer_ids[range].to_vec();
2570 let hash = *hash;
2571 let hash_key = hash_key.clone();
2572 async move {
2573 let mut sent = 0usize;
2574 for peer_id in wave_peer_ids {
2575 if self
2576 .send_request_to_peer(&peer_id, &hash, request_htl, None)
2577 .await
2578 {
2579 sent += 1;
2580 self.reserve_peer_request(&peer_id).await;
2581 if let Some(pending) =
2582 self.pending_requests.write().await.get_mut(&hash_key)
2583 {
2584 pending.queried_peers.push(peer_id);
2585 }
2586 }
2587 }
2588 sent
2589 }
2590 },
2591 |wait| {
2592 let rx = rx.clone();
2593 async move {
2594 let mut rx = rx.lock().await;
2595 match tokio::time::timeout(wait, &mut *rx).await {
2596 Ok(Ok(Some(data))) if hashtree_core::sha256(&data) == *hash => {
2597 HedgedWaveAction::Success(data)
2598 }
2599 Ok(Ok(Some(_))) => HedgedWaveAction::Continue,
2600 Ok(Ok(None)) | Ok(Err(_)) => HedgedWaveAction::Abort,
2601 Err(_) => HedgedWaveAction::Continue,
2602 }
2603 }
2604 },
2605 )
2606 .await;
2607
2608 let Some(data) = result else {
2609 if let Some(pending) = self.pending_requests.write().await.remove(&hash_key) {
2610 self.release_queried_peer_requests(&pending.queried_peers)
2611 .await;
2612 for peer_id in pending.queried_peers {
2613 self.peer_selector.write().await.record_timeout(&peer_id);
2614 }
2615 }
2616 let _ = self.take_forward_requesters(&hash_key).await;
2617 return RouteFetchOutcome::Timeout;
2618 };
2619
2620 let _ = self.local_store.put(*hash, data.clone()).await;
2621 RouteFetchOutcome::Hit(data)
2622 }
2623
2624 async fn request_from_read_sources_inner(&self, hash: &Hash) -> RouteFetchOutcome {
2625 let ordered_sources = self.ordered_read_sources().await;
2626 if ordered_sources.is_empty() {
2627 return RouteFetchOutcome::Miss;
2628 }
2629
2630 let dispatch = normalize_dispatch_config(
2631 self.source_dispatch_for(ordered_sources.len()).await,
2632 ordered_sources.len(),
2633 );
2634 let wave_plan = build_hedged_wave_plan(ordered_sources.len(), dispatch);
2635 if wave_plan.is_empty() {
2636 return RouteFetchOutcome::Miss;
2637 }
2638
2639 let deadline = Instant::now() + self.request_timeout;
2640 let mut pending = FuturesUnordered::new();
2641 let mut pending_source_ids = HashSet::new();
2642 let mut saw_timeout = false;
2643 let mut next_source_idx = 0usize;
2644
2645 for (wave_idx, wave_size) in wave_plan.iter().copied().enumerate() {
2646 let from = next_source_idx;
2647 let to = (next_source_idx + wave_size).min(ordered_sources.len());
2648 next_source_idx = to;
2649
2650 for source in &ordered_sources[from..to] {
2651 let source = Arc::clone(source);
2652 let source_id = source.id().to_string();
2653 self.record_read_source_request(&source_id).await;
2654 pending_source_ids.insert(source_id.clone());
2655 let hash = *hash;
2656 pending.push(tokio::spawn(async move {
2657 let started_at = Instant::now();
2658 let result = std::panic::AssertUnwindSafe(source.get(&hash))
2659 .catch_unwind()
2660 .await;
2661 match result {
2662 Ok(Some(data)) => SourceFetchOutcome::Hit {
2663 source_id,
2664 data,
2665 elapsed_ms: started_at.elapsed().as_millis().max(1) as u64,
2666 },
2667 Ok(None) => SourceFetchOutcome::Miss { source_id },
2668 Err(_) => SourceFetchOutcome::Failure { source_id },
2669 }
2670 }));
2671 }
2672
2673 let is_last_wave =
2674 wave_idx + 1 == wave_plan.len() || next_source_idx >= ordered_sources.len();
2675 let window_end = if is_last_wave {
2676 deadline
2677 } else {
2678 (Instant::now() + Duration::from_millis(dispatch.hedge_interval_ms)).min(deadline)
2679 };
2680
2681 while Instant::now() < window_end {
2682 let remaining = window_end.saturating_duration_since(Instant::now());
2683 let Some(result) = tokio::time::timeout(remaining, pending.next())
2684 .await
2685 .ok()
2686 .flatten()
2687 else {
2688 break;
2689 };
2690 let Ok(outcome) = result else {
2691 continue;
2692 };
2693 match outcome {
2694 SourceFetchOutcome::Hit {
2695 source_id,
2696 data,
2697 elapsed_ms,
2698 } => {
2699 pending_source_ids.remove(&source_id);
2700 self.record_read_source_success(&source_id, elapsed_ms)
2701 .await;
2702 return RouteFetchOutcome::Hit(data);
2703 }
2704 SourceFetchOutcome::Miss { source_id } => {
2705 pending_source_ids.remove(&source_id);
2706 self.record_read_source_miss(&source_id).await;
2707 }
2708 SourceFetchOutcome::Failure { source_id } => {
2709 pending_source_ids.remove(&source_id);
2710 self.record_read_source_failure(&source_id).await;
2711 }
2712 }
2713 }
2714
2715 if Instant::now() >= deadline {
2716 break;
2717 }
2718 }
2719
2720 for source_id in pending_source_ids {
2721 saw_timeout = true;
2722 self.record_read_source_timeout(&source_id).await;
2723 }
2724 if saw_timeout {
2725 RouteFetchOutcome::Timeout
2726 } else {
2727 RouteFetchOutcome::Miss
2728 }
2729 }
2730
2731 async fn request_from_read_sources(&self, hash: &Hash) -> RouteFetchOutcome {
2732 let hash_key = hash_to_key(hash);
2733 let existing_wait = {
2734 let mut inflight = self.inflight_source_fetches.lock().await;
2735 if let Some(existing) = inflight.get_mut(&hash_key) {
2736 let (tx, rx) = oneshot::channel();
2737 existing.waiters.push(tx);
2738 Some(rx)
2739 } else {
2740 inflight.insert(
2741 hash_key.clone(),
2742 InflightSourceFetch {
2743 waiters: Vec::new(),
2744 },
2745 );
2746 None
2747 }
2748 };
2749
2750 if let Some(wait) = existing_wait {
2751 return wait.await.unwrap_or(RouteFetchOutcome::Timeout);
2752 }
2753
2754 let result = self.request_from_read_sources_inner(hash).await;
2755 if let RouteFetchOutcome::Hit(hit) = &result {
2756 let _ = self.local_store.put(*hash, hit.clone()).await;
2757 }
2758 self.complete_inflight_source_fetch(&hash_key, result.clone())
2759 .await;
2760
2761 result
2762 }
2763
2764 async fn complete_inflight_source_fetch(&self, hash_key: &str, result: RouteFetchOutcome) {
2765 let waiters = self
2766 .inflight_source_fetches
2767 .lock()
2768 .await
2769 .remove(hash_key)
2770 .map(|inflight| inflight.waiters)
2771 .unwrap_or_default();
2772 for waiter in waiters {
2773 let _ = waiter.send(result.clone());
2774 }
2775 }
2776
2777 async fn cancel_pending_peer_route(&self, hash: &Hash) {
2778 let hash_key = hash_to_key(hash);
2779 if let Some(pending) = self.pending_requests.write().await.remove(&hash_key) {
2780 self.release_queried_peer_requests(&pending.queried_peers)
2781 .await;
2782 }
2783 }
2784
2785 async fn cancel_losing_route(&self, hash: &Hash, route: &ReadRoute, winner_data: &[u8]) {
2786 match route {
2787 ReadRoute::Peers(_) => self.cancel_pending_peer_route(hash).await,
2788 ReadRoute::Sources => {
2789 let hash_key = hash_to_key(hash);
2790 self.complete_inflight_source_fetch(
2791 &hash_key,
2792 RouteFetchOutcome::Hit(winner_data.to_vec()),
2793 )
2794 .await;
2795 }
2796 }
2797 }
2798
2799 async fn ranked_read_routes(&self, context: &MeshReadContext) -> Vec<RankedReadRoute> {
2800 let mut routes = Vec::new();
2801 let ordered_peers = if should_forward_htl(context.request_htl) {
2802 self.ordered_connected_peers(context.exclude_peer_id.as_deref())
2803 .await
2804 } else {
2805 Vec::new()
2806 };
2807 if !ordered_peers.is_empty() {
2808 let best_peer_id = ordered_peers[0].clone();
2809 let selector = self.peer_selector.read().await;
2810 let best_peer = selector.get_stats(&best_peer_id).cloned();
2811 let now = Instant::now();
2812 let (score, has_history) = match best_peer.as_ref() {
2813 Some(stats) => (
2814 peer_endpoint_score(stats, now),
2815 peer_endpoint_has_history(stats),
2816 ),
2817 None => (0.0, false),
2818 };
2819 routes.push(RankedReadRoute {
2820 route: ReadRoute::Peers(ordered_peers),
2821 best_endpoint_id: format!("peer:{best_peer_id}"),
2822 score,
2823 has_history,
2824 });
2825 }
2826 let ordered_sources = self.ordered_read_sources().await;
2827 if let Some(best_source) = ordered_sources.first() {
2828 let stats = self.read_source_stats.read().await;
2829 let best_source_stats = stats.get(best_source.id()).cloned().unwrap_or_default();
2830 let now = Instant::now();
2831 routes.push(RankedReadRoute {
2832 route: ReadRoute::Sources,
2833 best_endpoint_id: format!("source:{}", best_source.id()),
2834 score: adaptive_source_score(&best_source_stats, now),
2835 has_history: source_has_history(&best_source_stats),
2836 });
2837 }
2838 if routes.len() <= 1 {
2839 return routes;
2840 }
2841
2842 routes.sort_by(|left, right| {
2843 right
2844 .score
2845 .partial_cmp(&left.score)
2846 .unwrap_or(std::cmp::Ordering::Equal)
2847 .then_with(|| ranked_route_kind(&left.route).cmp(&ranked_route_kind(&right.route)))
2848 .then_with(|| left.best_endpoint_id.cmp(&right.best_endpoint_id))
2849 .then_with(|| left.route.id().cmp(right.route.id()))
2850 });
2851 routes
2852 }
2853
2854 fn should_probe_multiple_routes(&self, routes: &[RankedReadRoute]) -> bool {
2855 if routes.len() <= 1 {
2856 return false;
2857 }
2858 if !routes[0].has_history || !routes[1].has_history {
2859 return false;
2860 }
2861 (routes[0].score - routes[1].score) < SOURCE_SCORE_TIE_DELTA
2862 }
2863
2864 async fn run_read_route(
2865 &self,
2866 hash: &Hash,
2867 route: &ReadRoute,
2868 context: &MeshReadContext,
2869 ) -> RouteFetchOutcome {
2870 match route {
2871 ReadRoute::Peers(peer_ids) => {
2872 self.request_from_ordered_peers(hash, peer_ids, context.request_htl)
2873 .await
2874 }
2875 ReadRoute::Sources => self.request_from_read_sources(hash).await,
2876 }
2877 }
2878
2879 async fn request_from_mesh_with_context(
2880 &self,
2881 hash: &Hash,
2882 context: &MeshReadContext,
2883 ) -> Option<Vec<u8>> {
2884 let routes = self.ranked_read_routes(context).await;
2885 match routes.as_slice() {
2886 [] => None,
2887 [ranked] => match self.run_read_route(hash, &ranked.route, context).await {
2888 RouteFetchOutcome::Hit(data) => Some(data),
2889 RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => None,
2890 },
2891 [first, second, ..] => {
2892 if self.should_probe_multiple_routes(&routes) {
2893 let first_fut = self.run_read_route(hash, &first.route, context);
2894 let second_fut = self.run_read_route(hash, &second.route, context);
2895 tokio::pin!(first_fut);
2896 tokio::pin!(second_fut);
2897 let mut first_done = false;
2898 let mut second_done = false;
2899 loop {
2900 tokio::select! {
2901 result = &mut first_fut, if !first_done => {
2902 first_done = true;
2903 if let RouteFetchOutcome::Hit(data) = result {
2904 if !second_done {
2905 self.cancel_losing_route(hash, &second.route, &data).await;
2906 }
2907 return Some(data);
2908 }
2909 }
2910 result = &mut second_fut, if !second_done => {
2911 second_done = true;
2912 if let RouteFetchOutcome::Hit(data) = result {
2913 if !first_done {
2914 self.cancel_losing_route(hash, &first.route, &data).await;
2915 }
2916 return Some(data);
2917 }
2918 }
2919 else => break,
2920 }
2921 if first_done && second_done {
2922 break;
2923 }
2924 }
2925 None
2926 } else {
2927 match self.run_read_route(hash, &first.route, context).await {
2928 RouteFetchOutcome::Hit(data) => return Some(data),
2929 RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => {}
2930 }
2931 for ranked in routes.iter().skip(1) {
2932 match self.run_read_route(hash, &ranked.route, context).await {
2933 RouteFetchOutcome::Hit(data) => return Some(data),
2934 RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => {}
2935 }
2936 }
2937 None
2938 }
2939 }
2940 }
2941 }
2942
2943 async fn request_from_mesh(&self, hash: &Hash) -> Option<Vec<u8>> {
2944 self.request_from_mesh_with_context(hash, &MeshReadContext::default())
2945 .await
2946 }
2947
2948 async fn begin_forward_request(&self, hash_key: &str, requester_id: &str) -> bool {
2949 let mut pending = self.pending_forward_requests.write().await;
2950 if let Some(existing) = pending.get_mut(hash_key) {
2951 existing.requester_ids.insert(requester_id.to_string());
2952 return false;
2953 }
2954
2955 let mut requester_ids = HashSet::new();
2956 requester_ids.insert(requester_id.to_string());
2957 pending.insert(
2958 hash_key.to_string(),
2959 PendingForwardRequest { requester_ids },
2960 );
2961 true
2962 }
2963
2964 async fn was_recent_forward_miss(&self, hash_key: &str) -> bool {
2965 self.recent_forward_misses.lock().await.contains(hash_key)
2966 }
2967
2968 async fn mark_recent_forward_miss(&self, hash_key: &str) {
2969 let _ = self
2970 .recent_forward_misses
2971 .lock()
2972 .await
2973 .insert_if_new(hash_key.to_string());
2974 }
2975
2976 async fn take_forward_requesters(&self, hash_key: &str) -> Vec<String> {
2977 self.pending_forward_requests
2978 .write()
2979 .await
2980 .remove(hash_key)
2981 .map(|pending| pending.requester_ids.into_iter().collect())
2982 .unwrap_or_default()
2983 }
2984
2985 async fn complete_pending_response(
2986 self: &Arc<Self>,
2987 from_peer: &str,
2988 hash: &Hash,
2989 hash_key: String,
2990 payload: Vec<u8>,
2991 ) {
2992 let pending = self.pending_requests.write().await.remove(&hash_key);
2993 if let Some(pending) = pending {
2994 let payload_bytes = payload.len() as u64;
2995 self.record_useful_bytes_received_from_peer(from_peer, payload_bytes)
2996 .await;
2997 {
2998 let mut deliveries = self.verified_block_deliveries.lock().await;
2999 deliveries.deliveries.push_back(VerifiedBlockDelivery {
3000 hash: *hash,
3001 provider_peer_id: from_peer.to_string(),
3002 payload_bytes,
3003 });
3004 while deliveries.deliveries.len() > VERIFIED_BLOCK_DELIVERY_CAPACITY {
3005 deliveries.deliveries.pop_front();
3006 deliveries.dropped_since_last_drain =
3007 deliveries.dropped_since_last_drain.saturating_add(1);
3008 }
3009 }
3010 self.release_queried_peer_requests(&pending.queried_peers)
3011 .await;
3012 let rtt_ms = pending.started_at.elapsed().as_millis() as u64;
3013 self.peer_selector.write().await.record_success(
3014 from_peer,
3015 rtt_ms,
3016 payload.len() as u64,
3017 );
3018 let forward_requesters = self.take_forward_requesters(&hash_key).await;
3019 let response_bytes = if forward_requesters.is_empty() {
3020 None
3021 } else {
3022 Some(encode_response(&create_response(hash, payload.clone())))
3023 };
3024 let _ = pending.response_tx.send(Some(payload));
3025 if let Some(response_bytes) = response_bytes {
3026 for requester_id in forward_requesters {
3027 Arc::clone(self)
3028 .enqueue_response_send(requester_id, response_bytes.clone(), Instant::now())
3029 .await;
3030 }
3031 }
3032 }
3033 }
3034
3035 async fn handle_quote_response_message(&self, from_peer: &str, res: DataQuoteResponse) {
3036 if !res.a {
3037 return;
3038 }
3039
3040 let Some(quote_id) = res.q else {
3041 return;
3042 };
3043
3044 let hash_key = hash_to_key(&res.h);
3045 let (preferred_mint_url, offered_payment_sat) = {
3046 let pending_quotes = self.pending_quotes.read().await;
3047 let Some(pending) = pending_quotes.get(&hash_key) else {
3048 return;
3049 };
3050 (
3051 pending.preferred_mint_url.clone(),
3052 pending.offered_payment_sat,
3053 )
3054 };
3055 if !self
3056 .should_accept_quote_response(
3057 from_peer,
3058 preferred_mint_url.as_deref(),
3059 offered_payment_sat,
3060 &res,
3061 )
3062 .await
3063 {
3064 return;
3065 }
3066 let mut pending_quotes = self.pending_quotes.write().await;
3067 if let Some(pending) = pending_quotes.remove(&hash_key) {
3068 let _ = pending.response_tx.send(Some(NegotiatedQuote {
3069 peer_id: from_peer.to_string(),
3070 quote_id,
3071 mint_url: res.m,
3072 }));
3073 }
3074 }
3075
3076 async fn handle_response_message(
3077 self: &Arc<Self>,
3078 from_peer: &str,
3079 res: crate::protocol::DataResponse,
3080 ) {
3081 let hash_key = hash_to_key(&res.h);
3082 let hash = match crate::protocol::bytes_to_hash(&res.h) {
3083 Some(h) => h,
3084 None => return,
3085 };
3086
3087 if hashtree_core::sha256(&res.d) != hash {
3089 self.peer_selector.write().await.record_failure(from_peer);
3090 if self.debug {
3091 println!("[MeshStoreCore] Ignoring invalid response payload for {hash_key}");
3092 }
3093 return;
3094 }
3095
3096 self.complete_pending_response(from_peer, &hash, hash_key, res.d)
3097 .await;
3098 }
3099
3100 async fn handle_quote_request_message(&self, from_peer: &str, req: DataQuoteRequest) {
3101 let hash = match crate::protocol::bytes_to_hash(&req.h) {
3102 Some(h) => h,
3103 None => return,
3104 };
3105 let hash_key = hash_to_key(&hash);
3106
3107 {
3108 let selector = self.peer_selector.read().await;
3109 if self.should_refuse_requests_from_peer(&selector, from_peer) {
3110 if self.debug {
3111 println!(
3112 "[MeshStoreCore] Refusing quote request from delinquent peer {}",
3113 from_peer
3114 );
3115 }
3116 return;
3117 }
3118 }
3119
3120 let chosen_mint = self.choose_quote_mint(req.m.as_deref());
3121 let can_serve = self.local_store.has(&hash).await.ok().unwrap_or(false)
3122 && !self.should_drop_response(&hash)
3123 && !self.should_corrupt_response(&hash);
3124
3125 let res = if can_serve {
3126 let quote_id = self
3127 .issue_quote(from_peer, &hash_key, req.p, req.t, chosen_mint.as_deref())
3128 .await;
3129 create_quote_response_available(&hash, quote_id, req.p, req.t, chosen_mint.as_deref())
3130 } else {
3131 create_quote_response_unavailable(&hash)
3132 };
3133 let response_bytes = encode_quote_response(&res);
3134 if let Some(channel) = self.signaling.get_channel(from_peer).await {
3135 if channel.send(response_bytes.clone()).await.is_ok() {
3136 self.record_peer_wire_sent(from_peer, response_bytes.len() as u64)
3137 .await;
3138 }
3139 }
3140 }
3141
3142 async fn handle_request_message(
3143 self: &Arc<Self>,
3144 from_peer: &str,
3145 req: crate::protocol::DataRequest,
3146 ) {
3147 let hash = match crate::protocol::bytes_to_hash(&req.h) {
3148 Some(h) => h,
3149 None => return,
3150 };
3151 let hash_key = hash_to_key(&hash);
3152
3153 if let Some(quote_id) = req.q {
3154 if !self.take_valid_quote(from_peer, &hash_key, quote_id).await {
3155 if self.debug {
3156 println!(
3157 "[MeshStoreCore] Refusing request with invalid or expired quote {} from {}",
3158 quote_id, from_peer
3159 );
3160 }
3161 return;
3162 }
3163 }
3164
3165 let allow_peer_forwarding = {
3166 let selector = self.peer_selector.read().await;
3167 !self.should_refuse_requests_from_peer(&selector, from_peer)
3168 };
3169
3170 if let Ok(Some(mut data)) = self.local_store.get(&hash).await {
3172 if self.should_drop_response(&hash) {
3173 if self.debug {
3174 println!(
3175 "[MeshStoreCore] Dropping response for {} due to actor profile",
3176 hash_to_key(&hash)
3177 );
3178 }
3179 return;
3180 }
3181
3182 let response_delay = self.response_send_delay(&hash, data.len());
3183 if self.should_corrupt_response(&hash) {
3184 if data.is_empty() {
3185 data.push(0x80);
3186 } else {
3187 data[0] ^= 0x80;
3188 }
3189 }
3190
3191 let res = create_response(&hash, data);
3193 let response_bytes = encode_response(&res);
3194 let ready_at = Instant::now() + response_delay;
3195 Arc::clone(self)
3196 .enqueue_response_send(from_peer.to_string(), response_bytes, ready_at)
3197 .await;
3198 return;
3199 }
3200
3201 if self.pending_requests.read().await.contains_key(&hash_key) {
3202 let _ = self.begin_forward_request(&hash_key, from_peer).await;
3203 return;
3204 }
3205
3206 if self.was_recent_forward_miss(&hash_key).await {
3207 if self.debug {
3208 println!(
3209 "[MeshStoreCore] Suppressing recently missed forwarded request for {}",
3210 hash_key
3211 );
3212 }
3213 return;
3214 }
3215
3216 if !self.begin_forward_request(&hash_key, from_peer).await {
3217 return;
3218 }
3219
3220 let from_peer = from_peer.to_string();
3221 let this = Arc::clone(self);
3222 let request_htl = req.htl;
3223 tokio::spawn(async move {
3224 let result = if allow_peer_forwarding {
3225 let context = MeshReadContext {
3226 exclude_peer_id: Some(from_peer.clone()),
3227 request_htl,
3228 };
3229 this.request_from_mesh_with_context(&hash, &context).await
3230 } else {
3231 if this.debug {
3232 println!(
3233 "[MeshStoreCore] Serving request from delinquent peer {} via read sources only",
3234 from_peer
3235 );
3236 }
3237 match this.request_from_read_sources(&hash).await {
3238 RouteFetchOutcome::Hit(data) => Some(data),
3239 RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => None,
3240 }
3241 };
3242 let requester_ids = this.take_forward_requesters(&hash_key).await;
3243 if let Some(data) = result {
3244 let ready_at = Instant::now() + this.response_send_delay(&hash, data.len());
3245 let res = create_response(&hash, data);
3246 let response_bytes = encode_response(&res);
3247 for requester_id in requester_ids {
3248 Arc::clone(&this)
3249 .enqueue_response_send(requester_id, response_bytes.clone(), ready_at)
3250 .await;
3251 }
3252 } else {
3253 this.mark_recent_forward_miss(&hash_key).await;
3254 }
3255 });
3256 }
3257
3258 async fn handle_pubsub_interest_message(
3259 self: &Arc<Self>,
3260 from_peer: &str,
3261 mut interest: PubsubInterest,
3262 ) {
3263 if !self.apply_pubsub_interest_route(from_peer, &interest).await {
3264 return;
3265 }
3266
3267 if !self.routing.pubsub_forwarding || interest.htl <= 1 {
3268 return;
3269 }
3270 interest.htl = interest.htl.saturating_sub(1);
3271 let _ = self
3272 .send_pubsub_interest_to_peers(&interest, Some(from_peer))
3273 .await;
3274 }
3275
3276 async fn handle_pubsub_frame_message(
3277 self: &Arc<Self>,
3278 from_peer: &str,
3279 mut frame: PubsubFrame,
3280 wire_bytes: usize,
3281 ) {
3282 if frame.stream_id.is_empty() || frame.origin_peer_id.is_empty() {
3283 return;
3284 }
3285 if frame.origin_peer_id == self.signaling.peer_id() {
3286 return;
3287 }
3288
3289 let frame_key = Self::pubsub_frame_key(&frame);
3290 if !self
3291 .pubsub_seen_frames
3292 .lock()
3293 .await
3294 .insert_if_new(frame_key.clone())
3295 {
3296 return;
3297 }
3298 self.cache_pubsub_frame(frame_key.clone(), frame.clone())
3299 .await;
3300
3301 let local_interested = self
3302 .pubsub_local_interests
3303 .read()
3304 .await
3305 .contains(&frame.stream_id);
3306 let mut downstream_peers = if self.routing.pubsub_forwarding && frame.htl > 1 {
3307 match self.routing.pubsub_delivery_mode {
3308 PubsubDeliveryMode::InterestPush => {
3309 let mut peers = self
3310 .interested_pubsub_peers(&frame.stream_id, Some(from_peer))
3311 .await;
3312 peers.extend(
3313 self.take_pubsub_want_peers(&frame_key, Some(from_peer))
3314 .await,
3315 );
3316 peers.sort();
3317 peers.dedup();
3318 peers
3319 }
3320 PubsubDeliveryMode::HtlInvWant => {
3321 self.take_pubsub_want_peers(&frame_key, Some(from_peer))
3322 .await
3323 }
3324 }
3325 } else {
3326 Vec::new()
3327 };
3328 downstream_peers.retain(|peer_id| peer_id != from_peer);
3329
3330 if local_interested || !downstream_peers.is_empty() {
3331 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3332 .await;
3333 }
3334
3335 if local_interested {
3336 self.enqueue_pubsub_event(PubsubEvent {
3337 stream_id: frame.stream_id.clone(),
3338 seq: frame.seq,
3339 origin_peer_id: frame.origin_peer_id.clone(),
3340 from_peer_id: from_peer.to_string(),
3341 payload: frame.payload.clone(),
3342 })
3343 .await;
3344 }
3345
3346 if downstream_peers.is_empty() {
3347 return;
3348 }
3349
3350 frame.htl = frame.htl.saturating_sub(1);
3351 let _ = self
3352 .send_pubsub_frame_to_peers(&frame, &downstream_peers)
3353 .await;
3354 }
3355
3356 async fn handle_pubsub_inventory_message(
3357 self: &Arc<Self>,
3358 from_peer: &str,
3359 inv: PubsubInventory,
3360 wire_bytes: usize,
3361 ) {
3362 if inv.stream_id.is_empty() || inv.origin_peer_id.is_empty() {
3363 return;
3364 }
3365 if inv.origin_peer_id == self.signaling.peer_id() {
3366 return;
3367 }
3368
3369 let key = Self::pubsub_key(&inv.origin_peer_id, &inv.stream_id, inv.seq);
3370 if !self
3371 .pubsub_seen_inventories
3372 .lock()
3373 .await
3374 .insert_if_new(key.clone())
3375 {
3376 return;
3377 }
3378 {
3379 let mut routes = self.pubsub_inventory_routes.write().await;
3380 routes
3381 .entry(key.clone())
3382 .or_insert_with(|| from_peer.to_string());
3383 }
3384
3385 let local_interested = self
3386 .pubsub_local_interests
3387 .read()
3388 .await
3389 .contains(&inv.stream_id);
3390 let downstream_peers = if self.routing.pubsub_forwarding {
3391 self.interested_pubsub_peers(&inv.stream_id, Some(from_peer))
3392 .await
3393 } else {
3394 Vec::new()
3395 };
3396 if local_interested || !downstream_peers.is_empty() {
3397 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3398 .await;
3399 let want =
3400 create_pubsub_want(inv.stream_id.clone(), inv.seq, inv.origin_peer_id.clone());
3401 let _ = self.send_pubsub_want_upstream(&key, &want, None).await;
3402 }
3403
3404 if !self.routing.pubsub_forwarding
3405 || downstream_peers.is_empty()
3406 || !should_forward_htl(inv.htl)
3407 {
3408 return;
3409 }
3410 let _ = self
3411 .send_pubsub_inventory_to_peers(&inv, &downstream_peers)
3412 .await;
3413 }
3414
3415 async fn handle_pubsub_want_message(
3416 self: &Arc<Self>,
3417 from_peer: &str,
3418 want: PubsubWant,
3419 wire_bytes: usize,
3420 ) {
3421 if want.stream_id.is_empty() || want.origin_peer_id.is_empty() {
3422 return;
3423 }
3424 if want.origin_peer_id == from_peer {
3425 return;
3426 }
3427
3428 let key = Self::pubsub_key(&want.origin_peer_id, &want.stream_id, want.seq);
3429 let want_key = format!("{from_peer}:{key}");
3430 if !self.pubsub_seen_wants.lock().await.insert_if_new(want_key) {
3431 return;
3432 }
3433
3434 if let Some(frame) = self.cached_pubsub_frame(&key).await {
3435 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3436 .await;
3437 let peers = vec![from_peer.to_string()];
3438 let _ = self.send_pubsub_frame_to_peers(&frame, &peers).await;
3439 return;
3440 }
3441
3442 let has_upstream_route = self.pubsub_inventory_routes.read().await.contains_key(&key);
3443 if !has_upstream_route {
3444 return;
3445 }
3446
3447 if self.remember_pubsub_want_peer(key.clone(), from_peer).await {
3448 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3449 .await;
3450 }
3451 let _ = self
3452 .send_pubsub_want_upstream(&key, &want, Some(from_peer))
3453 .await;
3454 }
3455
3456 pub async fn handle_data_message(self: &Arc<Self>, from_peer: &str, data: &[u8]) {
3458 self.record_peer_wire_received(from_peer, data.len() as u64)
3459 .await;
3460 let parsed = match parse_message(data) {
3461 Some(m) => m,
3462 None => return,
3463 };
3464
3465 match parsed {
3466 DataMessage::Request(req) => {
3467 self.handle_request_message(from_peer, req).await;
3468 }
3469 DataMessage::Response(res) => {
3470 self.handle_response_message(from_peer, res).await;
3471 }
3472 DataMessage::QuoteRequest(req) => {
3473 self.handle_quote_request_message(from_peer, req).await;
3474 }
3475 DataMessage::QuoteResponse(res) => {
3476 self.handle_quote_response_message(from_peer, res).await;
3477 }
3478 DataMessage::PubsubInterest(interest) => {
3479 self.handle_pubsub_interest_message(from_peer, interest)
3480 .await;
3481 }
3482 DataMessage::PubsubFrame(frame) => {
3483 self.handle_pubsub_frame_message(from_peer, frame, data.len())
3484 .await;
3485 }
3486 DataMessage::PubsubInventory(inv) => {
3487 self.handle_pubsub_inventory_message(from_peer, inv, data.len())
3488 .await;
3489 }
3490 DataMessage::PubsubWant(want) => {
3491 self.handle_pubsub_want_message(from_peer, want, data.len())
3492 .await;
3493 }
3494 DataMessage::Payment(_)
3495 | DataMessage::PaymentAck(_)
3496 | DataMessage::Chunk(_)
3497 | DataMessage::PeerHints(_) => {}
3498 }
3499 }
3500}
3501
3502#[async_trait]
3503impl<S, R, F> Store for MeshStoreCore<S, R, F>
3504where
3505 S: Store + Send + Sync + 'static,
3506 R: SignalingTransport + Send + Sync + 'static,
3507 F: PeerLinkFactory + Send + Sync + 'static,
3508{
3509 async fn put(&self, hash: Hash, data: Vec<u8>) -> Result<bool, StoreError> {
3510 self.local_store.put(hash, data).await
3511 }
3512
3513 async fn get(&self, hash: &Hash) -> Result<Option<Vec<u8>>, StoreError> {
3514 if let Some(data) = self.local_store.get(hash).await? {
3516 return Ok(Some(data));
3517 }
3518
3519 Ok(self.request_from_mesh(hash).await)
3521 }
3522
3523 async fn has(&self, hash: &Hash) -> Result<bool, StoreError> {
3524 self.local_store.has(hash).await
3525 }
3526
3527 async fn delete(&self, hash: &Hash) -> Result<bool, StoreError> {
3528 self.local_store.delete(hash).await
3529 }
3530}
3531
3532#[cfg(test)]
3533mod delivery_tests;
3534
3535#[cfg(test)]
3536mod tests;
3537
3538pub type SimMeshStore<S> =
3540 MeshStoreCore<S, crate::mock::MockRelayTransport, crate::mock::MockConnectionFactory>;