1use async_trait::async_trait;
8use futures::{stream::FuturesUnordered, FutureExt, StreamExt};
9use std::collections::hash_map::DefaultHasher;
10use std::collections::{HashMap, HashSet, VecDeque};
11use std::future::Future;
12use std::hash::{Hash as _, Hasher};
13use std::ops::Range;
14use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
15use std::sync::Arc;
16use std::time::Duration;
17use tokio::sync::{oneshot, Mutex, Notify, RwLock};
18use tokio::time::Instant;
19
20use hashtree_core::{Hash, Store, StoreError};
21
22use crate::peer_selector::{PeerMetadataSnapshot, PeerSelector, SelectionStrategy};
23use crate::protocol::{
24 create_pubsub_frame, create_pubsub_interest, create_pubsub_inventory, create_pubsub_want,
25 create_quote_request, create_quote_response_available, create_quote_response_unavailable,
26 create_request, create_request_with_quote, create_response, encode_pubsub_frame,
27 encode_pubsub_interest, encode_pubsub_inventory, encode_pubsub_want, encode_quote_request,
28 encode_quote_response, encode_request, encode_response, hash_to_key, parse_message,
29 DataMessage, DataQuoteRequest, DataQuoteResponse, PubsubFrame, PubsubInterest, PubsubInventory,
30 PubsubWant,
31};
32use crate::pubsub_strategy::{
33 reciprocal_virtual_finish, select_reciprocal_outbound_job, OutboundJobCandidate,
34 PeerTrafficSnapshot, PubsubCandidate, PubsubSchedulerConfig,
35};
36use crate::signaling::MeshRouter;
37use crate::transport::{PeerLinkFactory, SignalingTransport, TransportError};
38use crate::types::{
39 should_forward_htl, PeerHTLConfig, SignalingMessage, TimedSeenSet, MAX_HTL, MESH_EVENT_POLICY,
40};
41
42const PEER_METADATA_POINTER_SLOT_KEY: &[u8] = b"hashtree-mesh/peer-metadata/latest/v1";
45const RECENT_FORWARD_MISS_CAPACITY: usize = 4096;
46const MIN_RECENT_FORWARD_MISS_TTL_MS: u64 = 250;
47const PUBSUB_SEEN_CAPACITY: usize = 16_384;
48const PUBSUB_INBOX_CAPACITY: usize = 4_096;
49const PUBSUB_FRAME_CACHE_CAPACITY: usize = 4_096;
50const PUBSUB_SEEN_TTL: Duration = Duration::from_secs(120);
51
52struct PendingRequest {
54 response_tx: oneshot::Sender<Option<Vec<u8>>>,
55 started_at: Instant,
56 queried_peers: Vec<String>,
57}
58
59struct PendingQuoteRequest {
60 response_tx: oneshot::Sender<Option<NegotiatedQuote>>,
61 preferred_mint_url: Option<String>,
62 offered_payment_sat: u64,
63}
64
65struct PendingForwardRequest {
66 requester_ids: HashSet<String>,
67}
68
69type PeerWireStats = PeerTrafficSnapshot;
70
71struct PendingResponseSend {
72 job_id: u64,
73 peer_id: String,
74 bytes: Vec<u8>,
75 ready_at: Instant,
76 queue_sequence: u64,
77}
78
79#[async_trait]
80pub trait MeshReadSource: Send + Sync {
81 fn id(&self) -> &str;
82
83 fn is_available(&self) -> bool {
84 true
85 }
86
87 async fn get(&self, hash: &Hash) -> Option<Vec<u8>>;
88}
89
90#[derive(Debug, Clone)]
91struct NegotiatedQuote {
92 peer_id: String,
93 quote_id: u64,
94 #[allow(dead_code)]
95 mint_url: Option<String>,
96}
97
98struct IssuedQuote {
99 expires_at: Instant,
100 #[allow(dead_code)]
101 payment_sat: u64,
102 #[allow(dead_code)]
103 mint_url: Option<String>,
104}
105
106#[derive(Debug, Clone, Default)]
107struct AdaptiveSourceStats {
108 requests: u64,
109 successes: u64,
110 misses: u64,
111 failures: u64,
112 timeouts: u64,
113 srtt_ms: f64,
114 rttvar_ms: f64,
115 backoff_level: u32,
116 backed_off_until: Option<Instant>,
117 last_success_at: Option<Instant>,
118 last_failure_at: Option<Instant>,
119}
120
121#[derive(Debug, Clone)]
122enum RouteFetchOutcome {
123 Hit(Vec<u8>),
124 Miss,
125 Timeout,
126}
127
128struct InflightSourceFetch {
129 waiters: Vec<oneshot::Sender<RouteFetchOutcome>>,
130}
131
132enum SourceFetchOutcome {
133 Hit {
134 source_id: String,
135 data: Vec<u8>,
136 elapsed_ms: u64,
137 },
138 Miss {
139 source_id: String,
140 },
141 Failure {
142 source_id: String,
143 },
144}
145
146const INITIAL_SOURCE_BACKOFF_MS: u64 = 250;
147const MAX_SOURCE_BACKOFF_MS: u64 = 10_000;
148const SOURCE_SCORE_TIE_DELTA: f64 = 0.15;
149const RECENT_SOURCE_SUCCESS_WINDOW: Duration = Duration::from_secs(60);
150const ACTIVE_PEER_REQUEST_RANK_PENALTY: usize = 3;
151
152fn source_reliability_score(stats: &AdaptiveSourceStats) -> f64 {
153 (stats.successes as f64 + 1.0) / (stats.requests as f64 + 2.0)
154}
155
156fn source_latency_score(stats: &AdaptiveSourceStats) -> f64 {
157 if stats.srtt_ms <= 0.0 {
158 return 0.5;
159 }
160 (500.0 / (stats.srtt_ms + 50.0)).min(1.0)
161}
162
163fn source_has_history(stats: &AdaptiveSourceStats) -> bool {
164 stats.requests > 0
165 || stats.successes > 0
166 || stats.misses > 0
167 || stats.failures > 0
168 || stats.timeouts > 0
169}
170
171fn adaptive_source_score(stats: &AdaptiveSourceStats, now: Instant) -> f64 {
172 if let Some(backed_off_until) = stats.backed_off_until {
173 if backed_off_until > now {
174 return f64::NEG_INFINITY;
175 }
176 }
177
178 let miss_penalty = if stats.requests > 0 {
179 (stats.misses as f64 / stats.requests as f64) * 0.15
180 } else {
181 0.0
182 };
183 let failure_penalty = if stats.requests > 0 {
184 ((stats.failures + stats.timeouts) as f64 / stats.requests as f64) * 0.3
185 } else {
186 0.0
187 };
188 let recency_bonus = if stats
189 .last_success_at
190 .is_some_and(|last| now.duration_since(last) < RECENT_SOURCE_SUCCESS_WINDOW)
191 {
192 0.1
193 } else {
194 0.0
195 };
196
197 0.6 * source_reliability_score(stats) + 0.3 * source_latency_score(stats) + recency_bonus
198 - miss_penalty
199 - failure_penalty
200}
201
202fn peer_endpoint_has_history(stats: &crate::peer_selector::PeerStats) -> bool {
203 stats.requests_sent > 0 || stats.successes > 0 || stats.failures > 0 || stats.timeouts > 0
204}
205
206fn peer_endpoint_score(stats: &crate::peer_selector::PeerStats, now: Instant) -> f64 {
207 if stats.backed_off_until.is_some_and(|until| until > now) {
208 return f64::NEG_INFINITY;
209 }
210
211 let miss_penalty = 0.0;
212 let failure_penalty = if stats.requests_sent > 0 {
213 ((stats.failures + stats.timeouts) as f64 / stats.requests_sent as f64) * 0.3
214 } else {
215 0.0
216 };
217 let recency_bonus = if stats
218 .last_success
219 .is_some_and(|last| now.duration_since(last) < RECENT_SOURCE_SUCCESS_WINDOW)
220 {
221 0.1
222 } else {
223 0.0
224 };
225
226 0.6 * stats.success_rate()
227 + 0.3
228 * source_latency_score(&AdaptiveSourceStats {
229 srtt_ms: stats.srtt_ms,
230 ..AdaptiveSourceStats::default()
231 })
232 + recency_bonus
233 - miss_penalty
234 - failure_penalty
235}
236
237#[derive(Clone)]
238enum ReadRoute {
239 Peers(Vec<String>),
240 Sources,
241}
242
243impl ReadRoute {
244 fn id(&self) -> &'static str {
245 match self {
246 Self::Peers(_) => "peers",
247 Self::Sources => "sources",
248 }
249 }
250}
251
252struct RankedReadRoute {
253 route: ReadRoute,
254 best_endpoint_id: String,
255 score: f64,
256 has_history: bool,
257}
258
259fn ranked_route_kind(route: &ReadRoute) -> u8 {
260 match route {
261 ReadRoute::Sources => 0,
262 ReadRoute::Peers(_) => 1,
263 }
264}
265
266#[derive(Debug, Clone)]
267struct MeshReadContext {
268 exclude_peer_id: Option<String>,
269 request_htl: u8,
270}
271
272impl Default for MeshReadContext {
273 fn default() -> Self {
274 Self {
275 exclude_peer_id: None,
276 request_htl: MAX_HTL,
277 }
278 }
279}
280
281#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
283pub struct DataPumpStats {
284 pub processed: usize,
285 pub request_messages: usize,
286 pub response_messages: usize,
287 pub quote_request_messages: u64,
288 pub quote_response_messages: u64,
289 pub pubsub_interest_messages: u64,
290 pub pubsub_frame_messages: u64,
291 pub pubsub_inventory_messages: u64,
292 pub pubsub_want_messages: u64,
293 pub processed_bytes: u64,
294}
295
296#[derive(Debug, Clone, PartialEq, Eq)]
298pub struct PubsubEvent {
299 pub stream_id: String,
300 pub seq: u64,
301 pub origin_peer_id: String,
302 pub from_peer_id: String,
303 pub payload: Vec<u8>,
304}
305
306#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
308pub struct PubsubPublishStats {
309 pub selected_peers: usize,
310 pub sent_peers: usize,
311 pub sent_bytes: u64,
312 pub deferred_peers: usize,
313}
314
315#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
317pub enum PubsubDeliveryMode {
318 InterestPush,
320 #[default]
322 HtlInvWant,
323}
324
325#[derive(Debug, Clone, Copy)]
331pub struct RequestDispatchConfig {
332 pub initial_fanout: usize,
334 pub hedge_fanout: usize,
336 pub max_fanout: usize,
338 pub hedge_interval_ms: u64,
340}
341
342impl Default for RequestDispatchConfig {
343 fn default() -> Self {
344 Self {
345 initial_fanout: usize::MAX,
346 hedge_fanout: usize::MAX,
347 max_fanout: usize::MAX,
348 hedge_interval_ms: 0,
349 }
350 }
351}
352
353pub fn normalize_dispatch_config(
355 dispatch: RequestDispatchConfig,
356 available_peers: usize,
357) -> RequestDispatchConfig {
358 let mut cfg = dispatch;
359 let cap = if cfg.max_fanout == 0 {
360 available_peers
361 } else {
362 cfg.max_fanout.min(available_peers)
363 };
364 cfg.max_fanout = cap;
365 cfg.initial_fanout = if cfg.initial_fanout == 0 {
366 1
367 } else {
368 cfg.initial_fanout.min(cap.max(1))
369 };
370 cfg.hedge_fanout = if cfg.hedge_fanout == 0 {
371 1
372 } else {
373 cfg.hedge_fanout.min(cap.max(1))
374 };
375 cfg
376}
377
378pub fn build_hedged_wave_plan(peer_count: usize, dispatch: RequestDispatchConfig) -> Vec<usize> {
380 if peer_count == 0 {
381 return Vec::new();
382 }
383 let cap = dispatch.max_fanout.min(peer_count);
384 if cap == 0 {
385 return Vec::new();
386 }
387
388 let mut plan = Vec::new();
389 let mut sent = 0usize;
390 let first = dispatch.initial_fanout.min(cap).max(1);
391 plan.push(first);
392 sent += first;
393
394 while sent < cap {
395 let next = dispatch.hedge_fanout.min(cap - sent).max(1);
396 plan.push(next);
397 sent += next;
398 }
399 plan
400}
401
402#[derive(Debug)]
404pub enum HedgedWaveAction<T> {
405 Continue,
406 Success(T),
407 Abort,
408}
409
410pub async fn run_hedged_waves<T, SendWave, SendWaveFut, WaitWave, WaitWaveFut>(
415 peer_count: usize,
416 dispatch: RequestDispatchConfig,
417 request_timeout: Duration,
418 mut send_wave: SendWave,
419 mut wait_wave: WaitWave,
420) -> Option<T>
421where
422 SendWave: FnMut(Range<usize>) -> SendWaveFut,
423 SendWaveFut: Future<Output = usize>,
424 WaitWave: FnMut(Duration) -> WaitWaveFut,
425 WaitWaveFut: Future<Output = HedgedWaveAction<T>>,
426{
427 let dispatch = normalize_dispatch_config(dispatch, peer_count);
428 let wave_plan = build_hedged_wave_plan(peer_count, dispatch);
429 if wave_plan.is_empty() {
430 return None;
431 }
432
433 let deadline = Instant::now() + request_timeout;
434 let mut sent_total = 0usize;
435 let mut next_peer_idx = 0usize;
436
437 for (wave_idx, wave_size) in wave_plan.iter().copied().enumerate() {
438 let from = next_peer_idx;
439 let to = (next_peer_idx + wave_size).min(peer_count);
440 next_peer_idx = to;
441
442 if from == to {
443 continue;
444 }
445
446 sent_total += send_wave(from..to).await;
447 if sent_total == 0 {
448 if next_peer_idx >= peer_count {
449 break;
450 }
451 continue;
452 }
453
454 let now = Instant::now();
455 if now >= deadline {
456 break;
457 }
458 let remaining = deadline.saturating_duration_since(now);
459 let is_last_wave = wave_idx + 1 == wave_plan.len() || next_peer_idx >= peer_count;
460 let wait = if is_last_wave {
461 remaining
462 } else if dispatch.hedge_interval_ms == 0 {
463 Duration::ZERO
464 } else {
465 Duration::from_millis(dispatch.hedge_interval_ms).min(remaining)
466 };
467
468 if wait.is_zero() {
469 continue;
470 }
471
472 match wait_wave(wait).await {
473 HedgedWaveAction::Continue => {}
474 HedgedWaveAction::Success(value) => return Some(value),
475 HedgedWaveAction::Abort => break,
476 }
477 }
478
479 None
480}
481
482pub async fn sync_selector_peers(selector: &RwLock<PeerSelector>, current_peer_ids: &[String]) {
484 let mut selector = selector.write().await;
485 let current: HashSet<&str> = current_peer_ids.iter().map(String::as_str).collect();
486 let known: Vec<String> = selector.all_stats().map(|s| s.peer_id.clone()).collect();
487 for peer_id in known {
488 if !current.contains(peer_id.as_str()) {
489 selector.remove_peer(&peer_id);
490 }
491 }
492 for peer_id in current_peer_ids {
493 selector.add_peer(peer_id.clone());
494 }
495}
496
497#[derive(Debug, Clone, Copy)]
501pub struct ResponseBehaviorConfig {
502 pub drop_response_prob: f64,
504 pub corrupt_response_prob: f64,
506 pub extra_delay_ms: u64,
508 pub first_byte_delay_ms: u64,
510 pub bytes_per_second: u64,
512 pub stall_response_prob: f64,
514 pub stall_delay_ms: u64,
516}
517
518impl Default for ResponseBehaviorConfig {
519 fn default() -> Self {
520 Self {
521 drop_response_prob: 0.0,
522 corrupt_response_prob: 0.0,
523 extra_delay_ms: 0,
524 first_byte_delay_ms: 0,
525 bytes_per_second: 0,
526 stall_response_prob: 0.0,
527 stall_delay_ms: 0,
528 }
529 }
530}
531
532impl ResponseBehaviorConfig {
533 fn normalized(self) -> Self {
534 Self {
535 drop_response_prob: self.drop_response_prob.clamp(0.0, 1.0),
536 corrupt_response_prob: self.corrupt_response_prob.clamp(0.0, 1.0),
537 extra_delay_ms: self.extra_delay_ms,
538 first_byte_delay_ms: self.first_byte_delay_ms,
539 bytes_per_second: self.bytes_per_second,
540 stall_response_prob: self.stall_response_prob.clamp(0.0, 1.0),
541 stall_delay_ms: self.stall_delay_ms,
542 }
543 }
544}
545
546#[derive(Debug, Clone)]
548pub struct MeshRoutingConfig {
549 pub selection_strategy: SelectionStrategy,
550 pub fairness_enabled: bool,
551 pub cashu_payment_weight: f64,
553 pub cashu_payment_default_block_threshold: u64,
556 pub cashu_accepted_mints: Vec<String>,
558 pub cashu_default_mint: Option<String>,
560 pub cashu_peer_suggested_mint_base_cap_sat: u64,
562 pub cashu_peer_suggested_mint_success_step_sat: u64,
564 pub cashu_peer_suggested_mint_receipt_step_sat: u64,
566 pub cashu_peer_suggested_mint_max_cap_sat: u64,
568 pub dispatch: RequestDispatchConfig,
569 pub response_behavior: ResponseBehaviorConfig,
570 pub pubsub_scheduler: PubsubSchedulerConfig,
571 pub pubsub_delivery_mode: PubsubDeliveryMode,
572 pub pubsub_forwarding: bool,
574 pub pubsub_max_htl: u8,
576}
577
578impl Default for MeshRoutingConfig {
579 fn default() -> Self {
580 Self {
581 selection_strategy: SelectionStrategy::Weighted,
582 fairness_enabled: true,
583 cashu_payment_weight: 0.0,
584 cashu_payment_default_block_threshold: 0,
585 cashu_accepted_mints: Vec::new(),
586 cashu_default_mint: None,
587 cashu_peer_suggested_mint_base_cap_sat: 0,
588 cashu_peer_suggested_mint_success_step_sat: 0,
589 cashu_peer_suggested_mint_receipt_step_sat: 0,
590 cashu_peer_suggested_mint_max_cap_sat: 0,
591 dispatch: RequestDispatchConfig::default(),
592 response_behavior: ResponseBehaviorConfig::default(),
593 pubsub_scheduler: PubsubSchedulerConfig::default(),
594 pubsub_delivery_mode: PubsubDeliveryMode::HtlInvWant,
595 pubsub_forwarding: true,
596 pubsub_max_htl: MESH_EVENT_POLICY.max_htl,
597 }
598 }
599}
600
601impl MeshRoutingConfig {
602 fn pubsub_initial_htl(&self) -> u8 {
603 self.pubsub_max_htl.clamp(1, MAX_HTL)
604 }
605}
606
607pub struct MeshStoreCore<S, R, F>
614where
615 S: Store + Send + Sync + 'static,
616 R: SignalingTransport + Send + Sync + 'static,
617 F: PeerLinkFactory + Send + Sync + 'static,
618{
619 local_store: Arc<S>,
621 signaling: Arc<MeshRouter<R, F>>,
623 htl_configs: RwLock<HashMap<String, PeerHTLConfig>>,
625 pending_requests: RwLock<HashMap<String, PendingRequest>>,
627 pending_quotes: RwLock<HashMap<String, PendingQuoteRequest>>,
629 pending_forward_requests: RwLock<HashMap<String, PendingForwardRequest>>,
631 recent_forward_misses: Mutex<TimedSeenSet>,
633 issued_quotes: RwLock<HashMap<(String, String, u64), IssuedQuote>>,
635 next_quote_id: RwLock<u64>,
637 read_sources: RwLock<HashMap<String, Arc<dyn MeshReadSource>>>,
639 read_source_stats: RwLock<HashMap<String, AdaptiveSourceStats>>,
641 inflight_source_fetches: Mutex<HashMap<String, InflightSourceFetch>>,
643 peer_selector: RwLock<PeerSelector>,
645 peer_active_requests: RwLock<HashMap<String, usize>>,
647 peer_wire_stats: RwLock<HashMap<String, PeerWireStats>>,
649 pubsub_local_interests: RwLock<HashSet<String>>,
651 pubsub_local_interest_versions: RwLock<HashMap<String, u64>>,
653 pubsub_peer_interests: RwLock<HashMap<String, HashSet<String>>>,
655 pubsub_interest_routes: RwLock<HashMap<(String, String), String>>,
657 pubsub_interest_versions: RwLock<HashMap<(String, String), u64>>,
659 pubsub_seen_interests: Mutex<TimedSeenSet>,
661 pubsub_seen_frames: Mutex<TimedSeenSet>,
663 pubsub_seen_inventories: Mutex<TimedSeenSet>,
665 pubsub_seen_wants: Mutex<TimedSeenSet>,
667 pubsub_inventory_routes: RwLock<HashMap<String, String>>,
669 pubsub_want_routes: RwLock<HashMap<String, HashSet<String>>>,
671 pubsub_upstream_wants: Mutex<TimedSeenSet>,
673 pubsub_frame_cache: Mutex<VecDeque<(String, PubsubFrame)>>,
675 pubsub_inbox: Mutex<VecDeque<PubsubEvent>>,
677 pubsub_notify: Notify,
679 pubsub_deferred_counts: RwLock<HashMap<(String, String), u64>>,
681 next_pubsub_interest_seq: AtomicU64,
683 pending_response_sends: Mutex<Vec<PendingResponseSend>>,
685 response_scheduler_running: AtomicBool,
687 next_response_job_id: AtomicU64,
689 routing: MeshRoutingConfig,
691 request_timeout: Duration,
693 debug: bool,
695 running: RwLock<bool>,
697}
698
699impl<S, R, F> MeshStoreCore<S, R, F>
700where
701 S: Store + Send + Sync + 'static,
702 R: SignalingTransport + Send + Sync + 'static,
703 F: PeerLinkFactory + Send + Sync + 'static,
704{
705 pub fn new(
707 local_store: Arc<S>,
708 signaling: Arc<MeshRouter<R, F>>,
709 request_timeout: Duration,
710 debug: bool,
711 ) -> Self {
712 Self::new_with_routing(
713 local_store,
714 signaling,
715 request_timeout,
716 debug,
717 Default::default(),
718 )
719 }
720
721 pub fn new_with_routing(
723 local_store: Arc<S>,
724 signaling: Arc<MeshRouter<R, F>>,
725 request_timeout: Duration,
726 debug: bool,
727 routing: MeshRoutingConfig,
728 ) -> Self {
729 let mut selector = PeerSelector::with_strategy(routing.selection_strategy);
730 selector.set_fairness(routing.fairness_enabled);
731 selector.set_cashu_payment_weight(routing.cashu_payment_weight);
732 Self {
733 local_store,
734 signaling,
735 htl_configs: RwLock::new(HashMap::new()),
736 pending_requests: RwLock::new(HashMap::new()),
737 pending_quotes: RwLock::new(HashMap::new()),
738 pending_forward_requests: RwLock::new(HashMap::new()),
739 recent_forward_misses: Mutex::new(TimedSeenSet::new(
740 RECENT_FORWARD_MISS_CAPACITY,
741 Self::recent_forward_miss_ttl(request_timeout),
742 )),
743 issued_quotes: RwLock::new(HashMap::new()),
744 next_quote_id: RwLock::new(1),
745 read_sources: RwLock::new(HashMap::new()),
746 read_source_stats: RwLock::new(HashMap::new()),
747 inflight_source_fetches: Mutex::new(HashMap::new()),
748 peer_selector: RwLock::new(selector),
749 peer_active_requests: RwLock::new(HashMap::new()),
750 peer_wire_stats: RwLock::new(HashMap::new()),
751 pubsub_local_interests: RwLock::new(HashSet::new()),
752 pubsub_local_interest_versions: RwLock::new(HashMap::new()),
753 pubsub_peer_interests: RwLock::new(HashMap::new()),
754 pubsub_interest_routes: RwLock::new(HashMap::new()),
755 pubsub_interest_versions: RwLock::new(HashMap::new()),
756 pubsub_seen_interests: Mutex::new(TimedSeenSet::new(
757 PUBSUB_SEEN_CAPACITY,
758 PUBSUB_SEEN_TTL,
759 )),
760 pubsub_seen_frames: Mutex::new(TimedSeenSet::new(
761 PUBSUB_SEEN_CAPACITY,
762 PUBSUB_SEEN_TTL,
763 )),
764 pubsub_seen_inventories: Mutex::new(TimedSeenSet::new(
765 PUBSUB_SEEN_CAPACITY,
766 PUBSUB_SEEN_TTL,
767 )),
768 pubsub_seen_wants: Mutex::new(TimedSeenSet::new(PUBSUB_SEEN_CAPACITY, PUBSUB_SEEN_TTL)),
769 pubsub_inventory_routes: RwLock::new(HashMap::new()),
770 pubsub_want_routes: RwLock::new(HashMap::new()),
771 pubsub_upstream_wants: Mutex::new(TimedSeenSet::new(
772 PUBSUB_SEEN_CAPACITY,
773 PUBSUB_SEEN_TTL,
774 )),
775 pubsub_frame_cache: Mutex::new(VecDeque::new()),
776 pubsub_inbox: Mutex::new(VecDeque::new()),
777 pubsub_notify: Notify::new(),
778 pubsub_deferred_counts: RwLock::new(HashMap::new()),
779 next_pubsub_interest_seq: AtomicU64::new(1),
780 pending_response_sends: Mutex::new(Vec::new()),
781 response_scheduler_running: AtomicBool::new(false),
782 next_response_job_id: AtomicU64::new(1),
783 routing,
784 request_timeout,
785 debug,
786 running: RwLock::new(false),
787 }
788 }
789
790 fn recent_forward_miss_ttl(request_timeout: Duration) -> Duration {
791 let ttl_ms = request_timeout
792 .as_millis()
793 .saturating_mul(2)
794 .max(MIN_RECENT_FORWARD_MISS_TTL_MS as u128)
795 .min(u64::MAX as u128) as u64;
796 Duration::from_millis(ttl_ms)
797 }
798
799 pub async fn start(&self) -> Result<(), TransportError> {
801 *self.running.write().await = true;
802
803 self.signaling.send_hello(vec![]).await?;
805
806 Ok(())
807 }
808
809 pub async fn stop(&self) {
811 *self.running.write().await = false;
812 }
813
814 pub async fn process_signaling(&self, msg: SignalingMessage) -> Result<(), TransportError> {
816 let peer_id = msg.peer_id().to_string();
818 {
819 let mut configs = self.htl_configs.write().await;
820 if !configs.contains_key(&peer_id) {
821 configs.insert(peer_id.clone(), PeerHTLConfig::random());
822 }
823 }
824 self.peer_selector.write().await.add_peer(peer_id.clone());
825
826 let result = self.signaling.handle_message(msg).await;
827 if result.is_ok() {
828 self.announce_pubsub_interests_to_peer(&peer_id).await;
829 }
830 result
831 }
832
833 pub fn signaling(&self) -> &Arc<MeshRouter<R, F>> {
835 &self.signaling
836 }
837
838 fn response_behavior(&self) -> ResponseBehaviorConfig {
839 self.routing.response_behavior.normalized()
840 }
841
842 async fn record_peer_wire_sent(&self, peer_id: &str, bytes: u64) {
843 if bytes == 0 {
844 return;
845 }
846 let mut stats = self.peer_wire_stats.write().await;
847 let entry = stats.entry(peer_id.to_string()).or_default();
848 entry.bytes_sent = entry.bytes_sent.saturating_add(bytes);
849 }
850
851 async fn record_peer_wire_received(&self, peer_id: &str, bytes: u64) {
852 if bytes == 0 {
853 return;
854 }
855 let mut stats = self.peer_wire_stats.write().await;
856 let entry = stats.entry(peer_id.to_string()).or_default();
857 entry.bytes_received = entry.bytes_received.saturating_add(bytes);
858 }
859
860 pub async fn record_useful_bytes_received_from_peer(&self, peer_id: &str, bytes: u64) {
865 if bytes == 0 {
866 return;
867 }
868 let mut stats = self.peer_wire_stats.write().await;
869 let entry = stats.entry(peer_id.to_string()).or_default();
870 entry.useful_bytes_received = entry.useful_bytes_received.saturating_add(bytes);
871 }
872
873 pub async fn peer_traffic_snapshot(&self, peer_id: &str) -> PeerTrafficSnapshot {
875 self.peer_wire_stats
876 .read()
877 .await
878 .get(peer_id)
879 .copied()
880 .unwrap_or_default()
881 }
882
883 pub async fn peer_traffic_snapshots(&self) -> HashMap<String, PeerTrafficSnapshot> {
885 self.peer_wire_stats.read().await.clone()
886 }
887
888 fn pubsub_key(origin_peer_id: &str, stream_id: &str, seq: u64) -> String {
889 format!("{origin_peer_id}:{stream_id}:{seq}")
890 }
891
892 fn pubsub_frame_key(frame: &PubsubFrame) -> String {
893 Self::pubsub_key(&frame.origin_peer_id, &frame.stream_id, frame.seq)
894 }
895
896 fn pubsub_interest_key(interest: &PubsubInterest) -> String {
897 format!(
898 "{}:{}:{}:{}",
899 interest.subscriber_peer_id, interest.stream_id, interest.seq, interest.active
900 )
901 }
902
903 fn next_pubsub_interest_seq(&self) -> u64 {
904 self.next_pubsub_interest_seq
905 .fetch_add(1, Ordering::Relaxed)
906 }
907
908 async fn record_peer_pubsub_wire_sent(&self, peer_id: &str, bytes: u64, bandwidth_debt: f64) {
909 if bytes == 0 {
910 return;
911 }
912 let mut stats = self.peer_wire_stats.write().await;
913 let entry = stats.entry(peer_id.to_string()).or_default();
914 entry.bytes_sent = entry.bytes_sent.saturating_add(bytes);
915 entry.bandwidth_debt = bandwidth_debt;
916 }
917
918 async fn send_pubsub_interest_to_peers(
919 &self,
920 interest: &PubsubInterest,
921 exclude_peer_id: Option<&str>,
922 ) -> PubsubPublishStats {
923 if !should_forward_htl(interest.htl) {
924 return PubsubPublishStats::default();
925 }
926
927 let mut peer_ids = self.signaling.peer_ids().await;
928 peer_ids.sort();
929 peer_ids.retain(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude));
930
931 let bytes = encode_pubsub_interest(interest);
932 let mut stats = PubsubPublishStats {
933 selected_peers: peer_ids.len(),
934 ..Default::default()
935 };
936 for peer_id in peer_ids {
937 let Some(channel) = self.signaling.get_channel(&peer_id).await else {
938 continue;
939 };
940 if channel.send(bytes.clone()).await.is_ok() {
941 stats.sent_peers += 1;
942 stats.sent_bytes = stats.sent_bytes.saturating_add(bytes.len() as u64);
943 self.record_peer_wire_sent(&peer_id, bytes.len() as u64)
944 .await;
945 }
946 }
947 stats
948 }
949
950 async fn announce_pubsub_interests_to_peer(&self, peer_id: &str) {
951 let mut interests = self
952 .pubsub_local_interests
953 .read()
954 .await
955 .iter()
956 .cloned()
957 .collect::<Vec<_>>();
958 interests.sort();
959 if interests.is_empty() {
960 return;
961 }
962
963 let interests = {
964 let versions = self.pubsub_local_interest_versions.read().await;
965 interests
966 .into_iter()
967 .filter_map(|stream_id| {
968 versions
969 .get(&stream_id)
970 .copied()
971 .map(|seq| (stream_id, seq))
972 })
973 .collect::<Vec<_>>()
974 };
975
976 for (stream_id, seq) in interests {
977 let interest = create_pubsub_interest(
978 stream_id,
979 self.signaling.peer_id().to_string(),
980 seq,
981 true,
982 MAX_HTL,
983 );
984 let Some(channel) = self.signaling.get_channel(peer_id).await else {
985 continue;
986 };
987 let bytes = encode_pubsub_interest(&interest);
988 if channel.send(bytes.clone()).await.is_ok() {
989 self.record_peer_wire_sent(peer_id, bytes.len() as u64)
990 .await;
991 }
992 }
993 }
994
995 fn remove_pubsub_peer_interest(
996 peer_interests: &mut HashMap<String, HashSet<String>>,
997 routes: &HashMap<(String, String), String>,
998 stream_id: &str,
999 peer_id: &str,
1000 ) {
1001 let still_has_route = routes
1002 .iter()
1003 .any(|((stream, _subscriber), peer)| stream == stream_id && peer == peer_id);
1004 if still_has_route {
1005 return;
1006 }
1007 if let Some(peers) = peer_interests.get_mut(stream_id) {
1008 peers.remove(peer_id);
1009 if peers.is_empty() {
1010 peer_interests.remove(stream_id);
1011 }
1012 }
1013 }
1014
1015 async fn apply_pubsub_interest_route(
1016 &self,
1017 from_peer: &str,
1018 interest: &PubsubInterest,
1019 ) -> bool {
1020 if interest.stream_id.is_empty() || interest.subscriber_peer_id.is_empty() {
1021 return false;
1022 }
1023 if interest.subscriber_peer_id == self.signaling.peer_id() {
1024 return false;
1025 }
1026
1027 let interest_key = Self::pubsub_interest_key(interest);
1028 if !self
1029 .pubsub_seen_interests
1030 .lock()
1031 .await
1032 .insert_if_new(interest_key)
1033 {
1034 return false;
1035 }
1036
1037 let route_key = (
1038 interest.stream_id.clone(),
1039 interest.subscriber_peer_id.clone(),
1040 );
1041 {
1042 let mut versions = self.pubsub_interest_versions.write().await;
1043 if versions
1044 .get(&route_key)
1045 .is_some_and(|latest| *latest >= interest.seq)
1046 {
1047 return false;
1048 }
1049 versions.insert(route_key.clone(), interest.seq);
1050 }
1051
1052 let mut peer_interests = self.pubsub_peer_interests.write().await;
1053 let mut routes = self.pubsub_interest_routes.write().await;
1054 if interest.active {
1055 if let Some(previous_peer) = routes.insert(route_key, from_peer.to_string()) {
1056 if previous_peer != from_peer {
1057 Self::remove_pubsub_peer_interest(
1058 &mut peer_interests,
1059 &routes,
1060 &interest.stream_id,
1061 &previous_peer,
1062 );
1063 }
1064 }
1065 peer_interests
1066 .entry(interest.stream_id.clone())
1067 .or_default()
1068 .insert(from_peer.to_string());
1069 } else if let Some(previous_peer) = routes.remove(&route_key) {
1070 Self::remove_pubsub_peer_interest(
1071 &mut peer_interests,
1072 &routes,
1073 &interest.stream_id,
1074 &previous_peer,
1075 );
1076 } else {
1077 Self::remove_pubsub_peer_interest(
1078 &mut peer_interests,
1079 &routes,
1080 &interest.stream_id,
1081 from_peer,
1082 );
1083 }
1084
1085 true
1086 }
1087
1088 async fn interested_pubsub_peers(
1089 &self,
1090 stream_id: &str,
1091 exclude_peer_id: Option<&str>,
1092 ) -> Vec<String> {
1093 let connected = self
1094 .signaling
1095 .peer_ids()
1096 .await
1097 .into_iter()
1098 .collect::<HashSet<_>>();
1099 let mut peers = self
1100 .pubsub_peer_interests
1101 .read()
1102 .await
1103 .get(stream_id)
1104 .map(|peers| peers.iter().cloned().collect::<Vec<_>>())
1105 .unwrap_or_default();
1106 peers.retain(|peer_id| {
1107 connected.contains(peer_id) && exclude_peer_id.is_none_or(|exclude| peer_id != exclude)
1108 });
1109 peers.sort();
1110 peers
1111 }
1112
1113 async fn decrement_pubsub_htl_for_peer(&self, peer_id: &str, htl: u8) -> u8 {
1114 let htl_config = {
1115 let configs = self.htl_configs.read().await;
1116 configs
1117 .get(peer_id)
1118 .cloned()
1119 .unwrap_or_else(PeerHTLConfig::random)
1120 };
1121 htl_config.decrement_with_policy(htl, &MESH_EVENT_POLICY)
1122 }
1123
1124 async fn send_pubsub_inventory_to_peers(
1125 &self,
1126 inv: &PubsubInventory,
1127 peer_ids: &[String],
1128 ) -> PubsubPublishStats {
1129 if peer_ids.is_empty() || !should_forward_htl(inv.htl) {
1130 return PubsubPublishStats::default();
1131 }
1132
1133 let mut stats = PubsubPublishStats {
1134 selected_peers: peer_ids.len(),
1135 ..Default::default()
1136 };
1137 for peer_id in peer_ids {
1138 let send_htl = self.decrement_pubsub_htl_for_peer(peer_id, inv.htl).await;
1139 if !should_forward_htl(send_htl) {
1140 continue;
1141 }
1142 let Some(channel) = self.signaling.get_channel(peer_id).await else {
1143 continue;
1144 };
1145 let mut outgoing = inv.clone();
1146 outgoing.htl = send_htl;
1147 let bytes = encode_pubsub_inventory(&outgoing);
1148 let message_bytes = bytes.len() as u64;
1149 if channel.send(bytes).await.is_ok() {
1150 stats.sent_peers += 1;
1151 stats.sent_bytes = stats.sent_bytes.saturating_add(message_bytes);
1152 self.record_peer_wire_sent(peer_id, message_bytes).await;
1153 }
1154 }
1155 stats
1156 }
1157
1158 async fn send_pubsub_want_to_peer(&self, want: &PubsubWant, peer_id: &str) -> bool {
1159 let Some(channel) = self.signaling.get_channel(peer_id).await else {
1160 return false;
1161 };
1162 let bytes = encode_pubsub_want(want);
1163 let message_bytes = bytes.len() as u64;
1164 match channel.send(bytes).await {
1165 Ok(()) => {
1166 self.record_peer_wire_sent(peer_id, message_bytes).await;
1167 true
1168 }
1169 Err(_) => false,
1170 }
1171 }
1172
1173 async fn send_pubsub_want_upstream(
1174 &self,
1175 key: &str,
1176 want: &PubsubWant,
1177 exclude_peer_id: Option<&str>,
1178 ) -> bool {
1179 let upstream = {
1180 let routes = self.pubsub_inventory_routes.read().await;
1181 routes.get(key).cloned()
1182 };
1183 let Some(upstream) = upstream else {
1184 return false;
1185 };
1186 if exclude_peer_id.is_some_and(|exclude| exclude == upstream) {
1187 return false;
1188 }
1189 let want_key = format!("{key}:{upstream}");
1190 if !self
1191 .pubsub_upstream_wants
1192 .lock()
1193 .await
1194 .insert_if_new(want_key)
1195 {
1196 return false;
1197 }
1198 self.send_pubsub_want_to_peer(want, &upstream).await
1199 }
1200
1201 async fn cache_pubsub_frame(&self, key: String, frame: PubsubFrame) {
1202 let mut cache = self.pubsub_frame_cache.lock().await;
1203 if let Some(index) = cache.iter().position(|(cached_key, _)| cached_key == &key) {
1204 cache.remove(index);
1205 }
1206 cache.push_back((key, frame));
1207 while cache.len() > PUBSUB_FRAME_CACHE_CAPACITY {
1208 cache.pop_front();
1209 }
1210 }
1211
1212 async fn cached_pubsub_frame(&self, key: &str) -> Option<PubsubFrame> {
1213 self.pubsub_frame_cache
1214 .lock()
1215 .await
1216 .iter()
1217 .find_map(|(cached_key, frame)| {
1218 if cached_key == key {
1219 Some(frame.clone())
1220 } else {
1221 None
1222 }
1223 })
1224 }
1225
1226 async fn remember_pubsub_want_peer(&self, key: String, from_peer: &str) -> bool {
1227 let mut routes = self.pubsub_want_routes.write().await;
1228 routes.entry(key).or_default().insert(from_peer.to_string())
1229 }
1230
1231 async fn take_pubsub_want_peers(
1232 &self,
1233 key: &str,
1234 exclude_peer_id: Option<&str>,
1235 ) -> Vec<String> {
1236 let connected = self
1237 .signaling
1238 .peer_ids()
1239 .await
1240 .into_iter()
1241 .collect::<HashSet<_>>();
1242 let mut peers = self
1243 .pubsub_want_routes
1244 .write()
1245 .await
1246 .remove(key)
1247 .map(|peers| peers.into_iter().collect::<Vec<_>>())
1248 .unwrap_or_default();
1249 peers.retain(|peer_id| {
1250 connected.contains(peer_id) && exclude_peer_id.is_none_or(|exclude| peer_id != exclude)
1251 });
1252 peers.sort();
1253 peers
1254 }
1255
1256 async fn select_pubsub_peers(
1257 &self,
1258 stream_id: &str,
1259 seq: u64,
1260 message_bytes: u64,
1261 peer_ids: &[String],
1262 ) -> (Vec<String>, Vec<String>) {
1263 let traffic = self.peer_wire_stats.read().await;
1264 let deferred_counts = self.pubsub_deferred_counts.read().await;
1265 let candidates = peer_ids
1266 .iter()
1267 .map(|peer_id| PubsubCandidate {
1268 peer_id: peer_id.clone(),
1269 traffic: traffic.get(peer_id).copied().unwrap_or_default(),
1270 deferred_count: deferred_counts
1271 .get(&(stream_id.to_string(), peer_id.clone()))
1272 .copied()
1273 .unwrap_or_default(),
1274 })
1275 .collect::<Vec<_>>();
1276 drop(deferred_counts);
1277 drop(traffic);
1278
1279 let selection = self.routing.pubsub_scheduler.select(
1280 stream_id,
1281 seq,
1282 self.signaling.peer_id(),
1283 message_bytes,
1284 &candidates,
1285 );
1286
1287 {
1288 let mut deferred_counts = self.pubsub_deferred_counts.write().await;
1289 for peer_id in &selection.deferred {
1290 *deferred_counts
1291 .entry((stream_id.to_string(), peer_id.clone()))
1292 .or_insert(0) += 1;
1293 }
1294 for peer_id in &selection.selected {
1295 deferred_counts.remove(&(stream_id.to_string(), peer_id.clone()));
1296 }
1297 }
1298
1299 (selection.selected, selection.deferred)
1300 }
1301
1302 async fn send_pubsub_frame_to_peers(
1303 &self,
1304 frame: &PubsubFrame,
1305 peer_ids: &[String],
1306 ) -> PubsubPublishStats {
1307 if peer_ids.is_empty() || !should_forward_htl(frame.htl) {
1308 return PubsubPublishStats::default();
1309 }
1310
1311 let bytes = encode_pubsub_frame(frame);
1312 let message_bytes = bytes.len() as u64;
1313 let (selected, deferred) = self
1314 .select_pubsub_peers(&frame.stream_id, frame.seq, message_bytes, peer_ids)
1315 .await;
1316 let mut stats = PubsubPublishStats {
1317 selected_peers: selected.len(),
1318 deferred_peers: deferred.len(),
1319 ..Default::default()
1320 };
1321
1322 for peer_id in selected {
1323 let Some(channel) = self.signaling.get_channel(&peer_id).await else {
1324 continue;
1325 };
1326 let snapshot = self.peer_traffic_snapshot(&peer_id).await;
1327 let bandwidth_debt = reciprocal_virtual_finish(snapshot, message_bytes);
1328 if channel.send(bytes.clone()).await.is_ok() {
1329 stats.sent_peers += 1;
1330 stats.sent_bytes = stats.sent_bytes.saturating_add(message_bytes);
1331 self.record_peer_pubsub_wire_sent(&peer_id, message_bytes, bandwidth_debt)
1332 .await;
1333 }
1334 }
1335
1336 stats
1337 }
1338
1339 async fn enqueue_pubsub_event(&self, event: PubsubEvent) {
1340 let mut inbox = self.pubsub_inbox.lock().await;
1341 inbox.push_back(event);
1342 while inbox.len() > PUBSUB_INBOX_CAPACITY {
1343 inbox.pop_front();
1344 }
1345 self.pubsub_notify.notify_one();
1346 }
1347
1348 pub async fn subscribe_pubsub(
1350 self: &Arc<Self>,
1351 stream_id: impl Into<String>,
1352 ) -> PubsubPublishStats {
1353 let stream_id = stream_id.into();
1354 if stream_id.is_empty() {
1355 return PubsubPublishStats::default();
1356 }
1357 self.pubsub_local_interests
1358 .write()
1359 .await
1360 .insert(stream_id.clone());
1361 let seq = {
1362 let mut versions = self.pubsub_local_interest_versions.write().await;
1363 match versions.get(&stream_id).copied() {
1364 Some(seq) => seq,
1365 None => {
1366 let seq = self.next_pubsub_interest_seq();
1367 versions.insert(stream_id.clone(), seq);
1368 seq
1369 }
1370 }
1371 };
1372 let interest = create_pubsub_interest(
1373 stream_id,
1374 self.signaling.peer_id().to_string(),
1375 seq,
1376 true,
1377 self.routing.pubsub_initial_htl(),
1378 );
1379 self.send_pubsub_interest_to_peers(&interest, None).await
1380 }
1381
1382 pub async fn unsubscribe_pubsub(
1384 self: &Arc<Self>,
1385 stream_id: impl Into<String>,
1386 ) -> PubsubPublishStats {
1387 let stream_id = stream_id.into();
1388 if stream_id.is_empty() {
1389 return PubsubPublishStats::default();
1390 }
1391 self.pubsub_local_interests.write().await.remove(&stream_id);
1392 self.pubsub_local_interest_versions
1393 .write()
1394 .await
1395 .remove(&stream_id);
1396 let interest = create_pubsub_interest(
1397 stream_id,
1398 self.signaling.peer_id().to_string(),
1399 self.next_pubsub_interest_seq(),
1400 false,
1401 self.routing.pubsub_initial_htl(),
1402 );
1403 self.send_pubsub_interest_to_peers(&interest, None).await
1404 }
1405
1406 pub async fn publish_pubsub(
1408 self: &Arc<Self>,
1409 stream_id: impl Into<String>,
1410 seq: u64,
1411 payload: Vec<u8>,
1412 ) -> PubsubPublishStats {
1413 let stream_id = stream_id.into();
1414 if stream_id.is_empty() {
1415 return PubsubPublishStats::default();
1416 }
1417 let payload_bytes = payload.len() as u64;
1418 let frame = create_pubsub_frame(
1419 stream_id.clone(),
1420 seq,
1421 self.signaling.peer_id().to_string(),
1422 payload.clone(),
1423 self.routing.pubsub_initial_htl(),
1424 );
1425 let frame_key = Self::pubsub_frame_key(&frame);
1426 self.pubsub_seen_frames
1427 .lock()
1428 .await
1429 .insert_if_new(frame_key.clone());
1430 self.cache_pubsub_frame(frame_key, frame.clone()).await;
1431
1432 if self
1433 .pubsub_local_interests
1434 .read()
1435 .await
1436 .contains(&stream_id)
1437 {
1438 self.enqueue_pubsub_event(PubsubEvent {
1439 stream_id: stream_id.clone(),
1440 seq,
1441 origin_peer_id: self.signaling.peer_id().to_string(),
1442 from_peer_id: self.signaling.peer_id().to_string(),
1443 payload,
1444 })
1445 .await;
1446 }
1447
1448 match self.routing.pubsub_delivery_mode {
1449 PubsubDeliveryMode::InterestPush => {
1450 let peers = self.interested_pubsub_peers(&stream_id, None).await;
1451 self.send_pubsub_frame_to_peers(&frame, &peers).await
1452 }
1453 PubsubDeliveryMode::HtlInvWant => {
1454 let inv = create_pubsub_inventory(
1455 stream_id,
1456 seq,
1457 self.signaling.peer_id().to_string(),
1458 payload_bytes,
1459 self.routing.pubsub_initial_htl(),
1460 );
1461 let peers = self.interested_pubsub_peers(&inv.stream_id, None).await;
1462 self.send_pubsub_inventory_to_peers(&inv, &peers).await
1463 }
1464 }
1465 }
1466
1467 pub async fn drain_pubsub_events(&self) -> Vec<PubsubEvent> {
1469 self.pubsub_inbox.lock().await.drain(..).collect()
1470 }
1471
1472 pub async fn recv_pubsub_event(&self) -> PubsubEvent {
1474 loop {
1475 if let Some(event) = self.pubsub_inbox.lock().await.pop_front() {
1476 return event;
1477 }
1478 self.pubsub_notify.notified().await;
1479 }
1480 }
1481
1482 pub async fn pubsub_interest_peers(&self, stream_id: &str) -> Vec<String> {
1484 self.interested_pubsub_peers(stream_id, None).await
1485 }
1486
1487 fn choose_ready_response_job(
1488 ready_jobs: &[(u64, String, usize, Instant, u64)],
1489 stats: &HashMap<String, PeerWireStats>,
1490 ) -> Option<(u64, f64)> {
1491 let jobs = ready_jobs
1492 .iter()
1493 .map(|job| OutboundJobCandidate {
1494 job_id: job.0,
1495 peer_id: job.1.clone(),
1496 message_bytes: job.2 as u64,
1497 queue_sequence: job.4,
1498 })
1499 .collect::<Vec<_>>();
1500 select_reciprocal_outbound_job(&jobs, |peer_id| {
1501 stats.get(peer_id).copied().unwrap_or_default()
1502 })
1503 .map(|choice| (choice.job_id, choice.virtual_finish))
1504 }
1505
1506 async fn enqueue_response_send(
1507 self: &Arc<Self>,
1508 peer_id: String,
1509 bytes: Vec<u8>,
1510 ready_at: Instant,
1511 ) {
1512 let job_id = self.next_response_job_id.fetch_add(1, Ordering::Relaxed);
1513 {
1514 let mut queue = self.pending_response_sends.lock().await;
1515 queue.push(PendingResponseSend {
1516 job_id,
1517 peer_id,
1518 bytes,
1519 ready_at,
1520 queue_sequence: job_id,
1521 });
1522 }
1523
1524 if self
1525 .response_scheduler_running
1526 .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
1527 .is_ok()
1528 {
1529 let this = Arc::clone(self);
1530 tokio::spawn(async move {
1531 this.run_response_scheduler().await;
1532 });
1533 }
1534 }
1535
1536 async fn run_response_scheduler(self: Arc<Self>) {
1537 loop {
1538 let snapshot = {
1539 let queue = self.pending_response_sends.lock().await;
1540 if queue.is_empty() {
1541 self.response_scheduler_running
1542 .store(false, Ordering::Release);
1543 return;
1544 }
1545 queue
1546 .iter()
1547 .map(|job| {
1548 (
1549 job.job_id,
1550 job.peer_id.clone(),
1551 job.bytes.len(),
1552 job.ready_at,
1553 job.queue_sequence,
1554 )
1555 })
1556 .collect::<Vec<_>>()
1557 };
1558
1559 let now = Instant::now();
1560 let mut earliest_ready_at: Option<Instant> = None;
1561 let mut ready_jobs = Vec::new();
1562 for job in &snapshot {
1563 if job.3 <= now {
1564 ready_jobs.push(job.clone());
1565 } else {
1566 earliest_ready_at = Some(match earliest_ready_at {
1567 Some(current) => current.min(job.3),
1568 None => job.3,
1569 });
1570 }
1571 }
1572
1573 if ready_jobs.is_empty() {
1574 if let Some(ready_at) = earliest_ready_at {
1575 tokio::time::sleep(ready_at.saturating_duration_since(Instant::now())).await;
1576 continue;
1577 }
1578 self.response_scheduler_running
1579 .store(false, Ordering::Release);
1580 return;
1581 }
1582
1583 let (selected_job_id, selected_finish) = {
1584 let stats = self.peer_wire_stats.read().await;
1585 Self::choose_ready_response_job(&ready_jobs, &stats).expect("ready response job")
1586 };
1587
1588 let selected = {
1589 let mut queue = self.pending_response_sends.lock().await;
1590 let Some(index) = queue.iter().position(|job| job.job_id == selected_job_id) else {
1591 continue;
1592 };
1593 queue.swap_remove(index)
1594 };
1595
1596 let sent = if let Some(channel) = self.signaling.get_channel(&selected.peer_id).await {
1597 channel.send(selected.bytes.clone()).await.is_ok()
1598 } else {
1599 false
1600 };
1601
1602 let queued_peers = {
1603 let queue = self.pending_response_sends.lock().await;
1604 queue
1605 .iter()
1606 .map(|job| job.peer_id.clone())
1607 .collect::<HashSet<_>>()
1608 };
1609 let mut stats = self.peer_wire_stats.write().await;
1610 let entry = stats.entry(selected.peer_id.clone()).or_default();
1611 if sent {
1612 entry.bytes_sent = entry.bytes_sent.saturating_add(selected.bytes.len() as u64);
1613 entry.bandwidth_debt = selected_finish;
1614 }
1615 if queued_peers.is_empty() {
1616 for peer_stats in stats.values_mut() {
1617 peer_stats.bandwidth_debt = 0.0;
1618 }
1619 } else {
1620 let floor = queued_peers
1621 .iter()
1622 .filter_map(|peer_id| stats.get(peer_id).map(|peer| peer.bandwidth_debt))
1623 .fold(f64::INFINITY, f64::min);
1624 if floor.is_finite() && floor > 0.0 {
1625 for peer_id in queued_peers {
1626 if let Some(peer_stats) = stats.get_mut(&peer_id) {
1627 peer_stats.bandwidth_debt =
1628 (peer_stats.bandwidth_debt - floor).max(0.0);
1629 }
1630 }
1631 }
1632 }
1633 }
1634 }
1635
1636 fn deterministic_actor_draw_for(peer_id: &str, hash: &Hash, salt: u64) -> f64 {
1637 let mut hasher = DefaultHasher::new();
1638 peer_id.hash(&mut hasher);
1639 hash.hash(&mut hasher);
1640 salt.hash(&mut hasher);
1641 let v = hasher.finish();
1642 (v as f64) / (u64::MAX as f64)
1643 }
1644
1645 fn deterministic_actor_draw(&self, hash: &Hash, salt: u64) -> f64 {
1646 Self::deterministic_actor_draw_for(self.signaling.peer_id(), hash, salt)
1647 }
1648
1649 fn peer_metadata_pointer_slot_hash() -> Hash {
1650 hashtree_core::sha256(PEER_METADATA_POINTER_SLOT_KEY)
1651 }
1652
1653 fn decode_hash_hex(hash_hex: &str) -> Result<Hash, StoreError> {
1654 let bytes = hex::decode(hash_hex)
1655 .map_err(|e| StoreError::Other(format!("Invalid hash hex: {e}")))?;
1656 if bytes.len() != 32 {
1657 return Err(StoreError::Other(format!(
1658 "Invalid hash length {}, expected 32 bytes",
1659 bytes.len()
1660 )));
1661 }
1662 let mut hash = [0u8; 32];
1663 hash.copy_from_slice(&bytes);
1664 Ok(hash)
1665 }
1666
1667 fn should_drop_response(&self, hash: &Hash) -> bool {
1668 let p = self.response_behavior().drop_response_prob;
1669 if p <= 0.0 {
1670 return false;
1671 }
1672 self.deterministic_actor_draw(hash, 0xD0_D0_D0_D0_D0_D0_D0_D0) < p
1673 }
1674
1675 fn should_corrupt_response(&self, hash: &Hash) -> bool {
1676 let p = self.response_behavior().corrupt_response_prob;
1677 if p <= 0.0 {
1678 return false;
1679 }
1680 self.deterministic_actor_draw(hash, 0xC0_C0_C0_C0_C0_C0_C0_C0) < p
1681 }
1682
1683 fn should_stall_response(&self, hash: &Hash) -> bool {
1684 let p = self.response_behavior().stall_response_prob;
1685 if p <= 0.0 {
1686 return false;
1687 }
1688 self.deterministic_actor_draw(hash, 0x5A_11_5A_11_5A_11_5A_11) < p
1689 }
1690
1691 fn response_send_delay(&self, hash: &Hash, payload_len: usize) -> Duration {
1692 let behavior = self.response_behavior();
1693 let mut total_ms = behavior
1694 .extra_delay_ms
1695 .saturating_add(behavior.first_byte_delay_ms);
1696
1697 if behavior.bytes_per_second > 0 && payload_len > 0 {
1698 let throughput_ms = ((payload_len as u128) * 1000)
1699 .div_ceil(behavior.bytes_per_second as u128)
1700 .min(u64::MAX as u128) as u64;
1701 total_ms = total_ms.saturating_add(throughput_ms);
1702 }
1703
1704 if behavior.stall_delay_ms > 0 && self.should_stall_response(hash) {
1705 total_ms = total_ms.saturating_add(behavior.stall_delay_ms);
1706 }
1707
1708 Duration::from_millis(total_ms)
1709 }
1710
1711 async fn ordered_connected_peers(&self, exclude_peer_id: Option<&str>) -> Vec<String> {
1712 let current_peer_ids = self.signaling.peer_ids().await;
1713 if current_peer_ids.is_empty() {
1714 return Vec::new();
1715 }
1716
1717 sync_selector_peers(&self.peer_selector, ¤t_peer_ids).await;
1718 let hash_get_peer_ids: HashSet<String> = self
1719 .signaling
1720 .hash_get_peer_ids()
1721 .await
1722 .into_iter()
1723 .collect();
1724 let mut candidate_peer_ids: Vec<String> = current_peer_ids
1725 .into_iter()
1726 .filter(|peer_id| hash_get_peer_ids.contains(peer_id))
1727 .filter(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude))
1728 .collect();
1729 if candidate_peer_ids.is_empty() {
1730 return Vec::new();
1731 }
1732
1733 let current_set: HashSet<&str> = candidate_peer_ids.iter().map(String::as_str).collect();
1734 let mut selector = self.peer_selector.write().await;
1735 let mut selector_order = selector.select_peers();
1736 selector_order.retain(|peer_id| current_set.contains(peer_id.as_str()));
1737 if selector_order.is_empty() {
1738 let mut fallback = candidate_peer_ids;
1739 fallback.sort();
1740 return fallback;
1741 }
1742 let backed_off: HashMap<String, bool> = candidate_peer_ids
1743 .iter()
1744 .map(|peer_id| (peer_id.clone(), selector.is_peer_backed_off(peer_id)))
1745 .collect();
1746 drop(selector);
1747
1748 let rank: HashMap<&str, usize> = selector_order
1749 .iter()
1750 .enumerate()
1751 .map(|(idx, peer_id)| (peer_id.as_str(), idx))
1752 .collect();
1753 let active = self.peer_active_requests.read().await;
1754 candidate_peer_ids.sort_by(|left, right| {
1755 let left_backed_off = backed_off.get(left).copied().unwrap_or(false);
1756 let right_backed_off = backed_off.get(right).copied().unwrap_or(false);
1757 if left_backed_off != right_backed_off {
1758 return if left_backed_off {
1759 std::cmp::Ordering::Greater
1760 } else {
1761 std::cmp::Ordering::Less
1762 };
1763 }
1764 let left_rank = rank.get(left.as_str()).copied().unwrap_or(usize::MAX / 2);
1765 let right_rank = rank.get(right.as_str()).copied().unwrap_or(usize::MAX / 2);
1766 let left_load = active.get(left).copied().unwrap_or(0);
1767 let right_load = active.get(right).copied().unwrap_or(0);
1768 (left_rank + left_load.saturating_mul(ACTIVE_PEER_REQUEST_RANK_PENALTY))
1769 .cmp(&(right_rank + right_load.saturating_mul(ACTIVE_PEER_REQUEST_RANK_PENALTY)))
1770 .then_with(|| left.cmp(right))
1771 });
1772 candidate_peer_ids
1773 }
1774
1775 async fn reserve_peer_request(&self, peer_id: &str) {
1776 let mut active = self.peer_active_requests.write().await;
1777 *active.entry(peer_id.to_string()).or_insert(0) += 1;
1778 }
1779
1780 async fn release_peer_request(&self, peer_id: &str) {
1781 let mut active = self.peer_active_requests.write().await;
1782 let Some(count) = active.get_mut(peer_id) else {
1783 return;
1784 };
1785 if *count <= 1 {
1786 active.remove(peer_id);
1787 } else {
1788 *count -= 1;
1789 }
1790 }
1791
1792 async fn release_queried_peer_requests(&self, peer_ids: &[String]) {
1793 for peer_id in peer_ids {
1794 self.release_peer_request(peer_id).await;
1795 }
1796 }
1797
1798 fn requested_quote_mint(&self) -> Option<&str> {
1799 if let Some(default_mint) = self.routing.cashu_default_mint.as_deref() {
1800 if self.routing.cashu_accepted_mints.is_empty()
1801 || self
1802 .routing
1803 .cashu_accepted_mints
1804 .iter()
1805 .any(|mint| mint == default_mint)
1806 {
1807 return Some(default_mint);
1808 }
1809 }
1810
1811 self.routing
1812 .cashu_accepted_mints
1813 .first()
1814 .map(String::as_str)
1815 }
1816
1817 fn choose_quote_mint(&self, requested_mint: Option<&str>) -> Option<String> {
1818 if let Some(requested_mint) = requested_mint {
1819 if self.accepts_quote_mint(Some(requested_mint)) {
1820 return Some(requested_mint.to_string());
1821 }
1822 }
1823 if let Some(default_mint) = self.routing.cashu_default_mint.as_ref() {
1824 return Some(default_mint.clone());
1825 }
1826 if let Some(first_mint) = self.routing.cashu_accepted_mints.first() {
1827 return Some(first_mint.clone());
1828 }
1829 requested_mint.map(str::to_string)
1830 }
1831
1832 fn accepts_quote_mint(&self, mint_url: Option<&str>) -> bool {
1833 if self.routing.cashu_accepted_mints.is_empty() {
1834 return true;
1835 }
1836
1837 let Some(mint_url) = mint_url else {
1838 return false;
1839 };
1840 self.routing
1841 .cashu_accepted_mints
1842 .iter()
1843 .any(|mint| mint == mint_url)
1844 }
1845
1846 fn trusts_quote_mint(&self, mint_url: Option<&str>) -> bool {
1847 let Some(mint_url) = mint_url else {
1848 return self.routing.cashu_default_mint.is_none()
1849 && self.routing.cashu_accepted_mints.is_empty();
1850 };
1851 self.routing.cashu_default_mint.as_deref() == Some(mint_url)
1852 || self
1853 .routing
1854 .cashu_accepted_mints
1855 .iter()
1856 .any(|mint| mint == mint_url)
1857 }
1858
1859 async fn peer_suggested_mint_cap_sat(&self, peer_id: &str) -> u64 {
1860 let base = self.routing.cashu_peer_suggested_mint_base_cap_sat;
1861 if base == 0 {
1862 return 0;
1863 }
1864
1865 let selector = self.peer_selector.read().await;
1866 let Some(stats) = selector.get_stats(peer_id) else {
1867 let max_cap = self.routing.cashu_peer_suggested_mint_max_cap_sat;
1868 return if max_cap > 0 { base.min(max_cap) } else { base };
1869 };
1870
1871 if stats.cashu_payment_defaults > 0
1872 && stats.cashu_payment_defaults >= stats.cashu_payment_receipts
1873 {
1874 return 0;
1875 }
1876
1877 let success_bonus = stats
1878 .successes
1879 .saturating_mul(self.routing.cashu_peer_suggested_mint_success_step_sat);
1880 let receipt_bonus = stats
1881 .cashu_payment_receipts
1882 .saturating_mul(self.routing.cashu_peer_suggested_mint_receipt_step_sat);
1883 let mut cap = base
1884 .saturating_add(success_bonus)
1885 .saturating_add(receipt_bonus);
1886 let max_cap = self.routing.cashu_peer_suggested_mint_max_cap_sat;
1887 if max_cap > 0 {
1888 cap = cap.min(max_cap);
1889 }
1890 cap
1891 }
1892
1893 async fn should_accept_quote_response(
1894 &self,
1895 from_peer: &str,
1896 preferred_mint_url: Option<&str>,
1897 offered_payment_sat: u64,
1898 res: &DataQuoteResponse,
1899 ) -> bool {
1900 let Some(payment_sat) = res.p else {
1901 return false;
1902 };
1903 if payment_sat > offered_payment_sat {
1904 return false;
1905 }
1906
1907 let response_mint = res.m.as_deref();
1908 if response_mint == preferred_mint_url {
1909 return true;
1910 }
1911 if self.trusts_quote_mint(response_mint) {
1912 return true;
1913 }
1914 if response_mint.is_none() {
1915 return false;
1916 }
1917
1918 payment_sat <= self.peer_suggested_mint_cap_sat(from_peer).await
1919 }
1920
1921 async fn issue_quote(
1922 &self,
1923 peer_id: &str,
1924 hash_key: &str,
1925 payment_sat: u64,
1926 ttl_ms: u32,
1927 mint_url: Option<&str>,
1928 ) -> u64 {
1929 let quote_id = {
1930 let mut next = self.next_quote_id.write().await;
1931 let quote_id = *next;
1932 *next = next.saturating_add(1);
1933 quote_id
1934 };
1935
1936 let expires_at = Instant::now() + Duration::from_millis(ttl_ms as u64);
1937 self.issued_quotes.write().await.insert(
1938 (peer_id.to_string(), hash_key.to_string(), quote_id),
1939 IssuedQuote {
1940 expires_at,
1941 payment_sat,
1942 mint_url: mint_url.map(str::to_string),
1943 },
1944 );
1945 quote_id
1946 }
1947
1948 async fn take_valid_quote(&self, peer_id: &str, hash_key: &str, quote_id: u64) -> bool {
1949 let key = (peer_id.to_string(), hash_key.to_string(), quote_id);
1950 let Some(quote) = self.issued_quotes.write().await.remove(&key) else {
1951 return false;
1952 };
1953 quote.expires_at > Instant::now()
1954 }
1955
1956 async fn send_request_to_peer(
1957 &self,
1958 peer_id: &str,
1959 hash: &Hash,
1960 request_htl: u8,
1961 quote_id: Option<u64>,
1962 ) -> bool {
1963 if !should_forward_htl(request_htl) {
1964 return false;
1965 }
1966
1967 let channel = match self.signaling.get_channel(peer_id).await {
1968 Some(c) => c,
1969 None => return false,
1970 };
1971
1972 let htl_config = {
1973 let configs = self.htl_configs.read().await;
1974 configs
1975 .get(peer_id)
1976 .cloned()
1977 .unwrap_or_else(PeerHTLConfig::random)
1978 };
1979
1980 let send_htl = htl_config.decrement(request_htl);
1981 let req = match quote_id {
1982 Some(quote_id) => create_request_with_quote(hash, send_htl, quote_id),
1983 None => create_request(hash, send_htl),
1984 };
1985 let request_bytes = encode_request(&req);
1986 let request_len = request_bytes.len() as u64;
1987
1988 {
1989 let mut selector = self.peer_selector.write().await;
1990 selector.record_request(peer_id, request_len);
1991 }
1992
1993 match channel.send(request_bytes).await {
1994 Ok(()) => {
1995 self.record_peer_wire_sent(peer_id, request_len).await;
1996 true
1997 }
1998 Err(_) => {
1999 self.peer_selector.write().await.record_failure(peer_id);
2000 false
2001 }
2002 }
2003 }
2004
2005 async fn send_quote_request_to_peer(
2006 &self,
2007 peer_id: &str,
2008 hash: &Hash,
2009 payment_sat: u64,
2010 ttl_ms: u32,
2011 mint_url: Option<&str>,
2012 ) -> bool {
2013 let channel = match self.signaling.get_channel(peer_id).await {
2014 Some(c) => c,
2015 None => return false,
2016 };
2017
2018 let req = create_quote_request(hash, ttl_ms, payment_sat, mint_url);
2019 let request_bytes = encode_quote_request(&req);
2020 let request_len = request_bytes.len() as u64;
2021
2022 match channel.send(request_bytes).await {
2023 Ok(()) => {
2024 self.record_peer_wire_sent(peer_id, request_len).await;
2025 true
2026 }
2027 Err(_) => false,
2028 }
2029 }
2030
2031 pub async fn set_read_sources(&self, sources: Vec<Arc<dyn MeshReadSource>>) {
2032 let mut by_id = HashMap::new();
2033 let mut stats = self.read_source_stats.write().await;
2034 for source in sources {
2035 let source_id = source.id().to_string();
2036 by_id.insert(source_id.clone(), source);
2037 stats
2038 .entry(source_id)
2039 .or_insert_with(AdaptiveSourceStats::default);
2040 }
2041 *self.read_sources.write().await = by_id;
2042 }
2043
2044 async fn record_read_source_request(&self, source_id: &str) {
2045 let mut stats = self.read_source_stats.write().await;
2046 stats
2047 .entry(source_id.to_string())
2048 .or_insert_with(AdaptiveSourceStats::default)
2049 .requests += 1;
2050 }
2051
2052 async fn record_read_source_miss(&self, source_id: &str) {
2053 let mut stats = self.read_source_stats.write().await;
2054 stats
2055 .entry(source_id.to_string())
2056 .or_insert_with(AdaptiveSourceStats::default)
2057 .misses += 1;
2058 }
2059
2060 async fn record_read_source_success(&self, source_id: &str, elapsed_ms: u64) {
2061 let now = Instant::now();
2062 let mut stats = self.read_source_stats.write().await;
2063 let stats = stats
2064 .entry(source_id.to_string())
2065 .or_insert_with(AdaptiveSourceStats::default);
2066 stats.successes += 1;
2067 stats.last_success_at = Some(now);
2068 stats.backoff_level = 0;
2069 stats.backed_off_until = None;
2070 if stats.srtt_ms <= 0.0 {
2071 stats.srtt_ms = elapsed_ms as f64;
2072 stats.rttvar_ms = elapsed_ms as f64 / 2.0;
2073 return;
2074 }
2075 let elapsed = elapsed_ms as f64;
2076 stats.rttvar_ms = 0.75 * stats.rttvar_ms + 0.25 * (stats.srtt_ms - elapsed).abs();
2077 stats.srtt_ms = 0.875 * stats.srtt_ms + 0.125 * elapsed;
2078 }
2079
2080 async fn record_read_source_failure(&self, source_id: &str) {
2081 let now = Instant::now();
2082 let mut stats = self.read_source_stats.write().await;
2083 let stats = stats
2084 .entry(source_id.to_string())
2085 .or_insert_with(AdaptiveSourceStats::default);
2086 stats.failures += 1;
2087 stats.last_failure_at = Some(now);
2088 Self::apply_source_backoff(stats, now);
2089 }
2090
2091 async fn record_read_source_timeout(&self, source_id: &str) {
2092 let now = Instant::now();
2093 let mut stats = self.read_source_stats.write().await;
2094 let stats = stats
2095 .entry(source_id.to_string())
2096 .or_insert_with(AdaptiveSourceStats::default);
2097 stats.timeouts += 1;
2098 stats.last_failure_at = Some(now);
2099 Self::apply_source_backoff(stats, now);
2100 }
2101
2102 fn apply_source_backoff(stats: &mut AdaptiveSourceStats, now: Instant) {
2103 stats.backoff_level = stats.backoff_level.saturating_add(1);
2104 let backoff_ms = (INITIAL_SOURCE_BACKOFF_MS
2105 .saturating_mul(2u64.saturating_pow(stats.backoff_level.saturating_sub(1))))
2106 .min(MAX_SOURCE_BACKOFF_MS);
2107 stats.backed_off_until = Some(now + Duration::from_millis(backoff_ms));
2108 }
2109
2110 async fn ordered_read_sources(&self) -> Vec<Arc<dyn MeshReadSource>> {
2111 let sources = self.read_sources.read().await;
2112 if sources.is_empty() {
2113 return Vec::new();
2114 }
2115
2116 let mut available: Vec<Arc<dyn MeshReadSource>> = sources
2117 .values()
2118 .filter(|source| source.is_available())
2119 .cloned()
2120 .collect();
2121 if available.is_empty() {
2122 return Vec::new();
2123 }
2124
2125 let now = Instant::now();
2126 let stats = self.read_source_stats.read().await;
2127 let mut healthy: Vec<Arc<dyn MeshReadSource>> = available
2128 .iter()
2129 .filter(|source| {
2130 stats
2131 .get(source.id())
2132 .and_then(|s| s.backed_off_until)
2133 .is_none_or(|until| until <= now)
2134 })
2135 .cloned()
2136 .collect();
2137 if !healthy.is_empty() {
2138 available = std::mem::take(&mut healthy);
2139 }
2140
2141 available.sort_by(|left, right| {
2142 let left_stats = stats.get(left.id()).cloned().unwrap_or_default();
2143 let right_stats = stats.get(right.id()).cloned().unwrap_or_default();
2144 adaptive_source_score(&right_stats, now)
2145 .partial_cmp(&adaptive_source_score(&left_stats, now))
2146 .unwrap_or(std::cmp::Ordering::Equal)
2147 .then_with(|| left.id().cmp(right.id()))
2148 });
2149 available
2150 }
2151
2152 async fn should_probe_multiple_read_sources(
2153 &self,
2154 ordered_sources: &[Arc<dyn MeshReadSource>],
2155 ) -> bool {
2156 if ordered_sources.len() <= 1 {
2157 return false;
2158 }
2159 let stats = self.read_source_stats.read().await;
2160 let best = stats
2161 .get(ordered_sources[0].id())
2162 .cloned()
2163 .unwrap_or_default();
2164 let second = stats
2165 .get(ordered_sources[1].id())
2166 .cloned()
2167 .unwrap_or_default();
2168 if !source_has_history(&best) || !source_has_history(&second) {
2169 return false;
2170 }
2171 let now = Instant::now();
2172 adaptive_source_score(&best, now) - adaptive_source_score(&second, now)
2173 < SOURCE_SCORE_TIE_DELTA
2174 }
2175
2176 async fn source_dispatch_for(&self, source_count: usize) -> RequestDispatchConfig {
2177 if source_count == 0 {
2178 return self.routing.dispatch;
2179 }
2180 let ordered_sources = self.ordered_read_sources().await;
2181 let probe_multiple = self
2182 .should_probe_multiple_read_sources(&ordered_sources)
2183 .await;
2184 let initial_fanout = if probe_multiple {
2185 source_count.min(2)
2186 } else {
2187 1
2188 };
2189 RequestDispatchConfig {
2190 initial_fanout,
2191 hedge_fanout: self.routing.dispatch.hedge_fanout,
2192 max_fanout: self.routing.dispatch.max_fanout.min(source_count),
2193 hedge_interval_ms: self.routing.dispatch.hedge_interval_ms,
2194 }
2195 }
2196
2197 pub async fn peer_count(&self) -> usize {
2199 self.signaling.peer_count().await
2200 }
2201
2202 pub async fn peer_ids(&self) -> Vec<String> {
2204 self.signaling.peer_ids().await
2205 }
2206
2207 pub async fn needs_peers(&self) -> bool {
2209 self.signaling.needs_peers().await
2210 }
2211
2212 pub async fn send_hello(&self) -> Result<(), TransportError> {
2214 self.signaling.send_hello(vec![]).await
2215 }
2216
2217 pub async fn drain_available_data_messages(self: &Arc<Self>) -> DataPumpStats {
2222 let mut stats = DataPumpStats::default();
2223 let peer_ids = self.signaling.peer_ids().await;
2224 for peer_id in peer_ids {
2225 let Some(channel) = self.signaling.get_channel(&peer_id).await else {
2226 continue;
2227 };
2228
2229 while let Some(data) = channel.try_recv() {
2230 stats.processed += 1;
2231 stats.processed_bytes += data.len() as u64;
2232 if let Some(msg) = parse_message(&data) {
2233 match msg {
2234 DataMessage::Request(_) => stats.request_messages += 1,
2235 DataMessage::Response(_) => stats.response_messages += 1,
2236 DataMessage::QuoteRequest(_) => stats.quote_request_messages += 1,
2237 DataMessage::QuoteResponse(_) => stats.quote_response_messages += 1,
2238 DataMessage::PubsubInterest(_) => stats.pubsub_interest_messages += 1,
2239 DataMessage::PubsubFrame(_) => stats.pubsub_frame_messages += 1,
2240 DataMessage::PubsubInventory(_) => stats.pubsub_inventory_messages += 1,
2241 DataMessage::PubsubWant(_) => stats.pubsub_want_messages += 1,
2242 DataMessage::Payment(_)
2243 | DataMessage::PaymentAck(_)
2244 | DataMessage::Chunk(_)
2245 | DataMessage::PeerHints(_) => {}
2246 }
2247 }
2248 self.handle_data_message(&peer_id, &data).await;
2249 }
2250 }
2251 stats
2252 }
2253
2254 pub async fn record_cashu_payment_for_peer(&self, peer_id: &str, amount_sat: u64) {
2256 self.peer_selector
2257 .write()
2258 .await
2259 .record_cashu_payment(peer_id, amount_sat);
2260 }
2261
2262 pub async fn record_cashu_receipt_from_peer(&self, peer_id: &str, amount_sat: u64) {
2264 self.peer_selector
2265 .write()
2266 .await
2267 .record_cashu_receipt(peer_id, amount_sat);
2268 }
2269
2270 pub async fn record_cashu_payment_default_from_peer(&self, peer_id: &str) {
2272 self.peer_selector
2273 .write()
2274 .await
2275 .record_cashu_payment_default(peer_id);
2276 }
2277
2278 pub async fn selector_summary(&self) -> crate::peer_selector::SelectorSummary {
2280 self.peer_selector.read().await.summary()
2281 }
2282
2283 fn should_refuse_requests_from_peer(&self, selector: &PeerSelector, peer_id: &str) -> bool {
2284 selector.is_peer_blocked_for_payment_defaults(
2285 peer_id,
2286 self.routing.cashu_payment_default_block_threshold,
2287 )
2288 }
2289
2290 pub async fn peer_metadata_snapshot(&self) -> PeerMetadataSnapshot {
2292 self.peer_selector
2293 .read()
2294 .await
2295 .export_peer_metadata_snapshot()
2296 }
2297
2298 pub async fn persist_peer_metadata(&self) -> Result<Hash, StoreError> {
2303 let snapshot = self
2304 .peer_selector
2305 .read()
2306 .await
2307 .export_peer_metadata_snapshot();
2308 let bytes = serde_json::to_vec(&snapshot).map_err(|e| {
2309 StoreError::Other(format!("Failed to encode peer metadata snapshot: {e}"))
2310 })?;
2311 let snapshot_hash = hashtree_core::sha256(&bytes);
2312 let _ = self.local_store.put(snapshot_hash, bytes).await?;
2313
2314 let pointer_slot = Self::peer_metadata_pointer_slot_hash();
2315 let pointer_bytes = hex::encode(snapshot_hash).into_bytes();
2316 let _ = self.local_store.delete(&pointer_slot).await?;
2317 let _ = self.local_store.put(pointer_slot, pointer_bytes).await?;
2318
2319 Ok(snapshot_hash)
2320 }
2321
2322 pub async fn load_peer_metadata(&self) -> Result<bool, StoreError> {
2324 let pointer_slot = Self::peer_metadata_pointer_slot_hash();
2325 let Some(pointer_bytes) = self.local_store.get(&pointer_slot).await? else {
2326 return Ok(false);
2327 };
2328 let pointer_hex = std::str::from_utf8(&pointer_bytes).map_err(|e| {
2329 StoreError::Other(format!("Peer metadata pointer is not valid UTF-8: {e}"))
2330 })?;
2331 let snapshot_hash = Self::decode_hash_hex(pointer_hex.trim())?;
2332
2333 let Some(snapshot_bytes) = self.local_store.get(&snapshot_hash).await? else {
2334 return Ok(false);
2335 };
2336 let snapshot: PeerMetadataSnapshot =
2337 serde_json::from_slice(&snapshot_bytes).map_err(|e| {
2338 StoreError::Other(format!("Failed to decode peer metadata snapshot: {e}"))
2339 })?;
2340 self.peer_selector
2341 .write()
2342 .await
2343 .import_peer_metadata_snapshot(&snapshot);
2344 Ok(true)
2345 }
2346
2347 pub async fn get_with_quote(
2352 &self,
2353 hash: &Hash,
2354 payment_sat: u64,
2355 quote_ttl: Duration,
2356 ) -> Result<Option<Vec<u8>>, StoreError> {
2357 if let Some(data) = self.local_store.get(hash).await? {
2358 return Ok(Some(data));
2359 }
2360 Ok(self
2361 .request_from_peers_with_quote(hash, payment_sat, quote_ttl)
2362 .await)
2363 }
2364
2365 async fn request_from_peers_with_quote(
2366 &self,
2367 hash: &Hash,
2368 payment_sat: u64,
2369 quote_ttl: Duration,
2370 ) -> Option<Vec<u8>> {
2371 let ordered_peer_ids = self.ordered_connected_peers(None).await;
2372 if ordered_peer_ids.is_empty() {
2373 return None;
2374 }
2375
2376 if let Some(quote) = self
2377 .request_quote_from_peers(hash, payment_sat, quote_ttl, &ordered_peer_ids)
2378 .await
2379 {
2380 if let Some(data) = self
2381 .request_from_single_peer(hash, "e.peer_id, MAX_HTL, Some(quote.quote_id))
2382 .await
2383 {
2384 return Some(data);
2385 }
2386 }
2387
2388 self.request_from_mesh(hash).await
2389 }
2390
2391 async fn request_quote_from_peers(
2392 &self,
2393 hash: &Hash,
2394 payment_sat: u64,
2395 quote_ttl: Duration,
2396 ordered_peer_ids: &[String],
2397 ) -> Option<NegotiatedQuote> {
2398 if ordered_peer_ids.is_empty() {
2399 return None;
2400 }
2401 let ttl_ms = quote_ttl.as_millis().min(u32::MAX as u128) as u32;
2402 if ttl_ms == 0 {
2403 return None;
2404 }
2405 let requested_mint = self.requested_quote_mint().map(str::to_string);
2406
2407 let hash_key = hash_to_key(hash);
2408 let (tx, rx) = oneshot::channel();
2409 self.pending_quotes.write().await.insert(
2410 hash_key.clone(),
2411 PendingQuoteRequest {
2412 response_tx: tx,
2413 preferred_mint_url: requested_mint.clone(),
2414 offered_payment_sat: payment_sat,
2415 },
2416 );
2417
2418 let rx = Arc::new(Mutex::new(rx));
2419 let result = run_hedged_waves(
2420 ordered_peer_ids.len(),
2421 self.routing.dispatch,
2422 self.request_timeout,
2423 |range| {
2424 let wave_peer_ids = ordered_peer_ids[range].to_vec();
2425 let requested_mint = requested_mint.clone();
2426 let hash = *hash;
2427 async move {
2428 let mut sent = 0usize;
2429 for peer_id in wave_peer_ids {
2430 if self
2431 .send_quote_request_to_peer(
2432 &peer_id,
2433 &hash,
2434 payment_sat,
2435 ttl_ms,
2436 requested_mint.as_deref(),
2437 )
2438 .await
2439 {
2440 sent += 1;
2441 }
2442 }
2443 sent
2444 }
2445 },
2446 |wait| {
2447 let rx = rx.clone();
2448 async move {
2449 let mut rx = rx.lock().await;
2450 match tokio::time::timeout(wait, &mut *rx).await {
2451 Ok(Ok(Some(quote))) => HedgedWaveAction::Success(quote),
2452 Ok(Ok(None)) | Ok(Err(_)) => HedgedWaveAction::Abort,
2453 Err(_) => HedgedWaveAction::Continue,
2454 }
2455 }
2456 },
2457 )
2458 .await;
2459 let _ = self.pending_quotes.write().await.remove(&hash_key);
2460 result
2461 }
2462
2463 async fn request_from_single_peer(
2464 &self,
2465 hash: &Hash,
2466 peer_id: &str,
2467 request_htl: u8,
2468 quote_id: Option<u64>,
2469 ) -> Option<Vec<u8>> {
2470 let hash_key = hash_to_key(hash);
2471 let (tx, rx) = oneshot::channel();
2472 self.pending_requests.write().await.insert(
2473 hash_key.clone(),
2474 PendingRequest {
2475 response_tx: tx,
2476 started_at: Instant::now(),
2477 queried_peers: vec![peer_id.to_string()],
2478 },
2479 );
2480
2481 let mut rx = rx;
2482 if !self
2483 .send_request_to_peer(peer_id, hash, request_htl, quote_id)
2484 .await
2485 {
2486 let _ = self.pending_requests.write().await.remove(&hash_key);
2487 return None;
2488 }
2489 self.reserve_peer_request(peer_id).await;
2490
2491 if let Ok(Ok(Some(data))) = tokio::time::timeout(self.request_timeout, &mut rx).await {
2492 if hashtree_core::sha256(&data) == *hash {
2493 let _ = self.local_store.put(*hash, data.clone()).await;
2494 return Some(data);
2495 }
2496 }
2497
2498 if let Some(pending) = self.pending_requests.write().await.remove(&hash_key) {
2499 self.release_queried_peer_requests(&pending.queried_peers)
2500 .await;
2501 for peer_id in pending.queried_peers {
2502 self.peer_selector.write().await.record_timeout(&peer_id);
2503 }
2504 }
2505 let _ = self.take_forward_requesters(&hash_key).await;
2506 None
2507 }
2508
2509 async fn request_from_ordered_peers(
2510 &self,
2511 hash: &Hash,
2512 ordered_peer_ids: &[String],
2513 request_htl: u8,
2514 ) -> RouteFetchOutcome {
2515 let hash_key = hash_to_key(hash);
2516 let (tx, rx) = oneshot::channel();
2517 self.pending_requests.write().await.insert(
2518 hash_key.clone(),
2519 PendingRequest {
2520 response_tx: tx,
2521 started_at: Instant::now(),
2522 queried_peers: Vec::new(),
2523 },
2524 );
2525
2526 let rx = Arc::new(Mutex::new(rx));
2527 let result = run_hedged_waves(
2528 ordered_peer_ids.len(),
2529 self.routing.dispatch,
2530 self.request_timeout,
2531 |range| {
2532 let wave_peer_ids = ordered_peer_ids[range].to_vec();
2533 let hash = *hash;
2534 let hash_key = hash_key.clone();
2535 async move {
2536 let mut sent = 0usize;
2537 for peer_id in wave_peer_ids {
2538 if self
2539 .send_request_to_peer(&peer_id, &hash, request_htl, None)
2540 .await
2541 {
2542 sent += 1;
2543 self.reserve_peer_request(&peer_id).await;
2544 if let Some(pending) =
2545 self.pending_requests.write().await.get_mut(&hash_key)
2546 {
2547 pending.queried_peers.push(peer_id);
2548 }
2549 }
2550 }
2551 sent
2552 }
2553 },
2554 |wait| {
2555 let rx = rx.clone();
2556 async move {
2557 let mut rx = rx.lock().await;
2558 match tokio::time::timeout(wait, &mut *rx).await {
2559 Ok(Ok(Some(data))) if hashtree_core::sha256(&data) == *hash => {
2560 HedgedWaveAction::Success(data)
2561 }
2562 Ok(Ok(Some(_))) => HedgedWaveAction::Continue,
2563 Ok(Ok(None)) | Ok(Err(_)) => HedgedWaveAction::Abort,
2564 Err(_) => HedgedWaveAction::Continue,
2565 }
2566 }
2567 },
2568 )
2569 .await;
2570
2571 let Some(data) = result else {
2572 if let Some(pending) = self.pending_requests.write().await.remove(&hash_key) {
2573 self.release_queried_peer_requests(&pending.queried_peers)
2574 .await;
2575 for peer_id in pending.queried_peers {
2576 self.peer_selector.write().await.record_timeout(&peer_id);
2577 }
2578 }
2579 let _ = self.take_forward_requesters(&hash_key).await;
2580 return RouteFetchOutcome::Timeout;
2581 };
2582
2583 let _ = self.local_store.put(*hash, data.clone()).await;
2584 RouteFetchOutcome::Hit(data)
2585 }
2586
2587 async fn request_from_read_sources_inner(&self, hash: &Hash) -> RouteFetchOutcome {
2588 let ordered_sources = self.ordered_read_sources().await;
2589 if ordered_sources.is_empty() {
2590 return RouteFetchOutcome::Miss;
2591 }
2592
2593 let dispatch = normalize_dispatch_config(
2594 self.source_dispatch_for(ordered_sources.len()).await,
2595 ordered_sources.len(),
2596 );
2597 let wave_plan = build_hedged_wave_plan(ordered_sources.len(), dispatch);
2598 if wave_plan.is_empty() {
2599 return RouteFetchOutcome::Miss;
2600 }
2601
2602 let deadline = Instant::now() + self.request_timeout;
2603 let mut pending = FuturesUnordered::new();
2604 let mut pending_source_ids = HashSet::new();
2605 let mut saw_timeout = false;
2606 let mut next_source_idx = 0usize;
2607
2608 for (wave_idx, wave_size) in wave_plan.iter().copied().enumerate() {
2609 let from = next_source_idx;
2610 let to = (next_source_idx + wave_size).min(ordered_sources.len());
2611 next_source_idx = to;
2612
2613 for source in &ordered_sources[from..to] {
2614 let source = Arc::clone(source);
2615 let source_id = source.id().to_string();
2616 self.record_read_source_request(&source_id).await;
2617 pending_source_ids.insert(source_id.clone());
2618 let hash = *hash;
2619 pending.push(tokio::spawn(async move {
2620 let started_at = Instant::now();
2621 let result = std::panic::AssertUnwindSafe(source.get(&hash))
2622 .catch_unwind()
2623 .await;
2624 match result {
2625 Ok(Some(data)) => SourceFetchOutcome::Hit {
2626 source_id,
2627 data,
2628 elapsed_ms: started_at.elapsed().as_millis().max(1) as u64,
2629 },
2630 Ok(None) => SourceFetchOutcome::Miss { source_id },
2631 Err(_) => SourceFetchOutcome::Failure { source_id },
2632 }
2633 }));
2634 }
2635
2636 let is_last_wave =
2637 wave_idx + 1 == wave_plan.len() || next_source_idx >= ordered_sources.len();
2638 let window_end = if is_last_wave {
2639 deadline
2640 } else {
2641 (Instant::now() + Duration::from_millis(dispatch.hedge_interval_ms)).min(deadline)
2642 };
2643
2644 while Instant::now() < window_end {
2645 let remaining = window_end.saturating_duration_since(Instant::now());
2646 let Some(result) = tokio::time::timeout(remaining, pending.next())
2647 .await
2648 .ok()
2649 .flatten()
2650 else {
2651 break;
2652 };
2653 let Ok(outcome) = result else {
2654 continue;
2655 };
2656 match outcome {
2657 SourceFetchOutcome::Hit {
2658 source_id,
2659 data,
2660 elapsed_ms,
2661 } => {
2662 pending_source_ids.remove(&source_id);
2663 self.record_read_source_success(&source_id, elapsed_ms)
2664 .await;
2665 return RouteFetchOutcome::Hit(data);
2666 }
2667 SourceFetchOutcome::Miss { source_id } => {
2668 pending_source_ids.remove(&source_id);
2669 self.record_read_source_miss(&source_id).await;
2670 }
2671 SourceFetchOutcome::Failure { source_id } => {
2672 pending_source_ids.remove(&source_id);
2673 self.record_read_source_failure(&source_id).await;
2674 }
2675 }
2676 }
2677
2678 if Instant::now() >= deadline {
2679 break;
2680 }
2681 }
2682
2683 for source_id in pending_source_ids {
2684 saw_timeout = true;
2685 self.record_read_source_timeout(&source_id).await;
2686 }
2687 if saw_timeout {
2688 RouteFetchOutcome::Timeout
2689 } else {
2690 RouteFetchOutcome::Miss
2691 }
2692 }
2693
2694 async fn request_from_read_sources(&self, hash: &Hash) -> RouteFetchOutcome {
2695 let hash_key = hash_to_key(hash);
2696 let existing_wait = {
2697 let mut inflight = self.inflight_source_fetches.lock().await;
2698 if let Some(existing) = inflight.get_mut(&hash_key) {
2699 let (tx, rx) = oneshot::channel();
2700 existing.waiters.push(tx);
2701 Some(rx)
2702 } else {
2703 inflight.insert(
2704 hash_key.clone(),
2705 InflightSourceFetch {
2706 waiters: Vec::new(),
2707 },
2708 );
2709 None
2710 }
2711 };
2712
2713 if let Some(wait) = existing_wait {
2714 return wait.await.unwrap_or(RouteFetchOutcome::Timeout);
2715 }
2716
2717 let result = self.request_from_read_sources_inner(hash).await;
2718 if let RouteFetchOutcome::Hit(hit) = &result {
2719 let _ = self.local_store.put(*hash, hit.clone()).await;
2720 }
2721 self.complete_inflight_source_fetch(&hash_key, result.clone())
2722 .await;
2723
2724 result
2725 }
2726
2727 async fn complete_inflight_source_fetch(&self, hash_key: &str, result: RouteFetchOutcome) {
2728 let waiters = self
2729 .inflight_source_fetches
2730 .lock()
2731 .await
2732 .remove(hash_key)
2733 .map(|inflight| inflight.waiters)
2734 .unwrap_or_default();
2735 for waiter in waiters {
2736 let _ = waiter.send(result.clone());
2737 }
2738 }
2739
2740 async fn cancel_pending_peer_route(&self, hash: &Hash) {
2741 let hash_key = hash_to_key(hash);
2742 if let Some(pending) = self.pending_requests.write().await.remove(&hash_key) {
2743 self.release_queried_peer_requests(&pending.queried_peers)
2744 .await;
2745 }
2746 }
2747
2748 async fn cancel_losing_route(&self, hash: &Hash, route: &ReadRoute, winner_data: &[u8]) {
2749 match route {
2750 ReadRoute::Peers(_) => self.cancel_pending_peer_route(hash).await,
2751 ReadRoute::Sources => {
2752 let hash_key = hash_to_key(hash);
2753 self.complete_inflight_source_fetch(
2754 &hash_key,
2755 RouteFetchOutcome::Hit(winner_data.to_vec()),
2756 )
2757 .await;
2758 }
2759 }
2760 }
2761
2762 async fn ranked_read_routes(&self, context: &MeshReadContext) -> Vec<RankedReadRoute> {
2763 let mut routes = Vec::new();
2764 let ordered_peers = if should_forward_htl(context.request_htl) {
2765 self.ordered_connected_peers(context.exclude_peer_id.as_deref())
2766 .await
2767 } else {
2768 Vec::new()
2769 };
2770 if !ordered_peers.is_empty() {
2771 let best_peer_id = ordered_peers[0].clone();
2772 let selector = self.peer_selector.read().await;
2773 let best_peer = selector.get_stats(&best_peer_id).cloned();
2774 let now = Instant::now();
2775 let (score, has_history) = match best_peer.as_ref() {
2776 Some(stats) => (
2777 peer_endpoint_score(stats, now),
2778 peer_endpoint_has_history(stats),
2779 ),
2780 None => (0.0, false),
2781 };
2782 routes.push(RankedReadRoute {
2783 route: ReadRoute::Peers(ordered_peers),
2784 best_endpoint_id: format!("peer:{best_peer_id}"),
2785 score,
2786 has_history,
2787 });
2788 }
2789 let ordered_sources = self.ordered_read_sources().await;
2790 if let Some(best_source) = ordered_sources.first() {
2791 let stats = self.read_source_stats.read().await;
2792 let best_source_stats = stats.get(best_source.id()).cloned().unwrap_or_default();
2793 let now = Instant::now();
2794 routes.push(RankedReadRoute {
2795 route: ReadRoute::Sources,
2796 best_endpoint_id: format!("source:{}", best_source.id()),
2797 score: adaptive_source_score(&best_source_stats, now),
2798 has_history: source_has_history(&best_source_stats),
2799 });
2800 }
2801 if routes.len() <= 1 {
2802 return routes;
2803 }
2804
2805 routes.sort_by(|left, right| {
2806 right
2807 .score
2808 .partial_cmp(&left.score)
2809 .unwrap_or(std::cmp::Ordering::Equal)
2810 .then_with(|| ranked_route_kind(&left.route).cmp(&ranked_route_kind(&right.route)))
2811 .then_with(|| left.best_endpoint_id.cmp(&right.best_endpoint_id))
2812 .then_with(|| left.route.id().cmp(right.route.id()))
2813 });
2814 routes
2815 }
2816
2817 fn should_probe_multiple_routes(&self, routes: &[RankedReadRoute]) -> bool {
2818 if routes.len() <= 1 {
2819 return false;
2820 }
2821 if !routes[0].has_history || !routes[1].has_history {
2822 return false;
2823 }
2824 (routes[0].score - routes[1].score) < SOURCE_SCORE_TIE_DELTA
2825 }
2826
2827 async fn run_read_route(
2828 &self,
2829 hash: &Hash,
2830 route: &ReadRoute,
2831 context: &MeshReadContext,
2832 ) -> RouteFetchOutcome {
2833 match route {
2834 ReadRoute::Peers(peer_ids) => {
2835 self.request_from_ordered_peers(hash, peer_ids, context.request_htl)
2836 .await
2837 }
2838 ReadRoute::Sources => self.request_from_read_sources(hash).await,
2839 }
2840 }
2841
2842 async fn request_from_mesh_with_context(
2843 &self,
2844 hash: &Hash,
2845 context: &MeshReadContext,
2846 ) -> Option<Vec<u8>> {
2847 let routes = self.ranked_read_routes(context).await;
2848 match routes.as_slice() {
2849 [] => None,
2850 [ranked] => match self.run_read_route(hash, &ranked.route, context).await {
2851 RouteFetchOutcome::Hit(data) => Some(data),
2852 RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => None,
2853 },
2854 [first, second, ..] => {
2855 if self.should_probe_multiple_routes(&routes) {
2856 let first_fut = self.run_read_route(hash, &first.route, context);
2857 let second_fut = self.run_read_route(hash, &second.route, context);
2858 tokio::pin!(first_fut);
2859 tokio::pin!(second_fut);
2860 let mut first_done = false;
2861 let mut second_done = false;
2862 loop {
2863 tokio::select! {
2864 result = &mut first_fut, if !first_done => {
2865 first_done = true;
2866 if let RouteFetchOutcome::Hit(data) = result {
2867 if !second_done {
2868 self.cancel_losing_route(hash, &second.route, &data).await;
2869 }
2870 return Some(data);
2871 }
2872 }
2873 result = &mut second_fut, if !second_done => {
2874 second_done = true;
2875 if let RouteFetchOutcome::Hit(data) = result {
2876 if !first_done {
2877 self.cancel_losing_route(hash, &first.route, &data).await;
2878 }
2879 return Some(data);
2880 }
2881 }
2882 else => break,
2883 }
2884 if first_done && second_done {
2885 break;
2886 }
2887 }
2888 None
2889 } else {
2890 match self.run_read_route(hash, &first.route, context).await {
2891 RouteFetchOutcome::Hit(data) => return Some(data),
2892 RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => {}
2893 }
2894 for ranked in routes.iter().skip(1) {
2895 match self.run_read_route(hash, &ranked.route, context).await {
2896 RouteFetchOutcome::Hit(data) => return Some(data),
2897 RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => {}
2898 }
2899 }
2900 None
2901 }
2902 }
2903 }
2904 }
2905
2906 async fn request_from_mesh(&self, hash: &Hash) -> Option<Vec<u8>> {
2907 self.request_from_mesh_with_context(hash, &MeshReadContext::default())
2908 .await
2909 }
2910
2911 async fn begin_forward_request(&self, hash_key: &str, requester_id: &str) -> bool {
2912 let mut pending = self.pending_forward_requests.write().await;
2913 if let Some(existing) = pending.get_mut(hash_key) {
2914 existing.requester_ids.insert(requester_id.to_string());
2915 return false;
2916 }
2917
2918 let mut requester_ids = HashSet::new();
2919 requester_ids.insert(requester_id.to_string());
2920 pending.insert(
2921 hash_key.to_string(),
2922 PendingForwardRequest { requester_ids },
2923 );
2924 true
2925 }
2926
2927 async fn was_recent_forward_miss(&self, hash_key: &str) -> bool {
2928 self.recent_forward_misses.lock().await.contains(hash_key)
2929 }
2930
2931 async fn mark_recent_forward_miss(&self, hash_key: &str) {
2932 let _ = self
2933 .recent_forward_misses
2934 .lock()
2935 .await
2936 .insert_if_new(hash_key.to_string());
2937 }
2938
2939 async fn take_forward_requesters(&self, hash_key: &str) -> Vec<String> {
2940 self.pending_forward_requests
2941 .write()
2942 .await
2943 .remove(hash_key)
2944 .map(|pending| pending.requester_ids.into_iter().collect())
2945 .unwrap_or_default()
2946 }
2947
2948 async fn complete_pending_response(
2949 self: &Arc<Self>,
2950 from_peer: &str,
2951 hash: &Hash,
2952 hash_key: String,
2953 payload: Vec<u8>,
2954 ) {
2955 if let Some(pending) = self.pending_requests.write().await.remove(&hash_key) {
2956 self.release_queried_peer_requests(&pending.queried_peers)
2957 .await;
2958 let rtt_ms = pending.started_at.elapsed().as_millis() as u64;
2959 self.peer_selector.write().await.record_success(
2960 from_peer,
2961 rtt_ms,
2962 payload.len() as u64,
2963 );
2964 let forward_requesters = self.take_forward_requesters(&hash_key).await;
2965 let response_bytes = if forward_requesters.is_empty() {
2966 None
2967 } else {
2968 Some(encode_response(&create_response(hash, payload.clone())))
2969 };
2970 let _ = pending.response_tx.send(Some(payload));
2971 if let Some(response_bytes) = response_bytes {
2972 for requester_id in forward_requesters {
2973 Arc::clone(self)
2974 .enqueue_response_send(requester_id, response_bytes.clone(), Instant::now())
2975 .await;
2976 }
2977 }
2978 }
2979 }
2980
2981 async fn handle_quote_response_message(&self, from_peer: &str, res: DataQuoteResponse) {
2982 if !res.a {
2983 return;
2984 }
2985
2986 let Some(quote_id) = res.q else {
2987 return;
2988 };
2989
2990 let hash_key = hash_to_key(&res.h);
2991 let (preferred_mint_url, offered_payment_sat) = {
2992 let pending_quotes = self.pending_quotes.read().await;
2993 let Some(pending) = pending_quotes.get(&hash_key) else {
2994 return;
2995 };
2996 (
2997 pending.preferred_mint_url.clone(),
2998 pending.offered_payment_sat,
2999 )
3000 };
3001 if !self
3002 .should_accept_quote_response(
3003 from_peer,
3004 preferred_mint_url.as_deref(),
3005 offered_payment_sat,
3006 &res,
3007 )
3008 .await
3009 {
3010 return;
3011 }
3012 let mut pending_quotes = self.pending_quotes.write().await;
3013 if let Some(pending) = pending_quotes.remove(&hash_key) {
3014 let _ = pending.response_tx.send(Some(NegotiatedQuote {
3015 peer_id: from_peer.to_string(),
3016 quote_id,
3017 mint_url: res.m,
3018 }));
3019 }
3020 }
3021
3022 async fn handle_response_message(
3023 self: &Arc<Self>,
3024 from_peer: &str,
3025 res: crate::protocol::DataResponse,
3026 ) {
3027 let hash_key = hash_to_key(&res.h);
3028 let hash = match crate::protocol::bytes_to_hash(&res.h) {
3029 Some(h) => h,
3030 None => return,
3031 };
3032
3033 if hashtree_core::sha256(&res.d) != hash {
3035 self.peer_selector.write().await.record_failure(from_peer);
3036 if self.debug {
3037 println!("[MeshStoreCore] Ignoring invalid response payload for {hash_key}");
3038 }
3039 return;
3040 }
3041
3042 self.record_useful_bytes_received_from_peer(from_peer, res.d.len() as u64)
3043 .await;
3044 self.complete_pending_response(from_peer, &hash, hash_key, res.d)
3045 .await;
3046 }
3047
3048 async fn handle_quote_request_message(&self, from_peer: &str, req: DataQuoteRequest) {
3049 let hash = match crate::protocol::bytes_to_hash(&req.h) {
3050 Some(h) => h,
3051 None => return,
3052 };
3053 let hash_key = hash_to_key(&hash);
3054
3055 {
3056 let selector = self.peer_selector.read().await;
3057 if self.should_refuse_requests_from_peer(&selector, from_peer) {
3058 if self.debug {
3059 println!(
3060 "[MeshStoreCore] Refusing quote request from delinquent peer {}",
3061 from_peer
3062 );
3063 }
3064 return;
3065 }
3066 }
3067
3068 let chosen_mint = self.choose_quote_mint(req.m.as_deref());
3069 let can_serve = self.local_store.has(&hash).await.ok().unwrap_or(false)
3070 && !self.should_drop_response(&hash)
3071 && !self.should_corrupt_response(&hash);
3072
3073 let res = if can_serve {
3074 let quote_id = self
3075 .issue_quote(from_peer, &hash_key, req.p, req.t, chosen_mint.as_deref())
3076 .await;
3077 create_quote_response_available(&hash, quote_id, req.p, req.t, chosen_mint.as_deref())
3078 } else {
3079 create_quote_response_unavailable(&hash)
3080 };
3081 let response_bytes = encode_quote_response(&res);
3082 if let Some(channel) = self.signaling.get_channel(from_peer).await {
3083 if channel.send(response_bytes.clone()).await.is_ok() {
3084 self.record_peer_wire_sent(from_peer, response_bytes.len() as u64)
3085 .await;
3086 }
3087 }
3088 }
3089
3090 async fn handle_request_message(
3091 self: &Arc<Self>,
3092 from_peer: &str,
3093 req: crate::protocol::DataRequest,
3094 ) {
3095 let hash = match crate::protocol::bytes_to_hash(&req.h) {
3096 Some(h) => h,
3097 None => return,
3098 };
3099 let hash_key = hash_to_key(&hash);
3100
3101 if let Some(quote_id) = req.q {
3102 if !self.take_valid_quote(from_peer, &hash_key, quote_id).await {
3103 if self.debug {
3104 println!(
3105 "[MeshStoreCore] Refusing request with invalid or expired quote {} from {}",
3106 quote_id, from_peer
3107 );
3108 }
3109 return;
3110 }
3111 }
3112
3113 let allow_peer_forwarding = {
3114 let selector = self.peer_selector.read().await;
3115 !self.should_refuse_requests_from_peer(&selector, from_peer)
3116 };
3117
3118 if let Ok(Some(mut data)) = self.local_store.get(&hash).await {
3120 if self.should_drop_response(&hash) {
3121 if self.debug {
3122 println!(
3123 "[MeshStoreCore] Dropping response for {} due to actor profile",
3124 hash_to_key(&hash)
3125 );
3126 }
3127 return;
3128 }
3129
3130 let response_delay = self.response_send_delay(&hash, data.len());
3131 if self.should_corrupt_response(&hash) {
3132 if data.is_empty() {
3133 data.push(0x80);
3134 } else {
3135 data[0] ^= 0x80;
3136 }
3137 }
3138
3139 let res = create_response(&hash, data);
3141 let response_bytes = encode_response(&res);
3142 let ready_at = Instant::now() + response_delay;
3143 Arc::clone(self)
3144 .enqueue_response_send(from_peer.to_string(), response_bytes, ready_at)
3145 .await;
3146 return;
3147 }
3148
3149 if self.pending_requests.read().await.contains_key(&hash_key) {
3150 let _ = self.begin_forward_request(&hash_key, from_peer).await;
3151 return;
3152 }
3153
3154 if self.was_recent_forward_miss(&hash_key).await {
3155 if self.debug {
3156 println!(
3157 "[MeshStoreCore] Suppressing recently missed forwarded request for {}",
3158 hash_key
3159 );
3160 }
3161 return;
3162 }
3163
3164 if !self.begin_forward_request(&hash_key, from_peer).await {
3165 return;
3166 }
3167
3168 let from_peer = from_peer.to_string();
3169 let this = Arc::clone(self);
3170 let request_htl = req.htl;
3171 tokio::spawn(async move {
3172 let result = if allow_peer_forwarding {
3173 let context = MeshReadContext {
3174 exclude_peer_id: Some(from_peer.clone()),
3175 request_htl,
3176 };
3177 this.request_from_mesh_with_context(&hash, &context).await
3178 } else {
3179 if this.debug {
3180 println!(
3181 "[MeshStoreCore] Serving request from delinquent peer {} via read sources only",
3182 from_peer
3183 );
3184 }
3185 match this.request_from_read_sources(&hash).await {
3186 RouteFetchOutcome::Hit(data) => Some(data),
3187 RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => None,
3188 }
3189 };
3190 let requester_ids = this.take_forward_requesters(&hash_key).await;
3191 if let Some(data) = result {
3192 let ready_at = Instant::now() + this.response_send_delay(&hash, data.len());
3193 let res = create_response(&hash, data);
3194 let response_bytes = encode_response(&res);
3195 for requester_id in requester_ids {
3196 Arc::clone(&this)
3197 .enqueue_response_send(requester_id, response_bytes.clone(), ready_at)
3198 .await;
3199 }
3200 } else {
3201 this.mark_recent_forward_miss(&hash_key).await;
3202 }
3203 });
3204 }
3205
3206 async fn handle_pubsub_interest_message(
3207 self: &Arc<Self>,
3208 from_peer: &str,
3209 mut interest: PubsubInterest,
3210 ) {
3211 if !self.apply_pubsub_interest_route(from_peer, &interest).await {
3212 return;
3213 }
3214
3215 if !self.routing.pubsub_forwarding || interest.htl <= 1 {
3216 return;
3217 }
3218 interest.htl = interest.htl.saturating_sub(1);
3219 let _ = self
3220 .send_pubsub_interest_to_peers(&interest, Some(from_peer))
3221 .await;
3222 }
3223
3224 async fn handle_pubsub_frame_message(
3225 self: &Arc<Self>,
3226 from_peer: &str,
3227 mut frame: PubsubFrame,
3228 wire_bytes: usize,
3229 ) {
3230 if frame.stream_id.is_empty() || frame.origin_peer_id.is_empty() {
3231 return;
3232 }
3233 if frame.origin_peer_id == self.signaling.peer_id() {
3234 return;
3235 }
3236
3237 let frame_key = Self::pubsub_frame_key(&frame);
3238 if !self
3239 .pubsub_seen_frames
3240 .lock()
3241 .await
3242 .insert_if_new(frame_key.clone())
3243 {
3244 return;
3245 }
3246 self.cache_pubsub_frame(frame_key.clone(), frame.clone())
3247 .await;
3248
3249 let local_interested = self
3250 .pubsub_local_interests
3251 .read()
3252 .await
3253 .contains(&frame.stream_id);
3254 let mut downstream_peers = if self.routing.pubsub_forwarding && frame.htl > 1 {
3255 match self.routing.pubsub_delivery_mode {
3256 PubsubDeliveryMode::InterestPush => {
3257 let mut peers = self
3258 .interested_pubsub_peers(&frame.stream_id, Some(from_peer))
3259 .await;
3260 peers.extend(
3261 self.take_pubsub_want_peers(&frame_key, Some(from_peer))
3262 .await,
3263 );
3264 peers.sort();
3265 peers.dedup();
3266 peers
3267 }
3268 PubsubDeliveryMode::HtlInvWant => {
3269 self.take_pubsub_want_peers(&frame_key, Some(from_peer))
3270 .await
3271 }
3272 }
3273 } else {
3274 Vec::new()
3275 };
3276 downstream_peers.retain(|peer_id| peer_id != from_peer);
3277
3278 if local_interested || !downstream_peers.is_empty() {
3279 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3280 .await;
3281 }
3282
3283 if local_interested {
3284 self.enqueue_pubsub_event(PubsubEvent {
3285 stream_id: frame.stream_id.clone(),
3286 seq: frame.seq,
3287 origin_peer_id: frame.origin_peer_id.clone(),
3288 from_peer_id: from_peer.to_string(),
3289 payload: frame.payload.clone(),
3290 })
3291 .await;
3292 }
3293
3294 if downstream_peers.is_empty() {
3295 return;
3296 }
3297
3298 frame.htl = frame.htl.saturating_sub(1);
3299 let _ = self
3300 .send_pubsub_frame_to_peers(&frame, &downstream_peers)
3301 .await;
3302 }
3303
3304 async fn handle_pubsub_inventory_message(
3305 self: &Arc<Self>,
3306 from_peer: &str,
3307 inv: PubsubInventory,
3308 wire_bytes: usize,
3309 ) {
3310 if inv.stream_id.is_empty() || inv.origin_peer_id.is_empty() {
3311 return;
3312 }
3313 if inv.origin_peer_id == self.signaling.peer_id() {
3314 return;
3315 }
3316
3317 let key = Self::pubsub_key(&inv.origin_peer_id, &inv.stream_id, inv.seq);
3318 if !self
3319 .pubsub_seen_inventories
3320 .lock()
3321 .await
3322 .insert_if_new(key.clone())
3323 {
3324 return;
3325 }
3326 {
3327 let mut routes = self.pubsub_inventory_routes.write().await;
3328 routes
3329 .entry(key.clone())
3330 .or_insert_with(|| from_peer.to_string());
3331 }
3332
3333 let local_interested = self
3334 .pubsub_local_interests
3335 .read()
3336 .await
3337 .contains(&inv.stream_id);
3338 let downstream_peers = if self.routing.pubsub_forwarding {
3339 self.interested_pubsub_peers(&inv.stream_id, Some(from_peer))
3340 .await
3341 } else {
3342 Vec::new()
3343 };
3344 if local_interested || !downstream_peers.is_empty() {
3345 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3346 .await;
3347 let want =
3348 create_pubsub_want(inv.stream_id.clone(), inv.seq, inv.origin_peer_id.clone());
3349 let _ = self.send_pubsub_want_upstream(&key, &want, None).await;
3350 }
3351
3352 if !self.routing.pubsub_forwarding
3353 || downstream_peers.is_empty()
3354 || !should_forward_htl(inv.htl)
3355 {
3356 return;
3357 }
3358 let _ = self
3359 .send_pubsub_inventory_to_peers(&inv, &downstream_peers)
3360 .await;
3361 }
3362
3363 async fn handle_pubsub_want_message(
3364 self: &Arc<Self>,
3365 from_peer: &str,
3366 want: PubsubWant,
3367 wire_bytes: usize,
3368 ) {
3369 if want.stream_id.is_empty() || want.origin_peer_id.is_empty() {
3370 return;
3371 }
3372 if want.origin_peer_id == from_peer {
3373 return;
3374 }
3375
3376 let key = Self::pubsub_key(&want.origin_peer_id, &want.stream_id, want.seq);
3377 let want_key = format!("{from_peer}:{key}");
3378 if !self.pubsub_seen_wants.lock().await.insert_if_new(want_key) {
3379 return;
3380 }
3381
3382 if let Some(frame) = self.cached_pubsub_frame(&key).await {
3383 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3384 .await;
3385 let peers = vec![from_peer.to_string()];
3386 let _ = self.send_pubsub_frame_to_peers(&frame, &peers).await;
3387 return;
3388 }
3389
3390 let has_upstream_route = self.pubsub_inventory_routes.read().await.contains_key(&key);
3391 if !has_upstream_route {
3392 return;
3393 }
3394
3395 if self.remember_pubsub_want_peer(key.clone(), from_peer).await {
3396 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3397 .await;
3398 }
3399 let _ = self
3400 .send_pubsub_want_upstream(&key, &want, Some(from_peer))
3401 .await;
3402 }
3403
3404 pub async fn handle_data_message(self: &Arc<Self>, from_peer: &str, data: &[u8]) {
3406 self.record_peer_wire_received(from_peer, data.len() as u64)
3407 .await;
3408 let parsed = match parse_message(data) {
3409 Some(m) => m,
3410 None => return,
3411 };
3412
3413 match parsed {
3414 DataMessage::Request(req) => {
3415 self.handle_request_message(from_peer, req).await;
3416 }
3417 DataMessage::Response(res) => {
3418 self.handle_response_message(from_peer, res).await;
3419 }
3420 DataMessage::QuoteRequest(req) => {
3421 self.handle_quote_request_message(from_peer, req).await;
3422 }
3423 DataMessage::QuoteResponse(res) => {
3424 self.handle_quote_response_message(from_peer, res).await;
3425 }
3426 DataMessage::PubsubInterest(interest) => {
3427 self.handle_pubsub_interest_message(from_peer, interest)
3428 .await;
3429 }
3430 DataMessage::PubsubFrame(frame) => {
3431 self.handle_pubsub_frame_message(from_peer, frame, data.len())
3432 .await;
3433 }
3434 DataMessage::PubsubInventory(inv) => {
3435 self.handle_pubsub_inventory_message(from_peer, inv, data.len())
3436 .await;
3437 }
3438 DataMessage::PubsubWant(want) => {
3439 self.handle_pubsub_want_message(from_peer, want, data.len())
3440 .await;
3441 }
3442 DataMessage::Payment(_)
3443 | DataMessage::PaymentAck(_)
3444 | DataMessage::Chunk(_)
3445 | DataMessage::PeerHints(_) => {}
3446 }
3447 }
3448}
3449
3450#[async_trait]
3451impl<S, R, F> Store for MeshStoreCore<S, R, F>
3452where
3453 S: Store + Send + Sync + 'static,
3454 R: SignalingTransport + Send + Sync + 'static,
3455 F: PeerLinkFactory + Send + Sync + 'static,
3456{
3457 async fn put(&self, hash: Hash, data: Vec<u8>) -> Result<bool, StoreError> {
3458 self.local_store.put(hash, data).await
3459 }
3460
3461 async fn get(&self, hash: &Hash) -> Result<Option<Vec<u8>>, StoreError> {
3462 if let Some(data) = self.local_store.get(hash).await? {
3464 return Ok(Some(data));
3465 }
3466
3467 Ok(self.request_from_mesh(hash).await)
3469 }
3470
3471 async fn has(&self, hash: &Hash) -> Result<bool, StoreError> {
3472 self.local_store.has(hash).await
3473 }
3474
3475 async fn delete(&self, hash: &Hash) -> Result<bool, StoreError> {
3476 self.local_store.delete(hash).await
3477 }
3478}
3479
3480#[cfg(test)]
3481mod tests;
3482
3483pub type SimMeshStore<S> =
3485 MeshStoreCore<S, crate::mock::MockRelayTransport, crate::mock::MockConnectionFactory>;