1use async_trait::async_trait;
8use futures::{stream::FuturesUnordered, FutureExt, StreamExt};
9use std::collections::hash_map::DefaultHasher;
10use std::collections::{HashMap, HashSet, VecDeque};
11use std::future::Future;
12use std::hash::{Hash as _, Hasher};
13use std::ops::Range;
14use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
15use std::sync::Arc;
16use std::time::Duration;
17use tokio::sync::{oneshot, Mutex, Notify, RwLock};
18use tokio::time::Instant;
19
20use hashtree_core::{Hash, Store, StoreError};
21
22use crate::peer_selector::{PeerMetadataSnapshot, PeerSelector, SelectionStrategy};
23use crate::protocol::{
24 create_pubsub_frame, create_pubsub_interest, create_pubsub_inventory, create_pubsub_want,
25 create_quote_request, create_quote_response_available, create_quote_response_unavailable,
26 create_request, create_request_with_quote, create_response, encode_pubsub_frame,
27 encode_pubsub_interest, encode_pubsub_inventory, encode_pubsub_want, encode_quote_request,
28 encode_quote_response, encode_request, encode_response, hash_to_key, parse_message,
29 DataMessage, DataQuoteRequest, DataQuoteResponse, PubsubFrame, PubsubInterest, PubsubInventory,
30 PubsubWant,
31};
32use crate::pubsub_strategy::{
33 reciprocal_virtual_finish, select_reciprocal_outbound_job, OutboundJobCandidate,
34 PeerTrafficSnapshot, PubsubCandidate, PubsubSchedulerConfig,
35};
36use crate::signaling::MeshRouter;
37use crate::transport::{PeerLinkFactory, SignalingTransport, TransportError};
38use crate::types::{
39 should_forward_htl, PeerHTLConfig, SignalingMessage, TimedSeenSet, MAX_HTL, MESH_EVENT_POLICY,
40};
41
42const PEER_METADATA_POINTER_SLOT_KEY: &[u8] = b"hashtree-mesh/peer-metadata/latest/v1";
45const RECENT_FORWARD_MISS_CAPACITY: usize = 4096;
46const MIN_RECENT_FORWARD_MISS_TTL_MS: u64 = 250;
47const PUBSUB_SEEN_CAPACITY: usize = 16_384;
48const PUBSUB_INBOX_CAPACITY: usize = 4_096;
49const PUBSUB_FRAME_CACHE_CAPACITY: usize = 4_096;
50const PUBSUB_SEEN_TTL: Duration = Duration::from_secs(120);
51
52struct PendingRequest {
54 response_tx: oneshot::Sender<Option<Vec<u8>>>,
55 started_at: Instant,
56 queried_peers: Vec<String>,
57}
58
59struct PendingQuoteRequest {
60 response_tx: oneshot::Sender<Option<NegotiatedQuote>>,
61 preferred_mint_url: Option<String>,
62 offered_payment_sat: u64,
63}
64
65struct PendingForwardRequest {
66 requester_ids: HashSet<String>,
67}
68
69type PeerWireStats = PeerTrafficSnapshot;
70
71struct PendingResponseSend {
72 job_id: u64,
73 peer_id: String,
74 bytes: Vec<u8>,
75 ready_at: Instant,
76 queue_sequence: u64,
77}
78
79#[async_trait]
80pub trait MeshReadSource: Send + Sync {
81 fn id(&self) -> &str;
82
83 fn is_available(&self) -> bool {
84 true
85 }
86
87 async fn get(&self, hash: &Hash) -> Option<Vec<u8>>;
88}
89
90#[derive(Debug, Clone)]
91struct NegotiatedQuote {
92 peer_id: String,
93 quote_id: u64,
94 #[allow(dead_code)]
95 mint_url: Option<String>,
96}
97
98struct IssuedQuote {
99 expires_at: Instant,
100 #[allow(dead_code)]
101 payment_sat: u64,
102 #[allow(dead_code)]
103 mint_url: Option<String>,
104}
105
106#[derive(Debug, Clone, Default)]
107struct AdaptiveSourceStats {
108 requests: u64,
109 successes: u64,
110 misses: u64,
111 failures: u64,
112 timeouts: u64,
113 srtt_ms: f64,
114 rttvar_ms: f64,
115 backoff_level: u32,
116 backed_off_until: Option<Instant>,
117 last_success_at: Option<Instant>,
118 last_failure_at: Option<Instant>,
119}
120
121#[derive(Debug, Clone)]
122enum RouteFetchOutcome {
123 Hit(Vec<u8>),
124 Miss,
125 Timeout,
126}
127
128struct InflightSourceFetch {
129 waiters: Vec<oneshot::Sender<RouteFetchOutcome>>,
130}
131
132enum SourceFetchOutcome {
133 Hit {
134 source_id: String,
135 data: Vec<u8>,
136 elapsed_ms: u64,
137 },
138 Miss {
139 source_id: String,
140 },
141 Failure {
142 source_id: String,
143 },
144}
145
146const INITIAL_SOURCE_BACKOFF_MS: u64 = 250;
147const MAX_SOURCE_BACKOFF_MS: u64 = 10_000;
148const SOURCE_SCORE_TIE_DELTA: f64 = 0.15;
149const RECENT_SOURCE_SUCCESS_WINDOW: Duration = Duration::from_secs(60);
150const ACTIVE_PEER_REQUEST_RANK_PENALTY: usize = 3;
151
152fn source_reliability_score(stats: &AdaptiveSourceStats) -> f64 {
153 (stats.successes as f64 + 1.0) / (stats.requests as f64 + 2.0)
154}
155
156fn source_latency_score(stats: &AdaptiveSourceStats) -> f64 {
157 if stats.srtt_ms <= 0.0 {
158 return 0.5;
159 }
160 (500.0 / (stats.srtt_ms + 50.0)).min(1.0)
161}
162
163fn source_has_history(stats: &AdaptiveSourceStats) -> bool {
164 stats.requests > 0
165 || stats.successes > 0
166 || stats.misses > 0
167 || stats.failures > 0
168 || stats.timeouts > 0
169}
170
171fn adaptive_source_score(stats: &AdaptiveSourceStats, now: Instant) -> f64 {
172 if let Some(backed_off_until) = stats.backed_off_until {
173 if backed_off_until > now {
174 return f64::NEG_INFINITY;
175 }
176 }
177
178 let miss_penalty = if stats.requests > 0 {
179 (stats.misses as f64 / stats.requests as f64) * 0.15
180 } else {
181 0.0
182 };
183 let failure_penalty = if stats.requests > 0 {
184 ((stats.failures + stats.timeouts) as f64 / stats.requests as f64) * 0.3
185 } else {
186 0.0
187 };
188 let recency_bonus = if stats
189 .last_success_at
190 .is_some_and(|last| now.duration_since(last) < RECENT_SOURCE_SUCCESS_WINDOW)
191 {
192 0.1
193 } else {
194 0.0
195 };
196
197 0.6 * source_reliability_score(stats) + 0.3 * source_latency_score(stats) + recency_bonus
198 - miss_penalty
199 - failure_penalty
200}
201
202fn peer_endpoint_has_history(stats: &crate::peer_selector::PeerStats) -> bool {
203 stats.requests_sent > 0 || stats.successes > 0 || stats.failures > 0 || stats.timeouts > 0
204}
205
206fn peer_endpoint_score(stats: &crate::peer_selector::PeerStats, now: Instant) -> f64 {
207 if stats.backed_off_until.is_some_and(|until| until > now) {
208 return f64::NEG_INFINITY;
209 }
210
211 let miss_penalty = 0.0;
212 let failure_penalty = if stats.requests_sent > 0 {
213 ((stats.failures + stats.timeouts) as f64 / stats.requests_sent as f64) * 0.3
214 } else {
215 0.0
216 };
217 let recency_bonus = if stats
218 .last_success
219 .is_some_and(|last| now.duration_since(last) < RECENT_SOURCE_SUCCESS_WINDOW)
220 {
221 0.1
222 } else {
223 0.0
224 };
225
226 0.6 * stats.success_rate()
227 + 0.3
228 * source_latency_score(&AdaptiveSourceStats {
229 srtt_ms: stats.srtt_ms,
230 ..AdaptiveSourceStats::default()
231 })
232 + recency_bonus
233 - miss_penalty
234 - failure_penalty
235}
236
237#[derive(Clone)]
238enum ReadRoute {
239 Peers(Vec<String>),
240 Sources,
241}
242
243impl ReadRoute {
244 fn id(&self) -> &'static str {
245 match self {
246 Self::Peers(_) => "peers",
247 Self::Sources => "sources",
248 }
249 }
250}
251
252struct RankedReadRoute {
253 route: ReadRoute,
254 best_endpoint_id: String,
255 score: f64,
256 has_history: bool,
257}
258
259fn ranked_route_kind(route: &ReadRoute) -> u8 {
260 match route {
261 ReadRoute::Sources => 0,
262 ReadRoute::Peers(_) => 1,
263 }
264}
265
266#[derive(Debug, Clone)]
267struct MeshReadContext {
268 exclude_peer_id: Option<String>,
269 request_htl: u8,
270}
271
272impl Default for MeshReadContext {
273 fn default() -> Self {
274 Self {
275 exclude_peer_id: None,
276 request_htl: MAX_HTL,
277 }
278 }
279}
280
281#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
283pub struct DataPumpStats {
284 pub processed: usize,
285 pub request_messages: usize,
286 pub response_messages: usize,
287 pub quote_request_messages: u64,
288 pub quote_response_messages: u64,
289 pub pubsub_interest_messages: u64,
290 pub pubsub_frame_messages: u64,
291 pub pubsub_inventory_messages: u64,
292 pub pubsub_want_messages: u64,
293 pub processed_bytes: u64,
294}
295
296#[derive(Debug, Clone, PartialEq, Eq)]
298pub struct PubsubEvent {
299 pub stream_id: String,
300 pub seq: u64,
301 pub origin_peer_id: String,
302 pub from_peer_id: String,
303 pub payload: Vec<u8>,
304}
305
306#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
308pub struct PubsubPublishStats {
309 pub selected_peers: usize,
310 pub sent_peers: usize,
311 pub sent_bytes: u64,
312 pub deferred_peers: usize,
313}
314
315#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
317pub enum PubsubDeliveryMode {
318 InterestPush,
320 #[default]
322 HtlInvWant,
323}
324
325#[derive(Debug, Clone, Copy)]
331pub struct RequestDispatchConfig {
332 pub initial_fanout: usize,
334 pub hedge_fanout: usize,
336 pub max_fanout: usize,
338 pub hedge_interval_ms: u64,
340}
341
342impl Default for RequestDispatchConfig {
343 fn default() -> Self {
344 Self {
345 initial_fanout: usize::MAX,
346 hedge_fanout: usize::MAX,
347 max_fanout: usize::MAX,
348 hedge_interval_ms: 0,
349 }
350 }
351}
352
353pub fn normalize_dispatch_config(
355 dispatch: RequestDispatchConfig,
356 available_peers: usize,
357) -> RequestDispatchConfig {
358 let mut cfg = dispatch;
359 let cap = if cfg.max_fanout == 0 {
360 available_peers
361 } else {
362 cfg.max_fanout.min(available_peers)
363 };
364 cfg.max_fanout = cap;
365 cfg.initial_fanout = if cfg.initial_fanout == 0 {
366 1
367 } else {
368 cfg.initial_fanout.min(cap.max(1))
369 };
370 cfg.hedge_fanout = if cfg.hedge_fanout == 0 {
371 1
372 } else {
373 cfg.hedge_fanout.min(cap.max(1))
374 };
375 cfg
376}
377
378pub fn build_hedged_wave_plan(peer_count: usize, dispatch: RequestDispatchConfig) -> Vec<usize> {
380 if peer_count == 0 {
381 return Vec::new();
382 }
383 let cap = dispatch.max_fanout.min(peer_count);
384 if cap == 0 {
385 return Vec::new();
386 }
387
388 let mut plan = Vec::new();
389 let mut sent = 0usize;
390 let first = dispatch.initial_fanout.min(cap).max(1);
391 plan.push(first);
392 sent += first;
393
394 while sent < cap {
395 let next = dispatch.hedge_fanout.min(cap - sent).max(1);
396 plan.push(next);
397 sent += next;
398 }
399 plan
400}
401
402#[derive(Debug)]
404pub enum HedgedWaveAction<T> {
405 Continue,
406 Success(T),
407 Abort,
408}
409
410pub async fn run_hedged_waves<T, SendWave, SendWaveFut, WaitWave, WaitWaveFut>(
415 peer_count: usize,
416 dispatch: RequestDispatchConfig,
417 request_timeout: Duration,
418 mut send_wave: SendWave,
419 mut wait_wave: WaitWave,
420) -> Option<T>
421where
422 SendWave: FnMut(Range<usize>) -> SendWaveFut,
423 SendWaveFut: Future<Output = usize>,
424 WaitWave: FnMut(Duration) -> WaitWaveFut,
425 WaitWaveFut: Future<Output = HedgedWaveAction<T>>,
426{
427 let dispatch = normalize_dispatch_config(dispatch, peer_count);
428 let wave_plan = build_hedged_wave_plan(peer_count, dispatch);
429 if wave_plan.is_empty() {
430 return None;
431 }
432
433 let deadline = Instant::now() + request_timeout;
434 let mut sent_total = 0usize;
435 let mut next_peer_idx = 0usize;
436
437 for (wave_idx, wave_size) in wave_plan.iter().copied().enumerate() {
438 let from = next_peer_idx;
439 let to = (next_peer_idx + wave_size).min(peer_count);
440 next_peer_idx = to;
441
442 if from == to {
443 continue;
444 }
445
446 sent_total += send_wave(from..to).await;
447 if sent_total == 0 {
448 if next_peer_idx >= peer_count {
449 break;
450 }
451 continue;
452 }
453
454 let now = Instant::now();
455 if now >= deadline {
456 break;
457 }
458 let remaining = deadline.saturating_duration_since(now);
459 let is_last_wave = wave_idx + 1 == wave_plan.len() || next_peer_idx >= peer_count;
460 let wait = if is_last_wave {
461 remaining
462 } else if dispatch.hedge_interval_ms == 0 {
463 Duration::ZERO
464 } else {
465 Duration::from_millis(dispatch.hedge_interval_ms).min(remaining)
466 };
467
468 if wait.is_zero() {
469 continue;
470 }
471
472 match wait_wave(wait).await {
473 HedgedWaveAction::Continue => {}
474 HedgedWaveAction::Success(value) => return Some(value),
475 HedgedWaveAction::Abort => break,
476 }
477 }
478
479 None
480}
481
482pub async fn sync_selector_peers(selector: &RwLock<PeerSelector>, current_peer_ids: &[String]) {
484 let mut selector = selector.write().await;
485 let current: HashSet<&str> = current_peer_ids.iter().map(String::as_str).collect();
486 let known: Vec<String> = selector.all_stats().map(|s| s.peer_id.clone()).collect();
487 for peer_id in known {
488 if !current.contains(peer_id.as_str()) {
489 selector.remove_peer(&peer_id);
490 }
491 }
492 for peer_id in current_peer_ids {
493 selector.add_peer(peer_id.clone());
494 }
495}
496
497#[derive(Debug, Clone, Copy)]
501pub struct ResponseBehaviorConfig {
502 pub drop_response_prob: f64,
504 pub corrupt_response_prob: f64,
506 pub extra_delay_ms: u64,
508 pub first_byte_delay_ms: u64,
510 pub bytes_per_second: u64,
512 pub stall_response_prob: f64,
514 pub stall_delay_ms: u64,
516}
517
518impl Default for ResponseBehaviorConfig {
519 fn default() -> Self {
520 Self {
521 drop_response_prob: 0.0,
522 corrupt_response_prob: 0.0,
523 extra_delay_ms: 0,
524 first_byte_delay_ms: 0,
525 bytes_per_second: 0,
526 stall_response_prob: 0.0,
527 stall_delay_ms: 0,
528 }
529 }
530}
531
532impl ResponseBehaviorConfig {
533 fn normalized(self) -> Self {
534 Self {
535 drop_response_prob: self.drop_response_prob.clamp(0.0, 1.0),
536 corrupt_response_prob: self.corrupt_response_prob.clamp(0.0, 1.0),
537 extra_delay_ms: self.extra_delay_ms,
538 first_byte_delay_ms: self.first_byte_delay_ms,
539 bytes_per_second: self.bytes_per_second,
540 stall_response_prob: self.stall_response_prob.clamp(0.0, 1.0),
541 stall_delay_ms: self.stall_delay_ms,
542 }
543 }
544}
545
546#[derive(Debug, Clone)]
548pub struct MeshRoutingConfig {
549 pub selection_strategy: SelectionStrategy,
550 pub fairness_enabled: bool,
551 pub cashu_payment_weight: f64,
553 pub cashu_payment_default_block_threshold: u64,
556 pub cashu_accepted_mints: Vec<String>,
558 pub cashu_default_mint: Option<String>,
560 pub cashu_peer_suggested_mint_base_cap_sat: u64,
562 pub cashu_peer_suggested_mint_success_step_sat: u64,
564 pub cashu_peer_suggested_mint_receipt_step_sat: u64,
566 pub cashu_peer_suggested_mint_max_cap_sat: u64,
568 pub dispatch: RequestDispatchConfig,
569 pub response_behavior: ResponseBehaviorConfig,
570 pub pubsub_scheduler: PubsubSchedulerConfig,
571 pub pubsub_delivery_mode: PubsubDeliveryMode,
572}
573
574impl Default for MeshRoutingConfig {
575 fn default() -> Self {
576 Self {
577 selection_strategy: SelectionStrategy::Weighted,
578 fairness_enabled: true,
579 cashu_payment_weight: 0.0,
580 cashu_payment_default_block_threshold: 0,
581 cashu_accepted_mints: Vec::new(),
582 cashu_default_mint: None,
583 cashu_peer_suggested_mint_base_cap_sat: 0,
584 cashu_peer_suggested_mint_success_step_sat: 0,
585 cashu_peer_suggested_mint_receipt_step_sat: 0,
586 cashu_peer_suggested_mint_max_cap_sat: 0,
587 dispatch: RequestDispatchConfig::default(),
588 response_behavior: ResponseBehaviorConfig::default(),
589 pubsub_scheduler: PubsubSchedulerConfig::default(),
590 pubsub_delivery_mode: PubsubDeliveryMode::HtlInvWant,
591 }
592 }
593}
594
595pub struct MeshStoreCore<S, R, F>
602where
603 S: Store + Send + Sync + 'static,
604 R: SignalingTransport + Send + Sync + 'static,
605 F: PeerLinkFactory + Send + Sync + 'static,
606{
607 local_store: Arc<S>,
609 signaling: Arc<MeshRouter<R, F>>,
611 htl_configs: RwLock<HashMap<String, PeerHTLConfig>>,
613 pending_requests: RwLock<HashMap<String, PendingRequest>>,
615 pending_quotes: RwLock<HashMap<String, PendingQuoteRequest>>,
617 pending_forward_requests: RwLock<HashMap<String, PendingForwardRequest>>,
619 recent_forward_misses: Mutex<TimedSeenSet>,
621 issued_quotes: RwLock<HashMap<(String, String, u64), IssuedQuote>>,
623 next_quote_id: RwLock<u64>,
625 read_sources: RwLock<HashMap<String, Arc<dyn MeshReadSource>>>,
627 read_source_stats: RwLock<HashMap<String, AdaptiveSourceStats>>,
629 inflight_source_fetches: Mutex<HashMap<String, InflightSourceFetch>>,
631 peer_selector: RwLock<PeerSelector>,
633 peer_active_requests: RwLock<HashMap<String, usize>>,
635 peer_wire_stats: RwLock<HashMap<String, PeerWireStats>>,
637 pubsub_local_interests: RwLock<HashSet<String>>,
639 pubsub_local_interest_versions: RwLock<HashMap<String, u64>>,
641 pubsub_peer_interests: RwLock<HashMap<String, HashSet<String>>>,
643 pubsub_interest_routes: RwLock<HashMap<(String, String), String>>,
645 pubsub_interest_versions: RwLock<HashMap<(String, String), u64>>,
647 pubsub_seen_interests: Mutex<TimedSeenSet>,
649 pubsub_seen_frames: Mutex<TimedSeenSet>,
651 pubsub_seen_inventories: Mutex<TimedSeenSet>,
653 pubsub_seen_wants: Mutex<TimedSeenSet>,
655 pubsub_inventory_routes: RwLock<HashMap<String, String>>,
657 pubsub_want_routes: RwLock<HashMap<String, HashSet<String>>>,
659 pubsub_upstream_wants: Mutex<TimedSeenSet>,
661 pubsub_frame_cache: Mutex<VecDeque<(String, PubsubFrame)>>,
663 pubsub_inbox: Mutex<VecDeque<PubsubEvent>>,
665 pubsub_notify: Notify,
667 pubsub_deferred_counts: RwLock<HashMap<(String, String), u64>>,
669 next_pubsub_interest_seq: AtomicU64,
671 pending_response_sends: Mutex<Vec<PendingResponseSend>>,
673 response_scheduler_running: AtomicBool,
675 next_response_job_id: AtomicU64,
677 routing: MeshRoutingConfig,
679 request_timeout: Duration,
681 debug: bool,
683 running: RwLock<bool>,
685}
686
687impl<S, R, F> MeshStoreCore<S, R, F>
688where
689 S: Store + Send + Sync + 'static,
690 R: SignalingTransport + Send + Sync + 'static,
691 F: PeerLinkFactory + Send + Sync + 'static,
692{
693 pub fn new(
695 local_store: Arc<S>,
696 signaling: Arc<MeshRouter<R, F>>,
697 request_timeout: Duration,
698 debug: bool,
699 ) -> Self {
700 Self::new_with_routing(
701 local_store,
702 signaling,
703 request_timeout,
704 debug,
705 Default::default(),
706 )
707 }
708
709 pub fn new_with_routing(
711 local_store: Arc<S>,
712 signaling: Arc<MeshRouter<R, F>>,
713 request_timeout: Duration,
714 debug: bool,
715 routing: MeshRoutingConfig,
716 ) -> Self {
717 let mut selector = PeerSelector::with_strategy(routing.selection_strategy);
718 selector.set_fairness(routing.fairness_enabled);
719 selector.set_cashu_payment_weight(routing.cashu_payment_weight);
720 Self {
721 local_store,
722 signaling,
723 htl_configs: RwLock::new(HashMap::new()),
724 pending_requests: RwLock::new(HashMap::new()),
725 pending_quotes: RwLock::new(HashMap::new()),
726 pending_forward_requests: RwLock::new(HashMap::new()),
727 recent_forward_misses: Mutex::new(TimedSeenSet::new(
728 RECENT_FORWARD_MISS_CAPACITY,
729 Self::recent_forward_miss_ttl(request_timeout),
730 )),
731 issued_quotes: RwLock::new(HashMap::new()),
732 next_quote_id: RwLock::new(1),
733 read_sources: RwLock::new(HashMap::new()),
734 read_source_stats: RwLock::new(HashMap::new()),
735 inflight_source_fetches: Mutex::new(HashMap::new()),
736 peer_selector: RwLock::new(selector),
737 peer_active_requests: RwLock::new(HashMap::new()),
738 peer_wire_stats: RwLock::new(HashMap::new()),
739 pubsub_local_interests: RwLock::new(HashSet::new()),
740 pubsub_local_interest_versions: RwLock::new(HashMap::new()),
741 pubsub_peer_interests: RwLock::new(HashMap::new()),
742 pubsub_interest_routes: RwLock::new(HashMap::new()),
743 pubsub_interest_versions: RwLock::new(HashMap::new()),
744 pubsub_seen_interests: Mutex::new(TimedSeenSet::new(
745 PUBSUB_SEEN_CAPACITY,
746 PUBSUB_SEEN_TTL,
747 )),
748 pubsub_seen_frames: Mutex::new(TimedSeenSet::new(
749 PUBSUB_SEEN_CAPACITY,
750 PUBSUB_SEEN_TTL,
751 )),
752 pubsub_seen_inventories: Mutex::new(TimedSeenSet::new(
753 PUBSUB_SEEN_CAPACITY,
754 PUBSUB_SEEN_TTL,
755 )),
756 pubsub_seen_wants: Mutex::new(TimedSeenSet::new(PUBSUB_SEEN_CAPACITY, PUBSUB_SEEN_TTL)),
757 pubsub_inventory_routes: RwLock::new(HashMap::new()),
758 pubsub_want_routes: RwLock::new(HashMap::new()),
759 pubsub_upstream_wants: Mutex::new(TimedSeenSet::new(
760 PUBSUB_SEEN_CAPACITY,
761 PUBSUB_SEEN_TTL,
762 )),
763 pubsub_frame_cache: Mutex::new(VecDeque::new()),
764 pubsub_inbox: Mutex::new(VecDeque::new()),
765 pubsub_notify: Notify::new(),
766 pubsub_deferred_counts: RwLock::new(HashMap::new()),
767 next_pubsub_interest_seq: AtomicU64::new(1),
768 pending_response_sends: Mutex::new(Vec::new()),
769 response_scheduler_running: AtomicBool::new(false),
770 next_response_job_id: AtomicU64::new(1),
771 routing,
772 request_timeout,
773 debug,
774 running: RwLock::new(false),
775 }
776 }
777
778 fn recent_forward_miss_ttl(request_timeout: Duration) -> Duration {
779 let ttl_ms = request_timeout
780 .as_millis()
781 .saturating_mul(2)
782 .max(MIN_RECENT_FORWARD_MISS_TTL_MS as u128)
783 .min(u64::MAX as u128) as u64;
784 Duration::from_millis(ttl_ms)
785 }
786
787 pub async fn start(&self) -> Result<(), TransportError> {
789 *self.running.write().await = true;
790
791 self.signaling.send_hello(vec![]).await?;
793
794 Ok(())
795 }
796
797 pub async fn stop(&self) {
799 *self.running.write().await = false;
800 }
801
802 pub async fn process_signaling(&self, msg: SignalingMessage) -> Result<(), TransportError> {
804 let peer_id = msg.peer_id().to_string();
806 {
807 let mut configs = self.htl_configs.write().await;
808 if !configs.contains_key(&peer_id) {
809 configs.insert(peer_id.clone(), PeerHTLConfig::random());
810 }
811 }
812 self.peer_selector.write().await.add_peer(peer_id.clone());
813
814 let result = self.signaling.handle_message(msg).await;
815 if result.is_ok() {
816 self.announce_pubsub_interests_to_peer(&peer_id).await;
817 }
818 result
819 }
820
821 pub fn signaling(&self) -> &Arc<MeshRouter<R, F>> {
823 &self.signaling
824 }
825
826 fn response_behavior(&self) -> ResponseBehaviorConfig {
827 self.routing.response_behavior.normalized()
828 }
829
830 async fn record_peer_wire_sent(&self, peer_id: &str, bytes: u64) {
831 if bytes == 0 {
832 return;
833 }
834 let mut stats = self.peer_wire_stats.write().await;
835 let entry = stats.entry(peer_id.to_string()).or_default();
836 entry.bytes_sent = entry.bytes_sent.saturating_add(bytes);
837 }
838
839 async fn record_peer_wire_received(&self, peer_id: &str, bytes: u64) {
840 if bytes == 0 {
841 return;
842 }
843 let mut stats = self.peer_wire_stats.write().await;
844 let entry = stats.entry(peer_id.to_string()).or_default();
845 entry.bytes_received = entry.bytes_received.saturating_add(bytes);
846 }
847
848 pub async fn record_useful_bytes_received_from_peer(&self, peer_id: &str, bytes: u64) {
853 if bytes == 0 {
854 return;
855 }
856 let mut stats = self.peer_wire_stats.write().await;
857 let entry = stats.entry(peer_id.to_string()).or_default();
858 entry.useful_bytes_received = entry.useful_bytes_received.saturating_add(bytes);
859 }
860
861 pub async fn peer_traffic_snapshot(&self, peer_id: &str) -> PeerTrafficSnapshot {
863 self.peer_wire_stats
864 .read()
865 .await
866 .get(peer_id)
867 .copied()
868 .unwrap_or_default()
869 }
870
871 pub async fn peer_traffic_snapshots(&self) -> HashMap<String, PeerTrafficSnapshot> {
873 self.peer_wire_stats.read().await.clone()
874 }
875
876 fn pubsub_key(origin_peer_id: &str, stream_id: &str, seq: u64) -> String {
877 format!("{origin_peer_id}:{stream_id}:{seq}")
878 }
879
880 fn pubsub_frame_key(frame: &PubsubFrame) -> String {
881 Self::pubsub_key(&frame.origin_peer_id, &frame.stream_id, frame.seq)
882 }
883
884 fn pubsub_interest_key(interest: &PubsubInterest) -> String {
885 format!(
886 "{}:{}:{}:{}",
887 interest.subscriber_peer_id, interest.stream_id, interest.seq, interest.active
888 )
889 }
890
891 fn next_pubsub_interest_seq(&self) -> u64 {
892 self.next_pubsub_interest_seq
893 .fetch_add(1, Ordering::Relaxed)
894 }
895
896 async fn record_peer_pubsub_wire_sent(&self, peer_id: &str, bytes: u64, bandwidth_debt: f64) {
897 if bytes == 0 {
898 return;
899 }
900 let mut stats = self.peer_wire_stats.write().await;
901 let entry = stats.entry(peer_id.to_string()).or_default();
902 entry.bytes_sent = entry.bytes_sent.saturating_add(bytes);
903 entry.bandwidth_debt = bandwidth_debt;
904 }
905
906 async fn send_pubsub_interest_to_peers(
907 &self,
908 interest: &PubsubInterest,
909 exclude_peer_id: Option<&str>,
910 ) -> PubsubPublishStats {
911 if !should_forward_htl(interest.htl) {
912 return PubsubPublishStats::default();
913 }
914
915 let mut peer_ids = self.signaling.peer_ids().await;
916 peer_ids.sort();
917 peer_ids.retain(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude));
918
919 let bytes = encode_pubsub_interest(interest);
920 let mut stats = PubsubPublishStats {
921 selected_peers: peer_ids.len(),
922 ..Default::default()
923 };
924 for peer_id in peer_ids {
925 let Some(channel) = self.signaling.get_channel(&peer_id).await else {
926 continue;
927 };
928 if channel.send(bytes.clone()).await.is_ok() {
929 stats.sent_peers += 1;
930 stats.sent_bytes = stats.sent_bytes.saturating_add(bytes.len() as u64);
931 self.record_peer_wire_sent(&peer_id, bytes.len() as u64)
932 .await;
933 }
934 }
935 stats
936 }
937
938 async fn announce_pubsub_interests_to_peer(&self, peer_id: &str) {
939 let mut interests = self
940 .pubsub_local_interests
941 .read()
942 .await
943 .iter()
944 .cloned()
945 .collect::<Vec<_>>();
946 interests.sort();
947 if interests.is_empty() {
948 return;
949 }
950
951 let interests = {
952 let versions = self.pubsub_local_interest_versions.read().await;
953 interests
954 .into_iter()
955 .filter_map(|stream_id| {
956 versions
957 .get(&stream_id)
958 .copied()
959 .map(|seq| (stream_id, seq))
960 })
961 .collect::<Vec<_>>()
962 };
963
964 for (stream_id, seq) in interests {
965 let interest = create_pubsub_interest(
966 stream_id,
967 self.signaling.peer_id().to_string(),
968 seq,
969 true,
970 MAX_HTL,
971 );
972 let Some(channel) = self.signaling.get_channel(peer_id).await else {
973 continue;
974 };
975 let bytes = encode_pubsub_interest(&interest);
976 if channel.send(bytes.clone()).await.is_ok() {
977 self.record_peer_wire_sent(peer_id, bytes.len() as u64)
978 .await;
979 }
980 }
981 }
982
983 fn remove_pubsub_peer_interest(
984 peer_interests: &mut HashMap<String, HashSet<String>>,
985 routes: &HashMap<(String, String), String>,
986 stream_id: &str,
987 peer_id: &str,
988 ) {
989 let still_has_route = routes
990 .iter()
991 .any(|((stream, _subscriber), peer)| stream == stream_id && peer == peer_id);
992 if still_has_route {
993 return;
994 }
995 if let Some(peers) = peer_interests.get_mut(stream_id) {
996 peers.remove(peer_id);
997 if peers.is_empty() {
998 peer_interests.remove(stream_id);
999 }
1000 }
1001 }
1002
1003 async fn apply_pubsub_interest_route(
1004 &self,
1005 from_peer: &str,
1006 interest: &PubsubInterest,
1007 ) -> bool {
1008 if interest.stream_id.is_empty() || interest.subscriber_peer_id.is_empty() {
1009 return false;
1010 }
1011 if interest.subscriber_peer_id == self.signaling.peer_id() {
1012 return false;
1013 }
1014
1015 let interest_key = Self::pubsub_interest_key(interest);
1016 if !self
1017 .pubsub_seen_interests
1018 .lock()
1019 .await
1020 .insert_if_new(interest_key)
1021 {
1022 return false;
1023 }
1024
1025 let route_key = (
1026 interest.stream_id.clone(),
1027 interest.subscriber_peer_id.clone(),
1028 );
1029 {
1030 let mut versions = self.pubsub_interest_versions.write().await;
1031 if versions
1032 .get(&route_key)
1033 .is_some_and(|latest| *latest >= interest.seq)
1034 {
1035 return false;
1036 }
1037 versions.insert(route_key.clone(), interest.seq);
1038 }
1039
1040 let mut peer_interests = self.pubsub_peer_interests.write().await;
1041 let mut routes = self.pubsub_interest_routes.write().await;
1042 if interest.active {
1043 if let Some(previous_peer) = routes.insert(route_key, from_peer.to_string()) {
1044 if previous_peer != from_peer {
1045 Self::remove_pubsub_peer_interest(
1046 &mut peer_interests,
1047 &routes,
1048 &interest.stream_id,
1049 &previous_peer,
1050 );
1051 }
1052 }
1053 peer_interests
1054 .entry(interest.stream_id.clone())
1055 .or_default()
1056 .insert(from_peer.to_string());
1057 } else if let Some(previous_peer) = routes.remove(&route_key) {
1058 Self::remove_pubsub_peer_interest(
1059 &mut peer_interests,
1060 &routes,
1061 &interest.stream_id,
1062 &previous_peer,
1063 );
1064 } else {
1065 Self::remove_pubsub_peer_interest(
1066 &mut peer_interests,
1067 &routes,
1068 &interest.stream_id,
1069 from_peer,
1070 );
1071 }
1072
1073 true
1074 }
1075
1076 async fn interested_pubsub_peers(
1077 &self,
1078 stream_id: &str,
1079 exclude_peer_id: Option<&str>,
1080 ) -> Vec<String> {
1081 let connected = self
1082 .signaling
1083 .peer_ids()
1084 .await
1085 .into_iter()
1086 .collect::<HashSet<_>>();
1087 let mut peers = self
1088 .pubsub_peer_interests
1089 .read()
1090 .await
1091 .get(stream_id)
1092 .map(|peers| peers.iter().cloned().collect::<Vec<_>>())
1093 .unwrap_or_default();
1094 peers.retain(|peer_id| {
1095 connected.contains(peer_id) && exclude_peer_id.is_none_or(|exclude| peer_id != exclude)
1096 });
1097 peers.sort();
1098 peers
1099 }
1100
1101 async fn decrement_pubsub_htl_for_peer(&self, peer_id: &str, htl: u8) -> u8 {
1102 let htl_config = {
1103 let configs = self.htl_configs.read().await;
1104 configs
1105 .get(peer_id)
1106 .cloned()
1107 .unwrap_or_else(PeerHTLConfig::random)
1108 };
1109 htl_config.decrement_with_policy(htl, &MESH_EVENT_POLICY)
1110 }
1111
1112 async fn send_pubsub_inventory_to_peers(
1113 &self,
1114 inv: &PubsubInventory,
1115 peer_ids: &[String],
1116 ) -> PubsubPublishStats {
1117 if peer_ids.is_empty() || !should_forward_htl(inv.htl) {
1118 return PubsubPublishStats::default();
1119 }
1120
1121 let mut stats = PubsubPublishStats {
1122 selected_peers: peer_ids.len(),
1123 ..Default::default()
1124 };
1125 for peer_id in peer_ids {
1126 let send_htl = self.decrement_pubsub_htl_for_peer(peer_id, inv.htl).await;
1127 if !should_forward_htl(send_htl) {
1128 continue;
1129 }
1130 let Some(channel) = self.signaling.get_channel(peer_id).await else {
1131 continue;
1132 };
1133 let mut outgoing = inv.clone();
1134 outgoing.htl = send_htl;
1135 let bytes = encode_pubsub_inventory(&outgoing);
1136 let message_bytes = bytes.len() as u64;
1137 if channel.send(bytes).await.is_ok() {
1138 stats.sent_peers += 1;
1139 stats.sent_bytes = stats.sent_bytes.saturating_add(message_bytes);
1140 self.record_peer_wire_sent(peer_id, message_bytes).await;
1141 }
1142 }
1143 stats
1144 }
1145
1146 async fn flood_pubsub_inventory(
1147 &self,
1148 inv: &PubsubInventory,
1149 exclude_peer_id: Option<&str>,
1150 ) -> PubsubPublishStats {
1151 let mut peer_ids = self.signaling.peer_ids().await;
1152 peer_ids.sort();
1153 peer_ids.retain(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude));
1154 self.send_pubsub_inventory_to_peers(inv, &peer_ids).await
1155 }
1156
1157 async fn send_pubsub_want_to_peer(&self, want: &PubsubWant, peer_id: &str) -> bool {
1158 let Some(channel) = self.signaling.get_channel(peer_id).await else {
1159 return false;
1160 };
1161 let bytes = encode_pubsub_want(want);
1162 let message_bytes = bytes.len() as u64;
1163 match channel.send(bytes).await {
1164 Ok(()) => {
1165 self.record_peer_wire_sent(peer_id, message_bytes).await;
1166 true
1167 }
1168 Err(_) => false,
1169 }
1170 }
1171
1172 async fn send_pubsub_want_upstream(
1173 &self,
1174 key: &str,
1175 want: &PubsubWant,
1176 exclude_peer_id: Option<&str>,
1177 ) -> bool {
1178 let upstream = {
1179 let routes = self.pubsub_inventory_routes.read().await;
1180 routes.get(key).cloned()
1181 };
1182 let Some(upstream) = upstream else {
1183 return false;
1184 };
1185 if exclude_peer_id.is_some_and(|exclude| exclude == upstream) {
1186 return false;
1187 }
1188 let want_key = format!("{key}:{upstream}");
1189 if !self
1190 .pubsub_upstream_wants
1191 .lock()
1192 .await
1193 .insert_if_new(want_key)
1194 {
1195 return false;
1196 }
1197 self.send_pubsub_want_to_peer(want, &upstream).await
1198 }
1199
1200 async fn cache_pubsub_frame(&self, key: String, frame: PubsubFrame) {
1201 let mut cache = self.pubsub_frame_cache.lock().await;
1202 if let Some(index) = cache.iter().position(|(cached_key, _)| cached_key == &key) {
1203 cache.remove(index);
1204 }
1205 cache.push_back((key, frame));
1206 while cache.len() > PUBSUB_FRAME_CACHE_CAPACITY {
1207 cache.pop_front();
1208 }
1209 }
1210
1211 async fn cached_pubsub_frame(&self, key: &str) -> Option<PubsubFrame> {
1212 self.pubsub_frame_cache
1213 .lock()
1214 .await
1215 .iter()
1216 .find_map(|(cached_key, frame)| {
1217 if cached_key == key {
1218 Some(frame.clone())
1219 } else {
1220 None
1221 }
1222 })
1223 }
1224
1225 async fn remember_pubsub_want_peer(&self, key: String, from_peer: &str) -> bool {
1226 let mut routes = self.pubsub_want_routes.write().await;
1227 routes.entry(key).or_default().insert(from_peer.to_string())
1228 }
1229
1230 async fn take_pubsub_want_peers(
1231 &self,
1232 key: &str,
1233 exclude_peer_id: Option<&str>,
1234 ) -> Vec<String> {
1235 let connected = self
1236 .signaling
1237 .peer_ids()
1238 .await
1239 .into_iter()
1240 .collect::<HashSet<_>>();
1241 let mut peers = self
1242 .pubsub_want_routes
1243 .write()
1244 .await
1245 .remove(key)
1246 .map(|peers| peers.into_iter().collect::<Vec<_>>())
1247 .unwrap_or_default();
1248 peers.retain(|peer_id| {
1249 connected.contains(peer_id) && exclude_peer_id.is_none_or(|exclude| peer_id != exclude)
1250 });
1251 peers.sort();
1252 peers
1253 }
1254
1255 async fn select_pubsub_peers(
1256 &self,
1257 stream_id: &str,
1258 seq: u64,
1259 message_bytes: u64,
1260 peer_ids: &[String],
1261 ) -> (Vec<String>, Vec<String>) {
1262 let traffic = self.peer_wire_stats.read().await;
1263 let deferred_counts = self.pubsub_deferred_counts.read().await;
1264 let candidates = peer_ids
1265 .iter()
1266 .map(|peer_id| PubsubCandidate {
1267 peer_id: peer_id.clone(),
1268 traffic: traffic.get(peer_id).copied().unwrap_or_default(),
1269 deferred_count: deferred_counts
1270 .get(&(stream_id.to_string(), peer_id.clone()))
1271 .copied()
1272 .unwrap_or_default(),
1273 })
1274 .collect::<Vec<_>>();
1275 drop(deferred_counts);
1276 drop(traffic);
1277
1278 let selection = self.routing.pubsub_scheduler.select(
1279 stream_id,
1280 seq,
1281 self.signaling.peer_id(),
1282 message_bytes,
1283 &candidates,
1284 );
1285
1286 {
1287 let mut deferred_counts = self.pubsub_deferred_counts.write().await;
1288 for peer_id in &selection.deferred {
1289 *deferred_counts
1290 .entry((stream_id.to_string(), peer_id.clone()))
1291 .or_insert(0) += 1;
1292 }
1293 for peer_id in &selection.selected {
1294 deferred_counts.remove(&(stream_id.to_string(), peer_id.clone()));
1295 }
1296 }
1297
1298 (selection.selected, selection.deferred)
1299 }
1300
1301 async fn send_pubsub_frame_to_peers(
1302 &self,
1303 frame: &PubsubFrame,
1304 peer_ids: &[String],
1305 ) -> PubsubPublishStats {
1306 if peer_ids.is_empty() || !should_forward_htl(frame.htl) {
1307 return PubsubPublishStats::default();
1308 }
1309
1310 let bytes = encode_pubsub_frame(frame);
1311 let message_bytes = bytes.len() as u64;
1312 let (selected, deferred) = self
1313 .select_pubsub_peers(&frame.stream_id, frame.seq, message_bytes, peer_ids)
1314 .await;
1315 let mut stats = PubsubPublishStats {
1316 selected_peers: selected.len(),
1317 deferred_peers: deferred.len(),
1318 ..Default::default()
1319 };
1320
1321 for peer_id in selected {
1322 let Some(channel) = self.signaling.get_channel(&peer_id).await else {
1323 continue;
1324 };
1325 let snapshot = self.peer_traffic_snapshot(&peer_id).await;
1326 let bandwidth_debt = reciprocal_virtual_finish(snapshot, message_bytes);
1327 if channel.send(bytes.clone()).await.is_ok() {
1328 stats.sent_peers += 1;
1329 stats.sent_bytes = stats.sent_bytes.saturating_add(message_bytes);
1330 self.record_peer_pubsub_wire_sent(&peer_id, message_bytes, bandwidth_debt)
1331 .await;
1332 }
1333 }
1334
1335 stats
1336 }
1337
1338 async fn enqueue_pubsub_event(&self, event: PubsubEvent) {
1339 let mut inbox = self.pubsub_inbox.lock().await;
1340 inbox.push_back(event);
1341 while inbox.len() > PUBSUB_INBOX_CAPACITY {
1342 inbox.pop_front();
1343 }
1344 self.pubsub_notify.notify_one();
1345 }
1346
1347 pub async fn subscribe_pubsub(
1349 self: &Arc<Self>,
1350 stream_id: impl Into<String>,
1351 ) -> PubsubPublishStats {
1352 let stream_id = stream_id.into();
1353 if stream_id.is_empty() {
1354 return PubsubPublishStats::default();
1355 }
1356 self.pubsub_local_interests
1357 .write()
1358 .await
1359 .insert(stream_id.clone());
1360 let seq = {
1361 let mut versions = self.pubsub_local_interest_versions.write().await;
1362 match versions.get(&stream_id).copied() {
1363 Some(seq) => seq,
1364 None => {
1365 let seq = self.next_pubsub_interest_seq();
1366 versions.insert(stream_id.clone(), seq);
1367 seq
1368 }
1369 }
1370 };
1371 let interest = create_pubsub_interest(
1372 stream_id,
1373 self.signaling.peer_id().to_string(),
1374 seq,
1375 true,
1376 MAX_HTL,
1377 );
1378 self.send_pubsub_interest_to_peers(&interest, None).await
1379 }
1380
1381 pub async fn unsubscribe_pubsub(
1383 self: &Arc<Self>,
1384 stream_id: impl Into<String>,
1385 ) -> PubsubPublishStats {
1386 let stream_id = stream_id.into();
1387 if stream_id.is_empty() {
1388 return PubsubPublishStats::default();
1389 }
1390 self.pubsub_local_interests.write().await.remove(&stream_id);
1391 self.pubsub_local_interest_versions
1392 .write()
1393 .await
1394 .remove(&stream_id);
1395 let interest = create_pubsub_interest(
1396 stream_id,
1397 self.signaling.peer_id().to_string(),
1398 self.next_pubsub_interest_seq(),
1399 false,
1400 MAX_HTL,
1401 );
1402 self.send_pubsub_interest_to_peers(&interest, None).await
1403 }
1404
1405 pub async fn publish_pubsub(
1407 self: &Arc<Self>,
1408 stream_id: impl Into<String>,
1409 seq: u64,
1410 payload: Vec<u8>,
1411 ) -> PubsubPublishStats {
1412 let stream_id = stream_id.into();
1413 if stream_id.is_empty() {
1414 return PubsubPublishStats::default();
1415 }
1416 let payload_bytes = payload.len() as u64;
1417 let frame = create_pubsub_frame(
1418 stream_id.clone(),
1419 seq,
1420 self.signaling.peer_id().to_string(),
1421 payload.clone(),
1422 MAX_HTL,
1423 );
1424 let frame_key = Self::pubsub_frame_key(&frame);
1425 self.pubsub_seen_frames
1426 .lock()
1427 .await
1428 .insert_if_new(frame_key.clone());
1429 self.cache_pubsub_frame(frame_key, frame.clone()).await;
1430
1431 if self
1432 .pubsub_local_interests
1433 .read()
1434 .await
1435 .contains(&stream_id)
1436 {
1437 self.enqueue_pubsub_event(PubsubEvent {
1438 stream_id: stream_id.clone(),
1439 seq,
1440 origin_peer_id: self.signaling.peer_id().to_string(),
1441 from_peer_id: self.signaling.peer_id().to_string(),
1442 payload,
1443 })
1444 .await;
1445 }
1446
1447 match self.routing.pubsub_delivery_mode {
1448 PubsubDeliveryMode::InterestPush => {
1449 let peers = self.interested_pubsub_peers(&stream_id, None).await;
1450 self.send_pubsub_frame_to_peers(&frame, &peers).await
1451 }
1452 PubsubDeliveryMode::HtlInvWant => {
1453 let inv = create_pubsub_inventory(
1454 stream_id,
1455 seq,
1456 self.signaling.peer_id().to_string(),
1457 payload_bytes,
1458 MESH_EVENT_POLICY.max_htl,
1459 );
1460 self.flood_pubsub_inventory(&inv, None).await
1461 }
1462 }
1463 }
1464
1465 pub async fn drain_pubsub_events(&self) -> Vec<PubsubEvent> {
1467 self.pubsub_inbox.lock().await.drain(..).collect()
1468 }
1469
1470 pub async fn recv_pubsub_event(&self) -> PubsubEvent {
1472 loop {
1473 if let Some(event) = self.pubsub_inbox.lock().await.pop_front() {
1474 return event;
1475 }
1476 self.pubsub_notify.notified().await;
1477 }
1478 }
1479
1480 pub async fn pubsub_interest_peers(&self, stream_id: &str) -> Vec<String> {
1482 self.interested_pubsub_peers(stream_id, None).await
1483 }
1484
1485 fn choose_ready_response_job(
1486 ready_jobs: &[(u64, String, usize, Instant, u64)],
1487 stats: &HashMap<String, PeerWireStats>,
1488 ) -> Option<(u64, f64)> {
1489 let jobs = ready_jobs
1490 .iter()
1491 .map(|job| OutboundJobCandidate {
1492 job_id: job.0,
1493 peer_id: job.1.clone(),
1494 message_bytes: job.2 as u64,
1495 queue_sequence: job.4,
1496 })
1497 .collect::<Vec<_>>();
1498 select_reciprocal_outbound_job(&jobs, |peer_id| {
1499 stats.get(peer_id).copied().unwrap_or_default()
1500 })
1501 .map(|choice| (choice.job_id, choice.virtual_finish))
1502 }
1503
1504 async fn enqueue_response_send(
1505 self: &Arc<Self>,
1506 peer_id: String,
1507 bytes: Vec<u8>,
1508 ready_at: Instant,
1509 ) {
1510 let job_id = self.next_response_job_id.fetch_add(1, Ordering::Relaxed);
1511 {
1512 let mut queue = self.pending_response_sends.lock().await;
1513 queue.push(PendingResponseSend {
1514 job_id,
1515 peer_id,
1516 bytes,
1517 ready_at,
1518 queue_sequence: job_id,
1519 });
1520 }
1521
1522 if self
1523 .response_scheduler_running
1524 .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
1525 .is_ok()
1526 {
1527 let this = Arc::clone(self);
1528 tokio::spawn(async move {
1529 this.run_response_scheduler().await;
1530 });
1531 }
1532 }
1533
1534 async fn run_response_scheduler(self: Arc<Self>) {
1535 loop {
1536 let snapshot = {
1537 let queue = self.pending_response_sends.lock().await;
1538 if queue.is_empty() {
1539 self.response_scheduler_running
1540 .store(false, Ordering::Release);
1541 return;
1542 }
1543 queue
1544 .iter()
1545 .map(|job| {
1546 (
1547 job.job_id,
1548 job.peer_id.clone(),
1549 job.bytes.len(),
1550 job.ready_at,
1551 job.queue_sequence,
1552 )
1553 })
1554 .collect::<Vec<_>>()
1555 };
1556
1557 let now = Instant::now();
1558 let mut earliest_ready_at: Option<Instant> = None;
1559 let mut ready_jobs = Vec::new();
1560 for job in &snapshot {
1561 if job.3 <= now {
1562 ready_jobs.push(job.clone());
1563 } else {
1564 earliest_ready_at = Some(match earliest_ready_at {
1565 Some(current) => current.min(job.3),
1566 None => job.3,
1567 });
1568 }
1569 }
1570
1571 if ready_jobs.is_empty() {
1572 if let Some(ready_at) = earliest_ready_at {
1573 tokio::time::sleep(ready_at.saturating_duration_since(Instant::now())).await;
1574 continue;
1575 }
1576 self.response_scheduler_running
1577 .store(false, Ordering::Release);
1578 return;
1579 }
1580
1581 let (selected_job_id, selected_finish) = {
1582 let stats = self.peer_wire_stats.read().await;
1583 Self::choose_ready_response_job(&ready_jobs, &stats).expect("ready response job")
1584 };
1585
1586 let selected = {
1587 let mut queue = self.pending_response_sends.lock().await;
1588 let Some(index) = queue.iter().position(|job| job.job_id == selected_job_id) else {
1589 continue;
1590 };
1591 queue.swap_remove(index)
1592 };
1593
1594 let sent = if let Some(channel) = self.signaling.get_channel(&selected.peer_id).await {
1595 channel.send(selected.bytes.clone()).await.is_ok()
1596 } else {
1597 false
1598 };
1599
1600 let queued_peers = {
1601 let queue = self.pending_response_sends.lock().await;
1602 queue
1603 .iter()
1604 .map(|job| job.peer_id.clone())
1605 .collect::<HashSet<_>>()
1606 };
1607 let mut stats = self.peer_wire_stats.write().await;
1608 let entry = stats.entry(selected.peer_id.clone()).or_default();
1609 if sent {
1610 entry.bytes_sent = entry.bytes_sent.saturating_add(selected.bytes.len() as u64);
1611 entry.bandwidth_debt = selected_finish;
1612 }
1613 if queued_peers.is_empty() {
1614 for peer_stats in stats.values_mut() {
1615 peer_stats.bandwidth_debt = 0.0;
1616 }
1617 } else {
1618 let floor = queued_peers
1619 .iter()
1620 .filter_map(|peer_id| stats.get(peer_id).map(|peer| peer.bandwidth_debt))
1621 .fold(f64::INFINITY, f64::min);
1622 if floor.is_finite() && floor > 0.0 {
1623 for peer_id in queued_peers {
1624 if let Some(peer_stats) = stats.get_mut(&peer_id) {
1625 peer_stats.bandwidth_debt =
1626 (peer_stats.bandwidth_debt - floor).max(0.0);
1627 }
1628 }
1629 }
1630 }
1631 }
1632 }
1633
1634 fn deterministic_actor_draw_for(peer_id: &str, hash: &Hash, salt: u64) -> f64 {
1635 let mut hasher = DefaultHasher::new();
1636 peer_id.hash(&mut hasher);
1637 hash.hash(&mut hasher);
1638 salt.hash(&mut hasher);
1639 let v = hasher.finish();
1640 (v as f64) / (u64::MAX as f64)
1641 }
1642
1643 fn deterministic_actor_draw(&self, hash: &Hash, salt: u64) -> f64 {
1644 Self::deterministic_actor_draw_for(self.signaling.peer_id(), hash, salt)
1645 }
1646
1647 fn peer_metadata_pointer_slot_hash() -> Hash {
1648 hashtree_core::sha256(PEER_METADATA_POINTER_SLOT_KEY)
1649 }
1650
1651 fn decode_hash_hex(hash_hex: &str) -> Result<Hash, StoreError> {
1652 let bytes = hex::decode(hash_hex)
1653 .map_err(|e| StoreError::Other(format!("Invalid hash hex: {e}")))?;
1654 if bytes.len() != 32 {
1655 return Err(StoreError::Other(format!(
1656 "Invalid hash length {}, expected 32 bytes",
1657 bytes.len()
1658 )));
1659 }
1660 let mut hash = [0u8; 32];
1661 hash.copy_from_slice(&bytes);
1662 Ok(hash)
1663 }
1664
1665 fn should_drop_response(&self, hash: &Hash) -> bool {
1666 let p = self.response_behavior().drop_response_prob;
1667 if p <= 0.0 {
1668 return false;
1669 }
1670 self.deterministic_actor_draw(hash, 0xD0_D0_D0_D0_D0_D0_D0_D0) < p
1671 }
1672
1673 fn should_corrupt_response(&self, hash: &Hash) -> bool {
1674 let p = self.response_behavior().corrupt_response_prob;
1675 if p <= 0.0 {
1676 return false;
1677 }
1678 self.deterministic_actor_draw(hash, 0xC0_C0_C0_C0_C0_C0_C0_C0) < p
1679 }
1680
1681 fn should_stall_response(&self, hash: &Hash) -> bool {
1682 let p = self.response_behavior().stall_response_prob;
1683 if p <= 0.0 {
1684 return false;
1685 }
1686 self.deterministic_actor_draw(hash, 0x5A_11_5A_11_5A_11_5A_11) < p
1687 }
1688
1689 fn response_send_delay(&self, hash: &Hash, payload_len: usize) -> Duration {
1690 let behavior = self.response_behavior();
1691 let mut total_ms = behavior
1692 .extra_delay_ms
1693 .saturating_add(behavior.first_byte_delay_ms);
1694
1695 if behavior.bytes_per_second > 0 && payload_len > 0 {
1696 let throughput_ms = ((payload_len as u128) * 1000)
1697 .div_ceil(behavior.bytes_per_second as u128)
1698 .min(u64::MAX as u128) as u64;
1699 total_ms = total_ms.saturating_add(throughput_ms);
1700 }
1701
1702 if behavior.stall_delay_ms > 0 && self.should_stall_response(hash) {
1703 total_ms = total_ms.saturating_add(behavior.stall_delay_ms);
1704 }
1705
1706 Duration::from_millis(total_ms)
1707 }
1708
1709 async fn ordered_connected_peers(&self, exclude_peer_id: Option<&str>) -> Vec<String> {
1710 let current_peer_ids = self.signaling.peer_ids().await;
1711 if current_peer_ids.is_empty() {
1712 return Vec::new();
1713 }
1714
1715 sync_selector_peers(&self.peer_selector, ¤t_peer_ids).await;
1716 let hash_get_peer_ids: HashSet<String> = self
1717 .signaling
1718 .hash_get_peer_ids()
1719 .await
1720 .into_iter()
1721 .collect();
1722 let mut candidate_peer_ids: Vec<String> = current_peer_ids
1723 .into_iter()
1724 .filter(|peer_id| hash_get_peer_ids.contains(peer_id))
1725 .filter(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude))
1726 .collect();
1727 if candidate_peer_ids.is_empty() {
1728 return Vec::new();
1729 }
1730
1731 let current_set: HashSet<&str> = candidate_peer_ids.iter().map(String::as_str).collect();
1732 let mut selector = self.peer_selector.write().await;
1733 let mut selector_order = selector.select_peers();
1734 selector_order.retain(|peer_id| current_set.contains(peer_id.as_str()));
1735 if selector_order.is_empty() {
1736 let mut fallback = candidate_peer_ids;
1737 fallback.sort();
1738 return fallback;
1739 }
1740 let backed_off: HashMap<String, bool> = candidate_peer_ids
1741 .iter()
1742 .map(|peer_id| (peer_id.clone(), selector.is_peer_backed_off(peer_id)))
1743 .collect();
1744 drop(selector);
1745
1746 let rank: HashMap<&str, usize> = selector_order
1747 .iter()
1748 .enumerate()
1749 .map(|(idx, peer_id)| (peer_id.as_str(), idx))
1750 .collect();
1751 let active = self.peer_active_requests.read().await;
1752 candidate_peer_ids.sort_by(|left, right| {
1753 let left_backed_off = backed_off.get(left).copied().unwrap_or(false);
1754 let right_backed_off = backed_off.get(right).copied().unwrap_or(false);
1755 if left_backed_off != right_backed_off {
1756 return if left_backed_off {
1757 std::cmp::Ordering::Greater
1758 } else {
1759 std::cmp::Ordering::Less
1760 };
1761 }
1762 let left_rank = rank.get(left.as_str()).copied().unwrap_or(usize::MAX / 2);
1763 let right_rank = rank.get(right.as_str()).copied().unwrap_or(usize::MAX / 2);
1764 let left_load = active.get(left).copied().unwrap_or(0);
1765 let right_load = active.get(right).copied().unwrap_or(0);
1766 (left_rank + left_load.saturating_mul(ACTIVE_PEER_REQUEST_RANK_PENALTY))
1767 .cmp(&(right_rank + right_load.saturating_mul(ACTIVE_PEER_REQUEST_RANK_PENALTY)))
1768 .then_with(|| left.cmp(right))
1769 });
1770 candidate_peer_ids
1771 }
1772
1773 async fn reserve_peer_request(&self, peer_id: &str) {
1774 let mut active = self.peer_active_requests.write().await;
1775 *active.entry(peer_id.to_string()).or_insert(0) += 1;
1776 }
1777
1778 async fn release_peer_request(&self, peer_id: &str) {
1779 let mut active = self.peer_active_requests.write().await;
1780 let Some(count) = active.get_mut(peer_id) else {
1781 return;
1782 };
1783 if *count <= 1 {
1784 active.remove(peer_id);
1785 } else {
1786 *count -= 1;
1787 }
1788 }
1789
1790 async fn release_queried_peer_requests(&self, peer_ids: &[String]) {
1791 for peer_id in peer_ids {
1792 self.release_peer_request(peer_id).await;
1793 }
1794 }
1795
1796 fn requested_quote_mint(&self) -> Option<&str> {
1797 if let Some(default_mint) = self.routing.cashu_default_mint.as_deref() {
1798 if self.routing.cashu_accepted_mints.is_empty()
1799 || self
1800 .routing
1801 .cashu_accepted_mints
1802 .iter()
1803 .any(|mint| mint == default_mint)
1804 {
1805 return Some(default_mint);
1806 }
1807 }
1808
1809 self.routing
1810 .cashu_accepted_mints
1811 .first()
1812 .map(String::as_str)
1813 }
1814
1815 fn choose_quote_mint(&self, requested_mint: Option<&str>) -> Option<String> {
1816 if let Some(requested_mint) = requested_mint {
1817 if self.accepts_quote_mint(Some(requested_mint)) {
1818 return Some(requested_mint.to_string());
1819 }
1820 }
1821 if let Some(default_mint) = self.routing.cashu_default_mint.as_ref() {
1822 return Some(default_mint.clone());
1823 }
1824 if let Some(first_mint) = self.routing.cashu_accepted_mints.first() {
1825 return Some(first_mint.clone());
1826 }
1827 requested_mint.map(str::to_string)
1828 }
1829
1830 fn accepts_quote_mint(&self, mint_url: Option<&str>) -> bool {
1831 if self.routing.cashu_accepted_mints.is_empty() {
1832 return true;
1833 }
1834
1835 let Some(mint_url) = mint_url else {
1836 return false;
1837 };
1838 self.routing
1839 .cashu_accepted_mints
1840 .iter()
1841 .any(|mint| mint == mint_url)
1842 }
1843
1844 fn trusts_quote_mint(&self, mint_url: Option<&str>) -> bool {
1845 let Some(mint_url) = mint_url else {
1846 return self.routing.cashu_default_mint.is_none()
1847 && self.routing.cashu_accepted_mints.is_empty();
1848 };
1849 self.routing.cashu_default_mint.as_deref() == Some(mint_url)
1850 || self
1851 .routing
1852 .cashu_accepted_mints
1853 .iter()
1854 .any(|mint| mint == mint_url)
1855 }
1856
1857 async fn peer_suggested_mint_cap_sat(&self, peer_id: &str) -> u64 {
1858 let base = self.routing.cashu_peer_suggested_mint_base_cap_sat;
1859 if base == 0 {
1860 return 0;
1861 }
1862
1863 let selector = self.peer_selector.read().await;
1864 let Some(stats) = selector.get_stats(peer_id) else {
1865 let max_cap = self.routing.cashu_peer_suggested_mint_max_cap_sat;
1866 return if max_cap > 0 { base.min(max_cap) } else { base };
1867 };
1868
1869 if stats.cashu_payment_defaults > 0
1870 && stats.cashu_payment_defaults >= stats.cashu_payment_receipts
1871 {
1872 return 0;
1873 }
1874
1875 let success_bonus = stats
1876 .successes
1877 .saturating_mul(self.routing.cashu_peer_suggested_mint_success_step_sat);
1878 let receipt_bonus = stats
1879 .cashu_payment_receipts
1880 .saturating_mul(self.routing.cashu_peer_suggested_mint_receipt_step_sat);
1881 let mut cap = base
1882 .saturating_add(success_bonus)
1883 .saturating_add(receipt_bonus);
1884 let max_cap = self.routing.cashu_peer_suggested_mint_max_cap_sat;
1885 if max_cap > 0 {
1886 cap = cap.min(max_cap);
1887 }
1888 cap
1889 }
1890
1891 async fn should_accept_quote_response(
1892 &self,
1893 from_peer: &str,
1894 preferred_mint_url: Option<&str>,
1895 offered_payment_sat: u64,
1896 res: &DataQuoteResponse,
1897 ) -> bool {
1898 let Some(payment_sat) = res.p else {
1899 return false;
1900 };
1901 if payment_sat > offered_payment_sat {
1902 return false;
1903 }
1904
1905 let response_mint = res.m.as_deref();
1906 if response_mint == preferred_mint_url {
1907 return true;
1908 }
1909 if self.trusts_quote_mint(response_mint) {
1910 return true;
1911 }
1912 if response_mint.is_none() {
1913 return false;
1914 }
1915
1916 payment_sat <= self.peer_suggested_mint_cap_sat(from_peer).await
1917 }
1918
1919 async fn issue_quote(
1920 &self,
1921 peer_id: &str,
1922 hash_key: &str,
1923 payment_sat: u64,
1924 ttl_ms: u32,
1925 mint_url: Option<&str>,
1926 ) -> u64 {
1927 let quote_id = {
1928 let mut next = self.next_quote_id.write().await;
1929 let quote_id = *next;
1930 *next = next.saturating_add(1);
1931 quote_id
1932 };
1933
1934 let expires_at = Instant::now() + Duration::from_millis(ttl_ms as u64);
1935 self.issued_quotes.write().await.insert(
1936 (peer_id.to_string(), hash_key.to_string(), quote_id),
1937 IssuedQuote {
1938 expires_at,
1939 payment_sat,
1940 mint_url: mint_url.map(str::to_string),
1941 },
1942 );
1943 quote_id
1944 }
1945
1946 async fn take_valid_quote(&self, peer_id: &str, hash_key: &str, quote_id: u64) -> bool {
1947 let key = (peer_id.to_string(), hash_key.to_string(), quote_id);
1948 let Some(quote) = self.issued_quotes.write().await.remove(&key) else {
1949 return false;
1950 };
1951 quote.expires_at > Instant::now()
1952 }
1953
1954 async fn send_request_to_peer(
1955 &self,
1956 peer_id: &str,
1957 hash: &Hash,
1958 request_htl: u8,
1959 quote_id: Option<u64>,
1960 ) -> bool {
1961 if !should_forward_htl(request_htl) {
1962 return false;
1963 }
1964
1965 let channel = match self.signaling.get_channel(peer_id).await {
1966 Some(c) => c,
1967 None => return false,
1968 };
1969
1970 let htl_config = {
1971 let configs = self.htl_configs.read().await;
1972 configs
1973 .get(peer_id)
1974 .cloned()
1975 .unwrap_or_else(PeerHTLConfig::random)
1976 };
1977
1978 let send_htl = htl_config.decrement(request_htl);
1979 let req = match quote_id {
1980 Some(quote_id) => create_request_with_quote(hash, send_htl, quote_id),
1981 None => create_request(hash, send_htl),
1982 };
1983 let request_bytes = encode_request(&req);
1984 let request_len = request_bytes.len() as u64;
1985
1986 {
1987 let mut selector = self.peer_selector.write().await;
1988 selector.record_request(peer_id, request_len);
1989 }
1990
1991 match channel.send(request_bytes).await {
1992 Ok(()) => {
1993 self.record_peer_wire_sent(peer_id, request_len).await;
1994 true
1995 }
1996 Err(_) => {
1997 self.peer_selector.write().await.record_failure(peer_id);
1998 false
1999 }
2000 }
2001 }
2002
2003 async fn send_quote_request_to_peer(
2004 &self,
2005 peer_id: &str,
2006 hash: &Hash,
2007 payment_sat: u64,
2008 ttl_ms: u32,
2009 mint_url: Option<&str>,
2010 ) -> bool {
2011 let channel = match self.signaling.get_channel(peer_id).await {
2012 Some(c) => c,
2013 None => return false,
2014 };
2015
2016 let req = create_quote_request(hash, ttl_ms, payment_sat, mint_url);
2017 let request_bytes = encode_quote_request(&req);
2018 let request_len = request_bytes.len() as u64;
2019
2020 match channel.send(request_bytes).await {
2021 Ok(()) => {
2022 self.record_peer_wire_sent(peer_id, request_len).await;
2023 true
2024 }
2025 Err(_) => false,
2026 }
2027 }
2028
2029 pub async fn set_read_sources(&self, sources: Vec<Arc<dyn MeshReadSource>>) {
2030 let mut by_id = HashMap::new();
2031 let mut stats = self.read_source_stats.write().await;
2032 for source in sources {
2033 let source_id = source.id().to_string();
2034 by_id.insert(source_id.clone(), source);
2035 stats
2036 .entry(source_id)
2037 .or_insert_with(AdaptiveSourceStats::default);
2038 }
2039 *self.read_sources.write().await = by_id;
2040 }
2041
2042 async fn record_read_source_request(&self, source_id: &str) {
2043 let mut stats = self.read_source_stats.write().await;
2044 stats
2045 .entry(source_id.to_string())
2046 .or_insert_with(AdaptiveSourceStats::default)
2047 .requests += 1;
2048 }
2049
2050 async fn record_read_source_miss(&self, source_id: &str) {
2051 let mut stats = self.read_source_stats.write().await;
2052 stats
2053 .entry(source_id.to_string())
2054 .or_insert_with(AdaptiveSourceStats::default)
2055 .misses += 1;
2056 }
2057
2058 async fn record_read_source_success(&self, source_id: &str, elapsed_ms: u64) {
2059 let now = Instant::now();
2060 let mut stats = self.read_source_stats.write().await;
2061 let stats = stats
2062 .entry(source_id.to_string())
2063 .or_insert_with(AdaptiveSourceStats::default);
2064 stats.successes += 1;
2065 stats.last_success_at = Some(now);
2066 stats.backoff_level = 0;
2067 stats.backed_off_until = None;
2068 if stats.srtt_ms <= 0.0 {
2069 stats.srtt_ms = elapsed_ms as f64;
2070 stats.rttvar_ms = elapsed_ms as f64 / 2.0;
2071 return;
2072 }
2073 let elapsed = elapsed_ms as f64;
2074 stats.rttvar_ms = 0.75 * stats.rttvar_ms + 0.25 * (stats.srtt_ms - elapsed).abs();
2075 stats.srtt_ms = 0.875 * stats.srtt_ms + 0.125 * elapsed;
2076 }
2077
2078 async fn record_read_source_failure(&self, source_id: &str) {
2079 let now = Instant::now();
2080 let mut stats = self.read_source_stats.write().await;
2081 let stats = stats
2082 .entry(source_id.to_string())
2083 .or_insert_with(AdaptiveSourceStats::default);
2084 stats.failures += 1;
2085 stats.last_failure_at = Some(now);
2086 Self::apply_source_backoff(stats, now);
2087 }
2088
2089 async fn record_read_source_timeout(&self, source_id: &str) {
2090 let now = Instant::now();
2091 let mut stats = self.read_source_stats.write().await;
2092 let stats = stats
2093 .entry(source_id.to_string())
2094 .or_insert_with(AdaptiveSourceStats::default);
2095 stats.timeouts += 1;
2096 stats.last_failure_at = Some(now);
2097 Self::apply_source_backoff(stats, now);
2098 }
2099
2100 fn apply_source_backoff(stats: &mut AdaptiveSourceStats, now: Instant) {
2101 stats.backoff_level = stats.backoff_level.saturating_add(1);
2102 let backoff_ms = (INITIAL_SOURCE_BACKOFF_MS
2103 .saturating_mul(2u64.saturating_pow(stats.backoff_level.saturating_sub(1))))
2104 .min(MAX_SOURCE_BACKOFF_MS);
2105 stats.backed_off_until = Some(now + Duration::from_millis(backoff_ms));
2106 }
2107
2108 async fn ordered_read_sources(&self) -> Vec<Arc<dyn MeshReadSource>> {
2109 let sources = self.read_sources.read().await;
2110 if sources.is_empty() {
2111 return Vec::new();
2112 }
2113
2114 let mut available: Vec<Arc<dyn MeshReadSource>> = sources
2115 .values()
2116 .filter(|source| source.is_available())
2117 .cloned()
2118 .collect();
2119 if available.is_empty() {
2120 return Vec::new();
2121 }
2122
2123 let now = Instant::now();
2124 let stats = self.read_source_stats.read().await;
2125 let mut healthy: Vec<Arc<dyn MeshReadSource>> = available
2126 .iter()
2127 .filter(|source| {
2128 stats
2129 .get(source.id())
2130 .and_then(|s| s.backed_off_until)
2131 .is_none_or(|until| until <= now)
2132 })
2133 .cloned()
2134 .collect();
2135 if !healthy.is_empty() {
2136 available = std::mem::take(&mut healthy);
2137 }
2138
2139 available.sort_by(|left, right| {
2140 let left_stats = stats.get(left.id()).cloned().unwrap_or_default();
2141 let right_stats = stats.get(right.id()).cloned().unwrap_or_default();
2142 adaptive_source_score(&right_stats, now)
2143 .partial_cmp(&adaptive_source_score(&left_stats, now))
2144 .unwrap_or(std::cmp::Ordering::Equal)
2145 .then_with(|| left.id().cmp(right.id()))
2146 });
2147 available
2148 }
2149
2150 async fn should_probe_multiple_read_sources(
2151 &self,
2152 ordered_sources: &[Arc<dyn MeshReadSource>],
2153 ) -> bool {
2154 if ordered_sources.len() <= 1 {
2155 return false;
2156 }
2157 let stats = self.read_source_stats.read().await;
2158 let best = stats
2159 .get(ordered_sources[0].id())
2160 .cloned()
2161 .unwrap_or_default();
2162 let second = stats
2163 .get(ordered_sources[1].id())
2164 .cloned()
2165 .unwrap_or_default();
2166 if !source_has_history(&best) || !source_has_history(&second) {
2167 return false;
2168 }
2169 let now = Instant::now();
2170 adaptive_source_score(&best, now) - adaptive_source_score(&second, now)
2171 < SOURCE_SCORE_TIE_DELTA
2172 }
2173
2174 async fn source_dispatch_for(&self, source_count: usize) -> RequestDispatchConfig {
2175 if source_count == 0 {
2176 return self.routing.dispatch;
2177 }
2178 let ordered_sources = self.ordered_read_sources().await;
2179 let probe_multiple = self
2180 .should_probe_multiple_read_sources(&ordered_sources)
2181 .await;
2182 let initial_fanout = if probe_multiple {
2183 source_count.min(2)
2184 } else {
2185 1
2186 };
2187 RequestDispatchConfig {
2188 initial_fanout,
2189 hedge_fanout: self.routing.dispatch.hedge_fanout,
2190 max_fanout: self.routing.dispatch.max_fanout.min(source_count),
2191 hedge_interval_ms: self.routing.dispatch.hedge_interval_ms,
2192 }
2193 }
2194
2195 pub async fn peer_count(&self) -> usize {
2197 self.signaling.peer_count().await
2198 }
2199
2200 pub async fn peer_ids(&self) -> Vec<String> {
2202 self.signaling.peer_ids().await
2203 }
2204
2205 pub async fn needs_peers(&self) -> bool {
2207 self.signaling.needs_peers().await
2208 }
2209
2210 pub async fn send_hello(&self) -> Result<(), TransportError> {
2212 self.signaling.send_hello(vec![]).await
2213 }
2214
2215 pub async fn drain_available_data_messages(self: &Arc<Self>) -> DataPumpStats {
2220 let mut stats = DataPumpStats::default();
2221 let peer_ids = self.signaling.peer_ids().await;
2222 for peer_id in peer_ids {
2223 let Some(channel) = self.signaling.get_channel(&peer_id).await else {
2224 continue;
2225 };
2226
2227 while let Some(data) = channel.try_recv() {
2228 stats.processed += 1;
2229 stats.processed_bytes += data.len() as u64;
2230 if let Some(msg) = parse_message(&data) {
2231 match msg {
2232 DataMessage::Request(_) => stats.request_messages += 1,
2233 DataMessage::Response(_) => stats.response_messages += 1,
2234 DataMessage::QuoteRequest(_) => stats.quote_request_messages += 1,
2235 DataMessage::QuoteResponse(_) => stats.quote_response_messages += 1,
2236 DataMessage::PubsubInterest(_) => stats.pubsub_interest_messages += 1,
2237 DataMessage::PubsubFrame(_) => stats.pubsub_frame_messages += 1,
2238 DataMessage::PubsubInventory(_) => stats.pubsub_inventory_messages += 1,
2239 DataMessage::PubsubWant(_) => stats.pubsub_want_messages += 1,
2240 DataMessage::Payment(_)
2241 | DataMessage::PaymentAck(_)
2242 | DataMessage::Chunk(_)
2243 | DataMessage::PeerHints(_) => {}
2244 }
2245 }
2246 self.handle_data_message(&peer_id, &data).await;
2247 }
2248 }
2249 stats
2250 }
2251
2252 pub async fn record_cashu_payment_for_peer(&self, peer_id: &str, amount_sat: u64) {
2254 self.peer_selector
2255 .write()
2256 .await
2257 .record_cashu_payment(peer_id, amount_sat);
2258 }
2259
2260 pub async fn record_cashu_receipt_from_peer(&self, peer_id: &str, amount_sat: u64) {
2262 self.peer_selector
2263 .write()
2264 .await
2265 .record_cashu_receipt(peer_id, amount_sat);
2266 }
2267
2268 pub async fn record_cashu_payment_default_from_peer(&self, peer_id: &str) {
2270 self.peer_selector
2271 .write()
2272 .await
2273 .record_cashu_payment_default(peer_id);
2274 }
2275
2276 pub async fn selector_summary(&self) -> crate::peer_selector::SelectorSummary {
2278 self.peer_selector.read().await.summary()
2279 }
2280
2281 fn should_refuse_requests_from_peer(&self, selector: &PeerSelector, peer_id: &str) -> bool {
2282 selector.is_peer_blocked_for_payment_defaults(
2283 peer_id,
2284 self.routing.cashu_payment_default_block_threshold,
2285 )
2286 }
2287
2288 pub async fn peer_metadata_snapshot(&self) -> PeerMetadataSnapshot {
2290 self.peer_selector
2291 .read()
2292 .await
2293 .export_peer_metadata_snapshot()
2294 }
2295
2296 pub async fn persist_peer_metadata(&self) -> Result<Hash, StoreError> {
2301 let snapshot = self
2302 .peer_selector
2303 .read()
2304 .await
2305 .export_peer_metadata_snapshot();
2306 let bytes = serde_json::to_vec(&snapshot).map_err(|e| {
2307 StoreError::Other(format!("Failed to encode peer metadata snapshot: {e}"))
2308 })?;
2309 let snapshot_hash = hashtree_core::sha256(&bytes);
2310 let _ = self.local_store.put(snapshot_hash, bytes).await?;
2311
2312 let pointer_slot = Self::peer_metadata_pointer_slot_hash();
2313 let pointer_bytes = hex::encode(snapshot_hash).into_bytes();
2314 let _ = self.local_store.delete(&pointer_slot).await?;
2315 let _ = self.local_store.put(pointer_slot, pointer_bytes).await?;
2316
2317 Ok(snapshot_hash)
2318 }
2319
2320 pub async fn load_peer_metadata(&self) -> Result<bool, StoreError> {
2322 let pointer_slot = Self::peer_metadata_pointer_slot_hash();
2323 let Some(pointer_bytes) = self.local_store.get(&pointer_slot).await? else {
2324 return Ok(false);
2325 };
2326 let pointer_hex = std::str::from_utf8(&pointer_bytes).map_err(|e| {
2327 StoreError::Other(format!("Peer metadata pointer is not valid UTF-8: {e}"))
2328 })?;
2329 let snapshot_hash = Self::decode_hash_hex(pointer_hex.trim())?;
2330
2331 let Some(snapshot_bytes) = self.local_store.get(&snapshot_hash).await? else {
2332 return Ok(false);
2333 };
2334 let snapshot: PeerMetadataSnapshot =
2335 serde_json::from_slice(&snapshot_bytes).map_err(|e| {
2336 StoreError::Other(format!("Failed to decode peer metadata snapshot: {e}"))
2337 })?;
2338 self.peer_selector
2339 .write()
2340 .await
2341 .import_peer_metadata_snapshot(&snapshot);
2342 Ok(true)
2343 }
2344
2345 pub async fn get_with_quote(
2350 &self,
2351 hash: &Hash,
2352 payment_sat: u64,
2353 quote_ttl: Duration,
2354 ) -> Result<Option<Vec<u8>>, StoreError> {
2355 if let Some(data) = self.local_store.get(hash).await? {
2356 return Ok(Some(data));
2357 }
2358 Ok(self
2359 .request_from_peers_with_quote(hash, payment_sat, quote_ttl)
2360 .await)
2361 }
2362
2363 async fn request_from_peers_with_quote(
2364 &self,
2365 hash: &Hash,
2366 payment_sat: u64,
2367 quote_ttl: Duration,
2368 ) -> Option<Vec<u8>> {
2369 let ordered_peer_ids = self.ordered_connected_peers(None).await;
2370 if ordered_peer_ids.is_empty() {
2371 return None;
2372 }
2373
2374 if let Some(quote) = self
2375 .request_quote_from_peers(hash, payment_sat, quote_ttl, &ordered_peer_ids)
2376 .await
2377 {
2378 if let Some(data) = self
2379 .request_from_single_peer(hash, "e.peer_id, MAX_HTL, Some(quote.quote_id))
2380 .await
2381 {
2382 return Some(data);
2383 }
2384 }
2385
2386 self.request_from_mesh(hash).await
2387 }
2388
2389 async fn request_quote_from_peers(
2390 &self,
2391 hash: &Hash,
2392 payment_sat: u64,
2393 quote_ttl: Duration,
2394 ordered_peer_ids: &[String],
2395 ) -> Option<NegotiatedQuote> {
2396 if ordered_peer_ids.is_empty() {
2397 return None;
2398 }
2399 let ttl_ms = quote_ttl.as_millis().min(u32::MAX as u128) as u32;
2400 if ttl_ms == 0 {
2401 return None;
2402 }
2403 let requested_mint = self.requested_quote_mint().map(str::to_string);
2404
2405 let hash_key = hash_to_key(hash);
2406 let (tx, rx) = oneshot::channel();
2407 self.pending_quotes.write().await.insert(
2408 hash_key.clone(),
2409 PendingQuoteRequest {
2410 response_tx: tx,
2411 preferred_mint_url: requested_mint.clone(),
2412 offered_payment_sat: payment_sat,
2413 },
2414 );
2415
2416 let rx = Arc::new(Mutex::new(rx));
2417 let result = run_hedged_waves(
2418 ordered_peer_ids.len(),
2419 self.routing.dispatch,
2420 self.request_timeout,
2421 |range| {
2422 let wave_peer_ids = ordered_peer_ids[range].to_vec();
2423 let requested_mint = requested_mint.clone();
2424 let hash = *hash;
2425 async move {
2426 let mut sent = 0usize;
2427 for peer_id in wave_peer_ids {
2428 if self
2429 .send_quote_request_to_peer(
2430 &peer_id,
2431 &hash,
2432 payment_sat,
2433 ttl_ms,
2434 requested_mint.as_deref(),
2435 )
2436 .await
2437 {
2438 sent += 1;
2439 }
2440 }
2441 sent
2442 }
2443 },
2444 |wait| {
2445 let rx = rx.clone();
2446 async move {
2447 let mut rx = rx.lock().await;
2448 match tokio::time::timeout(wait, &mut *rx).await {
2449 Ok(Ok(Some(quote))) => HedgedWaveAction::Success(quote),
2450 Ok(Ok(None)) | Ok(Err(_)) => HedgedWaveAction::Abort,
2451 Err(_) => HedgedWaveAction::Continue,
2452 }
2453 }
2454 },
2455 )
2456 .await;
2457 let _ = self.pending_quotes.write().await.remove(&hash_key);
2458 result
2459 }
2460
2461 async fn request_from_single_peer(
2462 &self,
2463 hash: &Hash,
2464 peer_id: &str,
2465 request_htl: u8,
2466 quote_id: Option<u64>,
2467 ) -> Option<Vec<u8>> {
2468 let hash_key = hash_to_key(hash);
2469 let (tx, rx) = oneshot::channel();
2470 self.pending_requests.write().await.insert(
2471 hash_key.clone(),
2472 PendingRequest {
2473 response_tx: tx,
2474 started_at: Instant::now(),
2475 queried_peers: vec![peer_id.to_string()],
2476 },
2477 );
2478
2479 let mut rx = rx;
2480 if !self
2481 .send_request_to_peer(peer_id, hash, request_htl, quote_id)
2482 .await
2483 {
2484 let _ = self.pending_requests.write().await.remove(&hash_key);
2485 return None;
2486 }
2487 self.reserve_peer_request(peer_id).await;
2488
2489 if let Ok(Ok(Some(data))) = tokio::time::timeout(self.request_timeout, &mut rx).await {
2490 if hashtree_core::sha256(&data) == *hash {
2491 let _ = self.local_store.put(*hash, data.clone()).await;
2492 return Some(data);
2493 }
2494 }
2495
2496 if let Some(pending) = self.pending_requests.write().await.remove(&hash_key) {
2497 self.release_queried_peer_requests(&pending.queried_peers)
2498 .await;
2499 for peer_id in pending.queried_peers {
2500 self.peer_selector.write().await.record_timeout(&peer_id);
2501 }
2502 }
2503 let _ = self.take_forward_requesters(&hash_key).await;
2504 None
2505 }
2506
2507 async fn request_from_ordered_peers(
2508 &self,
2509 hash: &Hash,
2510 ordered_peer_ids: &[String],
2511 request_htl: u8,
2512 ) -> RouteFetchOutcome {
2513 let hash_key = hash_to_key(hash);
2514 let (tx, rx) = oneshot::channel();
2515 self.pending_requests.write().await.insert(
2516 hash_key.clone(),
2517 PendingRequest {
2518 response_tx: tx,
2519 started_at: Instant::now(),
2520 queried_peers: Vec::new(),
2521 },
2522 );
2523
2524 let rx = Arc::new(Mutex::new(rx));
2525 let result = run_hedged_waves(
2526 ordered_peer_ids.len(),
2527 self.routing.dispatch,
2528 self.request_timeout,
2529 |range| {
2530 let wave_peer_ids = ordered_peer_ids[range].to_vec();
2531 let hash = *hash;
2532 let hash_key = hash_key.clone();
2533 async move {
2534 let mut sent = 0usize;
2535 for peer_id in wave_peer_ids {
2536 if self
2537 .send_request_to_peer(&peer_id, &hash, request_htl, None)
2538 .await
2539 {
2540 sent += 1;
2541 self.reserve_peer_request(&peer_id).await;
2542 if let Some(pending) =
2543 self.pending_requests.write().await.get_mut(&hash_key)
2544 {
2545 pending.queried_peers.push(peer_id);
2546 }
2547 }
2548 }
2549 sent
2550 }
2551 },
2552 |wait| {
2553 let rx = rx.clone();
2554 async move {
2555 let mut rx = rx.lock().await;
2556 match tokio::time::timeout(wait, &mut *rx).await {
2557 Ok(Ok(Some(data))) if hashtree_core::sha256(&data) == *hash => {
2558 HedgedWaveAction::Success(data)
2559 }
2560 Ok(Ok(Some(_))) => HedgedWaveAction::Continue,
2561 Ok(Ok(None)) | Ok(Err(_)) => HedgedWaveAction::Abort,
2562 Err(_) => HedgedWaveAction::Continue,
2563 }
2564 }
2565 },
2566 )
2567 .await;
2568
2569 let Some(data) = result else {
2570 if let Some(pending) = self.pending_requests.write().await.remove(&hash_key) {
2571 self.release_queried_peer_requests(&pending.queried_peers)
2572 .await;
2573 for peer_id in pending.queried_peers {
2574 self.peer_selector.write().await.record_timeout(&peer_id);
2575 }
2576 }
2577 let _ = self.take_forward_requesters(&hash_key).await;
2578 return RouteFetchOutcome::Timeout;
2579 };
2580
2581 let _ = self.local_store.put(*hash, data.clone()).await;
2582 RouteFetchOutcome::Hit(data)
2583 }
2584
2585 async fn request_from_read_sources_inner(&self, hash: &Hash) -> RouteFetchOutcome {
2586 let ordered_sources = self.ordered_read_sources().await;
2587 if ordered_sources.is_empty() {
2588 return RouteFetchOutcome::Miss;
2589 }
2590
2591 let dispatch = normalize_dispatch_config(
2592 self.source_dispatch_for(ordered_sources.len()).await,
2593 ordered_sources.len(),
2594 );
2595 let wave_plan = build_hedged_wave_plan(ordered_sources.len(), dispatch);
2596 if wave_plan.is_empty() {
2597 return RouteFetchOutcome::Miss;
2598 }
2599
2600 let deadline = Instant::now() + self.request_timeout;
2601 let mut pending = FuturesUnordered::new();
2602 let mut pending_source_ids = HashSet::new();
2603 let mut saw_timeout = false;
2604 let mut next_source_idx = 0usize;
2605
2606 for (wave_idx, wave_size) in wave_plan.iter().copied().enumerate() {
2607 let from = next_source_idx;
2608 let to = (next_source_idx + wave_size).min(ordered_sources.len());
2609 next_source_idx = to;
2610
2611 for source in &ordered_sources[from..to] {
2612 let source = Arc::clone(source);
2613 let source_id = source.id().to_string();
2614 self.record_read_source_request(&source_id).await;
2615 pending_source_ids.insert(source_id.clone());
2616 let hash = *hash;
2617 pending.push(tokio::spawn(async move {
2618 let started_at = Instant::now();
2619 let result = std::panic::AssertUnwindSafe(source.get(&hash))
2620 .catch_unwind()
2621 .await;
2622 match result {
2623 Ok(Some(data)) => SourceFetchOutcome::Hit {
2624 source_id,
2625 data,
2626 elapsed_ms: started_at.elapsed().as_millis().max(1) as u64,
2627 },
2628 Ok(None) => SourceFetchOutcome::Miss { source_id },
2629 Err(_) => SourceFetchOutcome::Failure { source_id },
2630 }
2631 }));
2632 }
2633
2634 let is_last_wave =
2635 wave_idx + 1 == wave_plan.len() || next_source_idx >= ordered_sources.len();
2636 let window_end = if is_last_wave {
2637 deadline
2638 } else {
2639 (Instant::now() + Duration::from_millis(dispatch.hedge_interval_ms)).min(deadline)
2640 };
2641
2642 while Instant::now() < window_end {
2643 let remaining = window_end.saturating_duration_since(Instant::now());
2644 let Some(result) = tokio::time::timeout(remaining, pending.next())
2645 .await
2646 .ok()
2647 .flatten()
2648 else {
2649 break;
2650 };
2651 let Ok(outcome) = result else {
2652 continue;
2653 };
2654 match outcome {
2655 SourceFetchOutcome::Hit {
2656 source_id,
2657 data,
2658 elapsed_ms,
2659 } => {
2660 pending_source_ids.remove(&source_id);
2661 self.record_read_source_success(&source_id, elapsed_ms)
2662 .await;
2663 return RouteFetchOutcome::Hit(data);
2664 }
2665 SourceFetchOutcome::Miss { source_id } => {
2666 pending_source_ids.remove(&source_id);
2667 self.record_read_source_miss(&source_id).await;
2668 }
2669 SourceFetchOutcome::Failure { source_id } => {
2670 pending_source_ids.remove(&source_id);
2671 self.record_read_source_failure(&source_id).await;
2672 }
2673 }
2674 }
2675
2676 if Instant::now() >= deadline {
2677 break;
2678 }
2679 }
2680
2681 for source_id in pending_source_ids {
2682 saw_timeout = true;
2683 self.record_read_source_timeout(&source_id).await;
2684 }
2685 if saw_timeout {
2686 RouteFetchOutcome::Timeout
2687 } else {
2688 RouteFetchOutcome::Miss
2689 }
2690 }
2691
2692 async fn request_from_read_sources(&self, hash: &Hash) -> RouteFetchOutcome {
2693 let hash_key = hash_to_key(hash);
2694 let existing_wait = {
2695 let mut inflight = self.inflight_source_fetches.lock().await;
2696 if let Some(existing) = inflight.get_mut(&hash_key) {
2697 let (tx, rx) = oneshot::channel();
2698 existing.waiters.push(tx);
2699 Some(rx)
2700 } else {
2701 inflight.insert(
2702 hash_key.clone(),
2703 InflightSourceFetch {
2704 waiters: Vec::new(),
2705 },
2706 );
2707 None
2708 }
2709 };
2710
2711 if let Some(wait) = existing_wait {
2712 return wait.await.unwrap_or(RouteFetchOutcome::Timeout);
2713 }
2714
2715 let result = self.request_from_read_sources_inner(hash).await;
2716 if let RouteFetchOutcome::Hit(hit) = &result {
2717 let _ = self.local_store.put(*hash, hit.clone()).await;
2718 }
2719 self.complete_inflight_source_fetch(&hash_key, result.clone())
2720 .await;
2721
2722 result
2723 }
2724
2725 async fn complete_inflight_source_fetch(&self, hash_key: &str, result: RouteFetchOutcome) {
2726 let waiters = self
2727 .inflight_source_fetches
2728 .lock()
2729 .await
2730 .remove(hash_key)
2731 .map(|inflight| inflight.waiters)
2732 .unwrap_or_default();
2733 for waiter in waiters {
2734 let _ = waiter.send(result.clone());
2735 }
2736 }
2737
2738 async fn cancel_pending_peer_route(&self, hash: &Hash) {
2739 let hash_key = hash_to_key(hash);
2740 if let Some(pending) = self.pending_requests.write().await.remove(&hash_key) {
2741 self.release_queried_peer_requests(&pending.queried_peers)
2742 .await;
2743 }
2744 }
2745
2746 async fn cancel_losing_route(&self, hash: &Hash, route: &ReadRoute, winner_data: &[u8]) {
2747 match route {
2748 ReadRoute::Peers(_) => self.cancel_pending_peer_route(hash).await,
2749 ReadRoute::Sources => {
2750 let hash_key = hash_to_key(hash);
2751 self.complete_inflight_source_fetch(
2752 &hash_key,
2753 RouteFetchOutcome::Hit(winner_data.to_vec()),
2754 )
2755 .await;
2756 }
2757 }
2758 }
2759
2760 async fn ranked_read_routes(&self, context: &MeshReadContext) -> Vec<RankedReadRoute> {
2761 let mut routes = Vec::new();
2762 let ordered_peers = if should_forward_htl(context.request_htl) {
2763 self.ordered_connected_peers(context.exclude_peer_id.as_deref())
2764 .await
2765 } else {
2766 Vec::new()
2767 };
2768 if !ordered_peers.is_empty() {
2769 let best_peer_id = ordered_peers[0].clone();
2770 let selector = self.peer_selector.read().await;
2771 let best_peer = selector.get_stats(&best_peer_id).cloned();
2772 let now = Instant::now();
2773 let (score, has_history) = match best_peer.as_ref() {
2774 Some(stats) => (
2775 peer_endpoint_score(stats, now),
2776 peer_endpoint_has_history(stats),
2777 ),
2778 None => (0.0, false),
2779 };
2780 routes.push(RankedReadRoute {
2781 route: ReadRoute::Peers(ordered_peers),
2782 best_endpoint_id: format!("peer:{best_peer_id}"),
2783 score,
2784 has_history,
2785 });
2786 }
2787 let ordered_sources = self.ordered_read_sources().await;
2788 if let Some(best_source) = ordered_sources.first() {
2789 let stats = self.read_source_stats.read().await;
2790 let best_source_stats = stats.get(best_source.id()).cloned().unwrap_or_default();
2791 let now = Instant::now();
2792 routes.push(RankedReadRoute {
2793 route: ReadRoute::Sources,
2794 best_endpoint_id: format!("source:{}", best_source.id()),
2795 score: adaptive_source_score(&best_source_stats, now),
2796 has_history: source_has_history(&best_source_stats),
2797 });
2798 }
2799 if routes.len() <= 1 {
2800 return routes;
2801 }
2802
2803 routes.sort_by(|left, right| {
2804 right
2805 .score
2806 .partial_cmp(&left.score)
2807 .unwrap_or(std::cmp::Ordering::Equal)
2808 .then_with(|| ranked_route_kind(&left.route).cmp(&ranked_route_kind(&right.route)))
2809 .then_with(|| left.best_endpoint_id.cmp(&right.best_endpoint_id))
2810 .then_with(|| left.route.id().cmp(right.route.id()))
2811 });
2812 routes
2813 }
2814
2815 fn should_probe_multiple_routes(&self, routes: &[RankedReadRoute]) -> bool {
2816 if routes.len() <= 1 {
2817 return false;
2818 }
2819 if !routes[0].has_history || !routes[1].has_history {
2820 return false;
2821 }
2822 (routes[0].score - routes[1].score) < SOURCE_SCORE_TIE_DELTA
2823 }
2824
2825 async fn run_read_route(
2826 &self,
2827 hash: &Hash,
2828 route: &ReadRoute,
2829 context: &MeshReadContext,
2830 ) -> RouteFetchOutcome {
2831 match route {
2832 ReadRoute::Peers(peer_ids) => {
2833 self.request_from_ordered_peers(hash, peer_ids, context.request_htl)
2834 .await
2835 }
2836 ReadRoute::Sources => self.request_from_read_sources(hash).await,
2837 }
2838 }
2839
2840 async fn request_from_mesh_with_context(
2841 &self,
2842 hash: &Hash,
2843 context: &MeshReadContext,
2844 ) -> Option<Vec<u8>> {
2845 let routes = self.ranked_read_routes(context).await;
2846 match routes.as_slice() {
2847 [] => None,
2848 [ranked] => match self.run_read_route(hash, &ranked.route, context).await {
2849 RouteFetchOutcome::Hit(data) => Some(data),
2850 RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => None,
2851 },
2852 [first, second, ..] => {
2853 if self.should_probe_multiple_routes(&routes) {
2854 let first_fut = self.run_read_route(hash, &first.route, context);
2855 let second_fut = self.run_read_route(hash, &second.route, context);
2856 tokio::pin!(first_fut);
2857 tokio::pin!(second_fut);
2858 let mut first_done = false;
2859 let mut second_done = false;
2860 loop {
2861 tokio::select! {
2862 result = &mut first_fut, if !first_done => {
2863 first_done = true;
2864 if let RouteFetchOutcome::Hit(data) = result {
2865 if !second_done {
2866 self.cancel_losing_route(hash, &second.route, &data).await;
2867 }
2868 return Some(data);
2869 }
2870 }
2871 result = &mut second_fut, if !second_done => {
2872 second_done = true;
2873 if let RouteFetchOutcome::Hit(data) = result {
2874 if !first_done {
2875 self.cancel_losing_route(hash, &first.route, &data).await;
2876 }
2877 return Some(data);
2878 }
2879 }
2880 else => break,
2881 }
2882 if first_done && second_done {
2883 break;
2884 }
2885 }
2886 None
2887 } else {
2888 match self.run_read_route(hash, &first.route, context).await {
2889 RouteFetchOutcome::Hit(data) => return Some(data),
2890 RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => {}
2891 }
2892 for ranked in routes.iter().skip(1) {
2893 match self.run_read_route(hash, &ranked.route, context).await {
2894 RouteFetchOutcome::Hit(data) => return Some(data),
2895 RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => {}
2896 }
2897 }
2898 None
2899 }
2900 }
2901 }
2902 }
2903
2904 async fn request_from_mesh(&self, hash: &Hash) -> Option<Vec<u8>> {
2905 self.request_from_mesh_with_context(hash, &MeshReadContext::default())
2906 .await
2907 }
2908
2909 async fn begin_forward_request(&self, hash_key: &str, requester_id: &str) -> bool {
2910 let mut pending = self.pending_forward_requests.write().await;
2911 if let Some(existing) = pending.get_mut(hash_key) {
2912 existing.requester_ids.insert(requester_id.to_string());
2913 return false;
2914 }
2915
2916 let mut requester_ids = HashSet::new();
2917 requester_ids.insert(requester_id.to_string());
2918 pending.insert(
2919 hash_key.to_string(),
2920 PendingForwardRequest { requester_ids },
2921 );
2922 true
2923 }
2924
2925 async fn was_recent_forward_miss(&self, hash_key: &str) -> bool {
2926 self.recent_forward_misses.lock().await.contains(hash_key)
2927 }
2928
2929 async fn mark_recent_forward_miss(&self, hash_key: &str) {
2930 let _ = self
2931 .recent_forward_misses
2932 .lock()
2933 .await
2934 .insert_if_new(hash_key.to_string());
2935 }
2936
2937 async fn take_forward_requesters(&self, hash_key: &str) -> Vec<String> {
2938 self.pending_forward_requests
2939 .write()
2940 .await
2941 .remove(hash_key)
2942 .map(|pending| pending.requester_ids.into_iter().collect())
2943 .unwrap_or_default()
2944 }
2945
2946 async fn complete_pending_response(
2947 self: &Arc<Self>,
2948 from_peer: &str,
2949 hash: &Hash,
2950 hash_key: String,
2951 payload: Vec<u8>,
2952 ) {
2953 if let Some(pending) = self.pending_requests.write().await.remove(&hash_key) {
2954 self.release_queried_peer_requests(&pending.queried_peers)
2955 .await;
2956 let rtt_ms = pending.started_at.elapsed().as_millis() as u64;
2957 self.peer_selector.write().await.record_success(
2958 from_peer,
2959 rtt_ms,
2960 payload.len() as u64,
2961 );
2962 let forward_requesters = self.take_forward_requesters(&hash_key).await;
2963 let response_bytes = if forward_requesters.is_empty() {
2964 None
2965 } else {
2966 Some(encode_response(&create_response(hash, payload.clone())))
2967 };
2968 let _ = pending.response_tx.send(Some(payload));
2969 if let Some(response_bytes) = response_bytes {
2970 for requester_id in forward_requesters {
2971 Arc::clone(self)
2972 .enqueue_response_send(requester_id, response_bytes.clone(), Instant::now())
2973 .await;
2974 }
2975 }
2976 }
2977 }
2978
2979 async fn handle_quote_response_message(&self, from_peer: &str, res: DataQuoteResponse) {
2980 if !res.a {
2981 return;
2982 }
2983
2984 let Some(quote_id) = res.q else {
2985 return;
2986 };
2987
2988 let hash_key = hash_to_key(&res.h);
2989 let (preferred_mint_url, offered_payment_sat) = {
2990 let pending_quotes = self.pending_quotes.read().await;
2991 let Some(pending) = pending_quotes.get(&hash_key) else {
2992 return;
2993 };
2994 (
2995 pending.preferred_mint_url.clone(),
2996 pending.offered_payment_sat,
2997 )
2998 };
2999 if !self
3000 .should_accept_quote_response(
3001 from_peer,
3002 preferred_mint_url.as_deref(),
3003 offered_payment_sat,
3004 &res,
3005 )
3006 .await
3007 {
3008 return;
3009 }
3010 let mut pending_quotes = self.pending_quotes.write().await;
3011 if let Some(pending) = pending_quotes.remove(&hash_key) {
3012 let _ = pending.response_tx.send(Some(NegotiatedQuote {
3013 peer_id: from_peer.to_string(),
3014 quote_id,
3015 mint_url: res.m,
3016 }));
3017 }
3018 }
3019
3020 async fn handle_response_message(
3021 self: &Arc<Self>,
3022 from_peer: &str,
3023 res: crate::protocol::DataResponse,
3024 ) {
3025 let hash_key = hash_to_key(&res.h);
3026 let hash = match crate::protocol::bytes_to_hash(&res.h) {
3027 Some(h) => h,
3028 None => return,
3029 };
3030
3031 if hashtree_core::sha256(&res.d) != hash {
3033 self.peer_selector.write().await.record_failure(from_peer);
3034 if self.debug {
3035 println!("[MeshStoreCore] Ignoring invalid response payload for {hash_key}");
3036 }
3037 return;
3038 }
3039
3040 self.record_useful_bytes_received_from_peer(from_peer, res.d.len() as u64)
3041 .await;
3042 self.complete_pending_response(from_peer, &hash, hash_key, res.d)
3043 .await;
3044 }
3045
3046 async fn handle_quote_request_message(&self, from_peer: &str, req: DataQuoteRequest) {
3047 let hash = match crate::protocol::bytes_to_hash(&req.h) {
3048 Some(h) => h,
3049 None => return,
3050 };
3051 let hash_key = hash_to_key(&hash);
3052
3053 {
3054 let selector = self.peer_selector.read().await;
3055 if self.should_refuse_requests_from_peer(&selector, from_peer) {
3056 if self.debug {
3057 println!(
3058 "[MeshStoreCore] Refusing quote request from delinquent peer {}",
3059 from_peer
3060 );
3061 }
3062 return;
3063 }
3064 }
3065
3066 let chosen_mint = self.choose_quote_mint(req.m.as_deref());
3067 let can_serve = self.local_store.has(&hash).await.ok().unwrap_or(false)
3068 && !self.should_drop_response(&hash)
3069 && !self.should_corrupt_response(&hash);
3070
3071 let res = if can_serve {
3072 let quote_id = self
3073 .issue_quote(from_peer, &hash_key, req.p, req.t, chosen_mint.as_deref())
3074 .await;
3075 create_quote_response_available(&hash, quote_id, req.p, req.t, chosen_mint.as_deref())
3076 } else {
3077 create_quote_response_unavailable(&hash)
3078 };
3079 let response_bytes = encode_quote_response(&res);
3080 if let Some(channel) = self.signaling.get_channel(from_peer).await {
3081 if channel.send(response_bytes.clone()).await.is_ok() {
3082 self.record_peer_wire_sent(from_peer, response_bytes.len() as u64)
3083 .await;
3084 }
3085 }
3086 }
3087
3088 async fn handle_request_message(
3089 self: &Arc<Self>,
3090 from_peer: &str,
3091 req: crate::protocol::DataRequest,
3092 ) {
3093 let hash = match crate::protocol::bytes_to_hash(&req.h) {
3094 Some(h) => h,
3095 None => return,
3096 };
3097 let hash_key = hash_to_key(&hash);
3098
3099 if let Some(quote_id) = req.q {
3100 if !self.take_valid_quote(from_peer, &hash_key, quote_id).await {
3101 if self.debug {
3102 println!(
3103 "[MeshStoreCore] Refusing request with invalid or expired quote {} from {}",
3104 quote_id, from_peer
3105 );
3106 }
3107 return;
3108 }
3109 }
3110
3111 let allow_peer_forwarding = {
3112 let selector = self.peer_selector.read().await;
3113 !self.should_refuse_requests_from_peer(&selector, from_peer)
3114 };
3115
3116 if let Ok(Some(mut data)) = self.local_store.get(&hash).await {
3118 if self.should_drop_response(&hash) {
3119 if self.debug {
3120 println!(
3121 "[MeshStoreCore] Dropping response for {} due to actor profile",
3122 hash_to_key(&hash)
3123 );
3124 }
3125 return;
3126 }
3127
3128 let response_delay = self.response_send_delay(&hash, data.len());
3129 if self.should_corrupt_response(&hash) {
3130 if data.is_empty() {
3131 data.push(0x80);
3132 } else {
3133 data[0] ^= 0x80;
3134 }
3135 }
3136
3137 let res = create_response(&hash, data);
3139 let response_bytes = encode_response(&res);
3140 let ready_at = Instant::now() + response_delay;
3141 Arc::clone(self)
3142 .enqueue_response_send(from_peer.to_string(), response_bytes, ready_at)
3143 .await;
3144 return;
3145 }
3146
3147 if self.pending_requests.read().await.contains_key(&hash_key) {
3148 let _ = self.begin_forward_request(&hash_key, from_peer).await;
3149 return;
3150 }
3151
3152 if self.was_recent_forward_miss(&hash_key).await {
3153 if self.debug {
3154 println!(
3155 "[MeshStoreCore] Suppressing recently missed forwarded request for {}",
3156 hash_key
3157 );
3158 }
3159 return;
3160 }
3161
3162 if !self.begin_forward_request(&hash_key, from_peer).await {
3163 return;
3164 }
3165
3166 let from_peer = from_peer.to_string();
3167 let this = Arc::clone(self);
3168 let request_htl = req.htl;
3169 tokio::spawn(async move {
3170 let result = if allow_peer_forwarding {
3171 let context = MeshReadContext {
3172 exclude_peer_id: Some(from_peer.clone()),
3173 request_htl,
3174 };
3175 this.request_from_mesh_with_context(&hash, &context).await
3176 } else {
3177 if this.debug {
3178 println!(
3179 "[MeshStoreCore] Serving request from delinquent peer {} via read sources only",
3180 from_peer
3181 );
3182 }
3183 match this.request_from_read_sources(&hash).await {
3184 RouteFetchOutcome::Hit(data) => Some(data),
3185 RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => None,
3186 }
3187 };
3188 let requester_ids = this.take_forward_requesters(&hash_key).await;
3189 if let Some(data) = result {
3190 let ready_at = Instant::now() + this.response_send_delay(&hash, data.len());
3191 let res = create_response(&hash, data);
3192 let response_bytes = encode_response(&res);
3193 for requester_id in requester_ids {
3194 Arc::clone(&this)
3195 .enqueue_response_send(requester_id, response_bytes.clone(), ready_at)
3196 .await;
3197 }
3198 } else {
3199 this.mark_recent_forward_miss(&hash_key).await;
3200 }
3201 });
3202 }
3203
3204 async fn handle_pubsub_interest_message(
3205 self: &Arc<Self>,
3206 from_peer: &str,
3207 mut interest: PubsubInterest,
3208 ) {
3209 if !self.apply_pubsub_interest_route(from_peer, &interest).await {
3210 return;
3211 }
3212
3213 if interest.htl <= 1 {
3214 return;
3215 }
3216 interest.htl = interest.htl.saturating_sub(1);
3217 let _ = self
3218 .send_pubsub_interest_to_peers(&interest, Some(from_peer))
3219 .await;
3220 }
3221
3222 async fn handle_pubsub_frame_message(
3223 self: &Arc<Self>,
3224 from_peer: &str,
3225 mut frame: PubsubFrame,
3226 wire_bytes: usize,
3227 ) {
3228 if frame.stream_id.is_empty() || frame.origin_peer_id.is_empty() {
3229 return;
3230 }
3231 if frame.origin_peer_id == self.signaling.peer_id() {
3232 return;
3233 }
3234
3235 let frame_key = Self::pubsub_frame_key(&frame);
3236 if !self
3237 .pubsub_seen_frames
3238 .lock()
3239 .await
3240 .insert_if_new(frame_key.clone())
3241 {
3242 return;
3243 }
3244 self.cache_pubsub_frame(frame_key.clone(), frame.clone())
3245 .await;
3246
3247 let local_interested = self
3248 .pubsub_local_interests
3249 .read()
3250 .await
3251 .contains(&frame.stream_id);
3252 let mut downstream_peers = if frame.htl > 1 {
3253 match self.routing.pubsub_delivery_mode {
3254 PubsubDeliveryMode::InterestPush => {
3255 let mut peers = self
3256 .interested_pubsub_peers(&frame.stream_id, Some(from_peer))
3257 .await;
3258 peers.extend(
3259 self.take_pubsub_want_peers(&frame_key, Some(from_peer))
3260 .await,
3261 );
3262 peers.sort();
3263 peers.dedup();
3264 peers
3265 }
3266 PubsubDeliveryMode::HtlInvWant => {
3267 self.take_pubsub_want_peers(&frame_key, Some(from_peer))
3268 .await
3269 }
3270 }
3271 } else {
3272 Vec::new()
3273 };
3274 downstream_peers.retain(|peer_id| peer_id != from_peer);
3275
3276 if local_interested || !downstream_peers.is_empty() {
3277 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3278 .await;
3279 }
3280
3281 if local_interested {
3282 self.enqueue_pubsub_event(PubsubEvent {
3283 stream_id: frame.stream_id.clone(),
3284 seq: frame.seq,
3285 origin_peer_id: frame.origin_peer_id.clone(),
3286 from_peer_id: from_peer.to_string(),
3287 payload: frame.payload.clone(),
3288 })
3289 .await;
3290 }
3291
3292 if downstream_peers.is_empty() {
3293 return;
3294 }
3295
3296 frame.htl = frame.htl.saturating_sub(1);
3297 let _ = self
3298 .send_pubsub_frame_to_peers(&frame, &downstream_peers)
3299 .await;
3300 }
3301
3302 async fn handle_pubsub_inventory_message(
3303 self: &Arc<Self>,
3304 from_peer: &str,
3305 inv: PubsubInventory,
3306 wire_bytes: usize,
3307 ) {
3308 if inv.stream_id.is_empty() || inv.origin_peer_id.is_empty() {
3309 return;
3310 }
3311 if inv.origin_peer_id == self.signaling.peer_id() {
3312 return;
3313 }
3314
3315 let key = Self::pubsub_key(&inv.origin_peer_id, &inv.stream_id, inv.seq);
3316 if !self
3317 .pubsub_seen_inventories
3318 .lock()
3319 .await
3320 .insert_if_new(key.clone())
3321 {
3322 return;
3323 }
3324 {
3325 let mut routes = self.pubsub_inventory_routes.write().await;
3326 routes
3327 .entry(key.clone())
3328 .or_insert_with(|| from_peer.to_string());
3329 }
3330
3331 let local_interested = self
3332 .pubsub_local_interests
3333 .read()
3334 .await
3335 .contains(&inv.stream_id);
3336 let downstream_peers = self
3337 .interested_pubsub_peers(&inv.stream_id, Some(from_peer))
3338 .await;
3339 if local_interested || !downstream_peers.is_empty() {
3340 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3341 .await;
3342 let want =
3343 create_pubsub_want(inv.stream_id.clone(), inv.seq, inv.origin_peer_id.clone());
3344 let _ = self.send_pubsub_want_upstream(&key, &want, None).await;
3345 }
3346
3347 if !should_forward_htl(inv.htl) {
3348 return;
3349 }
3350 let _ = self.flood_pubsub_inventory(&inv, Some(from_peer)).await;
3351 }
3352
3353 async fn handle_pubsub_want_message(
3354 self: &Arc<Self>,
3355 from_peer: &str,
3356 want: PubsubWant,
3357 wire_bytes: usize,
3358 ) {
3359 if want.stream_id.is_empty() || want.origin_peer_id.is_empty() {
3360 return;
3361 }
3362 if want.origin_peer_id == from_peer {
3363 return;
3364 }
3365
3366 let key = Self::pubsub_key(&want.origin_peer_id, &want.stream_id, want.seq);
3367 let want_key = format!("{from_peer}:{key}");
3368 if !self.pubsub_seen_wants.lock().await.insert_if_new(want_key) {
3369 return;
3370 }
3371
3372 if let Some(frame) = self.cached_pubsub_frame(&key).await {
3373 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3374 .await;
3375 let peers = vec![from_peer.to_string()];
3376 let _ = self.send_pubsub_frame_to_peers(&frame, &peers).await;
3377 return;
3378 }
3379
3380 let has_upstream_route = self.pubsub_inventory_routes.read().await.contains_key(&key);
3381 if !has_upstream_route {
3382 return;
3383 }
3384
3385 if self.remember_pubsub_want_peer(key.clone(), from_peer).await {
3386 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
3387 .await;
3388 }
3389 let _ = self
3390 .send_pubsub_want_upstream(&key, &want, Some(from_peer))
3391 .await;
3392 }
3393
3394 pub async fn handle_data_message(self: &Arc<Self>, from_peer: &str, data: &[u8]) {
3396 self.record_peer_wire_received(from_peer, data.len() as u64)
3397 .await;
3398 let parsed = match parse_message(data) {
3399 Some(m) => m,
3400 None => return,
3401 };
3402
3403 match parsed {
3404 DataMessage::Request(req) => {
3405 self.handle_request_message(from_peer, req).await;
3406 }
3407 DataMessage::Response(res) => {
3408 self.handle_response_message(from_peer, res).await;
3409 }
3410 DataMessage::QuoteRequest(req) => {
3411 self.handle_quote_request_message(from_peer, req).await;
3412 }
3413 DataMessage::QuoteResponse(res) => {
3414 self.handle_quote_response_message(from_peer, res).await;
3415 }
3416 DataMessage::PubsubInterest(interest) => {
3417 self.handle_pubsub_interest_message(from_peer, interest)
3418 .await;
3419 }
3420 DataMessage::PubsubFrame(frame) => {
3421 self.handle_pubsub_frame_message(from_peer, frame, data.len())
3422 .await;
3423 }
3424 DataMessage::PubsubInventory(inv) => {
3425 self.handle_pubsub_inventory_message(from_peer, inv, data.len())
3426 .await;
3427 }
3428 DataMessage::PubsubWant(want) => {
3429 self.handle_pubsub_want_message(from_peer, want, data.len())
3430 .await;
3431 }
3432 DataMessage::Payment(_)
3433 | DataMessage::PaymentAck(_)
3434 | DataMessage::Chunk(_)
3435 | DataMessage::PeerHints(_) => {}
3436 }
3437 }
3438}
3439
3440#[async_trait]
3441impl<S, R, F> Store for MeshStoreCore<S, R, F>
3442where
3443 S: Store + Send + Sync + 'static,
3444 R: SignalingTransport + Send + Sync + 'static,
3445 F: PeerLinkFactory + Send + Sync + 'static,
3446{
3447 async fn put(&self, hash: Hash, data: Vec<u8>) -> Result<bool, StoreError> {
3448 self.local_store.put(hash, data).await
3449 }
3450
3451 async fn get(&self, hash: &Hash) -> Result<Option<Vec<u8>>, StoreError> {
3452 if let Some(data) = self.local_store.get(hash).await? {
3454 return Ok(Some(data));
3455 }
3456
3457 Ok(self.request_from_mesh(hash).await)
3459 }
3460
3461 async fn has(&self, hash: &Hash) -> Result<bool, StoreError> {
3462 self.local_store.has(hash).await
3463 }
3464
3465 async fn delete(&self, hash: &Hash) -> Result<bool, StoreError> {
3466 self.local_store.delete(hash).await
3467 }
3468}
3469
3470#[cfg(test)]
3471mod tests;
3472
3473pub type SimMeshStore<S> =
3475 MeshStoreCore<S, crate::mock::MockRelayTransport, crate::mock::MockConnectionFactory>;