1use async_trait::async_trait;
8use std::collections::hash_map::DefaultHasher;
9use std::collections::{HashMap, HashSet, VecDeque};
10use std::future::Future;
11use std::hash::{Hash as _, Hasher};
12use std::ops::Range;
13use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
14use std::sync::Arc;
15use std::time::Duration;
16use tokio::sync::{oneshot, Mutex, Notify, RwLock};
17use tokio::time::Instant;
18
19use hashtree_core::{
20 BlobReply, BlobRequest, BlobRoute, BlobRouteContext, Hash, Store, StoreError, BLOB_MAX_BYTES,
21};
22
23use crate::peer_selector::{PeerMetadataSnapshot, PeerSelector, SelectionStrategy};
24use crate::protocol::{
25 create_pubsub_frame, create_pubsub_interest, create_pubsub_inventory, create_pubsub_want,
26 create_quote_request, create_quote_response_available, create_quote_response_unavailable,
27 create_request, create_request_with_quote, create_response, encode_pubsub_frame,
28 encode_pubsub_interest, encode_pubsub_inventory, encode_pubsub_want, encode_quote_request,
29 encode_quote_response, encode_request, encode_response, hash_to_key, parse_message,
30 DataMessage, DataQuoteRequest, DataQuoteResponse, PubsubFrame, PubsubInterest, PubsubInventory,
31 PubsubWant,
32};
33use crate::pubsub_strategy::{
34 reciprocal_virtual_finish, select_reciprocal_outbound_job, OutboundJobCandidate,
35 PeerTrafficSnapshot, PubsubCandidate, PubsubSchedulerConfig,
36};
37use crate::signaling::MeshRouter;
38use crate::transport::{PeerLinkFactory, SignalingTransport, TransportError};
39use crate::types::{
40 should_forward_htl, PeerHTLConfig, SignalingMessage, TimedSeenSet, MAX_HTL, MESH_EVENT_POLICY,
41};
42
43const PEER_METADATA_POINTER_SLOT_KEY: &[u8] = b"hashtree-mesh/peer-metadata/latest/v1";
46const PUBSUB_SEEN_CAPACITY: usize = 16_384;
47const PUBSUB_INBOX_CAPACITY: usize = 4_096;
48const PUBSUB_FRAME_CACHE_CAPACITY: usize = 4_096;
49const VERIFIED_BLOCK_DELIVERY_CAPACITY: usize = 4_096;
50const PUBSUB_SEEN_TTL: Duration = Duration::from_secs(120);
51
52struct PendingRequest {
54 owner: Arc<()>,
55 response_tx: oneshot::Sender<Option<Vec<u8>>>,
56 started_at: Instant,
57 queried_peers: Vec<String>,
58}
59
60#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
61struct PendingRequestKey {
62 hash: Hash,
63 htl: u8,
64}
65
66impl PendingRequestKey {
67 fn new(hash: Hash, htl: u8) -> Self {
68 Self { hash, htl }
69 }
70}
71
72struct PendingQuoteRequest {
73 response_tx: oneshot::Sender<Option<NegotiatedQuote>>,
74 preferred_mint_url: Option<String>,
75 offered_payment_sat: u64,
76}
77
78struct PendingForwardRequest {
79 requester_ids: HashSet<String>,
80}
81
82type PeerWireStats = PeerTrafficSnapshot;
83
84struct PendingResponseSend {
85 job_id: u64,
86 peer_id: String,
87 bytes: Vec<u8>,
88 ready_at: Instant,
89 queue_sequence: u64,
90}
91
92#[derive(Debug, Clone)]
93struct NegotiatedQuote {
94 peer_id: String,
95 quote_id: u64,
96 #[allow(dead_code)]
97 mint_url: Option<String>,
98}
99
100struct IssuedQuote {
101 expires_at: Instant,
102 #[allow(dead_code)]
103 payment_sat: u64,
104 #[allow(dead_code)]
105 mint_url: Option<String>,
106}
107
108#[derive(Debug, Clone)]
109enum RouteFetchOutcome {
110 Hit(Vec<u8>),
111 Miss,
112 Timeout,
113}
114
115const ACTIVE_PEER_REQUEST_RANK_PENALTY: usize = 3;
116
117#[derive(Debug, Clone)]
118struct MeshReadContext {
119 exclude_peer_id: Option<String>,
120 request_htl: u8,
121 deadline: Option<Instant>,
122 attempt_budget: Option<usize>,
123}
124
125impl Default for MeshReadContext {
126 fn default() -> Self {
127 Self {
128 exclude_peer_id: None,
129 request_htl: MAX_HTL,
130 deadline: None,
131 attempt_budget: None,
132 }
133 }
134}
135
136#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
138pub struct DataPumpStats {
139 pub processed: usize,
140 pub request_messages: usize,
141 pub response_messages: usize,
142 pub quote_request_messages: u64,
143 pub quote_response_messages: u64,
144 pub pubsub_interest_messages: u64,
145 pub pubsub_frame_messages: u64,
146 pub pubsub_inventory_messages: u64,
147 pub pubsub_want_messages: u64,
148 pub processed_bytes: u64,
149}
150
151#[derive(Debug, Clone, PartialEq, Eq)]
153pub struct PubsubEvent {
154 pub stream_id: String,
155 pub seq: u64,
156 pub origin_peer_id: String,
157 pub from_peer_id: String,
158 pub payload: Vec<u8>,
159}
160
161#[derive(Debug, Clone, PartialEq, Eq)]
163pub struct VerifiedBlockDelivery {
164 pub hash: Hash,
165 pub provider_peer_id: String,
166 pub payload_bytes: u64,
167}
168
169#[derive(Debug, Clone, Default, PartialEq, Eq)]
174pub struct VerifiedBlockDeliveryBatch {
175 pub deliveries: Vec<VerifiedBlockDelivery>,
176 pub dropped_since_last_drain: u64,
177}
178
179#[derive(Default)]
180struct VerifiedBlockDeliveryBuffer {
181 deliveries: VecDeque<VerifiedBlockDelivery>,
182 dropped_since_last_drain: u64,
183}
184
185#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
187pub struct PubsubPublishStats {
188 pub selected_peers: usize,
189 pub sent_peers: usize,
190 pub sent_bytes: u64,
191 pub deferred_peers: usize,
192}
193
194#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
196pub enum PubsubDeliveryMode {
197 InterestPush,
199 #[default]
201 HtlInvWant,
202}
203
204#[derive(Debug, Clone, Copy)]
210pub struct RequestDispatchConfig {
211 pub initial_fanout: usize,
213 pub hedge_fanout: usize,
215 pub max_fanout: usize,
217 pub hedge_interval_ms: u64,
219}
220
221impl Default for RequestDispatchConfig {
222 fn default() -> Self {
223 Self {
224 initial_fanout: 2,
225 hedge_fanout: 1,
226 max_fanout: 4,
227 hedge_interval_ms: 50,
228 }
229 }
230}
231
232pub fn normalize_dispatch_config(
234 dispatch: RequestDispatchConfig,
235 available_peers: usize,
236) -> RequestDispatchConfig {
237 let mut cfg = dispatch;
238 let cap = if cfg.max_fanout == 0 {
239 available_peers
240 } else {
241 cfg.max_fanout.min(available_peers)
242 };
243 cfg.max_fanout = cap;
244 cfg.initial_fanout = if cfg.initial_fanout == 0 {
245 1
246 } else {
247 cfg.initial_fanout.min(cap.max(1))
248 };
249 cfg.hedge_fanout = if cfg.hedge_fanout == 0 {
250 1
251 } else {
252 cfg.hedge_fanout.min(cap.max(1))
253 };
254 cfg
255}
256
257pub fn build_hedged_wave_plan(peer_count: usize, dispatch: RequestDispatchConfig) -> Vec<usize> {
259 if peer_count == 0 {
260 return Vec::new();
261 }
262 let cap = dispatch.max_fanout.min(peer_count);
263 if cap == 0 {
264 return Vec::new();
265 }
266
267 let mut plan = Vec::new();
268 let mut sent = 0usize;
269 let first = dispatch.initial_fanout.min(cap).max(1);
270 plan.push(first);
271 sent += first;
272
273 while sent < cap {
274 let next = dispatch.hedge_fanout.min(cap - sent).max(1);
275 plan.push(next);
276 sent += next;
277 }
278 plan
279}
280
281#[derive(Debug)]
283pub enum HedgedWaveAction<T> {
284 Continue,
285 Success(T),
286 Abort,
287}
288
289pub async fn run_hedged_waves<T, SendWave, SendWaveFut, WaitWave, WaitWaveFut>(
294 peer_count: usize,
295 dispatch: RequestDispatchConfig,
296 request_timeout: Duration,
297 mut send_wave: SendWave,
298 mut wait_wave: WaitWave,
299) -> Option<T>
300where
301 SendWave: FnMut(Range<usize>) -> SendWaveFut,
302 SendWaveFut: Future<Output = usize>,
303 WaitWave: FnMut(Duration) -> WaitWaveFut,
304 WaitWaveFut: Future<Output = HedgedWaveAction<T>>,
305{
306 let dispatch = normalize_dispatch_config(dispatch, peer_count);
307 let wave_plan = build_hedged_wave_plan(peer_count, dispatch);
308 if wave_plan.is_empty() {
309 return None;
310 }
311
312 let deadline = Instant::now() + request_timeout;
313 let mut sent_total = 0usize;
314 let mut next_peer_idx = 0usize;
315
316 for (wave_idx, wave_size) in wave_plan.iter().copied().enumerate() {
317 let from = next_peer_idx;
318 let to = (next_peer_idx + wave_size).min(peer_count);
319 next_peer_idx = to;
320
321 if from == to {
322 continue;
323 }
324
325 sent_total += send_wave(from..to).await;
326 if sent_total == 0 {
327 if next_peer_idx >= peer_count {
328 break;
329 }
330 continue;
331 }
332
333 let now = Instant::now();
334 if now >= deadline {
335 break;
336 }
337 let remaining = deadline.saturating_duration_since(now);
338 let is_last_wave = wave_idx + 1 == wave_plan.len() || next_peer_idx >= peer_count;
339 let wait = if is_last_wave {
340 remaining
341 } else if dispatch.hedge_interval_ms == 0 {
342 Duration::ZERO
343 } else {
344 Duration::from_millis(dispatch.hedge_interval_ms).min(remaining)
345 };
346
347 if wait.is_zero() {
348 continue;
349 }
350
351 match wait_wave(wait).await {
352 HedgedWaveAction::Continue => {}
353 HedgedWaveAction::Success(value) => return Some(value),
354 HedgedWaveAction::Abort => break,
355 }
356 }
357
358 None
359}
360
361pub async fn sync_selector_peers(selector: &RwLock<PeerSelector>, current_peer_ids: &[String]) {
363 let mut selector = selector.write().await;
364 let current: HashSet<&str> = current_peer_ids.iter().map(String::as_str).collect();
365 let known: Vec<String> = selector.all_stats().map(|s| s.peer_id.clone()).collect();
366 for peer_id in known {
367 if !current.contains(peer_id.as_str()) {
368 selector.remove_peer(&peer_id);
369 }
370 }
371 for peer_id in current_peer_ids {
372 selector.add_peer(peer_id.clone());
373 }
374}
375
376#[derive(Debug, Clone, Copy)]
380pub struct ResponseBehaviorConfig {
381 pub drop_response_prob: f64,
383 pub corrupt_response_prob: f64,
385 pub extra_delay_ms: u64,
387 pub first_byte_delay_ms: u64,
389 pub bytes_per_second: u64,
391 pub stall_response_prob: f64,
393 pub stall_delay_ms: u64,
395}
396
397impl Default for ResponseBehaviorConfig {
398 fn default() -> Self {
399 Self {
400 drop_response_prob: 0.0,
401 corrupt_response_prob: 0.0,
402 extra_delay_ms: 0,
403 first_byte_delay_ms: 0,
404 bytes_per_second: 0,
405 stall_response_prob: 0.0,
406 stall_delay_ms: 0,
407 }
408 }
409}
410
411impl ResponseBehaviorConfig {
412 fn normalized(self) -> Self {
413 Self {
414 drop_response_prob: self.drop_response_prob.clamp(0.0, 1.0),
415 corrupt_response_prob: self.corrupt_response_prob.clamp(0.0, 1.0),
416 extra_delay_ms: self.extra_delay_ms,
417 first_byte_delay_ms: self.first_byte_delay_ms,
418 bytes_per_second: self.bytes_per_second,
419 stall_response_prob: self.stall_response_prob.clamp(0.0, 1.0),
420 stall_delay_ms: self.stall_delay_ms,
421 }
422 }
423}
424
425#[derive(Debug, Clone)]
427pub struct MeshRoutingConfig {
428 pub selection_strategy: SelectionStrategy,
429 pub fairness_enabled: bool,
430 pub cashu_payment_weight: f64,
432 pub cashu_payment_default_block_threshold: u64,
435 pub cashu_accepted_mints: Vec<String>,
437 pub cashu_default_mint: Option<String>,
439 pub cashu_peer_suggested_mint_base_cap_sat: u64,
441 pub cashu_peer_suggested_mint_success_step_sat: u64,
443 pub cashu_peer_suggested_mint_receipt_step_sat: u64,
445 pub cashu_peer_suggested_mint_max_cap_sat: u64,
447 pub dispatch: RequestDispatchConfig,
448 pub response_behavior: ResponseBehaviorConfig,
449 pub pubsub_scheduler: PubsubSchedulerConfig,
450 pub pubsub_delivery_mode: PubsubDeliveryMode,
451 pub pubsub_forwarding: bool,
453 pub pubsub_max_htl: u8,
455}
456
457impl Default for MeshRoutingConfig {
458 fn default() -> Self {
459 Self {
460 selection_strategy: SelectionStrategy::Weighted,
461 fairness_enabled: true,
462 cashu_payment_weight: 0.0,
463 cashu_payment_default_block_threshold: 0,
464 cashu_accepted_mints: Vec::new(),
465 cashu_default_mint: None,
466 cashu_peer_suggested_mint_base_cap_sat: 0,
467 cashu_peer_suggested_mint_success_step_sat: 0,
468 cashu_peer_suggested_mint_receipt_step_sat: 0,
469 cashu_peer_suggested_mint_max_cap_sat: 0,
470 dispatch: RequestDispatchConfig::default(),
471 response_behavior: ResponseBehaviorConfig::default(),
472 pubsub_scheduler: PubsubSchedulerConfig::default(),
473 pubsub_delivery_mode: PubsubDeliveryMode::HtlInvWant,
474 pubsub_forwarding: true,
475 pubsub_max_htl: MESH_EVENT_POLICY.max_htl,
476 }
477 }
478}
479
480impl MeshRoutingConfig {
481 fn pubsub_initial_htl(&self) -> u8 {
482 self.pubsub_max_htl.clamp(1, MAX_HTL)
483 }
484}
485
486pub struct MeshStoreCore<S, R, F>
493where
494 S: Store + Send + Sync + 'static,
495 R: SignalingTransport + Send + Sync + 'static,
496 F: PeerLinkFactory + Send + Sync + 'static,
497{
498 local_store: Arc<S>,
500 signaling: Arc<MeshRouter<R, F>>,
502 htl_configs: RwLock<HashMap<String, PeerHTLConfig>>,
504 pending_requests: RwLock<HashMap<PendingRequestKey, Vec<PendingRequest>>>,
506 pending_quotes: RwLock<HashMap<String, PendingQuoteRequest>>,
508 pending_forward_requests: RwLock<HashMap<PendingRequestKey, PendingForwardRequest>>,
510 issued_quotes: RwLock<HashMap<(String, String, u64), IssuedQuote>>,
513 next_quote_id: RwLock<u64>,
515 peer_selector: RwLock<PeerSelector>,
517 peer_active_requests: RwLock<HashMap<String, usize>>,
519 peer_wire_stats: RwLock<HashMap<String, PeerWireStats>>,
521 pubsub_local_interests: RwLock<HashSet<String>>,
523 pubsub_local_interest_versions: RwLock<HashMap<String, u64>>,
525 pubsub_peer_interests: RwLock<HashMap<String, HashSet<String>>>,
527 pubsub_interest_routes: RwLock<HashMap<(String, String), String>>,
529 pubsub_interest_versions: RwLock<HashMap<(String, String), u64>>,
531 pubsub_seen_interests: Mutex<TimedSeenSet>,
533 pubsub_seen_frames: Mutex<TimedSeenSet>,
535 pubsub_seen_inventories: Mutex<TimedSeenSet>,
537 pubsub_seen_wants: Mutex<TimedSeenSet>,
539 pubsub_inventory_routes: RwLock<HashMap<String, String>>,
541 pubsub_want_routes: RwLock<HashMap<String, HashSet<String>>>,
543 pubsub_upstream_wants: Mutex<TimedSeenSet>,
545 pubsub_frame_cache: Mutex<VecDeque<(String, PubsubFrame)>>,
547 pubsub_inbox: Mutex<VecDeque<PubsubEvent>>,
549 verified_block_deliveries: Mutex<VerifiedBlockDeliveryBuffer>,
551 pubsub_notify: Notify,
553 pubsub_deferred_counts: RwLock<HashMap<(String, String), u64>>,
555 next_pubsub_interest_seq: AtomicU64,
557 pending_response_sends: Mutex<Vec<PendingResponseSend>>,
559 response_scheduler_running: AtomicBool,
561 next_response_job_id: AtomicU64,
563 routing: MeshRoutingConfig,
565 request_timeout: Duration,
567 debug: bool,
569 running: RwLock<bool>,
571}
572
573impl<S, R, F> MeshStoreCore<S, R, F>
574where
575 S: Store + Send + Sync + 'static,
576 R: SignalingTransport + Send + Sync + 'static,
577 F: PeerLinkFactory + Send + Sync + 'static,
578{
579 pub fn new(
581 local_store: Arc<S>,
582 signaling: Arc<MeshRouter<R, F>>,
583 request_timeout: Duration,
584 debug: bool,
585 ) -> Self {
586 Self::new_with_routing(
587 local_store,
588 signaling,
589 request_timeout,
590 debug,
591 Default::default(),
592 )
593 }
594
595 pub fn new_with_routing(
597 local_store: Arc<S>,
598 signaling: Arc<MeshRouter<R, F>>,
599 request_timeout: Duration,
600 debug: bool,
601 routing: MeshRoutingConfig,
602 ) -> Self {
603 let mut selector = PeerSelector::with_strategy(routing.selection_strategy);
604 selector.set_fairness(routing.fairness_enabled);
605 selector.set_cashu_payment_weight(routing.cashu_payment_weight);
606 Self {
607 local_store,
608 signaling,
609 htl_configs: RwLock::new(HashMap::new()),
610 pending_requests: RwLock::new(HashMap::new()),
611 pending_quotes: RwLock::new(HashMap::new()),
612 pending_forward_requests: RwLock::new(HashMap::new()),
613 issued_quotes: RwLock::new(HashMap::new()),
614 next_quote_id: RwLock::new(1),
615 peer_selector: RwLock::new(selector),
616 peer_active_requests: RwLock::new(HashMap::new()),
617 peer_wire_stats: RwLock::new(HashMap::new()),
618 pubsub_local_interests: RwLock::new(HashSet::new()),
619 pubsub_local_interest_versions: RwLock::new(HashMap::new()),
620 pubsub_peer_interests: RwLock::new(HashMap::new()),
621 pubsub_interest_routes: RwLock::new(HashMap::new()),
622 pubsub_interest_versions: RwLock::new(HashMap::new()),
623 pubsub_seen_interests: Mutex::new(TimedSeenSet::new(
624 PUBSUB_SEEN_CAPACITY,
625 PUBSUB_SEEN_TTL,
626 )),
627 pubsub_seen_frames: Mutex::new(TimedSeenSet::new(
628 PUBSUB_SEEN_CAPACITY,
629 PUBSUB_SEEN_TTL,
630 )),
631 pubsub_seen_inventories: Mutex::new(TimedSeenSet::new(
632 PUBSUB_SEEN_CAPACITY,
633 PUBSUB_SEEN_TTL,
634 )),
635 pubsub_seen_wants: Mutex::new(TimedSeenSet::new(PUBSUB_SEEN_CAPACITY, PUBSUB_SEEN_TTL)),
636 pubsub_inventory_routes: RwLock::new(HashMap::new()),
637 pubsub_want_routes: RwLock::new(HashMap::new()),
638 pubsub_upstream_wants: Mutex::new(TimedSeenSet::new(
639 PUBSUB_SEEN_CAPACITY,
640 PUBSUB_SEEN_TTL,
641 )),
642 pubsub_frame_cache: Mutex::new(VecDeque::new()),
643 pubsub_inbox: Mutex::new(VecDeque::new()),
644 verified_block_deliveries: Mutex::new(VerifiedBlockDeliveryBuffer::default()),
645 pubsub_notify: Notify::new(),
646 pubsub_deferred_counts: RwLock::new(HashMap::new()),
647 next_pubsub_interest_seq: AtomicU64::new(1),
648 pending_response_sends: Mutex::new(Vec::new()),
649 response_scheduler_running: AtomicBool::new(false),
650 next_response_job_id: AtomicU64::new(1),
651 routing,
652 request_timeout,
653 debug,
654 running: RwLock::new(false),
655 }
656 }
657
658 pub async fn start(&self) -> Result<(), TransportError> {
660 *self.running.write().await = true;
661
662 self.signaling.send_hello(vec![]).await?;
664
665 Ok(())
666 }
667
668 pub async fn stop(&self) {
670 *self.running.write().await = false;
671 }
672
673 pub async fn process_signaling(&self, msg: SignalingMessage) -> Result<(), TransportError> {
675 let peer_id = msg.peer_id().to_string();
677 {
678 let mut configs = self.htl_configs.write().await;
679 if !configs.contains_key(&peer_id) {
680 configs.insert(peer_id.clone(), PeerHTLConfig::random());
681 }
682 }
683 self.peer_selector.write().await.add_peer(peer_id.clone());
684
685 let result = self.signaling.handle_message(msg).await;
686 if result.is_ok() {
687 self.announce_pubsub_interests_to_peer(&peer_id).await;
688 }
689 result
690 }
691
692 pub fn signaling(&self) -> &Arc<MeshRouter<R, F>> {
694 &self.signaling
695 }
696
697 fn response_behavior(&self) -> ResponseBehaviorConfig {
698 self.routing.response_behavior.normalized()
699 }
700
701 async fn record_peer_wire_sent(&self, peer_id: &str, bytes: u64) {
702 if bytes == 0 {
703 return;
704 }
705 let mut stats = self.peer_wire_stats.write().await;
706 let entry = stats.entry(peer_id.to_string()).or_default();
707 entry.bytes_sent = entry.bytes_sent.saturating_add(bytes);
708 }
709
710 async fn record_peer_wire_received(&self, peer_id: &str, bytes: u64) {
711 if bytes == 0 {
712 return;
713 }
714 let mut stats = self.peer_wire_stats.write().await;
715 let entry = stats.entry(peer_id.to_string()).or_default();
716 entry.bytes_received = entry.bytes_received.saturating_add(bytes);
717 }
718
719 pub async fn record_useful_bytes_received_from_peer(&self, peer_id: &str, bytes: u64) {
724 if bytes == 0 {
725 return;
726 }
727 let mut stats = self.peer_wire_stats.write().await;
728 let entry = stats.entry(peer_id.to_string()).or_default();
729 entry.useful_bytes_received = entry.useful_bytes_received.saturating_add(bytes);
730 }
731
732 pub async fn peer_traffic_snapshot(&self, peer_id: &str) -> PeerTrafficSnapshot {
734 self.peer_wire_stats
735 .read()
736 .await
737 .get(peer_id)
738 .copied()
739 .unwrap_or_default()
740 }
741
742 pub async fn peer_traffic_snapshots(&self) -> HashMap<String, PeerTrafficSnapshot> {
744 self.peer_wire_stats.read().await.clone()
745 }
746
747 fn pubsub_key(origin_peer_id: &str, stream_id: &str, seq: u64) -> String {
748 format!("{origin_peer_id}:{stream_id}:{seq}")
749 }
750
751 fn pubsub_frame_key(frame: &PubsubFrame) -> String {
752 Self::pubsub_key(&frame.origin_peer_id, &frame.stream_id, frame.seq)
753 }
754
755 fn pubsub_interest_key(interest: &PubsubInterest) -> String {
756 format!(
757 "{}:{}:{}:{}",
758 interest.subscriber_peer_id, interest.stream_id, interest.seq, interest.active
759 )
760 }
761
762 fn next_pubsub_interest_seq(&self) -> u64 {
763 self.next_pubsub_interest_seq
764 .fetch_add(1, Ordering::Relaxed)
765 }
766
767 async fn record_peer_pubsub_wire_sent(&self, peer_id: &str, bytes: u64, bandwidth_debt: f64) {
768 if bytes == 0 {
769 return;
770 }
771 let mut stats = self.peer_wire_stats.write().await;
772 let entry = stats.entry(peer_id.to_string()).or_default();
773 entry.bytes_sent = entry.bytes_sent.saturating_add(bytes);
774 entry.bandwidth_debt = bandwidth_debt;
775 }
776
777 async fn send_pubsub_interest_to_peers(
778 &self,
779 interest: &PubsubInterest,
780 exclude_peer_id: Option<&str>,
781 ) -> PubsubPublishStats {
782 if !should_forward_htl(interest.htl) {
783 return PubsubPublishStats::default();
784 }
785
786 let mut peer_ids = self.signaling.peer_ids().await;
787 peer_ids.sort();
788 peer_ids.retain(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude));
789
790 let bytes = encode_pubsub_interest(interest);
791 let mut stats = PubsubPublishStats {
792 selected_peers: peer_ids.len(),
793 ..Default::default()
794 };
795 for peer_id in peer_ids {
796 let Some(channel) = self.signaling.get_channel(&peer_id).await else {
797 continue;
798 };
799 if channel.send(bytes.clone()).await.is_ok() {
800 stats.sent_peers += 1;
801 stats.sent_bytes = stats.sent_bytes.saturating_add(bytes.len() as u64);
802 self.record_peer_wire_sent(&peer_id, bytes.len() as u64)
803 .await;
804 }
805 }
806 stats
807 }
808
809 async fn announce_pubsub_interests_to_peer(&self, peer_id: &str) {
810 let mut interests = self
811 .pubsub_local_interests
812 .read()
813 .await
814 .iter()
815 .cloned()
816 .collect::<Vec<_>>();
817 interests.sort();
818 if interests.is_empty() {
819 return;
820 }
821
822 let interests = {
823 let versions = self.pubsub_local_interest_versions.read().await;
824 interests
825 .into_iter()
826 .filter_map(|stream_id| {
827 versions
828 .get(&stream_id)
829 .copied()
830 .map(|seq| (stream_id, seq))
831 })
832 .collect::<Vec<_>>()
833 };
834
835 for (stream_id, seq) in interests {
836 let interest = create_pubsub_interest(
837 stream_id,
838 self.signaling.peer_id().to_string(),
839 seq,
840 true,
841 MAX_HTL,
842 );
843 let Some(channel) = self.signaling.get_channel(peer_id).await else {
844 continue;
845 };
846 let bytes = encode_pubsub_interest(&interest);
847 if channel.send(bytes.clone()).await.is_ok() {
848 self.record_peer_wire_sent(peer_id, bytes.len() as u64)
849 .await;
850 }
851 }
852 }
853
854 fn remove_pubsub_peer_interest(
855 peer_interests: &mut HashMap<String, HashSet<String>>,
856 routes: &HashMap<(String, String), String>,
857 stream_id: &str,
858 peer_id: &str,
859 ) {
860 let still_has_route = routes
861 .iter()
862 .any(|((stream, _subscriber), peer)| stream == stream_id && peer == peer_id);
863 if still_has_route {
864 return;
865 }
866 if let Some(peers) = peer_interests.get_mut(stream_id) {
867 peers.remove(peer_id);
868 if peers.is_empty() {
869 peer_interests.remove(stream_id);
870 }
871 }
872 }
873
874 async fn apply_pubsub_interest_route(
875 &self,
876 from_peer: &str,
877 interest: &PubsubInterest,
878 ) -> bool {
879 if interest.stream_id.is_empty() || interest.subscriber_peer_id.is_empty() {
880 return false;
881 }
882 if interest.subscriber_peer_id == self.signaling.peer_id() {
883 return false;
884 }
885
886 let interest_key = Self::pubsub_interest_key(interest);
887 if !self
888 .pubsub_seen_interests
889 .lock()
890 .await
891 .insert_if_new(interest_key)
892 {
893 return false;
894 }
895
896 let route_key = (
897 interest.stream_id.clone(),
898 interest.subscriber_peer_id.clone(),
899 );
900 {
901 let mut versions = self.pubsub_interest_versions.write().await;
902 if versions
903 .get(&route_key)
904 .is_some_and(|latest| *latest >= interest.seq)
905 {
906 return false;
907 }
908 versions.insert(route_key.clone(), interest.seq);
909 }
910
911 let mut peer_interests = self.pubsub_peer_interests.write().await;
912 let mut routes = self.pubsub_interest_routes.write().await;
913 if interest.active {
914 if let Some(previous_peer) = routes.insert(route_key, from_peer.to_string()) {
915 if previous_peer != from_peer {
916 Self::remove_pubsub_peer_interest(
917 &mut peer_interests,
918 &routes,
919 &interest.stream_id,
920 &previous_peer,
921 );
922 }
923 }
924 peer_interests
925 .entry(interest.stream_id.clone())
926 .or_default()
927 .insert(from_peer.to_string());
928 } else if let Some(previous_peer) = routes.remove(&route_key) {
929 Self::remove_pubsub_peer_interest(
930 &mut peer_interests,
931 &routes,
932 &interest.stream_id,
933 &previous_peer,
934 );
935 } else {
936 Self::remove_pubsub_peer_interest(
937 &mut peer_interests,
938 &routes,
939 &interest.stream_id,
940 from_peer,
941 );
942 }
943
944 true
945 }
946
947 async fn interested_pubsub_peers(
948 &self,
949 stream_id: &str,
950 exclude_peer_id: Option<&str>,
951 ) -> Vec<String> {
952 let connected = self
953 .signaling
954 .peer_ids()
955 .await
956 .into_iter()
957 .collect::<HashSet<_>>();
958 let mut peers = self
959 .pubsub_peer_interests
960 .read()
961 .await
962 .get(stream_id)
963 .map(|peers| peers.iter().cloned().collect::<Vec<_>>())
964 .unwrap_or_default();
965 peers.retain(|peer_id| {
966 connected.contains(peer_id) && exclude_peer_id.is_none_or(|exclude| peer_id != exclude)
967 });
968 peers.sort();
969 peers
970 }
971
972 async fn decrement_pubsub_htl_for_peer(&self, peer_id: &str, htl: u8) -> u8 {
973 let htl_config = {
974 let configs = self.htl_configs.read().await;
975 configs
976 .get(peer_id)
977 .cloned()
978 .unwrap_or_else(PeerHTLConfig::random)
979 };
980 htl_config.decrement_with_policy(htl, &MESH_EVENT_POLICY)
981 }
982
983 async fn send_pubsub_inventory_to_peers(
984 &self,
985 inv: &PubsubInventory,
986 peer_ids: &[String],
987 ) -> PubsubPublishStats {
988 if peer_ids.is_empty() || !should_forward_htl(inv.htl) {
989 return PubsubPublishStats::default();
990 }
991
992 let mut stats = PubsubPublishStats {
993 selected_peers: peer_ids.len(),
994 ..Default::default()
995 };
996 for peer_id in peer_ids {
997 let send_htl = self.decrement_pubsub_htl_for_peer(peer_id, inv.htl).await;
998 if !should_forward_htl(send_htl) {
999 continue;
1000 }
1001 let Some(channel) = self.signaling.get_channel(peer_id).await else {
1002 continue;
1003 };
1004 let mut outgoing = inv.clone();
1005 outgoing.htl = send_htl;
1006 let bytes = encode_pubsub_inventory(&outgoing);
1007 let message_bytes = bytes.len() as u64;
1008 if channel.send(bytes).await.is_ok() {
1009 stats.sent_peers += 1;
1010 stats.sent_bytes = stats.sent_bytes.saturating_add(message_bytes);
1011 self.record_peer_wire_sent(peer_id, message_bytes).await;
1012 }
1013 }
1014 stats
1015 }
1016
1017 async fn send_pubsub_want_to_peer(&self, want: &PubsubWant, peer_id: &str) -> bool {
1018 let Some(channel) = self.signaling.get_channel(peer_id).await else {
1019 return false;
1020 };
1021 let bytes = encode_pubsub_want(want);
1022 let message_bytes = bytes.len() as u64;
1023 match channel.send(bytes).await {
1024 Ok(()) => {
1025 self.record_peer_wire_sent(peer_id, message_bytes).await;
1026 true
1027 }
1028 Err(_) => false,
1029 }
1030 }
1031
1032 async fn send_pubsub_want_upstream(
1033 &self,
1034 key: &str,
1035 want: &PubsubWant,
1036 exclude_peer_id: Option<&str>,
1037 ) -> bool {
1038 let upstream = {
1039 let routes = self.pubsub_inventory_routes.read().await;
1040 routes.get(key).cloned()
1041 };
1042 let Some(upstream) = upstream else {
1043 return false;
1044 };
1045 if exclude_peer_id.is_some_and(|exclude| exclude == upstream) {
1046 return false;
1047 }
1048 let want_key = format!("{key}:{upstream}");
1049 if !self
1050 .pubsub_upstream_wants
1051 .lock()
1052 .await
1053 .insert_if_new(want_key)
1054 {
1055 return false;
1056 }
1057 self.send_pubsub_want_to_peer(want, &upstream).await
1058 }
1059
1060 async fn cache_pubsub_frame(&self, key: String, frame: PubsubFrame) {
1061 let mut cache = self.pubsub_frame_cache.lock().await;
1062 if let Some(index) = cache.iter().position(|(cached_key, _)| cached_key == &key) {
1063 cache.remove(index);
1064 }
1065 cache.push_back((key, frame));
1066 while cache.len() > PUBSUB_FRAME_CACHE_CAPACITY {
1067 cache.pop_front();
1068 }
1069 }
1070
1071 async fn cached_pubsub_frame(&self, key: &str) -> Option<PubsubFrame> {
1072 self.pubsub_frame_cache
1073 .lock()
1074 .await
1075 .iter()
1076 .find_map(|(cached_key, frame)| {
1077 if cached_key == key {
1078 Some(frame.clone())
1079 } else {
1080 None
1081 }
1082 })
1083 }
1084
1085 async fn remember_pubsub_want_peer(&self, key: String, from_peer: &str) -> bool {
1086 let mut routes = self.pubsub_want_routes.write().await;
1087 routes.entry(key).or_default().insert(from_peer.to_string())
1088 }
1089
1090 async fn take_pubsub_want_peers(
1091 &self,
1092 key: &str,
1093 exclude_peer_id: Option<&str>,
1094 ) -> Vec<String> {
1095 let connected = self
1096 .signaling
1097 .peer_ids()
1098 .await
1099 .into_iter()
1100 .collect::<HashSet<_>>();
1101 let mut peers = self
1102 .pubsub_want_routes
1103 .write()
1104 .await
1105 .remove(key)
1106 .map(|peers| peers.into_iter().collect::<Vec<_>>())
1107 .unwrap_or_default();
1108 peers.retain(|peer_id| {
1109 connected.contains(peer_id) && exclude_peer_id.is_none_or(|exclude| peer_id != exclude)
1110 });
1111 peers.sort();
1112 peers
1113 }
1114
1115 async fn select_pubsub_peers(
1116 &self,
1117 stream_id: &str,
1118 seq: u64,
1119 message_bytes: u64,
1120 peer_ids: &[String],
1121 ) -> (Vec<String>, Vec<String>) {
1122 let traffic = self.peer_wire_stats.read().await;
1123 let deferred_counts = self.pubsub_deferred_counts.read().await;
1124 let candidates = peer_ids
1125 .iter()
1126 .map(|peer_id| PubsubCandidate {
1127 peer_id: peer_id.clone(),
1128 traffic: traffic.get(peer_id).copied().unwrap_or_default(),
1129 deferred_count: deferred_counts
1130 .get(&(stream_id.to_string(), peer_id.clone()))
1131 .copied()
1132 .unwrap_or_default(),
1133 })
1134 .collect::<Vec<_>>();
1135 drop(deferred_counts);
1136 drop(traffic);
1137
1138 let selection = self.routing.pubsub_scheduler.select(
1139 stream_id,
1140 seq,
1141 self.signaling.peer_id(),
1142 message_bytes,
1143 &candidates,
1144 );
1145
1146 {
1147 let mut deferred_counts = self.pubsub_deferred_counts.write().await;
1148 for peer_id in &selection.deferred {
1149 *deferred_counts
1150 .entry((stream_id.to_string(), peer_id.clone()))
1151 .or_insert(0) += 1;
1152 }
1153 for peer_id in &selection.selected {
1154 deferred_counts.remove(&(stream_id.to_string(), peer_id.clone()));
1155 }
1156 }
1157
1158 (selection.selected, selection.deferred)
1159 }
1160
1161 async fn send_pubsub_frame_to_peers(
1162 &self,
1163 frame: &PubsubFrame,
1164 peer_ids: &[String],
1165 ) -> PubsubPublishStats {
1166 if peer_ids.is_empty() || !should_forward_htl(frame.htl) {
1167 return PubsubPublishStats::default();
1168 }
1169
1170 let bytes = encode_pubsub_frame(frame);
1171 let message_bytes = bytes.len() as u64;
1172 let (selected, deferred) = self
1173 .select_pubsub_peers(&frame.stream_id, frame.seq, message_bytes, peer_ids)
1174 .await;
1175 let mut stats = PubsubPublishStats {
1176 selected_peers: selected.len(),
1177 deferred_peers: deferred.len(),
1178 ..Default::default()
1179 };
1180
1181 for peer_id in selected {
1182 let Some(channel) = self.signaling.get_channel(&peer_id).await else {
1183 continue;
1184 };
1185 let snapshot = self.peer_traffic_snapshot(&peer_id).await;
1186 let bandwidth_debt = reciprocal_virtual_finish(snapshot, message_bytes);
1187 if channel.send(bytes.clone()).await.is_ok() {
1188 stats.sent_peers += 1;
1189 stats.sent_bytes = stats.sent_bytes.saturating_add(message_bytes);
1190 self.record_peer_pubsub_wire_sent(&peer_id, message_bytes, bandwidth_debt)
1191 .await;
1192 }
1193 }
1194
1195 stats
1196 }
1197
1198 async fn enqueue_pubsub_event(&self, event: PubsubEvent) {
1199 let mut inbox = self.pubsub_inbox.lock().await;
1200 inbox.push_back(event);
1201 while inbox.len() > PUBSUB_INBOX_CAPACITY {
1202 inbox.pop_front();
1203 }
1204 self.pubsub_notify.notify_one();
1205 }
1206
1207 pub async fn subscribe_pubsub(
1209 self: &Arc<Self>,
1210 stream_id: impl Into<String>,
1211 ) -> PubsubPublishStats {
1212 let stream_id = stream_id.into();
1213 if stream_id.is_empty() {
1214 return PubsubPublishStats::default();
1215 }
1216 self.pubsub_local_interests
1217 .write()
1218 .await
1219 .insert(stream_id.clone());
1220 let seq = {
1221 let mut versions = self.pubsub_local_interest_versions.write().await;
1222 match versions.get(&stream_id).copied() {
1223 Some(seq) => seq,
1224 None => {
1225 let seq = self.next_pubsub_interest_seq();
1226 versions.insert(stream_id.clone(), seq);
1227 seq
1228 }
1229 }
1230 };
1231 let interest = create_pubsub_interest(
1232 stream_id,
1233 self.signaling.peer_id().to_string(),
1234 seq,
1235 true,
1236 self.routing.pubsub_initial_htl(),
1237 );
1238 self.send_pubsub_interest_to_peers(&interest, None).await
1239 }
1240
1241 pub async fn unsubscribe_pubsub(
1243 self: &Arc<Self>,
1244 stream_id: impl Into<String>,
1245 ) -> PubsubPublishStats {
1246 let stream_id = stream_id.into();
1247 if stream_id.is_empty() {
1248 return PubsubPublishStats::default();
1249 }
1250 self.pubsub_local_interests.write().await.remove(&stream_id);
1251 self.pubsub_local_interest_versions
1252 .write()
1253 .await
1254 .remove(&stream_id);
1255 let interest = create_pubsub_interest(
1256 stream_id,
1257 self.signaling.peer_id().to_string(),
1258 self.next_pubsub_interest_seq(),
1259 false,
1260 self.routing.pubsub_initial_htl(),
1261 );
1262 self.send_pubsub_interest_to_peers(&interest, None).await
1263 }
1264
1265 pub async fn publish_pubsub(
1267 self: &Arc<Self>,
1268 stream_id: impl Into<String>,
1269 seq: u64,
1270 payload: Vec<u8>,
1271 ) -> PubsubPublishStats {
1272 let stream_id = stream_id.into();
1273 if stream_id.is_empty() {
1274 return PubsubPublishStats::default();
1275 }
1276 let payload_bytes = payload.len() as u64;
1277 let frame = create_pubsub_frame(
1278 stream_id.clone(),
1279 seq,
1280 self.signaling.peer_id().to_string(),
1281 payload.clone(),
1282 self.routing.pubsub_initial_htl(),
1283 );
1284 let frame_key = Self::pubsub_frame_key(&frame);
1285 self.pubsub_seen_frames
1286 .lock()
1287 .await
1288 .insert_if_new(frame_key.clone());
1289 self.cache_pubsub_frame(frame_key, frame.clone()).await;
1290
1291 if self
1292 .pubsub_local_interests
1293 .read()
1294 .await
1295 .contains(&stream_id)
1296 {
1297 self.enqueue_pubsub_event(PubsubEvent {
1298 stream_id: stream_id.clone(),
1299 seq,
1300 origin_peer_id: self.signaling.peer_id().to_string(),
1301 from_peer_id: self.signaling.peer_id().to_string(),
1302 payload,
1303 })
1304 .await;
1305 }
1306
1307 match self.routing.pubsub_delivery_mode {
1308 PubsubDeliveryMode::InterestPush => {
1309 let peers = self.interested_pubsub_peers(&stream_id, None).await;
1310 self.send_pubsub_frame_to_peers(&frame, &peers).await
1311 }
1312 PubsubDeliveryMode::HtlInvWant => {
1313 let inv = create_pubsub_inventory(
1314 stream_id,
1315 seq,
1316 self.signaling.peer_id().to_string(),
1317 payload_bytes,
1318 self.routing.pubsub_initial_htl(),
1319 );
1320 let peers = self.interested_pubsub_peers(&inv.stream_id, None).await;
1321 self.send_pubsub_inventory_to_peers(&inv, &peers).await
1322 }
1323 }
1324 }
1325
1326 pub async fn drain_pubsub_events(&self) -> Vec<PubsubEvent> {
1328 self.pubsub_inbox.lock().await.drain(..).collect()
1329 }
1330
1331 pub async fn drain_verified_block_deliveries(&self) -> VerifiedBlockDeliveryBatch {
1333 let mut buffer = self.verified_block_deliveries.lock().await;
1334 VerifiedBlockDeliveryBatch {
1335 deliveries: buffer.deliveries.drain(..).collect(),
1336 dropped_since_last_drain: std::mem::take(&mut buffer.dropped_since_last_drain),
1337 }
1338 }
1339
1340 pub async fn recv_pubsub_event(&self) -> PubsubEvent {
1342 loop {
1343 if let Some(event) = self.pubsub_inbox.lock().await.pop_front() {
1344 return event;
1345 }
1346 self.pubsub_notify.notified().await;
1347 }
1348 }
1349
1350 pub async fn pubsub_interest_peers(&self, stream_id: &str) -> Vec<String> {
1352 self.interested_pubsub_peers(stream_id, None).await
1353 }
1354
1355 fn choose_ready_response_job(
1356 ready_jobs: &[(u64, String, usize, Instant, u64)],
1357 stats: &HashMap<String, PeerWireStats>,
1358 ) -> Option<(u64, f64)> {
1359 let jobs = ready_jobs
1360 .iter()
1361 .map(|job| OutboundJobCandidate {
1362 job_id: job.0,
1363 peer_id: job.1.clone(),
1364 message_bytes: job.2 as u64,
1365 queue_sequence: job.4,
1366 })
1367 .collect::<Vec<_>>();
1368 select_reciprocal_outbound_job(&jobs, |peer_id| {
1369 stats.get(peer_id).copied().unwrap_or_default()
1370 })
1371 .map(|choice| (choice.job_id, choice.virtual_finish))
1372 }
1373
1374 async fn enqueue_response_send(
1375 self: &Arc<Self>,
1376 peer_id: String,
1377 bytes: Vec<u8>,
1378 ready_at: Instant,
1379 ) {
1380 let job_id = self.next_response_job_id.fetch_add(1, Ordering::Relaxed);
1381 {
1382 let mut queue = self.pending_response_sends.lock().await;
1383 queue.push(PendingResponseSend {
1384 job_id,
1385 peer_id,
1386 bytes,
1387 ready_at,
1388 queue_sequence: job_id,
1389 });
1390 }
1391
1392 if self
1393 .response_scheduler_running
1394 .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
1395 .is_ok()
1396 {
1397 let this = Arc::clone(self);
1398 tokio::spawn(async move {
1399 this.run_response_scheduler().await;
1400 });
1401 }
1402 }
1403
1404 async fn run_response_scheduler(self: Arc<Self>) {
1405 loop {
1406 let snapshot = {
1407 let queue = self.pending_response_sends.lock().await;
1408 if queue.is_empty() {
1409 self.response_scheduler_running
1410 .store(false, Ordering::Release);
1411 return;
1412 }
1413 queue
1414 .iter()
1415 .map(|job| {
1416 (
1417 job.job_id,
1418 job.peer_id.clone(),
1419 job.bytes.len(),
1420 job.ready_at,
1421 job.queue_sequence,
1422 )
1423 })
1424 .collect::<Vec<_>>()
1425 };
1426
1427 let now = Instant::now();
1428 let mut earliest_ready_at: Option<Instant> = None;
1429 let mut ready_jobs = Vec::new();
1430 for job in &snapshot {
1431 if job.3 <= now {
1432 ready_jobs.push(job.clone());
1433 } else {
1434 earliest_ready_at = Some(match earliest_ready_at {
1435 Some(current) => current.min(job.3),
1436 None => job.3,
1437 });
1438 }
1439 }
1440
1441 if ready_jobs.is_empty() {
1442 if let Some(ready_at) = earliest_ready_at {
1443 tokio::time::sleep(ready_at.saturating_duration_since(Instant::now())).await;
1444 continue;
1445 }
1446 self.response_scheduler_running
1447 .store(false, Ordering::Release);
1448 return;
1449 }
1450
1451 let (selected_job_id, selected_finish) = {
1452 let stats = self.peer_wire_stats.read().await;
1453 Self::choose_ready_response_job(&ready_jobs, &stats).expect("ready response job")
1454 };
1455
1456 let selected = {
1457 let mut queue = self.pending_response_sends.lock().await;
1458 let Some(index) = queue.iter().position(|job| job.job_id == selected_job_id) else {
1459 continue;
1460 };
1461 queue.swap_remove(index)
1462 };
1463
1464 let sent = if let Some(channel) = self.signaling.get_channel(&selected.peer_id).await {
1465 channel.send(selected.bytes.clone()).await.is_ok()
1466 } else {
1467 false
1468 };
1469
1470 let queued_peers = {
1471 let queue = self.pending_response_sends.lock().await;
1472 queue
1473 .iter()
1474 .map(|job| job.peer_id.clone())
1475 .collect::<HashSet<_>>()
1476 };
1477 let mut stats = self.peer_wire_stats.write().await;
1478 let entry = stats.entry(selected.peer_id.clone()).or_default();
1479 if sent {
1480 entry.bytes_sent = entry.bytes_sent.saturating_add(selected.bytes.len() as u64);
1481 entry.bandwidth_debt = selected_finish;
1482 }
1483 if queued_peers.is_empty() {
1484 for peer_stats in stats.values_mut() {
1485 peer_stats.bandwidth_debt = 0.0;
1486 }
1487 } else {
1488 let floor = queued_peers
1489 .iter()
1490 .filter_map(|peer_id| stats.get(peer_id).map(|peer| peer.bandwidth_debt))
1491 .fold(f64::INFINITY, f64::min);
1492 if floor.is_finite() && floor > 0.0 {
1493 for peer_id in queued_peers {
1494 if let Some(peer_stats) = stats.get_mut(&peer_id) {
1495 peer_stats.bandwidth_debt =
1496 (peer_stats.bandwidth_debt - floor).max(0.0);
1497 }
1498 }
1499 }
1500 }
1501 }
1502 }
1503
1504 fn deterministic_actor_draw_for(peer_id: &str, hash: &Hash, salt: u64) -> f64 {
1505 let mut hasher = DefaultHasher::new();
1506 peer_id.hash(&mut hasher);
1507 hash.hash(&mut hasher);
1508 salt.hash(&mut hasher);
1509 let v = hasher.finish();
1510 (v as f64) / (u64::MAX as f64)
1511 }
1512
1513 fn deterministic_actor_draw(&self, hash: &Hash, salt: u64) -> f64 {
1514 Self::deterministic_actor_draw_for(self.signaling.peer_id(), hash, salt)
1515 }
1516
1517 fn peer_metadata_pointer_slot_hash() -> Hash {
1518 hashtree_core::sha256(PEER_METADATA_POINTER_SLOT_KEY)
1519 }
1520
1521 fn decode_hash_hex(hash_hex: &str) -> Result<Hash, StoreError> {
1522 let bytes = hex::decode(hash_hex)
1523 .map_err(|e| StoreError::Other(format!("Invalid hash hex: {e}")))?;
1524 if bytes.len() != 32 {
1525 return Err(StoreError::Other(format!(
1526 "Invalid hash length {}, expected 32 bytes",
1527 bytes.len()
1528 )));
1529 }
1530 let mut hash = [0u8; 32];
1531 hash.copy_from_slice(&bytes);
1532 Ok(hash)
1533 }
1534
1535 fn should_drop_response(&self, hash: &Hash) -> bool {
1536 let p = self.response_behavior().drop_response_prob;
1537 if p <= 0.0 {
1538 return false;
1539 }
1540 self.deterministic_actor_draw(hash, 0xD0_D0_D0_D0_D0_D0_D0_D0) < p
1541 }
1542
1543 fn should_corrupt_response(&self, hash: &Hash) -> bool {
1544 let p = self.response_behavior().corrupt_response_prob;
1545 if p <= 0.0 {
1546 return false;
1547 }
1548 self.deterministic_actor_draw(hash, 0xC0_C0_C0_C0_C0_C0_C0_C0) < p
1549 }
1550
1551 fn should_stall_response(&self, hash: &Hash) -> bool {
1552 let p = self.response_behavior().stall_response_prob;
1553 if p <= 0.0 {
1554 return false;
1555 }
1556 self.deterministic_actor_draw(hash, 0x5A_11_5A_11_5A_11_5A_11) < p
1557 }
1558
1559 fn response_send_delay(&self, hash: &Hash, payload_len: usize) -> Duration {
1560 let behavior = self.response_behavior();
1561 let mut total_ms = behavior
1562 .extra_delay_ms
1563 .saturating_add(behavior.first_byte_delay_ms);
1564
1565 if behavior.bytes_per_second > 0 && payload_len > 0 {
1566 let throughput_ms = ((payload_len as u128) * 1000)
1567 .div_ceil(behavior.bytes_per_second as u128)
1568 .min(u64::MAX as u128) as u64;
1569 total_ms = total_ms.saturating_add(throughput_ms);
1570 }
1571
1572 if behavior.stall_delay_ms > 0 && self.should_stall_response(hash) {
1573 total_ms = total_ms.saturating_add(behavior.stall_delay_ms);
1574 }
1575
1576 Duration::from_millis(total_ms)
1577 }
1578
1579 async fn ordered_connected_peers(&self, exclude_peer_id: Option<&str>) -> Vec<String> {
1580 let current_peer_ids = self.signaling.peer_ids().await;
1581 if current_peer_ids.is_empty() {
1582 return Vec::new();
1583 }
1584
1585 sync_selector_peers(&self.peer_selector, ¤t_peer_ids).await;
1586 let hash_get_peer_ids: HashSet<String> = self
1587 .signaling
1588 .hash_get_peer_ids()
1589 .await
1590 .into_iter()
1591 .collect();
1592 let mut candidate_peer_ids: Vec<String> = current_peer_ids
1593 .into_iter()
1594 .filter(|peer_id| hash_get_peer_ids.contains(peer_id))
1595 .filter(|peer_id| exclude_peer_id.is_none_or(|exclude| peer_id != exclude))
1596 .collect();
1597 if candidate_peer_ids.is_empty() {
1598 return Vec::new();
1599 }
1600
1601 let current_set: HashSet<&str> = candidate_peer_ids.iter().map(String::as_str).collect();
1602 let mut selector = self.peer_selector.write().await;
1603 let mut selector_order = selector.select_peers();
1604 selector_order.retain(|peer_id| current_set.contains(peer_id.as_str()));
1605 if selector_order.is_empty() {
1606 let mut fallback = candidate_peer_ids;
1607 fallback.sort();
1608 return fallback;
1609 }
1610 let backed_off: HashMap<String, bool> = candidate_peer_ids
1611 .iter()
1612 .map(|peer_id| (peer_id.clone(), selector.is_peer_backed_off(peer_id)))
1613 .collect();
1614 drop(selector);
1615
1616 let rank: HashMap<&str, usize> = selector_order
1617 .iter()
1618 .enumerate()
1619 .map(|(idx, peer_id)| (peer_id.as_str(), idx))
1620 .collect();
1621 let active = self.peer_active_requests.read().await;
1622 candidate_peer_ids.sort_by(|left, right| {
1623 let left_backed_off = backed_off.get(left).copied().unwrap_or(false);
1624 let right_backed_off = backed_off.get(right).copied().unwrap_or(false);
1625 if left_backed_off != right_backed_off {
1626 return if left_backed_off {
1627 std::cmp::Ordering::Greater
1628 } else {
1629 std::cmp::Ordering::Less
1630 };
1631 }
1632 let left_rank = rank.get(left.as_str()).copied().unwrap_or(usize::MAX / 2);
1633 let right_rank = rank.get(right.as_str()).copied().unwrap_or(usize::MAX / 2);
1634 let left_load = active.get(left).copied().unwrap_or(0);
1635 let right_load = active.get(right).copied().unwrap_or(0);
1636 (left_rank + left_load.saturating_mul(ACTIVE_PEER_REQUEST_RANK_PENALTY))
1637 .cmp(&(right_rank + right_load.saturating_mul(ACTIVE_PEER_REQUEST_RANK_PENALTY)))
1638 .then_with(|| left.cmp(right))
1639 });
1640 candidate_peer_ids
1641 }
1642
1643 async fn reserve_peer_request(&self, peer_id: &str) {
1644 let mut active = self.peer_active_requests.write().await;
1645 *active.entry(peer_id.to_string()).or_insert(0) += 1;
1646 }
1647
1648 async fn release_peer_request(&self, peer_id: &str) {
1649 let mut active = self.peer_active_requests.write().await;
1650 let Some(count) = active.get_mut(peer_id) else {
1651 return;
1652 };
1653 if *count <= 1 {
1654 active.remove(peer_id);
1655 } else {
1656 *count -= 1;
1657 }
1658 }
1659
1660 async fn release_queried_peer_requests(&self, peer_ids: &[String]) {
1661 for peer_id in peer_ids {
1662 self.release_peer_request(peer_id).await;
1663 }
1664 }
1665
1666 fn requested_quote_mint(&self) -> Option<&str> {
1667 if let Some(default_mint) = self.routing.cashu_default_mint.as_deref() {
1668 if self.routing.cashu_accepted_mints.is_empty()
1669 || self
1670 .routing
1671 .cashu_accepted_mints
1672 .iter()
1673 .any(|mint| mint == default_mint)
1674 {
1675 return Some(default_mint);
1676 }
1677 }
1678
1679 self.routing
1680 .cashu_accepted_mints
1681 .first()
1682 .map(String::as_str)
1683 }
1684
1685 fn choose_quote_mint(&self, requested_mint: Option<&str>) -> Option<String> {
1686 if let Some(requested_mint) = requested_mint {
1687 if self.accepts_quote_mint(Some(requested_mint)) {
1688 return Some(requested_mint.to_string());
1689 }
1690 }
1691 if let Some(default_mint) = self.routing.cashu_default_mint.as_ref() {
1692 return Some(default_mint.clone());
1693 }
1694 if let Some(first_mint) = self.routing.cashu_accepted_mints.first() {
1695 return Some(first_mint.clone());
1696 }
1697 requested_mint.map(str::to_string)
1698 }
1699
1700 fn accepts_quote_mint(&self, mint_url: Option<&str>) -> bool {
1701 if self.routing.cashu_accepted_mints.is_empty() {
1702 return true;
1703 }
1704
1705 let Some(mint_url) = mint_url else {
1706 return false;
1707 };
1708 self.routing
1709 .cashu_accepted_mints
1710 .iter()
1711 .any(|mint| mint == mint_url)
1712 }
1713
1714 fn trusts_quote_mint(&self, mint_url: Option<&str>) -> bool {
1715 let Some(mint_url) = mint_url else {
1716 return self.routing.cashu_default_mint.is_none()
1717 && self.routing.cashu_accepted_mints.is_empty();
1718 };
1719 self.routing.cashu_default_mint.as_deref() == Some(mint_url)
1720 || self
1721 .routing
1722 .cashu_accepted_mints
1723 .iter()
1724 .any(|mint| mint == mint_url)
1725 }
1726
1727 async fn peer_suggested_mint_cap_sat(&self, peer_id: &str) -> u64 {
1728 let base = self.routing.cashu_peer_suggested_mint_base_cap_sat;
1729 if base == 0 {
1730 return 0;
1731 }
1732
1733 let selector = self.peer_selector.read().await;
1734 let Some(stats) = selector.get_stats(peer_id) else {
1735 let max_cap = self.routing.cashu_peer_suggested_mint_max_cap_sat;
1736 return if max_cap > 0 { base.min(max_cap) } else { base };
1737 };
1738
1739 if stats.cashu_payment_defaults > 0
1740 && stats.cashu_payment_defaults >= stats.cashu_payment_receipts
1741 {
1742 return 0;
1743 }
1744
1745 let success_bonus = stats
1746 .successes
1747 .saturating_mul(self.routing.cashu_peer_suggested_mint_success_step_sat);
1748 let receipt_bonus = stats
1749 .cashu_payment_receipts
1750 .saturating_mul(self.routing.cashu_peer_suggested_mint_receipt_step_sat);
1751 let mut cap = base
1752 .saturating_add(success_bonus)
1753 .saturating_add(receipt_bonus);
1754 let max_cap = self.routing.cashu_peer_suggested_mint_max_cap_sat;
1755 if max_cap > 0 {
1756 cap = cap.min(max_cap);
1757 }
1758 cap
1759 }
1760
1761 async fn should_accept_quote_response(
1762 &self,
1763 from_peer: &str,
1764 preferred_mint_url: Option<&str>,
1765 offered_payment_sat: u64,
1766 res: &DataQuoteResponse,
1767 ) -> bool {
1768 let Some(payment_sat) = res.p else {
1769 return false;
1770 };
1771 if payment_sat > offered_payment_sat {
1772 return false;
1773 }
1774
1775 let response_mint = res.m.as_deref();
1776 if response_mint == preferred_mint_url {
1777 return true;
1778 }
1779 if self.trusts_quote_mint(response_mint) {
1780 return true;
1781 }
1782 if response_mint.is_none() {
1783 return false;
1784 }
1785
1786 payment_sat <= self.peer_suggested_mint_cap_sat(from_peer).await
1787 }
1788
1789 async fn issue_quote(
1790 &self,
1791 peer_id: &str,
1792 hash_key: &str,
1793 payment_sat: u64,
1794 ttl_ms: u32,
1795 mint_url: Option<&str>,
1796 ) -> u64 {
1797 let quote_id = {
1798 let mut next = self.next_quote_id.write().await;
1799 let quote_id = *next;
1800 *next = next.saturating_add(1);
1801 quote_id
1802 };
1803
1804 let expires_at = Instant::now() + Duration::from_millis(ttl_ms as u64);
1805 self.issued_quotes.write().await.insert(
1806 (peer_id.to_string(), hash_key.to_string(), quote_id),
1807 IssuedQuote {
1808 expires_at,
1809 payment_sat,
1810 mint_url: mint_url.map(str::to_string),
1811 },
1812 );
1813 quote_id
1814 }
1815
1816 async fn take_valid_quote(&self, peer_id: &str, hash_key: &str, quote_id: u64) -> bool {
1817 let key = (peer_id.to_string(), hash_key.to_string(), quote_id);
1818 let Some(quote) = self.issued_quotes.write().await.remove(&key) else {
1819 return false;
1820 };
1821 quote.expires_at > Instant::now()
1822 }
1823
1824 async fn send_request_to_peer(
1825 &self,
1826 peer_id: &str,
1827 hash: &Hash,
1828 request_htl: u8,
1829 quote_id: Option<u64>,
1830 ) -> bool {
1831 if !should_forward_htl(request_htl) {
1832 return false;
1833 }
1834
1835 let channel = match self.signaling.get_channel(peer_id).await {
1836 Some(c) => c,
1837 None => return false,
1838 };
1839
1840 let send_htl = request_htl.saturating_sub(1);
1844 let req = match quote_id {
1845 Some(quote_id) => create_request_with_quote(hash, send_htl, quote_id),
1846 None => create_request(hash, send_htl),
1847 };
1848 let request_bytes = encode_request(&req);
1849 let request_len = request_bytes.len() as u64;
1850
1851 {
1852 let mut selector = self.peer_selector.write().await;
1853 selector.record_request(peer_id, request_len);
1854 }
1855
1856 match channel.send(request_bytes).await {
1857 Ok(()) => {
1858 self.record_peer_wire_sent(peer_id, request_len).await;
1859 true
1860 }
1861 Err(_) => {
1862 self.peer_selector.write().await.record_failure(peer_id);
1863 false
1864 }
1865 }
1866 }
1867
1868 async fn send_quote_request_to_peer(
1869 &self,
1870 peer_id: &str,
1871 hash: &Hash,
1872 payment_sat: u64,
1873 ttl_ms: u32,
1874 mint_url: Option<&str>,
1875 ) -> bool {
1876 let channel = match self.signaling.get_channel(peer_id).await {
1877 Some(c) => c,
1878 None => return false,
1879 };
1880
1881 let req = create_quote_request(hash, ttl_ms, payment_sat, mint_url);
1882 let request_bytes = encode_quote_request(&req);
1883 let request_len = request_bytes.len() as u64;
1884
1885 match channel.send(request_bytes).await {
1886 Ok(()) => {
1887 self.record_peer_wire_sent(peer_id, request_len).await;
1888 true
1889 }
1890 Err(_) => false,
1891 }
1892 }
1893
1894 pub async fn peer_count(&self) -> usize {
1896 self.signaling.peer_count().await
1897 }
1898
1899 pub async fn peer_ids(&self) -> Vec<String> {
1901 self.signaling.peer_ids().await
1902 }
1903
1904 pub async fn needs_peers(&self) -> bool {
1906 self.signaling.needs_peers().await
1907 }
1908
1909 pub async fn send_hello(&self) -> Result<(), TransportError> {
1911 self.signaling.send_hello(vec![]).await
1912 }
1913
1914 pub async fn drain_available_data_messages(self: &Arc<Self>) -> DataPumpStats {
1919 let mut stats = DataPumpStats::default();
1920 let peer_ids = self.signaling.peer_ids().await;
1921 for peer_id in peer_ids {
1922 let Some(channel) = self.signaling.get_channel(&peer_id).await else {
1923 continue;
1924 };
1925
1926 while let Some(data) = channel.try_recv() {
1927 stats.processed += 1;
1928 stats.processed_bytes += data.len() as u64;
1929 if let Some(msg) = parse_message(&data) {
1930 match msg {
1931 DataMessage::Request(_) => stats.request_messages += 1,
1932 DataMessage::Response(_) => stats.response_messages += 1,
1933 DataMessage::QuoteRequest(_) => stats.quote_request_messages += 1,
1934 DataMessage::QuoteResponse(_) => stats.quote_response_messages += 1,
1935 DataMessage::PubsubInterest(_) => stats.pubsub_interest_messages += 1,
1936 DataMessage::PubsubFrame(_) => stats.pubsub_frame_messages += 1,
1937 DataMessage::PubsubInventory(_) => stats.pubsub_inventory_messages += 1,
1938 DataMessage::PubsubWant(_) => stats.pubsub_want_messages += 1,
1939 DataMessage::Payment(_)
1940 | DataMessage::PaymentAck(_)
1941 | DataMessage::Chunk(_)
1942 | DataMessage::PeerHints(_) => {}
1943 }
1944 }
1945 self.handle_data_message(&peer_id, &data).await;
1946 }
1947 }
1948 stats
1949 }
1950
1951 pub async fn record_cashu_payment_for_peer(&self, peer_id: &str, amount_sat: u64) {
1953 self.peer_selector
1954 .write()
1955 .await
1956 .record_cashu_payment(peer_id, amount_sat);
1957 }
1958
1959 pub async fn record_cashu_receipt_from_peer(&self, peer_id: &str, amount_sat: u64) {
1961 self.peer_selector
1962 .write()
1963 .await
1964 .record_cashu_receipt(peer_id, amount_sat);
1965 }
1966
1967 pub async fn record_cashu_payment_default_from_peer(&self, peer_id: &str) {
1969 self.peer_selector
1970 .write()
1971 .await
1972 .record_cashu_payment_default(peer_id);
1973 }
1974
1975 pub async fn selector_summary(&self) -> crate::peer_selector::SelectorSummary {
1977 self.peer_selector.read().await.summary()
1978 }
1979
1980 fn should_refuse_requests_from_peer(&self, selector: &PeerSelector, peer_id: &str) -> bool {
1981 selector.is_peer_blocked_for_payment_defaults(
1982 peer_id,
1983 self.routing.cashu_payment_default_block_threshold,
1984 )
1985 }
1986
1987 pub async fn peer_metadata_snapshot(&self) -> PeerMetadataSnapshot {
1989 self.peer_selector
1990 .read()
1991 .await
1992 .export_peer_metadata_snapshot()
1993 }
1994
1995 pub async fn persist_peer_metadata(&self) -> Result<Hash, StoreError> {
2000 let snapshot = self
2001 .peer_selector
2002 .read()
2003 .await
2004 .export_peer_metadata_snapshot();
2005 let bytes = serde_json::to_vec(&snapshot).map_err(|e| {
2006 StoreError::Other(format!("Failed to encode peer metadata snapshot: {e}"))
2007 })?;
2008 let snapshot_hash = hashtree_core::sha256(&bytes);
2009 let _ = self.local_store.put(snapshot_hash, bytes).await?;
2010
2011 let pointer_slot = Self::peer_metadata_pointer_slot_hash();
2012 let pointer_bytes = hex::encode(snapshot_hash).into_bytes();
2013 let _ = self.local_store.delete(&pointer_slot).await?;
2014 let _ = self.local_store.put(pointer_slot, pointer_bytes).await?;
2015
2016 Ok(snapshot_hash)
2017 }
2018
2019 pub async fn load_peer_metadata(&self) -> Result<bool, StoreError> {
2021 let pointer_slot = Self::peer_metadata_pointer_slot_hash();
2022 let Some(pointer_bytes) = self.local_store.get(&pointer_slot).await? else {
2023 return Ok(false);
2024 };
2025 let pointer_hex = std::str::from_utf8(&pointer_bytes).map_err(|e| {
2026 StoreError::Other(format!("Peer metadata pointer is not valid UTF-8: {e}"))
2027 })?;
2028 let snapshot_hash = Self::decode_hash_hex(pointer_hex.trim())?;
2029
2030 let Some(snapshot_bytes) = self.local_store.get(&snapshot_hash).await? else {
2031 return Ok(false);
2032 };
2033 let snapshot: PeerMetadataSnapshot =
2034 serde_json::from_slice(&snapshot_bytes).map_err(|e| {
2035 StoreError::Other(format!("Failed to decode peer metadata snapshot: {e}"))
2036 })?;
2037 self.peer_selector
2038 .write()
2039 .await
2040 .import_peer_metadata_snapshot(&snapshot);
2041 Ok(true)
2042 }
2043
2044 pub async fn get_with_quote(
2049 &self,
2050 hash: &Hash,
2051 payment_sat: u64,
2052 quote_ttl: Duration,
2053 ) -> Result<Option<Vec<u8>>, StoreError> {
2054 if let Some(data) = self.local_store.get(hash).await? {
2055 if hashtree_core::sha256(&data) != *hash {
2056 return Err(StoreError::Other(
2057 "local store returned corrupt content".to_string(),
2058 ));
2059 }
2060 return Ok(Some(data));
2061 }
2062 self.request_from_peers_with_quote(hash, payment_sat, quote_ttl)
2063 .await
2064 }
2065
2066 async fn request_from_peers_with_quote(
2067 &self,
2068 hash: &Hash,
2069 payment_sat: u64,
2070 quote_ttl: Duration,
2071 ) -> Result<Option<Vec<u8>>, StoreError> {
2072 let ordered_peer_ids = self.ordered_connected_peers(None).await;
2073 if ordered_peer_ids.is_empty() {
2074 return Ok(None);
2075 }
2076
2077 if let Some(quote) = self
2078 .request_quote_from_peers(hash, payment_sat, quote_ttl, &ordered_peer_ids)
2079 .await
2080 {
2081 if let Some(data) = self
2082 .request_from_single_peer(hash, "e.peer_id, MAX_HTL, Some(quote.quote_id))
2083 .await
2084 {
2085 return Ok(Some(data));
2086 }
2087 }
2088
2089 match self
2090 .request_from_mesh_with_context(hash, &MeshReadContext::default())
2091 .await
2092 {
2093 RouteFetchOutcome::Hit(data) => Ok(Some(data)),
2094 RouteFetchOutcome::Miss => Ok(None),
2095 RouteFetchOutcome::Timeout => Err(StoreError::Other(
2096 "blob retrieval deadline expired before the search completed".to_string(),
2097 )),
2098 }
2099 }
2100
2101 async fn request_quote_from_peers(
2102 &self,
2103 hash: &Hash,
2104 payment_sat: u64,
2105 quote_ttl: Duration,
2106 ordered_peer_ids: &[String],
2107 ) -> Option<NegotiatedQuote> {
2108 if ordered_peer_ids.is_empty() {
2109 return None;
2110 }
2111 let ttl_ms = quote_ttl.as_millis().min(u32::MAX as u128) as u32;
2112 if ttl_ms == 0 {
2113 return None;
2114 }
2115 let requested_mint = self.requested_quote_mint().map(str::to_string);
2116
2117 let hash_key = hash_to_key(hash);
2118 let (tx, rx) = oneshot::channel();
2119 self.pending_quotes.write().await.insert(
2120 hash_key.clone(),
2121 PendingQuoteRequest {
2122 response_tx: tx,
2123 preferred_mint_url: requested_mint.clone(),
2124 offered_payment_sat: payment_sat,
2125 },
2126 );
2127
2128 let rx = Arc::new(Mutex::new(rx));
2129 let result = run_hedged_waves(
2130 ordered_peer_ids.len(),
2131 self.routing.dispatch,
2132 self.request_timeout,
2133 |range| {
2134 let wave_peer_ids = ordered_peer_ids[range].to_vec();
2135 let requested_mint = requested_mint.clone();
2136 let hash = *hash;
2137 async move {
2138 let mut sent = 0usize;
2139 for peer_id in wave_peer_ids {
2140 if self
2141 .send_quote_request_to_peer(
2142 &peer_id,
2143 &hash,
2144 payment_sat,
2145 ttl_ms,
2146 requested_mint.as_deref(),
2147 )
2148 .await
2149 {
2150 sent += 1;
2151 }
2152 }
2153 sent
2154 }
2155 },
2156 |wait| {
2157 let rx = rx.clone();
2158 async move {
2159 let mut rx = rx.lock().await;
2160 match tokio::time::timeout(wait, &mut *rx).await {
2161 Ok(Ok(Some(quote))) => HedgedWaveAction::Success(quote),
2162 Ok(Ok(None)) | Ok(Err(_)) => HedgedWaveAction::Abort,
2163 Err(_) => HedgedWaveAction::Continue,
2164 }
2165 }
2166 },
2167 )
2168 .await;
2169 let _ = self.pending_quotes.write().await.remove(&hash_key);
2170 result
2171 }
2172
2173 async fn register_pending_request(
2174 &self,
2175 request_key: PendingRequestKey,
2176 queried_peers: Vec<String>,
2177 ) -> (Arc<()>, oneshot::Receiver<Option<Vec<u8>>>) {
2178 let owner = Arc::new(());
2179 let (response_tx, response_rx) = oneshot::channel();
2180 self.pending_requests
2181 .write()
2182 .await
2183 .entry(request_key)
2184 .or_default()
2185 .push(PendingRequest {
2186 owner: owner.clone(),
2187 response_tx,
2188 started_at: Instant::now(),
2189 queried_peers,
2190 });
2191 (owner, response_rx)
2192 }
2193
2194 async fn take_pending_request(
2195 &self,
2196 request_key: PendingRequestKey,
2197 owner: &Arc<()>,
2198 ) -> Option<(PendingRequest, bool)> {
2199 let mut pending = self.pending_requests.write().await;
2200 let (request, remove_key) = {
2201 let requests = pending.get_mut(&request_key)?;
2202 let index = requests
2203 .iter()
2204 .position(|request| Arc::ptr_eq(&request.owner, owner))?;
2205 let request = requests.swap_remove(index);
2206 (request, requests.is_empty())
2207 };
2208 if remove_key {
2209 pending.remove(&request_key);
2210 }
2211 Some((request, remove_key))
2212 }
2213
2214 async fn request_from_single_peer(
2215 &self,
2216 hash: &Hash,
2217 peer_id: &str,
2218 request_htl: u8,
2219 quote_id: Option<u64>,
2220 ) -> Option<Vec<u8>> {
2221 let request_key = PendingRequestKey::new(*hash, request_htl);
2222 let (owner, rx) = self
2223 .register_pending_request(request_key, vec![peer_id.to_string()])
2224 .await;
2225
2226 let mut rx = rx;
2227 if !self
2228 .send_request_to_peer(peer_id, hash, request_htl, quote_id)
2229 .await
2230 {
2231 if self
2232 .take_pending_request(request_key, &owner)
2233 .await
2234 .is_some_and(|(_, last)| last)
2235 {
2236 let _ = self.take_forward_requesters(request_key).await;
2237 }
2238 return None;
2239 }
2240 self.reserve_peer_request(peer_id).await;
2241
2242 if let Ok(Ok(Some(data))) = tokio::time::timeout(self.request_timeout, &mut rx).await {
2243 if data.len() <= BLOB_MAX_BYTES && hashtree_core::sha256(&data) == *hash {
2244 let _ = self.local_store.put(*hash, data.clone()).await;
2245 return Some(data);
2246 }
2247 }
2248
2249 if let Some((pending, last)) = self.take_pending_request(request_key, &owner).await {
2250 self.release_queried_peer_requests(&pending.queried_peers)
2251 .await;
2252 for peer_id in pending.queried_peers {
2253 self.peer_selector.write().await.record_timeout(&peer_id);
2254 }
2255 if last {
2256 let _ = self.take_forward_requesters(request_key).await;
2257 }
2258 }
2259 None
2260 }
2261
2262 async fn request_from_ordered_peers(
2263 &self,
2264 hash: &Hash,
2265 ordered_peer_ids: &[String],
2266 request_htl: u8,
2267 timeout: Duration,
2268 ) -> RouteFetchOutcome {
2269 let request_key = PendingRequestKey::new(*hash, request_htl);
2270 let (owner, rx) = self.register_pending_request(request_key, Vec::new()).await;
2271
2272 let rx = Arc::new(Mutex::new(rx));
2273 let result = run_hedged_waves(
2274 ordered_peer_ids.len(),
2275 normalize_dispatch_config(self.routing.dispatch, ordered_peer_ids.len()),
2276 timeout,
2277 |range| {
2278 let wave_peer_ids = ordered_peer_ids[range].to_vec();
2279 let hash = *hash;
2280 let owner = owner.clone();
2281 async move {
2282 let mut sent = 0usize;
2283 for peer_id in wave_peer_ids {
2284 if self
2285 .send_request_to_peer(&peer_id, &hash, request_htl, None)
2286 .await
2287 {
2288 sent += 1;
2289 self.reserve_peer_request(&peer_id).await;
2290 if let Some(pending) = self
2291 .pending_requests
2292 .write()
2293 .await
2294 .get_mut(&request_key)
2295 .and_then(|requests| {
2296 requests
2297 .iter_mut()
2298 .find(|request| Arc::ptr_eq(&request.owner, &owner))
2299 })
2300 {
2301 pending.queried_peers.push(peer_id);
2302 }
2303 }
2304 }
2305 sent
2306 }
2307 },
2308 |wait| {
2309 let rx = rx.clone();
2310 async move {
2311 let mut rx = rx.lock().await;
2312 match tokio::time::timeout(wait, &mut *rx).await {
2313 Ok(Ok(Some(data)))
2314 if data.len() <= BLOB_MAX_BYTES
2315 && hashtree_core::sha256(&data) == *hash =>
2316 {
2317 HedgedWaveAction::Success(data)
2318 }
2319 Ok(Ok(Some(_))) => HedgedWaveAction::Continue,
2320 Ok(Ok(None)) | Ok(Err(_)) => HedgedWaveAction::Abort,
2321 Err(_) => HedgedWaveAction::Continue,
2322 }
2323 }
2324 },
2325 )
2326 .await;
2327
2328 let Some(data) = result else {
2329 if let Some((pending, last)) = self.take_pending_request(request_key, &owner).await {
2330 self.release_queried_peer_requests(&pending.queried_peers)
2331 .await;
2332 for peer_id in pending.queried_peers {
2333 self.peer_selector.write().await.record_timeout(&peer_id);
2334 }
2335 if last {
2336 let _ = self.take_forward_requesters(request_key).await;
2337 }
2338 }
2339 return RouteFetchOutcome::Timeout;
2340 };
2341
2342 let _ = self.local_store.put(*hash, data.clone()).await;
2343 RouteFetchOutcome::Hit(data)
2344 }
2345
2346 async fn request_from_mesh_with_context(
2347 &self,
2348 hash: &Hash,
2349 context: &MeshReadContext,
2350 ) -> RouteFetchOutcome {
2351 if !should_forward_htl(context.request_htl) {
2352 return RouteFetchOutcome::Miss;
2353 }
2354 let mut peers = self
2355 .ordered_connected_peers(context.exclude_peer_id.as_deref())
2356 .await;
2357 if let Some(attempt_budget) = context.attempt_budget {
2358 peers.truncate(attempt_budget);
2359 }
2360 if peers.is_empty() {
2361 return RouteFetchOutcome::Miss;
2362 }
2363 let timeout = context
2364 .deadline
2365 .map(|deadline| deadline.saturating_duration_since(Instant::now()))
2366 .unwrap_or(self.request_timeout)
2367 .min(self.request_timeout);
2368 if timeout.is_zero() {
2369 return RouteFetchOutcome::Timeout;
2370 }
2371 self.request_from_ordered_peers(hash, &peers, context.request_htl, timeout)
2372 .await
2373 }
2374
2375 async fn begin_forward_request(
2376 &self,
2377 request_key: PendingRequestKey,
2378 requester_id: &str,
2379 ) -> bool {
2380 let mut pending = self.pending_forward_requests.write().await;
2381 if let Some(existing) = pending.get_mut(&request_key) {
2382 existing.requester_ids.insert(requester_id.to_string());
2383 return false;
2384 }
2385
2386 let mut requester_ids = HashSet::new();
2387 requester_ids.insert(requester_id.to_string());
2388 pending.insert(request_key, PendingForwardRequest { requester_ids });
2389 true
2390 }
2391
2392 async fn take_forward_requesters(&self, request_key: PendingRequestKey) -> Vec<String> {
2393 self.pending_forward_requests
2394 .write()
2395 .await
2396 .remove(&request_key)
2397 .map(|pending| pending.requester_ids.into_iter().collect())
2398 .unwrap_or_default()
2399 }
2400
2401 async fn complete_pending_response(
2402 self: &Arc<Self>,
2403 from_peer: &str,
2404 hash: &Hash,
2405 payload: Vec<u8>,
2406 ) {
2407 let pending = {
2408 let mut requests = self.pending_requests.write().await;
2409 let matching_keys: Vec<_> = requests
2410 .keys()
2411 .filter(|key| key.hash == *hash)
2412 .copied()
2413 .collect();
2414 matching_keys
2415 .into_iter()
2416 .flat_map(|key| {
2417 requests
2418 .remove(&key)
2419 .into_iter()
2420 .flatten()
2421 .map(move |request| (key, request))
2422 })
2423 .collect::<Vec<_>>()
2424 };
2425 if pending.is_empty() {
2426 return;
2427 }
2428
2429 let payload_bytes = payload.len() as u64;
2430 self.record_useful_bytes_received_from_peer(from_peer, payload_bytes)
2431 .await;
2432 {
2433 let mut deliveries = self.verified_block_deliveries.lock().await;
2434 deliveries.deliveries.push_back(VerifiedBlockDelivery {
2435 hash: *hash,
2436 provider_peer_id: from_peer.to_string(),
2437 payload_bytes,
2438 });
2439 while deliveries.deliveries.len() > VERIFIED_BLOCK_DELIVERY_CAPACITY {
2440 deliveries.deliveries.pop_front();
2441 deliveries.dropped_since_last_drain =
2442 deliveries.dropped_since_last_drain.saturating_add(1);
2443 }
2444 }
2445
2446 let rtt_ms = pending
2447 .iter()
2448 .map(|(_, request)| request.started_at.elapsed().as_millis() as u64)
2449 .max()
2450 .unwrap_or_default();
2451 let queried_peers: Vec<_> = pending
2452 .iter()
2453 .flat_map(|(_, request)| request.queried_peers.iter().cloned())
2454 .collect();
2455 self.release_queried_peer_requests(&queried_peers).await;
2456 self.peer_selector
2457 .write()
2458 .await
2459 .record_success(from_peer, rtt_ms, payload_bytes);
2460
2461 let mut forward_requesters = HashSet::new();
2462 for (request_key, pending) in pending {
2463 forward_requesters.extend(self.take_forward_requesters(request_key).await);
2464 let _ = pending.response_tx.send(Some(payload.clone()));
2465 }
2466 if !forward_requesters.is_empty() {
2467 let response_bytes = encode_response(&create_response(hash, payload));
2468 for requester_id in forward_requesters {
2469 Arc::clone(self)
2470 .enqueue_response_send(requester_id, response_bytes.clone(), Instant::now())
2471 .await;
2472 }
2473 }
2474 }
2475
2476 async fn handle_quote_response_message(&self, from_peer: &str, res: DataQuoteResponse) {
2477 if !res.a {
2478 return;
2479 }
2480
2481 let Some(quote_id) = res.q else {
2482 return;
2483 };
2484
2485 let hash_key = hash_to_key(&res.h);
2486 let (preferred_mint_url, offered_payment_sat) = {
2487 let pending_quotes = self.pending_quotes.read().await;
2488 let Some(pending) = pending_quotes.get(&hash_key) else {
2489 return;
2490 };
2491 (
2492 pending.preferred_mint_url.clone(),
2493 pending.offered_payment_sat,
2494 )
2495 };
2496 if !self
2497 .should_accept_quote_response(
2498 from_peer,
2499 preferred_mint_url.as_deref(),
2500 offered_payment_sat,
2501 &res,
2502 )
2503 .await
2504 {
2505 return;
2506 }
2507 let mut pending_quotes = self.pending_quotes.write().await;
2508 if let Some(pending) = pending_quotes.remove(&hash_key) {
2509 let _ = pending.response_tx.send(Some(NegotiatedQuote {
2510 peer_id: from_peer.to_string(),
2511 quote_id,
2512 mint_url: res.m,
2513 }));
2514 }
2515 }
2516
2517 async fn handle_response_message(
2518 self: &Arc<Self>,
2519 from_peer: &str,
2520 res: crate::protocol::DataResponse,
2521 ) {
2522 let hash_key = hash_to_key(&res.h);
2523 let hash = match crate::protocol::bytes_to_hash(&res.h) {
2524 Some(h) => h,
2525 None => return,
2526 };
2527
2528 if hashtree_core::sha256(&res.d) != hash {
2530 self.peer_selector.write().await.record_failure(from_peer);
2531 if self.debug {
2532 println!("[MeshStoreCore] Ignoring invalid response payload for {hash_key}");
2533 }
2534 return;
2535 }
2536
2537 self.complete_pending_response(from_peer, &hash, res.d)
2538 .await;
2539 }
2540
2541 async fn handle_quote_request_message(&self, from_peer: &str, req: DataQuoteRequest) {
2542 let hash = match crate::protocol::bytes_to_hash(&req.h) {
2543 Some(h) => h,
2544 None => return,
2545 };
2546 let hash_key = hash_to_key(&hash);
2547
2548 {
2549 let selector = self.peer_selector.read().await;
2550 if self.should_refuse_requests_from_peer(&selector, from_peer) {
2551 if self.debug {
2552 println!(
2553 "[MeshStoreCore] Refusing quote request from delinquent peer {}",
2554 from_peer
2555 );
2556 }
2557 return;
2558 }
2559 }
2560
2561 let chosen_mint = self.choose_quote_mint(req.m.as_deref());
2562 let can_serve = self.local_store.has(&hash).await.ok().unwrap_or(false)
2563 && !self.should_drop_response(&hash)
2564 && !self.should_corrupt_response(&hash);
2565
2566 let res = if can_serve {
2567 let quote_id = self
2568 .issue_quote(from_peer, &hash_key, req.p, req.t, chosen_mint.as_deref())
2569 .await;
2570 create_quote_response_available(&hash, quote_id, req.p, req.t, chosen_mint.as_deref())
2571 } else {
2572 create_quote_response_unavailable(&hash)
2573 };
2574 let response_bytes = encode_quote_response(&res);
2575 if let Some(channel) = self.signaling.get_channel(from_peer).await {
2576 if channel.send(response_bytes.clone()).await.is_ok() {
2577 self.record_peer_wire_sent(from_peer, response_bytes.len() as u64)
2578 .await;
2579 }
2580 }
2581 }
2582
2583 async fn handle_request_message(
2584 self: &Arc<Self>,
2585 from_peer: &str,
2586 req: crate::protocol::DataRequest,
2587 ) {
2588 if req.htl > MAX_HTL {
2589 return;
2590 }
2591 let hash = match crate::protocol::bytes_to_hash(&req.h) {
2592 Some(h) => h,
2593 None => return,
2594 };
2595 let hash_key = hash_to_key(&hash);
2596 let request_key = PendingRequestKey::new(hash, req.htl);
2597
2598 if let Some(quote_id) = req.q {
2599 if !self.take_valid_quote(from_peer, &hash_key, quote_id).await {
2600 if self.debug {
2601 println!(
2602 "[MeshStoreCore] Refusing request with invalid or expired quote {} from {}",
2603 quote_id, from_peer
2604 );
2605 }
2606 return;
2607 }
2608 }
2609
2610 let allow_peer_forwarding = {
2611 let selector = self.peer_selector.read().await;
2612 !self.should_refuse_requests_from_peer(&selector, from_peer)
2613 };
2614
2615 if let Ok(Some(mut data)) = self.local_store.get(&hash).await {
2617 if data.len() <= BLOB_MAX_BYTES && hashtree_core::sha256(&data) == hash {
2618 if self.should_drop_response(&hash) {
2619 if self.debug {
2620 println!(
2621 "[MeshStoreCore] Dropping response for {} due to actor profile",
2622 hash_to_key(&hash)
2623 );
2624 }
2625 return;
2626 }
2627
2628 let response_delay = self.response_send_delay(&hash, data.len());
2629 if self.should_corrupt_response(&hash) {
2630 if data.is_empty() {
2631 data.push(0x80);
2632 } else {
2633 data[0] ^= 0x80;
2634 }
2635 }
2636
2637 let res = create_response(&hash, data);
2638 let response_bytes = encode_response(&res);
2639 let ready_at = Instant::now() + response_delay;
2640 Arc::clone(self)
2641 .enqueue_response_send(from_peer.to_string(), response_bytes, ready_at)
2642 .await;
2643 return;
2644 }
2645 }
2646
2647 if self
2648 .pending_requests
2649 .read()
2650 .await
2651 .contains_key(&request_key)
2652 {
2653 let _ = self.begin_forward_request(request_key, from_peer).await;
2654 return;
2655 }
2656
2657 if !self.begin_forward_request(request_key, from_peer).await {
2658 return;
2659 }
2660
2661 let from_peer = from_peer.to_string();
2662 let this = Arc::clone(self);
2663 let request_htl = req.htl;
2664 tokio::spawn(async move {
2665 let result = if allow_peer_forwarding {
2666 let context = MeshReadContext {
2667 exclude_peer_id: Some(from_peer.clone()),
2668 request_htl,
2669 deadline: None,
2670 attempt_budget: None,
2671 };
2672 this.request_from_mesh_with_context(&hash, &context).await
2673 } else {
2674 if this.debug {
2675 println!(
2676 "[MeshStoreCore] Refusing to forward request from delinquent peer {}",
2677 from_peer
2678 );
2679 }
2680 RouteFetchOutcome::Miss
2681 };
2682 let requester_ids = this.take_forward_requesters(request_key).await;
2683 match result {
2684 RouteFetchOutcome::Hit(data) => {
2685 let ready_at = Instant::now() + this.response_send_delay(&hash, data.len());
2686 let res = create_response(&hash, data);
2687 let response_bytes = encode_response(&res);
2688 for requester_id in requester_ids {
2689 Arc::clone(&this)
2690 .enqueue_response_send(requester_id, response_bytes.clone(), ready_at)
2691 .await;
2692 }
2693 }
2694 RouteFetchOutcome::Miss | RouteFetchOutcome::Timeout => {}
2695 }
2696 });
2697 }
2698
2699 async fn handle_pubsub_interest_message(
2700 self: &Arc<Self>,
2701 from_peer: &str,
2702 mut interest: PubsubInterest,
2703 ) {
2704 if !self.apply_pubsub_interest_route(from_peer, &interest).await {
2705 return;
2706 }
2707
2708 if !self.routing.pubsub_forwarding || interest.htl <= 1 {
2709 return;
2710 }
2711 interest.htl = interest.htl.saturating_sub(1);
2712 let _ = self
2713 .send_pubsub_interest_to_peers(&interest, Some(from_peer))
2714 .await;
2715 }
2716
2717 async fn handle_pubsub_frame_message(
2718 self: &Arc<Self>,
2719 from_peer: &str,
2720 mut frame: PubsubFrame,
2721 wire_bytes: usize,
2722 ) {
2723 if frame.stream_id.is_empty() || frame.origin_peer_id.is_empty() {
2724 return;
2725 }
2726 if frame.origin_peer_id == self.signaling.peer_id() {
2727 return;
2728 }
2729
2730 let frame_key = Self::pubsub_frame_key(&frame);
2731 if !self
2732 .pubsub_seen_frames
2733 .lock()
2734 .await
2735 .insert_if_new(frame_key.clone())
2736 {
2737 return;
2738 }
2739 self.cache_pubsub_frame(frame_key.clone(), frame.clone())
2740 .await;
2741
2742 let local_interested = self
2743 .pubsub_local_interests
2744 .read()
2745 .await
2746 .contains(&frame.stream_id);
2747 let mut downstream_peers = if self.routing.pubsub_forwarding && frame.htl > 1 {
2748 match self.routing.pubsub_delivery_mode {
2749 PubsubDeliveryMode::InterestPush => {
2750 let mut peers = self
2751 .interested_pubsub_peers(&frame.stream_id, Some(from_peer))
2752 .await;
2753 peers.extend(
2754 self.take_pubsub_want_peers(&frame_key, Some(from_peer))
2755 .await,
2756 );
2757 peers.sort();
2758 peers.dedup();
2759 peers
2760 }
2761 PubsubDeliveryMode::HtlInvWant => {
2762 self.take_pubsub_want_peers(&frame_key, Some(from_peer))
2763 .await
2764 }
2765 }
2766 } else {
2767 Vec::new()
2768 };
2769 downstream_peers.retain(|peer_id| peer_id != from_peer);
2770
2771 if local_interested || !downstream_peers.is_empty() {
2772 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
2773 .await;
2774 }
2775
2776 if local_interested {
2777 self.enqueue_pubsub_event(PubsubEvent {
2778 stream_id: frame.stream_id.clone(),
2779 seq: frame.seq,
2780 origin_peer_id: frame.origin_peer_id.clone(),
2781 from_peer_id: from_peer.to_string(),
2782 payload: frame.payload.clone(),
2783 })
2784 .await;
2785 }
2786
2787 if downstream_peers.is_empty() {
2788 return;
2789 }
2790
2791 frame.htl = frame.htl.saturating_sub(1);
2792 let _ = self
2793 .send_pubsub_frame_to_peers(&frame, &downstream_peers)
2794 .await;
2795 }
2796
2797 async fn handle_pubsub_inventory_message(
2798 self: &Arc<Self>,
2799 from_peer: &str,
2800 inv: PubsubInventory,
2801 wire_bytes: usize,
2802 ) {
2803 if inv.stream_id.is_empty() || inv.origin_peer_id.is_empty() {
2804 return;
2805 }
2806 if inv.origin_peer_id == self.signaling.peer_id() {
2807 return;
2808 }
2809
2810 let key = Self::pubsub_key(&inv.origin_peer_id, &inv.stream_id, inv.seq);
2811 if !self
2812 .pubsub_seen_inventories
2813 .lock()
2814 .await
2815 .insert_if_new(key.clone())
2816 {
2817 return;
2818 }
2819 {
2820 let mut routes = self.pubsub_inventory_routes.write().await;
2821 routes
2822 .entry(key.clone())
2823 .or_insert_with(|| from_peer.to_string());
2824 }
2825
2826 let local_interested = self
2827 .pubsub_local_interests
2828 .read()
2829 .await
2830 .contains(&inv.stream_id);
2831 let downstream_peers = if self.routing.pubsub_forwarding {
2832 self.interested_pubsub_peers(&inv.stream_id, Some(from_peer))
2833 .await
2834 } else {
2835 Vec::new()
2836 };
2837 if local_interested || !downstream_peers.is_empty() {
2838 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
2839 .await;
2840 let want =
2841 create_pubsub_want(inv.stream_id.clone(), inv.seq, inv.origin_peer_id.clone());
2842 let _ = self.send_pubsub_want_upstream(&key, &want, None).await;
2843 }
2844
2845 if !self.routing.pubsub_forwarding
2846 || downstream_peers.is_empty()
2847 || !should_forward_htl(inv.htl)
2848 {
2849 return;
2850 }
2851 let _ = self
2852 .send_pubsub_inventory_to_peers(&inv, &downstream_peers)
2853 .await;
2854 }
2855
2856 async fn handle_pubsub_want_message(
2857 self: &Arc<Self>,
2858 from_peer: &str,
2859 want: PubsubWant,
2860 wire_bytes: usize,
2861 ) {
2862 if want.stream_id.is_empty() || want.origin_peer_id.is_empty() {
2863 return;
2864 }
2865 if want.origin_peer_id == from_peer {
2866 return;
2867 }
2868
2869 let key = Self::pubsub_key(&want.origin_peer_id, &want.stream_id, want.seq);
2870 let want_key = format!("{from_peer}:{key}");
2871 if !self.pubsub_seen_wants.lock().await.insert_if_new(want_key) {
2872 return;
2873 }
2874
2875 if let Some(frame) = self.cached_pubsub_frame(&key).await {
2876 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
2877 .await;
2878 let peers = vec![from_peer.to_string()];
2879 let _ = self.send_pubsub_frame_to_peers(&frame, &peers).await;
2880 return;
2881 }
2882
2883 let has_upstream_route = self.pubsub_inventory_routes.read().await.contains_key(&key);
2884 if !has_upstream_route {
2885 return;
2886 }
2887
2888 if self.remember_pubsub_want_peer(key.clone(), from_peer).await {
2889 self.record_useful_bytes_received_from_peer(from_peer, wire_bytes as u64)
2890 .await;
2891 }
2892 let _ = self
2893 .send_pubsub_want_upstream(&key, &want, Some(from_peer))
2894 .await;
2895 }
2896
2897 pub async fn handle_data_message(self: &Arc<Self>, from_peer: &str, data: &[u8]) {
2899 self.record_peer_wire_received(from_peer, data.len() as u64)
2900 .await;
2901 let parsed = match parse_message(data) {
2902 Some(m) => m,
2903 None => return,
2904 };
2905
2906 match parsed {
2907 DataMessage::Request(req) => {
2908 self.handle_request_message(from_peer, req).await;
2909 }
2910 DataMessage::Response(res) => {
2911 self.handle_response_message(from_peer, res).await;
2912 }
2913 DataMessage::QuoteRequest(req) => {
2914 self.handle_quote_request_message(from_peer, req).await;
2915 }
2916 DataMessage::QuoteResponse(res) => {
2917 self.handle_quote_response_message(from_peer, res).await;
2918 }
2919 DataMessage::PubsubInterest(interest) => {
2920 self.handle_pubsub_interest_message(from_peer, interest)
2921 .await;
2922 }
2923 DataMessage::PubsubFrame(frame) => {
2924 self.handle_pubsub_frame_message(from_peer, frame, data.len())
2925 .await;
2926 }
2927 DataMessage::PubsubInventory(inv) => {
2928 self.handle_pubsub_inventory_message(from_peer, inv, data.len())
2929 .await;
2930 }
2931 DataMessage::PubsubWant(want) => {
2932 self.handle_pubsub_want_message(from_peer, want, data.len())
2933 .await;
2934 }
2935 DataMessage::Payment(_)
2936 | DataMessage::PaymentAck(_)
2937 | DataMessage::Chunk(_)
2938 | DataMessage::PeerHints(_) => {}
2939 }
2940 }
2941}
2942
2943#[async_trait]
2944impl<S, R, F> BlobRoute for MeshStoreCore<S, R, F>
2945where
2946 S: Store + Send + Sync + 'static,
2947 R: SignalingTransport + Send + Sync + 'static,
2948 F: PeerLinkFactory + Send + Sync + 'static,
2949{
2950 async fn route(&self, request: BlobRequest) -> Result<BlobReply, StoreError> {
2951 self.route_with_context(
2952 request,
2953 BlobRouteContext {
2954 deadline: (Instant::now() + self.request_timeout).into(),
2955 attempt_budget: self.routing.dispatch.max_fanout.max(1),
2956 },
2957 )
2958 .await
2959 }
2960
2961 async fn route_with_context(
2962 &self,
2963 request: BlobRequest,
2964 route_context: BlobRouteContext,
2965 ) -> Result<BlobReply, StoreError> {
2966 if request.htl > MAX_HTL {
2967 return Err(StoreError::Other(format!(
2968 "Hashtree blob HTL {} exceeds the maximum of {MAX_HTL}",
2969 request.htl
2970 )));
2971 }
2972 if let Some(data) = self.local_store.get(&request.hash).await? {
2973 if data.len() > BLOB_MAX_BYTES {
2974 return Err(StoreError::Other(format!(
2975 "local store returned {} bytes, exceeding the {BLOB_MAX_BYTES}-byte limit",
2976 data.len()
2977 )));
2978 }
2979 if hashtree_core::sha256(&data) != request.hash {
2980 return Err(StoreError::Other(
2981 "local store returned corrupt content".to_string(),
2982 ));
2983 }
2984 return Ok(BlobReply::Data(data));
2985 }
2986
2987 if request.htl == 0 {
2988 return Ok(BlobReply::NoResult);
2989 }
2990
2991 let context = MeshReadContext {
2992 exclude_peer_id: None,
2993 request_htl: request.htl,
2994 deadline: Some(route_context.deadline.into()),
2995 attempt_budget: Some(route_context.attempt_budget),
2996 };
2997 match self
2998 .request_from_mesh_with_context(&request.hash, &context)
2999 .await
3000 {
3001 RouteFetchOutcome::Hit(data) => {
3002 if data.len() > BLOB_MAX_BYTES {
3003 return Err(StoreError::Other(format!(
3004 "blob route returned {} bytes, exceeding the {BLOB_MAX_BYTES}-byte limit",
3005 data.len()
3006 )));
3007 }
3008 if hashtree_core::sha256(&data) != request.hash {
3009 return Err(StoreError::Other(
3010 "blob route returned corrupt content".to_string(),
3011 ));
3012 }
3013 Ok(BlobReply::Data(data))
3014 }
3015 RouteFetchOutcome::Miss => Ok(BlobReply::NoResult),
3016 RouteFetchOutcome::Timeout => Err(StoreError::Other(
3017 "blob retrieval deadline expired before the search completed".to_string(),
3018 )),
3019 }
3020 }
3021}
3022
3023#[async_trait]
3024impl<S, R, F> Store for MeshStoreCore<S, R, F>
3025where
3026 S: Store + Send + Sync + 'static,
3027 R: SignalingTransport + Send + Sync + 'static,
3028 F: PeerLinkFactory + Send + Sync + 'static,
3029{
3030 async fn put(&self, hash: Hash, data: Vec<u8>) -> Result<bool, StoreError> {
3031 self.local_store.put(hash, data).await
3032 }
3033
3034 async fn get(&self, hash: &Hash) -> Result<Option<Vec<u8>>, StoreError> {
3035 Ok(
3036 match self
3037 .route(BlobRequest {
3038 hash: *hash,
3039 htl: MAX_HTL,
3040 })
3041 .await?
3042 {
3043 BlobReply::Data(data) => Some(data),
3044 BlobReply::NoResult => None,
3045 },
3046 )
3047 }
3048
3049 async fn has(&self, hash: &Hash) -> Result<bool, StoreError> {
3050 self.local_store.has(hash).await
3051 }
3052
3053 async fn delete(&self, hash: &Hash) -> Result<bool, StoreError> {
3054 self.local_store.delete(hash).await
3055 }
3056}
3057
3058#[cfg(test)]
3059mod delivery_tests;
3060
3061#[cfg(test)]
3062mod tests;
3063
3064pub type SimMeshStore<S> =
3066 MeshStoreCore<S, crate::mock::MockRelayTransport, crate::mock::MockConnectionFactory>;