1use crate::{
13 DialLimit, Error,
14 engine::Engine,
15 model::{OverlayPeerStatus, OverlayStatus, RecordReceipt},
16 peer::PeerSource,
17 peer_protocol::{InventoryPage, Placement},
18 recorded::{RecordedError, unix_seconds},
19};
20use libp2p::PeerId;
21use std::{
22 sync::{
23 Arc, Mutex,
24 atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering},
25 },
26 time::{Duration, Instant},
27};
28
29pub type ConnectedPeers = Arc<dyn Fn() -> u64 + Send + Sync>;
30
31pub struct OverlayInfo {
33 pub peer_id: String,
34 pub listen: Option<String>,
35 pub peer_book: u64,
36 pub connected: ConnectedPeers,
37}
38
39pub struct PeerSync {
41 pub peers: Vec<(String, PeerSource)>,
42 pub info: OverlayInfo,
43}
44
45struct Range {
47 from: u64,
48 to: u64,
49 after: Option<Placement>,
50 entries: Vec<Placement>,
51 next: usize,
52 more: bool,
53 scanned_through: u64,
54 generation: u64,
55 missing: u32,
57}
58
59struct Peer {
60 label: String,
61 id: String,
62 peer_id: PeerId,
63 source: PeerSource,
64 range: Mutex<Option<Range>>,
65 last_request: Mutex<Option<Instant>>,
66 last_error: Mutex<Option<RecordedError>>,
67 backoff: Mutex<Backoff>,
68 busy_turns: AtomicU32,
69 idle_turns: AtomicU32,
70 last_progress: Mutex<Option<u64>>,
71 connected_since: Mutex<Option<u64>>,
73}
74
75struct Backoff {
78 failures: u32,
79 until: Option<u64>,
80}
81
82impl Peer {
83 fn backoff_until(&self, now: u64) -> Result<Option<u64>, Error> {
84 let backoff = self.backoff.lock().map_err(|_| poisoned())?;
85 Ok(backoff.until.filter(|until| *until > now))
86 }
87
88 fn last_progress(&self) -> Result<Option<u64>, Error> {
89 Ok(*self.last_progress.lock().map_err(|_| poisoned())?)
90 }
91
92 fn hold_off(&self, now: u64, seconds: u64) -> Result<(), Error> {
95 let until = now.checked_add(seconds).unwrap_or(u64::MAX);
96 let mut backoff = self.backoff.lock().map_err(|_| poisoned())?;
97 backoff.until = Some(backoff.until.map_or(until, |current| current.max(until)));
98 Ok(())
99 }
100
101 fn wait_closed(&self, stop: &AtomicBool, deadline: Instant) -> bool {
104 let until = deadline.min(Instant::now() + Scheduler::EVICT_WAIT);
105 while self.source.connected() {
106 if stop.load(Ordering::Acquire) || Instant::now() >= until {
107 return false;
108 }
109 std::thread::sleep(Duration::from_millis(25));
110 }
111 true
112 }
113}
114
115enum Pause {
118 Done,
119 Abandon,
120 Error(Error),
121 Blame(Error),
122}
123
124enum Turn {
127 Quiet,
128 Busy(Error),
129 Retry(Error),
130 Blamed(Error),
131}
132
133fn blame(error: Error) -> Pause {
136 match error {
137 Error::Busy(_)
138 | Error::PeerBusy
139 | Error::Transport(_)
140 | Error::DialDenied(_)
141 | Error::Stale => Pause::Error(error),
142 other => Pause::Blame(other),
143 }
144}
145
146impl From<Error> for Pause {
147 fn from(error: Error) -> Self {
148 Pause::Error(error)
149 }
150}
151
152struct Round<'a> {
156 stop: &'a AtomicBool,
157 deadline: Instant,
158 fetched: usize,
159 requested_page: bool,
160 contacted: bool,
161 progressed: bool,
162}
163
164impl Round<'_> {
165 fn open(&self) -> Result<(), Pause> {
166 if self.stop.load(Ordering::Acquire) || Instant::now() >= self.deadline {
167 return Err(Pause::Done);
168 }
169 Ok(())
170 }
171}
172
173pub(crate) struct Scheduler {
174 peers: Mutex<Vec<Arc<Peer>>>,
175 turn: AtomicUsize,
176 info: OverlayInfo,
177 admitted: AtomicU64,
178 rejected: AtomicU64,
179 penalized: Mutex<Vec<PeerId>>,
180 steps: AtomicU64,
181}
182
183fn poisoned() -> Error {
184 Error::Invalid("peer synchronization state poisoned".into())
185}
186
187fn evictable(peer: &Peer, step: u64, now: u64) -> Result<bool, Error> {
190 let settled = peer
191 .connected_since
192 .lock()
193 .map_err(|_| poisoned())?
194 .is_some_and(|since| step >= since.saturating_add(2));
195 let idle = peer
196 .last_progress()?
197 .is_none_or(|at| now >= at.saturating_add(Scheduler::IDLE_BACKOFF));
198 Ok(settled && idle && peer.source.connected())
199}
200
201fn inventory_peer(label: String, source: PeerSource) -> Result<Peer, Error> {
202 let peer_id = match label.parse::<PeerId>() {
203 Ok(id) => id,
204 Err(error) => {
205 return Err(Error::Invalid(format!("inventory peer label: {error}")));
206 }
207 };
208 Ok(Peer {
209 label,
210 id: peer_id.to_string(),
211 peer_id,
212 source,
213 range: Mutex::new(None),
214 last_request: Mutex::new(None),
215 last_error: Mutex::new(None),
216 backoff: Mutex::new(Backoff {
217 failures: 0,
218 until: None,
219 }),
220 busy_turns: AtomicU32::new(0),
221 idle_turns: AtomicU32::new(0),
222 last_progress: Mutex::new(None),
223 connected_since: Mutex::new(None),
224 })
225}
226
227impl Scheduler {
228 pub const RANGE_HEIGHTS: u64 = 4096;
230 pub const PAGE_ENTRIES: u32 = 512;
232 pub const ROUND_FETCHES: usize = 16;
234 pub const ROUND_TIME: Duration = Duration::from_secs(20);
236 pub const REQUEST_SPACING: Duration = Duration::from_millis(250);
238 pub const BACKOFF_SECONDS: u64 = 30;
240 pub const BACKOFF_CAP: u64 = 86_400;
241 pub const BUSY_TURNS: u32 = 10;
243 pub const MISSING_ANSWERS: u32 = 2;
245 pub const IDLE_TURNS: u32 = 3;
250 pub const IDLE_BACKOFF: u64 = 60;
251 pub const EVICT_WAIT: Duration = Duration::from_secs(2);
253 const MAX_PEERS: usize = 1024;
254
255 pub(crate) fn new(sync: PeerSync) -> Result<Self, Error> {
256 if sync.peers.len() > Self::MAX_PEERS {
257 return Err(Error::Invalid("too many inventory peers".into()));
258 }
259 let mut peers: Vec<Arc<Peer>> = Vec::new();
260 for (label, source) in sync.peers {
261 let peer = inventory_peer(label, source)?;
262 if peers.iter().any(|known| known.id == peer.id) {
263 return Err(Error::Invalid("duplicate inventory peer".into()));
264 }
265 peers.push(Arc::new(peer));
266 }
267 Ok(Self {
268 peers: Mutex::new(peers),
269 turn: AtomicUsize::new(0),
270 info: sync.info,
271 admitted: AtomicU64::new(0),
272 rejected: AtomicU64::new(0),
273 penalized: Mutex::new(Vec::new()),
274 steps: AtomicU64::new(0),
275 })
276 }
277
278 pub(crate) fn step(&self, engine: &Engine, stop: &AtomicBool) -> Result<(), Error> {
282 let omitting = engine.omitting_peers()?;
283 let now = unix_seconds()?;
284 let step = self.steps.fetch_add(1, Ordering::Relaxed);
285 self.observe_connections(step)?;
286 let Some(peer) = self.next_peer(&omitting, now)? else {
287 return Ok(());
288 };
289 let deadline = Instant::now() + Self::ROUND_TIME;
290 let denied = self.turn(&peer, engine, stop, deadline, now)?;
291 if denied && self.free_slot_for(&peer, step, stop, deadline, now)? {
292 self.turn(&peer, engine, stop, deadline, now)?;
293 }
294 Ok(())
295 }
296
297 fn turn(
300 &self,
301 peer: &Peer,
302 engine: &Engine,
303 stop: &AtomicBool,
304 deadline: Instant,
305 now: u64,
306 ) -> Result<bool, Error> {
307 let mut round = Round {
308 stop,
309 deadline,
310 fetched: 0,
311 requested_page: false,
312 contacted: false,
313 progressed: false,
314 };
315 let outcome = self.advance(peer, engine, &mut round);
316 if stop.load(Ordering::Acquire) {
317 return Ok(false);
318 }
319 self.track_progress(peer, &outcome, &round, now)?;
320 let denied = matches!(
321 &outcome,
322 Ok(Turn::Retry(Error::DialDenied(DialLimit::Established)))
323 );
324 let result = self.judge(peer, outcome, now)?;
325 *peer.last_error.lock().map_err(|_| poisoned())? = RecordedError::from_result(&result);
326 Ok(denied)
327 }
328
329 fn observe_connections(&self, step: u64) -> Result<(), Error> {
331 for peer in self.snapshot()? {
332 let mut since = peer.connected_since.lock().map_err(|_| poisoned())?;
333 *since = match (peer.source.connected(), *since) {
334 (true, Some(first)) => Some(first),
335 (true, None) => Some(step),
336 (false, _gone) => None,
337 };
338 }
339 Ok(())
340 }
341
342 pub(crate) fn backed_off(&self, now: u64) -> Result<Vec<String>, Error> {
344 let mut skipped = Vec::new();
345 for peer in self.snapshot()? {
346 if peer.backoff_until(now)?.is_some() {
347 skipped.push(peer.id.clone());
348 }
349 }
350 Ok(skipped)
351 }
352
353 fn track_progress(
358 &self,
359 peer: &Peer,
360 outcome: &Result<Turn, Error>,
361 round: &Round,
362 now: u64,
363 ) -> Result<(), Error> {
364 if round.progressed {
365 peer.idle_turns.store(0, Ordering::Relaxed);
366 *peer.last_progress.lock().map_err(|_| poisoned())? = Some(now);
367 return Ok(());
368 }
369 if !round.contacted || !matches!(outcome, Ok(Turn::Quiet)) {
370 return Ok(());
371 }
372 let idle = peer.idle_turns.fetch_add(1, Ordering::Relaxed) + 1;
373 if idle < Self::IDLE_TURNS {
374 return Ok(());
375 }
376 peer.idle_turns.store(0, Ordering::Relaxed);
377 peer.source.disconnect();
378 peer.hold_off(now, Self::IDLE_BACKOFF)
379 }
380
381 fn free_slot_for(
391 &self,
392 starved: &Peer,
393 step: u64,
394 stop: &AtomicBool,
395 deadline: Instant,
396 now: u64,
397 ) -> Result<bool, Error> {
398 let mut oldest: Option<(Arc<Peer>, Option<u64>)> = None;
399 for peer in self.snapshot()? {
400 if peer.id == starved.id || !evictable(&peer, step, now)? {
401 continue;
402 }
403 let progress = peer.last_progress()?;
404 if oldest.as_ref().is_none_or(|(_, best)| progress < *best) {
405 oldest = Some((peer, progress));
406 }
407 }
408 let Some((victim, _progress)) = oldest else {
409 return Ok(false);
410 };
411 victim.source.disconnect();
412 victim.hold_off(now, Self::IDLE_BACKOFF)?;
413 Ok(victim.wait_closed(stop, deadline))
414 }
415
416 fn judge(
420 &self,
421 peer: &Peer,
422 outcome: Result<Turn, Error>,
423 now: u64,
424 ) -> Result<Result<(), Error>, Error> {
425 match outcome {
426 Ok(Turn::Quiet) => {
427 peer.busy_turns.store(0, Ordering::Relaxed);
428 peer.backoff.lock().map_err(|_| poisoned())?.failures = 0;
429 Ok(Ok(()))
430 }
431 Ok(Turn::Busy(error)) => {
432 let streak = peer.busy_turns.fetch_add(1, Ordering::Relaxed) + 1;
433 if streak < Self::BUSY_TURNS {
434 return Ok(Err(error));
435 }
436 peer.busy_turns.store(0, Ordering::Relaxed);
437 peer.source.disconnect();
438 Ok(Err(Error::Busy(format!(
439 "{error}; busy for {streak} turns, disconnected"
440 ))))
441 }
442 Ok(Turn::Blamed(error)) => {
443 peer.busy_turns.store(0, Ordering::Relaxed);
444 self.penalize(peer, now)?;
445 Ok(Err(error))
446 }
447 Ok(Turn::Retry(error)) | Err(error) => Ok(Err(error)),
448 }
449 }
450
451 fn penalize(&self, peer: &Peer, now: u64) -> Result<(), Error> {
452 peer.source.disconnect();
453 let mut backoff = peer.backoff.lock().map_err(|_| poisoned())?;
454 backoff.failures = backoff.failures.checked_add(1).unwrap_or(u32::MAX);
455 let delay = 2u64
456 .checked_pow(backoff.failures - 1)
457 .and_then(|factor| factor.checked_mul(Self::BACKOFF_SECONDS))
458 .map_or(Self::BACKOFF_CAP, |delay| delay.min(Self::BACKOFF_CAP));
459 backoff.until = Some(now.checked_add(delay).unwrap_or(u64::MAX));
460 drop(backoff);
461 let mut penalized = self.penalized.lock().map_err(|_| poisoned())?;
462 if !penalized.contains(&peer.peer_id) {
463 penalized.push(peer.peer_id);
464 }
465 Ok(())
466 }
467
468 pub(crate) fn drain_penalized(&self) -> Result<Vec<PeerId>, Error> {
470 let mut penalized = self.penalized.lock().map_err(|_| poisoned())?;
471 Ok(std::mem::take(&mut *penalized))
472 }
473
474 pub(crate) fn refresh(&self, offered: Vec<(String, PeerSource)>) -> Result<(), Error> {
477 if offered.len() > Self::MAX_PEERS {
478 return Err(Error::Invalid("too many inventory peers".into()));
479 }
480 let mut peers = self.peers.lock().map_err(|_| poisoned())?;
481 let mut next: Vec<Arc<Peer>> = Vec::with_capacity(offered.len());
482 for (label, source) in offered {
483 let candidate = inventory_peer(label, source)?;
484 if next.iter().any(|kept| kept.id == candidate.id) {
485 continue;
486 }
487 match peers.iter().find(|known| known.id == candidate.id) {
488 Some(known) => next.push(Arc::clone(known)),
489 None => next.push(Arc::new(candidate)),
490 }
491 }
492 *peers = next;
493 Ok(())
494 }
495
496 fn snapshot(&self) -> Result<Vec<Arc<Peer>>, Error> {
497 Ok(self.peers.lock().map_err(|_| poisoned())?.clone())
498 }
499
500 fn next_peer(&self, omitting: &[String], now: u64) -> Result<Option<Arc<Peer>>, Error> {
501 let peers = self.snapshot()?;
502 let count = peers.len();
503 for _ in 0..count {
504 let peer = &peers[self.turn.fetch_add(1, Ordering::Relaxed) % count];
505 if !omitting.contains(&peer.id) && peer.backoff_until(now)?.is_none() {
506 return Ok(Some(Arc::clone(peer)));
507 }
508 }
509 Ok(None)
510 }
511
512 fn advance(&self, peer: &Peer, engine: &Engine, round: &mut Round) -> Result<Turn, Error> {
513 let mut slot = peer.range.lock().map_err(|_| poisoned())?;
514 if slot.is_none() {
515 *slot = open(peer, engine)?;
516 round.progressed |= slot.is_none();
519 }
520 match self.walk(peer, engine, &mut slot, round) {
523 Ok(()) | Err(Pause::Done) => Ok(Turn::Quiet),
524 Err(Pause::Abandon) | Err(Pause::Error(Error::Stale)) => {
525 *slot = None;
526 Ok(Turn::Quiet)
527 }
528 Err(Pause::Error(Error::PeerBusy)) => Ok(Turn::Busy(Error::PeerBusy)),
529 Err(Pause::Error(
530 retry @ (Error::Busy(_) | Error::Transport(_) | Error::DialDenied(_)),
531 )) => Ok(Turn::Retry(retry)),
532 Err(Pause::Blame(error)) => {
533 *slot = None;
534 Ok(Turn::Blamed(error))
535 }
536 Err(Pause::Error(error)) => {
537 *slot = None;
538 Err(error)
539 }
540 }
541 }
542
543 fn walk(
544 &self,
545 peer: &Peer,
546 engine: &Engine,
547 slot: &mut Option<Range>,
548 round: &mut Round,
549 ) -> Result<(), Pause> {
550 while let Some(range) = slot.as_mut() {
551 round.open()?;
552 if let Some(&entry) = range.entries.get(range.next) {
553 if !selected(engine, &entry)? {
554 return Err(Pause::Abandon);
555 }
556 if engine.record(entry.txid)?.is_none() {
557 if round.fetched == Self::ROUND_FETCHES {
558 return Err(Pause::Done);
559 }
560 round.fetched += 1;
561 pace(peer, round)?;
562 let missing = &mut range.missing;
563 round.progressed |=
564 self.acquire(peer, engine, &entry, missing, round.deadline)?;
565 } else {
566 round.progressed = true;
567 }
568 range.next += 1;
569 } else if range.more {
570 if round.requested_page {
571 return Err(Pause::Done);
572 }
573 round.requested_page = true;
574 pace(peer, round)?;
575 let Some(page) = peer
576 .source
577 .inventory_until(
578 (range.from, range.to),
579 range.after,
580 Self::PAGE_ENTRIES,
581 round.stop,
582 round.deadline,
583 )
584 .map_err(blame)?
585 else {
586 return Err(Pause::Done);
587 };
588 if !accept(range, page).map_err(Pause::Blame)? {
589 *slot = None;
590 }
591 } else {
592 let through = range.to.min(range.scanned_through);
593 engine.record_peer_coverage(&peer.id, range.from, through, range.generation)?;
594 round.progressed = true;
595 *slot = None;
596 }
597 }
598 Ok(())
599 }
600
601 fn acquire(
607 &self,
608 peer: &Peer,
609 engine: &Engine,
610 entry: &Placement,
611 missing: &mut u32,
612 deadline: Instant,
613 ) -> Result<bool, Pause> {
614 let fetched = peer
615 .source
616 .fetch_until(entry.txid, deadline)
617 .map_err(blame)?;
618 let Some(package) = fetched else {
619 *missing += 1;
620 if *missing < Self::MISSING_ANSWERS {
621 return Ok(false);
622 }
623 return Err(Pause::Blame(Error::Invalid(
624 "peer reported advertised records missing".into(),
625 )));
626 };
627 *missing = 0;
628 let receipt = match engine.admit_record(package) {
629 Ok(receipt) => receipt,
630 Err(invalid @ Error::Invalid(_)) => {
631 self.rejected.fetch_add(1, Ordering::Relaxed);
632 return Err(Pause::Blame(invalid));
633 }
634 Err(local) => return Err(Pause::Error(local)),
635 };
636 self.admitted.fetch_add(1, Ordering::Relaxed);
637 if !placed(&receipt, entry) {
638 return Err(Pause::Blame(Error::Invalid(
639 "peer inventory placement disagrees with its proven record".into(),
640 )));
641 }
642 Ok(true)
643 }
644
645 pub(crate) fn status(&self, engine: &Engine) -> Result<OverlayStatus, Error> {
646 let start = engine.status()?.application_start.height;
647 let now = unix_seconds()?;
648 let known = self.snapshot()?;
649 let mut peers = Vec::with_capacity(known.len());
650 for peer in &known {
651 let cursor = engine.peer_cursor(&peer.id)?;
652 let synced_through = cursor.checked_sub(1).filter(|height| *height >= start);
653 let recorded = peer.last_error.lock().map_err(|_| poisoned())?.clone();
654 let (last_error, last_error_at) = RecordedError::split(recorded);
655 peers.push(OverlayPeerStatus {
656 label: peer.label.clone(),
657 cursor,
658 synced_through,
659 last_error,
660 last_error_at,
661 backoff_until: peer.backoff_until(now)?,
662 last_progress_at: peer.last_progress()?,
663 });
664 }
665 Ok(OverlayStatus {
666 peer_id: self.info.peer_id.clone(),
667 listen: self.info.listen.clone(),
668 peer_book: self.info.peer_book,
669 connected: (self.info.connected)(),
670 admitted: self.admitted.load(Ordering::Relaxed),
671 rejected: self.rejected.load(Ordering::Relaxed),
672 peers,
673 discovery: None,
674 })
675 }
676}
677
678fn pace(peer: &Peer, round: &mut Round) -> Result<(), Pause> {
682 let mut last = peer.last_request.lock().map_err(|_| poisoned())?;
683 if let Some(previous) = *last {
684 let ready = previous + Scheduler::REQUEST_SPACING;
685 while Instant::now() < ready {
686 round.open()?;
687 let wait = ready.saturating_duration_since(Instant::now());
688 std::thread::sleep(wait.min(Duration::from_millis(50)));
689 }
690 }
691 round.open()?;
692 *last = Some(Instant::now());
693 round.contacted = true;
694 Ok(())
695}
696
697fn open(peer: &Peer, engine: &Engine) -> Result<Option<Range>, Error> {
701 let status = engine.status()?;
702 let Some(tip) = status.selected_tip else {
703 return Ok(None);
704 };
705 let from = engine.peer_cursor(&peer.id)?;
706 if from > tip.height {
707 return Ok(None);
708 }
709 let span_end = from
710 .checked_add(Scheduler::RANGE_HEIGHTS - 1)
711 .ok_or_else(|| Error::Invalid("peer cursor overflows the height range".into()))?;
712 Ok(Some(Range {
713 from,
714 to: tip.height.min(span_end),
715 after: None,
716 entries: Vec::new(),
717 next: 0,
718 more: true,
719 scanned_through: 0,
720 generation: status.generation,
721 missing: 0,
722 }))
723}
724
725fn accept(range: &mut Range, page: InventoryPage) -> Result<bool, Error> {
731 let Some(scanned_through) = page.scanned_through.filter(|height| *height >= range.from) else {
732 return Ok(false);
733 };
734 let limit = usize::try_from(Scheduler::PAGE_ENTRIES)
735 .map_err(|error| Error::Invalid(format!("page limit: {error}")))?;
736 if page.entries.len() > limit || (page.more && page.entries.is_empty()) {
737 return Err(Error::Invalid("peer inventory page size is invalid".into()));
738 }
739 let mut previous = range.after;
740 for entry in &page.entries {
741 if entry.height < range.from
742 || entry.height > range.to
743 || previous
744 .is_some_and(|last| (entry.height, entry.position) <= (last.height, last.position))
745 {
746 return Err(Error::Invalid(
747 "peer inventory entry is out of range or order".into(),
748 ));
749 }
750 previous = Some(*entry);
751 }
752 range.after = previous;
753 range.entries = page
754 .entries
755 .into_iter()
756 .filter(|entry| entry.height <= scanned_through)
757 .collect();
758 range.next = 0;
759 range.more = page.more;
760 range.scanned_through = scanned_through;
761 Ok(true)
762}
763
764fn selected(engine: &Engine, entry: &Placement) -> Result<bool, Error> {
767 match engine.archive()?.ticket(entry.height) {
768 Ok(ticket) => Ok(ticket.point().hash == entry.block_hash),
769 Err(Error::Invalid(_outside_selected_view)) => Ok(false),
770 Err(error) => Err(error),
771 }
772}
773
774fn placed(receipt: &RecordReceipt, entry: &Placement) -> bool {
775 receipt.block.height == entry.height
776 && receipt.block.hash == entry.block_hash
777 && receipt.position == entry.position
778 && receipt.txid == entry.txid
779 && receipt.wtxid == entry.wtxid
780}