Skip to main content

urma_node/
peer_sync.rs

1//! Fast inventory synchronization from configured URMA peers. A peer page is an
2//! untrusted claim: every fetched package is proof-verified by `admit_record`
3//! against the independently selected header chain, and coverage is recorded
4//! only after a complete range has every advertised record present locally.
5//! Full-block audit (`indexed_through`) remains the separate, stronger claim.
6//! A peer that answers with an invalid page, package or reply is disconnected
7//! and backed off; one that stays Busy for many turns is disconnected so the
8//! connection slot it holds frees up for others. So is one that answers turn
9//! after turn without anything we can use (stale listings, unscanned pages),
10//! and a peer refused by the local connection limit gets the slot of the
11//! connected peer that progressed longest ago.
12use crate::{
13    DialLimit, Error,
14    engine::Engine,
15    model::{OverlayPeerStatus, OverlayStatus, RecordReceipt},
16    peer::PeerSource,
17    peer_protocol::{InventoryPage, Placement},
18    recorded::{RecordedError, unix_seconds},
19};
20use libp2p::PeerId;
21use std::{
22    sync::{
23        Arc, Mutex,
24        atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering},
25    },
26    time::{Duration, Instant},
27};
28
29pub type ConnectedPeers = Arc<dyn Fn() -> u64 + Send + Sync>;
30
31/// Public overlay identity facts reported in status; never secrets.
32pub struct OverlayInfo {
33    pub peer_id: String,
34    pub listen: Option<String>,
35    pub peer_book: u64,
36    pub connected: ConnectedPeers,
37}
38
39/// Configured inventory peers. Each label is the peer's libp2p identity.
40pub struct PeerSync {
41    pub peers: Vec<(String, PeerSource)>,
42    pub info: OverlayInfo,
43}
44
45/// One claimed range being walked page by page; lives only in memory.
46struct Range {
47    from: u64,
48    to: u64,
49    after: Option<Placement>,
50    entries: Vec<Placement>,
51    next: usize,
52    more: bool,
53    scanned_through: u64,
54    generation: u64,
55    /// Consecutive Missing answers for records this range advertised.
56    missing: u32,
57}
58
59struct Peer {
60    label: String,
61    id: String,
62    peer_id: PeerId,
63    source: PeerSource,
64    range: Mutex<Option<Range>>,
65    last_request: Mutex<Option<Instant>>,
66    last_error: Mutex<Option<RecordedError>>,
67    backoff: Mutex<Backoff>,
68    busy_turns: AtomicU32,
69    idle_turns: AtomicU32,
70    last_progress: Mutex<Option<u64>>,
71    /// Scheduler step at which this peer was first seen connected, if it is.
72    connected_since: Mutex<Option<u64>>,
73}
74
75/// Sync backoff after peer errors: consecutive blamed turns and the Unix
76/// second before which the peer is skipped.
77struct Backoff {
78    failures: u32,
79    until: Option<u64>,
80}
81
82impl Peer {
83    fn backoff_until(&self, now: u64) -> Result<Option<u64>, Error> {
84        let backoff = self.backoff.lock().map_err(|_| poisoned())?;
85        Ok(backoff.until.filter(|until| *until > now))
86    }
87
88    fn last_progress(&self) -> Result<Option<u64>, Error> {
89        Ok(*self.last_progress.lock().map_err(|_| poisoned())?)
90    }
91
92    /// Skip this peer until at least `now + seconds`, unblamed: failures and
93    /// the book are untouched, and a longer backoff is never shortened.
94    fn hold_off(&self, now: u64, seconds: u64) -> Result<(), Error> {
95        let until = now.checked_add(seconds).unwrap_or(u64::MAX);
96        let mut backoff = self.backoff.lock().map_err(|_| poisoned())?;
97        backoff.until = Some(backoff.until.map_or(until, |current| current.max(until)));
98        Ok(())
99    }
100
101    /// Wait for an evicted connection to close so its slot is really free,
102    /// bounded by `EVICT_WAIT`, the step deadline and cancellation.
103    fn wait_closed(&self, stop: &AtomicBool, deadline: Instant) -> bool {
104        let until = deadline.min(Instant::now() + Scheduler::EVICT_WAIT);
105        while self.source.connected() {
106            if stop.load(Ordering::Acquire) || Instant::now() >= until {
107                return false;
108            }
109            std::thread::sleep(Duration::from_millis(25));
110        }
111        true
112    }
113}
114
115/// Why a step stopped early. `Blame` is a protocol violation by the peer;
116/// `Error` is ours, transport or retryable.
117enum Pause {
118    Done,
119    Abandon,
120    Error(Error),
121    Blame(Error),
122}
123
124/// How one step ended for the peer it ran with: remote Busy builds a streak,
125/// local Busy and transport failures retry quietly.
126enum Turn {
127    Quiet,
128    Busy(Error),
129    Retry(Error),
130    Blamed(Error),
131}
132
133/// A failed request blames the peer only for a protocol violation: never for
134/// capacity (ours or theirs), transport failure or our reorg.
135fn blame(error: Error) -> Pause {
136    match error {
137        Error::Busy(_)
138        | Error::PeerBusy
139        | Error::Transport(_)
140        | Error::DialDenied(_)
141        | Error::Stale => Pause::Error(error),
142        other => Pause::Blame(other),
143    }
144}
145
146impl From<Error> for Pause {
147    fn from(error: Error) -> Self {
148        Pause::Error(error)
149    }
150}
151
152/// Per-round budgets for one peer: request spacing, fetch count, wall clock;
153/// whether the peer was asked anything and whether an admission or coverage
154/// came of it.
155struct Round<'a> {
156    stop: &'a AtomicBool,
157    deadline: Instant,
158    fetched: usize,
159    requested_page: bool,
160    contacted: bool,
161    progressed: bool,
162}
163
164impl Round<'_> {
165    fn open(&self) -> Result<(), Pause> {
166        if self.stop.load(Ordering::Acquire) || Instant::now() >= self.deadline {
167            return Err(Pause::Done);
168        }
169        Ok(())
170    }
171}
172
173pub(crate) struct Scheduler {
174    peers: Mutex<Vec<Arc<Peer>>>,
175    turn: AtomicUsize,
176    info: OverlayInfo,
177    admitted: AtomicU64,
178    rejected: AtomicU64,
179    penalized: Mutex<Vec<PeerId>>,
180    steps: AtomicU64,
181}
182
183fn poisoned() -> Error {
184    Error::Invalid("peer synchronization state poisoned".into())
185}
186
187/// A connected peer that has held its slot for more than one step without
188/// progress for `IDLE_BACKOFF` seconds, or ever.
189fn evictable(peer: &Peer, step: u64, now: u64) -> Result<bool, Error> {
190    let settled = peer
191        .connected_since
192        .lock()
193        .map_err(|_| poisoned())?
194        .is_some_and(|since| step >= since.saturating_add(2));
195    let idle = peer
196        .last_progress()?
197        .is_none_or(|at| now >= at.saturating_add(Scheduler::IDLE_BACKOFF));
198    Ok(settled && idle && peer.source.connected())
199}
200
201fn inventory_peer(label: String, source: PeerSource) -> Result<Peer, Error> {
202    let peer_id = match label.parse::<PeerId>() {
203        Ok(id) => id,
204        Err(error) => {
205            return Err(Error::Invalid(format!("inventory peer label: {error}")));
206        }
207    };
208    Ok(Peer {
209        label,
210        id: peer_id.to_string(),
211        peer_id,
212        source,
213        range: Mutex::new(None),
214        last_request: Mutex::new(None),
215        last_error: Mutex::new(None),
216        backoff: Mutex::new(Backoff {
217            failures: 0,
218            until: None,
219        }),
220        busy_turns: AtomicU32::new(0),
221        idle_turns: AtomicU32::new(0),
222        last_progress: Mutex::new(None),
223        connected_since: Mutex::new(None),
224    })
225}
226
227impl Scheduler {
228    /// Heights per claimed range, inclusive of both ends.
229    pub const RANGE_HEIGHTS: u64 = 4096;
230    /// Entries requested per inventory page.
231    pub const PAGE_ENTRIES: u32 = 512;
232    /// Package fetches per worker round.
233    pub const ROUND_FETCHES: usize = 16;
234    /// Wall-clock budget of one step, checked between requests.
235    pub const ROUND_TIME: Duration = Duration::from_secs(20);
236    /// Minimum spacing between requests to one peer (its 4/s token bucket).
237    pub const REQUEST_SPACING: Duration = Duration::from_millis(250);
238    /// Sync backoff after a peer error: 30 s, doubling per repeat, capped at a day.
239    pub const BACKOFF_SECONDS: u64 = 30;
240    pub const BACKOFF_CAP: u64 = 86_400;
241    /// Consecutive Busy turns after which a peer is disconnected.
242    pub const BUSY_TURNS: u32 = 10;
243    /// Consecutive Missing answers for advertised records that blame a peer.
244    pub const MISSING_ANSWERS: u32 = 2;
245    /// Consecutive answered turns without progress after which a peer's slot
246    /// is released for `IDLE_BACKOFF` seconds, unblamed. The same span guards
247    /// eviction: only a peer without progress for that long can be evicted,
248    /// and an evicted peer is held off for it too.
249    pub const IDLE_TURNS: u32 = 3;
250    pub const IDLE_BACKOFF: u64 = 60;
251    /// Longest wait for an evicted connection to close before the retry.
252    pub const EVICT_WAIT: Duration = Duration::from_secs(2);
253    const MAX_PEERS: usize = 1024;
254
255    pub(crate) fn new(sync: PeerSync) -> Result<Self, Error> {
256        if sync.peers.len() > Self::MAX_PEERS {
257            return Err(Error::Invalid("too many inventory peers".into()));
258        }
259        let mut peers: Vec<Arc<Peer>> = Vec::new();
260        for (label, source) in sync.peers {
261            let peer = inventory_peer(label, source)?;
262            if peers.iter().any(|known| known.id == peer.id) {
263                return Err(Error::Invalid("duplicate inventory peer".into()));
264            }
265            peers.push(Arc::new(peer));
266        }
267        Ok(Self {
268            peers: Mutex::new(peers),
269            turn: AtomicUsize::new(0),
270            info: sync.info,
271            admitted: AtomicU64::new(0),
272            rejected: AtomicU64::new(0),
273            penalized: Mutex::new(Vec::new()),
274            steps: AtomicU64::new(0),
275        })
276    }
277
278    /// One bounded step with the next non-omitting, non-backed-off peer in
279    /// rotation. Busy, deadline and cancellation keep the range; a step
280    /// interrupted by `stop` records nothing about the peer.
281    pub(crate) fn step(&self, engine: &Engine, stop: &AtomicBool) -> Result<(), Error> {
282        let omitting = engine.omitting_peers()?;
283        let now = unix_seconds()?;
284        let step = self.steps.fetch_add(1, Ordering::Relaxed);
285        self.observe_connections(step)?;
286        let Some(peer) = self.next_peer(&omitting, now)? else {
287            return Ok(());
288        };
289        let deadline = Instant::now() + Self::ROUND_TIME;
290        let denied = self.turn(&peer, engine, stop, deadline, now)?;
291        if denied && self.free_slot_for(&peer, step, stop, deadline, now)? {
292            self.turn(&peer, engine, stop, deadline, now)?;
293        }
294        Ok(())
295    }
296
297    /// One turn with `peer`. Returns whether our established-connection
298    /// limit refused its dial, the only refusal an eviction can cure.
299    fn turn(
300        &self,
301        peer: &Peer,
302        engine: &Engine,
303        stop: &AtomicBool,
304        deadline: Instant,
305        now: u64,
306    ) -> Result<bool, Error> {
307        let mut round = Round {
308            stop,
309            deadline,
310            fetched: 0,
311            requested_page: false,
312            contacted: false,
313            progressed: false,
314        };
315        let outcome = self.advance(peer, engine, &mut round);
316        if stop.load(Ordering::Acquire) {
317            return Ok(false);
318        }
319        self.track_progress(peer, &outcome, &round, now)?;
320        let denied = matches!(
321            &outcome,
322            Ok(Turn::Retry(Error::DialDenied(DialLimit::Established)))
323        );
324        let result = self.judge(peer, outcome, now)?;
325        *peer.last_error.lock().map_err(|_| poisoned())? = RecordedError::from_result(&result);
326        Ok(denied)
327    }
328
329    /// Note at which step each inventory peer was first seen connected.
330    fn observe_connections(&self, step: u64) -> Result<(), Error> {
331        for peer in self.snapshot()? {
332            let mut since = peer.connected_since.lock().map_err(|_| poisoned())?;
333            *since = match (peer.source.connected(), *since) {
334                (true, Some(first)) => Some(first),
335                (true, None) => Some(step),
336                (false, _gone) => None,
337            };
338        }
339        Ok(())
340    }
341
342    /// Peers sync is currently skipping, for discovery not to redial them.
343    pub(crate) fn backed_off(&self, now: u64) -> Result<Vec<String>, Error> {
344        let mut skipped = Vec::new();
345        for peer in self.snapshot()? {
346            if peer.backoff_until(now)?.is_some() {
347                skipped.push(peer.id.clone());
348            }
349        }
350        Ok(skipped)
351    }
352
353    /// An admission, coverage or an already-held listed record is progress
354    /// and resets the idle streak. `IDLE_TURNS` quiet turns in a row that
355    /// asked the peer and got nothing usable release its slot for
356    /// `IDLE_BACKOFF`: no blame, no book penalty, failures untouched.
357    fn track_progress(
358        &self,
359        peer: &Peer,
360        outcome: &Result<Turn, Error>,
361        round: &Round,
362        now: u64,
363    ) -> Result<(), Error> {
364        if round.progressed {
365            peer.idle_turns.store(0, Ordering::Relaxed);
366            *peer.last_progress.lock().map_err(|_| poisoned())? = Some(now);
367            return Ok(());
368        }
369        if !round.contacted || !matches!(outcome, Ok(Turn::Quiet)) {
370            return Ok(());
371        }
372        let idle = peer.idle_turns.fetch_add(1, Ordering::Relaxed) + 1;
373        if idle < Self::IDLE_TURNS {
374            return Ok(());
375        }
376        peer.idle_turns.store(0, Ordering::Relaxed);
377        peer.source.disconnect();
378        peer.hold_off(now, Self::IDLE_BACKOFF)
379    }
380
381    /// Our established-connection limit refused `starved`: evict the
382    /// connected inventory peer that progressed longest ago, but only one
383    /// without progress for `IDLE_BACKOFF` (or ever) that has stayed connected
384    /// for more than one step. The victim is held off for `IDLE_BACKOFF`, in
385    /// sync and in discovery, so nothing redials it into the freed slot.
386    /// Returns whether its connection closed in time for `starved` to retry
387    /// in this step. One eviction per step, and no second one for `starved`
388    /// before that retry; the scheduler runs one request at a time, so no
389    /// other inventory peer is mid-request.
390    fn free_slot_for(
391        &self,
392        starved: &Peer,
393        step: u64,
394        stop: &AtomicBool,
395        deadline: Instant,
396        now: u64,
397    ) -> Result<bool, Error> {
398        let mut oldest: Option<(Arc<Peer>, Option<u64>)> = None;
399        for peer in self.snapshot()? {
400            if peer.id == starved.id || !evictable(&peer, step, now)? {
401                continue;
402            }
403            let progress = peer.last_progress()?;
404            if oldest.as_ref().is_none_or(|(_, best)| progress < *best) {
405                oldest = Some((peer, progress));
406            }
407        }
408        let Some((victim, _progress)) = oldest else {
409            return Ok(false);
410        };
411        victim.source.disconnect();
412        victim.hold_off(now, Self::IDLE_BACKOFF)?;
413        Ok(victim.wait_closed(stop, deadline))
414    }
415
416    /// A blamed turn disconnects the peer and backs it off; remote Busy turns
417    /// build a streak that disconnects the peer at `BUSY_TURNS`; a quiet turn
418    /// resets both. Retries and our own failures say nothing about the peer.
419    fn judge(
420        &self,
421        peer: &Peer,
422        outcome: Result<Turn, Error>,
423        now: u64,
424    ) -> Result<Result<(), Error>, Error> {
425        match outcome {
426            Ok(Turn::Quiet) => {
427                peer.busy_turns.store(0, Ordering::Relaxed);
428                peer.backoff.lock().map_err(|_| poisoned())?.failures = 0;
429                Ok(Ok(()))
430            }
431            Ok(Turn::Busy(error)) => {
432                let streak = peer.busy_turns.fetch_add(1, Ordering::Relaxed) + 1;
433                if streak < Self::BUSY_TURNS {
434                    return Ok(Err(error));
435                }
436                peer.busy_turns.store(0, Ordering::Relaxed);
437                peer.source.disconnect();
438                Ok(Err(Error::Busy(format!(
439                    "{error}; busy for {streak} turns, disconnected"
440                ))))
441            }
442            Ok(Turn::Blamed(error)) => {
443                peer.busy_turns.store(0, Ordering::Relaxed);
444                self.penalize(peer, now)?;
445                Ok(Err(error))
446            }
447            Ok(Turn::Retry(error)) | Err(error) => Ok(Err(error)),
448        }
449    }
450
451    fn penalize(&self, peer: &Peer, now: u64) -> Result<(), Error> {
452        peer.source.disconnect();
453        let mut backoff = peer.backoff.lock().map_err(|_| poisoned())?;
454        backoff.failures = backoff.failures.checked_add(1).unwrap_or(u32::MAX);
455        let delay = 2u64
456            .checked_pow(backoff.failures - 1)
457            .and_then(|factor| factor.checked_mul(Self::BACKOFF_SECONDS))
458            .map_or(Self::BACKOFF_CAP, |delay| delay.min(Self::BACKOFF_CAP));
459        backoff.until = Some(now.checked_add(delay).unwrap_or(u64::MAX));
460        drop(backoff);
461        let mut penalized = self.penalized.lock().map_err(|_| poisoned())?;
462        if !penalized.contains(&peer.peer_id) {
463            penalized.push(peer.peer_id);
464        }
465        Ok(())
466    }
467
468    /// Peers backed off since the last call, for the peer book to record.
469    pub(crate) fn drain_penalized(&self) -> Result<Vec<PeerId>, Error> {
470        let mut penalized = self.penalized.lock().map_err(|_| poisoned())?;
471        Ok(std::mem::take(&mut *penalized))
472    }
473
474    /// Reconcile with the peer book: known identities keep their peer state and
475    /// in-flight range, new ones join, ones no longer offered are dropped.
476    pub(crate) fn refresh(&self, offered: Vec<(String, PeerSource)>) -> Result<(), Error> {
477        if offered.len() > Self::MAX_PEERS {
478            return Err(Error::Invalid("too many inventory peers".into()));
479        }
480        let mut peers = self.peers.lock().map_err(|_| poisoned())?;
481        let mut next: Vec<Arc<Peer>> = Vec::with_capacity(offered.len());
482        for (label, source) in offered {
483            let candidate = inventory_peer(label, source)?;
484            if next.iter().any(|kept| kept.id == candidate.id) {
485                continue;
486            }
487            match peers.iter().find(|known| known.id == candidate.id) {
488                Some(known) => next.push(Arc::clone(known)),
489                None => next.push(Arc::new(candidate)),
490            }
491        }
492        *peers = next;
493        Ok(())
494    }
495
496    fn snapshot(&self) -> Result<Vec<Arc<Peer>>, Error> {
497        Ok(self.peers.lock().map_err(|_| poisoned())?.clone())
498    }
499
500    fn next_peer(&self, omitting: &[String], now: u64) -> Result<Option<Arc<Peer>>, Error> {
501        let peers = self.snapshot()?;
502        let count = peers.len();
503        for _ in 0..count {
504            let peer = &peers[self.turn.fetch_add(1, Ordering::Relaxed) % count];
505            if !omitting.contains(&peer.id) && peer.backoff_until(now)?.is_none() {
506                return Ok(Some(Arc::clone(peer)));
507            }
508        }
509        Ok(None)
510    }
511
512    fn advance(&self, peer: &Peer, engine: &Engine, round: &mut Round) -> Result<Turn, Error> {
513        let mut slot = peer.range.lock().map_err(|_| poisoned())?;
514        if slot.is_none() {
515            *slot = open(peer, engine)?;
516            // Nothing left to ask a fully synced peer: as fresh as progress,
517            // so surplus synced peers are not evicted every IDLE_BACKOFF.
518            round.progressed |= slot.is_none();
519        }
520        // A peer error or our own failure abandons the range; it is requested
521        // again from the persisted cursor on this peer's next turn.
522        match self.walk(peer, engine, &mut slot, round) {
523            Ok(()) | Err(Pause::Done) => Ok(Turn::Quiet),
524            Err(Pause::Abandon) | Err(Pause::Error(Error::Stale)) => {
525                *slot = None;
526                Ok(Turn::Quiet)
527            }
528            Err(Pause::Error(Error::PeerBusy)) => Ok(Turn::Busy(Error::PeerBusy)),
529            Err(Pause::Error(
530                retry @ (Error::Busy(_) | Error::Transport(_) | Error::DialDenied(_)),
531            )) => Ok(Turn::Retry(retry)),
532            Err(Pause::Blame(error)) => {
533                *slot = None;
534                Ok(Turn::Blamed(error))
535            }
536            Err(Pause::Error(error)) => {
537                *slot = None;
538                Err(error)
539            }
540        }
541    }
542
543    fn walk(
544        &self,
545        peer: &Peer,
546        engine: &Engine,
547        slot: &mut Option<Range>,
548        round: &mut Round,
549    ) -> Result<(), Pause> {
550        while let Some(range) = slot.as_mut() {
551            round.open()?;
552            if let Some(&entry) = range.entries.get(range.next) {
553                if !selected(engine, &entry)? {
554                    return Err(Pause::Abandon);
555                }
556                if engine.record(entry.txid)?.is_none() {
557                    if round.fetched == Self::ROUND_FETCHES {
558                        return Err(Pause::Done);
559                    }
560                    round.fetched += 1;
561                    pace(peer, round)?;
562                    let missing = &mut range.missing;
563                    round.progressed |=
564                        self.acquire(peer, engine, &entry, missing, round.deadline)?;
565                } else {
566                    round.progressed = true;
567                }
568                range.next += 1;
569            } else if range.more {
570                if round.requested_page {
571                    return Err(Pause::Done);
572                }
573                round.requested_page = true;
574                pace(peer, round)?;
575                let Some(page) = peer
576                    .source
577                    .inventory_until(
578                        (range.from, range.to),
579                        range.after,
580                        Self::PAGE_ENTRIES,
581                        round.stop,
582                        round.deadline,
583                    )
584                    .map_err(blame)?
585                else {
586                    return Err(Pause::Done);
587                };
588                if !accept(range, page).map_err(Pause::Blame)? {
589                    *slot = None;
590                }
591            } else {
592                let through = range.to.min(range.scanned_through);
593                engine.record_peer_coverage(&peer.id, range.from, through, range.generation)?;
594                round.progressed = true;
595                *slot = None;
596            }
597        }
598        Ok(())
599    }
600
601    /// Fetch one advertised record from the advertising peer and admit it. A
602    /// peer that answers Missing leaves the entry listed-only: it neither blocks
603    /// coverage nor is retried within this range. `MISSING_ANSWERS` Missing
604    /// answers in a row within one range blame the peer; an invalid package
605    /// always does. Returns whether a record was admitted.
606    fn acquire(
607        &self,
608        peer: &Peer,
609        engine: &Engine,
610        entry: &Placement,
611        missing: &mut u32,
612        deadline: Instant,
613    ) -> Result<bool, Pause> {
614        let fetched = peer
615            .source
616            .fetch_until(entry.txid, deadline)
617            .map_err(blame)?;
618        let Some(package) = fetched else {
619            *missing += 1;
620            if *missing < Self::MISSING_ANSWERS {
621                return Ok(false);
622            }
623            return Err(Pause::Blame(Error::Invalid(
624                "peer reported advertised records missing".into(),
625            )));
626        };
627        *missing = 0;
628        let receipt = match engine.admit_record(package) {
629            Ok(receipt) => receipt,
630            Err(invalid @ Error::Invalid(_)) => {
631                self.rejected.fetch_add(1, Ordering::Relaxed);
632                return Err(Pause::Blame(invalid));
633            }
634            Err(local) => return Err(Pause::Error(local)),
635        };
636        self.admitted.fetch_add(1, Ordering::Relaxed);
637        if !placed(&receipt, entry) {
638            return Err(Pause::Blame(Error::Invalid(
639                "peer inventory placement disagrees with its proven record".into(),
640            )));
641        }
642        Ok(true)
643    }
644
645    pub(crate) fn status(&self, engine: &Engine) -> Result<OverlayStatus, Error> {
646        let start = engine.status()?.application_start.height;
647        let now = unix_seconds()?;
648        let known = self.snapshot()?;
649        let mut peers = Vec::with_capacity(known.len());
650        for peer in &known {
651            let cursor = engine.peer_cursor(&peer.id)?;
652            let synced_through = cursor.checked_sub(1).filter(|height| *height >= start);
653            let recorded = peer.last_error.lock().map_err(|_| poisoned())?.clone();
654            let (last_error, last_error_at) = RecordedError::split(recorded);
655            peers.push(OverlayPeerStatus {
656                label: peer.label.clone(),
657                cursor,
658                synced_through,
659                last_error,
660                last_error_at,
661                backoff_until: peer.backoff_until(now)?,
662                last_progress_at: peer.last_progress()?,
663            });
664        }
665        Ok(OverlayStatus {
666            peer_id: self.info.peer_id.clone(),
667            listen: self.info.listen.clone(),
668            peer_book: self.info.peer_book,
669            connected: (self.info.connected)(),
670            admitted: self.admitted.load(Ordering::Relaxed),
671            rejected: self.rejected.load(Ordering::Relaxed),
672            peers,
673            discovery: None,
674        })
675    }
676}
677
678/// Keep requests to one peer at least `REQUEST_SPACING` apart, waiting in
679/// short slices so cancellation and the round deadline stay responsive. The
680/// round counts as having asked the peer once pacing lets a request through.
681fn pace(peer: &Peer, round: &mut Round) -> Result<(), Pause> {
682    let mut last = peer.last_request.lock().map_err(|_| poisoned())?;
683    if let Some(previous) = *last {
684        let ready = previous + Scheduler::REQUEST_SPACING;
685        while Instant::now() < ready {
686            round.open()?;
687            let wait = ready.saturating_duration_since(Instant::now());
688            std::thread::sleep(wait.min(Duration::from_millis(50)));
689        }
690    }
691    round.open()?;
692    *last = Some(Instant::now());
693    round.contacted = true;
694    Ok(())
695}
696
697/// The next unclaimed range for this peer, bounded by the local selected tip.
698/// The range keeps the generation observed here for its whole lifetime, so a
699/// reorg between rounds makes its coverage claim Stale instead of mislabelled.
700fn open(peer: &Peer, engine: &Engine) -> Result<Option<Range>, Error> {
701    let status = engine.status()?;
702    let Some(tip) = status.selected_tip else {
703        return Ok(None);
704    };
705    let from = engine.peer_cursor(&peer.id)?;
706    if from > tip.height {
707        return Ok(None);
708    }
709    let span_end = from
710        .checked_add(Scheduler::RANGE_HEIGHTS - 1)
711        .ok_or_else(|| Error::Invalid("peer cursor overflows the height range".into()))?;
712    Ok(Some(Range {
713        from,
714        to: tip.height.min(span_end),
715        after: None,
716        entries: Vec::new(),
717        next: 0,
718        more: true,
719        scanned_through: 0,
720        generation: status.generation,
721        missing: 0,
722    }))
723}
724
725/// Install a page. Returns false when the peer has not scanned this range yet.
726/// Entries must be strictly ordered after the page cursor and inside the
727/// requested range; anything else is a peer error. Entries above the peer's
728/// own scanned prefix (blocks still waiting on a dependency) are dropped
729/// without blame: they are outside what the page can claim.
730fn accept(range: &mut Range, page: InventoryPage) -> Result<bool, Error> {
731    let Some(scanned_through) = page.scanned_through.filter(|height| *height >= range.from) else {
732        return Ok(false);
733    };
734    let limit = usize::try_from(Scheduler::PAGE_ENTRIES)
735        .map_err(|error| Error::Invalid(format!("page limit: {error}")))?;
736    if page.entries.len() > limit || (page.more && page.entries.is_empty()) {
737        return Err(Error::Invalid("peer inventory page size is invalid".into()));
738    }
739    let mut previous = range.after;
740    for entry in &page.entries {
741        if entry.height < range.from
742            || entry.height > range.to
743            || previous
744                .is_some_and(|last| (entry.height, entry.position) <= (last.height, last.position))
745        {
746            return Err(Error::Invalid(
747                "peer inventory entry is out of range or order".into(),
748            ));
749        }
750        previous = Some(*entry);
751    }
752    range.after = previous;
753    range.entries = page
754        .entries
755        .into_iter()
756        .filter(|entry| entry.height <= scanned_through)
757        .collect();
758    range.next = 0;
759    range.more = page.more;
760    range.scanned_through = scanned_through;
761    Ok(true)
762}
763
764/// A listing whose block differs from our selected header at that height comes
765/// from another view of the chain (a reorg in flight); it is not a peer error.
766fn selected(engine: &Engine, entry: &Placement) -> Result<bool, Error> {
767    match engine.archive()?.ticket(entry.height) {
768        Ok(ticket) => Ok(ticket.point().hash == entry.block_hash),
769        Err(Error::Invalid(_outside_selected_view)) => Ok(false),
770        Err(error) => Err(error),
771    }
772}
773
774fn placed(receipt: &RecordReceipt, entry: &Placement) -> bool {
775    receipt.block.height == entry.height
776        && receipt.block.hash == entry.block_hash
777        && receipt.position == entry.position
778        && receipt.txid == entry.txid
779        && receipt.wtxid == entry.wtxid
780}