Skip to main content

tycho_simulation/snapshot_feed/
http.rs

1//! Feeds that poll: the shared poll loop, the source trait a provider implements for it, and
2//! the one-request helper those sources fetch with.
3
4use std::{future::Future, time::Duration};
5
6use async_stream::try_stream;
7use bytes::Bytes;
8use futures::Stream;
9use reqwest::{RequestBuilder, Response};
10use serde::de::DeserializeOwned;
11use tokio::time::{interval, sleep, timeout, MissedTickBehavior};
12use tracing::{debug_span, error, info, warn, Instrument};
13
14use super::{errors::FeedError, failures::FailureTracker, publisher::Publisher};
15
16/// Feed tuning for HTTP-polling feeds. Set through the feed builder.
17///
18/// There is no `Default`: start from the builder's `default_feed_config()`, which holds the
19/// values that fit that venue, and change individual fields with struct-update syntax.
20#[derive(Clone, Debug)]
21pub struct HttpFeedConfig {
22    /// Minimum time between the starts of two polls, whatever a poll costs: one that outruns the
23    /// interval is followed by the next as soon as it answers, and the cadence runs on from
24    /// there. Must not be zero: the feed future resolves with `InvalidInput` right away.
25    pub poll_interval: Duration,
26    /// Deadline for one poll's requests; a hung poll counts as a failed poll.
27    pub request_timeout: Duration,
28    /// How many times a run of failed polls may double the gap between them before it stops
29    /// growing, so a provider that is down or rate-limiting is not polled at the healthy cadence
30    /// for the whole outage. The first failure waits one `poll_interval`, the second two, and so
31    /// on up to `poll_interval * 2^max_backoff_exp`; `0` polls a failing provider at the healthy
32    /// cadence.
33    pub max_backoff_exp: u32,
34    /// Consecutive failed polls before the feed gives up and resolves with an error.
35    /// `None` retries forever.
36    pub max_consecutive_failures: Option<u32>,
37    /// Withdraw the published snapshot (publish `None`) once it is this old, so consumers stop
38    /// using data nobody is refreshing; the next successful poll restores it. Age runs from the
39    /// last publish on the monotonic clock, so it bounds staleness whether a failing provider
40    /// refuses instantly or hangs to `request_timeout`. An age that does not exceed
41    /// `poll_interval` is a duty cycle: each snapshot served for this long, then nothing until
42    /// the next poll. Zero is `InvalidInput`; `None` keeps the last snapshot until the feed
43    /// ends.
44    pub max_snapshot_age: Option<Duration>,
45}
46
47/// The values every HTTP feed builder starts from. The relation among them worth keeping through
48/// a retune: the withdrawal age outlasts a poll that hangs to its deadline, so one slow poll does
49/// not blank the book.
50pub fn default_http_feed_config() -> HttpFeedConfig {
51    HttpFeedConfig {
52        poll_interval: Duration::from_secs(5),
53        request_timeout: Duration::from_secs(10),
54        max_backoff_exp: 3,
55        max_consecutive_failures: None,
56        max_snapshot_age: Some(Duration::from_secs(30)),
57    }
58}
59
60/// The poll loop shared by all HTTP-based feeds.
61///
62/// Polls `source` on the cadence `config` asks for and publishes every snapshot it answers with,
63/// withdrawing one that goes `max_snapshot_age` without a refresh. What each knob buys is on its
64/// own field; this is where they are checked and handed to the pieces that act on them.
65///
66/// Resolves `Err` when the feed gives up: on a fatal poll error, whatever the failure budget,
67/// after `max_consecutive_failures` failed polls where one is configured, or right away with
68/// `InvalidInput` for a zero `poll_interval` or a zero `max_snapshot_age`. Resolves `Ok(())` as
69/// soon as the last receiver goes away, mid-poll included, since nothing it fetched from then on
70/// could reach anyone; polling itself never runs out, so a feed is otherwise stopped by dropping
71/// its future, which withdraws the published snapshot on the way out.
72pub async fn run_http_poll_feed<S: HttpSource>(
73    config: HttpFeedConfig,
74    publisher: Publisher<S::Snapshot>,
75    source: S,
76) -> Result<(), FeedError> {
77    // Where the config comes apart, and the only place that sees all of it: the loop waits on
78    // the two deadlines and spends the failure budget, the publisher withdraws what has gone
79    // stale.
80    let HttpFeedConfig {
81        poll_interval,
82        request_timeout,
83        max_backoff_exp,
84        max_consecutive_failures,
85        max_snapshot_age,
86    } = config;
87
88    if poll_interval.is_zero() {
89        return Err(FeedError::InvalidInput("poll_interval must not be zero".to_string()));
90    }
91    if let Some(age) = max_snapshot_age {
92        if age.is_zero() {
93            return Err(FeedError::InvalidInput("max_snapshot_age must not be zero".to_string()));
94        }
95        if age <= poll_interval {
96            // Deliberate for a venue whose rate limit sets the cadence, a mistake otherwise, and
97            // indistinguishable from here — so it is said once rather than refused.
98            warn!(
99                max_snapshot_age = ?age,
100                ?poll_interval,
101                "max_snapshot_age does not exceed poll_interval: this feed will serve each \
102                 snapshot for max_snapshot_age and nothing until the next poll"
103            );
104        }
105    }
106
107    let snapshots = polled_snapshots(
108        poll_interval,
109        request_timeout,
110        max_consecutive_failures,
111        max_backoff_exp,
112        source,
113    );
114    publisher
115        .publishing(max_snapshot_age, snapshots)
116        .await
117}
118
119/// The provider-specific half of an HTTP-polling feed: one call fetches one complete snapshot.
120/// [`run_http_poll_feed`] owns the source and drives it by reference.
121pub trait HttpSource: Send + Sync {
122    /// The complete snapshot one poll yields. Each snapshot replaces the previous one, so it must
123    /// not be a delta.
124    type Snapshot: Send + Sync + 'static;
125
126    /// Fetches one complete snapshot. The loop calls this once per poll and gives it the config's
127    /// `request_timeout`; an implementation may be written as `async fn` as long as its future is
128    /// `Send`.
129    ///
130    /// Return [`FeedError::Fatal`] when polling again cannot produce a different answer; this
131    /// ends the feed. Any other error, like a poll that overruns `request_timeout`, counts as one
132    /// failed poll toward `max_consecutive_failures` (unset by default, which polls forever) and
133    /// backs off before the next poll. Only `Ok` clears the failure count.
134    fn fetch(&self) -> impl Future<Output = Result<Self::Snapshot, FeedError>> + Send;
135}
136
137/// Every snapshot the provider answers with, and as a last item the error the feed gives up on.
138///
139/// One fetch is one poll, started every `poll_interval` and given `request_timeout` to answer; an
140/// error or an overrun is one failed poll. Failures back off, and run out, as [`HttpFeedConfig`]
141/// describes. Each fetch runs in a `poll` span.
142///
143/// Waiting out the cadence and the backoff happens in here, so the one who awaits the next
144/// snapshot waits exactly as long as it takes — which is how a published one can go stale on time
145/// while a poll is still in flight.
146fn polled_snapshots<S: HttpSource>(
147    poll_interval: Duration,
148    request_timeout: Duration,
149    max_consecutive_failures: Option<u32>,
150    max_backoff_exp: u32,
151    source: S,
152) -> impl Stream<Item = Result<S::Snapshot, FeedError>> {
153    // A poller backs off by stretching the cadence it already has, so one poll interval is the
154    // unit its failures double — which is what lets a backoff stand in for a tick below.
155    let mut failures =
156        FailureTracker::new(max_consecutive_failures, poll_interval, max_backoff_exp);
157
158    let mut ticker = interval(poll_interval);
159    // A poll slower than the interval runs through ticks. `Delay` drops them and re-anchors the
160    // cadence on the poll that outran them, which is the only behaviour that keeps every gap at
161    // the interval: `Burst` owes one poll per tick missed and fires them at once, and `Skip`
162    // stays on the original phase, so its next poll can follow a late one by a fraction of the
163    // interval.
164    ticker.set_missed_tick_behavior(MissedTickBehavior::Delay);
165
166    // Set by a failed poll and taken by the wait that serves it out, so a poll that succeeds
167    // leaves nothing behind for the next one to wait through.
168    let mut backoff = None;
169
170    info!(?poll_interval, "starting polling");
171
172    try_stream! {
173        loop {
174            // The backoff stands in for the tick: it is already a multiple of the interval, and
175            // the ticker's own deadline expires while it waits.
176            match backoff.take() {
177                Some(backoff) => {
178                    sleep(backoff).await;
179                    // The tick this backoff stood in for came due while it was waiting, and an
180                    // overdue tick completes at once: without restarting the interval here, the
181                    // poll after a recovered one would follow it immediately.
182                    ticker.reset();
183                }
184                None => {
185                    ticker.tick().await;
186                }
187            }
188            let polled = timeout(request_timeout, source.fetch())
189                .instrument(debug_span!("poll"))
190                .await;
191
192            let error = match polled {
193                Ok(Ok(snapshot)) => {
194                    let cleared = failures.record_success();
195                    if cleared > 0 {
196                        info!(after_failures = cleared, "poll succeeded again");
197                    }
198                    yield snapshot;
199                    continue;
200                }
201                Ok(Err(e)) if e.is_fatal() => Err(e)?,
202                Ok(Err(e)) => e,
203                Err(_) => FeedError::Connection(format!(
204                    "poll timed out after {request_timeout:?}"
205                )),
206            };
207
208            let retry = failures
209                .record_failure()
210                .inspect(|retry| {
211                    warn!(
212                        consecutive = retry.consecutive,
213                        backoff = ?retry.backoff,
214                        %error,
215                        "poll failed, backing off"
216                    )
217                })
218                // The feed ends with the failure that ended it, classified as its source
219                // classified it: a run of unparseable answers is not a connection failure, and
220                // how long the feed tolerated it is for the log rather than for the caller to
221                // act on.
222                .map_err(|consecutive| {
223                    error!(consecutive, "giving up on the feed");
224                    error
225                })?;
226            backoff = Some(retry.backoff);
227        }
228    }
229}
230
231/// Whether a failing response's body says the venue will never serve this request, such as a
232/// chain it does not price. A refusal is a [`FeedError::Fatal`] rather than the retryable
233/// [`FeedError::Connection`], since another attempt gets the same answer. Every fetch here
234/// takes one, because what a failing status means is the venue's to say and not this module's.
235///
236/// A plain `fn`, so the verdict rests on what the venue said and on nothing else.
237pub type RefusalCheck = fn(&str) -> bool;
238
239/// For a venue that refuses nothing outright, so every failing answer is worth another attempt.
240pub fn nothing_refused(_body: &str) -> bool {
241    false
242}
243
244/// Sends `request` and returns its body. A transport failure is a [`FeedError::Connection`], a
245/// failing status whatever `refused` makes of it; `what` names the resource in the message
246/// (e.g. "Hashflow price levels").
247///
248/// For an API that reports failures inside the body of an otherwise successful response, which
249/// therefore has to be classified before it is parsed. [`fetch_json`] is the plain case.
250pub(crate) async fn fetch_bytes(
251    request: RequestBuilder,
252    what: &str,
253    refused: RefusalCheck,
254) -> Result<Bytes, FeedError> {
255    let response = request
256        .send()
257        .await
258        .map_err(|e| FeedError::Connection(format!("Failed to fetch {what}: {e}")))?;
259    read_bytes(response, what, refused).await
260}
261
262/// Reads `response`'s body, failing a non-success status with whatever the server said — as a
263/// refusal when `refused` recognises it there. `what` names the resource, as in [`fetch_bytes`].
264async fn read_bytes(
265    response: Response,
266    what: &str,
267    refused: RefusalCheck,
268) -> Result<Bytes, FeedError> {
269    // The body is read before the status is judged, so a failing response can report what the
270    // server said, and so the venue can be asked what it meant by it.
271    let status = response.status();
272    let body = response
273        .bytes()
274        .await
275        .map_err(|e| FeedError::Connection(format!("Failed to read {what} response: {e}")))?;
276
277    if !status.is_success() {
278        let said = String::from_utf8_lossy(&body);
279        let message = format!("{what} HTTP error {status}: {said}");
280        return Err(if refused(&said) {
281            FeedError::Fatal(message)
282        } else {
283            FeedError::Connection(message)
284        });
285    }
286    Ok(body)
287}
288
289/// Sends `request` and parses its JSON body, fetching as `fetch_bytes` does; an unparseable
290/// body is a [`FeedError::Parsing`] naming `what`.
291pub async fn fetch_json<T: DeserializeOwned>(
292    request: RequestBuilder,
293    what: &str,
294    refused: RefusalCheck,
295) -> Result<T, FeedError> {
296    parse_json(&fetch_bytes(request, what, refused).await?, what)
297}
298
299/// Parses `response`'s JSON body, the second half of [`fetch_json`] for a caller that sends the
300/// request itself.
301pub(crate) async fn read_json<T: DeserializeOwned>(
302    response: Response,
303    what: &str,
304    refused: RefusalCheck,
305) -> Result<T, FeedError> {
306    parse_json(&read_bytes(response, what, refused).await?, what)
307}
308
309fn parse_json<T: DeserializeOwned>(body: &[u8], what: &str) -> Result<T, FeedError> {
310    serde_json::from_slice(body)
311        .map_err(|e| FeedError::Parsing(format!("Failed to parse {what} response: {e}")))
312}
313
314/// A local HTTP/1.1 server for feed tests: answers every request with `respond`'s status line
315/// and JSON body. Serves until the runtime drops it.
316#[cfg(test)]
317pub(crate) mod test_support {
318    use std::{
319        net::SocketAddr,
320        sync::{
321            atomic::{AtomicUsize, Ordering},
322            Arc,
323        },
324        time::Duration,
325    };
326
327    use tokio::{
328        io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader},
329        net::TcpListener,
330    };
331
332    /// One canned HTTP response; `(status_line, body)` converts into it.
333    pub struct MockResponse {
334        pub status: &'static str,
335        pub body: String,
336        /// Wait before answering, for tests that exercise request deadlines.
337        pub delay: Duration,
338    }
339
340    impl From<(&'static str, String)> for MockResponse {
341        fn from((status, body): (&'static str, String)) -> Self {
342            MockResponse { status, body, delay: Duration::ZERO }
343        }
344    }
345
346    /// One request as the mock server read it.
347    pub struct MockRequest {
348        /// Path and query string, as the request line carries them.
349        pub target: String,
350        /// The body, empty where the request had none. Read in full, so a client that sends one
351        /// is not left writing into a socket nobody drains.
352        pub body: String,
353    }
354
355    /// A running mock server: where it listens and how many requests it has answered.
356    pub struct MockHttpServer {
357        pub address: SocketAddr,
358        pub requests: Arc<AtomicUsize>,
359    }
360
361    impl MockHttpServer {
362        pub fn url(&self) -> String {
363            format!("http://{}", self.address)
364        }
365
366        pub fn request_count(&self) -> usize {
367            self.requests.load(Ordering::SeqCst)
368        }
369    }
370
371    /// The headers and however many bytes of body `Content-Length` announced. A request that
372    /// announces none has none: nothing here speaks chunked transfer encoding.
373    async fn read_body(reader: &mut BufReader<tokio::net::TcpStream>) -> String {
374        let mut length = 0;
375        loop {
376            let mut header = String::new();
377            match reader.read_line(&mut header).await {
378                Ok(0) => break,
379                Err(_) => break,
380                Ok(_) if header.trim().is_empty() => break,
381                Ok(_) => {}
382            }
383            if let Some((name, value)) = header.split_once(':') {
384                if name.eq_ignore_ascii_case("content-length") {
385                    length = value.trim().parse().unwrap_or(0);
386                }
387            }
388        }
389        let mut body = vec![0; length];
390        if length > 0 {
391            #[expect(
392                clippy::let_underscore_must_use,
393                reason = "the fake peer may already be gone; that is not a test failure"
394            )]
395            let _ = reader.read_exact(&mut body).await;
396        }
397        String::from_utf8_lossy(&body).into_owned()
398    }
399
400    /// Serves HTTP/1.1 requests until the runtime drops it. `route` maps a request to a
401    /// response; a request it returns `None` for gets a 404. Each request is counted before
402    /// `route` runs, so a route closure holding its own counter can answer per attempt.
403    ///
404    /// Each connection is served by its own task, so a response that waits out its `delay` holds
405    /// up nobody else: a client that retries gets its second answer a delay after it asked, not
406    /// a delay after the first one finished.
407    pub async fn spawn_http_server<R: Into<MockResponse>>(
408        route: impl Fn(&MockRequest) -> Option<R> + Send + Sync + 'static,
409    ) -> MockHttpServer {
410        let listener = TcpListener::bind("127.0.0.1:0")
411            .await
412            .unwrap();
413        let address = listener.local_addr().unwrap();
414        let requests = Arc::new(AtomicUsize::new(0));
415        let served = Arc::clone(&requests);
416        let route = Arc::new(route);
417        tokio::spawn(async move {
418            while let Ok((stream, _)) = listener.accept().await {
419                let (served, route) = (Arc::clone(&served), Arc::clone(&route));
420                tokio::spawn(async move {
421                    let mut reader = BufReader::new(stream);
422                    let mut request_line = String::new();
423                    if reader
424                        .read_line(&mut request_line)
425                        .await
426                        .is_err()
427                    {
428                        return;
429                    }
430                    served.fetch_add(1, Ordering::SeqCst);
431                    let request = MockRequest {
432                        target: request_line
433                            .split_whitespace()
434                            .nth(1)
435                            .unwrap_or_default()
436                            .to_string(),
437                        body: read_body(&mut reader).await,
438                    };
439                    let response = route(&request)
440                        .map(Into::into)
441                        .unwrap_or_else(|| {
442                            MockResponse::from((
443                                "404 Not Found",
444                                format!("no route for {}", request.target),
445                            ))
446                        });
447
448                    tokio::time::sleep(response.delay).await;
449
450                    let MockResponse { status, body, .. } = response;
451                    let payload = format!(
452                    "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
453                    body.len()
454                );
455                    let mut stream = reader.into_inner();
456                    #[expect(
457                        clippy::let_underscore_must_use,
458                        reason = "the fake peer may already be gone; that is not a test failure"
459                    )]
460                    let _ = stream
461                        .write_all(payload.as_bytes())
462                        .await;
463                    #[expect(
464                        clippy::let_underscore_must_use,
465                        reason = "the fake peer may already be gone; that is not a test failure"
466                    )]
467                    let _ = stream.shutdown().await;
468                });
469            }
470        });
471        MockHttpServer { address, requests }
472    }
473}
474
475#[cfg(test)]
476mod tests {
477    use std::{
478        future::Future,
479        sync::{
480            atomic::{AtomicBool, AtomicU32, Ordering},
481            Arc,
482        },
483    };
484
485    use futures::StreamExt as _;
486    use tokio::sync::watch;
487    use tokio_stream::wrappers::WatchStream;
488
489    use super::{super::expect_to_finish, *};
490
491    /// Lets a closure stand in for a provider's source.
492    impl<F, Fut, T> HttpSource for F
493    where
494        F: Fn() -> Fut + Send + Sync,
495        Fut: Future<Output = Result<T, FeedError>> + Send,
496        T: Send + Sync + 'static,
497    {
498        type Snapshot = T;
499
500        async fn fetch(&self) -> Result<T, FeedError> {
501            self().await
502        }
503    }
504
505    /// The one snapshot the mock sources here publish.
506    fn snapshot() -> u32 {
507        1
508    }
509
510    fn config(max_consecutive_failures: Option<u32>) -> HttpFeedConfig {
511        HttpFeedConfig {
512            poll_interval: Duration::from_millis(5),
513            request_timeout: Duration::from_millis(50),
514            // A failed poll retries at the polling cadence: the tests that fail on purpose are
515            // timing the loop, not the backoff curve.
516            max_backoff_exp: 0,
517            max_consecutive_failures,
518            max_snapshot_age: None,
519        }
520    }
521
522    #[tokio::test]
523    async fn publishes_a_snapshot_and_success_resets_failures() {
524        // One failed poll, then successes: with a limit of 2 the single failure must not
525        // accumulate towards termination once a poll succeeds.
526        let polls = Arc::new(AtomicU32::new(0));
527        let polls_clone = Arc::clone(&polls);
528        let (publisher, mut rx) = Publisher::channel();
529        let feed = run_http_poll_feed(config(Some(2)), publisher, move || {
530            let n = polls_clone.fetch_add(1, Ordering::SeqCst);
531            async move {
532                if n == 0 {
533                    Err(FeedError::Connection("boom".to_string()))
534                } else {
535                    Ok(snapshot())
536                }
537            }
538        });
539        let feed = tokio::spawn(feed);
540
541        expect_to_finish("no snapshot published", async {
542            loop {
543                rx.changed().await.expect("feed ended");
544                if rx.borrow_and_update().is_some() {
545                    return;
546                }
547            }
548        })
549        .await;
550
551        // Several more polls run without termination.
552        tokio::time::sleep(Duration::from_millis(50)).await;
553        assert!(!feed.is_finished());
554    }
555
556    #[tokio::test]
557    async fn withdraws_a_snapshot_that_goes_unrefreshed_and_restores_on_next_success() {
558        // One successful poll, then failures until the flag flips: the published snapshot
559        // must be withdrawn once it has gone max_snapshot_age without a refresh and come back
560        // with the next successful poll.
561        let polls = Arc::new(AtomicU32::new(0));
562        let polls_clone = Arc::clone(&polls);
563        let recovered = Arc::new(AtomicBool::new(false));
564        let recovered_clone = Arc::clone(&recovered);
565        let (publisher, mut rx) = Publisher::channel();
566        let config =
567            HttpFeedConfig { max_snapshot_age: Some(Duration::from_millis(20)), ..config(None) };
568        let feed = run_http_poll_feed(config, publisher, move || {
569            let n = polls_clone.fetch_add(1, Ordering::SeqCst);
570            let recovered = recovered_clone.load(Ordering::SeqCst);
571            async move {
572                if n == 0 || recovered {
573                    Ok(snapshot())
574                } else {
575                    Err(FeedError::Connection("boom".to_string()))
576                }
577            }
578        });
579        let _feed = tokio::spawn(feed);
580
581        let wait_for = |rx: &mut watch::Receiver<Option<u32>>, want_some: bool| {
582            let mut rx = rx.clone();
583            async move {
584                expect_to_finish("watch did not reach the expected state", async {
585                    loop {
586                        if rx.borrow_and_update().is_some() == want_some {
587                            return;
588                        }
589                        rx.changed().await.expect("feed ended");
590                    }
591                })
592                .await;
593            }
594        };
595        wait_for(&mut rx, true).await;
596        wait_for(&mut rx, false).await;
597        recovered.store(true, Ordering::SeqCst);
598        wait_for(&mut rx, true).await;
599    }
600
601    #[tokio::test]
602    async fn rejects_zero_poll_interval() {
603        let (publisher, _rx) = Publisher::<u32>::channel();
604        let config = HttpFeedConfig { poll_interval: Duration::ZERO, ..config(None) };
605        let result = run_http_poll_feed(config, publisher, || async { Ok(snapshot()) }).await;
606        assert!(matches!(result, Err(FeedError::InvalidInput(_))));
607    }
608
609    #[tokio::test]
610    async fn rejects_a_zero_max_snapshot_age() {
611        let (publisher, _rx) = Publisher::<u32>::channel();
612        let config = HttpFeedConfig { max_snapshot_age: Some(Duration::ZERO), ..config(None) };
613        let result = run_http_poll_feed(config, publisher, || async { Ok(snapshot()) }).await;
614        assert!(matches!(result, Err(FeedError::InvalidInput(_))));
615    }
616
617    /// A snapshot may be servable for less than the gap between polls: a consumer that would
618    /// rather decline than quote an old book asks for exactly that when the venue's rate limit
619    /// sets the cadence. The feed serves each snapshot for `max_snapshot_age` and withdraws it
620    /// until the next poll.
621    #[tokio::test(start_paused = true)]
622    async fn a_max_snapshot_age_below_the_poll_interval_is_a_duty_cycle() {
623        let (publisher, rx) = Publisher::<u32>::channel();
624        let mut changes = WatchStream::from_changes(rx);
625        let config = HttpFeedConfig {
626            poll_interval: Duration::from_millis(100),
627            max_snapshot_age: Some(Duration::from_millis(20)),
628            ..config(None)
629        };
630        let _feed =
631            tokio::spawn(run_http_poll_feed(config, publisher, || async { Ok(snapshot()) }));
632
633        let start = tokio::time::Instant::now();
634        let mut seen = Vec::new();
635        for _ in 0..4 {
636            let change = expect_to_finish("the feed stopped changing", changes.next()).await;
637            seen.push((change.expect("the feed is still running"), start.elapsed()));
638        }
639
640        assert_eq!(
641            seen,
642            [
643                // The first poll fires immediately; each snapshot stands for its 20 ms and the
644                // feed serves nothing for the remaining 80 ms of the interval.
645                (Some(snapshot()), Duration::ZERO),
646                (None, Duration::from_millis(20)),
647                (Some(snapshot()), Duration::from_millis(100)),
648                (None, Duration::from_millis(120)),
649            ]
650        );
651    }
652
653    #[tokio::test]
654    async fn gives_up_after_max_consecutive_failed_polls() {
655        // A provider that answers every time with something unparseable is not a connection
656        // failure, so the feed ends with the failure that ended it, as its source classified it.
657        let (publisher, _rx) = Publisher::<u32>::channel();
658        let feed = run_http_poll_feed(config(Some(3)), publisher, || async {
659            Err(FeedError::Parsing("not a snapshot".to_string()))
660        });
661        let feed = tokio::spawn(feed);
662
663        let result = expect_to_finish("feed did not give up", feed)
664            .await
665            .unwrap();
666        assert!(matches!(result, Err(FeedError::Parsing(msg)) if msg == "not a snapshot"));
667    }
668
669    #[tokio::test]
670    async fn gives_up_on_a_fatal_poll_error_despite_an_unlimited_failure_budget() {
671        // A rejected key or an unsupported chain answers every poll the same way, so the
672        // feed stops instead of asking forever — and the snapshot it had goes with it.
673        let (publisher, mut rx) = Publisher::channel();
674        let polls = Arc::new(AtomicU32::new(0));
675        let polls_clone = Arc::clone(&polls);
676        let feed = run_http_poll_feed(config(None), publisher, move || {
677            let n = polls_clone.fetch_add(1, Ordering::SeqCst);
678            async move {
679                if n == 0 {
680                    Ok(snapshot())
681                } else {
682                    Err(FeedError::Fatal("api key rejected".to_string()))
683                }
684            }
685        });
686        let feed = tokio::spawn(feed);
687
688        let result = expect_to_finish("feed did not give up", feed)
689            .await
690            .unwrap();
691
692        assert!(matches!(result, Err(FeedError::Fatal(_))));
693        assert_eq!(polls.load(Ordering::SeqCst), 2, "the feed polled again after the fatal");
694        assert!(rx.borrow_and_update().is_none(), "the snapshot must be withdrawn");
695    }
696
697    #[tokio::test]
698    async fn withdraws_the_published_snapshot_when_it_gives_up() {
699        // A snapshot published before the feed gave up must not stay servable: nothing is
700        // refreshing it, and a consumer holding the receiver would keep using it.
701        let polls = Arc::new(AtomicU32::new(0));
702        let polls_clone = Arc::clone(&polls);
703        let (publisher, mut rx) = Publisher::channel();
704        let feed = run_http_poll_feed(config(Some(2)), publisher, move || {
705            let n = polls_clone.fetch_add(1, Ordering::SeqCst);
706            async move {
707                if n == 0 {
708                    Ok(snapshot())
709                } else {
710                    Err(FeedError::Connection("boom".to_string()))
711                }
712            }
713        });
714        let feed = tokio::spawn(feed);
715
716        expect_to_finish("no snapshot published", async {
717            loop {
718                rx.changed()
719                    .await
720                    .expect("feed ended before publishing");
721                if rx.borrow_and_update().is_some() {
722                    return;
723                }
724            }
725        })
726        .await;
727
728        let result = expect_to_finish("feed did not give up", feed)
729            .await
730            .unwrap();
731        assert!(matches!(result, Err(FeedError::Connection(_))));
732        assert!(rx.borrow().is_none(), "the snapshot must be withdrawn when the feed gives up");
733    }
734
735    #[tokio::test]
736    async fn withdraws_the_published_snapshot_when_the_feed_is_dropped() {
737        // Dropping the feed future — the consumer aborting its task — leaves the receiver
738        // alive; the snapshot stops being refreshed and must stop being served with it.
739        let (publisher, mut rx) = Publisher::channel();
740        let feed = run_http_poll_feed(config(None), publisher, || async { Ok(snapshot()) });
741        let feed = tokio::spawn(feed);
742
743        expect_to_finish("no snapshot published", async {
744            loop {
745                rx.changed()
746                    .await
747                    .expect("feed ended before publishing");
748                if rx.borrow_and_update().is_some() {
749                    return;
750                }
751            }
752        })
753        .await;
754
755        feed.abort();
756        expect_to_finish("the snapshot was not withdrawn", rx.changed())
757            .await
758            .expect("the withdrawal must reach the receiver before the sender drops");
759        assert!(rx.borrow().is_none());
760    }
761
762    #[tokio::test(start_paused = true)]
763    async fn failing_polls_stretch_the_polling_cadence() {
764        // A 50 ms cadence doubling at most three times: a provider that is down is polled at 0
765        // and then after 50, 100, 200, 400 and 400 ms — 5 attempts in the first second, where the
766        // healthy cadence would make 20.
767        let attempts = Arc::new(std::sync::Mutex::new(Vec::new()));
768        let attempts_clone = Arc::clone(&attempts);
769        let start = tokio::time::Instant::now();
770        let (publisher, _rx) = Publisher::<u32>::channel();
771        let feed = tokio::spawn(run_http_poll_feed(
772            HttpFeedConfig {
773                poll_interval: Duration::from_millis(50),
774                max_backoff_exp: 3,
775                ..config(None)
776            },
777            publisher,
778            move || {
779                attempts_clone
780                    .lock()
781                    .unwrap()
782                    .push(start.elapsed().as_millis());
783                async { Err(FeedError::Connection("boom".to_string())) }
784            },
785        ));
786
787        tokio::time::sleep(Duration::from_secs(1)).await;
788        feed.abort();
789
790        assert_eq!(*attempts.lock().unwrap(), vec![0, 50, 150, 350, 750]);
791    }
792
793    #[tokio::test(start_paused = true)]
794    async fn a_recovered_poll_resumes_the_cadence_instead_of_catching_up() {
795        // Two failures stretch the 50 ms cadence to 50 and 100 ms, so the third attempt polls at
796        // 150 ms and succeeds. The polls after it are one plain interval apart: the backoff took
797        // the place of those ticks, so none of them are owed.
798        let attempts = Arc::new(std::sync::Mutex::new(Vec::new()));
799        let attempts_clone = Arc::clone(&attempts);
800        let start = tokio::time::Instant::now();
801        let (publisher, _rx) = Publisher::<u32>::channel();
802        let feed = tokio::spawn(run_http_poll_feed(
803            HttpFeedConfig {
804                poll_interval: Duration::from_millis(50),
805                max_backoff_exp: 3,
806                ..config(None)
807            },
808            publisher,
809            move || {
810                let mut attempts = attempts_clone.lock().unwrap();
811                attempts.push(start.elapsed().as_millis());
812                let failing = attempts.len() <= 2;
813                async move {
814                    if failing {
815                        Err(FeedError::Connection("boom".to_string()))
816                    } else {
817                        Ok(snapshot())
818                    }
819                }
820            },
821        ));
822
823        tokio::time::sleep(Duration::from_millis(300)).await;
824        feed.abort();
825
826        assert_eq!(*attempts.lock().unwrap(), vec![0, 50, 150, 200, 250]);
827    }
828
829    #[tokio::test(start_paused = true)]
830    async fn a_slow_poll_never_shortens_the_gap_to_the_next_one() {
831        // The third poll starts at 100 ms and answers only at 220 ms, having run through two
832        // ticks. The polls that follow it are a full 50 ms interval apart — the ticks it ran
833        // through are not owed back, and none of them lands sooner than the cadence allows, which
834        // is what a venue's rate limit is promised.
835        let attempts = Arc::new(std::sync::Mutex::new(Vec::new()));
836        let attempts_clone = Arc::clone(&attempts);
837        let start = tokio::time::Instant::now();
838        let (publisher, _rx) = Publisher::<u32>::channel();
839        let feed = tokio::spawn(run_http_poll_feed(
840            HttpFeedConfig {
841                poll_interval: Duration::from_millis(50),
842                request_timeout: Duration::from_millis(500),
843                ..config(None)
844            },
845            publisher,
846            move || {
847                let mut attempts = attempts_clone.lock().unwrap();
848                attempts.push(start.elapsed().as_millis());
849                let slow = attempts.len() == 3;
850                async move {
851                    if slow {
852                        tokio::time::sleep(Duration::from_millis(120)).await;
853                    }
854                    Ok(snapshot())
855                }
856            },
857        ));
858
859        tokio::time::sleep(Duration::from_millis(350)).await;
860        feed.abort();
861
862        assert_eq!(*attempts.lock().unwrap(), vec![0, 50, 100, 220, 270, 320]);
863    }
864
865    #[tokio::test]
866    async fn retries_forever_without_limit() {
867        let polls = Arc::new(AtomicU32::new(0));
868        let polls_clone = Arc::clone(&polls);
869        let (publisher, _rx) = Publisher::<u32>::channel();
870        let feed = run_http_poll_feed(config(None), publisher, move || {
871            polls_clone.fetch_add(1, Ordering::SeqCst);
872            async { Err(FeedError::Connection("boom".to_string())) }
873        });
874        let feed = tokio::spawn(feed);
875
876        tokio::time::sleep(Duration::from_millis(100)).await;
877        assert!(polls.load(Ordering::SeqCst) > 5, "should keep polling");
878        assert!(!feed.is_finished());
879    }
880
881    #[tokio::test]
882    async fn hung_poll_counts_as_failure() {
883        let (publisher, _rx) = Publisher::<u32>::channel();
884        let feed = run_http_poll_feed(
885            HttpFeedConfig { request_timeout: Duration::from_millis(5), ..config(Some(2)) },
886            publisher,
887            || async {
888                tokio::time::sleep(Duration::from_secs(3600)).await;
889                Ok(snapshot())
890            },
891        );
892        let feed = tokio::spawn(feed);
893
894        let result = expect_to_finish("feed did not give up", feed)
895            .await
896            .unwrap();
897        assert!(matches!(result, Err(FeedError::Connection(msg)) if msg.contains("timed out")));
898    }
899
900    mod fetch_json {
901
902        use rstest::rstest;
903        use serde::Deserialize;
904
905        use super::{test_support::spawn_http_server, *};
906
907        #[derive(Debug, Deserialize, PartialEq)]
908        struct Payload {
909            value: u32,
910        }
911
912        #[tokio::test]
913        async fn parses_a_successful_json_body() {
914            let server =
915                spawn_http_server(|_| Some(("200 OK", r#"{"value":7}"#.to_string()))).await;
916
917            let payload: Payload = fetch_json(
918                reqwest::Client::new().get(format!("{}/x", server.url())),
919                "thing",
920                nothing_refused,
921            )
922            .await
923            .unwrap();
924
925            assert_eq!(payload, Payload { value: 7 });
926        }
927
928        /// A transport failure or a non-success status is a connection error naming the resource
929        /// (and, for a status, carrying it and the body); a body that is not the expected JSON is a
930        /// parsing error.
931        #[rstest]
932        #[case::non_success_status_with_body("503 Service Unavailable", "maintenance", false)]
933        #[case::unparseable_success_body("200 OK", "<html>", true)]
934        #[tokio::test]
935        async fn classifies_status_and_body_failures(
936            #[case] status: &'static str,
937            #[case] body: &'static str,
938            #[case] expect_parsing_error: bool,
939        ) {
940            let server = spawn_http_server(move |_| Some((status, body.to_string()))).await;
941
942            let result: Result<Payload, FeedError> = fetch_json(
943                reqwest::Client::new().get(format!("{}/x", server.url())),
944                "thing",
945                nothing_refused,
946            )
947            .await;
948
949            match result {
950                Err(FeedError::Parsing(msg)) if expect_parsing_error => {
951                    assert!(msg.contains("thing"), "{msg}")
952                }
953                Err(FeedError::Connection(msg)) if !expect_parsing_error => {
954                    assert!(
955                        msg.contains("thing") && msg.contains("503") && msg.contains(body),
956                        "{msg}"
957                    )
958                }
959                other => panic!("unexpected result: {other:?}"),
960            }
961        }
962
963        #[tokio::test]
964        async fn refused_connection_is_a_connection_error() {
965            // Bind and drop so the port is known to be closed.
966            let address = tokio::net::TcpListener::bind("127.0.0.1:0")
967                .await
968                .unwrap()
969                .local_addr()
970                .unwrap();
971
972            let result: Result<Payload, FeedError> = fetch_json(
973                reqwest::Client::new().get(format!("http://{address}/x")),
974                "thing",
975                nothing_refused,
976            )
977            .await;
978
979            assert!(matches!(result, Err(FeedError::Connection(msg)) if msg.contains("thing")));
980        }
981    }
982}