Skip to main content

tycho_simulation/snapshot_feed/
http.rs

1//! Feeds that poll: the shared poll loop, the source trait a provider implements for it, and
2//! the one-request helper those sources fetch with.
3
4use std::{future::Future, time::Duration};
5
6use async_stream::try_stream;
7use bytes::Bytes;
8use futures::Stream;
9use reqwest::{RequestBuilder, Response};
10use serde::de::DeserializeOwned;
11use tokio::time::{interval, sleep, timeout, MissedTickBehavior};
12use tracing::{debug_span, error, info, warn, Instrument};
13
14use super::{errors::FeedError, failures::FailureTracker, publisher::Publisher};
15
16/// Feed tuning for HTTP-polling feeds. Set through the feed builder.
17///
18/// There is no `Default`: start from the builder's `default_feed_config()`, which holds the
19/// values that fit that venue, and change individual fields with struct-update syntax.
20#[derive(Clone, Debug)]
21pub struct HttpFeedConfig {
22    /// Minimum time between the starts of two polls, whatever a poll costs: one that outruns the
23    /// interval is followed by the next as soon as it answers, and the cadence runs on from
24    /// there. Must not be zero: the feed future resolves with `InvalidInput` right away.
25    pub poll_interval: Duration,
26    /// Deadline for one poll's requests; a hung poll counts as a failed poll.
27    pub request_timeout: Duration,
28    /// How many times a run of failed polls may double the gap between them before it stops
29    /// growing, so a provider that is down or rate-limiting is not polled at the healthy cadence
30    /// for the whole outage. The first failure waits one `poll_interval`, the second two, and so
31    /// on up to `poll_interval * 2^max_backoff_exp`; `0` polls a failing provider at the healthy
32    /// cadence.
33    pub max_backoff_exp: u32,
34    /// Consecutive failed polls before the feed gives up and resolves with an error.
35    /// `None` retries forever.
36    pub max_consecutive_failures: Option<u32>,
37    /// Withdraw the published snapshot (publish `None`) once it is this old, so consumers stop
38    /// using data nobody is refreshing; the next successful poll restores it. Age runs from the
39    /// last publish on the monotonic clock, so it bounds staleness whether a failing provider
40    /// refuses instantly or hangs to `request_timeout`. An age that does not exceed
41    /// `poll_interval` is a duty cycle: each snapshot served for this long, then nothing until
42    /// the next poll. Zero is `InvalidInput`; `None` keeps the last snapshot until the feed
43    /// ends.
44    pub max_snapshot_age: Option<Duration>,
45}
46
47/// The values every HTTP feed builder starts from. The relation among them worth keeping through
48/// a retune: the withdrawal age outlasts a poll that hangs to its deadline, so one slow poll does
49/// not blank the book.
50pub fn default_http_feed_config() -> HttpFeedConfig {
51    HttpFeedConfig {
52        poll_interval: Duration::from_secs(5),
53        request_timeout: Duration::from_secs(10),
54        max_backoff_exp: 3,
55        max_consecutive_failures: None,
56        max_snapshot_age: Some(Duration::from_secs(30)),
57    }
58}
59
60/// The poll loop shared by all HTTP-based feeds.
61///
62/// Polls `source` on the cadence `config` asks for and publishes every snapshot it answers with,
63/// withdrawing one that goes `max_snapshot_age` without a refresh. What each knob buys is on its
64/// own field; this is where they are checked and handed to the pieces that act on them.
65///
66/// Resolves `Err` when the feed gives up: on a fatal poll error, whatever the failure budget,
67/// after `max_consecutive_failures` failed polls where one is configured, or right away with
68/// `InvalidInput` for a zero `poll_interval` or a zero `max_snapshot_age`. Resolves `Ok(())` as
69/// soon as the last receiver goes away, mid-poll included, since nothing it fetched from then on
70/// could reach anyone; polling itself never runs out, so a feed is otherwise stopped by dropping
71/// its future, which withdraws the published snapshot on the way out.
72pub async fn run_http_poll_feed<S: HttpSource>(
73    config: HttpFeedConfig,
74    publisher: Publisher<S::Snapshot>,
75    source: S,
76) -> Result<(), FeedError> {
77    // Where the config comes apart, and the only place that sees all of it: the loop waits on
78    // the two deadlines and spends the failure budget, the publisher withdraws what has gone
79    // stale.
80    let HttpFeedConfig {
81        poll_interval,
82        request_timeout,
83        max_backoff_exp,
84        max_consecutive_failures,
85        max_snapshot_age,
86    } = config;
87
88    if poll_interval.is_zero() {
89        return Err(FeedError::InvalidInput("poll_interval must not be zero".to_string()));
90    }
91    if let Some(age) = max_snapshot_age {
92        if age.is_zero() {
93            return Err(FeedError::InvalidInput("max_snapshot_age must not be zero".to_string()));
94        }
95        if age <= poll_interval {
96            // Deliberate for a venue whose rate limit sets the cadence, a mistake otherwise, and
97            // indistinguishable from here — so it is said once rather than refused.
98            warn!(
99                max_snapshot_age = ?age,
100                ?poll_interval,
101                "max_snapshot_age does not exceed poll_interval: this feed will serve each \
102                 snapshot for max_snapshot_age and nothing until the next poll"
103            );
104        }
105    }
106
107    let snapshots = polled_snapshots(
108        poll_interval,
109        request_timeout,
110        max_consecutive_failures,
111        max_backoff_exp,
112        source,
113    );
114    publisher
115        .publishing(max_snapshot_age, snapshots)
116        .await
117}
118
119/// The provider-specific half of an HTTP-polling feed: one call fetches one complete snapshot.
120/// [`run_http_poll_feed`] owns the source and drives it by reference.
121pub trait HttpSource: Send + Sync {
122    /// The complete snapshot one poll yields. Each snapshot replaces the previous one, so it must
123    /// not be a delta.
124    type Snapshot: Send + Sync + 'static;
125
126    /// Fetches one complete snapshot. The loop calls this once per poll and gives it the config's
127    /// `request_timeout`; an implementation may be written as `async fn` as long as its future is
128    /// `Send`.
129    ///
130    /// Return [`FeedError::Fatal`] when polling again cannot produce a different answer; this
131    /// ends the feed. Any other error, like a poll that overruns `request_timeout`, counts as one
132    /// failed poll toward `max_consecutive_failures` (unset by default, which polls forever) and
133    /// backs off before the next poll. Only `Ok` clears the failure count.
134    fn fetch(&self) -> impl Future<Output = Result<Self::Snapshot, FeedError>> + Send;
135}
136
137/// Every snapshot the provider answers with, and as a last item the error the feed gives up on.
138///
139/// One fetch is one poll, started every `poll_interval` and given `request_timeout` to answer; an
140/// error or an overrun is one failed poll. Failures back off, and run out, as [`HttpFeedConfig`]
141/// describes. Each fetch runs in a `poll` span.
142///
143/// Waiting out the cadence and the backoff happens in here, so the one who awaits the next
144/// snapshot waits exactly as long as it takes — which is how a published one can go stale on time
145/// while a poll is still in flight.
146fn polled_snapshots<S: HttpSource>(
147    poll_interval: Duration,
148    request_timeout: Duration,
149    max_consecutive_failures: Option<u32>,
150    max_backoff_exp: u32,
151    source: S,
152) -> impl Stream<Item = Result<S::Snapshot, FeedError>> {
153    // A poller backs off by stretching the cadence it already has, so one poll interval is the
154    // unit its failures double — which is what lets a backoff stand in for a tick below.
155    let mut failures =
156        FailureTracker::new(max_consecutive_failures, poll_interval, max_backoff_exp);
157
158    let mut ticker = interval(poll_interval);
159    // A poll slower than the interval runs through ticks. `Delay` drops them and re-anchors the
160    // cadence on the poll that outran them, which is the only behaviour that keeps every gap at
161    // the interval: `Burst` owes one poll per tick missed and fires them at once, and `Skip`
162    // stays on the original phase, so its next poll can follow a late one by a fraction of the
163    // interval.
164    ticker.set_missed_tick_behavior(MissedTickBehavior::Delay);
165
166    // Set by a failed poll and taken by the wait that serves it out, so a poll that succeeds
167    // leaves nothing behind for the next one to wait through.
168    let mut backoff = None;
169
170    info!(?poll_interval, "starting polling");
171
172    try_stream! {
173        loop {
174            // The backoff stands in for the tick: it is already a multiple of the interval, and
175            // the ticker's own deadline expires while it waits.
176            match backoff.take() {
177                Some(backoff) => {
178                    sleep(backoff).await;
179                    // The tick this backoff stood in for came due while it was waiting, and an
180                    // overdue tick completes at once: without restarting the interval here, the
181                    // poll after a recovered one would follow it immediately.
182                    ticker.reset();
183                }
184                None => {
185                    ticker.tick().await;
186                }
187            }
188            let polled = timeout(request_timeout, source.fetch())
189                .instrument(debug_span!("poll"))
190                .await;
191
192            let error = match polled {
193                Ok(Ok(snapshot)) => {
194                    let cleared = failures.record_success();
195                    if cleared > 0 {
196                        info!(after_failures = cleared, "poll succeeded again");
197                    }
198                    yield snapshot;
199                    continue;
200                }
201                Ok(Err(e)) if e.is_fatal() => Err(e)?,
202                Ok(Err(e)) => e,
203                Err(_) => FeedError::Connection(format!(
204                    "poll timed out after {request_timeout:?}"
205                )),
206            };
207
208            let retry = failures
209                .record_failure()
210                .inspect(|retry| {
211                    warn!(
212                        consecutive = retry.consecutive,
213                        backoff = ?retry.backoff,
214                        %error,
215                        "poll failed, backing off"
216                    )
217                })
218                // The feed ends with the failure that ended it, classified as its source
219                // classified it: a run of unparseable answers is not a connection failure, and
220                // how long the feed tolerated it is for the log rather than for the caller to
221                // act on.
222                .map_err(|consecutive| {
223                    error!(consecutive, "giving up on the feed");
224                    error
225                })?;
226            backoff = Some(retry.backoff);
227        }
228    }
229}
230
231/// Whether a failing response's body says the venue will never serve this request, such as a
232/// chain it does not price. A refusal is a [`FeedError::Fatal`] rather than the retryable
233/// [`FeedError::Connection`], since another attempt gets the same answer. Every fetch here
234/// takes one, because what a failing status means is the venue's to say and not this module's.
235///
236/// A plain `fn`, so the verdict rests on what the venue said and on nothing else.
237pub type RefusalCheck = fn(&str) -> bool;
238
239/// For a venue that refuses nothing outright, so every failing answer is worth another attempt.
240pub fn nothing_refused(_body: &str) -> bool {
241    false
242}
243
244/// Sends `request` and returns its body. A transport failure is a [`FeedError::Connection`], a
245/// failing status whatever `refused` makes of it; `what` names the resource in the message
246/// (e.g. "Hashflow price levels").
247///
248/// For an API that reports failures inside the body of an otherwise successful response, which
249/// therefore has to be classified before it is parsed. [`fetch_json`] is the plain case.
250pub(crate) async fn fetch_bytes(
251    request: RequestBuilder,
252    what: &str,
253    refused: RefusalCheck,
254) -> Result<Bytes, FeedError> {
255    let response = request
256        .send()
257        .await
258        .map_err(|e| FeedError::Connection(format!("Failed to fetch {what}: {e}")))?;
259    read_bytes(response, what, refused).await
260}
261
262/// Reads `response`'s body, failing a non-success status with whatever the server said — as a
263/// refusal when `refused` recognises it there. `what` names the resource, as in [`fetch_bytes`].
264async fn read_bytes(
265    response: Response,
266    what: &str,
267    refused: RefusalCheck,
268) -> Result<Bytes, FeedError> {
269    // The body is read before the status is judged, so a failing response can report what the
270    // server said, and so the venue can be asked what it meant by it.
271    let status = response.status();
272    let body = response
273        .bytes()
274        .await
275        .map_err(|e| FeedError::Connection(format!("Failed to read {what} response: {e}")))?;
276
277    if !status.is_success() {
278        let said = String::from_utf8_lossy(&body);
279        let message = format!("{what} HTTP error {status}: {said}");
280        return Err(if refused(&said) {
281            FeedError::Fatal(message)
282        } else {
283            FeedError::Connection(message)
284        });
285    }
286    Ok(body)
287}
288
289/// Sends `request` and parses its JSON body, fetching as `fetch_bytes` does; an unparseable
290/// body is a [`FeedError::Parsing`] naming `what`.
291pub async fn fetch_json<T: DeserializeOwned>(
292    request: RequestBuilder,
293    what: &str,
294    refused: RefusalCheck,
295) -> Result<T, FeedError> {
296    parse_json(&fetch_bytes(request, what, refused).await?, what)
297}
298
299/// Parses `response`'s JSON body, the second half of [`fetch_json`] for a caller that sends the
300/// request itself.
301pub(crate) async fn read_json<T: DeserializeOwned>(
302    response: Response,
303    what: &str,
304    refused: RefusalCheck,
305) -> Result<T, FeedError> {
306    parse_json(&read_bytes(response, what, refused).await?, what)
307}
308
309fn parse_json<T: DeserializeOwned>(body: &[u8], what: &str) -> Result<T, FeedError> {
310    serde_json::from_slice(body)
311        .map_err(|e| FeedError::Parsing(format!("Failed to parse {what} response: {e}")))
312}
313
314/// A local HTTP/1.1 server for feed tests: answers every request with `respond`'s status line
315/// and JSON body. Serves until the runtime drops it.
316#[cfg(test)]
317pub(crate) mod test_support {
318    use std::{
319        net::SocketAddr,
320        sync::{
321            atomic::{AtomicUsize, Ordering},
322            Arc,
323        },
324        time::Duration,
325    };
326
327    use tokio::{
328        io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader},
329        net::TcpListener,
330    };
331
332    /// One canned HTTP response; `(status_line, body)` converts into it.
333    pub struct MockResponse {
334        pub status: &'static str,
335        pub body: String,
336        /// Wait before answering, for tests that exercise request deadlines.
337        pub delay: Duration,
338    }
339
340    impl From<(&'static str, String)> for MockResponse {
341        fn from((status, body): (&'static str, String)) -> Self {
342            MockResponse { status, body, delay: Duration::ZERO }
343        }
344    }
345
346    /// One request as the mock server read it.
347    pub struct MockRequest {
348        /// Path and query string, as the request line carries them.
349        pub target: String,
350        /// The body, empty where the request had none. Read in full, so a client that sends one
351        /// is not left writing into a socket nobody drains.
352        pub body: String,
353    }
354
355    /// A running mock server: where it listens and how many requests it has answered.
356    pub struct MockHttpServer {
357        pub address: SocketAddr,
358        pub requests: Arc<AtomicUsize>,
359    }
360
361    impl MockHttpServer {
362        pub fn url(&self) -> String {
363            format!("http://{}", self.address)
364        }
365
366        pub fn request_count(&self) -> usize {
367            self.requests.load(Ordering::SeqCst)
368        }
369    }
370
371    /// The headers and however many bytes of body `Content-Length` announced. A request that
372    /// announces none has none: nothing here speaks chunked transfer encoding.
373    async fn read_body(reader: &mut BufReader<tokio::net::TcpStream>) -> String {
374        let mut length = 0;
375        loop {
376            let mut header = String::new();
377            match reader.read_line(&mut header).await {
378                Ok(0) => break,
379                Err(_) => break,
380                Ok(_) if header.trim().is_empty() => break,
381                Ok(_) => {}
382            }
383            if let Some((name, value)) = header.split_once(':') {
384                if name.eq_ignore_ascii_case("content-length") {
385                    length = value.trim().parse().unwrap_or(0);
386                }
387            }
388        }
389        let mut body = vec![0; length];
390        if length > 0 {
391            let _ = reader.read_exact(&mut body).await;
392        }
393        String::from_utf8_lossy(&body).into_owned()
394    }
395
396    /// Serves HTTP/1.1 requests until the runtime drops it. `route` maps a request to a
397    /// response; a request it returns `None` for gets a 404. Each request is counted before
398    /// `route` runs, so a route closure holding its own counter can answer per attempt.
399    ///
400    /// Each connection is served by its own task, so a response that waits out its `delay` holds
401    /// up nobody else: a client that retries gets its second answer a delay after it asked, not
402    /// a delay after the first one finished.
403    pub async fn spawn_http_server<R: Into<MockResponse>>(
404        route: impl Fn(&MockRequest) -> Option<R> + Send + Sync + 'static,
405    ) -> MockHttpServer {
406        let listener = TcpListener::bind("127.0.0.1:0")
407            .await
408            .unwrap();
409        let address = listener.local_addr().unwrap();
410        let requests = Arc::new(AtomicUsize::new(0));
411        let served = Arc::clone(&requests);
412        let route = Arc::new(route);
413        tokio::spawn(async move {
414            while let Ok((stream, _)) = listener.accept().await {
415                let (served, route) = (Arc::clone(&served), Arc::clone(&route));
416                tokio::spawn(async move {
417                    let mut reader = BufReader::new(stream);
418                    let mut request_line = String::new();
419                    if reader
420                        .read_line(&mut request_line)
421                        .await
422                        .is_err()
423                    {
424                        return;
425                    }
426                    served.fetch_add(1, Ordering::SeqCst);
427                    let request = MockRequest {
428                        target: request_line
429                            .split_whitespace()
430                            .nth(1)
431                            .unwrap_or_default()
432                            .to_string(),
433                        body: read_body(&mut reader).await,
434                    };
435                    let response = route(&request)
436                        .map(Into::into)
437                        .unwrap_or_else(|| {
438                            MockResponse::from((
439                                "404 Not Found",
440                                format!("no route for {}", request.target),
441                            ))
442                        });
443
444                    tokio::time::sleep(response.delay).await;
445
446                    let MockResponse { status, body, .. } = response;
447                    let payload = format!(
448                    "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
449                    body.len()
450                );
451                    let mut stream = reader.into_inner();
452                    let _ = stream
453                        .write_all(payload.as_bytes())
454                        .await;
455                    let _ = stream.shutdown().await;
456                });
457            }
458        });
459        MockHttpServer { address, requests }
460    }
461}
462
463#[cfg(test)]
464mod tests {
465    use std::{
466        future::Future,
467        sync::{
468            atomic::{AtomicBool, AtomicU32, Ordering},
469            Arc,
470        },
471    };
472
473    use futures::StreamExt as _;
474    use tokio::sync::watch;
475    use tokio_stream::wrappers::WatchStream;
476
477    use super::{super::expect_to_finish, *};
478
479    /// Lets a closure stand in for a provider's source.
480    impl<F, Fut, T> HttpSource for F
481    where
482        F: Fn() -> Fut + Send + Sync,
483        Fut: Future<Output = Result<T, FeedError>> + Send,
484        T: Send + Sync + 'static,
485    {
486        type Snapshot = T;
487
488        async fn fetch(&self) -> Result<T, FeedError> {
489            self().await
490        }
491    }
492
493    /// The one snapshot the mock sources here publish.
494    fn snapshot() -> u32 {
495        1
496    }
497
498    fn config(max_consecutive_failures: Option<u32>) -> HttpFeedConfig {
499        HttpFeedConfig {
500            poll_interval: Duration::from_millis(5),
501            request_timeout: Duration::from_millis(50),
502            // A failed poll retries at the polling cadence: the tests that fail on purpose are
503            // timing the loop, not the backoff curve.
504            max_backoff_exp: 0,
505            max_consecutive_failures,
506            max_snapshot_age: None,
507        }
508    }
509
510    #[tokio::test]
511    async fn publishes_a_snapshot_and_success_resets_failures() {
512        // One failed poll, then successes: with a limit of 2 the single failure must not
513        // accumulate towards termination once a poll succeeds.
514        let polls = Arc::new(AtomicU32::new(0));
515        let polls_clone = Arc::clone(&polls);
516        let (publisher, mut rx) = Publisher::channel();
517        let feed = run_http_poll_feed(config(Some(2)), publisher, move || {
518            let n = polls_clone.fetch_add(1, Ordering::SeqCst);
519            async move {
520                if n == 0 {
521                    Err(FeedError::Connection("boom".to_string()))
522                } else {
523                    Ok(snapshot())
524                }
525            }
526        });
527        let feed = tokio::spawn(feed);
528
529        expect_to_finish("no snapshot published", async {
530            loop {
531                rx.changed().await.expect("feed ended");
532                if rx.borrow_and_update().is_some() {
533                    return;
534                }
535            }
536        })
537        .await;
538
539        // Several more polls run without termination.
540        tokio::time::sleep(Duration::from_millis(50)).await;
541        assert!(!feed.is_finished());
542    }
543
544    #[tokio::test]
545    async fn withdraws_a_snapshot_that_goes_unrefreshed_and_restores_on_next_success() {
546        // One successful poll, then failures until the flag flips: the published snapshot
547        // must be withdrawn once it has gone max_snapshot_age without a refresh and come back
548        // with the next successful poll.
549        let polls = Arc::new(AtomicU32::new(0));
550        let polls_clone = Arc::clone(&polls);
551        let recovered = Arc::new(AtomicBool::new(false));
552        let recovered_clone = Arc::clone(&recovered);
553        let (publisher, mut rx) = Publisher::channel();
554        let config =
555            HttpFeedConfig { max_snapshot_age: Some(Duration::from_millis(20)), ..config(None) };
556        let feed = run_http_poll_feed(config, publisher, move || {
557            let n = polls_clone.fetch_add(1, Ordering::SeqCst);
558            let recovered = recovered_clone.load(Ordering::SeqCst);
559            async move {
560                if n == 0 || recovered {
561                    Ok(snapshot())
562                } else {
563                    Err(FeedError::Connection("boom".to_string()))
564                }
565            }
566        });
567        let _feed = tokio::spawn(feed);
568
569        let wait_for = |rx: &mut watch::Receiver<Option<u32>>, want_some: bool| {
570            let mut rx = rx.clone();
571            async move {
572                expect_to_finish("watch did not reach the expected state", async {
573                    loop {
574                        if rx.borrow_and_update().is_some() == want_some {
575                            return;
576                        }
577                        rx.changed().await.expect("feed ended");
578                    }
579                })
580                .await;
581            }
582        };
583        wait_for(&mut rx, true).await;
584        wait_for(&mut rx, false).await;
585        recovered.store(true, Ordering::SeqCst);
586        wait_for(&mut rx, true).await;
587    }
588
589    #[tokio::test]
590    async fn rejects_zero_poll_interval() {
591        let (publisher, _rx) = Publisher::<u32>::channel();
592        let config = HttpFeedConfig { poll_interval: Duration::ZERO, ..config(None) };
593        let result = run_http_poll_feed(config, publisher, || async { Ok(snapshot()) }).await;
594        assert!(matches!(result, Err(FeedError::InvalidInput(_))));
595    }
596
597    #[tokio::test]
598    async fn rejects_a_zero_max_snapshot_age() {
599        let (publisher, _rx) = Publisher::<u32>::channel();
600        let config = HttpFeedConfig { max_snapshot_age: Some(Duration::ZERO), ..config(None) };
601        let result = run_http_poll_feed(config, publisher, || async { Ok(snapshot()) }).await;
602        assert!(matches!(result, Err(FeedError::InvalidInput(_))));
603    }
604
605    /// A snapshot may be servable for less than the gap between polls: a consumer that would
606    /// rather decline than quote an old book asks for exactly that when the venue's rate limit
607    /// sets the cadence. The feed serves each snapshot for `max_snapshot_age` and withdraws it
608    /// until the next poll.
609    #[tokio::test(start_paused = true)]
610    async fn a_max_snapshot_age_below_the_poll_interval_is_a_duty_cycle() {
611        let (publisher, rx) = Publisher::<u32>::channel();
612        let mut changes = WatchStream::from_changes(rx);
613        let config = HttpFeedConfig {
614            poll_interval: Duration::from_millis(100),
615            max_snapshot_age: Some(Duration::from_millis(20)),
616            ..config(None)
617        };
618        let _feed =
619            tokio::spawn(run_http_poll_feed(config, publisher, || async { Ok(snapshot()) }));
620
621        let start = tokio::time::Instant::now();
622        let mut seen = Vec::new();
623        for _ in 0..4 {
624            let change = expect_to_finish("the feed stopped changing", changes.next()).await;
625            seen.push((change.expect("the feed is still running"), start.elapsed()));
626        }
627
628        assert_eq!(
629            seen,
630            [
631                // The first poll fires immediately; each snapshot stands for its 20 ms and the
632                // feed serves nothing for the remaining 80 ms of the interval.
633                (Some(snapshot()), Duration::ZERO),
634                (None, Duration::from_millis(20)),
635                (Some(snapshot()), Duration::from_millis(100)),
636                (None, Duration::from_millis(120)),
637            ]
638        );
639    }
640
641    #[tokio::test]
642    async fn gives_up_after_max_consecutive_failed_polls() {
643        // A provider that answers every time with something unparseable is not a connection
644        // failure, so the feed ends with the failure that ended it, as its source classified it.
645        let (publisher, _rx) = Publisher::<u32>::channel();
646        let feed = run_http_poll_feed(config(Some(3)), publisher, || async {
647            Err(FeedError::Parsing("not a snapshot".to_string()))
648        });
649        let feed = tokio::spawn(feed);
650
651        let result = expect_to_finish("feed did not give up", feed)
652            .await
653            .unwrap();
654        assert!(matches!(result, Err(FeedError::Parsing(msg)) if msg == "not a snapshot"));
655    }
656
657    #[tokio::test]
658    async fn gives_up_on_a_fatal_poll_error_despite_an_unlimited_failure_budget() {
659        // A rejected key or an unsupported chain answers every poll the same way, so the
660        // feed stops instead of asking forever — and the snapshot it had goes with it.
661        let (publisher, mut rx) = Publisher::channel();
662        let polls = Arc::new(AtomicU32::new(0));
663        let polls_clone = Arc::clone(&polls);
664        let feed = run_http_poll_feed(config(None), publisher, move || {
665            let n = polls_clone.fetch_add(1, Ordering::SeqCst);
666            async move {
667                if n == 0 {
668                    Ok(snapshot())
669                } else {
670                    Err(FeedError::Fatal("api key rejected".to_string()))
671                }
672            }
673        });
674        let feed = tokio::spawn(feed);
675
676        let result = expect_to_finish("feed did not give up", feed)
677            .await
678            .unwrap();
679
680        assert!(matches!(result, Err(FeedError::Fatal(_))));
681        assert_eq!(polls.load(Ordering::SeqCst), 2, "the feed polled again after the fatal");
682        assert!(rx.borrow_and_update().is_none(), "the snapshot must be withdrawn");
683    }
684
685    #[tokio::test]
686    async fn withdraws_the_published_snapshot_when_it_gives_up() {
687        // A snapshot published before the feed gave up must not stay servable: nothing is
688        // refreshing it, and a consumer holding the receiver would keep using it.
689        let polls = Arc::new(AtomicU32::new(0));
690        let polls_clone = Arc::clone(&polls);
691        let (publisher, mut rx) = Publisher::channel();
692        let feed = run_http_poll_feed(config(Some(2)), publisher, move || {
693            let n = polls_clone.fetch_add(1, Ordering::SeqCst);
694            async move {
695                if n == 0 {
696                    Ok(snapshot())
697                } else {
698                    Err(FeedError::Connection("boom".to_string()))
699                }
700            }
701        });
702        let feed = tokio::spawn(feed);
703
704        expect_to_finish("no snapshot published", async {
705            loop {
706                rx.changed()
707                    .await
708                    .expect("feed ended before publishing");
709                if rx.borrow_and_update().is_some() {
710                    return;
711                }
712            }
713        })
714        .await;
715
716        let result = expect_to_finish("feed did not give up", feed)
717            .await
718            .unwrap();
719        assert!(matches!(result, Err(FeedError::Connection(_))));
720        assert!(rx.borrow().is_none(), "the snapshot must be withdrawn when the feed gives up");
721    }
722
723    #[tokio::test]
724    async fn withdraws_the_published_snapshot_when_the_feed_is_dropped() {
725        // Dropping the feed future — the consumer aborting its task — leaves the receiver
726        // alive; the snapshot stops being refreshed and must stop being served with it.
727        let (publisher, mut rx) = Publisher::channel();
728        let feed = run_http_poll_feed(config(None), publisher, || async { Ok(snapshot()) });
729        let feed = tokio::spawn(feed);
730
731        expect_to_finish("no snapshot published", async {
732            loop {
733                rx.changed()
734                    .await
735                    .expect("feed ended before publishing");
736                if rx.borrow_and_update().is_some() {
737                    return;
738                }
739            }
740        })
741        .await;
742
743        feed.abort();
744        expect_to_finish("the snapshot was not withdrawn", rx.changed())
745            .await
746            .expect("the withdrawal must reach the receiver before the sender drops");
747        assert!(rx.borrow().is_none());
748    }
749
750    #[tokio::test(start_paused = true)]
751    async fn failing_polls_stretch_the_polling_cadence() {
752        // A 50 ms cadence doubling at most three times: a provider that is down is polled at 0
753        // and then after 50, 100, 200, 400 and 400 ms — 5 attempts in the first second, where the
754        // healthy cadence would make 20.
755        let attempts = Arc::new(std::sync::Mutex::new(Vec::new()));
756        let attempts_clone = Arc::clone(&attempts);
757        let start = tokio::time::Instant::now();
758        let (publisher, _rx) = Publisher::<u32>::channel();
759        let feed = tokio::spawn(run_http_poll_feed(
760            HttpFeedConfig {
761                poll_interval: Duration::from_millis(50),
762                max_backoff_exp: 3,
763                ..config(None)
764            },
765            publisher,
766            move || {
767                attempts_clone
768                    .lock()
769                    .unwrap()
770                    .push(start.elapsed().as_millis());
771                async { Err(FeedError::Connection("boom".to_string())) }
772            },
773        ));
774
775        tokio::time::sleep(Duration::from_secs(1)).await;
776        feed.abort();
777
778        assert_eq!(*attempts.lock().unwrap(), vec![0, 50, 150, 350, 750]);
779    }
780
781    #[tokio::test(start_paused = true)]
782    async fn a_recovered_poll_resumes_the_cadence_instead_of_catching_up() {
783        // Two failures stretch the 50 ms cadence to 50 and 100 ms, so the third attempt polls at
784        // 150 ms and succeeds. The polls after it are one plain interval apart: the backoff took
785        // the place of those ticks, so none of them are owed.
786        let attempts = Arc::new(std::sync::Mutex::new(Vec::new()));
787        let attempts_clone = Arc::clone(&attempts);
788        let start = tokio::time::Instant::now();
789        let (publisher, _rx) = Publisher::<u32>::channel();
790        let feed = tokio::spawn(run_http_poll_feed(
791            HttpFeedConfig {
792                poll_interval: Duration::from_millis(50),
793                max_backoff_exp: 3,
794                ..config(None)
795            },
796            publisher,
797            move || {
798                let mut attempts = attempts_clone.lock().unwrap();
799                attempts.push(start.elapsed().as_millis());
800                let failing = attempts.len() <= 2;
801                async move {
802                    if failing {
803                        Err(FeedError::Connection("boom".to_string()))
804                    } else {
805                        Ok(snapshot())
806                    }
807                }
808            },
809        ));
810
811        tokio::time::sleep(Duration::from_millis(300)).await;
812        feed.abort();
813
814        assert_eq!(*attempts.lock().unwrap(), vec![0, 50, 150, 200, 250]);
815    }
816
817    #[tokio::test(start_paused = true)]
818    async fn a_slow_poll_never_shortens_the_gap_to_the_next_one() {
819        // The third poll starts at 100 ms and answers only at 220 ms, having run through two
820        // ticks. The polls that follow it are a full 50 ms interval apart — the ticks it ran
821        // through are not owed back, and none of them lands sooner than the cadence allows, which
822        // is what a venue's rate limit is promised.
823        let attempts = Arc::new(std::sync::Mutex::new(Vec::new()));
824        let attempts_clone = Arc::clone(&attempts);
825        let start = tokio::time::Instant::now();
826        let (publisher, _rx) = Publisher::<u32>::channel();
827        let feed = tokio::spawn(run_http_poll_feed(
828            HttpFeedConfig {
829                poll_interval: Duration::from_millis(50),
830                request_timeout: Duration::from_millis(500),
831                ..config(None)
832            },
833            publisher,
834            move || {
835                let mut attempts = attempts_clone.lock().unwrap();
836                attempts.push(start.elapsed().as_millis());
837                let slow = attempts.len() == 3;
838                async move {
839                    if slow {
840                        tokio::time::sleep(Duration::from_millis(120)).await;
841                    }
842                    Ok(snapshot())
843                }
844            },
845        ));
846
847        tokio::time::sleep(Duration::from_millis(350)).await;
848        feed.abort();
849
850        assert_eq!(*attempts.lock().unwrap(), vec![0, 50, 100, 220, 270, 320]);
851    }
852
853    #[tokio::test]
854    async fn retries_forever_without_limit() {
855        let polls = Arc::new(AtomicU32::new(0));
856        let polls_clone = Arc::clone(&polls);
857        let (publisher, _rx) = Publisher::<u32>::channel();
858        let feed = run_http_poll_feed(config(None), publisher, move || {
859            polls_clone.fetch_add(1, Ordering::SeqCst);
860            async { Err(FeedError::Connection("boom".to_string())) }
861        });
862        let feed = tokio::spawn(feed);
863
864        tokio::time::sleep(Duration::from_millis(100)).await;
865        assert!(polls.load(Ordering::SeqCst) > 5, "should keep polling");
866        assert!(!feed.is_finished());
867    }
868
869    #[tokio::test]
870    async fn hung_poll_counts_as_failure() {
871        let (publisher, _rx) = Publisher::<u32>::channel();
872        let feed = run_http_poll_feed(
873            HttpFeedConfig { request_timeout: Duration::from_millis(5), ..config(Some(2)) },
874            publisher,
875            || async {
876                tokio::time::sleep(Duration::from_secs(3600)).await;
877                Ok(snapshot())
878            },
879        );
880        let feed = tokio::spawn(feed);
881
882        let result = expect_to_finish("feed did not give up", feed)
883            .await
884            .unwrap();
885        assert!(matches!(result, Err(FeedError::Connection(msg)) if msg.contains("timed out")));
886    }
887
888    mod fetch_json {
889
890        use rstest::rstest;
891        use serde::Deserialize;
892
893        use super::{test_support::spawn_http_server, *};
894
895        #[derive(Debug, Deserialize, PartialEq)]
896        struct Payload {
897            value: u32,
898        }
899
900        #[tokio::test]
901        async fn parses_a_successful_json_body() {
902            let server =
903                spawn_http_server(|_| Some(("200 OK", r#"{"value":7}"#.to_string()))).await;
904
905            let payload: Payload = fetch_json(
906                reqwest::Client::new().get(format!("{}/x", server.url())),
907                "thing",
908                nothing_refused,
909            )
910            .await
911            .unwrap();
912
913            assert_eq!(payload, Payload { value: 7 });
914        }
915
916        /// A transport failure or a non-success status is a connection error naming the resource
917        /// (and, for a status, carrying it and the body); a body that is not the expected JSON is a
918        /// parsing error.
919        #[rstest]
920        #[case::non_success_status_with_body("503 Service Unavailable", "maintenance", false)]
921        #[case::unparseable_success_body("200 OK", "<html>", true)]
922        #[tokio::test]
923        async fn classifies_status_and_body_failures(
924            #[case] status: &'static str,
925            #[case] body: &'static str,
926            #[case] expect_parsing_error: bool,
927        ) {
928            let server = spawn_http_server(move |_| Some((status, body.to_string()))).await;
929
930            let result: Result<Payload, FeedError> = fetch_json(
931                reqwest::Client::new().get(format!("{}/x", server.url())),
932                "thing",
933                nothing_refused,
934            )
935            .await;
936
937            match result {
938                Err(FeedError::Parsing(msg)) if expect_parsing_error => {
939                    assert!(msg.contains("thing"), "{msg}")
940                }
941                Err(FeedError::Connection(msg)) if !expect_parsing_error => {
942                    assert!(
943                        msg.contains("thing") && msg.contains("503") && msg.contains(body),
944                        "{msg}"
945                    )
946                }
947                other => panic!("unexpected result: {other:?}"),
948            }
949        }
950
951        #[tokio::test]
952        async fn refused_connection_is_a_connection_error() {
953            // Bind and drop so the port is known to be closed.
954            let address = tokio::net::TcpListener::bind("127.0.0.1:0")
955                .await
956                .unwrap()
957                .local_addr()
958                .unwrap();
959
960            let result: Result<Payload, FeedError> = fetch_json(
961                reqwest::Client::new().get(format!("http://{address}/x")),
962                "thing",
963                nothing_refused,
964            )
965            .await;
966
967            assert!(matches!(result, Err(FeedError::Connection(msg)) if msg.contains("thing")));
968        }
969    }
970}