Skip to main content

tycho_simulation/snapshot_feed/
http.rs

1//! Feeds that poll: the shared poll loop, the source trait a provider implements for it, and
2//! the one-request helper those sources fetch with.
3
4use std::time::Duration;
5
6use async_stream::try_stream;
7use bytes::Bytes;
8use futures::Stream;
9use reqwest::{RequestBuilder, Response};
10use serde::de::DeserializeOwned;
11use tokio::time::{interval, sleep, timeout, MissedTickBehavior};
12use tracing::{debug_span, error, info, warn, Instrument};
13
14use super::{errors::FeedError, failures::FailureTracker, publisher::Publisher};
15
16/// Feed tuning for HTTP-polling feeds. Set through the feed builder.
17///
18/// There is no `Default`: start from the builder's `default_feed_config()`, which holds the
19/// values that fit that venue, and change individual fields with struct-update syntax.
20#[derive(Clone, Debug)]
21pub struct HttpFeedConfig {
22    /// Minimum time between the starts of two polls, whatever a poll costs: one that outruns the
23    /// interval is followed by the next as soon as it answers, and the cadence runs on from
24    /// there. Must not be zero: the feed future resolves with `InvalidInput` right away.
25    pub poll_interval: Duration,
26    /// Deadline for one poll's requests; a hung poll counts as a failed poll.
27    pub request_timeout: Duration,
28    /// How many times a run of failed polls may double the gap between them before it stops
29    /// growing, so a provider that is down or rate-limiting is not polled at the healthy cadence
30    /// for the whole outage. The first failure waits one `poll_interval`, the second two, and so
31    /// on up to `poll_interval * 2^max_backoff_exp`; `0` polls a failing provider at the healthy
32    /// cadence.
33    pub max_backoff_exp: u32,
34    /// Consecutive failed polls before the feed gives up and resolves with an error.
35    /// `None` retries forever.
36    pub max_consecutive_failures: Option<u32>,
37    /// Withdraw the published snapshot (publish `None`) once it is this old, so consumers stop
38    /// using data nobody is refreshing; the next successful poll restores it. Age runs from the
39    /// last publish on the monotonic clock, so it bounds staleness whether a failing provider
40    /// refuses instantly or hangs to `request_timeout`. An age that does not exceed
41    /// `poll_interval` is a duty cycle: each snapshot served for this long, then nothing until
42    /// the next poll. Zero is `InvalidInput`; `None` keeps the last snapshot until the feed
43    /// ends.
44    pub max_snapshot_age: Option<Duration>,
45}
46
47/// The values every HTTP feed builder starts from. The relation among them worth keeping through
48/// a retune: the withdrawal age outlasts a poll that hangs to its deadline, so one slow poll does
49/// not blank the book.
50pub(crate) fn default_http_feed_config() -> HttpFeedConfig {
51    HttpFeedConfig {
52        poll_interval: Duration::from_secs(5),
53        request_timeout: Duration::from_secs(10),
54        max_backoff_exp: 3,
55        max_consecutive_failures: None,
56        max_snapshot_age: Some(Duration::from_secs(30)),
57    }
58}
59
60/// The poll loop shared by all HTTP-based feeds.
61///
62/// Polls `source` on the cadence `config` asks for and publishes every snapshot it answers with,
63/// withdrawing one that goes `max_snapshot_age` without a refresh. What each knob buys is on its
64/// own field; this is where they are checked and handed to the pieces that act on them.
65///
66/// Resolves `Err` when the feed gives up: on a fatal poll error, whatever the failure budget,
67/// after `max_consecutive_failures` failed polls where one is configured, or right away with
68/// `InvalidInput` for a zero `poll_interval` or a zero `max_snapshot_age`. Resolves `Ok(())` as
69/// soon as the last receiver goes away, mid-poll included, since nothing it fetched from then on
70/// could reach anyone; polling itself never runs out, so a feed is otherwise stopped by dropping
71/// its future, which withdraws the published snapshot on the way out.
72pub(crate) async fn run_http_poll_feed<S: HttpSource>(
73    config: HttpFeedConfig,
74    publisher: Publisher<S::Snapshot>,
75    source: S,
76) -> Result<(), FeedError> {
77    // Where the config comes apart, and the only place that sees all of it: the loop waits on
78    // the two deadlines and spends the failure budget, the publisher withdraws what has gone
79    // stale.
80    let HttpFeedConfig {
81        poll_interval,
82        request_timeout,
83        max_backoff_exp,
84        max_consecutive_failures,
85        max_snapshot_age,
86    } = config;
87
88    if poll_interval.is_zero() {
89        return Err(FeedError::InvalidInput("poll_interval must not be zero".to_string()));
90    }
91    if let Some(age) = max_snapshot_age {
92        if age.is_zero() {
93            return Err(FeedError::InvalidInput("max_snapshot_age must not be zero".to_string()));
94        }
95        if age <= poll_interval {
96            // Deliberate for a venue whose rate limit sets the cadence, a mistake otherwise, and
97            // indistinguishable from here — so it is said once rather than refused.
98            warn!(
99                max_snapshot_age = ?age,
100                ?poll_interval,
101                "max_snapshot_age does not exceed poll_interval: this feed will serve each \
102                 snapshot for max_snapshot_age and nothing until the next poll"
103            );
104        }
105    }
106
107    let snapshots = polled_snapshots(
108        poll_interval,
109        request_timeout,
110        max_consecutive_failures,
111        max_backoff_exp,
112        source,
113    );
114    publisher
115        .publishing(max_snapshot_age, snapshots)
116        .await
117}
118
119/// The provider-specific half of an HTTP-polling feed: one call fetches one complete snapshot.
120/// [`run_http_poll_feed`] owns the source and drives it by reference.
121pub(crate) trait HttpSource: Send + Sync {
122    type Snapshot: Send + Sync + 'static;
123
124    async fn fetch(&self) -> Result<Self::Snapshot, FeedError>;
125}
126
127/// Every snapshot the provider answers with, and as a last item the error the feed gives up on.
128///
129/// One fetch is one poll, started every `poll_interval` and given `request_timeout` to answer; an
130/// error or an overrun is one failed poll. Failures back off, and run out, as [`HttpFeedConfig`]
131/// describes. Each fetch runs in a `poll` span.
132///
133/// Waiting out the cadence and the backoff happens in here, so the one who awaits the next
134/// snapshot waits exactly as long as it takes — which is how a published one can go stale on time
135/// while a poll is still in flight.
136fn polled_snapshots<S: HttpSource>(
137    poll_interval: Duration,
138    request_timeout: Duration,
139    max_consecutive_failures: Option<u32>,
140    max_backoff_exp: u32,
141    source: S,
142) -> impl Stream<Item = Result<S::Snapshot, FeedError>> {
143    // A poller backs off by stretching the cadence it already has, so one poll interval is the
144    // unit its failures double — which is what lets a backoff stand in for a tick below.
145    let mut failures =
146        FailureTracker::new(max_consecutive_failures, poll_interval, max_backoff_exp);
147
148    let mut ticker = interval(poll_interval);
149    // A poll slower than the interval runs through ticks. `Delay` drops them and re-anchors the
150    // cadence on the poll that outran them, which is the only behaviour that keeps every gap at
151    // the interval: `Burst` owes one poll per tick missed and fires them at once, and `Skip`
152    // stays on the original phase, so its next poll can follow a late one by a fraction of the
153    // interval.
154    ticker.set_missed_tick_behavior(MissedTickBehavior::Delay);
155
156    // Set by a failed poll and taken by the wait that serves it out, so a poll that succeeds
157    // leaves nothing behind for the next one to wait through.
158    let mut backoff = None;
159
160    info!(?poll_interval, "starting polling");
161
162    try_stream! {
163        loop {
164            // The backoff stands in for the tick: it is already a multiple of the interval, and
165            // the ticker's own deadline expires while it waits.
166            match backoff.take() {
167                Some(backoff) => {
168                    sleep(backoff).await;
169                    // The tick this backoff stood in for came due while it was waiting, and an
170                    // overdue tick completes at once: without restarting the interval here, the
171                    // poll after a recovered one would follow it immediately.
172                    ticker.reset();
173                }
174                None => {
175                    ticker.tick().await;
176                }
177            }
178            let polled = timeout(request_timeout, source.fetch())
179                .instrument(debug_span!("poll"))
180                .await;
181
182            let error = match polled {
183                Ok(Ok(snapshot)) => {
184                    let cleared = failures.record_success();
185                    if cleared > 0 {
186                        info!(after_failures = cleared, "poll succeeded again");
187                    }
188                    yield snapshot;
189                    continue;
190                }
191                Ok(Err(e)) if e.is_fatal() => Err(e)?,
192                Ok(Err(e)) => e,
193                Err(_) => FeedError::Connection(format!(
194                    "poll timed out after {request_timeout:?}"
195                )),
196            };
197
198            let retry = failures
199                .record_failure()
200                .inspect(|retry| {
201                    warn!(
202                        consecutive = retry.consecutive,
203                        backoff = ?retry.backoff,
204                        %error,
205                        "poll failed, backing off"
206                    )
207                })
208                // The feed ends with the failure that ended it, classified as its source
209                // classified it: a run of unparseable answers is not a connection failure, and
210                // how long the feed tolerated it is for the log rather than for the caller to
211                // act on.
212                .map_err(|consecutive| {
213                    error!(consecutive, "giving up on the feed");
214                    error
215                })?;
216            backoff = Some(retry.backoff);
217        }
218    }
219}
220
221/// Whether a failing response's body says the venue will never serve this request, such as a
222/// chain it does not price. A refusal is a [`FeedError::Fatal`] rather than the retryable
223/// [`FeedError::Connection`], since another attempt gets the same answer. Every fetch here
224/// takes one, because what a failing status means is the venue's to say and not this module's.
225///
226/// A plain `fn`, so the verdict rests on what the venue said and on nothing else.
227pub(crate) type RefusalCheck = fn(&str) -> bool;
228
229/// For a venue that refuses nothing outright, so every failing answer is worth another attempt.
230pub(crate) fn nothing_refused(_body: &str) -> bool {
231    false
232}
233
234/// Sends `request` and returns its body. A transport failure is a [`FeedError::Connection`], a
235/// failing status whatever `refused` makes of it; `what` names the resource in the message
236/// (e.g. "Hashflow price levels").
237///
238/// For an API that reports failures inside the body of an otherwise successful response, which
239/// therefore has to be classified before it is parsed. [`fetch_json`] is the plain case.
240pub(crate) async fn fetch_bytes(
241    request: RequestBuilder,
242    what: &str,
243    refused: RefusalCheck,
244) -> Result<Bytes, FeedError> {
245    let response = request
246        .send()
247        .await
248        .map_err(|e| FeedError::Connection(format!("Failed to fetch {what}: {e}")))?;
249    read_bytes(response, what, refused).await
250}
251
252/// Reads `response`'s body, failing a non-success status with whatever the server said — as a
253/// refusal when `refused` recognises it there. `what` names the resource, as in [`fetch_bytes`].
254async fn read_bytes(
255    response: Response,
256    what: &str,
257    refused: RefusalCheck,
258) -> Result<Bytes, FeedError> {
259    // The body is read before the status is judged, so a failing response can report what the
260    // server said, and so the venue can be asked what it meant by it.
261    let status = response.status();
262    let body = response
263        .bytes()
264        .await
265        .map_err(|e| FeedError::Connection(format!("Failed to read {what} response: {e}")))?;
266
267    if !status.is_success() {
268        let said = String::from_utf8_lossy(&body);
269        let message = format!("{what} HTTP error {status}: {said}");
270        return Err(if refused(&said) {
271            FeedError::Fatal(message)
272        } else {
273            FeedError::Connection(message)
274        });
275    }
276    Ok(body)
277}
278
279/// Sends `request` and parses its JSON body, fetching as [`fetch_bytes`] does; an unparseable
280/// body is a [`FeedError::Parsing`] naming `what`.
281pub(crate) async fn fetch_json<T: DeserializeOwned>(
282    request: RequestBuilder,
283    what: &str,
284    refused: RefusalCheck,
285) -> Result<T, FeedError> {
286    parse_json(&fetch_bytes(request, what, refused).await?, what)
287}
288
289/// Parses `response`'s JSON body, the second half of [`fetch_json`] for a caller that sends the
290/// request itself.
291pub(crate) async fn read_json<T: DeserializeOwned>(
292    response: Response,
293    what: &str,
294    refused: RefusalCheck,
295) -> Result<T, FeedError> {
296    parse_json(&read_bytes(response, what, refused).await?, what)
297}
298
299fn parse_json<T: DeserializeOwned>(body: &[u8], what: &str) -> Result<T, FeedError> {
300    serde_json::from_slice(body)
301        .map_err(|e| FeedError::Parsing(format!("Failed to parse {what} response: {e}")))
302}
303
304/// A local HTTP/1.1 server for feed tests: answers every request with `respond`'s status line
305/// and JSON body. Serves until the runtime drops it.
306#[cfg(test)]
307pub(crate) mod test_support {
308    use std::{
309        net::SocketAddr,
310        sync::{
311            atomic::{AtomicUsize, Ordering},
312            Arc,
313        },
314        time::Duration,
315    };
316
317    use tokio::{
318        io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader},
319        net::TcpListener,
320    };
321
322    /// One canned HTTP response; `(status_line, body)` converts into it.
323    pub struct MockResponse {
324        pub status: &'static str,
325        pub body: String,
326        /// Wait before answering, for tests that exercise request deadlines.
327        pub delay: Duration,
328    }
329
330    impl From<(&'static str, String)> for MockResponse {
331        fn from((status, body): (&'static str, String)) -> Self {
332            MockResponse { status, body, delay: Duration::ZERO }
333        }
334    }
335
336    /// One request as the mock server read it.
337    pub struct MockRequest {
338        /// Path and query string, as the request line carries them.
339        pub target: String,
340        /// The body, empty where the request had none. Read in full, so a client that sends one
341        /// is not left writing into a socket nobody drains.
342        pub body: String,
343    }
344
345    /// A running mock server: where it listens and how many requests it has answered.
346    pub struct MockHttpServer {
347        pub address: SocketAddr,
348        pub requests: Arc<AtomicUsize>,
349    }
350
351    impl MockHttpServer {
352        pub fn url(&self) -> String {
353            format!("http://{}", self.address)
354        }
355
356        pub fn request_count(&self) -> usize {
357            self.requests.load(Ordering::SeqCst)
358        }
359    }
360
361    /// The headers and however many bytes of body `Content-Length` announced. A request that
362    /// announces none has none: nothing here speaks chunked transfer encoding.
363    async fn read_body(reader: &mut BufReader<tokio::net::TcpStream>) -> String {
364        let mut length = 0;
365        loop {
366            let mut header = String::new();
367            match reader.read_line(&mut header).await {
368                Ok(0) => break,
369                Err(_) => break,
370                Ok(_) if header.trim().is_empty() => break,
371                Ok(_) => {}
372            }
373            if let Some((name, value)) = header.split_once(':') {
374                if name.eq_ignore_ascii_case("content-length") {
375                    length = value.trim().parse().unwrap_or(0);
376                }
377            }
378        }
379        let mut body = vec![0; length];
380        if length > 0 {
381            let _ = reader.read_exact(&mut body).await;
382        }
383        String::from_utf8_lossy(&body).into_owned()
384    }
385
386    /// Serves HTTP/1.1 requests until the runtime drops it. `route` maps a request to a
387    /// response; a request it returns `None` for gets a 404. Each request is counted before
388    /// `route` runs, so a route closure holding its own counter can answer per attempt.
389    ///
390    /// Each connection is served by its own task, so a response that waits out its `delay` holds
391    /// up nobody else: a client that retries gets its second answer a delay after it asked, not
392    /// a delay after the first one finished.
393    pub async fn spawn_http_server<R: Into<MockResponse>>(
394        route: impl Fn(&MockRequest) -> Option<R> + Send + Sync + 'static,
395    ) -> MockHttpServer {
396        let listener = TcpListener::bind("127.0.0.1:0")
397            .await
398            .unwrap();
399        let address = listener.local_addr().unwrap();
400        let requests = Arc::new(AtomicUsize::new(0));
401        let served = Arc::clone(&requests);
402        let route = Arc::new(route);
403        tokio::spawn(async move {
404            while let Ok((stream, _)) = listener.accept().await {
405                let (served, route) = (Arc::clone(&served), Arc::clone(&route));
406                tokio::spawn(async move {
407                    let mut reader = BufReader::new(stream);
408                    let mut request_line = String::new();
409                    if reader
410                        .read_line(&mut request_line)
411                        .await
412                        .is_err()
413                    {
414                        return;
415                    }
416                    served.fetch_add(1, Ordering::SeqCst);
417                    let request = MockRequest {
418                        target: request_line
419                            .split_whitespace()
420                            .nth(1)
421                            .unwrap_or_default()
422                            .to_string(),
423                        body: read_body(&mut reader).await,
424                    };
425                    let response = route(&request)
426                        .map(Into::into)
427                        .unwrap_or_else(|| {
428                            MockResponse::from((
429                                "404 Not Found",
430                                format!("no route for {}", request.target),
431                            ))
432                        });
433
434                    tokio::time::sleep(response.delay).await;
435
436                    let MockResponse { status, body, .. } = response;
437                    let payload = format!(
438                    "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
439                    body.len()
440                );
441                    let mut stream = reader.into_inner();
442                    let _ = stream
443                        .write_all(payload.as_bytes())
444                        .await;
445                    let _ = stream.shutdown().await;
446                });
447            }
448        });
449        MockHttpServer { address, requests }
450    }
451}
452
453#[cfg(test)]
454mod tests {
455    use std::{
456        future::Future,
457        sync::{
458            atomic::{AtomicBool, AtomicU32, Ordering},
459            Arc,
460        },
461    };
462
463    use futures::StreamExt as _;
464    use tokio::sync::watch;
465    use tokio_stream::wrappers::WatchStream;
466
467    use super::{super::expect_to_finish, *};
468
469    /// Lets a closure stand in for a provider's source.
470    impl<F, Fut, T> HttpSource for F
471    where
472        F: Fn() -> Fut + Send + Sync,
473        Fut: Future<Output = Result<T, FeedError>> + Send,
474        T: Send + Sync + 'static,
475    {
476        type Snapshot = T;
477
478        async fn fetch(&self) -> Result<T, FeedError> {
479            self().await
480        }
481    }
482
483    /// The one snapshot the mock sources here publish.
484    fn snapshot() -> u32 {
485        1
486    }
487
488    fn config(max_consecutive_failures: Option<u32>) -> HttpFeedConfig {
489        HttpFeedConfig {
490            poll_interval: Duration::from_millis(5),
491            request_timeout: Duration::from_millis(50),
492            // A failed poll retries at the polling cadence: the tests that fail on purpose are
493            // timing the loop, not the backoff curve.
494            max_backoff_exp: 0,
495            max_consecutive_failures,
496            max_snapshot_age: None,
497        }
498    }
499
500    #[tokio::test]
501    async fn publishes_a_snapshot_and_success_resets_failures() {
502        // One failed poll, then successes: with a limit of 2 the single failure must not
503        // accumulate towards termination once a poll succeeds.
504        let polls = Arc::new(AtomicU32::new(0));
505        let polls_clone = Arc::clone(&polls);
506        let (publisher, mut rx) = Publisher::channel();
507        let feed = run_http_poll_feed(config(Some(2)), publisher, move || {
508            let n = polls_clone.fetch_add(1, Ordering::SeqCst);
509            async move {
510                if n == 0 {
511                    Err(FeedError::Connection("boom".to_string()))
512                } else {
513                    Ok(snapshot())
514                }
515            }
516        });
517        let feed = tokio::spawn(feed);
518
519        expect_to_finish("no snapshot published", async {
520            loop {
521                rx.changed().await.expect("feed ended");
522                if rx.borrow_and_update().is_some() {
523                    return;
524                }
525            }
526        })
527        .await;
528
529        // Several more polls run without termination.
530        tokio::time::sleep(Duration::from_millis(50)).await;
531        assert!(!feed.is_finished());
532    }
533
534    #[tokio::test]
535    async fn withdraws_a_snapshot_that_goes_unrefreshed_and_restores_on_next_success() {
536        // One successful poll, then failures until the flag flips: the published snapshot
537        // must be withdrawn once it has gone max_snapshot_age without a refresh and come back
538        // with the next successful poll.
539        let polls = Arc::new(AtomicU32::new(0));
540        let polls_clone = Arc::clone(&polls);
541        let recovered = Arc::new(AtomicBool::new(false));
542        let recovered_clone = Arc::clone(&recovered);
543        let (publisher, mut rx) = Publisher::channel();
544        let config =
545            HttpFeedConfig { max_snapshot_age: Some(Duration::from_millis(20)), ..config(None) };
546        let feed = run_http_poll_feed(config, publisher, move || {
547            let n = polls_clone.fetch_add(1, Ordering::SeqCst);
548            let recovered = recovered_clone.load(Ordering::SeqCst);
549            async move {
550                if n == 0 || recovered {
551                    Ok(snapshot())
552                } else {
553                    Err(FeedError::Connection("boom".to_string()))
554                }
555            }
556        });
557        let _feed = tokio::spawn(feed);
558
559        let wait_for = |rx: &mut watch::Receiver<Option<u32>>, want_some: bool| {
560            let mut rx = rx.clone();
561            async move {
562                expect_to_finish("watch did not reach the expected state", async {
563                    loop {
564                        if rx.borrow_and_update().is_some() == want_some {
565                            return;
566                        }
567                        rx.changed().await.expect("feed ended");
568                    }
569                })
570                .await;
571            }
572        };
573        wait_for(&mut rx, true).await;
574        wait_for(&mut rx, false).await;
575        recovered.store(true, Ordering::SeqCst);
576        wait_for(&mut rx, true).await;
577    }
578
579    #[tokio::test]
580    async fn rejects_zero_poll_interval() {
581        let (publisher, _rx) = Publisher::<u32>::channel();
582        let config = HttpFeedConfig { poll_interval: Duration::ZERO, ..config(None) };
583        let result = run_http_poll_feed(config, publisher, || async { Ok(snapshot()) }).await;
584        assert!(matches!(result, Err(FeedError::InvalidInput(_))));
585    }
586
587    #[tokio::test]
588    async fn rejects_a_zero_max_snapshot_age() {
589        let (publisher, _rx) = Publisher::<u32>::channel();
590        let config = HttpFeedConfig { max_snapshot_age: Some(Duration::ZERO), ..config(None) };
591        let result = run_http_poll_feed(config, publisher, || async { Ok(snapshot()) }).await;
592        assert!(matches!(result, Err(FeedError::InvalidInput(_))));
593    }
594
595    /// A snapshot may be servable for less than the gap between polls: a consumer that would
596    /// rather decline than quote an old book asks for exactly that when the venue's rate limit
597    /// sets the cadence. The feed serves each snapshot for `max_snapshot_age` and withdraws it
598    /// until the next poll.
599    #[tokio::test(start_paused = true)]
600    async fn a_max_snapshot_age_below_the_poll_interval_is_a_duty_cycle() {
601        let (publisher, rx) = Publisher::<u32>::channel();
602        let mut changes = WatchStream::from_changes(rx);
603        let config = HttpFeedConfig {
604            poll_interval: Duration::from_millis(100),
605            max_snapshot_age: Some(Duration::from_millis(20)),
606            ..config(None)
607        };
608        let _feed =
609            tokio::spawn(run_http_poll_feed(config, publisher, || async { Ok(snapshot()) }));
610
611        let start = tokio::time::Instant::now();
612        let mut seen = Vec::new();
613        for _ in 0..4 {
614            let change = expect_to_finish("the feed stopped changing", changes.next()).await;
615            seen.push((change.expect("the feed is still running"), start.elapsed()));
616        }
617
618        assert_eq!(
619            seen,
620            [
621                // The first poll fires immediately; each snapshot stands for its 20 ms and the
622                // feed serves nothing for the remaining 80 ms of the interval.
623                (Some(snapshot()), Duration::ZERO),
624                (None, Duration::from_millis(20)),
625                (Some(snapshot()), Duration::from_millis(100)),
626                (None, Duration::from_millis(120)),
627            ]
628        );
629    }
630
631    #[tokio::test]
632    async fn gives_up_after_max_consecutive_failed_polls() {
633        // A provider that answers every time with something unparseable is not a connection
634        // failure, so the feed ends with the failure that ended it, as its source classified it.
635        let (publisher, _rx) = Publisher::<u32>::channel();
636        let feed = run_http_poll_feed(config(Some(3)), publisher, || async {
637            Err(FeedError::Parsing("not a snapshot".to_string()))
638        });
639        let feed = tokio::spawn(feed);
640
641        let result = expect_to_finish("feed did not give up", feed)
642            .await
643            .unwrap();
644        assert!(matches!(result, Err(FeedError::Parsing(msg)) if msg == "not a snapshot"));
645    }
646
647    #[tokio::test]
648    async fn gives_up_on_a_fatal_poll_error_despite_an_unlimited_failure_budget() {
649        // A rejected key or an unsupported chain answers every poll the same way, so the
650        // feed stops instead of asking forever — and the snapshot it had goes with it.
651        let (publisher, mut rx) = Publisher::channel();
652        let polls = Arc::new(AtomicU32::new(0));
653        let polls_clone = Arc::clone(&polls);
654        let feed = run_http_poll_feed(config(None), publisher, move || {
655            let n = polls_clone.fetch_add(1, Ordering::SeqCst);
656            async move {
657                if n == 0 {
658                    Ok(snapshot())
659                } else {
660                    Err(FeedError::Fatal("api key rejected".to_string()))
661                }
662            }
663        });
664        let feed = tokio::spawn(feed);
665
666        let result = expect_to_finish("feed did not give up", feed)
667            .await
668            .unwrap();
669
670        assert!(matches!(result, Err(FeedError::Fatal(_))));
671        assert_eq!(polls.load(Ordering::SeqCst), 2, "the feed polled again after the fatal");
672        assert!(rx.borrow_and_update().is_none(), "the snapshot must be withdrawn");
673    }
674
675    #[tokio::test]
676    async fn withdraws_the_published_snapshot_when_it_gives_up() {
677        // A snapshot published before the feed gave up must not stay servable: nothing is
678        // refreshing it, and a consumer holding the receiver would keep using it.
679        let polls = Arc::new(AtomicU32::new(0));
680        let polls_clone = Arc::clone(&polls);
681        let (publisher, mut rx) = Publisher::channel();
682        let feed = run_http_poll_feed(config(Some(2)), publisher, move || {
683            let n = polls_clone.fetch_add(1, Ordering::SeqCst);
684            async move {
685                if n == 0 {
686                    Ok(snapshot())
687                } else {
688                    Err(FeedError::Connection("boom".to_string()))
689                }
690            }
691        });
692        let feed = tokio::spawn(feed);
693
694        expect_to_finish("no snapshot published", async {
695            loop {
696                rx.changed()
697                    .await
698                    .expect("feed ended before publishing");
699                if rx.borrow_and_update().is_some() {
700                    return;
701                }
702            }
703        })
704        .await;
705
706        let result = expect_to_finish("feed did not give up", feed)
707            .await
708            .unwrap();
709        assert!(matches!(result, Err(FeedError::Connection(_))));
710        assert!(rx.borrow().is_none(), "the snapshot must be withdrawn when the feed gives up");
711    }
712
713    #[tokio::test]
714    async fn withdraws_the_published_snapshot_when_the_feed_is_dropped() {
715        // Dropping the feed future — the consumer aborting its task — leaves the receiver
716        // alive; the snapshot stops being refreshed and must stop being served with it.
717        let (publisher, mut rx) = Publisher::channel();
718        let feed = run_http_poll_feed(config(None), publisher, || async { Ok(snapshot()) });
719        let feed = tokio::spawn(feed);
720
721        expect_to_finish("no snapshot published", async {
722            loop {
723                rx.changed()
724                    .await
725                    .expect("feed ended before publishing");
726                if rx.borrow_and_update().is_some() {
727                    return;
728                }
729            }
730        })
731        .await;
732
733        feed.abort();
734        expect_to_finish("the snapshot was not withdrawn", rx.changed())
735            .await
736            .expect("the withdrawal must reach the receiver before the sender drops");
737        assert!(rx.borrow().is_none());
738    }
739
740    #[tokio::test(start_paused = true)]
741    async fn failing_polls_stretch_the_polling_cadence() {
742        // A 50 ms cadence doubling at most three times: a provider that is down is polled at 0
743        // and then after 50, 100, 200, 400 and 400 ms — 5 attempts in the first second, where the
744        // healthy cadence would make 20.
745        let attempts = Arc::new(std::sync::Mutex::new(Vec::new()));
746        let attempts_clone = Arc::clone(&attempts);
747        let start = tokio::time::Instant::now();
748        let (publisher, _rx) = Publisher::<u32>::channel();
749        let feed = tokio::spawn(run_http_poll_feed(
750            HttpFeedConfig {
751                poll_interval: Duration::from_millis(50),
752                max_backoff_exp: 3,
753                ..config(None)
754            },
755            publisher,
756            move || {
757                attempts_clone
758                    .lock()
759                    .unwrap()
760                    .push(start.elapsed().as_millis());
761                async { Err(FeedError::Connection("boom".to_string())) }
762            },
763        ));
764
765        tokio::time::sleep(Duration::from_secs(1)).await;
766        feed.abort();
767
768        assert_eq!(*attempts.lock().unwrap(), vec![0, 50, 150, 350, 750]);
769    }
770
771    #[tokio::test(start_paused = true)]
772    async fn a_recovered_poll_resumes_the_cadence_instead_of_catching_up() {
773        // Two failures stretch the 50 ms cadence to 50 and 100 ms, so the third attempt polls at
774        // 150 ms and succeeds. The polls after it are one plain interval apart: the backoff took
775        // the place of those ticks, so none of them are owed.
776        let attempts = Arc::new(std::sync::Mutex::new(Vec::new()));
777        let attempts_clone = Arc::clone(&attempts);
778        let start = tokio::time::Instant::now();
779        let (publisher, _rx) = Publisher::<u32>::channel();
780        let feed = tokio::spawn(run_http_poll_feed(
781            HttpFeedConfig {
782                poll_interval: Duration::from_millis(50),
783                max_backoff_exp: 3,
784                ..config(None)
785            },
786            publisher,
787            move || {
788                let mut attempts = attempts_clone.lock().unwrap();
789                attempts.push(start.elapsed().as_millis());
790                let failing = attempts.len() <= 2;
791                async move {
792                    if failing {
793                        Err(FeedError::Connection("boom".to_string()))
794                    } else {
795                        Ok(snapshot())
796                    }
797                }
798            },
799        ));
800
801        tokio::time::sleep(Duration::from_millis(300)).await;
802        feed.abort();
803
804        assert_eq!(*attempts.lock().unwrap(), vec![0, 50, 150, 200, 250]);
805    }
806
807    #[tokio::test(start_paused = true)]
808    async fn a_slow_poll_never_shortens_the_gap_to_the_next_one() {
809        // The third poll starts at 100 ms and answers only at 220 ms, having run through two
810        // ticks. The polls that follow it are a full 50 ms interval apart — the ticks it ran
811        // through are not owed back, and none of them lands sooner than the cadence allows, which
812        // is what a venue's rate limit is promised.
813        let attempts = Arc::new(std::sync::Mutex::new(Vec::new()));
814        let attempts_clone = Arc::clone(&attempts);
815        let start = tokio::time::Instant::now();
816        let (publisher, _rx) = Publisher::<u32>::channel();
817        let feed = tokio::spawn(run_http_poll_feed(
818            HttpFeedConfig {
819                poll_interval: Duration::from_millis(50),
820                request_timeout: Duration::from_millis(500),
821                ..config(None)
822            },
823            publisher,
824            move || {
825                let mut attempts = attempts_clone.lock().unwrap();
826                attempts.push(start.elapsed().as_millis());
827                let slow = attempts.len() == 3;
828                async move {
829                    if slow {
830                        tokio::time::sleep(Duration::from_millis(120)).await;
831                    }
832                    Ok(snapshot())
833                }
834            },
835        ));
836
837        tokio::time::sleep(Duration::from_millis(350)).await;
838        feed.abort();
839
840        assert_eq!(*attempts.lock().unwrap(), vec![0, 50, 100, 220, 270, 320]);
841    }
842
843    #[tokio::test]
844    async fn retries_forever_without_limit() {
845        let polls = Arc::new(AtomicU32::new(0));
846        let polls_clone = Arc::clone(&polls);
847        let (publisher, _rx) = Publisher::<u32>::channel();
848        let feed = run_http_poll_feed(config(None), publisher, move || {
849            polls_clone.fetch_add(1, Ordering::SeqCst);
850            async { Err(FeedError::Connection("boom".to_string())) }
851        });
852        let feed = tokio::spawn(feed);
853
854        tokio::time::sleep(Duration::from_millis(100)).await;
855        assert!(polls.load(Ordering::SeqCst) > 5, "should keep polling");
856        assert!(!feed.is_finished());
857    }
858
859    #[tokio::test]
860    async fn hung_poll_counts_as_failure() {
861        let (publisher, _rx) = Publisher::<u32>::channel();
862        let feed = run_http_poll_feed(
863            HttpFeedConfig { request_timeout: Duration::from_millis(5), ..config(Some(2)) },
864            publisher,
865            || async {
866                tokio::time::sleep(Duration::from_secs(3600)).await;
867                Ok(snapshot())
868            },
869        );
870        let feed = tokio::spawn(feed);
871
872        let result = expect_to_finish("feed did not give up", feed)
873            .await
874            .unwrap();
875        assert!(matches!(result, Err(FeedError::Connection(msg)) if msg.contains("timed out")));
876    }
877
878    mod fetch_json {
879
880        use rstest::rstest;
881        use serde::Deserialize;
882
883        use super::{test_support::spawn_http_server, *};
884
885        #[derive(Debug, Deserialize, PartialEq)]
886        struct Payload {
887            value: u32,
888        }
889
890        #[tokio::test]
891        async fn parses_a_successful_json_body() {
892            let server =
893                spawn_http_server(|_| Some(("200 OK", r#"{"value":7}"#.to_string()))).await;
894
895            let payload: Payload = fetch_json(
896                reqwest::Client::new().get(format!("{}/x", server.url())),
897                "thing",
898                nothing_refused,
899            )
900            .await
901            .unwrap();
902
903            assert_eq!(payload, Payload { value: 7 });
904        }
905
906        /// A transport failure or a non-success status is a connection error naming the resource
907        /// (and, for a status, carrying it and the body); a body that is not the expected JSON is a
908        /// parsing error.
909        #[rstest]
910        #[case::non_success_status_with_body("503 Service Unavailable", "maintenance", false)]
911        #[case::unparseable_success_body("200 OK", "<html>", true)]
912        #[tokio::test]
913        async fn classifies_status_and_body_failures(
914            #[case] status: &'static str,
915            #[case] body: &'static str,
916            #[case] expect_parsing_error: bool,
917        ) {
918            let server = spawn_http_server(move |_| Some((status, body.to_string()))).await;
919
920            let result: Result<Payload, FeedError> = fetch_json(
921                reqwest::Client::new().get(format!("{}/x", server.url())),
922                "thing",
923                nothing_refused,
924            )
925            .await;
926
927            match result {
928                Err(FeedError::Parsing(msg)) if expect_parsing_error => {
929                    assert!(msg.contains("thing"), "{msg}")
930                }
931                Err(FeedError::Connection(msg)) if !expect_parsing_error => {
932                    assert!(
933                        msg.contains("thing") && msg.contains("503") && msg.contains(body),
934                        "{msg}"
935                    )
936                }
937                other => panic!("unexpected result: {other:?}"),
938            }
939        }
940
941        #[tokio::test]
942        async fn refused_connection_is_a_connection_error() {
943            // Bind and drop so the port is known to be closed.
944            let address = tokio::net::TcpListener::bind("127.0.0.1:0")
945                .await
946                .unwrap()
947                .local_addr()
948                .unwrap();
949
950            let result: Result<Payload, FeedError> = fetch_json(
951                reqwest::Client::new().get(format!("http://{address}/x")),
952                "thing",
953                nothing_refused,
954            )
955            .await;
956
957            assert!(matches!(result, Err(FeedError::Connection(msg)) if msg.contains("thing")));
958        }
959    }
960}