Skip to main content

fathom_charts/
rest.rs

1use std::collections::BTreeMap;
2use std::fmt;
3use std::pin::Pin;
4use std::sync::Arc;
5use std::time::{Duration, SystemTime, UNIX_EPOCH};
6
7use bytes::Bytes;
8use futures_util::{Stream, StreamExt};
9use reqwest::header::{AUTHORIZATION, CONTENT_DISPOSITION, CONTENT_TYPE, RETRY_AFTER};
10use reqwest::{Method, Response, StatusCode};
11use serde::de::DeserializeOwned;
12use serde::{Deserialize, Serialize};
13use serde_json::{Map, Value};
14
15use crate::error::{api_key_problem, misfit, short_type_name, spacing_problem, Error, ErrorCode};
16use crate::protocol::{Item, MaintenanceWindow, Mode, SourceState};
17
18/// A range of an indicator's history (exports).
19#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
20#[serde(rename_all = "camelCase")]
21pub struct RangeQuery {
22    /// Raw symbol or alias from GET /v1/instruments, or the raw symbol of another contract of the same market.
23    pub instrument: String,
24    pub indicator: String,
25    #[serde(default, skip_serializing_if = "Option::is_none")]
26    pub params: Option<Value>,
27    /// Unix nanoseconds, decimal strings; [from, to) on the reception time of the trades.
28    pub from: String,
29    pub to: String,
30    /// `Live` (default): every version published in real time; `Confirmed`: only the final ones. Same compute units.
31    #[serde(default, skip_serializing_if = "Option::is_none")]
32    pub mode: Option<Mode>,
33}
34
35impl RangeQuery {
36    pub fn new(instrument: impl Into<String>, indicator: impl Into<String>, from: impl Into<String>, to: impl Into<String>) -> RangeQuery {
37        RangeQuery {
38            instrument: instrument.into(),
39            indicator: indicator.into(),
40            params: None,
41            from: from.into(),
42            to: to.into(),
43            mode: None,
44        }
45    }
46
47    /// Parameters of the indicator, a JSON object (`serde_json::to_value` of a generated `<Name>Params`, or `json!({…})`).
48    pub fn params(mut self, params: Value) -> RangeQuery {
49        self.params = Some(params);
50        self
51    }
52
53    pub fn mode(mut self, mode: Mode) -> RangeQuery {
54        self.mode = Some(mode);
55        self
56    }
57}
58
59/// A range of an indicator's history, read page by page (`page`, `pages`, `history`, `estimate`).
60#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
61#[serde(rename_all = "camelCase")]
62pub struct PageQuery {
63    /// Raw symbol or alias from GET /v1/instruments, or the raw symbol of another contract of the same market.
64    pub instrument: String,
65    pub indicator: String,
66    #[serde(default, skip_serializing_if = "Option::is_none")]
67    pub params: Option<Value>,
68    /// Unix nanoseconds, decimal strings; [from, to) on the reception time of the trades.
69    pub from: String,
70    pub to: String,
71    /// `Live` (default): every version published in real time; `Confirmed`: only the final ones. Same compute units.
72    #[serde(default, skip_serializing_if = "Option::is_none")]
73    pub mode: Option<Mode>,
74    /// `next` of the previous page of the same query (same account and parameters).
75    #[serde(default, skip_serializing_if = "Option::is_none")]
76    pub cursor: Option<String>,
77    /// Stops at this stream cursor (join with a live subscription).
78    #[serde(default, skip_serializing_if = "Option::is_none")]
79    pub until_cursor: Option<String>,
80    #[serde(default, skip_serializing_if = "Option::is_none")]
81    pub limit: Option<u32>,
82    /// First page only: also the state at `from` (`snapshot` of the page), as a subscription snapshot.
83    #[serde(default, skip_serializing_if = "Option::is_none")]
84    pub snapshot: Option<bool>,
85}
86
87impl PageQuery {
88    pub fn new(instrument: impl Into<String>, indicator: impl Into<String>, from: impl Into<String>, to: impl Into<String>) -> PageQuery {
89        PageQuery::from(RangeQuery::new(instrument, indicator, from, to))
90    }
91
92    /// Parameters of the indicator, a JSON object (`serde_json::to_value` of a generated `<Name>Params`, or `json!({…})`).
93    pub fn params(mut self, params: Value) -> PageQuery {
94        self.params = Some(params);
95        self
96    }
97
98    pub fn mode(mut self, mode: Mode) -> PageQuery {
99        self.mode = Some(mode);
100        self
101    }
102
103    pub fn cursor(mut self, cursor: impl Into<String>) -> PageQuery {
104        self.cursor = Some(cursor.into());
105        self
106    }
107
108    pub fn until_cursor(mut self, cursor: impl Into<String>) -> PageQuery {
109        self.until_cursor = Some(cursor.into());
110        self
111    }
112
113    pub fn limit(mut self, limit: u32) -> PageQuery {
114        self.limit = Some(limit);
115        self
116    }
117
118    pub fn snapshot(mut self, snapshot: bool) -> PageQuery {
119        self.snapshot = Some(snapshot);
120        self
121    }
122}
123
124impl std::convert::From<RangeQuery> for PageQuery {
125    fn from(q: RangeQuery) -> PageQuery {
126        PageQuery {
127            instrument: q.instrument,
128            indicator: q.indicator,
129            params: q.params,
130            from: q.from,
131            to: q.to,
132            mode: q.mode,
133            cursor: None,
134            until_cursor: None,
135            limit: None,
136            snapshot: None,
137        }
138    }
139}
140
141/// Resumption of a cut export: `X-Export-Token` of its first response and cursor of the last complete line received.
142#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
143pub struct ExportResume {
144    pub token: String,
145    #[serde(default, skip_serializing_if = "Option::is_none")]
146    pub after: Option<String>,
147}
148
149#[derive(Debug, Clone, Copy, PartialEq, Eq)]
150pub struct ExportOptions {
151    /// Consecutive resumptions without a new line (connection lost, stream cut before its end line, 5xx) before failing. Default 5.
152    pub max_resumes: u32,
153    /// Delay before the first resumption (default 1 s), doubled for each consecutive one up to 30 s.
154    pub resume_delay: Duration,
155}
156
157impl Default for ExportOptions {
158    fn default() -> Self {
159        ExportOptions { max_resumes: 5, resume_delay: Duration::from_secs(1) }
160    }
161}
162
163#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
164#[serde(rename_all = "lowercase")]
165pub enum MutationKind {
166    Upsert,
167    Remove,
168}
169
170/// One mutation of a historical range: as the `upsert` and `remove` messages of a subscription, with their cursor.
171#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
172pub struct Mutation<T = Value> {
173    pub cursor: String,
174    pub t: MutationKind,
175    pub id: String,
176    #[serde(rename = "final")]
177    pub is_final: bool,
178    pub ts: String,
179    #[serde(default = "Option::default", skip_serializing_if = "Option::is_none")]
180    pub data: Option<T>,
181}
182
183/// State at `from` (`snapshot: true`): the objects then, as the `snapshot` of a subscription, and the cursor it covers.
184#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
185pub struct PageSnapshot<T = Value> {
186    pub cursor: String,
187    pub items: Vec<Item<T>>,
188}
189
190/// An element of [`FathomChartsRest::history`]: with `snapshot: true`, the snapshot comes first, then the mutations.
191#[derive(Debug, Clone, PartialEq)]
192pub enum HistoryItem<T = Value> {
193    Snapshot(PageSnapshot<T>),
194    Mutation(Mutation<T>),
195}
196
197#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
198#[serde(rename_all = "camelCase")]
199pub struct Page<T = Value> {
200    /// Contract read (an alias resolves to the one it designated at `from`).
201    pub instrument: String,
202    /// Price increment of that contract, in 1e-9 units.
203    pub tick_size_nanos: String,
204    /// With `snapshot: true`, on the first page.
205    #[serde(default = "Option::default", skip_serializing_if = "Option::is_none")]
206    pub snapshot: Option<PageSnapshot<T>>,
207    pub items: Vec<Mutation<T>>,
208    pub next: Option<String>,
209    pub compute_units: f64,
210}
211
212#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
213#[serde(rename_all = "camelCase")]
214pub struct Estimate {
215    pub instrument: String,
216    pub tick_size_nanos: String,
217    pub trades: f64,
218    pub compute_units: f64,
219    pub covered_from: Option<String>,
220    pub covered_to: Option<String>,
221    pub missing_days: Vec<String>,
222    pub remaining_compute_units: f64,
223}
224
225/// One instrument served (GET /v1/instruments).
226#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
227#[serde(rename_all = "camelCase")]
228pub struct Instrument {
229    /// Raw symbol of the contract, e.g. `NQZ6`.
230    pub symbol: String,
231    /// Alias that designates it, e.g. `NQ.front`, or none.
232    pub alias: Option<String>,
233    /// Price increment, in 1e-9 units (decimal string).
234    pub tick_size_nanos: String,
235    pub status: SourceState,
236    /// Unix nanoseconds (decimal string) of the last status change.
237    pub updated_at: String,
238}
239
240/// An instrument in the service status.
241#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
242#[serde(rename_all = "camelCase")]
243pub struct InstrumentStatus {
244    pub symbol: String,
245    pub alias: Option<String>,
246    pub status: SourceState,
247    pub updated_at: String,
248}
249
250#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
251pub struct EngineStatus {
252    /// `up` or `down`.
253    pub state: String,
254}
255
256/// Source and service state (GET /v1/status).
257#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
258pub struct ServiceStatus {
259    pub engine: EngineStatus,
260    /// Maintenance announced and not over, by start (also sent to stream connections as a `maintenance` notice).
261    pub maintenance: Vec<MaintenanceWindow>,
262    pub instruments: Vec<InstrumentStatus>,
263}
264
265/// One parameter of an indicator in the catalog.
266#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
267pub struct CatalogParam {
268    pub description: String,
269    /// `integer`, `number`, `boolean` or `choice`.
270    #[serde(rename = "type")]
271    pub kind: String,
272    /// `calc`: changes the computation; `view`: only filters or presents its output.
273    pub category: String,
274    /// Default value, for the products without their own (see [`CatalogIndicator::product_defaults`]).
275    pub default: Value,
276    #[serde(default, skip_serializing_if = "Option::is_none")]
277    pub minimum: Option<f64>,
278    #[serde(default, skip_serializing_if = "Option::is_none")]
279    pub maximum: Option<f64>,
280    #[serde(default, skip_serializing_if = "Option::is_none")]
281    pub unit: Option<String>,
282    /// Allowed values of a `choice`.
283    #[serde(default, skip_serializing_if = "Option::is_none")]
284    pub options: Option<Vec<Value>>,
285}
286
287/// One field of the `data` of an indicator object.
288#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
289pub struct CatalogField {
290    pub description: String,
291    #[serde(rename = "type")]
292    pub kind: String,
293    #[serde(default, skip_serializing_if = "Option::is_none")]
294    pub unit: Option<String>,
295    #[serde(default, rename = "enum", skip_serializing_if = "Option::is_none")]
296    pub values: Option<Vec<String>>,
297    #[serde(default, skip_serializing_if = "Option::is_none")]
298    pub optional: Option<bool>,
299}
300
301/// One kind of object an indicator publishes.
302#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
303#[serde(rename_all = "camelCase")]
304pub struct CatalogObject {
305    pub kind: String,
306    pub description: String,
307    pub id_pattern: String,
308    pub provisional: bool,
309    pub fields: BTreeMap<String, CatalogField>,
310}
311
312#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
313#[serde(rename_all = "camelCase")]
314pub struct CatalogIndicator {
315    pub id: String,
316    pub name: String,
317    pub summary: String,
318    pub description: String,
319    /// When the state of the indicator starts over: `none`, at each `session`, each `week`, or per `periods`.
320    pub origin: String,
321    pub inputs: Vec<String>,
322    pub compute_units_per_trade: f64,
323    pub params: BTreeMap<String, CatalogParam>,
324    /// Default parameters specific to a product (ticker root, e.g. `ES`), overriding `default` for its contracts; only
325    /// the parameters whose default differs are listed.
326    #[serde(default)]
327    pub product_defaults: BTreeMap<String, BTreeMap<String, Value>>,
328    pub objects: Vec<CatalogObject>,
329    /// JSON Schemas (2020-12) of `params` and of the `data` of the mutations.
330    pub params_schema: Value,
331    pub data_schema: Value,
332    /// `params` and `messages` of an example.
333    pub example: Value,
334}
335
336/// Indicator catalog (GET /v1/indicators).
337#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
338#[serde(rename_all = "camelCase")]
339pub struct Catalog {
340    pub version: u32,
341    pub engine_fingerprint: String,
342    pub indicators: Vec<CatalogIndicator>,
343}
344
345#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
346#[serde(rename_all = "camelCase")]
347pub struct Limits {
348    /// Real-time stream (Live); otherwise the stream is delayed by `delay_seconds`.
349    pub realtime: bool,
350    pub delay_seconds: u64,
351    pub max_connections: u64,
352    pub max_subscriptions: u64,
353    pub max_custom_configs: u64,
354    /// Depth of the historical API (queries, estimates, exports, `from: {time}`).
355    pub history_days: u64,
356    /// History starting within this many days consumes no compute budget (Live); 0: none.
357    pub included_history_days: u64,
358    /// Compute units and exported bytes per UTC month.
359    pub compute_budget: f64,
360    pub rest_per_minute: u64,
361    pub export_bytes: f64,
362}
363
364#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
365#[serde(rename_all = "camelCase")]
366pub struct UsageCounters {
367    pub compute_units: f64,
368    pub history_items: f64,
369    pub export_bytes: f64,
370    pub rest_requests: f64,
371}
372
373#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
374#[serde(rename_all = "camelCase")]
375pub struct UsageRemaining {
376    pub compute_units: f64,
377    pub export_bytes: f64,
378}
379
380/// Usage of the account this period (GET /v1/usage).
381#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
382#[serde(rename_all = "camelCase")]
383pub struct Usage {
384    pub period: String,
385    pub resets_at: String,
386    /// Plan of the account: the combination of the offers it owns (`live`, `history`, `live_history`), `sandbox` (free)
387    /// without either.
388    pub plan: String,
389    pub limits: Limits,
390    pub used: UsageCounters,
391    pub remaining: UsageRemaining,
392}
393
394/// Single-use stream token (POST /v1/stream-tokens).
395#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
396#[serde(rename_all = "camelCase")]
397pub struct StreamToken {
398    pub token: String,
399    pub expires_at: String,
400}
401
402/// A stream of bytes read at the consumer's pace.
403pub type ByteStream = Pin<Box<dyn Stream<Item = Result<Bytes, Error>> + Send>>;
404
405/// One export response.
406pub struct ExportStream {
407    /// The zstd-compressed NDJSON file (`application/zstd`), as sent: one mutation per line, then the end line `{"t":"end"}`.
408    pub body: ByteStream,
409    pub content_type: String,
410    pub filename: Option<String>,
411    /// `X-Export-Token`: resumes this export after a cut (`resume` of `export()`).
412    pub token: Option<String>,
413}
414
415impl fmt::Debug for ExportStream {
416    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
417        f.debug_struct("ExportStream")
418            .field("content_type", &self.content_type)
419            .field("filename", &self.filename)
420            .field("token", &self.token)
421            .finish_non_exhaustive()
422    }
423}
424
425const EXPORT_TYPE: &str = "application/zstd";
426const NEWLINE: u8 = b'\n';
427
428/// Whether `line` (with its newline) is the end line of an export, `{"t":"end"}`.
429fn is_end_line(line: &[u8]) -> bool {
430    line.starts_with(br#"{"t":"#) && serde_json::from_slice::<Value>(line).is_ok_and(|v| v.get("t") == Some(&Value::from("end")))
431}
432
433/// Cursor of the last line of `lines` (which ends with a newline): `{"cursor":"<day>.<rank>.<k>"` opens every line.
434fn last_cursor(lines: &[u8]) -> Result<String, Error> {
435    let body = &lines[..lines.len() - 1];
436    let start = body.iter().rposition(|&b| b == NEWLINE).map_or(0, |i| i + 1);
437    let head = &body[start..];
438    let cursor =
439        head.strip_prefix(br#"{"cursor":""#).and_then(|rest| rest.iter().position(|&b| b == b'"').map(|end| &rest[..end])).filter(|c| {
440            let parts: Vec<&[u8]> = c.split(|&b| b == b'.').collect();
441            parts.len() == 3 && parts.iter().all(|p| !p.is_empty() && p.iter().all(u8::is_ascii_digit))
442        });
443    match cursor {
444        Some(cursor) => Ok(String::from_utf8_lossy(cursor).into_owned()),
445        None => Err(Error::new(ErrorCode::UpstreamError, "Export line without cursor").with_status(502)),
446    }
447}
448
449/// `detail` of a problem document, followed by those of its invalid fields (`errors`) that it does not already state: the
450/// API's detail often names the fields itself (`from must not be in the future`), or only sums them up (`Invalid indicator
451/// parameters`). Every field stays in `Error::problem`.
452fn problem_message(problem: Option<&Map<String, Value>>, fallback: &str) -> String {
453    let detail = problem.and_then(|p| p.get("detail")).and_then(Value::as_str).unwrap_or(fallback);
454    let errors: Vec<String> = match problem.and_then(|p| p.get("errors")) {
455        Some(Value::Array(errors)) => errors
456            .iter()
457            .filter_map(|e| {
458                let message = e.get("message")?.as_str()?;
459                if detail.contains(message) {
460                    return None;
461                }
462                Some(match e.get("path").and_then(Value::as_str).filter(|p| !p.is_empty()) {
463                    Some(path) => format!("{path} {message}"),
464                    None => message.to_string(),
465                })
466            })
467            .collect(),
468        _ => Vec::new(),
469    };
470    if errors.is_empty() {
471        detail.to_string()
472    } else {
473        format!("{detail}: {}", errors.join("; "))
474    }
475}
476
477/// A failure that is not an answer of the API, as an [`Error`] whose `source` is the original error.
478fn wrapped(error: reqwest::Error) -> Error {
479    if error.is_timeout() {
480        return Error::new(ErrorCode::Aborted, "Request aborted by its timeout").with_source(error);
481    }
482    let mut reason: &dyn std::error::Error = &error;
483    while let Some(inner) = reason.source() {
484        reason = inner;
485    }
486    let message = format!("The API could not be reached or the connection was lost: {reason}");
487    Error::new(ErrorCode::NetworkError, message).with_source(error)
488}
489
490/// An answer of the API as `T`: UPSTREAM_ERROR when it is not JSON, or JSON that is not the document `T` reads.
491fn read_json<T: DeserializeOwned>(bytes: &[u8]) -> Result<T, Error> {
492    serde_json::from_slice(bytes).map_err(|e| {
493        let message = if e.is_data() {
494            format!("The answer of the API does not fit {}: {e}", short_type_name::<T>())
495        } else {
496            "The API answered with invalid JSON".to_string()
497        };
498        Error::new(ErrorCode::UpstreamError, message).with_source(e)
499    })
500}
501
502impl Page<Value> {
503    /// The page with the data of its objects read as `T`; CONFIGURATION when they do not fit it.
504    fn decode<T: DeserializeOwned>(self, indicator: &str) -> Result<Page<T>, Error> {
505        let data = |value: Value| serde_json::from_value::<T>(value).map_err(|e| misfit::<T>(indicator, e));
506        let snapshot = match self.snapshot {
507            Some(PageSnapshot { cursor, items }) => Some(PageSnapshot {
508                cursor,
509                items: items
510                    .into_iter()
511                    .map(|i| Ok(Item { id: i.id, is_final: i.is_final, ts: i.ts, data: data(i.data)? }))
512                    .collect::<Result<_, Error>>()?,
513            }),
514            None => None,
515        };
516        let items = self
517            .items
518            .into_iter()
519            .map(|m| {
520                let Mutation { cursor, t, id, is_final, ts, data: value } = m;
521                Ok(Mutation { cursor, t, id, is_final, ts, data: value.map(data).transpose()? })
522            })
523            .collect::<Result<_, Error>>()?;
524        Ok(Page {
525            instrument: self.instrument,
526            tick_size_nanos: self.tick_size_nanos,
527            snapshot,
528            items,
529            next: self.next,
530            compute_units: self.compute_units,
531        })
532    }
533}
534
535/// Options of a [`FathomChartsRest`].
536#[derive(Debug, Clone)]
537pub struct RestBuilder {
538    base_url: String,
539    api_key: Option<String>,
540    client: Option<reqwest::Client>,
541    rate_limit_retries: u32,
542}
543
544impl RestBuilder {
545    pub fn api_key(mut self, api_key: impl Into<String>) -> RestBuilder {
546        self.api_key = Some(api_key.into());
547        self
548    }
549
550    /// Retries (default 3) of a request the server did not process or that has no effect but its answer: a RATE_LIMITED answer
551    /// (429) or a 503 with Retry-After, after that delay (at most 60 s; a 429 without it: 1 s, 2 s, 4 s…); a GET that failed
552    /// without an answer (network error, connection closed, e.g. during an update of the service) or answered 502, after 0.5 s,
553    /// 1 s, 2 s…
554    pub fn rate_limit_retries(mut self, retries: u32) -> RestBuilder {
555        self.rate_limit_retries = retries;
556        self
557    }
558
559    /// The HTTP client (default: `reqwest::Client::new()`); a timeout of its own ends a call with ABORTED.
560    pub fn client(mut self, client: reqwest::Client) -> RestBuilder {
561        self.client = Some(client);
562        self
563    }
564
565    pub fn build(self) -> Result<FathomChartsRest, Error> {
566        if let Some(problem) = api_key_problem(self.api_key.as_deref()) {
567            return Err(Error::new(ErrorCode::Configuration, problem));
568        }
569        if let Some(problem) = self.api_key.as_deref().and_then(|key| spacing_problem("api_key", key)) {
570            return Err(Error::new(ErrorCode::Configuration, problem));
571        }
572        let url = reqwest::Url::parse(&self.base_url).ok();
573        if !url.is_some_and(|u| matches!(u.scheme(), "http" | "https") && u.has_host()) {
574            return Err(Error::new(
575                ErrorCode::Configuration,
576                format!("base_url must be an absolute http:// or https:// URL, not {:?}", self.base_url),
577            ));
578        }
579        let client = match self.client {
580            Some(client) => client,
581            None => reqwest::Client::builder()
582                .build()
583                .map_err(|e| Error::new(ErrorCode::Configuration, format!("The HTTP client could not be created: {e}")).with_source(e))?,
584        };
585        Ok(FathomChartsRest {
586            inner: Arc::new(RestInner {
587                base: self.base_url.trim_end_matches('/').to_string(),
588                api_key: self.api_key.unwrap_or_default(),
589                client,
590                retries: self.rate_limit_retries,
591            }),
592        })
593    }
594}
595
596#[derive(Debug)]
597struct RestInner {
598    base: String,
599    api_key: String,
600    client: reqwest::Client,
601    retries: u32,
602}
603
604/// REST helpers of the public API, authenticated by API key. A cheap handle: clones share the HTTP client.
605///
606/// Every call is cancelled by dropping its future (or its stream), including while it waits to retry.
607#[derive(Debug, Clone)]
608pub struct FathomChartsRest {
609    inner: Arc<RestInner>,
610}
611
612/// An export response whose content type was checked.
613struct Exported {
614    response: Response,
615    content_type: String,
616    filename: Option<String>,
617    token: Option<String>,
618}
619
620#[derive(Serialize)]
621struct ExportBody<'a> {
622    #[serde(flatten)]
623    query: &'a RangeQuery,
624    #[serde(skip_serializing_if = "Option::is_none")]
625    resume: Option<&'a ExportResume>,
626}
627
628impl FathomChartsRest {
629    /// A client of the API at `base_url`, e.g. `https://api.fathomcharts.com`.
630    pub fn new(base_url: impl Into<String>, api_key: impl Into<String>) -> Result<FathomChartsRest, Error> {
631        FathomChartsRest::builder(base_url).api_key(api_key).build()
632    }
633
634    pub fn builder(base_url: impl Into<String>) -> RestBuilder {
635        RestBuilder { base_url: base_url.into(), api_key: None, client: None, rate_limit_retries: 3 }
636    }
637
638    async fn request<B: Serialize + ?Sized>(&self, method: Method, path: &str, body: Option<&B>) -> Result<Response, Error> {
639        let inner = &self.inner;
640        let url = format!("{}{}", inner.base, path);
641        let mut attempt = 0u32;
642        loop {
643            let mut request = inner.client.request(method.clone(), &url).header(AUTHORIZATION, format!("Bearer {}", inner.api_key));
644            if let Some(body) = body {
645                request = request.json(body);
646            }
647            let response = match request.send().await {
648                Ok(response) => response,
649                Err(error) => {
650                    // A read has no effect but its answer: retried after a lost connection.
651                    if method != Method::GET || attempt >= inner.retries {
652                        return Err(wrapped(error));
653                    }
654                    tokio::time::sleep(Duration::from_millis(500 << attempt.min(16))).await;
655                    attempt += 1;
656                    continue;
657                }
658            };
659            let status = response.status();
660            if status.is_success() {
661                return Ok(response);
662            }
663            let retry_after =
664                response.headers().get(RETRY_AFTER).and_then(|v| v.to_str().ok()).and_then(|v| retry_after_seconds(v, SystemTime::now()));
665            let text = response.bytes().await.map_err(wrapped)?;
666            let problem = match serde_json::from_slice::<Value>(&text) {
667                Ok(Value::Object(problem)) => Some(problem),
668                _ => None,
669            };
670            let code = match problem.as_ref().and_then(|p| p.get("code")).and_then(Value::as_str) {
671                Some(code) => code.to_string(),
672                None => format!("HTTP_{}", status.as_u16()),
673            };
674            let wait = |seconds: f64| Duration::from_secs_f64(seconds.clamp(0.0, 60.0));
675            if status == StatusCode::TOO_MANY_REQUESTS && (code == "RATE_LIMITED" || code == "HTTP_429") && attempt < inner.retries {
676                tokio::time::sleep(wait(retry_after.unwrap_or(2f64.powi(attempt as i32)))).await;
677                attempt += 1;
678                continue;
679            }
680            // Not processed: the server says when to come back.
681            if status == StatusCode::SERVICE_UNAVAILABLE && retry_after.is_some() && attempt < inner.retries {
682                tokio::time::sleep(wait(retry_after.unwrap_or_default())).await;
683                attempt += 1;
684                continue;
685            }
686            if status == StatusCode::BAD_GATEWAY && method == Method::GET && attempt < inner.retries {
687                tokio::time::sleep(Duration::from_millis(500 << attempt.min(16))).await;
688                attempt += 1;
689                continue;
690            }
691            let fallback = match status.canonical_reason() {
692                Some(reason) => reason.to_string(),
693                None => format!("HTTP {}", status.as_u16()),
694            };
695            let message = problem_message(problem.as_ref(), &fallback);
696            return Err(Error::new(ErrorCode::parse(&code), message)
697                .with_status(status.as_u16())
698                .with_problem(problem)
699                .with_retry_after(retry_after.map(|s| Duration::from_secs_f64(s.max(0.0)))));
700        }
701    }
702
703    async fn json<T: DeserializeOwned, B: Serialize + ?Sized>(&self, method: Method, path: &str, body: Option<&B>) -> Result<T, Error> {
704        let response = self.request(method, path, body).await?;
705        let bytes = response.bytes().await.map_err(wrapped)?;
706        read_json(&bytes)
707    }
708
709    async fn get<T: DeserializeOwned>(&self, path: &str) -> Result<T, Error> {
710        self.json::<T, ()>(Method::GET, path, None).await
711    }
712
713    /// Indicator catalog: parameters, objects and JSON Schemas of each indicator (public).
714    pub async fn indicators(&self) -> Result<Catalog, Error> {
715        self.get("/v1/indicators").await
716    }
717
718    pub async fn instruments(&self) -> Result<Vec<Instrument>, Error> {
719        #[derive(Deserialize)]
720        struct Instruments {
721            instruments: Vec<Instrument>,
722        }
723        Ok(self.get::<Instruments>("/v1/instruments").await?.instruments)
724    }
725
726    pub async fn status(&self) -> Result<ServiceStatus, Error> {
727        self.get("/v1/status").await
728    }
729
730    pub async fn usage(&self) -> Result<Usage, Error> {
731        self.get("/v1/usage").await
732    }
733
734    /// Single-use stream token (60 s). Call it from your backend and hand the token to the application that streams.
735    pub async fn stream_token(&self) -> Result<StreamToken, Error> {
736        self.json::<StreamToken, ()>(Method::POST, "/v1/stream-tokens", None).await
737    }
738
739    /// Cost of a query without running it. Takes the same body as `page()`: `until_cursor` counts, `cursor`, `limit` and
740    /// `snapshot` are ignored.
741    pub async fn estimate(&self, query: &PageQuery) -> Result<Estimate, Error> {
742        check_query(&query.instrument, &query.indicator, &query.from, &query.to)?;
743        self.json(Method::POST, "/v1/indicator-queries/estimate", Some(query)).await
744    }
745
746    /// One page of a range. `T` is the type of the data of its objects (a generated `<Name>Data`, or `serde_json::Value`):
747    /// data that does not fit it fails with CONFIGURATION (the page is computed and charged all the same).
748    pub async fn page<T: DeserializeOwned>(&self, query: &PageQuery) -> Result<Page<T>, Error> {
749        check_query(&query.instrument, &query.indicator, &query.from, &query.to)?;
750        let page: Page<Value> = self.json(Method::POST, "/v1/indicator-queries", Some(query)).await?;
751        page.decode(&query.indicator)
752    }
753
754    /// Every page of a range, fetched one at a time as the consumer asks for them. A failure is yielded once, then the
755    /// stream ends.
756    pub fn pages<T: DeserializeOwned + Send + 'static>(
757        &self,
758        query: PageQuery,
759    ) -> impl Stream<Item = Result<Page<T>, Error>> + Send + 'static {
760        let rest = self.clone();
761        futures_util::stream::unfold(Some(query), move |query| {
762            let rest = rest.clone();
763            async move {
764                let query = query?;
765                match rest.page::<T>(&query).await {
766                    Ok(page) => {
767                        let next = page.next.clone().map(|cursor| PageQuery { cursor: Some(cursor), ..query });
768                        Some((Ok(page), next))
769                    }
770                    Err(error) => Some((Err(error), None)),
771                }
772            }
773        })
774    }
775
776    /// Every mutation of a range, in order, across pages. With `snapshot: true`, the state at `from` comes first, as a
777    /// [`HistoryItem::Snapshot`]. A failure is yielded once, then the stream ends.
778    pub fn history<T: DeserializeOwned + Send + 'static>(
779        &self,
780        query: PageQuery,
781    ) -> impl Stream<Item = Result<HistoryItem<T>, Error>> + Send + 'static {
782        self.pages::<T>(query).flat_map(|page| {
783            let items: Vec<Result<HistoryItem<T>, Error>> = match page {
784                Ok(page) => page
785                    .snapshot
786                    .map(HistoryItem::Snapshot)
787                    .into_iter()
788                    .chain(page.items.into_iter().map(HistoryItem::Mutation))
789                    .map(Ok)
790                    .collect(),
791                Err(error) => vec![Err(error)],
792            };
793            futures_util::stream::iter(items)
794        })
795    }
796
797    /// One export response: the zstd-compressed NDJSON file (one mutation per line, with its cursor, then the end line
798    /// `{"t":"end"}`), read at the consumer's pace. A response without its end line was cut: `resume` (its `token` and the
799    /// cursor of the last complete line) continues strictly after that line, as a new zstd file that ends with its own end
800    /// line. `export_ndjson()` does all of this.
801    pub async fn export(&self, query: &RangeQuery, resume: Option<&ExportResume>) -> Result<ExportStream, Error> {
802        let exported = self.exported(query, resume).await?;
803        let body = exported.response.bytes_stream().map(|chunk| chunk.map_err(wrapped));
804        Ok(ExportStream { body: Box::pin(body), content_type: exported.content_type, filename: exported.filename, token: exported.token })
805    }
806
807    async fn exported(&self, query: &RangeQuery, resume: Option<&ExportResume>) -> Result<Exported, Error> {
808        check_query(&query.instrument, &query.indicator, &query.from, &query.to)?;
809        let response = self.request(Method::POST, "/v1/exports", Some(&ExportBody { query, resume })).await?;
810        let header = |name: &str| response.headers().get(name).and_then(|v| v.to_str().ok()).map(str::to_string);
811        let content_type = header(CONTENT_TYPE.as_str()).unwrap_or_default();
812        if content_type.split(';').next().unwrap_or_default().trim().to_ascii_lowercase() != EXPORT_TYPE {
813            let shown = if content_type.is_empty() { "no content type" } else { &content_type };
814            return Err(Error::new(ErrorCode::UpstreamError, format!("Export answered as {shown}, not {EXPORT_TYPE}")));
815        }
816        let filename = header(CONTENT_DISPOSITION.as_str()).and_then(|d| {
817            let start = d.find("filename=\"")? + "filename=\"".len();
818            let name = &d[start..][..d[start..].find('"')?];
819            (!name.is_empty()).then(|| name.to_string())
820        });
821        let token = header("x-export-token");
822        Ok(Exported { response, content_type, filename, token })
823    }
824
825    /// Export of a range as decompressed NDJSON (mutation lines only, the end line removed), in chunks of whole lines, read
826    /// at the consumer's pace. A cut once the first response has started (connection lost, response without its end line,
827    /// deployment of the service, 5xx answer) is followed by a resumption after the last line received: the chunks
828    /// concatenate to exactly the mutation lines of an uninterrupted export, without gap or duplicate; compute units are
829    /// charged once. The first request is never sent again: when it fails, its error is yielded and the caller decides.
830    pub fn export_ndjson(&self, query: RangeQuery, options: ExportOptions) -> impl Stream<Item = Result<Bytes, Error>> + Send + 'static {
831        let export = NdjsonExport {
832            rest: self.clone(),
833            query,
834            options,
835            token: None,
836            after: None,
837            failures: 0,
838            body: None,
839            decoder: None,
840            rest_bytes: Vec::new(),
841            ended: false,
842            cut: None,
843            done: false,
844        };
845        futures_util::stream::unfold(export, |mut export| async move {
846            let chunk = export.next().await?;
847            Some((chunk, export))
848        })
849    }
850}
851
852/// The fields of a query, checked before anything is sent: CONFIGURATION when one is empty.
853fn check_query(instrument: &str, indicator: &str, from: &str, to: &str) -> Result<(), Error> {
854    for (value, name, what) in [
855        (instrument, "instrument", "a raw symbol or an alias, e.g. NQZ6"),
856        (indicator, "indicator", "an indicator id of the catalog, e.g. big-trades"),
857        (from, "from", "Unix nanoseconds as a decimal string"),
858        (to, "to", "Unix nanoseconds as a decimal string"),
859    ] {
860        if value.trim().is_empty() {
861            return Err(Error::new(ErrorCode::Configuration, format!("{name} of the query is missing: give {what}")));
862        }
863    }
864    Ok(())
865}
866
867/// A cut of an export response.
868enum Cut {
869    /// Network failure while reading the body: worth a resumption.
870    Network(reqwest::Error),
871    /// An error of the SDK or of the API: worth a resumption when it is a 5xx or a lost connection (NETWORK_ERROR).
872    Api(Error),
873}
874
875impl Cut {
876    /// Cuts worth a resumption: network and decompression errors, a resume request that lost its connection, 5xx answers.
877    fn resumable(&self) -> bool {
878        match self {
879            Cut::Network(_) => true,
880            Cut::Api(error) => *error.code() == ErrorCode::NetworkError || error.status().unwrap_or(0) >= 500,
881        }
882    }
883
884    fn into_error(self) -> Error {
885        match self {
886            Cut::Network(error) => wrapped(error),
887            Cut::Api(error) => error,
888        }
889    }
890}
891
892struct NdjsonExport {
893    rest: FathomChartsRest,
894    query: RangeQuery,
895    options: ExportOptions,
896    /// `X-Export-Token` of the first response; `None` before it.
897    token: Option<String>,
898    after: Option<String>,
899    failures: u32,
900    body: Option<Pin<Box<dyn Stream<Item = reqwest::Result<Bytes>> + Send>>>,
901    decoder: Option<zstd::stream::raw::Decoder<'static>>,
902    /// Decompressed bytes after the last newline.
903    rest_bytes: Vec<u8>,
904    ended: bool,
905    /// A cut met with lines still to hand over: raised once they are.
906    cut: Option<Cut>,
907    done: bool,
908}
909
910impl NdjsonExport {
911    async fn next(&mut self) -> Option<Result<Bytes, Error>> {
912        if self.done {
913            return None;
914        }
915        if self.token.is_none() {
916            if let Err(error) = self.first().await {
917                self.done = true;
918                return Some(Err(error));
919            }
920        }
921        loop {
922            match self.step().await {
923                Ok(Some(lines)) => return Some(Ok(lines)),
924                Ok(None) => {
925                    self.done = true;
926                    return None;
927                }
928                Err(cut) => {
929                    self.body = None;
930                    if !cut.resumable() || self.failures >= self.options.max_resumes {
931                        self.done = true;
932                        return Some(Err(cut.into_error()));
933                    }
934                    self.failures += 1;
935                    let delay = self.options.resume_delay.saturating_mul(1 << (self.failures - 1).min(16)).min(Duration::from_secs(30));
936                    tokio::time::sleep(delay).await;
937                }
938            }
939        }
940    }
941
942    /// The first request, never sent again.
943    async fn first(&mut self) -> Result<(), Error> {
944        let exported = self.rest.exported(&self.query, None).await?;
945        let Some(token) = exported.token else {
946            return Err(Error::new(ErrorCode::UpstreamError, "Export without X-Export-Token"));
947        };
948        self.token = Some(token);
949        self.open(exported.response)
950    }
951
952    fn open(&mut self, response: Response) -> Result<(), Error> {
953        self.body = Some(Box::pin(response.bytes_stream()));
954        let decoder = zstd::stream::raw::Decoder::new()
955            .map_err(|e| Error::new(ErrorCode::Configuration, format!("The zstd decoder could not be created: {e}")).with_source(e))?;
956        self.decoder = Some(decoder);
957        self.rest_bytes.clear();
958        self.ended = false;
959        Ok(())
960    }
961
962    /// The next chunk of whole lines; `None` once the end line is read.
963    async fn step(&mut self) -> Result<Option<Bytes>, Cut> {
964        if let Some(cut) = self.cut.take() {
965            return Err(cut);
966        }
967        loop {
968            if self.body.is_none() {
969                let token = self.token.clone().expect("token of the first response");
970                let resume = ExportResume { token, after: self.after.clone() };
971                let exported = self.rest.exported(&self.query, Some(&resume)).await.map_err(Cut::Api)?;
972                self.open(exported.response).map_err(Cut::Api)?;
973            }
974            let body = self.body.as_mut().expect("export body");
975            let mut data = std::mem::take(&mut self.rest_bytes);
976            let cut = match body.next().await {
977                Some(Ok(chunk)) => {
978                    let decoder = self.decoder.as_mut().expect("zstd decoder");
979                    // What was decoded before a cut stays valid.
980                    decompress(decoder, &chunk, &mut data).err().map(|e| {
981                        Cut::Api(
982                            Error::new(ErrorCode::UpstreamError, "Export body could not be decompressed").with_status(502).with_source(e),
983                        )
984                    })
985                }
986                Some(Err(error)) => Some(Cut::Network(error)),
987                None => {
988                    if self.ended && data.is_empty() {
989                        return Ok(None);
990                    }
991                    if self.ended {
992                        return Err(Cut::Api(Error::new(ErrorCode::UpstreamError, "Export lines after its end line")));
993                    }
994                    return Err(Cut::Api(Error::new(ErrorCode::UpstreamError, "Export cut before its end line").with_status(502)));
995                }
996            };
997            let lines = self.lines(data).map_err(Cut::Api)?;
998            match (lines, cut) {
999                (Some(lines), cut) => {
1000                    self.cut = cut;
1001                    return Ok(Some(lines));
1002                }
1003                (None, Some(cut)) => return Err(cut),
1004                (None, None) => {}
1005            }
1006        }
1007    }
1008
1009    /// Cuts `data` (the bytes after the last newline, then the new ones) into whole lines and keeps the rest; `None` when no
1010    /// mutation line is complete yet.
1011    fn lines(&mut self, mut data: Vec<u8>) -> Result<Option<Bytes>, Error> {
1012        let Some(end) = data.iter().rposition(|&b| b == NEWLINE) else {
1013            self.rest_bytes = data;
1014            return Ok(None);
1015        };
1016        if self.ended {
1017            return Err(Error::new(ErrorCode::UpstreamError, "Export lines after its end line"));
1018        }
1019        self.rest_bytes = data.split_off(end + 1);
1020        let last = data[..data.len() - 1].iter().rposition(|&b| b == NEWLINE).map_or(0, |i| i + 1);
1021        if is_end_line(&data[last..]) {
1022            self.ended = true;
1023            data.truncate(last);
1024        }
1025        if data.is_empty() {
1026            return Ok(None);
1027        }
1028        self.after = Some(last_cursor(&data)?);
1029        self.failures = 0;
1030        Ok(Some(Bytes::from(data)))
1031    }
1032}
1033
1034/// Decompresses `chunk` into `out`. A stream cut inside a zstd frame ends without error: its end line tells.
1035fn decompress(decoder: &mut zstd::stream::raw::Decoder<'static>, chunk: &[u8], out: &mut Vec<u8>) -> std::io::Result<()> {
1036    use zstd::stream::raw::{InBuffer, Operation, OutBuffer};
1037    let mut input = InBuffer::around(chunk);
1038    let mut buffer = vec![0u8; 128 * 1024];
1039    loop {
1040        let mut output = OutBuffer::around(&mut buffer[..]);
1041        decoder.run(&mut input, &mut output)?;
1042        let written = output.pos();
1043        out.extend_from_slice(&buffer[..written]);
1044        if input.pos() == chunk.len() && written < buffer.len() {
1045            return Ok(());
1046        }
1047    }
1048}
1049
1050/// The delay of a `Retry-After` header in seconds: a number of seconds, or an HTTP date (IMF-fixdate, `Sun, 06 Nov 1994
1051/// 08:49:37 GMT`) counted from `now` (negative when it is past). `None` when it is neither.
1052pub(crate) fn retry_after_seconds(value: &str, now: SystemTime) -> Option<f64> {
1053    let value = value.trim();
1054    if let Ok(seconds) = value.parse::<f64>() {
1055        return seconds.is_finite().then_some(seconds);
1056    }
1057    let at = http_date(value)?;
1058    let now = match now.duration_since(UNIX_EPOCH) {
1059        Ok(since) => since.as_secs_f64(),
1060        Err(before) => -before.duration().as_secs_f64(),
1061    };
1062    Some(at as f64 - now)
1063}
1064
1065/// Unix seconds of an IMF-fixdate (`Sun, 06 Nov 1994 08:49:37 GMT`).
1066fn http_date(value: &str) -> Option<i64> {
1067    let parts: Vec<&str> = value.split(' ').collect();
1068    let [weekday, day, month, year, time, "GMT"] = parts[..] else { return None };
1069    const DAYS: [&str; 7] = ["Mon,", "Tue,", "Wed,", "Thu,", "Fri,", "Sat,", "Sun,"];
1070    const MONTHS: [&str; 12] = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
1071    let number = |text: &str, digits: usize| {
1072        (text.len() == digits && text.bytes().all(|b| b.is_ascii_digit())).then(|| text.parse::<i64>().ok()).flatten()
1073    };
1074    if !DAYS.contains(&weekday) {
1075        return None;
1076    }
1077    let day = number(day, 2)?;
1078    let month = MONTHS.iter().position(|m| *m == month)? as i64 + 1;
1079    let year = number(year, 4)?;
1080    let [hour, minute, second] = time.split(':').collect::<Vec<_>>()[..] else { return None };
1081    let (hour, minute, second) = (number(hour, 2)?, number(minute, 2)?, number(second, 2)?);
1082    if !(1..=31).contains(&day) || hour > 23 || minute > 59 || second > 60 {
1083        return None;
1084    }
1085    // Days from the civil date (proleptic Gregorian calendar).
1086    let y = if month <= 2 { year - 1 } else { year };
1087    let era = y.div_euclid(400);
1088    let yoe = y - era * 400;
1089    let doy = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1;
1090    let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
1091    let days = era * 146_097 + doe - 719_468;
1092    Some(days * 86_400 + hour * 3_600 + minute * 60 + second)
1093}