1use std::collections::BTreeMap;
2use std::fmt;
3use std::pin::Pin;
4use std::sync::Arc;
5use std::time::{Duration, SystemTime, UNIX_EPOCH};
6
7use bytes::Bytes;
8use futures_util::{Stream, StreamExt};
9use reqwest::header::{AUTHORIZATION, CONTENT_DISPOSITION, CONTENT_TYPE, RETRY_AFTER};
10use reqwest::{Method, Response, StatusCode};
11use serde::de::DeserializeOwned;
12use serde::{Deserialize, Serialize};
13use serde_json::{Map, Value};
14
15use crate::error::{api_key_problem, misfit, short_type_name, spacing_problem, Error, ErrorCode};
16use crate::protocol::{Item, MaintenanceWindow, Mode, SourceState};
17
18#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
20#[serde(rename_all = "camelCase")]
21pub struct RangeQuery {
22 pub instrument: String,
24 pub indicator: String,
25 #[serde(default, skip_serializing_if = "Option::is_none")]
26 pub params: Option<Value>,
27 pub from: String,
29 pub to: String,
30 #[serde(default, skip_serializing_if = "Option::is_none")]
32 pub mode: Option<Mode>,
33}
34
35impl RangeQuery {
36 pub fn new(instrument: impl Into<String>, indicator: impl Into<String>, from: impl Into<String>, to: impl Into<String>) -> RangeQuery {
37 RangeQuery {
38 instrument: instrument.into(),
39 indicator: indicator.into(),
40 params: None,
41 from: from.into(),
42 to: to.into(),
43 mode: None,
44 }
45 }
46
47 pub fn params(mut self, params: Value) -> RangeQuery {
49 self.params = Some(params);
50 self
51 }
52
53 pub fn mode(mut self, mode: Mode) -> RangeQuery {
54 self.mode = Some(mode);
55 self
56 }
57}
58
59#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
61#[serde(rename_all = "camelCase")]
62pub struct PageQuery {
63 pub instrument: String,
65 pub indicator: String,
66 #[serde(default, skip_serializing_if = "Option::is_none")]
67 pub params: Option<Value>,
68 pub from: String,
70 pub to: String,
71 #[serde(default, skip_serializing_if = "Option::is_none")]
73 pub mode: Option<Mode>,
74 #[serde(default, skip_serializing_if = "Option::is_none")]
76 pub cursor: Option<String>,
77 #[serde(default, skip_serializing_if = "Option::is_none")]
79 pub until_cursor: Option<String>,
80 #[serde(default, skip_serializing_if = "Option::is_none")]
81 pub limit: Option<u32>,
82 #[serde(default, skip_serializing_if = "Option::is_none")]
84 pub snapshot: Option<bool>,
85}
86
87impl PageQuery {
88 pub fn new(instrument: impl Into<String>, indicator: impl Into<String>, from: impl Into<String>, to: impl Into<String>) -> PageQuery {
89 PageQuery::from(RangeQuery::new(instrument, indicator, from, to))
90 }
91
92 pub fn params(mut self, params: Value) -> PageQuery {
94 self.params = Some(params);
95 self
96 }
97
98 pub fn mode(mut self, mode: Mode) -> PageQuery {
99 self.mode = Some(mode);
100 self
101 }
102
103 pub fn cursor(mut self, cursor: impl Into<String>) -> PageQuery {
104 self.cursor = Some(cursor.into());
105 self
106 }
107
108 pub fn until_cursor(mut self, cursor: impl Into<String>) -> PageQuery {
109 self.until_cursor = Some(cursor.into());
110 self
111 }
112
113 pub fn limit(mut self, limit: u32) -> PageQuery {
114 self.limit = Some(limit);
115 self
116 }
117
118 pub fn snapshot(mut self, snapshot: bool) -> PageQuery {
119 self.snapshot = Some(snapshot);
120 self
121 }
122}
123
124impl std::convert::From<RangeQuery> for PageQuery {
125 fn from(q: RangeQuery) -> PageQuery {
126 PageQuery {
127 instrument: q.instrument,
128 indicator: q.indicator,
129 params: q.params,
130 from: q.from,
131 to: q.to,
132 mode: q.mode,
133 cursor: None,
134 until_cursor: None,
135 limit: None,
136 snapshot: None,
137 }
138 }
139}
140
141#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
143pub struct ExportResume {
144 pub token: String,
145 #[serde(default, skip_serializing_if = "Option::is_none")]
146 pub after: Option<String>,
147}
148
149#[derive(Debug, Clone, Copy, PartialEq, Eq)]
150pub struct ExportOptions {
151 pub max_resumes: u32,
153 pub resume_delay: Duration,
155}
156
157impl Default for ExportOptions {
158 fn default() -> Self {
159 ExportOptions { max_resumes: 5, resume_delay: Duration::from_secs(1) }
160 }
161}
162
163#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
164#[serde(rename_all = "lowercase")]
165pub enum MutationKind {
166 Upsert,
167 Remove,
168}
169
170#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
172pub struct Mutation<T = Value> {
173 pub cursor: String,
174 pub t: MutationKind,
175 pub id: String,
176 #[serde(rename = "final")]
177 pub is_final: bool,
178 pub ts: String,
179 #[serde(default = "Option::default", skip_serializing_if = "Option::is_none")]
180 pub data: Option<T>,
181}
182
183#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
185pub struct PageSnapshot<T = Value> {
186 pub cursor: String,
187 pub items: Vec<Item<T>>,
188}
189
190#[derive(Debug, Clone, PartialEq)]
192pub enum HistoryItem<T = Value> {
193 Snapshot(PageSnapshot<T>),
194 Mutation(Mutation<T>),
195}
196
197#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
198#[serde(rename_all = "camelCase")]
199pub struct Page<T = Value> {
200 pub instrument: String,
202 pub tick_size_nanos: String,
204 #[serde(default = "Option::default", skip_serializing_if = "Option::is_none")]
206 pub snapshot: Option<PageSnapshot<T>>,
207 pub items: Vec<Mutation<T>>,
208 pub next: Option<String>,
209 pub compute_units: f64,
210}
211
212#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
213#[serde(rename_all = "camelCase")]
214pub struct Estimate {
215 pub instrument: String,
216 pub tick_size_nanos: String,
217 pub trades: f64,
218 pub compute_units: f64,
219 pub covered_from: Option<String>,
220 pub covered_to: Option<String>,
221 pub missing_days: Vec<String>,
222 pub remaining_compute_units: f64,
223}
224
225#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
227#[serde(rename_all = "camelCase")]
228pub struct Instrument {
229 pub symbol: String,
231 pub alias: Option<String>,
233 pub tick_size_nanos: String,
235 pub status: SourceState,
236 pub updated_at: String,
238}
239
240#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
242#[serde(rename_all = "camelCase")]
243pub struct InstrumentStatus {
244 pub symbol: String,
245 pub alias: Option<String>,
246 pub status: SourceState,
247 pub updated_at: String,
248}
249
250#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
251pub struct EngineStatus {
252 pub state: String,
254}
255
256#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
258pub struct ServiceStatus {
259 pub engine: EngineStatus,
260 pub maintenance: Vec<MaintenanceWindow>,
262 pub instruments: Vec<InstrumentStatus>,
263}
264
265#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
267pub struct CatalogParam {
268 pub description: String,
269 #[serde(rename = "type")]
271 pub kind: String,
272 pub category: String,
274 pub default: Value,
276 #[serde(default, skip_serializing_if = "Option::is_none")]
277 pub minimum: Option<f64>,
278 #[serde(default, skip_serializing_if = "Option::is_none")]
279 pub maximum: Option<f64>,
280 #[serde(default, skip_serializing_if = "Option::is_none")]
281 pub unit: Option<String>,
282 #[serde(default, skip_serializing_if = "Option::is_none")]
284 pub options: Option<Vec<Value>>,
285}
286
287#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
289pub struct CatalogField {
290 pub description: String,
291 #[serde(rename = "type")]
292 pub kind: String,
293 #[serde(default, skip_serializing_if = "Option::is_none")]
294 pub unit: Option<String>,
295 #[serde(default, rename = "enum", skip_serializing_if = "Option::is_none")]
296 pub values: Option<Vec<String>>,
297 #[serde(default, skip_serializing_if = "Option::is_none")]
298 pub optional: Option<bool>,
299}
300
301#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
303#[serde(rename_all = "camelCase")]
304pub struct CatalogObject {
305 pub kind: String,
306 pub description: String,
307 pub id_pattern: String,
308 pub provisional: bool,
309 pub fields: BTreeMap<String, CatalogField>,
310}
311
312#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
313#[serde(rename_all = "camelCase")]
314pub struct CatalogIndicator {
315 pub id: String,
316 pub name: String,
317 pub summary: String,
318 pub description: String,
319 pub origin: String,
321 pub inputs: Vec<String>,
322 pub compute_units_per_trade: f64,
323 pub params: BTreeMap<String, CatalogParam>,
324 #[serde(default)]
327 pub product_defaults: BTreeMap<String, BTreeMap<String, Value>>,
328 pub objects: Vec<CatalogObject>,
329 pub params_schema: Value,
331 pub data_schema: Value,
332 pub example: Value,
334}
335
336#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
338#[serde(rename_all = "camelCase")]
339pub struct Catalog {
340 pub version: u32,
341 pub engine_fingerprint: String,
342 pub indicators: Vec<CatalogIndicator>,
343}
344
345#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
346#[serde(rename_all = "camelCase")]
347pub struct Limits {
348 pub realtime: bool,
350 pub delay_seconds: u64,
351 pub max_connections: u64,
352 pub max_subscriptions: u64,
353 pub max_custom_configs: u64,
354 pub history_days: u64,
356 pub included_history_days: u64,
358 pub compute_budget: f64,
360 pub rest_per_minute: u64,
361 pub export_bytes: f64,
362}
363
364#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
365#[serde(rename_all = "camelCase")]
366pub struct UsageCounters {
367 pub compute_units: f64,
368 pub history_items: f64,
369 pub export_bytes: f64,
370 pub rest_requests: f64,
371}
372
373#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
374#[serde(rename_all = "camelCase")]
375pub struct UsageRemaining {
376 pub compute_units: f64,
377 pub export_bytes: f64,
378}
379
380#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
382#[serde(rename_all = "camelCase")]
383pub struct Usage {
384 pub period: String,
385 pub resets_at: String,
386 pub plan: String,
389 pub limits: Limits,
390 pub used: UsageCounters,
391 pub remaining: UsageRemaining,
392}
393
394#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
396#[serde(rename_all = "camelCase")]
397pub struct StreamToken {
398 pub token: String,
399 pub expires_at: String,
400}
401
402pub type ByteStream = Pin<Box<dyn Stream<Item = Result<Bytes, Error>> + Send>>;
404
405pub struct ExportStream {
407 pub body: ByteStream,
409 pub content_type: String,
410 pub filename: Option<String>,
411 pub token: Option<String>,
413}
414
415impl fmt::Debug for ExportStream {
416 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
417 f.debug_struct("ExportStream")
418 .field("content_type", &self.content_type)
419 .field("filename", &self.filename)
420 .field("token", &self.token)
421 .finish_non_exhaustive()
422 }
423}
424
425const EXPORT_TYPE: &str = "application/zstd";
426const NEWLINE: u8 = b'\n';
427
428fn is_end_line(line: &[u8]) -> bool {
430 line.starts_with(br#"{"t":"#) && serde_json::from_slice::<Value>(line).is_ok_and(|v| v.get("t") == Some(&Value::from("end")))
431}
432
433fn last_cursor(lines: &[u8]) -> Result<String, Error> {
435 let body = &lines[..lines.len() - 1];
436 let start = body.iter().rposition(|&b| b == NEWLINE).map_or(0, |i| i + 1);
437 let head = &body[start..];
438 let cursor =
439 head.strip_prefix(br#"{"cursor":""#).and_then(|rest| rest.iter().position(|&b| b == b'"').map(|end| &rest[..end])).filter(|c| {
440 let parts: Vec<&[u8]> = c.split(|&b| b == b'.').collect();
441 parts.len() == 3 && parts.iter().all(|p| !p.is_empty() && p.iter().all(u8::is_ascii_digit))
442 });
443 match cursor {
444 Some(cursor) => Ok(String::from_utf8_lossy(cursor).into_owned()),
445 None => Err(Error::new(ErrorCode::UpstreamError, "Export line without cursor").with_status(502)),
446 }
447}
448
449fn problem_message(problem: Option<&Map<String, Value>>, fallback: &str) -> String {
453 let detail = problem.and_then(|p| p.get("detail")).and_then(Value::as_str).unwrap_or(fallback);
454 let errors: Vec<String> = match problem.and_then(|p| p.get("errors")) {
455 Some(Value::Array(errors)) => errors
456 .iter()
457 .filter_map(|e| {
458 let message = e.get("message")?.as_str()?;
459 if detail.contains(message) {
460 return None;
461 }
462 Some(match e.get("path").and_then(Value::as_str).filter(|p| !p.is_empty()) {
463 Some(path) => format!("{path} {message}"),
464 None => message.to_string(),
465 })
466 })
467 .collect(),
468 _ => Vec::new(),
469 };
470 if errors.is_empty() {
471 detail.to_string()
472 } else {
473 format!("{detail}: {}", errors.join("; "))
474 }
475}
476
477fn wrapped(error: reqwest::Error) -> Error {
479 if error.is_timeout() {
480 return Error::new(ErrorCode::Aborted, "Request aborted by its timeout").with_source(error);
481 }
482 let mut reason: &dyn std::error::Error = &error;
483 while let Some(inner) = reason.source() {
484 reason = inner;
485 }
486 let message = format!("The API could not be reached or the connection was lost: {reason}");
487 Error::new(ErrorCode::NetworkError, message).with_source(error)
488}
489
490fn read_json<T: DeserializeOwned>(bytes: &[u8]) -> Result<T, Error> {
492 serde_json::from_slice(bytes).map_err(|e| {
493 let message = if e.is_data() {
494 format!("The answer of the API does not fit {}: {e}", short_type_name::<T>())
495 } else {
496 "The API answered with invalid JSON".to_string()
497 };
498 Error::new(ErrorCode::UpstreamError, message).with_source(e)
499 })
500}
501
502impl Page<Value> {
503 fn decode<T: DeserializeOwned>(self, indicator: &str) -> Result<Page<T>, Error> {
505 let data = |value: Value| serde_json::from_value::<T>(value).map_err(|e| misfit::<T>(indicator, e));
506 let snapshot = match self.snapshot {
507 Some(PageSnapshot { cursor, items }) => Some(PageSnapshot {
508 cursor,
509 items: items
510 .into_iter()
511 .map(|i| Ok(Item { id: i.id, is_final: i.is_final, ts: i.ts, data: data(i.data)? }))
512 .collect::<Result<_, Error>>()?,
513 }),
514 None => None,
515 };
516 let items = self
517 .items
518 .into_iter()
519 .map(|m| {
520 let Mutation { cursor, t, id, is_final, ts, data: value } = m;
521 Ok(Mutation { cursor, t, id, is_final, ts, data: value.map(data).transpose()? })
522 })
523 .collect::<Result<_, Error>>()?;
524 Ok(Page {
525 instrument: self.instrument,
526 tick_size_nanos: self.tick_size_nanos,
527 snapshot,
528 items,
529 next: self.next,
530 compute_units: self.compute_units,
531 })
532 }
533}
534
535#[derive(Debug, Clone)]
537pub struct RestBuilder {
538 base_url: String,
539 api_key: Option<String>,
540 client: Option<reqwest::Client>,
541 rate_limit_retries: u32,
542}
543
544impl RestBuilder {
545 pub fn api_key(mut self, api_key: impl Into<String>) -> RestBuilder {
546 self.api_key = Some(api_key.into());
547 self
548 }
549
550 pub fn rate_limit_retries(mut self, retries: u32) -> RestBuilder {
555 self.rate_limit_retries = retries;
556 self
557 }
558
559 pub fn client(mut self, client: reqwest::Client) -> RestBuilder {
561 self.client = Some(client);
562 self
563 }
564
565 pub fn build(self) -> Result<FathomChartsRest, Error> {
566 if let Some(problem) = api_key_problem(self.api_key.as_deref()) {
567 return Err(Error::new(ErrorCode::Configuration, problem));
568 }
569 if let Some(problem) = self.api_key.as_deref().and_then(|key| spacing_problem("api_key", key)) {
570 return Err(Error::new(ErrorCode::Configuration, problem));
571 }
572 let url = reqwest::Url::parse(&self.base_url).ok();
573 if !url.is_some_and(|u| matches!(u.scheme(), "http" | "https") && u.has_host()) {
574 return Err(Error::new(
575 ErrorCode::Configuration,
576 format!("base_url must be an absolute http:// or https:// URL, not {:?}", self.base_url),
577 ));
578 }
579 let client = match self.client {
580 Some(client) => client,
581 None => reqwest::Client::builder()
582 .build()
583 .map_err(|e| Error::new(ErrorCode::Configuration, format!("The HTTP client could not be created: {e}")).with_source(e))?,
584 };
585 Ok(FathomChartsRest {
586 inner: Arc::new(RestInner {
587 base: self.base_url.trim_end_matches('/').to_string(),
588 api_key: self.api_key.unwrap_or_default(),
589 client,
590 retries: self.rate_limit_retries,
591 }),
592 })
593 }
594}
595
596#[derive(Debug)]
597struct RestInner {
598 base: String,
599 api_key: String,
600 client: reqwest::Client,
601 retries: u32,
602}
603
604#[derive(Debug, Clone)]
608pub struct FathomChartsRest {
609 inner: Arc<RestInner>,
610}
611
612struct Exported {
614 response: Response,
615 content_type: String,
616 filename: Option<String>,
617 token: Option<String>,
618}
619
620#[derive(Serialize)]
621struct ExportBody<'a> {
622 #[serde(flatten)]
623 query: &'a RangeQuery,
624 #[serde(skip_serializing_if = "Option::is_none")]
625 resume: Option<&'a ExportResume>,
626}
627
628impl FathomChartsRest {
629 pub fn new(base_url: impl Into<String>, api_key: impl Into<String>) -> Result<FathomChartsRest, Error> {
631 FathomChartsRest::builder(base_url).api_key(api_key).build()
632 }
633
634 pub fn builder(base_url: impl Into<String>) -> RestBuilder {
635 RestBuilder { base_url: base_url.into(), api_key: None, client: None, rate_limit_retries: 3 }
636 }
637
638 async fn request<B: Serialize + ?Sized>(&self, method: Method, path: &str, body: Option<&B>) -> Result<Response, Error> {
639 let inner = &self.inner;
640 let url = format!("{}{}", inner.base, path);
641 let mut attempt = 0u32;
642 loop {
643 let mut request = inner.client.request(method.clone(), &url).header(AUTHORIZATION, format!("Bearer {}", inner.api_key));
644 if let Some(body) = body {
645 request = request.json(body);
646 }
647 let response = match request.send().await {
648 Ok(response) => response,
649 Err(error) => {
650 if method != Method::GET || attempt >= inner.retries {
652 return Err(wrapped(error));
653 }
654 tokio::time::sleep(Duration::from_millis(500 << attempt.min(16))).await;
655 attempt += 1;
656 continue;
657 }
658 };
659 let status = response.status();
660 if status.is_success() {
661 return Ok(response);
662 }
663 let retry_after =
664 response.headers().get(RETRY_AFTER).and_then(|v| v.to_str().ok()).and_then(|v| retry_after_seconds(v, SystemTime::now()));
665 let text = response.bytes().await.map_err(wrapped)?;
666 let problem = match serde_json::from_slice::<Value>(&text) {
667 Ok(Value::Object(problem)) => Some(problem),
668 _ => None,
669 };
670 let code = match problem.as_ref().and_then(|p| p.get("code")).and_then(Value::as_str) {
671 Some(code) => code.to_string(),
672 None => format!("HTTP_{}", status.as_u16()),
673 };
674 let wait = |seconds: f64| Duration::from_secs_f64(seconds.clamp(0.0, 60.0));
675 if status == StatusCode::TOO_MANY_REQUESTS && (code == "RATE_LIMITED" || code == "HTTP_429") && attempt < inner.retries {
676 tokio::time::sleep(wait(retry_after.unwrap_or(2f64.powi(attempt as i32)))).await;
677 attempt += 1;
678 continue;
679 }
680 if status == StatusCode::SERVICE_UNAVAILABLE && retry_after.is_some() && attempt < inner.retries {
682 tokio::time::sleep(wait(retry_after.unwrap_or_default())).await;
683 attempt += 1;
684 continue;
685 }
686 if status == StatusCode::BAD_GATEWAY && method == Method::GET && attempt < inner.retries {
687 tokio::time::sleep(Duration::from_millis(500 << attempt.min(16))).await;
688 attempt += 1;
689 continue;
690 }
691 let fallback = match status.canonical_reason() {
692 Some(reason) => reason.to_string(),
693 None => format!("HTTP {}", status.as_u16()),
694 };
695 let message = problem_message(problem.as_ref(), &fallback);
696 return Err(Error::new(ErrorCode::parse(&code), message)
697 .with_status(status.as_u16())
698 .with_problem(problem)
699 .with_retry_after(retry_after.map(|s| Duration::from_secs_f64(s.max(0.0)))));
700 }
701 }
702
703 async fn json<T: DeserializeOwned, B: Serialize + ?Sized>(&self, method: Method, path: &str, body: Option<&B>) -> Result<T, Error> {
704 let response = self.request(method, path, body).await?;
705 let bytes = response.bytes().await.map_err(wrapped)?;
706 read_json(&bytes)
707 }
708
709 async fn get<T: DeserializeOwned>(&self, path: &str) -> Result<T, Error> {
710 self.json::<T, ()>(Method::GET, path, None).await
711 }
712
713 pub async fn indicators(&self) -> Result<Catalog, Error> {
715 self.get("/v1/indicators").await
716 }
717
718 pub async fn instruments(&self) -> Result<Vec<Instrument>, Error> {
719 #[derive(Deserialize)]
720 struct Instruments {
721 instruments: Vec<Instrument>,
722 }
723 Ok(self.get::<Instruments>("/v1/instruments").await?.instruments)
724 }
725
726 pub async fn status(&self) -> Result<ServiceStatus, Error> {
727 self.get("/v1/status").await
728 }
729
730 pub async fn usage(&self) -> Result<Usage, Error> {
731 self.get("/v1/usage").await
732 }
733
734 pub async fn stream_token(&self) -> Result<StreamToken, Error> {
736 self.json::<StreamToken, ()>(Method::POST, "/v1/stream-tokens", None).await
737 }
738
739 pub async fn estimate(&self, query: &PageQuery) -> Result<Estimate, Error> {
742 check_query(&query.instrument, &query.indicator, &query.from, &query.to)?;
743 self.json(Method::POST, "/v1/indicator-queries/estimate", Some(query)).await
744 }
745
746 pub async fn page<T: DeserializeOwned>(&self, query: &PageQuery) -> Result<Page<T>, Error> {
749 check_query(&query.instrument, &query.indicator, &query.from, &query.to)?;
750 let page: Page<Value> = self.json(Method::POST, "/v1/indicator-queries", Some(query)).await?;
751 page.decode(&query.indicator)
752 }
753
754 pub fn pages<T: DeserializeOwned + Send + 'static>(
757 &self,
758 query: PageQuery,
759 ) -> impl Stream<Item = Result<Page<T>, Error>> + Send + 'static {
760 let rest = self.clone();
761 futures_util::stream::unfold(Some(query), move |query| {
762 let rest = rest.clone();
763 async move {
764 let query = query?;
765 match rest.page::<T>(&query).await {
766 Ok(page) => {
767 let next = page.next.clone().map(|cursor| PageQuery { cursor: Some(cursor), ..query });
768 Some((Ok(page), next))
769 }
770 Err(error) => Some((Err(error), None)),
771 }
772 }
773 })
774 }
775
776 pub fn history<T: DeserializeOwned + Send + 'static>(
779 &self,
780 query: PageQuery,
781 ) -> impl Stream<Item = Result<HistoryItem<T>, Error>> + Send + 'static {
782 self.pages::<T>(query).flat_map(|page| {
783 let items: Vec<Result<HistoryItem<T>, Error>> = match page {
784 Ok(page) => page
785 .snapshot
786 .map(HistoryItem::Snapshot)
787 .into_iter()
788 .chain(page.items.into_iter().map(HistoryItem::Mutation))
789 .map(Ok)
790 .collect(),
791 Err(error) => vec![Err(error)],
792 };
793 futures_util::stream::iter(items)
794 })
795 }
796
797 pub async fn export(&self, query: &RangeQuery, resume: Option<&ExportResume>) -> Result<ExportStream, Error> {
802 let exported = self.exported(query, resume).await?;
803 let body = exported.response.bytes_stream().map(|chunk| chunk.map_err(wrapped));
804 Ok(ExportStream { body: Box::pin(body), content_type: exported.content_type, filename: exported.filename, token: exported.token })
805 }
806
807 async fn exported(&self, query: &RangeQuery, resume: Option<&ExportResume>) -> Result<Exported, Error> {
808 check_query(&query.instrument, &query.indicator, &query.from, &query.to)?;
809 let response = self.request(Method::POST, "/v1/exports", Some(&ExportBody { query, resume })).await?;
810 let header = |name: &str| response.headers().get(name).and_then(|v| v.to_str().ok()).map(str::to_string);
811 let content_type = header(CONTENT_TYPE.as_str()).unwrap_or_default();
812 if content_type.split(';').next().unwrap_or_default().trim().to_ascii_lowercase() != EXPORT_TYPE {
813 let shown = if content_type.is_empty() { "no content type" } else { &content_type };
814 return Err(Error::new(ErrorCode::UpstreamError, format!("Export answered as {shown}, not {EXPORT_TYPE}")));
815 }
816 let filename = header(CONTENT_DISPOSITION.as_str()).and_then(|d| {
817 let start = d.find("filename=\"")? + "filename=\"".len();
818 let name = &d[start..][..d[start..].find('"')?];
819 (!name.is_empty()).then(|| name.to_string())
820 });
821 let token = header("x-export-token");
822 Ok(Exported { response, content_type, filename, token })
823 }
824
825 pub fn export_ndjson(&self, query: RangeQuery, options: ExportOptions) -> impl Stream<Item = Result<Bytes, Error>> + Send + 'static {
831 let export = NdjsonExport {
832 rest: self.clone(),
833 query,
834 options,
835 token: None,
836 after: None,
837 failures: 0,
838 body: None,
839 decoder: None,
840 rest_bytes: Vec::new(),
841 ended: false,
842 cut: None,
843 done: false,
844 };
845 futures_util::stream::unfold(export, |mut export| async move {
846 let chunk = export.next().await?;
847 Some((chunk, export))
848 })
849 }
850}
851
852fn check_query(instrument: &str, indicator: &str, from: &str, to: &str) -> Result<(), Error> {
854 for (value, name, what) in [
855 (instrument, "instrument", "a raw symbol or an alias, e.g. NQZ6"),
856 (indicator, "indicator", "an indicator id of the catalog, e.g. big-trades"),
857 (from, "from", "Unix nanoseconds as a decimal string"),
858 (to, "to", "Unix nanoseconds as a decimal string"),
859 ] {
860 if value.trim().is_empty() {
861 return Err(Error::new(ErrorCode::Configuration, format!("{name} of the query is missing: give {what}")));
862 }
863 }
864 Ok(())
865}
866
867enum Cut {
869 Network(reqwest::Error),
871 Api(Error),
873}
874
875impl Cut {
876 fn resumable(&self) -> bool {
878 match self {
879 Cut::Network(_) => true,
880 Cut::Api(error) => *error.code() == ErrorCode::NetworkError || error.status().unwrap_or(0) >= 500,
881 }
882 }
883
884 fn into_error(self) -> Error {
885 match self {
886 Cut::Network(error) => wrapped(error),
887 Cut::Api(error) => error,
888 }
889 }
890}
891
892struct NdjsonExport {
893 rest: FathomChartsRest,
894 query: RangeQuery,
895 options: ExportOptions,
896 token: Option<String>,
898 after: Option<String>,
899 failures: u32,
900 body: Option<Pin<Box<dyn Stream<Item = reqwest::Result<Bytes>> + Send>>>,
901 decoder: Option<zstd::stream::raw::Decoder<'static>>,
902 rest_bytes: Vec<u8>,
904 ended: bool,
905 cut: Option<Cut>,
907 done: bool,
908}
909
910impl NdjsonExport {
911 async fn next(&mut self) -> Option<Result<Bytes, Error>> {
912 if self.done {
913 return None;
914 }
915 if self.token.is_none() {
916 if let Err(error) = self.first().await {
917 self.done = true;
918 return Some(Err(error));
919 }
920 }
921 loop {
922 match self.step().await {
923 Ok(Some(lines)) => return Some(Ok(lines)),
924 Ok(None) => {
925 self.done = true;
926 return None;
927 }
928 Err(cut) => {
929 self.body = None;
930 if !cut.resumable() || self.failures >= self.options.max_resumes {
931 self.done = true;
932 return Some(Err(cut.into_error()));
933 }
934 self.failures += 1;
935 let delay = self.options.resume_delay.saturating_mul(1 << (self.failures - 1).min(16)).min(Duration::from_secs(30));
936 tokio::time::sleep(delay).await;
937 }
938 }
939 }
940 }
941
942 async fn first(&mut self) -> Result<(), Error> {
944 let exported = self.rest.exported(&self.query, None).await?;
945 let Some(token) = exported.token else {
946 return Err(Error::new(ErrorCode::UpstreamError, "Export without X-Export-Token"));
947 };
948 self.token = Some(token);
949 self.open(exported.response)
950 }
951
952 fn open(&mut self, response: Response) -> Result<(), Error> {
953 self.body = Some(Box::pin(response.bytes_stream()));
954 let decoder = zstd::stream::raw::Decoder::new()
955 .map_err(|e| Error::new(ErrorCode::Configuration, format!("The zstd decoder could not be created: {e}")).with_source(e))?;
956 self.decoder = Some(decoder);
957 self.rest_bytes.clear();
958 self.ended = false;
959 Ok(())
960 }
961
962 async fn step(&mut self) -> Result<Option<Bytes>, Cut> {
964 if let Some(cut) = self.cut.take() {
965 return Err(cut);
966 }
967 loop {
968 if self.body.is_none() {
969 let token = self.token.clone().expect("token of the first response");
970 let resume = ExportResume { token, after: self.after.clone() };
971 let exported = self.rest.exported(&self.query, Some(&resume)).await.map_err(Cut::Api)?;
972 self.open(exported.response).map_err(Cut::Api)?;
973 }
974 let body = self.body.as_mut().expect("export body");
975 let mut data = std::mem::take(&mut self.rest_bytes);
976 let cut = match body.next().await {
977 Some(Ok(chunk)) => {
978 let decoder = self.decoder.as_mut().expect("zstd decoder");
979 decompress(decoder, &chunk, &mut data).err().map(|e| {
981 Cut::Api(
982 Error::new(ErrorCode::UpstreamError, "Export body could not be decompressed").with_status(502).with_source(e),
983 )
984 })
985 }
986 Some(Err(error)) => Some(Cut::Network(error)),
987 None => {
988 if self.ended && data.is_empty() {
989 return Ok(None);
990 }
991 if self.ended {
992 return Err(Cut::Api(Error::new(ErrorCode::UpstreamError, "Export lines after its end line")));
993 }
994 return Err(Cut::Api(Error::new(ErrorCode::UpstreamError, "Export cut before its end line").with_status(502)));
995 }
996 };
997 let lines = self.lines(data).map_err(Cut::Api)?;
998 match (lines, cut) {
999 (Some(lines), cut) => {
1000 self.cut = cut;
1001 return Ok(Some(lines));
1002 }
1003 (None, Some(cut)) => return Err(cut),
1004 (None, None) => {}
1005 }
1006 }
1007 }
1008
1009 fn lines(&mut self, mut data: Vec<u8>) -> Result<Option<Bytes>, Error> {
1012 let Some(end) = data.iter().rposition(|&b| b == NEWLINE) else {
1013 self.rest_bytes = data;
1014 return Ok(None);
1015 };
1016 if self.ended {
1017 return Err(Error::new(ErrorCode::UpstreamError, "Export lines after its end line"));
1018 }
1019 self.rest_bytes = data.split_off(end + 1);
1020 let last = data[..data.len() - 1].iter().rposition(|&b| b == NEWLINE).map_or(0, |i| i + 1);
1021 if is_end_line(&data[last..]) {
1022 self.ended = true;
1023 data.truncate(last);
1024 }
1025 if data.is_empty() {
1026 return Ok(None);
1027 }
1028 self.after = Some(last_cursor(&data)?);
1029 self.failures = 0;
1030 Ok(Some(Bytes::from(data)))
1031 }
1032}
1033
1034fn decompress(decoder: &mut zstd::stream::raw::Decoder<'static>, chunk: &[u8], out: &mut Vec<u8>) -> std::io::Result<()> {
1036 use zstd::stream::raw::{InBuffer, Operation, OutBuffer};
1037 let mut input = InBuffer::around(chunk);
1038 let mut buffer = vec![0u8; 128 * 1024];
1039 loop {
1040 let mut output = OutBuffer::around(&mut buffer[..]);
1041 decoder.run(&mut input, &mut output)?;
1042 let written = output.pos();
1043 out.extend_from_slice(&buffer[..written]);
1044 if input.pos() == chunk.len() && written < buffer.len() {
1045 return Ok(());
1046 }
1047 }
1048}
1049
1050pub(crate) fn retry_after_seconds(value: &str, now: SystemTime) -> Option<f64> {
1053 let value = value.trim();
1054 if let Ok(seconds) = value.parse::<f64>() {
1055 return seconds.is_finite().then_some(seconds);
1056 }
1057 let at = http_date(value)?;
1058 let now = match now.duration_since(UNIX_EPOCH) {
1059 Ok(since) => since.as_secs_f64(),
1060 Err(before) => -before.duration().as_secs_f64(),
1061 };
1062 Some(at as f64 - now)
1063}
1064
1065fn http_date(value: &str) -> Option<i64> {
1067 let parts: Vec<&str> = value.split(' ').collect();
1068 let [weekday, day, month, year, time, "GMT"] = parts[..] else { return None };
1069 const DAYS: [&str; 7] = ["Mon,", "Tue,", "Wed,", "Thu,", "Fri,", "Sat,", "Sun,"];
1070 const MONTHS: [&str; 12] = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
1071 let number = |text: &str, digits: usize| {
1072 (text.len() == digits && text.bytes().all(|b| b.is_ascii_digit())).then(|| text.parse::<i64>().ok()).flatten()
1073 };
1074 if !DAYS.contains(&weekday) {
1075 return None;
1076 }
1077 let day = number(day, 2)?;
1078 let month = MONTHS.iter().position(|m| *m == month)? as i64 + 1;
1079 let year = number(year, 4)?;
1080 let [hour, minute, second] = time.split(':').collect::<Vec<_>>()[..] else { return None };
1081 let (hour, minute, second) = (number(hour, 2)?, number(minute, 2)?, number(second, 2)?);
1082 if !(1..=31).contains(&day) || hour > 23 || minute > 59 || second > 60 {
1083 return None;
1084 }
1085 let y = if month <= 2 { year - 1 } else { year };
1087 let era = y.div_euclid(400);
1088 let yoe = y - era * 400;
1089 let doy = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1;
1090 let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
1091 let days = era * 146_097 + doe - 719_468;
1092 Some(days * 86_400 + hour * 3_600 + minute * 60 + second)
1093}