use crate::exchange::lse::PROVIDER_TIMESTAMP_FORMAT;
use crate::exchange::lse::error::LseError;
use crate::exchange::lse::market::candle_interval_str;
use crate::exchange::lse::vault::LseVaultClient;
use crate::subscription::candle::{
Candle, CandleInterval, IntervalStep, close_time_from_open, open_time_from_close,
};
use async_stream::try_stream;
use chrono::{DateTime, Duration as TimeDelta, NaiveDateTime, Utc};
use futures::{Stream, StreamExt};
use rust_decimal::Decimal;
use serde::{Deserialize, de::DeserializeOwned};
use tracing::debug;
const CURSOR_FORMAT: &str = "%Y-%m-%d %H:%M:%S";
const CURSOR_STEP_SECS: i64 = 1;
#[derive(Debug, Deserialize)]
struct LseCandleRow {
ts: String,
open: Decimal,
high: Decimal,
low: Decimal,
close: Decimal,
#[serde(default)]
volume: Option<Decimal>,
}
pub(super) trait VaultCandleRow: DeserializeOwned {
fn open_time(&self) -> Result<DateTime<Utc>, LseError>;
}
pub(super) fn parse_candle_open(label: &str) -> Result<DateTime<Utc>, LseError> {
NaiveDateTime::parse_from_str(label, PROVIDER_TIMESTAMP_FORMAT)
.map(|naive| naive.and_utc())
.map_err(|error| LseError::Deserialize {
message: format!("invalid candle timestamp {label:?}: {error}"),
})
}
impl VaultCandleRow for LseCandleRow {
fn open_time(&self) -> Result<DateTime<Utc>, LseError> {
parse_candle_open(&self.ts)
}
}
impl LseCandleRow {
fn into_candle(self, close_time: DateTime<Utc>) -> Candle {
Candle {
close_time,
open: self.open,
high: self.high,
low: self.low,
close: self.close,
volume: self.volume,
trade_count: None,
}
}
}
impl LseVaultClient {
#[must_use = "fetch_candles returns a lazy Stream that does nothing unless polled"]
pub fn fetch_candles<'a>(
&'a self,
symbol: &'a str,
interval: CandleInterval,
start: DateTime<Utc>,
end: DateTime<Utc>,
) -> impl Stream<Item = Result<Candle, LseError>> + 'a {
try_stream! {
let timeframe = candle_interval_str(interval)
.ok_or(LseError::UnsupportedInterval { interval })?;
let rows = self.candle_rows::<LseCandleRow>(
"candles",
symbol,
vec![
("symbol", symbol.to_owned()),
("timeframe", timeframe.to_owned()),
],
interval,
start,
end,
);
futures::pin_mut!(rows);
while let Some(row) = rows.next().await {
let (row, close_time) = row?;
yield row.into_candle(close_time);
}
}
}
pub(super) fn candle_rows<'a, R>(
&'a self,
path: &'static str,
symbol: &'a str,
selector: Vec<(&'static str, String)>,
interval: CandleInterval,
start: DateTime<Utc>,
end: DateTime<Utc>,
) -> impl Stream<Item = Result<(R, DateTime<Utc>), LseError>> + 'a
where
R: VaultCandleRow + 'a,
{
try_stream! {
if start > end {
Err(LseError::InvalidInput {
message: format!("start ({start}) must not be after end ({end})"),
})?;
}
let step = interval.to_step();
let request_start = open_time_from_close(start, step).unwrap_or(start);
let request_end_open = open_time_from_close(end, step).unwrap_or(end);
let range_end = request_end_open
.checked_add_signed(TimeDelta::seconds(CURSOR_STEP_SECS))
.unwrap_or(request_end_open);
let mut cursor = request_start;
let mut page = 0usize;
let mut previous_open: Option<DateTime<Utc>> = None;
loop {
let mut query = selector.clone();
query.extend([
("start", cursor.format(CURSOR_FORMAT).to_string()),
("end", range_end.format(CURSOR_FORMAT).to_string()),
("limit", self.page_limit().to_string()),
]);
let rows: Vec<R> = self.get_json(path, &query).await?;
page += 1;
debug!(path, symbol, %cursor, rows = rows.len(), page, "vault candle page received");
if rows.is_empty() {
break;
}
let mut max_open: Option<DateTime<Utc>> = None;
let mut reached_end = false;
let mut opens = Vec::with_capacity(rows.len());
for row in &rows {
let open = row.open_time()?;
if let Some(previous) = previous_open {
if open < previous {
Err(LseError::NonMonotonicCandlePage {
symbol: symbol.to_owned(),
page,
previous_open: previous,
open,
})?;
}
if let IntervalStep::Fixed(width) = step {
let actual = open - previous;
if actual < width {
Err(LseError::UnexpectedCandleResolution {
symbol: symbol.to_owned(),
interval,
previous_open: previous,
open,
actual,
})?;
}
}
}
previous_open = Some(open);
opens.push(open);
}
for (row, open) in rows.into_iter().zip(opens) {
let close_time = close_time_from_open(open, step)
.ok_or(LseError::TimestampOverflow { open, interval })?;
max_open = Some(max_open.map_or(open, |seen| seen.max(open)));
if close_time > end {
reached_end = true;
continue;
}
if close_time < start {
continue;
}
yield (row, close_time);
}
if reached_end {
Err(LseError::UnexpectedCandleRange {
symbol: symbol.to_owned(),
cursor,
page,
end,
})?;
}
let Some(last_open) = max_open else {
break;
};
let next_cursor = last_open
.checked_add_signed(TimeDelta::seconds(CURSOR_STEP_SECS))
.ok_or(LseError::CursorOverflow { last_open })?;
if next_cursor <= cursor {
Err(LseError::Api {
status: 200,
message: format!(
"page {page} did not advance past {cursor}: the range parameters \
appear to have been ignored"
),
})?;
}
cursor = next_cursor;
if cursor >= range_end {
break;
}
}
}
}
pub async fn collect_candles(
&self,
symbol: &str,
interval: CandleInterval,
start: DateTime<Utc>,
end: DateTime<Utc>,
) -> Result<Vec<Candle>, LseError> {
let stream = self.fetch_candles(symbol, interval, start, end);
futures::pin_mut!(stream);
let mut candles = Vec::new();
while let Some(candle) = stream.next().await {
candles.push(candle?);
}
Ok(candles)
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)] mod tests {
use super::*;
use rust_decimal_macros::dec;
fn row(ts: &str, volume: Option<Decimal>) -> LseCandleRow {
LseCandleRow {
ts: ts.to_owned(),
open: dec!(1),
high: dec!(2),
low: dec!(0.5),
close: dec!(1.5),
volume,
}
}
#[test]
fn an_equity_row_deserializes_float_prices_and_an_integer_volume() {
let json = r#"{"ts":"2024-01-02 00:00:00.000000","symbol":"TEST","open":10.0,
"high":10.1234567890123,"low":10.0,"close":10.0,"volume":1000000}"#;
let row: LseCandleRow = serde_json::from_str(json).unwrap();
assert_eq!(row.open, dec!(10.0));
assert_eq!(row.high, dec!(10.1234567890123));
assert_eq!(row.volume, Some(dec!(1000000)));
}
#[test]
fn an_fx_row_omitting_volume_deserializes_as_none() {
let json = r#"{"ts":"2024-01-02 00:00:00.000000","symbol":"AAA/BBB","open":2.0,
"high":2.0,"low":2.0,"close":2.0}"#;
let row: LseCandleRow = serde_json::from_str(json).unwrap();
assert_eq!(row.volume, None);
}
#[test]
fn timestamps_parse_as_utc() {
let open = row("2024-01-02 09:09:00.000000", None).open_time().unwrap();
assert_eq!(
open,
"2024-01-02T09:09:00Z".parse::<DateTime<Utc>>().unwrap()
);
}
#[test]
fn a_timestamp_without_a_fractional_part_still_parses() {
assert!(row("2024-01-02 09:09:00", None).open_time().is_ok());
}
#[test]
fn a_malformed_timestamp_is_a_typed_error() {
let error = row("not-a-timestamp", None).open_time().unwrap_err();
assert!(matches!(error, LseError::Deserialize { .. }));
assert!(error.to_string().contains("not-a-timestamp"));
}
#[test]
fn trade_count_is_always_none() {
let candle = row("2024-01-02 09:09:00.000000", Some(dec!(5))).into_candle(Utc::now());
assert_eq!(candle.trade_count, None);
}
#[test]
fn a_daily_bars_close_is_the_next_midnight_not_its_own_label() {
let open = row("2024-01-02 00:00:00.000000", None).open_time().unwrap();
let close = close_time_from_open(open, CandleInterval::Day1.to_step()).unwrap();
assert_eq!(
close,
"2024-01-03T00:00:00Z".parse::<DateTime<Utc>>().unwrap()
);
}
#[test]
fn a_monthly_bars_close_uses_calendar_arithmetic() {
let open = row("2024-02-01 00:00:00.000000", None).open_time().unwrap();
let close = close_time_from_open(open, CandleInterval::Month1.to_step()).unwrap();
assert_eq!(
close,
"2024-03-01T00:00:00Z".parse::<DateTime<Utc>>().unwrap()
);
}
}