use crate::exchange::http::{MAX_ERROR_BODY_DOWNLOAD_BYTES, read_body_capped};
use crate::exchange::lse::error::{LseError, extract_detail};
use reqwest::header::{HeaderMap, HeaderValue};
use std::{env, sync::Arc, time::Duration};
use tokio::{
sync::{Mutex, Semaphore, SemaphorePermit},
time::{Instant, sleep_until},
};
use tracing::debug;
const API_KEY_HEADER: &str = "x-api-key";
pub(crate) const API_KEY_ENV: &str = "LSE_API_KEY";
const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
const READ_TIMEOUT: Duration = Duration::from_secs(30);
const USER_AGENT: &str = concat!("rustrade-data/", env!("CARGO_PKG_VERSION"));
pub(crate) const DEFAULT_PACE: Duration = Duration::from_millis(300);
pub(crate) const DEFAULT_CONCURRENCY: usize = 2;
#[derive(Debug)]
struct RequestGate {
concurrency: usize,
permits: Semaphore,
next_slot: Mutex<Instant>,
}
impl RequestGate {
fn new(concurrency: usize) -> Self {
Self {
concurrency: concurrency.max(1),
permits: Semaphore::new(concurrency.max(1)),
next_slot: Mutex::new(Instant::now()),
}
}
async fn enter(&self, pace: Duration) -> Option<SemaphorePermit<'_>> {
let permit = self.permits.acquire().await.ok();
let slot = {
let mut next_slot = self.next_slot.lock().await;
let slot = (*next_slot).max(Instant::now());
*next_slot = slot + pace;
slot
};
sleep_until(slot).await;
permit
}
}
#[derive(Clone)]
pub(crate) struct LseHttpCore {
http: reqwest::Client,
base_url: String,
pace: Duration,
gate: Arc<RequestGate>,
}
impl std::fmt::Debug for LseHttpCore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("LseHttpCore")
.field("base_url", &self.base_url)
.field("pace", &self.pace)
.field("concurrency", &self.gate.concurrency)
.finish_non_exhaustive()
}
}
impl LseHttpCore {
pub(crate) fn new(api_key: &str, base_url: impl Into<String>) -> Result<Self, LseError> {
let mut headers = HeaderMap::new();
let mut key = HeaderValue::from_str(api_key)
.map_err(|error| LseError::InvalidCredential(format!("invalid API key: {error}")))?;
key.set_sensitive(true);
headers.insert(API_KEY_HEADER, key);
let http = reqwest::Client::builder()
.default_headers(headers)
.user_agent(USER_AGENT)
.timeout(REQUEST_TIMEOUT)
.read_timeout(READ_TIMEOUT)
.redirect(reqwest::redirect::Policy::none())
.build()?;
Ok(Self {
http,
base_url: base_url.into(),
pace: DEFAULT_PACE,
gate: Arc::new(RequestGate::new(DEFAULT_CONCURRENCY)),
})
}
pub(crate) fn from_env(base_url: impl Into<String>) -> Result<Self, LseError> {
let api_key = api_key_from_env().map_err(LseError::EnvVar)?;
Self::new(&api_key, base_url)
}
pub(crate) fn set_base_url(&mut self, base_url: impl Into<String>) {
self.base_url = base_url.into();
}
pub(crate) fn set_client(&mut self, client: reqwest::Client) {
self.http = client;
}
pub(crate) fn set_pace(&mut self, pace: Duration) {
self.pace = pace;
}
pub(crate) fn set_concurrency(&mut self, concurrency: usize) {
self.gate = Arc::new(RequestGate::new(concurrency));
}
pub(crate) async fn enter_gate(&self) -> Option<SemaphorePermit<'_>> {
self.gate.enter(self.pace).await
}
pub(crate) fn base_url(&self) -> &str {
&self.base_url
}
pub(crate) fn http(&self) -> &reqwest::Client {
&self.http
}
pub(crate) fn pace(&self) -> Duration {
self.pace
}
pub(crate) fn concurrency(&self) -> usize {
self.gate.concurrency
}
pub(crate) async fn get_json<T>(
&self,
path: &str,
query: &[(&str, String)],
) -> Result<T, LseError>
where
T: serde::de::DeserializeOwned,
{
let _permit = self.enter_gate().await;
let url = format!("{}/{path}", self.base_url);
let response = self.http.get(&url).query(query).send().await?;
let status = response.status();
if status == reqwest::StatusCode::TOO_MANY_REQUESTS {
let retry_after = parse_retry_after(response.headers());
let _ = read_body_capped(response, MAX_ERROR_BODY_DOWNLOAD_BYTES).await;
return Err(LseError::RateLimited { retry_after });
}
if !status.is_success() {
let body = read_body_capped(response, MAX_ERROR_BODY_DOWNLOAD_BYTES).await?;
return Err(LseError::Api {
status: status.as_u16(),
message: extract_detail(&body),
});
}
let body = response.bytes().await?;
debug!(len = body.len(), path, "lse response received");
serde_json::from_slice(&body).map_err(|error| LseError::Deserialize {
message: format!("{path}: {error}"),
})
}
}
pub(crate) fn api_key_from_env() -> Result<String, String> {
env::var(API_KEY_ENV).map_err(|error| match error {
env::VarError::NotPresent => format!("{API_KEY_ENV} is not set"),
env::VarError::NotUnicode(_) => format!("{API_KEY_ENV} is set but is not valid UTF-8"),
})
}
fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> Option<Duration> {
headers
.get(reqwest::header::RETRY_AFTER)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.trim().parse::<u64>().ok())
.map(Duration::from_secs)
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)] mod tests {
use super::*;
const TEST_BASE: &str = "https://example.invalid/base";
#[test]
fn debug_does_not_leak_the_api_key() {
let core = LseHttpCore::new("super-secret-key", TEST_BASE).unwrap();
assert!(!format!("{core:?}").contains("super-secret-key"));
}
#[test]
fn a_key_carrying_a_control_character_is_a_typed_error_not_a_panic() {
for key in [
"key-with-\nnewline",
"key-with-\rcarriage",
"key\u{0}with-nul",
] {
assert!(
matches!(
LseHttpCore::new(key, TEST_BASE).unwrap_err(),
LseError::InvalidCredential(_)
),
"{key:?} should be rejected"
);
}
}
#[test]
fn from_env_reports_the_variable_name_when_unset() {
temp_env::with_var_unset(API_KEY_ENV, || {
let error = LseHttpCore::from_env(TEST_BASE).unwrap_err();
assert!(matches!(error, LseError::EnvVar(_)));
assert!(error.to_string().contains(API_KEY_ENV));
});
}
#[cfg(unix)]
#[test]
fn from_env_never_reports_a_non_unicode_key() {
use std::{ffi::OsString, os::unix::ffi::OsStringExt};
let mut raw = b"lse-live-super-secret-".to_vec();
raw.push(0xff);
raw.extend_from_slice(b"-tail");
temp_env::with_var(API_KEY_ENV, Some(OsString::from_vec(raw)), || {
let error = LseHttpCore::from_env(TEST_BASE).unwrap_err();
let message = error.to_string();
assert!(matches!(error, LseError::EnvVar(_)));
assert!(message.contains(API_KEY_ENV));
assert!(!message.contains("lse-live-super-secret-"));
});
}
#[test]
fn the_base_url_is_whatever_the_host_client_supplied() {
let core = LseHttpCore::new("k", "https://data-api.example.invalid").unwrap();
assert_eq!(core.base_url(), "https://data-api.example.invalid");
}
#[test]
fn defaults_are_the_measured_provider_limits() {
let core = LseHttpCore::new("k", TEST_BASE).unwrap();
assert_eq!(core.pace(), DEFAULT_PACE);
assert_eq!(core.concurrency(), DEFAULT_CONCURRENCY);
}
#[test]
fn a_zero_concurrency_is_clamped_rather_than_parking_every_request() {
let mut core = LseHttpCore::new("k", TEST_BASE).unwrap();
core.set_concurrency(0);
assert_eq!(core.concurrency(), 1);
}
#[tokio::test(start_paused = true)]
async fn two_cores_built_separately_do_not_share_a_gate() {
let one = LseHttpCore::new("k", TEST_BASE).unwrap();
let two = LseHttpCore::new("k", TEST_BASE).unwrap();
let start = Instant::now();
let _a = one.enter_gate().await;
let _b = two.enter_gate().await;
assert_eq!(start.elapsed(), Duration::ZERO);
}
#[tokio::test(start_paused = true)]
async fn clones_of_one_core_share_its_gate() {
let core = LseHttpCore::new("k", TEST_BASE).unwrap();
let clone = core.clone();
let start = Instant::now();
let _a = core.enter_gate().await;
let _b = clone.enter_gate().await;
assert_eq!(start.elapsed(), DEFAULT_PACE);
}
#[test]
fn api_key_from_env_names_the_variable_when_unset() {
temp_env::with_var_unset(API_KEY_ENV, || {
let message = api_key_from_env().unwrap_err();
assert!(message.contains(API_KEY_ENV), "{message}");
});
}
#[test]
fn api_key_from_env_returns_the_key_when_set() {
temp_env::with_var(API_KEY_ENV, Some("test-key"), || {
assert_eq!(api_key_from_env().unwrap(), "test-key");
});
}
#[cfg(unix)]
#[test]
fn api_key_from_env_never_reports_a_non_unicode_key() {
use std::{ffi::OsString, os::unix::ffi::OsStringExt};
let mut raw = b"lse-live-super-secret-".to_vec();
raw.push(0xff);
raw.extend_from_slice(b"-tail");
temp_env::with_var(API_KEY_ENV, Some(OsString::from_vec(raw)), || {
let message = api_key_from_env().unwrap_err();
assert!(message.contains(API_KEY_ENV), "{message}");
assert!(!message.contains("lse-live-super-secret-"), "{message}");
});
}
#[test]
fn retry_after_reads_delta_seconds() {
let mut headers = HeaderMap::new();
headers.insert(reqwest::header::RETRY_AFTER, HeaderValue::from_static("42"));
assert_eq!(parse_retry_after(&headers), Some(Duration::from_secs(42)));
}
#[test]
fn retry_after_declines_to_guess_at_an_http_date() {
let mut headers = HeaderMap::new();
headers.insert(
reqwest::header::RETRY_AFTER,
HeaderValue::from_static("Wed, 21 Oct 2026 07:28:00 GMT"),
);
assert_eq!(parse_retry_after(&headers), None);
}
#[test]
fn retry_after_is_none_when_absent() {
assert_eq!(parse_retry_after(&HeaderMap::new()), None);
}
#[tokio::test]
async fn the_gate_never_admits_more_callers_than_its_concurrency() {
use std::sync::atomic::{AtomicUsize, Ordering};
let gate = RequestGate::new(2);
let in_flight = AtomicUsize::new(0);
let peak = AtomicUsize::new(0);
futures::future::join_all((0..8).map(|_| async {
let _permit = gate.enter(Duration::ZERO).await;
let entered = in_flight.fetch_add(1, Ordering::SeqCst) + 1;
peak.fetch_max(entered, Ordering::SeqCst);
tokio::task::yield_now().await;
in_flight.fetch_sub(1, Ordering::SeqCst);
}))
.await;
assert_eq!(peak.load(Ordering::SeqCst), 2);
}
#[tokio::test(start_paused = true)]
async fn the_gate_spaces_concurrent_callers_against_one_shared_schedule() {
let pace = Duration::from_millis(300);
let gate = RequestGate::new(8);
let started = Instant::now();
let admitted = futures::future::join_all((0..4).map(|_| async {
let _permit = gate.enter(pace).await;
started.elapsed()
}))
.await;
assert_eq!(admitted, vec![Duration::ZERO, pace, pace * 2, pace * 3]);
}
#[tokio::test(start_paused = true)]
async fn an_idle_gate_accrues_no_credit() {
let pace = Duration::from_millis(300);
let gate = RequestGate::new(8);
drop(gate.enter(pace).await);
tokio::time::sleep(Duration::from_secs(60)).await;
let started = Instant::now();
drop(gate.enter(pace).await);
assert_eq!(started.elapsed(), Duration::ZERO);
}
}