use crate::{
AccountEventKind, UnindexedAccountEvent,
error::{
ApiError, BorrowReject, BorrowRejectReason, ConnectivityError, OrderError, OrderField,
PrecisionLimit, PrecisionViolation, StreamTerminationReason, UnindexedClientError,
UnindexedOrderError,
},
fill_recovery::{FillRecoveryFailure, FillRecoveryGap, FillRecoveryScope},
order::{
Order, OrderKey, OrderKind, TimeInForce, TrailingOffsetType, UnindexedInactiveOrder,
UnindexedOrderKey,
id::{ClientOrderId, OrderId, StrategyId, VenueOrderId},
request::UnindexedOrderResponseCancel,
state::{
Cancelled, Expired, Filled, InactiveOrderState, Open, OrderState, UnindexedOrderState,
},
},
trade::{AssetFees, Trade, TradeId},
};
pub(crate) use crate::client::dedup::{
SharedDedupCache, dedup_key_from_event, is_duplicate, new_dedup_cache,
};
use crate::client::order_recovery::{
GAP_RETRY_BASE_SECS, GapFailure, MAX_GAP_RETRIES, PendingFills,
};
use crate::client::reconnect_budget::ReconnectBudget;
use binance_sdk::common::{
errors::{ConnectorError, WebsocketError},
models::{Interval, ParamBuildError, RateLimitType, RestApiResponse, WebsocketApiRateLimit},
websocket::Subscription,
};
use chrono::{DateTime, TimeZone, Utc};
use rust_decimal::Decimal;
use rustrade_instrument::{
Side, asset::name::AssetNameExchange, exchange::ExchangeId,
instrument::name::InstrumentNameExchange,
};
use serde::{
Deserialize, Deserializer,
de::{self, Visitor},
};
use smol_str::format_smolstr;
use std::{
fmt,
pin::Pin,
str::FromStr,
sync::{
Arc,
atomic::{AtomicU32, AtomicU64, Ordering},
},
task::{Context, Poll},
time::Duration,
};
use tokio::sync::mpsc;
use tracing::{debug, error, info, trace, warn};
pub(crate) struct AbortOnDropStream<S> {
inner: S,
handle: tokio::task::JoinHandle<()>,
}
impl<S> AbortOnDropStream<S> {
pub(crate) fn new(inner: S, handle: tokio::task::JoinHandle<()>) -> Self {
Self { inner, handle }
}
}
impl<S: futures::Stream + Unpin> futures::Stream for AbortOnDropStream<S> {
type Item = S::Item;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
Pin::new(&mut self.inner).poll_next(cx)
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.inner.size_hint()
}
}
impl<S> Drop for AbortOnDropStream<S> {
fn drop(&mut self) {
self.handle.abort();
}
}
pub(crate) trait BinanceWsConnection: Send + Sync + 'static {
fn close(&self) -> impl Future<Output = anyhow::Result<()>> + Send;
}
impl BinanceWsConnection for binance_sdk::spot::websocket_api::WebsocketApi {
fn close(&self) -> impl Future<Output = anyhow::Result<()>> + Send {
self.disconnect()
}
}
impl BinanceWsConnection for Arc<binance_sdk::common::websocket::WebsocketApi> {
async fn close(&self) -> anyhow::Result<()> {
self.disconnect().await.map_err(anyhow::Error::msg)
}
}
pub(crate) struct DisconnectOnDrop<W: BinanceWsConnection> {
ws: Option<W>,
label: &'static str,
}
impl<W: BinanceWsConnection> DisconnectOnDrop<W> {
pub(crate) fn new(ws: W, label: &'static str) -> Self {
Self {
ws: Some(ws),
label,
}
}
pub(crate) async fn disconnect(mut self) -> anyhow::Result<()> {
let result = match &self.ws {
Some(ws) => ws.close().await,
None => Ok(()),
};
self.ws = None;
result
}
}
impl<W: BinanceWsConnection> std::ops::Deref for DisconnectOnDrop<W> {
type Target = W;
#[allow(clippy::expect_used)] fn deref(&self) -> &W {
self.ws
.as_ref()
.expect("DisconnectOnDrop holds its connection until disconnected")
}
}
impl<W: BinanceWsConnection> Drop for DisconnectOnDrop<W> {
fn drop(&mut self) {
let Some(ws) = self.ws.take() else { return };
let label = self.label;
match tokio::runtime::Handle::try_current() {
Ok(runtime) => {
debug!(
connection = label,
"closing a dropped Binance WebSocket connection"
);
runtime.spawn(async move {
if let Err(e) = ws.close().await {
warn!(%e, connection = label, "failed to close a dropped Binance WebSocket connection");
}
});
}
Err(_) => warn!(
connection = label,
"Binance WebSocket connection dropped outside a Tokio runtime: it cannot be \
disconnected, so binance-sdk closes it once its last handle drops, without \
draining pending requests"
),
}
}
}
pub(crate) struct UnsubscribeOnDrop(Option<Subscription>);
impl UnsubscribeOnDrop {
pub(crate) fn new(subscription: Subscription) -> Self {
Self(Some(subscription))
}
pub(crate) fn unsubscribe(mut self) {
if let Some(subscription) = self.0.take() {
subscription.unsubscribe();
}
}
}
impl Drop for UnsubscribeOnDrop {
fn drop(&mut self) {
if let Some(subscription) = self.0.take() {
subscription.unsubscribe();
}
}
}
pub(crate) const HEARTBEAT_TIMEOUT_SECS: u64 = 30;
const _: () = assert!(
HEARTBEAT_TIMEOUT_SECS <= i64::MAX as u64,
"HEARTBEAT_TIMEOUT_SECS overflows i64"
);
pub(crate) const FILL_RECOVERY_TIMEOUT_SECS: u64 = 30;
pub(crate) const GAP_END_SLACK_SECS: i64 = 10;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct FillGap {
pub(crate) start_ms: i64,
pub(crate) end_ms: i64,
failures: u32,
due: tokio::time::Instant,
}
impl FillGap {
fn same_span(&self, other: &Self) -> bool {
self.start_ms == other.start_ms && self.end_ms == other.end_ms
}
}
#[derive(Debug, Default)]
pub(crate) struct UnrecoveredFills(fnv::FnvHashMap<InstrumentNameExchange, Vec<FillGap>>);
impl PendingFills for UnrecoveredFills {
fn pending(&self, instrument: &InstrumentNameExchange) -> bool {
self.covers(instrument)
}
}
impl UnrecoveredFills {
pub(crate) fn open(
&mut self,
instruments: &[InstrumentNameExchange],
disconnect_time: DateTime<Utc>,
now: DateTime<Utc>,
) {
let start_ms = disconnect_time.timestamp_millis();
let end_ms = (now + chrono::Duration::seconds(GAP_END_SLACK_SECS)).timestamp_millis();
let due = tokio::time::Instant::now();
for instrument in instruments {
let kept = self.0.get(instrument).map_or(&[][..], Vec::as_slice);
let start_ms = kept
.iter()
.filter(|kept| kept.start_ms <= end_ms && start_ms <= kept.end_ms)
.map(|kept| kept.end_ms + 1)
.fold(start_ms, i64::max);
if start_ms <= end_ms {
self.0.entry(instrument.clone()).or_default().push(FillGap {
start_ms,
end_ms,
failures: 0,
due,
});
}
}
}
pub(crate) fn due(&self, now: tokio::time::Instant) -> Vec<(InstrumentNameExchange, FillGap)> {
self.0
.iter()
.flat_map(|(instrument, gaps)| {
gaps.iter()
.filter(move |gap| gap.due <= now)
.map(move |gap| (instrument.clone(), *gap))
})
.collect()
}
pub(crate) fn covers(&self, instrument: &InstrumentNameExchange) -> bool {
self.0.contains_key(instrument)
}
pub(crate) fn next_due(&self) -> Option<tokio::time::Instant> {
self.0.values().flatten().map(|gap| gap.due).min()
}
#[cfg(test)]
pub(crate) fn is_empty(&self) -> bool {
self.0.is_empty()
}
#[cfg(test)]
pub(crate) fn make_due(&mut self) {
let now = tokio::time::Instant::now();
for gap in self.0.values_mut().flatten() {
gap.due = now;
}
}
pub(crate) fn recovered(&mut self, instrument: &InstrumentNameExchange, gap: &FillGap) {
self.remove_where(instrument, |kept| kept.same_span(gap));
}
pub(crate) fn failed(
&mut self,
instrument: &InstrumentNameExchange,
gap: &FillGap,
now: tokio::time::Instant,
) -> Option<GapFailure> {
let gaps = self.0.get_mut(instrument)?;
let index = gaps.iter().position(|kept| kept.same_span(gap))?;
gaps[index].failures += 1;
if gaps[index].failures > MAX_GAP_RETRIES {
gaps.remove(index);
if gaps.is_empty() {
self.0.remove(instrument);
}
return Some(GapFailure::GivenUp);
}
let delay = Duration::from_secs(GAP_RETRY_BASE_SECS << (gaps[index].failures - 1));
gaps[index].due = now + delay;
Some(GapFailure::Retry(delay))
}
fn remove_where(
&mut self,
instrument: &InstrumentNameExchange,
matches: impl Fn(&FillGap) -> bool,
) {
if let Some(gaps) = self.0.get_mut(instrument) {
gaps.retain(|gap| !matches(gap));
if gaps.is_empty() {
self.0.remove(instrument);
}
}
}
}
pub(crate) fn gap_time(ms: i64) -> DateTime<Utc> {
Utc.timestamp_millis_opt(ms).single().unwrap_or_default()
}
pub(crate) fn gap_failed(
unrecovered: &mut UnrecoveredFills,
tx: &mpsc::UnboundedSender<UnindexedAccountEvent>,
venue: ExchangeId,
instrument: &InstrumentNameExchange,
gap: &FillGap,
reason: FillRecoveryFailure,
) {
let (start, end) = (gap_time(gap.start_ms), gap_time(gap.end_ms));
match unrecovered.failed(instrument, gap, tokio::time::Instant::now()) {
Some(GapFailure::Retry(delay)) => warn!(
%venue,
%instrument,
%start,
%end,
retry_in_secs = delay.as_secs(),
%reason,
"Binance fill recovery did not read this gap, retrying it later"
),
Some(GapFailure::GivenUp) => {
error!(
%venue,
%instrument,
%start,
%end,
retries = MAX_GAP_RETRIES,
%reason,
"Binance fill recovery gave up on this gap: its fills are not delivered; read them \
with fetch_trades"
);
let gave_up = FillRecoveryGap::new(
FillRecoveryScope::Instruments(vec![instrument.clone()]),
start,
end,
MAX_GAP_RETRIES + 1,
reason,
);
if tx
.send(UnindexedAccountEvent::new(
venue,
AccountEventKind::FillRecoveryGaveUp(gave_up),
))
.is_err()
{
debug!(%venue, "Binance fill recovery: consumer dropped before the give-up was sent");
}
}
None => {}
}
}
pub(crate) fn drop_after<T: BinanceExecutionFields>(page: &mut Vec<T>, end_ms: i64) -> bool {
let len = page.len();
page.retain(|execution| execution.time().is_none_or(|time| time <= end_ms));
page.len() < len
}
pub(crate) const CONNECT_TIMEOUT_SECS: u64 = 15;
const _: () = assert!(
CONNECT_TIMEOUT_SECS <= i64::MAX as u64,
"CONNECT_TIMEOUT_SECS overflows i64"
);
pub(crate) const SIGNAL_RECOVERY_LOOKBACK_MS: i64 = 500;
pub(crate) const BINANCE_MAX_TRADES: usize = 1000;
const _: () = assert!(
BINANCE_MAX_TRADES <= i32::MAX as usize,
"BINANCE_MAX_TRADES overflows i32"
);
pub(crate) const DEFAULT_RATE_LIMIT_DELAY_SECS: u64 = 10;
const MAX_COOLDOWN_SLEEP: Duration = Duration::from_secs(DEFAULT_RATE_LIMIT_DELAY_SECS);
pub(crate) const MAX_RATE_LIMIT_RETRIES: u32 = 3;
#[derive(Debug, PartialEq)]
pub(crate) enum UserDataFrame<'a> {
Response,
Event {
subscription_id: Option<i64>,
event_type: &'a str,
event: &'a str,
},
Unrecognised,
}
pub(crate) fn parse_user_data_frame(frame: &str) -> UserDataFrame<'_> {
use serde_json::value::RawValue;
#[derive(serde::Deserialize)]
struct Envelope<'a> {
#[serde(borrow, default)]
id: Option<&'a RawValue>,
#[serde(borrow, default)]
event: Option<&'a RawValue>,
#[serde(rename = "subscriptionId", default)]
subscription_id: Option<i64>,
}
#[derive(serde::Deserialize)]
struct EventTag<'a> {
#[serde(borrow, default)]
e: Option<&'a str>,
}
let Ok(envelope) = serde_json::from_str::<Envelope<'_>>(frame) else {
return UserDataFrame::Unrecognised;
};
if envelope.id.is_some() {
return UserDataFrame::Response;
}
let Some(event) = envelope.event else {
return UserDataFrame::Unrecognised;
};
let event = event.get();
let Some(event_type) = serde_json::from_str::<EventTag<'_>>(event)
.ok()
.and_then(|tag| tag.e)
else {
return UserDataFrame::Unrecognised;
};
UserDataFrame::Event {
subscription_id: envelope.subscription_id,
event_type,
event,
}
}
pub(crate) fn log_unrecognised_frame(venue: &'static str, seen: &AtomicU64, frame: &str) {
let count = seen.fetch_add(1, Ordering::Relaxed) + 1;
if count == 1 || count.is_multiple_of(1000) {
warn!(
venue,
count,
frame = frame_excerpt(frame),
"Binance WS: unrecognised user-data frame (not an RPC response, nor an event envelope \
with a readable `e` tag), ignoring it; further ones are logged at trace, with a \
warning every 1000th"
);
} else {
trace!(
venue,
count,
frame = frame_excerpt(frame),
"Binance WS: unrecognised user-data frame, ignoring it"
);
}
}
pub(crate) struct UnhandledEvents {
count: AtomicU64,
types: parking_lot::Mutex<Vec<String>>,
}
impl UnhandledEvents {
const MAX_TYPES: usize = 32;
pub(crate) const fn new() -> Self {
Self {
count: AtomicU64::new(0),
types: parking_lot::Mutex::new(Vec::new()),
}
}
fn record(&self, event_type: &str) -> (u64, bool) {
let count = self.count.fetch_add(1, Ordering::Relaxed) + 1;
let first_of_type = {
let mut types = self.types.lock();
if types.iter().any(|seen| seen == event_type) || types.len() >= Self::MAX_TYPES {
false
} else {
types.push(event_type.to_owned());
true
}
};
(count, first_of_type || count.is_multiple_of(1000))
}
}
pub(crate) fn log_unhandled_event(
venue: &'static str,
seen: &UnhandledEvents,
event_type: &str,
event: &str,
) {
let (count, loud) = seen.record(event_type);
if loud {
warn!(
venue,
count,
event_type,
event = frame_excerpt(event),
"Binance WS: user-data event of a type this client does not handle, ignoring it; \
further ones of this type are logged at trace, with a warning every 1000th"
);
} else {
trace!(
venue,
count, event_type, "Binance WS: unhandled user-data event, ignoring it"
);
}
}
pub(crate) fn frame_excerpt(frame: &str) -> &str {
frame
.char_indices()
.nth(200)
.map_or(frame, |(end, _)| &frame[..end])
}
pub(crate) const SPOT_REQUEST_WEIGHT_PER_MINUTE: u32 = 6_000;
pub(crate) const SAPI_IP_WEIGHT_PER_MINUTE: u32 = 12_000;
const WEIGHT_PAUSE_PERCENT: u64 = 90;
pub(crate) const CLOCK_SKEW_SLACK_MS: u64 = 1_000;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum WeightPool {
Spot,
Sapi,
}
impl WeightPool {
fn default_limit(self) -> u32 {
match self {
Self::Spot => SPOT_REQUEST_WEIGHT_PER_MINUTE,
Self::Sapi => SAPI_IP_WEIGHT_PER_MINUTE,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum RequestKind {
Order,
Query,
Essential,
}
#[derive(Default)]
struct Deadlines {
blocked_until: Option<tokio::time::Instant>,
throttled_until: Option<tokio::time::Instant>,
}
pub(crate) struct RateLimitTracker {
deadlines: parking_lot::Mutex<Deadlines>,
pool: WeightPool,
weight_limit: AtomicU32,
}
impl RateLimitTracker {
pub(crate) fn new(pool: WeightPool) -> Self {
Self {
deadlines: parking_lot::Mutex::new(Deadlines::default()),
pool,
weight_limit: AtomicU32::new(pool.default_limit()),
}
}
pub(crate) async fn wait_if_blocked(&self, kind: RequestKind) -> Result<(), CooldownActive> {
loop {
let (blocked_until, deadline) = {
let deadlines = self.deadlines.lock();
let deadline = match kind {
RequestKind::Order | RequestKind::Essential => deadlines.blocked_until,
RequestKind::Query => deadlines.blocked_until.max(deadlines.throttled_until),
};
(deadlines.blocked_until, deadline)
};
let now = tokio::time::Instant::now();
if let Some(remaining) = blocked_until.map(|until| until.saturating_duration_since(now))
&& too_long_to_sleep(remaining)
{
return Err(CooldownActive { remaining });
}
match deadline {
None => return Ok(()),
Some(until) => {
if until <= now {
return Ok(());
}
#[allow(clippy::cast_possible_truncation)]
let delay_ms = (until - now).as_millis() as u64;
debug!(delay_ms, ?kind, "Binance request held back, waiting");
tokio::time::sleep_until(until).await;
}
}
}
}
#[cfg(test)]
pub(crate) fn weight_limit(&self) -> u32 {
self.weight_limit.load(Ordering::Relaxed)
}
#[cfg(test)]
pub(crate) fn throttle(&self, pause: Duration) {
let now = tokio::time::Instant::now();
extend_deadline(&mut self.deadlines.lock().throttled_until, now, pause);
}
pub(crate) fn cooldown_remaining(&self) -> Option<Duration> {
let now = tokio::time::Instant::now();
self.deadlines
.lock()
.blocked_until
.map(|until| until.saturating_duration_since(now))
.filter(|remaining| !remaining.is_zero())
}
#[cfg(test)]
pub(crate) fn is_blocked(&self) -> bool {
self.cooldown_remaining().is_some()
}
pub(crate) fn on_rate_limited(&self, retry_after: Option<Duration>) {
let delay = retry_after.unwrap_or(Duration::from_secs(DEFAULT_RATE_LIMIT_DELAY_SECS));
let now = tokio::time::Instant::now();
let was_blocked = extend_deadline(&mut self.deadlines.lock().blocked_until, now, delay);
if was_blocked {
debug!(
delay_secs = delay.as_secs(),
"Binance rate-limit cooldown extended"
);
} else {
warn!(
delay_secs = delay.as_secs(),
"Binance entering rate-limit degradation mode"
);
}
}
pub(crate) fn on_stream_refused(&self, e: &anyhow::Error) -> bool {
let retry_after = if let Some(refusal) = HandshakeRefusal::of(e) {
refusal.retry_after()
} else if let Some(RequestFailure::RateLimited(retry_after)) = ws_response_failure(e) {
retry_after
} else {
return false;
};
warn!(error = %format_args!("{e:#}"), "Binance refused an account-stream connect for a rate limit");
self.on_rate_limited(retry_after);
true
}
pub(crate) async fn before_stream_connect(&self) -> Result<(), StreamTerminationReason> {
self.wait_if_blocked(RequestKind::Essential)
.await
.map_err(|cooldown| end_for_cooldown(cooldown.remaining))
}
pub(crate) async fn before_stream_open(&self) -> Result<(), UnindexedClientError> {
self.wait_if_blocked(RequestKind::Essential)
.await
.map_err(|cooldown| {
UnindexedClientError::Api(ApiError::rate_limited(Some(cooldown.remaining)))
})
}
pub(crate) fn stream_open_error(
&self,
e: &anyhow::Error,
otherwise: impl FnOnce(&anyhow::Error) -> UnindexedClientError,
) -> UnindexedClientError {
if self.on_stream_refused(e) {
UnindexedClientError::Api(ApiError::rate_limited(self.cooldown_remaining()))
} else {
otherwise(e)
}
}
fn too_long_to_wait(&self) -> Option<Duration> {
self.cooldown_remaining()
.filter(|remaining| too_long_to_sleep(*remaining))
}
pub(crate) fn observe_rest<D>(&self, response: &RestApiResponse<D>, sent_ms: u128) {
let used = match self.pool {
WeightPool::Spot => response
.rate_limits
.iter()
.flatten()
.find(|limit| {
is_weight_per_minute(
&limit.rate_limit_type,
&limit.interval,
limit.interval_num,
)
})
.map(|limit| limit.count),
WeightPool::Sapi => response
.headers
.get("x-sapi-used-ip-weight-1m")
.and_then(|used| used.parse().ok()),
};
if let Some(used) = used {
self.observe_used_weight(used, sent_ms);
}
}
pub(crate) fn observe_ws_api(
&self,
rate_limits: Option<&[WebsocketApiRateLimit]>,
sent_ms: u128,
) {
debug_assert_eq!(self.pool, WeightPool::Spot, "WS-API usage is spot weight");
if self.pool != WeightPool::Spot {
return;
}
let Some(weight) = rate_limits.into_iter().flatten().find(|limit| {
is_weight_per_minute(&limit.rate_limit_type, &limit.interval, limit.interval_num)
}) else {
return;
};
let previous = self.weight_limit.load(Ordering::Relaxed);
if weight.limit > 0 && weight.limit != previous {
self.weight_limit.store(weight.limit, Ordering::Relaxed);
debug!(
previous,
limit = weight.limit,
"Binance request-weight limit per minute updated"
);
}
self.observe_used_weight(weight.count, sent_ms);
}
fn observe_used_weight(&self, used: u32, sent_ms: u128) {
let limit = self.weight_limit.load(Ordering::Relaxed);
if u64::from(used) * 100 < u64::from(limit) * WEIGHT_PAUSE_PERCENT {
return;
}
let Some(pause) = pause_after(sent_ms, unix_ms()) else {
return;
};
let now = tokio::time::Instant::now();
let was_throttled = extend_deadline(&mut self.deadlines.lock().throttled_until, now, pause);
if !was_throttled {
#[allow(clippy::cast_possible_truncation)]
let pause_ms = pause.as_millis() as u64;
info!(
used,
limit,
pause_ms,
"Binance request weight near the per-minute limit, pausing queries until the next minute"
);
}
}
}
fn extend_deadline(
deadline: &mut Option<tokio::time::Instant>,
now: tokio::time::Instant,
delay: Duration,
) -> bool {
let was_active = deadline.is_some_and(|until| until > now);
let new_deadline = now
.checked_add(delay)
.unwrap_or_else(|| now + Duration::from_secs(u64::from(u32::MAX)));
*deadline = Some(deadline.map_or(new_deadline, |existing| existing.max(new_deadline)));
was_active
}
fn is_weight_per_minute(kind: &RateLimitType, interval: &Interval, interval_num: u32) -> bool {
*kind == RateLimitType::RequestWeight && *interval == Interval::Minute && interval_num == 1
}
pub(crate) fn unix_ms() -> u128 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis()
}
fn pause_after(sent_ms: u128, now_ms: u128) -> Option<Duration> {
let resume_ms = sent_ms - sent_ms % 60_000 + 60_000 + u128::from(CLOCK_SKEW_SLACK_MS);
let remaining_ms = resume_ms.checked_sub(now_ms).filter(|&ms| ms > 0)?;
#[allow(clippy::cast_possible_truncation)]
Some(Duration::from_millis(remaining_ms as u64))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)]
#[error("Binance rate-limit cooldown active for {remaining:?} more; request not sent")]
pub(crate) struct CooldownActive {
pub(crate) remaining: Duration,
}
fn rest_retry_after(ce: &ConnectorError) -> Option<Duration> {
ce.retry_after()
.filter(|&secs| secs > 0)
.map(|secs| Duration::from_secs(u64::from(secs)))
}
fn wait_until_epoch_ms(until_ms: i64, now_ms: u128) -> Duration {
let resume_ms = u128::try_from(until_ms).unwrap_or(0) + u128::from(CLOCK_SKEW_SLACK_MS);
let wait_ms = resume_ms.saturating_sub(now_ms);
Duration::from_millis(u64::try_from(wait_ms).unwrap_or(u64::MAX))
}
pub(crate) fn is_rate_limit_error(e: &anyhow::Error) -> bool {
if let Some(ConnectorError::ForbiddenError { msg, code }) = e.downcast_ref::<ConnectorError>()
&& is_waf_block(&with_code(msg, *code))
{
return true;
}
for cause in e.chain() {
let msg = cause.to_string();
if msg.contains("Too many requests")
|| msg.contains("been banned for exceeding rate limits")
|| has_rate_limit_error_code(&msg)
{
return true;
}
}
false
}
pub(crate) const MIN_IP_BAN: Duration = Duration::from_secs(120);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum HandshakeRefusal {
Throttled,
Banned,
}
impl HandshakeRefusal {
pub(crate) fn of(e: &anyhow::Error) -> Option<Self> {
e.chain()
.find_map(|cause| match cause.downcast_ref::<WebsocketError>()? {
WebsocketError::Handshake(msg) if msg.contains("HTTP error: 429") => {
Some(Self::Throttled)
}
WebsocketError::Handshake(msg) if msg.contains("HTTP error: 418") => {
Some(Self::Banned)
}
_ => None,
})
}
pub(crate) fn retry_after(self) -> Option<Duration> {
match self {
Self::Throttled => None,
Self::Banned => Some(MIN_IP_BAN),
}
}
}
pub(crate) async fn retry_or_end(
budget: &mut ReconnectBudget,
rate_limiter: &RateLimitTracker,
last_error: impl FnOnce() -> String,
) -> Result<(), StreamTerminationReason> {
if let Some(remaining) = rate_limiter.too_long_to_wait() {
return Err(end_for_cooldown(remaining));
}
if budget.wait().await {
return Ok(());
}
error!(
attempts = budget.attempts(),
"Binance account stream: max reconnect attempts exhausted, stream terminating"
);
Err(StreamTerminationReason::ReconnectBudgetExhausted {
attempts: budget.attempts(),
last_error: last_error(),
})
}
pub(crate) fn disconnect_error(heartbeat_timed_out: bool) -> String {
if heartbeat_timed_out {
format!("heartbeat timeout ({HEARTBEAT_TIMEOUT_SECS}s)")
} else {
"WebSocket disconnected (server close/error)".to_string()
}
}
fn too_long_to_sleep(remaining: Duration) -> bool {
remaining > MAX_COOLDOWN_SLEEP
}
fn end_for_cooldown(remaining: Duration) -> StreamTerminationReason {
warn!(
retry_after_secs = remaining.as_secs(),
"Binance account stream ending: the rate-limit cooldown outlasts what it waits through"
);
StreamTerminationReason::RateLimited {
retry_after: remaining,
}
}
pub(crate) fn parse_side(s: &str) -> Option<Side> {
match s {
"BUY" => Some(Side::Buy),
"SELL" => Some(Side::Sell),
_ => {
warn!(side = s, "unknown Binance order side");
None
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum TrailingDelta {
Absent,
BasisPoints(u32),
Unknown,
}
impl TrailingDelta {
pub(crate) fn from_reported(delta: Option<i64>) -> Self {
match delta {
None | Some(0) => Self::Absent,
Some(bips) => u32::try_from(bips).map_or(Self::Unknown, Self::BasisPoints),
}
}
}
pub(crate) fn decode_order_kind(
exchange: ExchangeId,
order_id: Option<i64>,
order_type: &str,
stop_price: Option<&str>,
trailing_delta: TrailingDelta,
) -> Option<OrderKind> {
let parsed_stop_price = stop_price.map(Decimal::from_str);
let unparseable_stop_price = stop_price.filter(|_| matches!(parsed_stop_price, Some(Err(_))));
let stop_price = parsed_stop_price
.and_then(Result::ok)
.filter(|price| price.is_sign_positive() && !price.is_zero());
let triggered = |kind: fn(Decimal) -> OrderKind| {
if stop_price.is_none() {
warn!(%exchange, ?order_id, order_type, "Binance conditional order has no positive stopPrice, dropping it");
}
stop_price.map(kind)
};
match (order_type, trailing_delta) {
("MARKET", _) => Some(OrderKind::Market),
("LIMIT" | "LIMIT_MAKER", _) => Some(OrderKind::Limit),
(_, TrailingDelta::Unknown) if is_conditional_order_type(order_type) => {
static SEEN: AtomicU64 = AtomicU64::new(0);
let (count, warns) = sampled(&SEEN);
if warns {
warn!(%exchange, ?order_id, order_type, count, "Binance conditional order may be a trailing one and its trailingDelta is unknown, dropping it; further ones are logged at debug, with a warning every 1000th");
} else {
debug!(%exchange, ?order_id, order_type, count, "Binance conditional order may be a trailing one and its trailingDelta is unknown, dropping it");
}
None
}
("STOP_LOSS", TrailingDelta::Absent) => {
triggered(|trigger_price| OrderKind::Stop { trigger_price })
}
("STOP_LOSS_LIMIT", TrailingDelta::Absent) => {
triggered(|trigger_price| OrderKind::StopLimit { trigger_price })
}
("TAKE_PROFIT", TrailingDelta::Absent) => {
triggered(|trigger_price| OrderKind::TakeProfit { trigger_price })
}
("TAKE_PROFIT_LIMIT", TrailingDelta::Absent) => {
triggered(|trigger_price| OrderKind::TakeProfitLimit { trigger_price })
}
("STOP_LOSS" | "STOP_LOSS_LIMIT", TrailingDelta::BasisPoints(_))
if unparseable_stop_price.is_some() =>
{
warn!(%exchange, ?order_id, order_type, stop_price = ?unparseable_stop_price, "Binance trailing order's stopPrice, its activation price, does not parse, dropping it");
None
}
("STOP_LOSS", TrailingDelta::BasisPoints(bips)) => Some(OrderKind::TrailingStop {
offset: Decimal::from(bips),
offset_type: TrailingOffsetType::BasisPoints,
activation_price: stop_price,
}),
("STOP_LOSS_LIMIT", TrailingDelta::BasisPoints(bips)) => {
Some(OrderKind::TrailingStopFixedLimit {
offset: Decimal::from(bips),
offset_type: TrailingOffsetType::BasisPoints,
activation_price: stop_price,
})
}
("TAKE_PROFIT" | "TAKE_PROFIT_LIMIT", TrailingDelta::BasisPoints(_)) => {
static SEEN: AtomicU64 = AtomicU64::new(0);
let (count, warns) = sampled(&SEEN);
if warns {
warn!(%exchange, ?order_id, order_type, count, "Binance trailing order of a type OrderKind cannot describe, dropping it; further ones are logged at debug, with a warning every 1000th");
} else {
debug!(%exchange, ?order_id, order_type, count, "Binance trailing order of a type OrderKind cannot describe, dropping it");
}
None
}
_ => {
warn!(%exchange, ?order_id, order_type, "unsupported Binance order type");
None
}
}
}
fn sampled(seen: &AtomicU64) -> (u64, bool) {
let count = seen.fetch_add(1, Ordering::Relaxed) + 1;
(count, count == 1 || count.is_multiple_of(1000))
}
fn is_conditional_order_type(order_type: &str) -> bool {
matches!(
order_type,
"STOP_LOSS" | "STOP_LOSS_LIMIT" | "TAKE_PROFIT" | "TAKE_PROFIT_LIMIT"
)
}
fn kind_has_limit_price(kind: &OrderKind) -> bool {
match kind {
OrderKind::Limit
| OrderKind::StopLimit { .. }
| OrderKind::TakeProfitLimit { .. }
| OrderKind::TrailingStopLimit { .. }
| OrderKind::TrailingStopFixedLimit { .. } => true,
OrderKind::Market
| OrderKind::Stop { .. }
| OrderKind::TakeProfit { .. }
| OrderKind::TrailingStop { .. } => false,
}
}
pub(crate) fn parse_time_in_force(tif: &str) -> TimeInForce {
match tif {
"GTC" => TimeInForce::GoodUntilCancelled { post_only: false },
"GTX" => TimeInForce::GoodUntilCancelled { post_only: true },
"IOC" => TimeInForce::ImmediateOrCancel,
"FOK" => TimeInForce::FillOrKill,
"GTD" => TimeInForce::GoodUntilEndOfDay,
_ => {
warn!(
time_in_force = tif,
"unknown Binance TimeInForce, defaulting to GTC"
);
TimeInForce::GoodUntilCancelled { post_only: false }
}
}
}
pub(crate) trait BinanceOrderFields {
fn order_id(&self) -> Option<i64>;
fn client_order_id(&self) -> Option<&str>;
fn side(&self) -> Option<&str>;
fn price(&self) -> Option<&str>;
fn orig_qty(&self) -> Option<&str>;
fn executed_qty(&self) -> Option<&str>;
fn order_type(&self) -> Option<&str>;
fn time_in_force(&self) -> Option<&str>;
fn time(&self) -> Option<i64>;
fn update_time(&self) -> Option<i64>;
fn symbol(&self) -> Option<&str>;
fn status(&self) -> Option<&str>;
fn cumulative_quote_qty(&self) -> Option<&str>;
fn stop_price(&self) -> Option<&str>;
fn trailing_delta(&self) -> TrailingDelta;
}
macro_rules! impl_binance_order_fields {
(reports_trailing_delta: $($t:ty),* $(,)?) => {
impl_binance_order_fields!(@impl |order| TrailingDelta::from_reported(order.trailing_delta); $($t),*);
};
(omits_trailing_delta: $($t:ty),* $(,)?) => {
impl_binance_order_fields!(@impl |_order| TrailingDelta::Unknown; $($t),*);
};
(@impl |$order:ident| $trailing_delta:expr; $($t:ty),*) => {
$(
impl BinanceOrderFields for $t {
fn order_id(&self) -> Option<i64> { self.order_id }
fn client_order_id(&self) -> Option<&str> { self.client_order_id.as_deref() }
fn side(&self) -> Option<&str> { self.side.as_deref() }
fn price(&self) -> Option<&str> { self.price.as_deref() }
fn orig_qty(&self) -> Option<&str> { self.orig_qty.as_deref() }
fn executed_qty(&self) -> Option<&str> { self.executed_qty.as_deref() }
fn order_type(&self) -> Option<&str> { self.r#type.as_deref() }
fn time_in_force(&self) -> Option<&str> { self.time_in_force.as_deref() }
fn time(&self) -> Option<i64> { self.time }
fn update_time(&self) -> Option<i64> { self.update_time }
fn symbol(&self) -> Option<&str> { self.symbol.as_deref() }
fn status(&self) -> Option<&str> { self.status.as_deref() }
fn cumulative_quote_qty(&self) -> Option<&str> {
self.cummulative_quote_qty.as_deref()
}
fn stop_price(&self) -> Option<&str> { self.stop_price.as_deref() }
fn trailing_delta(&self) -> TrailingDelta {
let $order = self;
$trailing_delta
}
}
)*
};
}
impl_binance_order_fields!(
reports_trailing_delta: binance_sdk::spot::rest_api::AllOrdersResponseInner,
binance_sdk::spot::rest_api::GetOpenOrdersResponseInner,
binance_sdk::spot::rest_api::GetOrderResponse,
);
impl_binance_order_fields!(
omits_trailing_delta: binance_sdk::margin_trading::rest_api::QueryMarginAccountsOpenOrdersResponseInner,
binance_sdk::margin_trading::rest_api::QueryMarginAccountsOrderResponse,
);
fn rest_order_is_open(status: &str) -> bool {
matches!(status, "NEW" | "PARTIALLY_FILLED" | "PENDING_NEW")
}
pub(crate) fn convert_open_order<T: BinanceOrderFields>(
o: &T,
exchange: ExchangeId,
instrument: &InstrumentNameExchange,
) -> Option<Order<ExchangeId, InstrumentNameExchange, Open>> {
match o.status() {
Some(status) if rest_order_is_open(status) => {}
Some(status) => {
warn!(%exchange, %instrument, order_id = ?o.order_id(), status, "Binance order is not live, not converting it to an open order");
return None;
}
None => {
warn!(%exchange, %instrument, order_id = ?o.order_id(), "Binance open order missing status");
return None;
}
}
let row = convert_order_row(o, exchange, instrument, UnknownTrailingDelta::Drop)?;
Some(row.map_state(|row| {
Open::new(
VenueOrderId::Assigned(row.order_id),
row.time_exchange,
row.filled_qty.unwrap_or(Decimal::ZERO),
)
}))
}
pub(crate) fn convert_ended_order<T: BinanceOrderFields>(
o: &T,
exchange: ExchangeId,
key: &UnindexedOrderKey,
recorded_kind: Option<OrderKind>,
) -> Option<UnindexedInactiveOrder> {
let instrument = &key.instrument;
let Some(status) = o.status() else {
warn!(%exchange, %instrument, cid = %key.cid, "Binance order missing status");
return None;
};
if rest_order_is_open(status) {
return None;
}
let row = convert_order_row(
o,
exchange,
instrument,
UnknownTrailingDelta::Ended {
recorded: recorded_kind,
},
)?;
let Some(state) = ended_order_state(
status,
row.state.order_id.clone(),
row.state.time_exchange,
row.quantity,
row.state.filled_qty,
|filled_qty| {
binance_avg_price(
exchange,
&row.state.order_id,
o.cumulative_quote_qty(),
filled_qty,
)
},
) else {
warn!(%exchange, %instrument, cid = %key.cid, status, "Binance order has a status this version does not know, treating it as not ended");
return None;
};
let mut order = row.map_state(|_| state);
order.key = key.clone();
Some(order)
}
pub(crate) fn ended_order_state<AssetKey, InstrumentKey>(
status: &str,
order_id: OrderId,
time_exchange: DateTime<Utc>,
quantity: Decimal,
filled_qty: Option<Decimal>,
avg_price: impl FnOnce(Decimal) -> Option<Decimal>,
) -> Option<InactiveOrderState<AssetKey, InstrumentKey>> {
Some(match status {
"FILLED" => {
let filled_qty = filled_qty.unwrap_or(quantity);
InactiveOrderState::FullyFilled(Filled::new(
order_id,
time_exchange,
filled_qty,
avg_price(filled_qty),
))
}
"CANCELED" => {
InactiveOrderState::Cancelled(Cancelled::new(order_id, time_exchange, filled_qty))
}
"EXPIRED" | "EXPIRED_IN_MATCH" => {
InactiveOrderState::Expired(Expired::new(order_id, time_exchange, filled_qty))
}
"REJECTED" => InactiveOrderState::OpenFailed(OrderError::Rejected(
ApiError::OrderRejected(format!("Binance rejected order {order_id}")),
)),
_ => return None,
})
}
pub(crate) fn unreadable_answer(exchange: ExchangeId, problem: &str) -> UnindexedOrderError {
error!(%exchange, problem, "Binance answered an order request unreadably - outcome unknown");
OrderError::Connectivity(ConnectivityError::Socket(format!(
"{exchange} answered unreadably, outcome unknown: {problem}"
)))
}
#[derive(Debug, Clone)]
pub(crate) struct PlacementResponse<'a> {
pub(crate) status: Option<&'a str>,
pub(crate) order_id: OrderId,
pub(crate) time_exchange: DateTime<Utc>,
pub(crate) executed_qty: Option<&'a str>,
pub(crate) cumulative_quote_qty: Option<&'a str>,
}
pub(crate) fn placed_order_state(
exchange: ExchangeId,
instrument: &InstrumentNameExchange,
quantity: Decimal,
response: PlacementResponse<'_>,
) -> UnindexedOrderState {
let PlacementResponse {
status,
order_id,
time_exchange,
executed_qty,
cumulative_quote_qty,
} = response;
let filled_qty = match (status, executed_qty) {
(None, None) => None,
_ => binance_filled_qty(exchange, &order_id, executed_qty),
};
let avg_price =
|filled_qty| binance_avg_price(exchange, &order_id, cumulative_quote_qty, filled_qty);
if let Some(status) = status {
if let Some(ended) = ended_order_state(
status,
order_id.clone(),
time_exchange,
quantity,
filled_qty,
avg_price,
) {
return OrderState::Inactive(ended);
}
if !rest_order_is_open(status) {
warn!(%exchange, %instrument, %order_id, status, "Binance placed an order with a status this version does not know, reading it from what filled");
}
}
match filled_qty {
Some(filled_qty) if filled_qty >= quantity => {
let avg_price = avg_price(filled_qty);
OrderState::fully_filled(Filled::new(order_id, time_exchange, filled_qty, avg_price))
}
_ => OrderState::active(Open::new(
VenueOrderId::Assigned(order_id),
time_exchange,
filled_qty.unwrap_or(Decimal::ZERO),
)),
}
}
pub(crate) fn binance_filled_qty(
exchange: ExchangeId,
order_id: &OrderId,
filled_qty: Option<&str>,
) -> Option<Decimal> {
let Some(raw) = filled_qty else {
warn!(%exchange, %order_id, "Binance did not report how much the order filled");
return None;
};
match Decimal::from_str(raw) {
Ok(filled_qty) => Some(filled_qty),
Err(_) => {
warn!(%exchange, %order_id, filled_qty = raw, "Binance reported an unparseable filled quantity, treating it as unknown");
None
}
}
}
pub(crate) fn binance_avg_price(
exchange: ExchangeId,
order_id: &OrderId,
cumulative_quote_qty: Option<&str>,
filled_qty: Decimal,
) -> Option<Decimal> {
if filled_qty.is_zero() {
return None;
}
let quote = cumulative_quote_qty?;
match Decimal::from_str(quote) {
Ok(quote) if quote.is_sign_negative() => None,
Ok(quote) => quote.checked_div(filled_qty),
Err(_) => {
warn!(%exchange, %order_id, cummulative_quote_qty = quote, "Binance: failed to parse cummulativeQuoteQty; average price unavailable");
None
}
}
}
#[derive(Debug)]
struct OrderRow {
order_id: OrderId,
time_exchange: DateTime<Utc>,
filled_qty: Option<Decimal>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum UnknownTrailingDelta {
Drop,
Ended { recorded: Option<OrderKind> },
}
fn convert_order_row<T: BinanceOrderFields>(
o: &T,
exchange: ExchangeId,
instrument: &InstrumentNameExchange,
unknown_trailing_delta: UnknownTrailingDelta,
) -> Option<Order<ExchangeId, InstrumentNameExchange, OrderRow>> {
let Some(order_id_raw) = o.order_id() else {
warn!(%exchange, %instrument, "Binance order missing orderId");
return None;
};
let order_id = OrderId(format_smolstr!("{}", order_id_raw));
if o.client_order_id().is_none() {
warn!(%exchange, %instrument, order_id = %order_id_raw, "Binance order missing clientOrderId, using orderId as fallback — order may not reconcile with engine state");
}
let cid = ClientOrderId::new(
o.client_order_id()
.unwrap_or(&format_smolstr!("{}", order_id_raw)),
);
let side = match o.side() {
Some(s) => parse_side(s)?,
None => {
warn!(%exchange, %instrument, order_id = %order_id_raw, "Binance order missing side");
return None;
}
};
let quantity = match o.orig_qty().and_then(|s| Decimal::from_str(s).ok()) {
Some(v) => v,
None => {
warn!(%exchange, %instrument, order_id = %order_id_raw, "Binance order missing/unparseable origQty");
return None;
}
};
let filled_qty = binance_filled_qty(exchange, &order_id, o.executed_qty());
let kind = match o.order_type() {
Some(t) => {
let trailing_delta = o.trailing_delta();
let decode = |trailing_delta| {
decode_order_kind(
exchange,
Some(order_id_raw),
t,
o.stop_price(),
trailing_delta,
)
};
let undescribed =
trailing_delta == TrailingDelta::Unknown && is_conditional_order_type(t);
match unknown_trailing_delta {
UnknownTrailingDelta::Ended {
recorded: Some(recorded),
} if undescribed => {
debug!(%exchange, %instrument, order_id = %order_id_raw, order_type = t, kind = %recorded, "Binance order has ended and its row cannot tell whether it trailed; reporting it with the kind it was held with");
recorded
}
UnknownTrailingDelta::Ended { recorded: None } if undescribed => {
warn!(%exchange, %instrument, order_id = %order_id_raw, order_type = t, "Binance order has ended and may have been a trailing one, which its row cannot tell; reporting it with its fixed trigger");
decode(TrailingDelta::Absent)?
}
_ => decode(trailing_delta)?,
}
}
None => {
warn!(%exchange, %instrument, order_id = %order_id_raw, "Binance order missing type");
return None;
}
};
let price = if kind_has_limit_price(&kind) {
o.price().and_then(|s| Decimal::from_str(s).ok())
} else {
None
};
let time_in_force = parse_time_in_force(o.time_in_force().unwrap_or("GTC"));
let time_exchange = match o
.update_time()
.or_else(|| o.time())
.and_then(|ms| Utc.timestamp_millis_opt(ms).single())
{
Some(ts) => ts,
None => {
warn!(%exchange, %instrument, order_id = %order_id_raw, "Binance order missing/unparseable time, using now");
Utc::now()
}
};
Some(Order {
key: OrderKey::new(
exchange,
instrument.clone(),
StrategyId::unknown(),
cid,
),
side,
price,
quantity,
kind,
time_in_force,
state: OrderRow {
order_id,
time_exchange,
filled_qty,
},
})
}
#[derive(Debug)]
pub(crate) struct OpenOrderListing {
pub(crate) orders: Vec<Order<ExchangeId, InstrumentNameExchange, Open>>,
pub(crate) complete: bool,
}
pub(crate) fn convert_open_order_listing<T: BinanceOrderFields>(
rows: &[T],
exchange: ExchangeId,
instrument: &InstrumentNameExchange,
) -> OpenOrderListing {
let mut complete = true;
let orders = rows
.iter()
.filter_map(|row| {
let order = convert_open_order(row, exchange, instrument);
complete &= order.is_some() && row.client_order_id().is_some();
order
})
.collect();
OpenOrderListing { orders, complete }
}
pub(crate) fn convert_open_order_owned_symbol<T: BinanceOrderFields>(
o: &T,
exchange: ExchangeId,
) -> Option<Order<ExchangeId, InstrumentNameExchange, Open>> {
let instrument = match o.symbol() {
Some(s) => InstrumentNameExchange::new(s),
None => {
warn!(%exchange, "Binance open order missing symbol in return-all query, dropping order");
return None;
}
};
convert_open_order(o, exchange, &instrument)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum MyTradesFrom {
Span { start: i64, end: i64 },
Order(i64),
}
pub(crate) trait BinanceExecutionFields {
fn id(&self) -> Option<i64>;
fn order_id(&self) -> Option<i64>;
fn qty(&self) -> Option<&str>;
fn time(&self) -> Option<i64>;
}
macro_rules! impl_binance_execution_fields {
($($t:ty),* $(,)?) => {
$(
impl BinanceExecutionFields for $t {
fn id(&self) -> Option<i64> { self.id }
fn order_id(&self) -> Option<i64> { self.order_id }
fn qty(&self) -> Option<&str> { self.qty.as_deref() }
fn time(&self) -> Option<i64> { self.time }
}
)*
};
}
impl_binance_execution_fields!(
binance_sdk::spot::rest_api::MyTradesResponseInner,
binance_sdk::margin_trading::rest_api::QueryMarginAccountsTradeListResponseInner,
);
pub(crate) const ORDER_EXECUTIONS_BUDGET: Duration =
Duration::from_secs(FILL_RECOVERY_TIMEOUT_SECS / 2);
pub(crate) const ORDER_EXECUTIONS_IN_FLIGHT: usize = 4;
pub(crate) async fn recovered_order_totals<T, F, Fut>(
exchange: ExchangeId,
instrument: &InstrumentNameExchange,
recovered: &[T],
deadline: tokio::time::Instant,
fetch: F,
) -> fnv::FnvHashMap<i64, Decimal>
where
T: BinanceExecutionFields,
F: Fn(i64) -> Fut,
Fut: Future<Output = Result<Vec<T>, UnindexedClientError>>,
{
let mut totals = fnv::FnvHashMap::default();
for_each_recovered_order(
RecoveredOrderLookup {
exchange,
instrument,
reading: "orders' executions",
lost: "cumulative filled quantity",
},
recovered,
deadline,
fetch,
|order_id, executions| match order_running_totals(order_id, &executions) {
Some(running) => totals.extend(running),
None => warn!(
%exchange, %instrument, order_id, executions = executions.len(),
"fill recovery read an order's executions it cannot sum (a missing id or size, \
or another order's execution); its recovered fills carry no cumulative filled \
quantity"
),
},
)
.await;
totals
}
pub(crate) async fn recovered_order_cids<T, F, Fut>(
exchange: ExchangeId,
instrument: &InstrumentNameExchange,
recovered: &[T],
deadline: tokio::time::Instant,
fetch: F,
) -> fnv::FnvHashMap<i64, ClientOrderId>
where
T: BinanceExecutionFields,
F: Fn(i64) -> Fut,
Fut: Future<Output = Result<Option<ClientOrderId>, UnindexedClientError>>,
{
let mut cids = fnv::FnvHashMap::default();
for_each_recovered_order(
RecoveredOrderLookup {
exchange,
instrument,
reading: "orders",
lost: "client order id",
},
recovered,
deadline,
fetch,
|order_id, cid| {
if let Some(cid) = cid {
cids.insert(order_id, cid);
}
},
)
.await;
cids
}
struct RecoveredOrderLookup<'a> {
exchange: ExchangeId,
instrument: &'a InstrumentNameExchange,
reading: &'static str,
lost: &'static str,
}
async fn for_each_recovered_order<T, R, F, Fut>(
lookup: RecoveredOrderLookup<'_>,
recovered: &[T],
deadline: tokio::time::Instant,
fetch: F,
mut settle: impl FnMut(i64, R),
) where
T: BinanceExecutionFields,
F: Fn(i64) -> Fut,
Fut: Future<Output = Result<R, UnindexedClientError>>,
{
use futures::StreamExt as _;
let RecoveredOrderLookup {
exchange,
instrument,
reading,
lost,
} = lookup;
let mut order_ids: Vec<i64> = recovered.iter().filter_map(|t| t.order_id()).collect();
order_ids.sort_unstable();
order_ids.dedup();
if tokio::time::Instant::now() >= deadline {
if !order_ids.is_empty() {
warn!(
%exchange, %instrument,
orders_left = order_ids.len(),
budget = ?ORDER_EXECUTIONS_BUDGET,
"fill recovery ran out of time before reading {reading}; the orders' recovered \
fills carry no {lost}"
);
}
return;
}
let fetch = &fetch;
let mut lookups = futures::stream::iter(order_ids.iter().copied())
.map(|order_id| async move { (order_id, fetch(order_id).await) })
.buffer_unordered(ORDER_EXECUTIONS_IN_FLIGHT);
let mut settled = 0;
loop {
let (order_id, result) = match tokio::time::timeout_at(deadline, lookups.next()).await {
Ok(Some(lookup)) => lookup,
Ok(None) => break,
Err(_elapsed) => {
warn!(
%exchange, %instrument,
orders_left = order_ids.len() - settled,
budget = ?ORDER_EXECUTIONS_BUDGET,
"fill recovery ran out of time reading {reading}; the remaining orders' \
recovered fills carry no {lost}"
);
break;
}
};
settled += 1;
match result {
Ok(value) => settle(order_id, value),
Err(error) => warn!(
%exchange, %instrument, order_id, %error,
"fill recovery could not read {reading} for an order; its recovered fills carry \
no {lost}"
),
}
}
}
pub(crate) fn order_running_totals<T: BinanceExecutionFields>(
order_id: i64,
executions: &[T],
) -> Option<Vec<(i64, Decimal)>> {
let mut sized = executions
.iter()
.map(|execution| {
if execution.order_id() != Some(order_id) {
return None;
}
let qty = Decimal::from_str(execution.qty()?).ok()?;
Some((execution.id()?, qty))
})
.collect::<Option<Vec<_>>>()?;
sized.sort_unstable_by_key(|&(id, _)| id);
sized.dedup_by_key(|&mut (id, _)| id);
let mut cumulative = Decimal::ZERO;
Some(
sized
.into_iter()
.map(|(id, qty)| {
cumulative += qty;
(id, cumulative)
})
.collect(),
)
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)]
#[cfg_attr(test, derive(serde::Serialize))]
pub(crate) struct ExecutionReport {
#[serde(rename = "x")]
pub(crate) execution_type: Option<String>,
#[serde(rename = "X")]
pub(crate) order_status: Option<String>,
#[serde(rename = "s")]
pub(crate) symbol: Option<String>,
#[serde(rename = "S")]
pub(crate) side: Option<String>,
#[serde(rename = "i")]
pub(crate) order_id: Option<i64>,
#[serde(rename = "c")]
pub(crate) client_order_id: Option<String>,
#[serde(rename = "T")]
pub(crate) transaction_time: Option<i64>,
#[serde(rename = "t")]
pub(crate) trade_id: Option<i64>,
#[serde(rename = "L")]
pub(crate) last_executed_price: Option<String>,
#[serde(rename = "l")]
pub(crate) last_executed_quantity: Option<String>,
#[serde(rename = "n")]
pub(crate) commission_amount: Option<String>,
#[serde(rename = "N")]
pub(crate) commission_asset: Option<String>,
#[serde(rename = "z")]
pub(crate) cumulative_filled_quantity: Option<String>,
#[serde(rename = "r")]
pub(crate) reject_reason: Option<String>,
#[serde(rename = "o")]
pub(crate) order_type: Option<String>,
#[serde(rename = "p")]
pub(crate) price: Option<String>,
#[serde(rename = "q")]
pub(crate) order_quantity: Option<String>,
#[serde(rename = "f")]
pub(crate) time_in_force: Option<String>,
#[serde(rename = "P")]
pub(crate) stop_price: Option<String>,
#[serde(rename = "d")]
pub(crate) trailing_delta: Option<WireInt>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum WireInt {
Int(i64),
Unreadable(String),
}
impl<'de> Deserialize<'de> for WireInt {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
deserializer.deserialize_any(WireIntVisitor)
}
}
struct WireIntVisitor;
impl<'de> Visitor<'de> for WireIntVisitor {
type Value = WireInt;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("an integer, or a string holding one")
}
fn visit_i64<E: de::Error>(self, v: i64) -> Result<WireInt, E> {
Ok(WireInt::Int(v))
}
fn visit_u64<E: de::Error>(self, v: u64) -> Result<WireInt, E> {
Ok(i64::try_from(v).map_or_else(|_| WireInt::Unreadable(v.to_string()), WireInt::Int))
}
fn visit_f64<E: de::Error>(self, v: f64) -> Result<WireInt, E> {
Ok(WireInt::Unreadable(v.to_string()))
}
fn visit_bool<E: de::Error>(self, v: bool) -> Result<WireInt, E> {
Ok(WireInt::Unreadable(v.to_string()))
}
fn visit_str<E: de::Error>(self, v: &str) -> Result<WireInt, E> {
Ok(v.parse()
.map_or_else(|_| WireInt::Unreadable(v.to_owned()), WireInt::Int))
}
fn visit_unit<E: de::Error>(self) -> Result<WireInt, E> {
Ok(WireInt::Unreadable("null".to_owned()))
}
fn visit_seq<A: de::SeqAccess<'de>>(self, mut seq: A) -> Result<WireInt, A::Error> {
while seq.next_element::<de::IgnoredAny>()?.is_some() {}
Ok(WireInt::Unreadable("an array".to_owned()))
}
fn visit_map<A: de::MapAccess<'de>>(self, mut map: A) -> Result<WireInt, A::Error> {
while map
.next_entry::<de::IgnoredAny, de::IgnoredAny>()?
.is_some()
{}
Ok(WireInt::Unreadable("an object".to_owned()))
}
}
#[cfg(test)]
impl serde::Serialize for WireInt {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
match self {
Self::Int(int) => serializer.serialize_i64(*int),
Self::Unreadable(raw) => serializer.serialize_str(raw),
}
}
}
fn trade_order_is_live(status: &str) -> bool {
matches!(status, "PARTIALLY_FILLED" | "FILLED")
}
#[allow(clippy::cognitive_complexity)]
pub(crate) fn convert_execution_report(
report: &ExecutionReport,
exchange: ExchangeId,
buf: &mut Vec<UnindexedAccountEvent>,
) {
let Some(exec_type) = report.execution_type.as_deref() else {
warn!(%exchange, "Binance executionReport missing execution type (x), dropping");
return;
};
let Some(symbol) = report.symbol.as_deref().map(InstrumentNameExchange::new) else {
warn!(%exchange, "Binance executionReport missing symbol (s), dropping");
return;
};
let Some(order_id_raw) = report.order_id else {
warn!(%exchange, %symbol, "Binance executionReport missing orderId (i), dropping");
return;
};
let order_id = OrderId(format_smolstr!("{order_id_raw}"));
let cid = match report.client_order_id.as_deref() {
Some(c) => ClientOrderId::new(c),
None => ClientOrderId::new(order_id.0.as_str()),
};
let time_exchange = match report
.transaction_time
.and_then(|ms| Utc.timestamp_millis_opt(ms).single())
{
Some(t) => t,
None => {
warn!(%exchange, %symbol, "Binance executionReport missing/unparseable transaction time (T), using now");
Utc::now()
}
};
match exec_type {
"NEW" => {
buf.extend(convert_order_snapshot(
report,
exchange,
symbol,
cid,
order_id,
time_exchange,
));
}
"TRADE" => {
let Some(trade_id) = report.trade_id else {
warn!(%exchange, %symbol, "Binance TRADE event missing trade ID (t), dropping");
return;
};
let trade_id = TradeId(format_smolstr!("{trade_id}"));
let Some(side) = report.side.as_deref().and_then(parse_side) else {
warn!(%exchange, %symbol, "Binance TRADE event missing/unknown side (S), dropping");
return;
};
let last_price = match report.last_executed_price.as_deref() {
Some(s) => match Decimal::from_str(s) {
Ok(v) => v,
Err(e) => {
warn!(%exchange, %symbol, error = %e, raw = s, "Binance TRADE event unparseable last price (L), dropping fill");
return;
}
},
None => {
warn!(%exchange, %symbol, "Binance TRADE event missing last price (L), dropping fill");
return;
}
};
let last_qty = match report.last_executed_quantity.as_deref() {
Some(s) => match Decimal::from_str(s) {
Ok(v) => v,
Err(e) => {
warn!(%exchange, %symbol, error = %e, raw = s, "Binance TRADE event unparseable last qty (l), dropping fill");
return;
}
},
None => {
warn!(%exchange, %symbol, "Binance TRADE event missing last qty (l), dropping fill");
return;
}
};
let commission = match Decimal::from_str(
report.commission_amount.as_deref().unwrap_or("0"),
) {
Ok(v) => v,
Err(e) => {
warn!(%exchange, %symbol, error = %e, "Binance TRADE event unparseable commission (n), defaulting to 0");
Decimal::ZERO
}
};
let fee_asset = report
.commission_asset
.as_deref()
.map(AssetNameExchange::from)
.unwrap_or_else(|| AssetNameExchange::from("UNKNOWN"));
let order_filled_quantity = report
.cumulative_filled_quantity
.as_deref()
.and_then(|s| Decimal::from_str(s).ok());
let trade = Trade::new(
trade_id,
order_id.clone(),
symbol.clone(),
StrategyId::unknown(), time_exchange,
side,
last_price,
last_qty,
order_filled_quantity,
AssetFees::new(fee_asset, commission, None),
);
let trade = match report.client_order_id.as_deref() {
Some(c) if !c.is_empty() => trade.with_client_order_id(cid.clone()),
_ => trade,
};
buf.push(UnindexedAccountEvent::new(
exchange,
AccountEventKind::Trade(trade),
));
let order_status = report.order_status.as_deref().unwrap_or_default();
if trade_order_is_live(order_status) {
buf.extend(convert_order_snapshot(
report,
exchange,
symbol,
cid,
order_id,
time_exchange,
));
} else {
trace!(
%exchange,
%symbol,
status = order_status,
"Binance TRADE for an order the exchange no longer reports as working — \
emitting the execution without an order snapshot"
);
}
}
"CANCELED" | "EXPIRED" | "EXPIRED_IN_MATCH" => {
buf.push(cancelled_event(
report,
exchange,
symbol,
cid,
order_id,
time_exchange,
));
}
"REJECTED" => {
let reject_reason = report.reject_reason.as_deref().unwrap_or("unknown");
warn!(
%exchange, %symbol, %order_id, reason = reject_reason,
"Binance order REJECTED by matching engine"
);
let error = parse_binance_api_error(reject_reason.to_string(), &symbol);
let response = UnindexedOrderResponseCancel {
key: OrderKey::new(exchange, symbol, StrategyId::unknown(), cid),
state: Err(UnindexedOrderError::Rejected(error)),
};
buf.push(UnindexedAccountEvent::new(
exchange,
AccountEventKind::OrderCancelled(response),
));
}
"REPLACE" => {
buf.push(cancelled_event(
report,
exchange,
symbol,
cid,
order_id,
time_exchange,
));
}
_ => {
trace!(%exchange, exec_type, "Binance ignoring execution type");
}
}
}
fn cancelled_event(
report: &ExecutionReport,
exchange: ExchangeId,
symbol: InstrumentNameExchange,
cid: ClientOrderId,
order_id: OrderId,
time_exchange: DateTime<Utc>,
) -> UnindexedAccountEvent {
let filled_qty = binance_filled_qty(
exchange,
&order_id,
report.cumulative_filled_quantity.as_deref(),
);
let response = UnindexedOrderResponseCancel {
key: OrderKey::new(exchange, symbol, StrategyId::unknown(), cid),
state: Ok(Cancelled::new(order_id, time_exchange, filled_qty)),
};
UnindexedAccountEvent::new(exchange, AccountEventKind::OrderCancelled(response))
}
fn convert_order_snapshot(
report: &ExecutionReport,
exchange: ExchangeId,
symbol: InstrumentNameExchange,
cid: ClientOrderId,
order_id: OrderId,
time_exchange: DateTime<Utc>,
) -> Option<UnindexedAccountEvent> {
let Some(side) = report.side.as_deref().and_then(parse_side) else {
warn!(%exchange, %symbol, "Binance order report missing/unknown side (S), dropping snapshot");
return None;
};
let trailing_delta = match &report.trailing_delta {
None => TrailingDelta::Absent,
Some(WireInt::Int(delta)) => TrailingDelta::from_reported(Some(*delta)),
Some(WireInt::Unreadable(raw)) => {
static SEEN: AtomicU64 = AtomicU64::new(0);
let (count, warns) = sampled(&SEEN);
if warns {
warn!(%exchange, %symbol, %order_id, raw, count, "Binance order report has an unreadable trailing delta (d); further ones are logged at debug, with a warning every 1000th");
} else {
debug!(%exchange, %symbol, %order_id, raw, count, "Binance order report has an unreadable trailing delta (d)");
}
TrailingDelta::Unknown
}
};
let Some(order_type) = report.order_type.as_deref() else {
warn!(%exchange, %symbol, %order_id, "Binance order report missing order type (o), dropping snapshot");
return None;
};
let kind = decode_order_kind(
exchange,
report.order_id,
order_type,
report.stop_price.as_deref(),
trailing_delta,
)?;
let price = if kind_has_limit_price(&kind) {
match report.price.as_deref() {
Some(p) => match Decimal::from_str(p) {
Ok(v) if !v.is_zero() => Some(v),
Ok(_) => {
trace!(%exchange, %symbol, %kind, "Binance order report has zero price (p) on limit-type order, treating as no limit price");
None
}
Err(e) => {
warn!(%exchange, %symbol, price = p, error = %e, "Binance order report unparseable price (p), dropping snapshot");
return None;
}
},
None => {
warn!(%exchange, %symbol, "Binance limit-type order report missing price (p), dropping snapshot");
return None;
}
}
} else {
None
};
let quantity = match report.order_quantity.as_deref() {
Some(q) => match Decimal::from_str(q) {
Ok(v) => v,
Err(e) => {
warn!(%exchange, %symbol, qty = q, error = %e, "Binance order report unparseable quantity (q), dropping snapshot");
return None;
}
},
None => {
warn!(%exchange, %symbol, "Binance order report missing quantity (q), dropping snapshot");
return None;
}
};
let time_in_force = parse_time_in_force(report.time_in_force.as_deref().unwrap_or("GTC"));
let filled_qty = report
.cumulative_filled_quantity
.as_deref()
.and_then(|s| Decimal::from_str(s).ok())
.unwrap_or(Decimal::ZERO);
let order = Order {
key: OrderKey::new(
exchange,
symbol,
StrategyId::unknown(), cid,
),
side,
price,
quantity,
kind,
time_in_force,
state: OrderState::active(Open::new(
VenueOrderId::Assigned(order_id),
time_exchange,
filled_qty,
)),
};
Some(UnindexedAccountEvent::new(
exchange,
AccountEventKind::OrderSnapshot(rustrade_integration::collection::snapshot::Snapshot::new(
order,
)),
))
}
fn contains_ignore_case(haystack: &str, needle: &str) -> bool {
haystack
.as_bytes()
.windows(needle.len())
.any(|w| w.eq_ignore_ascii_case(needle.as_bytes()))
}
pub(crate) fn contains_error_code(msg: &str, code: &str) -> bool {
let code_len = code.len();
let mut start = 0;
while let Some(rel) = msg[start..].find(code) {
let pos = start + rel;
let prefix_ok = pos == 0 || !msg[..pos].ends_with(|c: char| c.is_ascii_digit());
let suffix_ok = !msg[pos + code_len..].starts_with(|c: char| c.is_ascii_digit());
if prefix_ok && suffix_ok {
return true;
}
start = pos + 1;
}
false
}
fn is_auth_failure(msg: &str) -> bool {
contains_error_code(msg, "-1002")
|| contains_error_code(msg, "-2015")
|| contains_error_code(msg, "-1022")
|| contains_error_code(msg, "-2014")
|| contains_ignore_case(msg, "invalid api-key")
|| contains_ignore_case(msg, "invalid signature")
|| contains_ignore_case(msg, "signature for this request is not valid")
}
fn has_venue_failure_code(msg: &str) -> bool {
["-1000", "-1001", "-1006", "-1007", "-1008"]
.iter()
.any(|code| contains_error_code(msg, code))
}
fn has_rate_limit_error_code(msg: &str) -> bool {
contains_error_code(msg, "-1003") || contains_error_code(msg, "-1015")
}
const BORROW_REJECT_CODES: [(&str, BorrowRejectReason); 5] = [
("-3006", BorrowRejectReason::BorrowLimitExceeded),
("-3008", BorrowRejectReason::BorrowDisabled),
("-3012", BorrowRejectReason::BorrowDisabled),
("-3014", BorrowRejectReason::BorrowDisabled),
("-3045", BorrowRejectReason::InventoryUnavailable),
];
fn borrow_reject_code(msg: &str) -> Option<(&'static str, BorrowRejectReason)> {
BORROW_REJECT_CODES
.iter()
.copied()
.find(|(code, _)| contains_error_code(msg, code))
}
pub(crate) fn parse_binance_api_error(
error_msg: String,
instrument: &InstrumentNameExchange,
) -> ApiError<AssetNameExchange, InstrumentNameExchange> {
if is_auth_failure(&error_msg) {
return ApiError::Unauthenticated(error_msg);
}
if has_rate_limit_error_code(&error_msg) {
return ApiError::rate_limited(None);
}
if contains_error_code(&error_msg, "-2011") {
return ApiError::OrderAlreadyCancelled;
}
if contains_error_code(&error_msg, "-2013") {
return ApiError::OrderAlreadyCancelled;
}
if contains_error_code(&error_msg, "-1121") {
return ApiError::InstrumentInvalid(instrument.clone(), error_msg);
}
if let Some((code, reason)) = borrow_reject_code(&error_msg) {
return ApiError::BorrowRejected(BorrowReject::new(
reason,
Some(code.to_owned()),
error_msg,
));
}
if contains_error_code(&error_msg, "-3041") || contains_error_code(&error_msg, "-3023") {
return ApiError::BalanceInsufficient(None, error_msg);
}
if contains_ignore_case(&error_msg, "duplicate order sent") {
ApiError::DuplicateClientOrderId(error_msg)
} else if contains_ignore_case(&error_msg, "insufficient")
|| contains_ignore_case(&error_msg, "not enough")
{
ApiError::BalanceInsufficient(None, error_msg)
} else if contains_ignore_case(&error_msg, "rate limit") {
ApiError::rate_limited(None)
} else if contains_ignore_case(&error_msg, "unknown order") {
ApiError::OrderRejected(error_msg)
} else if contains_ignore_case(&error_msg, "invalid symbol") {
ApiError::InstrumentInvalid(instrument.clone(), error_msg)
} else {
ApiError::OrderRejected(error_msg)
}
}
pub(crate) fn parse_binance_order_rejection(
msg: String,
instrument: &InstrumentNameExchange,
) -> UnindexedOrderError {
if has_venue_failure_code(&msg) {
OrderError::Connectivity(ConnectivityError::Socket(msg))
} else if contains_error_code(&msg, "-3111") {
OrderError::UnsupportedOrderType(msg)
} else {
OrderError::Rejected(parse_binance_api_error(msg, instrument))
}
}
fn parse_binance_query_rejection(
msg: String,
instrument: Option<&InstrumentNameExchange>,
) -> UnindexedClientError {
let api = if is_auth_failure(&msg) {
ApiError::Unauthenticated(msg)
} else if has_rate_limit_error_code(&msg) {
ApiError::rate_limited(None)
} else if has_venue_failure_code(&msg) || contains_error_code(&msg, "-1021") {
return UnindexedClientError::Connectivity(ConnectivityError::Socket(msg));
} else if let Some(instrument) = instrument.filter(|_| contains_error_code(&msg, "-1121")) {
ApiError::InstrumentInvalid(instrument.clone(), msg)
} else if let Some(instrument) = instrument {
ApiError::RequestRejected(format!("{}: {msg}", instrument.name()))
} else {
ApiError::RequestRejected(msg)
};
UnindexedClientError::Api(api)
}
pub(crate) fn classify_rest_query_error(
e: &anyhow::Error,
instrument: Option<&InstrumentNameExchange>,
) -> UnindexedClientError {
if let Some(cooldown) = e.downcast_ref::<CooldownActive>() {
return UnindexedClientError::Api(ApiError::rate_limited(Some(cooldown.remaining)));
}
if let Some(build) = e.downcast_ref::<ParamBuildError>() {
return UnindexedClientError::Internal(format!("building Binance request: {build}"));
}
if let Some(ce) = e.downcast_ref::<ConnectorError>() {
return match classify_connector_error(ce) {
RequestFailure::RateLimited(retry_after) => {
UnindexedClientError::Api(ApiError::rate_limited(retry_after))
}
RequestFailure::Unauthenticated(msg) => {
UnindexedClientError::Api(ApiError::Unauthenticated(msg))
}
RequestFailure::Transport(msg) => {
UnindexedClientError::Connectivity(ConnectivityError::Socket(msg))
}
RequestFailure::Rejected(msg) => parse_binance_query_rejection(msg, instrument),
};
}
let msg = format!("{e:#}");
if is_auth_failure(&msg) {
return UnindexedClientError::Api(ApiError::Unauthenticated(msg));
}
UnindexedClientError::Internal(msg)
}
pub(crate) fn is_unknown_order(e: &anyhow::Error) -> bool {
matches!(
e.downcast_ref::<ConnectorError>(),
Some(
ConnectorError::BadRequestError {
code: Some(-2013 | -1121),
..
} | ConnectorError::NotFoundError {
code: Some(-2013 | -1121),
..
} | ConnectorError::ConnectorClientError {
code: Some(-2013 | -1121),
..
}
)
)
}
pub(crate) fn response_decode_error(e: ConnectorError) -> UnindexedClientError {
UnindexedClientError::Internal(format!("decoding Binance response: {e}"))
}
pub(crate) async fn rest_call_with_retry<R, D>(
rest: &Arc<R>,
rate_limiter: &RateLimitTracker,
kind: RequestKind,
mut make_call: impl FnMut(
Arc<R>,
) -> Pin<
Box<dyn std::future::Future<Output = anyhow::Result<RestApiResponse<D>>> + Send>,
>,
) -> anyhow::Result<RestApiResponse<D>>
where
R: Send + Sync,
{
for attempt in 0..=MAX_RATE_LIMIT_RETRIES {
rate_limiter.wait_if_blocked(kind).await?;
let sent_ms = unix_ms();
match make_call(Arc::clone(rest)).await {
Ok(response) => {
rate_limiter.observe_rest(&response, sent_ms);
return Ok(response);
}
Err(e) if is_rate_limit_error(&e) => {
let refusal = e.downcast_ref::<ConnectorError>();
let venue_wait = refusal.and_then(rest_retry_after);
let banned = matches!(refusal, Some(ConnectorError::RateLimitBanError { .. }));
let backoff = Duration::from_secs(2u64.saturating_pow(attempt).min(30));
let delay = venue_wait.map_or(backoff, |wait| wait.max(backoff));
if attempt < MAX_RATE_LIMIT_RETRIES && !banned && delay <= MAX_COOLDOWN_SLEEP {
warn!(
attempt = attempt.saturating_add(1),
max = MAX_RATE_LIMIT_RETRIES,
delay_ms = u64::try_from(delay.as_millis()).unwrap_or(u64::MAX),
"Binance REST rate-limited, retrying"
);
rate_limiter.on_rate_limited(Some(delay));
} else {
warn!(
attempt = attempt.saturating_add(1),
banned,
venue_wait_ms = venue_wait
.map(|wait| u64::try_from(wait.as_millis()).unwrap_or(u64::MAX)),
"Binance REST rate-limited, not retrying"
);
rate_limiter.on_rate_limited(venue_wait);
return Err(e);
}
}
Err(e) => return Err(e),
}
}
unreachable!("Binance REST retries exhausted: loop invariant violated")
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum BinanceOrderType {
Market,
Limit,
LimitMaker,
StopLoss,
StopLossLimit,
TakeProfit,
TakeProfitLimit,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum BinanceTimeInForce {
Gtc,
Ioc,
Fok,
}
pub(crate) fn classify_order_kind_tif(
kind: OrderKind,
tif: TimeInForce,
) -> Option<(BinanceOrderType, Option<BinanceTimeInForce>)> {
match kind {
OrderKind::Market => Some((BinanceOrderType::Market, None)),
OrderKind::Limit => match tif {
TimeInForce::GoodUntilCancelled { post_only: false } => {
Some((BinanceOrderType::Limit, Some(BinanceTimeInForce::Gtc)))
}
TimeInForce::GoodUntilCancelled { post_only: true } => {
Some((BinanceOrderType::LimitMaker, None))
}
TimeInForce::FillOrKill => {
Some((BinanceOrderType::Limit, Some(BinanceTimeInForce::Fok)))
}
TimeInForce::ImmediateOrCancel => {
Some((BinanceOrderType::Limit, Some(BinanceTimeInForce::Ioc)))
}
TimeInForce::GoodUntilEndOfDay
| TimeInForce::GoodTillDate { .. }
| TimeInForce::AtOpen
| TimeInForce::AtClose => {
warn!(time_in_force = ?tif, "Binance does not support this TimeInForce");
None
}
},
OrderKind::Stop { .. } => Some((BinanceOrderType::StopLoss, None)),
OrderKind::TrailingStopFixedLimit {
offset_type: TrailingOffsetType::Absolute,
..
} => {
warn!("{ABSOLUTE_TRAILING_OFFSET_UNSUPPORTED}");
None
}
OrderKind::StopLimit { .. } | OrderKind::TrailingStopFixedLimit { .. } => match tif {
TimeInForce::GoodUntilCancelled { post_only: false } => Some((
BinanceOrderType::StopLossLimit,
Some(BinanceTimeInForce::Gtc),
)),
TimeInForce::FillOrKill => Some((
BinanceOrderType::StopLossLimit,
Some(BinanceTimeInForce::Fok),
)),
TimeInForce::ImmediateOrCancel => Some((
BinanceOrderType::StopLossLimit,
Some(BinanceTimeInForce::Ioc),
)),
_ => {
warn!(time_in_force = ?tif, "Binance StopLimit does not support this TimeInForce");
None
}
},
OrderKind::TakeProfit { .. } => Some((BinanceOrderType::TakeProfit, None)),
OrderKind::TakeProfitLimit { .. } => match tif {
TimeInForce::GoodUntilCancelled { post_only: false } => Some((
BinanceOrderType::TakeProfitLimit,
Some(BinanceTimeInForce::Gtc),
)),
TimeInForce::FillOrKill => Some((
BinanceOrderType::TakeProfitLimit,
Some(BinanceTimeInForce::Fok),
)),
TimeInForce::ImmediateOrCancel => Some((
BinanceOrderType::TakeProfitLimit,
Some(BinanceTimeInForce::Ioc),
)),
_ => {
warn!(time_in_force = ?tif, "Binance TakeProfitLimit does not support this TimeInForce");
None
}
},
OrderKind::TrailingStop { offset_type, .. } => match offset_type {
TrailingOffsetType::BasisPoints | TrailingOffsetType::Percentage => {
Some((BinanceOrderType::StopLoss, None))
}
TrailingOffsetType::Absolute => {
warn!("{ABSOLUTE_TRAILING_OFFSET_UNSUPPORTED}");
None
}
},
OrderKind::TrailingStopLimit { .. } => {
warn!("Binance does not support TrailingStopLimit orders");
None
}
}
}
const ABSOLUTE_TRAILING_OFFSET_UNSUPPORTED: &str = "Binance TrailingStop does not support an \
Absolute offset; convert it to basis points: (absolute / price) * 10000";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum TrailingDeltaError {
AbsoluteOffset,
Invalid(PrecisionViolation),
}
impl TrailingDeltaError {
pub(crate) fn into_order_error<AssetKey, InstrumentKey>(
self,
) -> OrderError<AssetKey, InstrumentKey> {
match self {
Self::AbsoluteOffset => {
OrderError::UnsupportedOrderType(ABSOLUTE_TRAILING_OFFSET_UNSUPPORTED.to_owned())
}
Self::Invalid(violation) => OrderError::InvalidPrecision(violation),
}
}
}
pub(crate) fn trailing_delta_basis_points(
offset: Decimal,
offset_type: TrailingOffsetType,
) -> Result<i32, TrailingDeltaError> {
let invalid = |limit| {
TrailingDeltaError::Invalid(PrecisionViolation::new(
OrderField::TrailingOffset,
offset,
limit,
))
};
let (basis_points, max_decimal_places) = match offset_type {
TrailingOffsetType::BasisPoints => (Some(offset), 0),
TrailingOffsetType::Percentage => (offset.checked_mul(Decimal::ONE_HUNDRED), 2),
TrailingOffsetType::Absolute => return Err(TrailingDeltaError::AbsoluteOffset),
};
if offset.is_sign_negative() || offset.is_zero() {
return Err(invalid(PrecisionLimit::NotPositive));
}
let Some(basis_points) = basis_points else {
return Err(invalid(PrecisionLimit::NotRepresentable));
};
if !basis_points.fract().is_zero() {
return Err(invalid(PrecisionLimit::DecimalPlaces {
max: max_decimal_places,
}));
}
i32::try_from(basis_points).map_err(|_| invalid(PrecisionLimit::NotRepresentable))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct TrailingParams {
pub trailing_delta: i32,
pub price: Option<Decimal>,
pub stop_price: Option<Decimal>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum TrailingOrderError {
Delta(TrailingDeltaError),
NonPositiveActivation(Decimal),
LimitPrice(Option<Decimal>),
}
impl TrailingOrderError {
pub(crate) fn into_order_error<AssetKey, InstrumentKey>(
self,
) -> OrderError<AssetKey, InstrumentKey> {
match self {
Self::Delta(refused) => refused.into_order_error(),
Self::NonPositiveActivation(price) => OrderError::Rejected(ApiError::OrderRejected(
format!("a trailing stop's activation_price must be positive, got {price}"),
)),
Self::LimitPrice(price) => OrderError::Rejected(ApiError::OrderRejected(format!(
"TrailingStopFixedLimit requires a positive price, its fixed limit, got {price:?}"
))),
}
}
}
pub(crate) fn trailing_params(
kind: &OrderKind,
price: Option<Decimal>,
) -> Result<Option<TrailingParams>, TrailingOrderError> {
let positive = |value: &Decimal| value.is_sign_positive() && !value.is_zero();
let (offset, offset_type, activation_price, fixed_limit) = match *kind {
OrderKind::TrailingStop {
offset,
offset_type,
activation_price,
} => (offset, offset_type, activation_price, false),
OrderKind::TrailingStopFixedLimit {
offset,
offset_type,
activation_price,
} => (offset, offset_type, activation_price, true),
OrderKind::Market
| OrderKind::Limit
| OrderKind::Stop { .. }
| OrderKind::StopLimit { .. }
| OrderKind::TakeProfit { .. }
| OrderKind::TakeProfitLimit { .. }
| OrderKind::TrailingStopLimit { .. } => return Ok(None),
};
let trailing_delta =
trailing_delta_basis_points(offset, offset_type).map_err(TrailingOrderError::Delta)?;
if let Some(activation) = activation_price.filter(|activation| !positive(activation)) {
return Err(TrailingOrderError::NonPositiveActivation(activation));
}
let price = if fixed_limit {
match price.filter(positive) {
Some(limit) => Some(limit),
None => return Err(TrailingOrderError::LimitPrice(price)),
}
} else {
None
};
Ok(Some(TrailingParams {
trailing_delta,
price,
stop_price: activation_price,
}))
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum RequestFailure {
RateLimited(Option<Duration>),
Unauthenticated(String),
Transport(String),
Rejected(String),
}
fn with_code(msg: &str, code: Option<i64>) -> String {
code.map_or_else(|| msg.to_owned(), |c| format!("{c} {msg}"))
}
fn is_waf_block(msg: &str) -> bool {
!is_auth_failure(msg)
}
fn classify_forbidden(msg: String) -> RequestFailure {
if is_waf_block(&msg) {
const LOG_EXCERPT_CHARS: usize = 256;
let excerpt = msg
.char_indices()
.nth(LOG_EXCERPT_CHARS)
.map_or(msg.as_str(), |(end, _)| &msg[..end]);
warn!(body = %excerpt, "Binance answered 403 (WAF limit); reporting RateLimit");
RequestFailure::RateLimited(None)
} else {
RequestFailure::Unauthenticated(msg)
}
}
fn classify_ws_response_error(
code: i64,
retry_after: Option<Duration>,
msg: String,
) -> RequestFailure {
match code {
429 | 418 | -1003 | -1015 => RequestFailure::RateLimited(retry_after),
401 => RequestFailure::Unauthenticated(msg),
403 => classify_forbidden(msg),
500..=599 => RequestFailure::Transport(msg),
_ => RequestFailure::Rejected(msg),
}
}
fn classify_connector_error(ce: &ConnectorError) -> RequestFailure {
match ce {
ConnectorError::TooManyRequestsError { .. } | ConnectorError::RateLimitBanError { .. } => {
RequestFailure::RateLimited(rest_retry_after(ce))
}
ConnectorError::UnauthorizedError { msg, code } => {
RequestFailure::Unauthenticated(with_code(msg, *code))
}
ConnectorError::ForbiddenError { msg, code } => classify_forbidden(with_code(msg, *code)),
ConnectorError::ServerError { msg, .. } | ConnectorError::NetworkError(msg) => {
RequestFailure::Transport(msg.clone())
}
ConnectorError::BadRequestError { msg, code }
| ConnectorError::NotFoundError { msg, code }
| ConnectorError::ConnectorClientError { msg, code } => {
const TRANSPORT_PREFIXES: [&str; 4] = [
"HTTP request failed",
"Failed to get response bytes",
"Failed to decompress gzip response",
"Failed to convert response to UTF-8",
];
if code.is_none() && TRANSPORT_PREFIXES.iter().any(|p| msg.starts_with(p)) {
RequestFailure::Transport(msg.clone())
} else {
RequestFailure::Rejected(with_code(msg, *code))
}
}
}
}
pub(crate) fn classify_rest_order_error(
e: &anyhow::Error,
instrument: &InstrumentNameExchange,
) -> OrderError<AssetNameExchange, InstrumentNameExchange> {
if let Some(cooldown) = e.downcast_ref::<CooldownActive>() {
return OrderError::Rejected(ApiError::rate_limited(Some(cooldown.remaining)));
}
let Some(ce) = e.downcast_ref::<ConnectorError>() else {
return OrderError::Connectivity(ConnectivityError::Socket(format!("{e:#}")));
};
order_error_from(classify_connector_error(ce), instrument)
}
pub(crate) fn classify_ws_order_error(
e: &anyhow::Error,
instrument: &InstrumentNameExchange,
) -> Option<UnindexedOrderError> {
ws_response_failure(e).map(|failure| order_error_from(failure, instrument))
}
fn ws_response_failure(e: &anyhow::Error) -> Option<RequestFailure> {
match e.downcast_ref::<WebsocketError>()? {
response @ WebsocketError::ResponseError {
code, retry_after, ..
} => Some(classify_ws_response_error(
*code,
retry_after
.map(|until_ms| wait_until_epoch_ms(until_ms, unix_ms()))
.filter(|wait| !wait.is_zero()),
response.to_string(),
)),
_ => None,
}
}
fn order_error_from(
failure: RequestFailure,
instrument: &InstrumentNameExchange,
) -> UnindexedOrderError {
match failure {
RequestFailure::RateLimited(retry_after) => {
OrderError::Rejected(ApiError::rate_limited(retry_after))
}
RequestFailure::Unauthenticated(msg) => {
OrderError::Rejected(ApiError::Unauthenticated(msg))
}
RequestFailure::Transport(msg) => OrderError::Connectivity(ConnectivityError::Socket(msg)),
RequestFailure::Rejected(msg) => parse_binance_order_rejection(msg, instrument),
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)] mod tests {
use super::*;
use crate::client::reconnect_budget::BackoffLimits;
use futures::FutureExt as _;
#[test]
fn the_heartbeat_and_cooldown_are_as_documented() {
assert_eq!(HEARTBEAT_TIMEOUT_SECS, 30);
assert_eq!(MAX_COOLDOWN_SLEEP, Duration::from_secs(10));
}
#[derive(Clone, Default)]
struct CountingWs {
closes: Arc<AtomicU32>,
stall: Arc<std::sync::atomic::AtomicBool>,
}
impl BinanceWsConnection for CountingWs {
async fn close(&self) -> anyhow::Result<()> {
self.closes.fetch_add(1, Ordering::SeqCst);
if self.stall.load(Ordering::SeqCst) {
std::future::pending::<()>().await;
}
Ok(())
}
}
impl CountingWs {
async fn closes(&self) -> u32 {
for _ in 0..10 {
tokio::task::yield_now().await;
}
self.closes.load(Ordering::SeqCst)
}
}
#[tokio::test]
async fn a_dropped_guard_closes_its_connection() {
let ws = CountingWs::default();
drop(DisconnectOnDrop::new(ws.clone(), "test"));
assert_eq!(ws.closes().await, 1);
}
#[tokio::test]
async fn a_disconnected_guard_does_not_close_again_on_drop() {
let ws = CountingWs::default();
let result = DisconnectOnDrop::new(ws.clone(), "test").disconnect().await;
assert!(result.is_ok());
assert_eq!(ws.closes().await, 1);
}
#[tokio::test]
async fn an_abandoned_disconnect_closes_again_on_drop() {
let ws = CountingWs::default();
ws.stall.store(true, Ordering::SeqCst);
let disconnect = DisconnectOnDrop::new(ws.clone(), "test").disconnect();
assert!(
tokio::time::timeout(Duration::from_millis(10), disconnect)
.await
.is_err()
);
assert_eq!(ws.closes().await, 2);
}
#[tokio::test]
async fn a_dropped_subscription_guard_stops_its_callback() {
let emitter = binance_sdk::common::websocket::WebsocketEventEmitter::new();
let callback_state = Arc::new(());
let held = Arc::clone(&callback_state);
let subscription = emitter.subscribe(move |_| {
let _ = &held;
});
drop(UnsubscribeOnDrop::new(subscription));
for _ in 0..10 {
tokio::task::yield_now().await;
}
assert_eq!(
Arc::strong_count(&callback_state),
1,
"the callback was dropped"
);
}
fn alive() -> usize {
tokio::runtime::Handle::current()
.metrics()
.num_alive_tasks()
}
async fn local_ws_api_config() -> binance_sdk::common::config::ConfigurationWebsocketApi {
let (url, _) = super::test_ws_api::serve(
|_| serde_json::Value::Null,
|_| super::test_ws_api::Lifetime::Open,
)
.await;
binance_sdk::common::config::ConfigurationWebsocketApi::builder()
.api_key("key")
.api_secret("secret")
.ws_url(url)
.build()
.unwrap()
}
async fn assert_tasks_return_to(before: usize) {
let settled = tokio::time::timeout(Duration::from_secs(5), async {
while alive() > before {
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await;
assert!(
settled.is_ok(),
"tasks left behind: {} of {before}",
alive()
);
assert_eq!(alive(), before);
}
#[tokio::test]
async fn a_disconnected_connection_leaves_no_task_behind() {
let config = local_ws_api_config().await;
let before = alive();
let session = binance_sdk::spot::SpotWsApi::from_config(config.clone())
.connect()
.await
.unwrap();
DisconnectOnDrop::new(session, "spot")
.disconnect()
.await
.unwrap();
let stream = binance_sdk::common::websocket::WebsocketApi::new(config, vec![]);
Arc::clone(&stream).connect().await.unwrap();
DisconnectOnDrop::new(stream, "margin")
.disconnect()
.await
.unwrap();
assert_tasks_return_to(before).await;
}
#[tokio::test]
async fn a_dropped_connection_leaves_no_task_behind() {
let config = local_ws_api_config().await;
let before = alive();
let session = binance_sdk::spot::SpotWsApi::from_config(config.clone())
.connect()
.await
.unwrap();
drop(DisconnectOnDrop::new(session, "spot"));
let stream = binance_sdk::common::websocket::WebsocketApi::new(config, vec![]);
Arc::clone(&stream).connect().await.unwrap();
let callback_state = Arc::new(());
let held = Arc::clone(&callback_state);
let _subscription = stream.common.events.subscribe(move |_| {
let _ = &held;
});
drop(stream);
assert_tasks_return_to(before).await;
assert_eq!(
Arc::strong_count(&callback_state),
1,
"the callback was dropped"
);
}
#[test]
fn a_guard_dropped_outside_a_runtime_does_not_panic() {
let ws = CountingWs::default();
drop(DisconnectOnDrop::new(ws.clone(), "test"));
assert_eq!(ws.closes.load(Ordering::SeqCst), 0);
}
#[test]
fn unhandled_events_warn_once_per_type_and_every_thousandth() {
let seen = UnhandledEvents::new();
assert_eq!(seen.record("frequent"), (1, true));
for count in 2..=10 {
assert_eq!(seen.record("frequent"), (count, false));
}
assert_eq!(
seen.record("renamed"),
(11, true),
"a new type warns despite the first"
);
assert_eq!(seen.record("renamed"), (12, false));
for count in 13..1000 {
assert_eq!(seen.record("frequent"), (count, false));
}
assert_eq!(seen.record("frequent"), (1000, true));
let capped = UnhandledEvents::new();
for i in 0..UnhandledEvents::MAX_TYPES {
assert!(capped.record(&format!("type{i}")).1);
}
assert!(!capped.record("one_too_many").1);
}
fn report(json: &str) -> ExecutionReport {
match serde_json::from_str(json) {
Ok(report) => report,
Err(error) => panic!("{json} should parse: {error}"),
}
}
#[test]
fn a_report_reads_an_absent_null_or_present_commission_asset() {
assert_eq!(report(r#"{"e":"executionReport"}"#).commission_asset, None);
assert_eq!(report(r#"{"N":null}"#).commission_asset, None);
assert_eq!(
report(r#"{"N":"BNB"}"#).commission_asset.as_deref(),
Some("BNB")
);
}
#[test]
fn a_report_reads_the_trailing_delta_as_a_number_or_a_string() {
assert_eq!(report(r#"{}"#).trailing_delta, None);
assert_eq!(report(r#"{"d":null}"#).trailing_delta, None);
assert_eq!(report(r#"{"d":25}"#).trailing_delta, Some(WireInt::Int(25)));
assert_eq!(
report(r#"{"d":"25"}"#).trailing_delta,
Some(WireInt::Int(25))
);
assert_eq!(
report(r#"{"d":"x"}"#).trailing_delta,
Some(WireInt::Unreadable("x".to_owned()))
);
assert_eq!(
report(r#"{"d":true}"#).trailing_delta,
Some(WireInt::Unreadable("true".to_owned()))
);
assert_eq!(
report(r#"{"d":[1],"x":"NEW"}"#).trailing_delta,
Some(WireInt::Unreadable("an array".to_owned()))
);
assert_eq!(
report(r#"{"d":{"a":1},"x":"NEW"}"#)
.execution_type
.as_deref(),
Some("NEW")
);
assert_eq!(
report(r#"{"d":2.5}"#).trailing_delta,
Some(WireInt::Unreadable("2.5".to_owned()))
);
}
#[test]
fn keys_the_converter_does_not_read_cannot_fail_a_report() {
for (unread, delta) in [
(
r#""D":1668680518494,"j":1,"J":1000000,"u":2,"U":3,"a":5,"v":3"#,
"4",
),
(
r#""D":"1668680518494","j":"1","J":"1000000","u":"2","U":"3","a":"5","v":"3""#,
r#""4""#,
),
(
r#""D":null,"j":null,"J":null,"u":null,"U":null,"a":null,"v":null"#,
"4",
),
] {
let json = format!(
r#"{{"e":"executionReport","x":"NEW","i":7,"o":"STOP_LOSS","P":"0","d":{delta},{unread},"W":1499405658658,"V":"NONE"}}"#
);
let parsed = report(&json);
assert_eq!(parsed.execution_type.as_deref(), Some("NEW"), "{json}");
assert_eq!(parsed.order_id, Some(7), "{json}");
assert_eq!(parsed.trailing_delta, Some(WireInt::Int(4)), "{json}");
}
}
#[test]
fn a_reported_trailing_delta_reads_absent_when_missing_or_zero() {
assert_eq!(TrailingDelta::from_reported(None), TrailingDelta::Absent);
assert_eq!(TrailingDelta::from_reported(Some(0)), TrailingDelta::Absent);
assert_eq!(
TrailingDelta::from_reported(Some(25)),
TrailingDelta::BasisPoints(25)
);
assert_eq!(
TrailingDelta::from_reported(Some(-1)),
TrailingDelta::Unknown
);
}
fn decode(
order_type: &str,
stop_price: Option<&str>,
trailing_delta: TrailingDelta,
) -> Option<OrderKind> {
decode_order_kind(
ExchangeId::BinanceSpot,
Some(1),
order_type,
stop_price,
trailing_delta,
)
}
#[test]
fn plain_types_decode_whatever_the_trailing_delta() {
for delta in [
TrailingDelta::Absent,
TrailingDelta::BasisPoints(25),
TrailingDelta::Unknown,
] {
assert_eq!(decode("MARKET", None, delta), Some(OrderKind::Market));
assert_eq!(decode("LIMIT", None, delta), Some(OrderKind::Limit));
assert_eq!(decode("LIMIT_MAKER", None, delta), Some(OrderKind::Limit));
}
assert_eq!(decode("UNKNOWN_TYPE", None, TrailingDelta::Absent), None);
}
#[test]
fn a_conditional_order_without_a_trailing_delta_decodes_with_its_trigger() {
let trigger_price = Decimal::from(100);
let cases = [
("STOP_LOSS", OrderKind::Stop { trigger_price }),
("STOP_LOSS_LIMIT", OrderKind::StopLimit { trigger_price }),
("TAKE_PROFIT", OrderKind::TakeProfit { trigger_price }),
(
"TAKE_PROFIT_LIMIT",
OrderKind::TakeProfitLimit { trigger_price },
),
];
for (order_type, kind) in cases {
assert_eq!(
decode(order_type, Some("100.00"), TrailingDelta::Absent),
Some(kind),
"{order_type}"
);
for stop_price in [None, Some("0.00000000"), Some("-1"), Some("x")] {
assert_eq!(
decode(order_type, stop_price, TrailingDelta::Absent),
None,
"{order_type} {stop_price:?}"
);
}
}
}
#[test]
fn a_trailing_stop_loss_decodes_with_its_activation_price() {
use rust_decimal_macros::dec;
for (stop_price, activation_price) in [
(None, None),
(Some("0.00000000"), None),
(Some("105.5"), Some(dec!(105.5))),
] {
assert_eq!(
decode("STOP_LOSS", stop_price, TrailingDelta::BasisPoints(25)),
Some(OrderKind::TrailingStop {
offset: Decimal::from(25),
offset_type: TrailingOffsetType::BasisPoints,
activation_price,
}),
"{stop_price:?}"
);
assert_eq!(
decode(
"STOP_LOSS_LIMIT",
stop_price,
TrailingDelta::BasisPoints(25)
),
Some(OrderKind::TrailingStopFixedLimit {
offset: Decimal::from(25),
offset_type: TrailingOffsetType::BasisPoints,
activation_price,
}),
"{stop_price:?}"
);
}
}
#[test]
fn conditional_orders_that_cannot_be_described_are_dropped() {
for order_type in ["STOP_LOSS", "STOP_LOSS_LIMIT"] {
assert_eq!(
decode(order_type, Some("x"), TrailingDelta::BasisPoints(25)),
None,
"{order_type}"
);
}
for order_type in ["TAKE_PROFIT", "TAKE_PROFIT_LIMIT"] {
assert_eq!(
decode(order_type, Some("100"), TrailingDelta::BasisPoints(25)),
None,
"{order_type}"
);
}
for order_type in [
"STOP_LOSS",
"STOP_LOSS_LIMIT",
"TAKE_PROFIT",
"TAKE_PROFIT_LIMIT",
] {
assert_eq!(
decode(order_type, Some("100"), TrailingDelta::Unknown),
None,
"{order_type}"
);
}
}
#[test]
fn a_trailing_offset_converts_to_whole_basis_points() {
use rust_decimal_macros::dec;
let ok = |offset, offset_type| trailing_delta_basis_points(offset, offset_type);
assert_eq!(ok(dec!(25), TrailingOffsetType::BasisPoints), Ok(25));
assert_eq!(ok(dec!(25.0), TrailingOffsetType::BasisPoints), Ok(25));
assert_eq!(ok(dec!(1.5), TrailingOffsetType::Percentage), Ok(150));
assert_eq!(ok(dec!(0.15), TrailingOffsetType::Percentage), Ok(15));
assert_eq!(
ok(dec!(1), TrailingOffsetType::Absolute),
Err(TrailingDeltaError::AbsoluteOffset)
);
}
#[test]
fn a_trailing_offset_that_is_not_whole_positive_basis_points_is_refused() {
use rust_decimal_macros::dec;
let refused = |offset, offset_type, limit| {
assert_eq!(
trailing_delta_basis_points(offset, offset_type),
Err(TrailingDeltaError::Invalid(PrecisionViolation::new(
OrderField::TrailingOffset,
offset,
limit
))),
"{offset} {offset_type:?}"
);
};
refused(
dec!(0.155),
TrailingOffsetType::Percentage,
PrecisionLimit::DecimalPlaces { max: 2 },
);
refused(
dec!(2.5),
TrailingOffsetType::BasisPoints,
PrecisionLimit::DecimalPlaces { max: 0 },
);
for offset_type in [
TrailingOffsetType::BasisPoints,
TrailingOffsetType::Percentage,
] {
refused(dec!(0), offset_type, PrecisionLimit::NotPositive);
refused(dec!(-10), offset_type, PrecisionLimit::NotPositive);
}
refused(
Decimal::from(i64::from(i32::MAX) + 1),
TrailingOffsetType::BasisPoints,
PrecisionLimit::NotRepresentable,
);
refused(
Decimal::MAX,
TrailingOffsetType::Percentage,
PrecisionLimit::NotRepresentable,
);
}
#[test]
fn a_trailing_offset_refusal_is_an_order_error() {
use rust_decimal_macros::dec;
let violation = PrecisionViolation::new(
OrderField::TrailingOffset,
dec!(2.5),
PrecisionLimit::NotPositive,
);
assert_eq!(
TrailingDeltaError::Invalid(violation).into_order_error::<(), ()>(),
OrderError::InvalidPrecision(violation)
);
assert!(matches!(
TrailingDeltaError::AbsoluteOffset.into_order_error::<(), ()>(),
OrderError::UnsupportedOrderType(_)
));
}
fn order_row(order_type: &str, status: &str, extra: serde_json::Value) -> serde_json::Value {
let mut row = serde_json::json!({
"symbol": "BTCUSDT", "orderId": 7, "clientOrderId": "cid-7", "price": "0.00000000",
"origQty": "2", "executedQty": "0", "cummulativeQuoteQty": "0", "status": status,
"timeInForce": "GTC", "type": order_type, "side": "SELL",
"stopPrice": "0.00000000", "time": 1_700_000_000_000_i64,
"updateTime": 1_700_000_060_000_i64,
});
if let (Some(row), serde_json::Value::Object(extra)) = (row.as_object_mut(), extra) {
row.extend(extra);
}
row
}
fn spot_row(row: serde_json::Value) -> binance_sdk::spot::rest_api::GetOpenOrdersResponseInner {
match serde_json::from_value(row) {
Ok(row) => row,
Err(error) => panic!("spot row should parse: {error}"),
}
}
fn margin_row(
row: serde_json::Value,
) -> binance_sdk::margin_trading::rest_api::QueryMarginAccountsOpenOrdersResponseInner {
match serde_json::from_value(row) {
Ok(row) => row,
Err(error) => panic!("margin row should parse: {error}"),
}
}
#[test]
fn a_trailing_stop_loss_limit_row_lists_with_its_limit_price() {
let btc = InstrumentNameExchange::new("BTCUSDT");
for (stop_price, activation_price) in
[("0.00000000", None), ("105", Some(Decimal::from(105)))]
{
let rows = [spot_row(order_row(
"STOP_LOSS_LIMIT",
"NEW",
serde_json::json!({
"trailingDelta": 25, "trailingTime": -1, "price": "95", "stopPrice": stop_price,
}),
))];
let listing = convert_open_order_listing(&rows, ExchangeId::BinanceSpot, &btc);
let [order] = listing.orders.as_slice() else {
panic!("one order: {:?}", listing.orders);
};
assert_eq!(
order.kind,
OrderKind::TrailingStopFixedLimit {
offset: Decimal::from(25),
offset_type: TrailingOffsetType::BasisPoints,
activation_price,
}
);
assert_eq!(order.price, Some(Decimal::from(95)));
}
}
#[test]
fn trailing_params_carry_the_delta_limit_and_activation_and_refuse_bad_prices() {
use rust_decimal_macros::dec;
let fixed_limit = |activation_price| OrderKind::TrailingStopFixedLimit {
offset: dec!(1.5),
offset_type: TrailingOffsetType::Percentage,
activation_price,
};
assert_eq!(trailing_params(&OrderKind::Limit, Some(dec!(1))), Ok(None));
assert_eq!(
trailing_params(&fixed_limit(Some(dec!(105))), Some(dec!(99))),
Ok(Some(TrailingParams {
trailing_delta: 150,
price: Some(dec!(99)),
stop_price: Some(dec!(105)),
}))
);
assert_eq!(
trailing_params(
&OrderKind::TrailingStop {
offset: dec!(25),
offset_type: TrailingOffsetType::BasisPoints,
activation_price: None,
},
Some(dec!(99))
),
Ok(Some(TrailingParams {
trailing_delta: 25,
price: None,
stop_price: None,
})),
"a trailing stop sends no price"
);
for activation in [dec!(0), dec!(-1)] {
assert_eq!(
trailing_params(&fixed_limit(Some(activation)), Some(dec!(99))),
Err(TrailingOrderError::NonPositiveActivation(activation))
);
}
for limit in [None, Some(dec!(0))] {
assert_eq!(
trailing_params(&fixed_limit(None), limit),
Err(TrailingOrderError::LimitPrice(limit))
);
}
assert!(matches!(
trailing_params(
&OrderKind::TrailingStopFixedLimit {
offset: dec!(1.555),
offset_type: TrailingOffsetType::Percentage,
activation_price: Some(dec!(0)),
},
None
),
Err(TrailingOrderError::Delta(TrailingDeltaError::Invalid(_)))
));
assert!(matches!(
TrailingOrderError::LimitPrice(None).into_order_error::<(), ()>(),
OrderError::Rejected(ApiError::OrderRejected(_))
));
assert!(matches!(
TrailingOrderError::NonPositiveActivation(dec!(0)).into_order_error::<(), ()>(),
OrderError::Rejected(ApiError::OrderRejected(_))
));
assert!(matches!(
TrailingOrderError::Delta(TrailingDeltaError::AbsoluteOffset)
.into_order_error::<(), ()>(),
OrderError::UnsupportedOrderType(_)
));
}
#[test]
fn spot_rows_list_conditional_and_trailing_orders() {
let btc = InstrumentNameExchange::new("BTCUSDT");
let rows = [
spot_row(order_row(
"STOP_LOSS",
"NEW",
serde_json::json!({"trailingDelta": 25, "trailingTime": -1}),
)),
spot_row(order_row(
"STOP_LOSS_LIMIT",
"NEW",
serde_json::json!({"clientOrderId": "cid-8", "price": "95", "stopPrice": "96"}),
)),
spot_row(order_row(
"STOP_LOSS",
"NEW",
serde_json::json!({"clientOrderId": "cid-9", "stopPrice": "96"}),
)),
];
let listing = convert_open_order_listing(&rows, ExchangeId::BinanceSpot, &btc);
assert!(listing.complete);
let described: Vec<_> = listing
.orders
.iter()
.map(|order| (order.key.cid.0.as_str(), order.kind, order.price))
.collect();
assert_eq!(
described,
[
(
"cid-7",
OrderKind::TrailingStop {
offset: Decimal::from(25),
offset_type: TrailingOffsetType::BasisPoints,
activation_price: None,
},
None,
),
(
"cid-8",
OrderKind::StopLimit {
trigger_price: Decimal::from(96)
},
Some(Decimal::from(95)),
),
(
"cid-9",
OrderKind::Stop {
trigger_price: Decimal::from(96)
},
None,
),
]
);
}
#[test]
fn margin_listings_leave_out_conditional_rows_and_read_incomplete() {
let btc = InstrumentNameExchange::new("BTCUSDT");
for order_type in [
"STOP_LOSS",
"STOP_LOSS_LIMIT",
"TAKE_PROFIT",
"TAKE_PROFIT_LIMIT",
] {
let rows = [
margin_row(order_row(
"LIMIT",
"NEW",
serde_json::json!({"clientOrderId": "cid-1", "price": "95"}),
)),
margin_row(order_row(
order_type,
"NEW",
serde_json::json!({"price": "95", "stopPrice": "96"}),
)),
];
let listing = convert_open_order_listing(&rows, ExchangeId::BinanceMargin, &btc);
assert!(!listing.complete, "{order_type}");
assert_eq!(listing.orders.len(), 1, "{order_type}");
assert_eq!(listing.orders[0].kind, OrderKind::Limit);
}
}
#[test]
fn an_ended_margin_conditional_order_reports_its_ending() {
let key = UnindexedOrderKey::new(
ExchangeId::BinanceMargin,
InstrumentNameExchange::new("BTCUSDT"),
StrategyId::unknown(),
ClientOrderId::new("cid-7"),
);
let row = margin_row(order_row(
"STOP_LOSS",
"CANCELED",
serde_json::json!({"stopPrice": "96"}),
));
let Some(order) = convert_ended_order(&row, ExchangeId::BinanceMargin, &key, None) else {
panic!("an ended stop should be reported");
};
assert_eq!(
order.kind,
OrderKind::Stop {
trigger_price: Decimal::from(96)
}
);
assert!(matches!(order.state, InactiveOrderState::Cancelled(_)));
let untriggered = margin_row(order_row("STOP_LOSS", "CANCELED", serde_json::json!({})));
assert!(convert_ended_order(&untriggered, ExchangeId::BinanceMargin, &key, None).is_none());
let trailing_stop = OrderKind::TrailingStop {
offset: Decimal::from(25),
offset_type: TrailingOffsetType::BasisPoints,
activation_price: None,
};
for row in [&row, &untriggered] {
let Some(order) =
convert_ended_order(row, ExchangeId::BinanceMargin, &key, Some(trailing_stop))
else {
panic!("an ended order held with its kind should be reported");
};
assert_eq!(order.kind, trailing_stop);
assert_eq!(order.price, None);
}
let limit = margin_row(order_row(
"LIMIT",
"CANCELED",
serde_json::json!({"price": "95"}),
));
let spot_stop = spot_row(order_row(
"STOP_LOSS",
"CANCELED",
serde_json::json!({"stopPrice": "96"}),
));
let read = [
convert_ended_order(&limit, ExchangeId::BinanceMargin, &key, Some(trailing_stop)),
convert_ended_order(
&spot_stop,
ExchangeId::BinanceSpot,
&key,
Some(trailing_stop),
),
]
.map(|order| order.map(|order| (order.kind, order.price)));
assert_eq!(
read,
[
Some((OrderKind::Limit, Some(Decimal::from(95)))),
Some((
OrderKind::Stop {
trigger_price: Decimal::from(96)
},
None
)),
]
);
}
#[tokio::test]
async fn a_fill_gap_is_retried_with_backoff_then_given_up() {
tokio::time::pause();
let btc = InstrumentNameExchange::new("BTCUSDT");
let eth = InstrumentNameExchange::new("ETHUSDT");
let disconnect = Utc.timestamp_millis_opt(1_000_000).unwrap();
let reconnect = Utc.timestamp_millis_opt(2_000_000).unwrap();
let mut unrecovered = UnrecoveredFills::default();
unrecovered.open(&[btc.clone(), eth.clone()], disconnect, reconnect);
let due = unrecovered.due(tokio::time::Instant::now());
assert_eq!(due.len(), 2);
let gap_of = |instrument: &InstrumentNameExchange| {
let Some((_, gap)) = due.iter().find(|(inst, _)| inst == instrument) else {
panic!("{instrument} has a gap");
};
*gap
};
let gap = gap_of(&btc);
assert_eq!(gap.start_ms, 1_000_000);
assert_eq!(gap.end_ms, 2_000_000 + GAP_END_SLACK_SECS * 1_000);
unrecovered.recovered(ð, &gap_of(ð));
for retry in 0..MAX_GAP_RETRIES {
let delay = Duration::from_secs(GAP_RETRY_BASE_SECS << retry);
let now = tokio::time::Instant::now();
assert_eq!(
unrecovered.failed(&btc, &gap, now),
Some(GapFailure::Retry(delay))
);
assert!(unrecovered.due(now).is_empty(), "not due before its delay");
assert_eq!(unrecovered.next_due(), Some(now + delay));
tokio::time::advance(delay).await;
assert_eq!(unrecovered.due(tokio::time::Instant::now()).len(), 1);
}
assert_eq!(
unrecovered.failed(&btc, &gap, tokio::time::Instant::now()),
Some(GapFailure::GivenUp)
);
assert!(unrecovered.is_empty());
assert_eq!(unrecovered.next_due(), None);
assert_eq!(
unrecovered.failed(&btc, &gap, tokio::time::Instant::now()),
None
);
}
#[tokio::test]
async fn a_given_up_fill_gap_is_reported_on_the_stream() {
tokio::time::pause();
let btc = InstrumentNameExchange::new("BTCUSDT");
let disconnect = Utc.timestamp_millis_opt(1_000_000).unwrap();
let reconnect = Utc.timestamp_millis_opt(2_000_000).unwrap();
let mut unrecovered = UnrecoveredFills::default();
unrecovered.open(std::slice::from_ref(&btc), disconnect, reconnect);
let Some((_, gap)) = unrecovered.due(tokio::time::Instant::now()).pop() else {
panic!("the gap is due");
};
let (tx, mut rx) = mpsc::unbounded_channel();
for _ in 0..MAX_GAP_RETRIES {
gap_failed(
&mut unrecovered,
&tx,
ExchangeId::BinanceSpot,
&btc,
&gap,
FillRecoveryFailure::Request("rejected".to_string()),
);
assert!(
rx.try_recv().is_err(),
"a gap left to retry is not reported"
);
unrecovered.make_due();
}
gap_failed(
&mut unrecovered,
&tx,
ExchangeId::BinanceSpot,
&btc,
&gap,
FillRecoveryFailure::TimedOut { timeout_secs: 30 },
);
let Ok(event) = rx.try_recv() else {
panic!("the given-up gap is reported");
};
assert_eq!(event.exchange, ExchangeId::BinanceSpot);
assert_eq!(
event.kind,
AccountEventKind::FillRecoveryGaveUp(FillRecoveryGap::new(
FillRecoveryScope::Instruments(vec![btc]),
disconnect,
reconnect + chrono::Duration::seconds(GAP_END_SLACK_SECS),
MAX_GAP_RETRIES + 1,
FillRecoveryFailure::TimedOut { timeout_secs: 30 },
))
);
assert!(rx.try_recv().is_err(), "reported once");
assert!(unrecovered.is_empty(), "and no longer kept");
}
#[tokio::test]
async fn an_overlapping_fill_gap_starts_after_the_kept_one() {
tokio::time::pause();
let btc = InstrumentNameExchange::new("BTCUSDT");
let at = |secs: i64| Utc.timestamp_millis_opt(secs * 1_000).unwrap();
let mut unrecovered = UnrecoveredFills::default();
unrecovered.open(std::slice::from_ref(&btc), at(1_000), at(1_100));
let (_, first) = unrecovered.due(tokio::time::Instant::now())[0].clone();
unrecovered.failed(&btc, &first, tokio::time::Instant::now());
unrecovered.open(std::slice::from_ref(&btc), at(1_105), at(1_200));
let due = unrecovered.due(tokio::time::Instant::now());
assert_eq!(due.len(), 1, "the failed gap waits for its retry");
assert_eq!(due[0].1.start_ms, first.end_ms + 1);
assert_eq!(due[0].1.end_ms, (1_200 + GAP_END_SLACK_SECS) * 1_000);
unrecovered.open(std::slice::from_ref(&btc), at(1_150), at(1_150));
assert_eq!(unrecovered.due(tokio::time::Instant::now()).len(), 1);
tokio::time::advance(Duration::from_secs(GAP_RETRY_BASE_SECS)).await;
assert_eq!(unrecovered.due(tokio::time::Instant::now()).len(), 2);
}
#[test]
fn drop_after_keeps_only_the_span() {
let execution =
|id: i64, time: Option<i64>| binance_sdk::spot::rest_api::MyTradesResponseInner {
id: Some(id),
time,
..Default::default()
};
let mut page = vec![
execution(1, Some(100)),
execution(2, None),
execution(3, Some(200)),
execution(4, Some(201)),
];
assert!(drop_after(&mut page, 200));
assert_eq!(
page.iter().map(|e| e.id).collect::<Vec<_>>(),
[Some(1), Some(2), Some(3)]
);
assert!(!drop_after(&mut page, 200));
}
#[test]
fn parse_user_data_frame_splits_responses_events_and_unknown_shapes() {
assert_eq!(
parse_user_data_frame(r#"{"id":"abc","status":200,"result":{"subscriptionId":0}}"#),
UserDataFrame::Response
);
assert_eq!(
parse_user_data_frame(
r#"{"subscriptionId":4,"event":{"e":"outboundAccountPosition","E":1}}"#
),
UserDataFrame::Event {
subscription_id: Some(4),
event_type: "outboundAccountPosition",
event: r#"{"e":"outboundAccountPosition","E":1}"#,
}
);
assert_eq!(
parse_user_data_frame(r#"{"id":1,"subscriptionId":4,"event":{"e":"x"}}"#),
UserDataFrame::Response
);
for frame in [
r#"{"e":"executionReport","i":1}"#,
r#"{"subscriptionId":4,"event":{"E":1}}"#,
r#"{"subscriptionId":4,"event":{"e":7}}"#,
r#"{"subscriptionId":4,"event":{"e":"executionRep\u006frt"}}"#,
"not json",
"[]",
] {
assert_eq!(
parse_user_data_frame(frame),
UserDataFrame::Unrecognised,
"{frame}"
);
}
}
#[test]
fn frame_excerpt_cuts_at_200_characters_on_a_char_boundary() {
assert_eq!(frame_excerpt("short"), "short");
let exact = "a".repeat(200);
assert_eq!(frame_excerpt(&exact), exact);
let long = "é".repeat(300);
assert_eq!(frame_excerpt(&long).chars().count(), 200);
}
#[derive(Debug, Clone, Copy)]
struct Execution {
id: Option<i64>,
order_id: Option<i64>,
qty: Option<&'static str>,
}
impl BinanceExecutionFields for Execution {
fn id(&self) -> Option<i64> {
self.id
}
fn order_id(&self) -> Option<i64> {
self.order_id
}
fn qty(&self) -> Option<&str> {
self.qty
}
fn time(&self) -> Option<i64> {
None
}
}
fn execution(id: i64, order_id: i64, qty: &'static str) -> Execution {
Execution {
id: Some(id),
order_id: Some(order_id),
qty: Some(qty),
}
}
fn btcusdt() -> InstrumentNameExchange {
InstrumentNameExchange::new("BTCUSDT")
}
#[test]
fn running_totals_accumulate_in_trade_id_order_counting_a_repeat_once() {
let executions = [
execution(12, 7, "0.5"),
execution(10, 7, "1"),
execution(12, 7, "0.5"),
execution(15, 7, "2"),
];
assert_eq!(
order_running_totals(7, &executions),
Some(vec![
(10, Decimal::ONE),
(12, Decimal::new(15, 1)),
(15, Decimal::new(35, 1)),
])
);
}
#[test]
fn running_totals_refuse_a_walk_holding_another_orders_execution() {
let executions = [execution(10, 7, "1"), execution(11, 8, "5")];
assert_eq!(order_running_totals(7, &executions), None);
}
#[test]
fn running_totals_refuse_a_walk_with_an_unreadable_execution() {
let unsized_execution = Execution {
qty: None,
..execution(11, 7, "0")
};
let unparseable = execution(11, 7, "abc");
let unidentified = Execution {
id: None,
..execution(11, 7, "1")
};
for broken in [unsized_execution, unparseable, unidentified] {
assert_eq!(
order_running_totals(7, &[execution(10, 7, "1"), broken]),
None,
"{broken:?}",
);
}
}
#[tokio::test]
async fn a_recovered_fill_counts_what_its_order_filled_before_the_window() {
let recovered = [execution(12, 7, "2")];
let order_walk = vec![execution(10, 7, "1"), execution(12, 7, "2")];
let totals = recovered_order_totals(
ExchangeId::BinanceSpot,
&btcusdt(),
&recovered,
tokio::time::Instant::now() + Duration::from_secs(5),
|order_id| {
assert_eq!(order_id, 7);
let walk = order_walk.clone();
async move { Ok(walk) }
},
)
.await;
assert_eq!(totals.get(&12), Some(&Decimal::new(3, 0)));
}
#[tokio::test]
async fn an_order_whose_walk_fails_is_left_out_and_the_others_are_not() {
let recovered = [execution(20, 1, "1"), execution(21, 2, "1")];
let totals = recovered_order_totals(
ExchangeId::BinanceMargin,
&btcusdt(),
&recovered,
tokio::time::Instant::now() + Duration::from_secs(5),
|order_id| async move {
match order_id {
1 => Err(UnindexedClientError::Connectivity(
ConnectivityError::Socket("connection reset".into()),
)),
_ => Ok(vec![execution(21, 2, "1")]),
}
},
)
.await;
assert_eq!(totals.get(&20), None);
assert_eq!(totals.get(&21), Some(&Decimal::ONE));
}
#[tokio::test]
async fn a_lookup_past_its_deadline_sends_nothing() {
let recovered = [execution(20, 1, "1")];
let cids = recovered_order_cids(
ExchangeId::BinanceSpot,
&btcusdt(),
&recovered,
tokio::time::Instant::now(),
|_| async { panic!("no read past the deadline") },
)
.await;
assert!(cids.is_empty());
}
#[tokio::test]
async fn recovered_orders_are_read_for_their_client_order_ids() {
let recovered = [
execution(20, 1, "1"),
execution(21, 1, "1"),
execution(22, 2, "1"),
execution(23, 3, "1"),
];
let reads = std::sync::atomic::AtomicUsize::new(0);
let cids = recovered_order_cids(
ExchangeId::BinanceMargin,
&btcusdt(),
&recovered,
tokio::time::Instant::now() + Duration::from_secs(5),
|order_id| {
reads.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
async move {
match order_id {
1 => Ok(Some(ClientOrderId::new("sweep-1"))),
2 => Ok(None),
_ => Err(UnindexedClientError::Connectivity(
ConnectivityError::Socket("connection reset".into()),
)),
}
}
},
)
.await;
assert_eq!(reads.into_inner(), 3, "one read per order");
assert_eq!(cids.len(), 1, "{cids:?}");
assert_eq!(cids.get(&1), Some(&ClientOrderId::new("sweep-1")));
}
#[tokio::test(start_paused = true)]
async fn lookups_stop_at_the_deadline_keeping_the_walks_that_answered() {
let recovered = [execution(20, 1, "1"), execution(21, 2, "1")];
let started = tokio::time::Instant::now();
let deadline = started + Duration::from_secs(5);
let totals = recovered_order_totals(
ExchangeId::BinanceSpot,
&btcusdt(),
&recovered,
deadline,
|order_id| async move {
if order_id == 1 {
std::future::pending::<()>().await;
}
Ok(vec![execution(21, 2, "1")])
},
)
.await;
assert_eq!(totals.get(&20), None);
assert_eq!(totals.get(&21), Some(&Decimal::ONE));
assert_eq!(tokio::time::Instant::now(), deadline);
}
#[tokio::test(start_paused = true)]
async fn lookups_run_side_by_side_up_to_the_in_flight_bound() {
let recovered: Vec<Execution> = (1_i64..)
.take(ORDER_EXECUTIONS_IN_FLIGHT + 1)
.map(|order_id| execution(100 + order_id, order_id, "1"))
.collect();
let over_bound = recovered.last().and_then(|t| t.id);
let totals = recovered_order_totals(
ExchangeId::BinanceSpot,
&btcusdt(),
&recovered,
tokio::time::Instant::now() + Duration::from_secs(5),
|order_id| async move {
tokio::time::sleep(Duration::from_secs(4)).await;
Ok(vec![execution(100 + order_id, order_id, "1")])
},
)
.await;
assert_eq!(totals.len(), ORDER_EXECUTIONS_IN_FLIGHT, "{totals:?}");
assert!(
over_bound.is_some_and(|id| !totals.contains_key(&id)),
"{totals:?}"
);
}
fn classify(
msg: &str,
code: Option<i64>,
) -> OrderError<AssetNameExchange, InstrumentNameExchange> {
let err = anyhow::Error::new(ConnectorError::ConnectorClientError {
msg: msg.to_string(),
code,
});
classify_rest_order_error(&err, &InstrumentNameExchange::new("BTCUSDT"))
}
#[test]
fn codeless_transport_failures_map_to_connectivity() {
for msg in [
"HTTP request failed: connection reset",
"Failed to get response bytes: error reading body",
"Failed to decompress gzip response",
"Failed to convert response to UTF-8: invalid utf-8 sequence",
] {
assert!(
matches!(classify(msg, None), OrderError::Connectivity(_)),
"expected Connectivity for {msg:?}"
);
}
}
#[test]
fn coded_error_maps_to_rejection() {
assert!(matches!(
classify("Account has insufficient balance.", Some(-2010)),
OrderError::Rejected(_)
));
}
#[test]
fn codeless_non_transport_status_error_maps_to_rejection() {
assert!(matches!(
classify("Conflict", None),
OrderError::Rejected(_)
));
}
fn classify_err(ce: ConnectorError) -> OrderError<AssetNameExchange, InstrumentNameExchange> {
classify_rest_order_error(
&anyhow::Error::new(ce),
&InstrumentNameExchange::new("BTCUSDT"),
)
}
#[test]
fn rate_limit_variants_map_to_rejection_ratelimit() {
for ce in [
ConnectorError::TooManyRequestsError {
msg: "Too many requests.".to_string(),
code: Some(-1003),
retry_after: None,
},
ConnectorError::RateLimitBanError {
msg: "IP banned.".to_string(),
code: Some(-1003),
retry_after: None,
},
] {
assert!(matches!(
classify_err(ce),
OrderError::Rejected(ApiError::RateLimit { .. })
));
}
}
#[test]
fn auth_variants_map_to_unauthenticated() {
for ce in [
ConnectorError::UnauthorizedError {
msg: "bad key".to_string(),
code: Some(-2014),
},
ConnectorError::ForbiddenError {
msg: "Invalid API-key, IP, or permissions for action.".to_string(),
code: Some(-2015),
},
] {
assert!(matches!(
classify_err(ce),
OrderError::Rejected(ApiError::Unauthenticated(_))
));
}
}
#[test]
fn server_and_network_variants_map_to_connectivity() {
for ce in [
ConnectorError::ServerError {
msg: "internal error".to_string(),
status_code: Some(503),
},
ConnectorError::NetworkError("connection reset".to_string()),
] {
assert!(matches!(classify_err(ce), OrderError::Connectivity(_)));
}
}
fn classify_query(ce: ConnectorError) -> UnindexedClientError {
classify_rest_query_error(
&anyhow::Error::new(ce),
Some(&InstrumentNameExchange::new("BTCUSDT")),
)
}
fn bad_request(code: i64, msg: &str) -> ConnectorError {
ConnectorError::BadRequestError {
msg: msg.to_string(),
code: Some(code),
}
}
#[test]
fn a_query_the_venue_refuses_is_a_request_rejection_carrying_its_code() {
let err = classify_query(bad_request(
-1127,
"More than 24 hours between startTime and endTime.",
));
let UnindexedClientError::Api(ApiError::RequestRejected(msg)) = &err else {
panic!("expected RequestRejected, got {err:?}");
};
assert!(msg.contains("-1127"), "code missing from {msg:?}");
assert!(!err.is_transient());
}
#[test]
fn a_rejected_query_naming_one_instrument_says_which() {
let err = classify_query(bad_request(
-11001,
"Isolated margin account does not exist.",
));
let UnindexedClientError::Api(ApiError::RequestRejected(msg)) = &err else {
panic!("expected RequestRejected, got {err:?}");
};
assert!(msg.starts_with("BTCUSDT: "), "symbol missing from {msg:?}");
assert!(msg.contains("-11001"), "code missing from {msg:?}");
let err = classify_rest_query_error(
&anyhow::Error::new(bad_request(-1127, "More than 24 hours.")),
None,
);
let UnindexedClientError::Api(ApiError::RequestRejected(msg)) = &err else {
panic!("expected RequestRejected, got {err:?}");
};
assert!(!msg.contains("BTCUSDT"), "{msg:?}");
}
#[test]
fn a_query_naming_an_invalid_symbol_is_an_invalid_instrument() {
let err = classify_query(bad_request(-1121, "Invalid symbol."));
assert!(
matches!(
&err,
UnindexedClientError::Api(ApiError::InstrumentInvalid(instrument, _))
if instrument.name().as_str() == "BTCUSDT"
),
"got {err:?}"
);
let err = classify_rest_query_error(
&anyhow::Error::new(bad_request(-1121, "Invalid symbol.")),
None,
);
assert!(
matches!(err, UnindexedClientError::Api(ApiError::RequestRejected(_))),
"got {err:?}"
);
}
#[test]
fn order_codes_do_not_leak_into_query_classification() {
for code in [-2011, -2013] {
let err = classify_query(bad_request(code, "Unknown order sent."));
assert!(
matches!(err, UnindexedClientError::Api(ApiError::RequestRejected(_))),
"{code}: got {err:?}"
);
}
}
#[test]
fn query_rejections_with_auth_or_throttle_codes_keep_their_meaning() {
for (code, msg) in [
(-1002, "You are not authorized to execute this request."),
(-2015, "Invalid API-key, IP, or permissions for action."),
(-1022, "Signature for this request is not valid."),
(-2014, "API-key format invalid."),
] {
let err = classify_query(bad_request(code, msg));
assert!(
matches!(err, UnindexedClientError::Api(ApiError::Unauthenticated(_))),
"{code}: got {err:?}"
);
}
for code in [-1003, -1015] {
let err = classify_query(bad_request(code, "Too much request weight used."));
assert!(
matches!(err, UnindexedClientError::Api(ApiError::RateLimit { .. })),
"{code}: got {err:?}"
);
}
}
#[test]
fn query_rejections_a_retry_can_cure_stay_transient() {
for code in [-1000, -1001, -1006, -1007, -1008, -1021] {
let err = classify_query(bad_request(code, "Internal error; please try again."));
assert!(
matches!(err, UnindexedClientError::Connectivity(_)),
"{code}: got {err:?}"
);
assert!(err.is_transient());
}
}
#[test]
fn query_transport_server_and_throttle_failures_classify_like_orders() {
for ce in [
ConnectorError::ConnectorClientError {
msg: "HTTP request failed: connection reset".to_string(),
code: None,
},
ConnectorError::ServerError {
msg: "Server error: 503".to_string(),
status_code: Some(503),
},
ConnectorError::NetworkError("connection reset".to_string()),
] {
let err = classify_query(ce);
assert!(
matches!(err, UnindexedClientError::Connectivity(_)),
"got {err:?}"
);
assert!(err.is_transient());
}
let err = classify_query(ConnectorError::TooManyRequestsError {
msg: "Too many requests.".to_string(),
code: Some(-1003),
retry_after: None,
});
assert!(matches!(
err,
UnindexedClientError::Api(ApiError::RateLimit { .. })
));
let err = classify_query(ConnectorError::UnauthorizedError {
msg: "API-key format invalid.".to_string(),
code: Some(-2014),
});
let UnindexedClientError::Api(ApiError::Unauthenticated(msg)) = &err else {
panic!("expected Unauthenticated, got {err:?}");
};
assert!(msg.contains("-2014"), "code missing from {msg:?}");
}
#[test]
fn a_request_the_client_failed_to_build_is_internal() {
let err = classify_rest_query_error(
&anyhow::Error::new(ParamBuildError::UninitializedField("symbol")),
None,
);
assert!(
matches!(&err, UnindexedClientError::Internal(msg) if msg.contains("symbol")),
"got {err:?}"
);
assert!(!err.is_transient());
}
#[test]
fn a_response_body_that_does_not_decode_is_internal() {
let err = response_decode_error(ConnectorError::ConnectorClientError {
msg: "invalid type: map, expected a sequence at line 1 column 0".to_string(),
code: None,
});
assert!(
matches!(&err, UnindexedClientError::Internal(msg) if msg.contains("invalid type")),
"got {err:?}"
);
assert!(!err.is_transient());
}
#[test]
fn untyped_query_errors_are_classified_by_their_text() {
for msg in [
"Error -1002: unauthorized",
"Error -2015: permission denied for action",
"invalid signature provided",
"The signature for this request is not valid.",
"Invalid API-key format",
] {
let err = classify_rest_query_error(&anyhow::anyhow!("{msg}"), None);
assert!(
matches!(err, UnindexedClientError::Api(ApiError::Unauthenticated(_))),
"{msg:?}: got {err:?}"
);
}
let err = classify_rest_query_error(
&anyhow::anyhow!("relative URL without a base").context("Failed to join base URL"),
None,
);
assert!(
matches!(err, UnindexedClientError::Internal(_)),
"got {err:?}"
);
assert!(!err.is_transient());
}
#[test]
fn a_forbidden_query_is_an_auth_failure_only_when_it_says_so() {
let err = classify_query(ConnectorError::ForbiddenError {
msg: "<html>403 Forbidden</html>".to_string(),
code: None,
});
assert!(
matches!(err, UnindexedClientError::Api(ApiError::RateLimit { .. })),
"got {err:?}"
);
let err = classify_query(ConnectorError::ForbiddenError {
msg: "Invalid API-key, IP, or permissions for action.".to_string(),
code: Some(-2015),
});
assert!(
matches!(err, UnindexedClientError::Api(ApiError::Unauthenticated(_))),
"got {err:?}"
);
}
#[test]
fn query_classification_reads_the_sdk_error_through_added_context() {
let err = classify_rest_query_error(
&anyhow::Error::new(bad_request(-1127, "More than 24 hours.")).context("outer"),
None,
);
assert!(
matches!(err, UnindexedClientError::Api(ApiError::RequestRejected(_))),
"got {err:?}"
);
}
#[test]
fn an_order_forbidden_by_the_waf_is_a_rate_limit() {
assert!(matches!(
classify_err(ConnectorError::ForbiddenError {
msg: "<html>403 Forbidden</html>".to_string(),
code: None,
}),
OrderError::Rejected(ApiError::RateLimit { .. })
));
}
#[test]
fn an_order_the_venue_failed_has_an_unknown_status() {
for code in [-1000, -1001, -1006, -1007, -1008] {
let err = classify_err(ConnectorError::BadRequestError {
msg: "Timeout waiting for response from backend server.".to_string(),
code: Some(code),
});
assert!(
matches!(err, OrderError::Connectivity(_)),
"{code}: got {err:?}"
);
assert!(err.is_transient());
}
let err = parse_binance_order_rejection(
"Server\u{2010}side response error (code -1007): Timeout waiting for response from \
backend server. Send status unknown; execution status unknown."
.to_string(),
&InstrumentNameExchange::new("BTCUSDT"),
);
assert!(matches!(err, OrderError::Connectivity(_)), "got {err:?}");
}
#[test]
fn an_order_with_a_bad_signature_or_key_format_is_unauthenticated() {
for (code, msg) in [
(-1022, "Signature for this request is not valid."),
(-2014, "API-key format invalid."),
] {
let err = classify_err(ConnectorError::BadRequestError {
msg: msg.to_string(),
code: Some(code),
});
assert!(
matches!(err, OrderError::Rejected(ApiError::Unauthenticated(_))),
"{code}: got {err:?}"
);
}
}
#[test]
fn a_definitive_order_rejection_is_still_a_rejection() {
let err = classify_err(ConnectorError::BadRequestError {
msg: "Account has insufficient balance for requested action.".to_string(),
code: Some(-2010),
});
assert!(
matches!(
err,
OrderError::Rejected(ApiError::BalanceInsufficient(None, _))
),
"got {err:?}"
);
}
#[test]
fn a_2010_rejection_is_read_by_its_message() {
let cases: [(&str, fn(&UnindexedOrderError) -> bool); 4] = [
(
"Account has insufficient balance for requested action.",
|err| {
matches!(
err,
OrderError::Rejected(ApiError::BalanceInsufficient(None, _))
)
},
),
("Order would trigger immediately.", |err| {
matches!(err, OrderError::Rejected(ApiError::OrderRejected(_)))
}),
("Duplicate order sent.", |err| {
matches!(
err,
OrderError::Rejected(ApiError::DuplicateClientOrderId(_))
)
}),
(
"Trailing stop orders are not supported for this symbol.",
|err| matches!(err, OrderError::Rejected(ApiError::OrderRejected(_))),
),
];
for (msg, expected) in cases {
let rest = classify_err(ConnectorError::BadRequestError {
msg: msg.to_string(),
code: Some(-2010),
});
assert!(expected(&rest), "REST {msg:?}: got {rest:?}");
let ws = classify_ws(-2010, msg);
assert!(
ws.as_ref().is_some_and(expected),
"WS API {msg:?}: got {ws:?}"
);
}
}
#[test]
fn a_borrow_refusal_is_read_by_its_code() {
for (code, msg, reason) in [
(
-3006,
"Your borrow amount has exceed maximum borrow amount.",
BorrowRejectReason::BorrowLimitExceeded,
),
(
-3008,
"Borrow not allowed.",
BorrowRejectReason::BorrowDisabled,
),
(
-3012,
"Borrow is banned for this asset.",
BorrowRejectReason::BorrowDisabled,
),
(
-3014,
"Borrow is banned for this account.",
BorrowRejectReason::BorrowDisabled,
),
(
-3045,
"The system doesn't have enough asset now.",
BorrowRejectReason::InventoryUnavailable,
),
] {
let rest = classify_err(ConnectorError::BadRequestError {
msg: msg.to_string(),
code: Some(code),
});
let expected =
BorrowReject::new(reason, Some(code.to_string()), format!("{code} {msg}"));
assert_eq!(
rest,
OrderError::Rejected(ApiError::BorrowRejected(expected)),
"REST {code}"
);
let ws = classify_ws(code, msg);
assert!(
matches!(
&ws,
Some(OrderError::Rejected(ApiError::BorrowRejected(reject)))
if reject.reason == reason
&& reject.venue_code.as_deref() == Some(code.to_string().as_str())
),
"WS API {code}: got {ws:?}"
);
}
}
#[test]
fn a_trailing_stop_limit_only_refusal_is_an_unsupported_order_type() {
let msg = "Trailing stop order only supports limit type.";
let rest = classify_err(ConnectorError::BadRequestError {
msg: msg.to_string(),
code: Some(-3111),
});
assert_eq!(
rest,
OrderError::UnsupportedOrderType(format!("-3111 {msg}"))
);
assert!(!rest.is_transient());
let ws = classify_ws(-3111, msg);
assert!(
matches!(
&ws,
Some(OrderError::UnsupportedOrderType(text))
if text.contains("-3111") && text.contains(msg)
),
"WS API: got {ws:?}"
);
}
#[test]
fn other_margin_codes_are_not_borrow_refusals() {
let instrument = InstrumentNameExchange::new("BTCUSDT");
for msg in [
"-3041 Balance is not enough",
"-3023 You can't transfer out/place order under current margin level.",
] {
assert_eq!(
parse_binance_api_error(msg.to_string(), &instrument),
ApiError::BalanceInsufficient(None, msg.to_string()),
"{msg:?}"
);
}
for msg in [
"-3013 Borrow amount less than minimum borrow amount.",
"-3021 Margin account are not allowed to trade this trading pair.",
"-3022 You account's trading is banned.",
] {
assert_eq!(
parse_binance_api_error(msg.to_string(), &instrument),
ApiError::OrderRejected(msg.to_string()),
"{msg:?}"
);
}
}
#[test]
fn order_rejections_keep_their_order_meaning() {
let instrument = InstrumentNameExchange::new("BTCUSDT");
for (msg, expected) in [
(
"-2013 Order does not exist.",
ApiError::OrderAlreadyCancelled,
),
("-2011 Unknown order sent.", ApiError::OrderAlreadyCancelled),
(
"-1021 Timestamp for this request is outside of the recvWindow.",
ApiError::OrderRejected(
"-1021 Timestamp for this request is outside of the recvWindow.".to_string(),
),
),
(
"-1102 Duplicate values for a parameter detected.",
ApiError::OrderRejected(
"-1102 Duplicate values for a parameter detected.".to_string(),
),
),
(
"Invalid API-key, IP, or permissions for action.",
ApiError::Unauthenticated(
"Invalid API-key, IP, or permissions for action.".to_string(),
),
),
] {
assert_eq!(
parse_binance_order_rejection(msg.to_string(), &instrument),
OrderError::Rejected(expected),
"{msg:?}"
);
}
}
fn classify_ws(code: i64, message: &str) -> Option<UnindexedOrderError> {
classify_ws_order_error(
&anyhow::Error::new(WebsocketError::ResponseError {
code,
message: message.to_string(),
retry_after: None,
}),
&InstrumentNameExchange::new("BTCUSDT"),
)
}
#[test]
fn a_dropped_connection_names_its_cause() {
assert_eq!(disconnect_error(true), "heartbeat timeout (30s)");
assert_eq!(
disconnect_error(false),
"WebSocket disconnected (server close/error)"
);
}
#[tokio::test(start_paused = true)]
async fn a_reconnect_ends_at_once_on_a_cooldown_too_long_to_wait_through() {
let tracker = RateLimitTracker::new(WeightPool::Spot);
tracker.on_rate_limited(Some(Duration::from_secs(60)));
let mut budget = ReconnectBudget::new("Binance", BackoffLimits::ACCOUNT_STREAM);
let started = tokio::time::Instant::now();
let ended = StreamTerminationReason::RateLimited {
retry_after: Duration::from_secs(60),
};
let reason = retry_or_end(&mut budget, &tracker, || {
unreachable!("the budget is not spent")
})
.await;
assert_eq!(reason, Err(ended.clone()));
assert_eq!(tracker.before_stream_connect().await, Err(ended));
assert_eq!(started.elapsed(), Duration::ZERO);
assert_eq!(budget.attempts(), 0);
}
#[tokio::test(start_paused = true)]
async fn a_short_cooldown_is_waited_out_without_spending_an_attempt() {
let tracker = RateLimitTracker::new(WeightPool::Spot);
tracker.on_rate_limited(Some(Duration::from_secs(5)));
let mut budget = ReconnectBudget::new("Binance", BackoffLimits::ACCOUNT_STREAM);
let started = tokio::time::Instant::now();
assert_eq!(
retry_or_end(&mut budget, &tracker, String::new).await,
Ok(())
);
assert_eq!(started.elapsed(), BackoffLimits::ACCOUNT_STREAM.initial);
assert_eq!(tracker.before_stream_connect().await, Ok(()));
assert_eq!(started.elapsed(), Duration::from_secs(5));
assert_eq!(budget.attempts(), 1);
}
#[tokio::test(start_paused = true)]
async fn refusals_naming_no_wait_spend_the_budget() {
let tracker = RateLimitTracker::new(WeightPool::Spot);
let mut budget = ReconnectBudget::new("Binance", BackoffLimits::ACCOUNT_STREAM);
let reason = loop {
assert_eq!(tracker.before_stream_connect().await, Ok(()));
tracker.on_rate_limited(None);
if let Err(reason) = retry_or_end(&mut budget, &tracker, || "refused".to_string()).await
{
break reason;
}
};
assert_eq!(
reason,
StreamTerminationReason::ReconnectBudgetExhausted {
attempts: BackoffLimits::ACCOUNT_STREAM.max_attempts,
last_error: "refused".to_string(),
}
);
}
#[tokio::test]
async fn only_a_rate_limit_refusal_of_a_stream_connect_starts_a_cooldown() {
let handshake = |msg: &str| anyhow::Error::new(WebsocketError::Handshake(msg.into()));
let response = |code, retry_after| {
anyhow::Error::new(WebsocketError::ResponseError {
code,
message: "refused".to_string(),
retry_after,
})
.context("subscribing")
};
let about = |tracker: &RateLimitTracker, wait: Duration| {
tracker
.cooldown_remaining()
.is_some_and(|left| left <= wait && left > wait - Duration::from_secs(5))
};
let tracker = RateLimitTracker::new(WeightPool::Spot);
assert!(!tracker.on_stream_refused(&handshake("HTTP error: 503 Service Unavailable")));
let unrelated = response(-2010, None);
assert!(!tracker.on_stream_refused(&unrelated));
assert!(!tracker.on_stream_refused(&anyhow::anyhow!("connection reset")));
assert_eq!(tracker.cooldown_remaining(), None);
assert!(tracker.on_stream_refused(&handshake("HTTP error: 429 Too Many Requests")));
let default = Duration::from_secs(DEFAULT_RATE_LIMIT_DELAY_SECS);
assert!(
about(&tracker, default),
"{:?}",
tracker.cooldown_remaining()
);
let tracker = RateLimitTracker::new(WeightPool::Spot);
assert!(tracker.on_stream_refused(&handshake("HTTP error: 418 I'm a teapot")));
assert!(
about(&tracker, MIN_IP_BAN),
"{:?}",
tracker.cooldown_remaining()
);
let tracker = RateLimitTracker::new(WeightPool::Spot);
let until = i64::try_from(unix_ms()).unwrap() + 120_000;
assert!(tracker.on_stream_refused(&response(-1003, Some(until))));
let ban = Duration::from_millis(120_000 + CLOCK_SKEW_SLACK_MS);
assert!(about(&tracker, ban), "{:?}", tracker.cooldown_remaining());
}
#[tokio::test]
async fn opening_a_stream_reports_a_rate_limit_with_the_cooldown_left() {
let tracker = RateLimitTracker::new(WeightPool::Spot);
let socket = |e: &anyhow::Error| {
UnindexedClientError::Connectivity(ConnectivityError::Socket(e.to_string()))
};
let other = anyhow::anyhow!("connection reset");
assert_eq!(tracker.stream_open_error(&other, socket), socket(&other));
assert_eq!(tracker.before_stream_open().await, Ok(()));
let refused = anyhow::Error::new(WebsocketError::Handshake(
"HTTP error: 429 Too Many Requests".into(),
));
let error = tracker.stream_open_error(&refused, socket);
assert!(
error
.retry_after()
.is_some_and(|wait| wait > Duration::from_secs(DEFAULT_RATE_LIMIT_DELAY_SECS - 5)),
"{error:?}"
);
tracker.on_rate_limited(Some(Duration::from_secs(60)));
let error = tracker.before_stream_open().await.unwrap_err();
assert!(
error
.retry_after()
.is_some_and(|wait| wait > Duration::from_secs(55)),
"{error:?}"
);
}
#[test]
fn a_ws_response_without_an_error_body_is_read_by_its_status() {
let err = classify_ws(503, "Unknown error");
assert!(
matches!(err, Some(OrderError::Connectivity(_))),
"a 5xx leaves the order's status unknown: {err:?}"
);
for code in [429, 418, 403] {
let err = classify_ws(code, "Unknown error");
assert!(
matches!(err, Some(OrderError::Rejected(ApiError::RateLimit { .. }))),
"{code}: got {err:?}"
);
}
let err = classify_ws(401, "Unknown error");
assert!(
matches!(
err,
Some(OrderError::Rejected(ApiError::Unauthenticated(_)))
),
"got {err:?}"
);
let err = classify_ws(409, "Unknown error");
assert!(
matches!(err, Some(OrderError::Rejected(ApiError::OrderRejected(_)))),
"got {err:?}"
);
}
#[test]
fn a_ws_response_with_a_binance_code_is_read_by_its_code() {
let err = classify_ws(-1007, "Timeout waiting for response from backend server.");
assert!(
matches!(err, Some(OrderError::Connectivity(_))),
"got {err:?}"
);
let wrapped = anyhow::Error::new(WebsocketError::ResponseError {
code: -1007,
message: "Timeout waiting for response from backend server.".to_string(),
retry_after: None,
})
.context("outer");
let err = classify_ws_order_error(&wrapped, &InstrumentNameExchange::new("BTCUSDT"));
assert!(
matches!(err, Some(OrderError::Connectivity(_))),
"got {err:?}"
);
let err = classify_ws(-1003, "Too much request weight used.");
assert!(
matches!(err, Some(OrderError::Rejected(ApiError::RateLimit { .. }))),
"got {err:?}"
);
}
#[test]
fn a_waf_403_is_backed_off_as_a_rate_limit_but_an_auth_403_is_not() {
let waf = anyhow::Error::new(ConnectorError::ForbiddenError {
msg: "<html>403 Forbidden</html>".to_string(),
code: None,
});
assert!(is_rate_limit_error(&waf));
let auth = anyhow::Error::new(ConnectorError::ForbiddenError {
msg: "Invalid API-key, IP, or permissions for action.".to_string(),
code: Some(-2015),
});
assert!(!is_rate_limit_error(&auth));
}
fn ws_weight(limit: u32, count: u32) -> Vec<WebsocketApiRateLimit> {
vec![
WebsocketApiRateLimit {
rate_limit_type: RateLimitType::Orders,
interval: Interval::Second,
interval_num: 10,
limit: 100,
count: 100,
},
WebsocketApiRateLimit {
rate_limit_type: RateLimitType::RequestWeight,
interval: Interval::Minute,
interval_num: 1,
limit,
count,
},
]
}
async fn waits(tracker: &RateLimitTracker, kind: RequestKind) -> bool {
tokio::time::timeout(Duration::from_millis(1), tracker.wait_if_blocked(kind))
.await
.is_err()
}
#[test]
fn pause_after_ends_just_past_the_send_minute() {
let minute = 29_000_000 * 60_000_u128;
let slack = CLOCK_SKEW_SLACK_MS;
let ms = |ms: u64| Some(Duration::from_millis(ms));
assert_eq!(
pause_after(minute + 15_000, minute + 15_000),
ms(45_000 + slack)
);
assert_eq!(pause_after(minute, minute + 100), ms(59_900 + slack));
assert_eq!(pause_after(minute + 59_999, minute + 59_999), ms(1 + slack));
assert_eq!(pause_after(minute - 100, minute + 400), ms(slack - 400));
assert_eq!(pause_after(minute - 100, minute + u128::from(slack)), None);
assert_eq!(pause_after(minute - 100, minute + 30_000), None);
}
#[tokio::test]
async fn weight_near_the_limit_pauses_queries_but_not_orders() {
tokio::time::pause();
let tracker = RateLimitTracker::new(WeightPool::Spot);
tracker.observe_ws_api(Some(&ws_weight(1_000, 899)), unix_ms());
assert!(!waits(&tracker, RequestKind::Query).await, "below 90%");
tracker.observe_ws_api(Some(&ws_weight(1_000, 900)), unix_ms());
assert!(waits(&tracker, RequestKind::Query).await, "at 90%");
assert!(!waits(&tracker, RequestKind::Order).await);
assert!(
!tracker.is_blocked(),
"a pause is not a rate-limit cooldown"
);
tokio::time::advance(Duration::from_millis(60_000 + CLOCK_SKEW_SLACK_MS)).await;
assert!(!waits(&tracker, RequestKind::Query).await, "the pause ends");
}
#[tokio::test]
async fn essential_reads_skip_the_pause_but_not_a_cooldown() {
tokio::time::pause();
let tracker = RateLimitTracker::new(WeightPool::Spot);
tracker.throttle(Duration::from_secs(30));
assert!(waits(&tracker, RequestKind::Query).await);
assert!(!waits(&tracker, RequestKind::Essential).await);
tracker.on_rate_limited(Some(Duration::from_secs(5)));
assert!(waits(&tracker, RequestKind::Essential).await);
}
#[tokio::test]
async fn a_rate_limit_cooldown_holds_back_both_kinds() {
tokio::time::pause();
let tracker = RateLimitTracker::new(WeightPool::Spot);
tracker.on_rate_limited(Some(Duration::from_secs(5)));
assert!(tracker.is_blocked());
assert!(waits(&tracker, RequestKind::Order).await);
assert!(waits(&tracker, RequestKind::Query).await);
tokio::time::advance(Duration::from_secs(5)).await;
assert!(!tracker.is_blocked());
assert!(!waits(&tracker, RequestKind::Order).await);
}
#[tokio::test]
async fn a_cooldown_longer_than_the_sleep_limit_fails_at_once() {
tokio::time::pause();
let tracker = RateLimitTracker::new(WeightPool::Spot);
tracker.on_rate_limited(Some(MAX_COOLDOWN_SLEEP));
assert!(waits(&tracker, RequestKind::Order).await, "at the limit");
let longer = MAX_COOLDOWN_SLEEP + Duration::from_millis(1);
tracker.on_rate_limited(Some(longer));
for kind in [
RequestKind::Order,
RequestKind::Query,
RequestKind::Essential,
] {
assert_eq!(
tracker.wait_if_blocked(kind).now_or_never(),
Some(Err(CooldownActive { remaining: longer })),
"{kind:?}"
);
}
assert_eq!(tracker.cooldown_remaining(), Some(longer));
}
#[tokio::test]
async fn a_cooldown_extended_past_the_limit_while_sleeping_fails() {
tokio::time::pause();
let tracker = Arc::new(RateLimitTracker::new(WeightPool::Spot));
tracker.on_rate_limited(Some(Duration::from_secs(5)));
let waiter = tokio::spawn({
let tracker = Arc::clone(&tracker);
async move { tracker.wait_if_blocked(RequestKind::Order).await }
});
tokio::time::advance(Duration::from_secs(1)).await;
tracker.on_rate_limited(Some(Duration::from_secs(120)));
tokio::time::advance(Duration::from_secs(4)).await;
let remaining = waiter.await.unwrap().unwrap_err().remaining;
assert!(
remaining <= Duration::from_secs(116)
&& remaining >= Duration::from_secs(116) - Duration::from_millis(1),
"{remaining:?}"
);
}
#[tokio::test]
async fn a_long_weight_pause_is_slept_through() {
tokio::time::pause();
let tracker = RateLimitTracker::new(WeightPool::Spot);
tracker.throttle(Duration::from_secs(30));
assert!(waits(&tracker, RequestKind::Query).await);
tokio::time::advance(Duration::from_secs(30)).await;
assert_eq!(
tracker.wait_if_blocked(RequestKind::Query).now_or_never(),
Some(Ok(()))
);
}
#[tokio::test]
async fn a_wait_beyond_the_clock_range_blocks_without_panicking() {
tokio::time::pause();
let tracker = RateLimitTracker::new(WeightPool::Spot);
tracker.on_rate_limited(Some(Duration::MAX));
assert_eq!(
tracker.cooldown_remaining(),
Some(Duration::from_secs(u64::from(u32::MAX)))
);
assert!(
tracker
.wait_if_blocked(RequestKind::Order)
.now_or_never()
.is_some_and(|waited| waited.is_err())
);
}
#[tokio::test]
async fn cooldown_remaining_reports_none_outside_a_cooldown() {
tokio::time::pause();
let tracker = RateLimitTracker::new(WeightPool::Spot);
assert_eq!(tracker.cooldown_remaining(), None);
tracker.on_rate_limited(None);
assert_eq!(
tracker.cooldown_remaining(),
Some(Duration::from_secs(DEFAULT_RATE_LIMIT_DELAY_SECS))
);
tokio::time::advance(Duration::from_secs(DEFAULT_RATE_LIMIT_DELAY_SECS)).await;
assert_eq!(tracker.cooldown_remaining(), None);
}
#[test]
fn wait_until_epoch_ms_runs_to_the_slack_past_the_time() {
let now = 1_700_000_000_000_u128;
let at = |offset: i128| i64::try_from(now.cast_signed() + offset).unwrap();
let ms = Duration::from_millis;
assert_eq!(
wait_until_epoch_ms(at(60_000), now),
ms(60_000 + CLOCK_SKEW_SLACK_MS)
);
assert_eq!(wait_until_epoch_ms(at(0), now), ms(CLOCK_SKEW_SLACK_MS));
assert_eq!(
wait_until_epoch_ms(at(-400), now),
ms(CLOCK_SKEW_SLACK_MS - 400)
);
let slack = i128::from(CLOCK_SKEW_SLACK_MS);
assert_eq!(wait_until_epoch_ms(at(-slack), now), Duration::ZERO);
assert_eq!(wait_until_epoch_ms(at(-60_000), now), Duration::ZERO);
assert_eq!(wait_until_epoch_ms(-1, now), Duration::ZERO);
assert_eq!(
wait_until_epoch_ms(i64::MAX, 0),
ms(i64::MAX.cast_unsigned() + CLOCK_SKEW_SLACK_MS)
);
}
fn epoch_in(ms: u64) -> i64 {
i64::try_from(unix_ms()).unwrap() + i64::try_from(ms).unwrap()
}
fn classify_ws_with_retry_after(code: i64, retry_after: Option<i64>) -> UnindexedOrderError {
classify_ws_order_error(
&anyhow::Error::new(WebsocketError::ResponseError {
code,
message: "Way too much request weight used; IP banned.".to_string(),
retry_after,
}),
&InstrumentNameExchange::new("BTCUSDT"),
)
.expect("a ResponseError is a venue response")
}
fn order_retry_after(err: &UnindexedOrderError) -> Option<Option<Duration>> {
match err {
OrderError::Rejected(api @ ApiError::RateLimit { .. }) => Some(api.retry_after()),
_ => None,
}
}
#[test]
fn a_ws_api_rate_limit_reports_the_wait_until_retry_after() {
for code in [-1003, -1015, 429, 418] {
let err = classify_ws_with_retry_after(code, Some(epoch_in(120_000)));
let wait = order_retry_after(&err)
.flatten()
.unwrap_or_else(|| panic!("{code}: got {err:?}"));
let expected = Duration::from_millis(120_000 + CLOCK_SKEW_SLACK_MS);
assert!(
wait <= expected && wait > expected - Duration::from_secs(5),
"{code}: {wait:?}"
);
let err = classify_ws_with_retry_after(code, None);
assert_eq!(order_retry_after(&err), Some(None), "{code}");
let past = epoch_in(0) - i64::try_from(CLOCK_SKEW_SLACK_MS).unwrap() - 1_000;
let err = classify_ws_with_retry_after(code, Some(past));
assert_eq!(order_retry_after(&err), Some(None), "{code}");
}
}
#[test]
fn a_retry_after_on_another_refusal_is_not_a_rate_limit() {
let err = classify_ws_with_retry_after(-2010, Some(epoch_in(60_000)));
assert!(
matches!(err, OrderError::Rejected(ApiError::OrderRejected(_))),
"got {err:?}"
);
}
#[test]
fn a_rest_rate_limit_reports_its_retry_after() {
let refusal = |banned: bool, retry_after| {
let msg = "Too many requests.".to_string();
let code = Some(-1003);
if banned {
ConnectorError::RateLimitBanError {
msg,
code,
retry_after,
}
} else {
ConnectorError::TooManyRequestsError {
msg,
code,
retry_after,
}
}
};
for (sent, expected) in [
(Some(120), Some(Duration::from_secs(120))),
(Some(0), None),
(None, None),
] {
for banned in [false, true] {
let order = classify_err(refusal(banned, sent));
assert_eq!(order_retry_after(&order), Some(expected), "{order:?}");
let query = classify_query(refusal(banned, sent));
assert!(
matches!(
&query,
UnindexedClientError::Api(api @ ApiError::RateLimit { .. })
if api.retry_after() == expected
),
"{query:?}"
);
}
}
}
#[tokio::test]
async fn a_throttle_is_retried_until_the_retries_run_out() {
tokio::time::pause();
let tracker = RateLimitTracker::new(WeightPool::Spot);
let started = tokio::time::Instant::now();
let mut calls = 0_u32;
let result =
rest_call_with_retry::<(), ()>(&Arc::new(()), &tracker, RequestKind::Query, |_| {
calls += 1;
Box::pin(async {
Err(anyhow::Error::new(ConnectorError::TooManyRequestsError {
msg: "Too much request weight used.".to_string(),
code: Some(-1003),
retry_after: None,
}))
})
})
.await;
assert_eq!(calls, MAX_RATE_LIMIT_RETRIES + 1);
let error = result.err().expect("every attempt was refused");
assert!(
matches!(
error.downcast_ref::<ConnectorError>(),
Some(ConnectorError::TooManyRequestsError { .. })
),
"{error:?}"
);
let backoffs = Duration::from_secs(1 + 2 + 4);
let elapsed = started.elapsed();
assert!(
elapsed >= backoffs && elapsed <= backoffs + Duration::from_millis(3),
"{elapsed:?}"
);
assert_eq!(
tracker.cooldown_remaining(),
Some(Duration::from_secs(DEFAULT_RATE_LIMIT_DELAY_SECS))
);
}
#[test]
fn a_held_back_request_reports_the_cooldown_left() {
let remaining = Duration::from_secs(42);
let held = || anyhow::Error::new(CooldownActive { remaining });
let order = classify_rest_order_error(&held(), &InstrumentNameExchange::new("BTCUSDT"));
assert_eq!(order_retry_after(&order), Some(Some(remaining)));
let query = classify_rest_query_error(&held(), None);
assert!(
matches!(
&query,
UnindexedClientError::Api(api @ ApiError::RateLimit { .. })
if api.retry_after() == Some(remaining)
),
"{query:?}"
);
}
#[tokio::test]
async fn the_ws_api_limit_replaces_the_default() {
tokio::time::pause();
let tracker = RateLimitTracker::new(WeightPool::Spot);
tracker.observe_ws_api(Some(&ws_weight(0, 1_000)), unix_ms());
assert_eq!(
tracker.weight_limit.load(Ordering::Relaxed),
SPOT_REQUEST_WEIGHT_PER_MINUTE
);
assert!(!waits(&tracker, RequestKind::Query).await);
tracker.observe_ws_api(Some(&ws_weight(1_100, 1_000)), unix_ms());
assert_eq!(tracker.weight_limit.load(Ordering::Relaxed), 1_100);
assert!(waits(&tracker, RequestKind::Query).await);
}
#[tokio::test]
async fn a_response_from_an_earlier_minute_pauses_nothing() {
tokio::time::pause();
let tracker = RateLimitTracker::new(WeightPool::Spot);
tracker.observe_ws_api(Some(&ws_weight(1_000, 1_000)), unix_ms() - 120_000);
assert!(!waits(&tracker, RequestKind::Query).await);
}
#[tokio::test]
#[cfg_attr(
debug_assertions,
should_panic(expected = "WS-API usage is spot weight")
)]
async fn a_margin_tracker_ignores_ws_api_usage() {
tokio::time::pause();
let tracker = RateLimitTracker::new(WeightPool::Sapi);
tracker.observe_ws_api(Some(&ws_weight(1_000, 1_000)), unix_ms());
assert_eq!(
tracker.weight_limit.load(Ordering::Relaxed),
SAPI_IP_WEIGHT_PER_MINUTE
);
assert!(!waits(&tracker, RequestKind::Query).await);
}
#[tokio::test]
async fn a_response_without_weight_changes_nothing() {
tokio::time::pause();
let tracker = RateLimitTracker::new(WeightPool::Spot);
tracker.observe_ws_api(None, unix_ms());
tracker.observe_ws_api(Some(&ws_weight(1_000, 1_000)[..1]), unix_ms());
assert_eq!(
tracker.weight_limit.load(Ordering::Relaxed),
SPOT_REQUEST_WEIGHT_PER_MINUTE
);
assert!(!waits(&tracker, RequestKind::Query).await);
}
#[test]
fn handshake_rate_limit_is_recognised() {
let handshake = |msg: &str| anyhow::Error::new(WebsocketError::Handshake(msg.into()));
let throttled = HandshakeRefusal::of(&handshake("HTTP error: 429 Too Many Requests"));
assert_eq!(throttled, Some(HandshakeRefusal::Throttled));
assert_eq!(HandshakeRefusal::Throttled.retry_after(), None);
let banned = HandshakeRefusal::of(&handshake("HTTP error: 418 I'm a teapot"));
assert_eq!(banned, Some(HandshakeRefusal::Banned));
assert_eq!(
HandshakeRefusal::Banned.retry_after(),
Some(Duration::from_secs(120))
);
assert_eq!(
HandshakeRefusal::of(
&handshake("HTTP error: 429 Too Many Requests").context("connecting")
),
Some(HandshakeRefusal::Throttled)
);
for other in [
handshake("HTTP error: 503 Service Unavailable"),
anyhow::Error::new(WebsocketError::Timeout),
anyhow::anyhow!("HTTP error: 429 Too Many Requests"),
] {
assert_eq!(HandshakeRefusal::of(&other), None, "{other:#}");
}
}
#[test]
fn a_placement_response_is_read_from_its_status() {
use rust_decimal_macros::dec;
let btc = InstrumentNameExchange::new("BTCUSDT");
let time = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap();
let placed = |status: Option<&str>, filled: Option<&str>, quote: Option<&str>| {
placed_order_state(
ExchangeId::BinanceSpot,
&btc,
dec!(2),
PlacementResponse {
status,
order_id: OrderId::new("7"),
time_exchange: time,
executed_qty: filled,
cumulative_quote_qty: quote,
},
)
};
let expired = |filled| {
OrderState::Inactive(InactiveOrderState::Expired(Expired::new(
OrderId::new("7"),
time,
filled,
)))
};
let open = |filled| {
OrderState::active(Open::new(
VenueOrderId::Assigned(OrderId::new("7")),
time,
filled,
))
};
assert_eq!(
placed(Some("EXPIRED"), Some("0"), Some("0")),
expired(Some(Decimal::ZERO)),
"an IOC or FOK order that found no liquidity"
);
assert_eq!(
placed(Some("EXPIRED"), Some("1"), Some("100")),
expired(Some(dec!(1))),
"an IOC order that partly filled and expired the rest"
);
assert_eq!(
placed(Some("EXPIRED_IN_MATCH"), Some("0"), None),
expired(Some(Decimal::ZERO)),
"expired by self-trade prevention"
);
assert_eq!(
placed(Some("FILLED"), Some("2"), Some("201")),
OrderState::fully_filled(Filled::new(
OrderId::new("7"),
time,
dec!(2),
Some(dec!(100.5))
))
);
assert_eq!(
placed(Some("CANCELED"), Some("1"), None),
OrderState::Inactive(InactiveOrderState::Cancelled(Cancelled::new(
OrderId::new("7"),
time,
Some(dec!(1))
)))
);
assert!(matches!(
placed(Some("REJECTED"), Some("0"), None),
OrderState::Inactive(InactiveOrderState::OpenFailed(OrderError::Rejected(
ApiError::OrderRejected(_)
)))
));
for status in [
Some("NEW"),
Some("PARTIALLY_FILLED"),
Some("PENDING_NEW"),
None,
Some("FROZEN"),
] {
assert_eq!(
placed(status, Some("1"), Some("100")),
open(dec!(1)),
"{status:?} reads as open from what filled"
);
}
assert!(
matches!(
placed(None, Some("2"), None),
OrderState::Inactive(InactiveOrderState::FullyFilled(_))
),
"an ACK that somehow reports everything filled"
);
}
#[test]
fn a_placement_response_without_a_fill_reports_it_unknown() {
use rust_decimal_macros::dec;
let btc = InstrumentNameExchange::new("BTCUSDT");
let time = Utc.timestamp_millis_opt(1_700_000_000_000).unwrap();
let placed = |status: Option<&str>, filled: Option<&str>| {
placed_order_state(
ExchangeId::BinanceSpot,
&btc,
dec!(2),
PlacementResponse {
status,
order_id: OrderId::new("7"),
time_exchange: time,
executed_qty: filled,
cumulative_quote_qty: Some("201"),
},
)
};
let id = || OrderId::new("7");
for filled in [None, Some("not a number")] {
assert_eq!(
placed(Some("EXPIRED"), filled),
OrderState::Inactive(InactiveOrderState::Expired(Expired::new(id(), time, None))),
"{filled:?}"
);
assert_eq!(
placed(Some("CANCELED"), filled),
OrderState::Inactive(InactiveOrderState::Cancelled(Cancelled::new(
id(),
time,
None
))),
"{filled:?}"
);
assert_eq!(
placed(Some("FILLED"), filled),
OrderState::fully_filled(Filled::new(id(), time, dec!(2), Some(dec!(100.5)))),
"a filled order filled its whole quantity, at the quote over it: {filled:?}"
);
assert_eq!(
placed(Some("NEW"), filled),
OrderState::active(Open::new(VenueOrderId::Assigned(id()), time, Decimal::ZERO)),
"a live order's unknown fill reads as nothing filled yet: {filled:?}"
);
}
assert_eq!(
placed(None, None),
OrderState::active(Open::new(VenueOrderId::Assigned(id()), time, Decimal::ZERO)),
"an ACK reports neither status nor fill"
);
}
#[test]
fn the_average_price_is_the_quote_traded_over_the_base() {
use rust_decimal_macros::dec;
let id = OrderId::new("7");
let avg = |quote, filled| binance_avg_price(ExchangeId::BinanceSpot, &id, quote, filled);
assert_eq!(avg(Some("100"), dec!(4)), Some(dec!(25)));
assert_eq!(avg(Some("100"), Decimal::ZERO), None, "nothing filled");
assert_eq!(avg(None, dec!(4)), None, "no quote");
assert_eq!(
avg(Some("-1"), dec!(4)),
None,
"Binance's \"not available\""
);
assert_eq!(avg(Some("not-a-number"), dec!(4)), None);
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used)] pub(super) mod test_ws_api {
use super::HEARTBEAT_TIMEOUT_SECS;
use crate::client::reconnect_budget::BackoffLimits;
use futures::{SinkExt as _, StreamExt as _};
use parking_lot::Mutex;
use std::{sync::Arc, time::Duration};
use tokio::time::Instant;
use tokio_tungstenite::tungstenite::Message;
#[derive(Debug, Clone, Copy)]
pub(in crate::client::binance) enum Lifetime {
Open,
Silent(Duration),
Pinging(Duration),
}
pub(in crate::client::binance) async fn serve(
answer: fn(&serde_json::Value) -> serde_json::Value,
lifetime: fn(usize) -> Lifetime,
) -> (String, Arc<Mutex<Vec<Instant>>>) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("ws://{}", listener.local_addr().unwrap());
let requests = Arc::new(Mutex::new(Vec::new()));
let seen = Arc::clone(&requests);
tokio::spawn(async move {
let mut connections = 0..;
while let Ok((stream, _)) = listener.accept().await {
stream.set_nodelay(true).unwrap();
let seen = Arc::clone(&seen);
let lifetime = connections.next().map_or(Lifetime::Open, lifetime);
tokio::spawn(async move {
let Ok(mut ws) = tokio_tungstenite::accept_async(stream).await else {
return;
};
while let Some(Ok(message)) = ws.next().await {
let Message::Text(text) = message else {
continue;
};
seen.lock().push(Instant::now());
let request: serde_json::Value = serde_json::from_str(&text).unwrap();
if ws
.send(Message::text(answer(&request).to_string()))
.await
.is_err()
{
return;
}
let (after, pinging) = match lifetime {
Lifetime::Open => continue,
Lifetime::Silent(after) => (after, false),
Lifetime::Pinging(after) => (after, true),
};
let closes = tokio::time::sleep(after);
tokio::pin!(closes);
let mut pings = tokio::time::interval(Duration::from_secs(1));
loop {
tokio::select! {
() = &mut closes => break,
_ = pings.tick(), if pinging => {
if ws.send(Message::Ping(Default::default())).await.is_err() {
return;
}
}
}
}
let _ = ws.close(None).await;
return;
}
});
}
});
(url, requests)
}
pub(in crate::client::binance) const STABLE: Duration =
Duration::from_secs(HEARTBEAT_TIMEOUT_SECS + 3);
const DROPPED: Duration = Duration::from_millis(100);
pub(in crate::client::binance) fn dropped_then_pinging(n: usize) -> Lifetime {
match n {
0 | 1 => Lifetime::Silent(DROPPED),
2 => Lifetime::Pinging(STABLE),
_ => Lifetime::Open,
}
}
pub(in crate::client::binance) fn dropped_then_silent(n: usize) -> Lifetime {
match n {
0 => Lifetime::Silent(DROPPED),
1 => Lifetime::Silent(STABLE),
_ => Lifetime::Open,
}
}
async fn requests_until(
requests: &Mutex<Vec<Instant>>,
count: usize,
label: &str,
) -> Vec<Instant> {
let deadline = Instant::now() + 2 * STABLE + Duration::from_secs(15);
while requests.lock().len() < count {
assert!(
Instant::now() < deadline,
"{label}: {} requests",
requests.lock().len()
);
tokio::time::sleep(Duration::from_millis(50)).await;
}
requests.lock().clone()
}
pub(in crate::client::binance) async fn assert_reset_once_pinged(
requests: &Mutex<Vec<Instant>>,
label: &str,
) {
let initial = BackoffLimits::ACCOUNT_STREAM.initial;
assert_eq!(
waits(&requests_until(requests, 4, label).await),
[DROPPED + initial, DROPPED + 2 * initial, STABLE + initial],
"{label}: the waits between connections"
);
}
pub(in crate::client::binance) async fn assert_no_reset_when_silent(
requests: &Mutex<Vec<Instant>>,
label: &str,
) {
let initial = BackoffLimits::ACCOUNT_STREAM.initial;
assert_eq!(
waits(&requests_until(requests, 3, label).await),
[DROPPED + initial, STABLE + 2 * initial],
"{label}: the waits between connections"
);
}
fn waits(at: &[Instant]) -> Vec<Duration> {
at.windows(2).map(|w| w[1] - w[0]).collect()
}
}