use crate::{
AccountEventKind, UnindexedAccountEvent,
error::{ApiError, ConnectivityError, OrderError, UnindexedClientError},
order::{OrderKind, TimeInForce, TrailingOffsetType, state::OrderState},
};
use binance_sdk::common::errors::{ConnectorError, WebsocketError};
use lru::LruCache;
use rustrade_instrument::{
Side, asset::name::AssetNameExchange, instrument::name::InstrumentNameExchange,
};
use smol_str::SmolStr;
use std::{
num::NonZeroUsize,
pin::Pin,
sync::Arc,
task::{Context, Poll},
time::Duration,
};
use tracing::{debug, warn};
pub(crate) struct AbortOnDropStream<S> {
inner: S,
handle: tokio::task::JoinHandle<()>,
}
impl<S> AbortOnDropStream<S> {
pub(crate) fn new(inner: S, handle: tokio::task::JoinHandle<()>) -> Self {
Self { inner, handle }
}
}
impl<S: futures::Stream + Unpin> futures::Stream for AbortOnDropStream<S> {
type Item = S::Item;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
Pin::new(&mut self.inner).poll_next(cx)
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.inner.size_hint()
}
}
impl<S> Drop for AbortOnDropStream<S> {
fn drop(&mut self) {
self.handle.abort();
}
}
pub(crate) const INITIAL_BACKOFF_MS: u64 = 1_000;
pub(crate) const MAX_BACKOFF_MS: u64 = 30_000;
pub(crate) const MAX_RECONNECT_ATTEMPTS: u32 = 10;
pub(crate) const HEARTBEAT_TIMEOUT_SECS: u64 = 30;
const _: () = assert!(
HEARTBEAT_TIMEOUT_SECS <= i64::MAX as u64,
"HEARTBEAT_TIMEOUT_SECS overflows i64"
);
pub(crate) const FILL_RECOVERY_TIMEOUT_SECS: u64 = 30;
pub(crate) const CONNECT_TIMEOUT_SECS: u64 = 15;
const _: () = assert!(
CONNECT_TIMEOUT_SECS <= i64::MAX as u64,
"CONNECT_TIMEOUT_SECS overflows i64"
);
pub(crate) const SIGNAL_RECOVERY_LOOKBACK_MS: i64 = 500;
pub(crate) const DEDUP_CACHE_SIZE: usize = 10_000;
pub(crate) const BINANCE_MAX_TRADES: usize = 1000;
const _: () = assert!(
BINANCE_MAX_TRADES <= i32::MAX as usize,
"BINANCE_MAX_TRADES overflows i32"
);
pub(crate) const DEFAULT_RATE_LIMIT_DELAY_SECS: u64 = 10;
pub(crate) const MAX_RATE_LIMIT_RETRIES: u32 = 3;
#[derive(Debug, Hash, Eq, PartialEq, Clone, Copy)]
pub(crate) enum DedupEventKind {
Trade,
New,
Cancelled,
}
#[derive(Debug, Hash, Eq, PartialEq)]
pub(crate) struct DedupKey {
pub(crate) instrument: SmolStr,
pub(crate) id: SmolStr,
pub(crate) kind: DedupEventKind,
}
pub(crate) type SharedDedupCache = Arc<parking_lot::Mutex<LruCache<DedupKey, ()>>>;
pub(crate) fn new_dedup_cache() -> SharedDedupCache {
#[allow(clippy::unwrap_used)]
Arc::new(parking_lot::Mutex::new(LruCache::new(
NonZeroUsize::new(DEDUP_CACHE_SIZE).unwrap(),
)))
}
pub(crate) fn dedup_key_from_event(event: &UnindexedAccountEvent) -> Option<DedupKey> {
match &event.kind {
AccountEventKind::Trade(trade) => Some(DedupKey {
instrument: trade.instrument.name().clone(),
id: trade.id.0.clone(),
kind: DedupEventKind::Trade,
}),
AccountEventKind::OrderSnapshot(snap) => {
match &snap.0.state {
OrderState::Active(active) => {
use crate::order::state::ActiveOrderState;
match active {
ActiveOrderState::Open(open) => Some(DedupKey {
instrument: snap.0.key.instrument.name().clone(),
id: open.id.0.clone(),
kind: DedupEventKind::New,
}),
_ => None,
}
}
_ => None,
}
}
AccountEventKind::OrderCancelled(resp) => match &resp.state {
Ok(cancelled) => Some(DedupKey {
instrument: resp.key.instrument.name().clone(),
id: cancelled.id.0.clone(),
kind: DedupEventKind::Cancelled,
}),
Err(_) => None, },
_ => None, }
}
pub(crate) fn is_duplicate(cache: &SharedDedupCache, key: DedupKey) -> bool {
let mut guard = cache.lock();
if guard.peek(&key).is_some() {
return true;
}
guard.put(key, ());
false
}
pub(crate) struct RateLimitTracker {
blocked_until: parking_lot::Mutex<Option<tokio::time::Instant>>,
}
impl RateLimitTracker {
pub(crate) fn new() -> Self {
Self {
blocked_until: parking_lot::Mutex::new(None),
}
}
pub(crate) async fn wait_if_blocked(&self) {
loop {
let deadline = *self.blocked_until.lock();
match deadline {
None => return,
Some(until) => {
let now = tokio::time::Instant::now();
if until <= now {
return;
}
#[allow(clippy::cast_possible_truncation)]
let delay_ms = (until - now).as_millis() as u64;
debug!(
delay_ms,
"Binance REST rate-limited, waiting before request"
);
tokio::time::sleep_until(until).await;
}
}
}
}
pub(crate) fn on_rate_limited(&self, retry_after: Option<Duration>) {
let delay = retry_after.unwrap_or(Duration::from_secs(DEFAULT_RATE_LIMIT_DELAY_SECS));
let new_deadline = tokio::time::Instant::now() + delay;
let mut guard = self.blocked_until.lock();
let was_blocked = guard.is_some();
*guard = Some(guard.map_or(new_deadline, |existing| existing.max(new_deadline)));
if was_blocked {
debug!(
delay_secs = delay.as_secs(),
"Binance rate-limit cooldown extended"
);
} else {
warn!(
delay_secs = delay.as_secs(),
"Binance entering rate-limit degradation mode"
);
}
}
}
pub(crate) fn is_rate_limit_error(e: &anyhow::Error) -> bool {
for cause in e.chain() {
let msg = cause.to_string();
if msg.contains("Too many requests")
|| msg.contains("been banned for exceeding rate limits")
|| contains_error_code(&msg, "-1015")
|| contains_error_code(&msg, "-1003")
{
return true;
}
}
false
}
pub(crate) fn is_api_rejection_error(e: &anyhow::Error) -> bool {
e.downcast_ref::<WebsocketError>()
.is_some_and(|we| matches!(we, WebsocketError::ResponseError { .. }))
}
pub(crate) struct ExponentialBackoff {
attempt: u32,
max_attempts: u32,
initial_ms: u64,
max_ms: u64,
}
impl ExponentialBackoff {
pub(crate) fn new() -> Self {
Self {
attempt: 0,
max_attempts: MAX_RECONNECT_ATTEMPTS,
initial_ms: INITIAL_BACKOFF_MS,
max_ms: MAX_BACKOFF_MS,
}
}
pub(crate) fn reset(&mut self) {
self.attempt = 0;
}
pub(crate) async fn wait(&mut self) -> bool {
if self.attempt >= self.max_attempts {
return false;
}
let delay_ms = self
.initial_ms
.saturating_mul(2u64.saturating_pow(self.attempt))
.min(self.max_ms);
self.attempt += 1;
debug!(
attempt = self.attempt,
max = self.max_attempts,
delay_ms,
"Binance reconnect backoff"
);
tokio::time::sleep(Duration::from_millis(delay_ms)).await;
true
}
}
pub(crate) fn parse_side(s: &str) -> Option<Side> {
match s {
"BUY" => Some(Side::Buy),
"SELL" => Some(Side::Sell),
_ => {
warn!(side = s, "unknown Binance order side");
None
}
}
}
pub(crate) fn parse_order_kind(t: &str) -> Option<OrderKind> {
match t {
"MARKET" => Some(OrderKind::Market),
"STOP_LOSS" | "TAKE_PROFIT" => {
warn!(
order_type = t,
"dropping conditional Binance order type (no OrderKind equivalent)"
);
None
}
"LIMIT" | "LIMIT_MAKER" | "STOP_LOSS_LIMIT" | "TAKE_PROFIT_LIMIT" => Some(OrderKind::Limit),
_ => {
warn!(order_type = t, "unsupported Binance order type");
None
}
}
}
pub(crate) fn parse_time_in_force(tif: &str) -> TimeInForce {
match tif {
"GTC" => TimeInForce::GoodUntilCancelled { post_only: false },
"GTX" => TimeInForce::GoodUntilCancelled { post_only: true },
"IOC" => TimeInForce::ImmediateOrCancel,
"FOK" => TimeInForce::FillOrKill,
"GTD" => TimeInForce::GoodUntilEndOfDay,
_ => {
warn!(
time_in_force = tif,
"unknown Binance TimeInForce, defaulting to GTC"
);
TimeInForce::GoodUntilCancelled { post_only: false }
}
}
}
fn contains_ignore_case(haystack: &str, needle: &str) -> bool {
haystack
.as_bytes()
.windows(needle.len())
.any(|w| w.eq_ignore_ascii_case(needle.as_bytes()))
}
pub(crate) fn contains_error_code(msg: &str, code: &str) -> bool {
let code_len = code.len();
let mut start = 0;
while let Some(rel) = msg[start..].find(code) {
let pos = start + rel;
let prefix_ok = pos == 0 || !msg[..pos].ends_with(|c: char| c.is_ascii_digit());
let suffix_ok = !msg[pos + code_len..].starts_with(|c: char| c.is_ascii_digit());
if prefix_ok && suffix_ok {
return true;
}
start = pos + 1;
}
false
}
pub(crate) fn parse_binance_api_error(
error_msg: String,
instrument: &InstrumentNameExchange,
) -> ApiError<AssetNameExchange, InstrumentNameExchange> {
if contains_error_code(&error_msg, "-1002") || contains_error_code(&error_msg, "-2015") {
return ApiError::Unauthenticated(error_msg);
}
if contains_error_code(&error_msg, "-1003") || contains_error_code(&error_msg, "-1015") {
return ApiError::RateLimit;
}
if contains_error_code(&error_msg, "-2011") {
return ApiError::OrderAlreadyCancelled;
}
if contains_error_code(&error_msg, "-2013") {
return ApiError::OrderAlreadyCancelled;
}
if contains_error_code(&error_msg, "-1121") {
return ApiError::InstrumentInvalid(instrument.clone(), error_msg);
}
if contains_error_code(&error_msg, "-2010") {
return ApiError::BalanceInsufficient(
AssetNameExchange::new(instrument.name().as_str()),
error_msg,
);
}
if contains_ignore_case(&error_msg, "insufficient")
|| contains_ignore_case(&error_msg, "not enough")
{
ApiError::BalanceInsufficient(
AssetNameExchange::new(instrument.name().as_str()),
error_msg,
)
} else if contains_ignore_case(&error_msg, "rate limit") {
ApiError::RateLimit
} else if contains_ignore_case(&error_msg, "unknown order") {
ApiError::OrderRejected(error_msg)
} else if contains_ignore_case(&error_msg, "invalid symbol") {
ApiError::InstrumentInvalid(instrument.clone(), error_msg)
} else {
ApiError::OrderRejected(error_msg)
}
}
pub(crate) fn connectivity_error(e: anyhow::Error) -> UnindexedClientError {
let msg = format!("{e:#}");
if contains_error_code(&msg, "-1002")
|| contains_error_code(&msg, "-2015")
|| contains_ignore_case(&msg, "invalid api-key")
|| contains_ignore_case(&msg, "invalid signature")
|| contains_ignore_case(&msg, "signature for this request is not valid")
{
return UnindexedClientError::Api(ApiError::Unauthenticated(msg));
}
UnindexedClientError::Connectivity(ConnectivityError::Socket(msg))
}
pub(crate) async fn rest_call_with_retry<R, T>(
rest: &Arc<R>,
rate_limiter: &RateLimitTracker,
mut make_call: impl FnMut(
Arc<R>,
)
-> Pin<Box<dyn std::future::Future<Output = anyhow::Result<T>> + Send>>,
) -> anyhow::Result<T>
where
R: Send + Sync,
{
for attempt in 0..=MAX_RATE_LIMIT_RETRIES {
rate_limiter.wait_if_blocked().await;
match make_call(Arc::clone(rest)).await {
Ok(v) => return Ok(v),
Err(e) if is_rate_limit_error(&e) && attempt < MAX_RATE_LIMIT_RETRIES => {
let delay = Duration::from_secs(2u64.saturating_pow(attempt).min(30));
warn!(
attempt = attempt.saturating_add(1),
max = MAX_RATE_LIMIT_RETRIES,
delay_secs = delay.as_secs(),
"Binance REST rate-limited, retrying"
);
rate_limiter.on_rate_limited(Some(delay));
}
Err(e) => return Err(e),
}
}
unreachable!("Binance REST retries exhausted: loop invariant violated")
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum BinanceOrderType {
Market,
Limit,
LimitMaker,
StopLoss,
StopLossLimit,
TakeProfit,
TakeProfitLimit,
}
impl BinanceOrderType {
pub(crate) fn as_binance_str(self) -> &'static str {
match self {
BinanceOrderType::Market => "MARKET",
BinanceOrderType::Limit => "LIMIT",
BinanceOrderType::LimitMaker => "LIMIT_MAKER",
BinanceOrderType::StopLoss => "STOP_LOSS",
BinanceOrderType::StopLossLimit => "STOP_LOSS_LIMIT",
BinanceOrderType::TakeProfit => "TAKE_PROFIT",
BinanceOrderType::TakeProfitLimit => "TAKE_PROFIT_LIMIT",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum BinanceTimeInForce {
Gtc,
Ioc,
Fok,
}
pub(crate) fn classify_order_kind_tif(
kind: OrderKind,
tif: TimeInForce,
) -> Option<(BinanceOrderType, Option<BinanceTimeInForce>)> {
match kind {
OrderKind::Market => Some((BinanceOrderType::Market, None)),
OrderKind::Limit => match tif {
TimeInForce::GoodUntilCancelled { post_only: false } => {
Some((BinanceOrderType::Limit, Some(BinanceTimeInForce::Gtc)))
}
TimeInForce::GoodUntilCancelled { post_only: true } => {
Some((BinanceOrderType::LimitMaker, None))
}
TimeInForce::FillOrKill => {
Some((BinanceOrderType::Limit, Some(BinanceTimeInForce::Fok)))
}
TimeInForce::ImmediateOrCancel => {
Some((BinanceOrderType::Limit, Some(BinanceTimeInForce::Ioc)))
}
TimeInForce::GoodUntilEndOfDay
| TimeInForce::GoodTillDate { .. }
| TimeInForce::AtOpen
| TimeInForce::AtClose => {
warn!(time_in_force = ?tif, "Binance does not support this TimeInForce");
None
}
},
OrderKind::Stop { .. } => Some((BinanceOrderType::StopLoss, None)),
OrderKind::StopLimit { .. } => match tif {
TimeInForce::GoodUntilCancelled { post_only: false } => Some((
BinanceOrderType::StopLossLimit,
Some(BinanceTimeInForce::Gtc),
)),
TimeInForce::FillOrKill => Some((
BinanceOrderType::StopLossLimit,
Some(BinanceTimeInForce::Fok),
)),
TimeInForce::ImmediateOrCancel => Some((
BinanceOrderType::StopLossLimit,
Some(BinanceTimeInForce::Ioc),
)),
_ => {
warn!(time_in_force = ?tif, "Binance StopLimit does not support this TimeInForce");
None
}
},
OrderKind::TakeProfit { .. } => Some((BinanceOrderType::TakeProfit, None)),
OrderKind::TakeProfitLimit { .. } => match tif {
TimeInForce::GoodUntilCancelled { post_only: false } => Some((
BinanceOrderType::TakeProfitLimit,
Some(BinanceTimeInForce::Gtc),
)),
TimeInForce::FillOrKill => Some((
BinanceOrderType::TakeProfitLimit,
Some(BinanceTimeInForce::Fok),
)),
TimeInForce::ImmediateOrCancel => Some((
BinanceOrderType::TakeProfitLimit,
Some(BinanceTimeInForce::Ioc),
)),
_ => {
warn!(time_in_force = ?tif, "Binance TakeProfitLimit does not support this TimeInForce");
None
}
},
OrderKind::TrailingStop { offset_type, .. } => match offset_type {
TrailingOffsetType::BasisPoints | TrailingOffsetType::Percentage => {
Some((BinanceOrderType::StopLoss, None))
}
TrailingOffsetType::Absolute => {
warn!(
"Binance TrailingStop does not support Absolute offset; \
convert to basis points: (absolute / price) * 10000"
);
None
}
},
OrderKind::TrailingStopLimit { .. } => {
warn!("Binance does not support TrailingStopLimit orders");
None
}
}
}
pub(crate) fn classify_rest_order_error(
e: &anyhow::Error,
instrument: &InstrumentNameExchange,
) -> OrderError<AssetNameExchange, InstrumentNameExchange> {
let Some(ce) = e.downcast_ref::<ConnectorError>() else {
return OrderError::Connectivity(ConnectivityError::Socket(format!("{e:#}")));
};
match ce {
ConnectorError::TooManyRequestsError { .. } | ConnectorError::RateLimitBanError { .. } => {
OrderError::Rejected(ApiError::RateLimit)
}
ConnectorError::UnauthorizedError { msg, code }
| ConnectorError::ForbiddenError { msg, code } => {
let msg_with_code = code.map_or_else(|| msg.clone(), |c| format!("{c} {msg}"));
OrderError::Rejected(ApiError::Unauthenticated(msg_with_code))
}
ConnectorError::ServerError { msg, .. } | ConnectorError::NetworkError(msg) => {
OrderError::Connectivity(ConnectivityError::Socket(msg.clone()))
}
ConnectorError::BadRequestError { msg, code }
| ConnectorError::NotFoundError { msg, code }
| ConnectorError::ConnectorClientError { msg, code } => {
const TRANSPORT_PREFIXES: [&str; 4] = [
"HTTP request failed",
"Failed to get response bytes",
"Failed to decompress gzip response",
"Failed to convert response to UTF-8",
];
if code.is_none() && TRANSPORT_PREFIXES.iter().any(|p| msg.starts_with(p)) {
return OrderError::Connectivity(ConnectivityError::Socket(msg.clone()));
}
let msg_with_code = code.map_or_else(|| msg.clone(), |c| format!("{c} {msg}"));
OrderError::Rejected(parse_binance_api_error(msg_with_code, instrument))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn classify(
msg: &str,
code: Option<i64>,
) -> OrderError<AssetNameExchange, InstrumentNameExchange> {
let err = anyhow::Error::new(ConnectorError::ConnectorClientError {
msg: msg.to_string(),
code,
});
classify_rest_order_error(&err, &InstrumentNameExchange::new("BTCUSDT"))
}
#[test]
fn codeless_transport_failures_map_to_connectivity() {
for msg in [
"HTTP request failed: connection reset",
"Failed to get response bytes: error reading body",
"Failed to decompress gzip response",
"Failed to convert response to UTF-8: invalid utf-8 sequence",
] {
assert!(
matches!(classify(msg, None), OrderError::Connectivity(_)),
"expected Connectivity for {msg:?}"
);
}
}
#[test]
fn coded_error_maps_to_rejection() {
assert!(matches!(
classify("Account has insufficient balance.", Some(-2010)),
OrderError::Rejected(_)
));
}
#[test]
fn codeless_non_transport_status_error_maps_to_rejection() {
assert!(matches!(
classify("Conflict", None),
OrderError::Rejected(_)
));
}
fn classify_err(ce: ConnectorError) -> OrderError<AssetNameExchange, InstrumentNameExchange> {
classify_rest_order_error(
&anyhow::Error::new(ce),
&InstrumentNameExchange::new("BTCUSDT"),
)
}
#[test]
fn rate_limit_variants_map_to_rejection_ratelimit() {
for ce in [
ConnectorError::TooManyRequestsError {
msg: "Too many requests.".to_string(),
code: Some(-1003),
},
ConnectorError::RateLimitBanError {
msg: "IP banned.".to_string(),
code: Some(-1003),
},
] {
assert!(matches!(
classify_err(ce),
OrderError::Rejected(ApiError::RateLimit)
));
}
}
#[test]
fn auth_variants_map_to_unauthenticated() {
for ce in [
ConnectorError::UnauthorizedError {
msg: "bad key".to_string(),
code: Some(-2014),
},
ConnectorError::ForbiddenError {
msg: "forbidden".to_string(),
code: None,
},
] {
assert!(matches!(
classify_err(ce),
OrderError::Rejected(ApiError::Unauthenticated(_))
));
}
}
#[test]
fn server_and_network_variants_map_to_connectivity() {
for ce in [
ConnectorError::ServerError {
msg: "internal error".to_string(),
status_code: Some(503),
},
ConnectorError::NetworkError("connection reset".to_string()),
] {
assert!(matches!(classify_err(ce), OrderError::Connectivity(_)));
}
}
}