use std::time::Duration;
use async_stream::stream;
use futures::{Stream, StreamExt};
use serde::Deserialize;
use tokio::time::{sleep, timeout, timeout_at, Instant};
use tokio_tungstenite::{connect_async, tungstenite::Message};
use tracing::{info, warn};
use tycho_common::Bytes;
use super::telemetry::{self, ReconnectReason, RejectReason};
pub(super) const TITAN_PRICE_LEVEL_URL: &str =
"wss://eu.data.titanbuilder.xyz/ws/pamm_price_levels_v2";
pub(super) const TITAN_PRICE_LEVEL_URL_ENV: &str = "TITAN_PAMM_PRICE_LEVEL_URL";
#[derive(Clone, Copy, Debug)]
pub(super) struct ConnectionSettings {
pub connect_timeout: Duration,
pub read_idle_timeout: Duration,
pub max_backoff: Duration,
}
impl Default for ConnectionSettings {
fn default() -> Self {
Self {
connect_timeout: Duration::from_secs(10),
read_idle_timeout: Duration::from_secs(10),
max_backoff: Duration::from_secs(32),
}
}
}
fn backoff(attempt: u32, max_backoff: Duration) -> Duration {
2u64.checked_pow(attempt)
.map_or(Duration::MAX, Duration::from_secs)
.min(max_backoff)
}
fn deadline_after(idle_timeout: Duration) -> Instant {
Instant::now() + idle_timeout.min(Duration::from_secs(u32::MAX.into()))
}
#[derive(Debug, Deserialize)]
pub(super) struct TitanPriceLevelMessage {
pub block: u64,
pub slot: Option<u64>,
pub ts: u64,
pub books: Vec<TitanBook>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(super) struct TitanBook {
#[cfg_attr(
not(test),
expect(
dead_code,
reason = "a display label, deliberately not part of any identity the stream records"
)
)]
pub maker: Option<String>,
pub quote_address: Bytes,
pub token1: String,
pub token2: String,
pub token1_address: Bytes,
pub token2_address: Bytes,
pub asks: Vec<TitanLevel>,
pub bids: Vec<TitanLevel>,
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub(super) struct TitanLevel {
pub price: f64,
pub cum_volume: f64,
}
pub(super) fn messages(
url: String,
settings: ConnectionSettings,
) -> impl Stream<Item = TitanPriceLevelMessage> + Send {
stream! {
let mut attempt: u32 = 0;
loop {
match timeout(settings.connect_timeout, connect_async(url.as_str())).await {
Ok(Ok((mut ws_stream, _))) => {
info!(%url, "Connected to Titan pAMM price level stream");
let mut idle_deadline = deadline_after(settings.read_idle_timeout);
loop {
let message = match timeout_at(idle_deadline, ws_stream.next()).await {
Ok(Some(message)) => message,
Ok(None) => {
warn!("Titan price level stream ended; reconnecting");
telemetry::record_reconnect(ReconnectReason::Ended);
break;
}
Err(_elapsed) => {
warn!(
idle_secs = settings.read_idle_timeout.as_secs(),
"No parsed Titan frame within idle timeout; reconnecting"
);
telemetry::record_reconnect(ReconnectReason::IdleTimeout);
break;
}
};
match message {
Ok(Message::Text(text)) => {
match serde_json::from_str::<TitanPriceLevelMessage>(text.as_str())
{
Ok(message) => {
attempt = 0;
yield message;
idle_deadline = deadline_after(settings.read_idle_timeout);
}
Err(e) => {
warn!(error = %e, "Failed to parse Titan price level message");
telemetry::record_frame_rejected(RejectReason::ParseError);
}
}
}
Ok(Message::Binary(bytes)) => {
warn!(len = bytes.len(), "Ignoring unexpected binary Titan frame");
}
Ok(Message::Ping(_)) | Ok(Message::Pong(_)) => {}
Ok(Message::Frame(_)) => {}
Ok(Message::Close(frame)) => {
warn!(?frame, "Titan price level stream closed by server; reconnecting");
telemetry::record_reconnect(ReconnectReason::Closed);
break;
}
Err(e) => {
warn!(error = %e, "Titan price level stream read error; reconnecting");
telemetry::record_reconnect(ReconnectReason::ReadError);
break;
}
}
}
}
Ok(Err(e)) => {
warn!(error = %e, "Failed to connect to Titan price level stream; retrying");
telemetry::record_reconnect(ReconnectReason::ConnectFailed);
}
Err(_elapsed) => {
warn!(
timeout_secs = settings.connect_timeout.as_secs(),
"Titan price level connect timed out; retrying"
);
telemetry::record_reconnect(ReconnectReason::ConnectTimeout);
}
}
attempt = attempt.saturating_add(1);
let backoff = backoff(attempt, settings.max_backoff);
warn!(seconds = backoff.as_secs(), attempt, "Backing off before reconnecting to Titan");
sleep(backoff).await;
}
}
}
#[cfg(test)]
mod tests {
use std::{collections::HashSet, str::FromStr, sync::atomic::Ordering};
use futures::SinkExt;
use rstest::rstest;
use super::{
super::{
telemetry::{
recorded::{counter_value, record_async},
FRAMES_REJECTED, RECONNECTS,
},
test_support::{
frame_text, frame_then_repeat, wall_nanos_now, FakeTitan, PAMM, USDC, WBTC,
},
},
*,
};
const CAPTURED_MESSAGE: &str =
include_str!("test_responses/pamm_price_levels_v2_1792115918290990905.json");
const NANOS_LOWER_BOUND: u64 = 1_700_000_000_000_000_000;
const NANOS_UPPER_BOUND: u64 = 2_000_000_000_000_000_000;
#[test]
fn parses_captured_live_message() {
let message: TitanPriceLevelMessage =
serde_json::from_str(CAPTURED_MESSAGE).expect("valid JSON");
assert_eq!(message.block, 26_201_953);
assert_eq!(message.slot, Some(15_440_991));
assert!(
(NANOS_LOWER_BOUND..NANOS_UPPER_BOUND).contains(&message.ts),
"ts is not nanosecond-scale: {}",
message.ts
);
let venues: HashSet<&Bytes> = message
.books
.iter()
.map(|book| &book.quote_address)
.collect();
assert!(venues.len() >= 2, "expected several venues, got {}", venues.len());
let fermi = message
.books
.iter()
.find(|book| {
book.quote_address == Bytes::from_str(PAMM).unwrap() &&
book.token1 == "WBTC" &&
book.token2 == "USDC"
})
.expect("Fermi WBTC/USDC book");
assert_eq!(fermi.maker.as_deref(), Some("Fermi"));
assert!(!fermi.asks.is_empty());
assert!(!fermi.bids.is_empty());
for book in &message.books {
for level in book.asks.iter().chain(&book.bids) {
assert!(level.cum_volume > 0.0, "non-positive cumVolume: {level:?}");
assert!(level.price > 0.0, "non-positive price: {level:?}");
}
}
}
#[test]
fn requires_ts_but_not_slot() {
let without_ts = r#"{"slot": 1, "block": 2, "books": []}"#;
assert!(serde_json::from_str::<TitanPriceLevelMessage>(without_ts).is_err());
let without_slot = r#"{"block": 2, "ts": 3, "books": []}"#;
let message: TitanPriceLevelMessage =
serde_json::from_str(without_slot).expect("valid JSON");
assert_eq!(message.slot, None);
}
#[rstest]
#[case::named_maker(r#""maker": "Fermi","#, Some("Fermi"))]
#[case::empty_maker(r#""maker": "","#, Some(""))]
#[case::null_maker(r#""maker": null,"#, None)]
#[case::absent_maker("", None)]
fn tolerates_internal_fields_and_unlabelled_makers(
#[case] maker: &str,
#[case] expected: Option<&str>,
) {
let json = format!(
r#"{{
{maker}
"quoteAddress": "{PAMM}",
"token1": "WBTC",
"token2": "USDC",
"token1Address": "{WBTC}",
"token2Address": "{USDC}",
"calculated_from": 283851543383726791766048878,
"asks": [{{"price": 80997.89405475458, "cumVolume": 0.00012346}}],
"bids": []
}}"#
);
let book: TitanBook = serde_json::from_str(&json).expect("valid book");
assert_eq!(book.maker.as_deref(), expected);
assert_eq!(book.quote_address, Bytes::from_str(PAMM).unwrap());
assert_eq!(book.asks.len(), 1);
assert_eq!(book.asks[0].price, 80997.89405475458);
assert_eq!(book.asks[0].cum_volume, 0.00012346);
assert!(book.bids.is_empty());
}
#[test]
fn backoff_grows_exponentially_up_to_the_cap() {
let max_backoff = ConnectionSettings::default().max_backoff;
assert_eq!(backoff(1, max_backoff), Duration::from_secs(2));
assert_eq!(backoff(4, max_backoff), Duration::from_secs(16));
assert_eq!(backoff(5, max_backoff), max_backoff);
assert_eq!(backoff(100, max_backoff), max_backoff);
assert_eq!(backoff(u32::MAX, max_backoff), max_backoff);
}
fn fast_settings() -> ConnectionSettings {
ConnectionSettings {
connect_timeout: Duration::from_secs(1),
read_idle_timeout: Duration::from_millis(100),
max_backoff: Duration::from_millis(10),
}
}
fn frame() -> Message {
Message::Text(frame_text(100, wall_nanos_now()).into())
}
async fn next_frame(stream: &mut (impl Stream<Item = TitanPriceLevelMessage> + Unpin)) {
tokio::time::timeout(Duration::from_secs(2), stream.next())
.await
.expect("a frame within two seconds")
.expect("the stream never ends");
}
#[rstest]
#[case::ping_only(Message::Ping(Vec::new().into()), false)]
#[case::malformed_text(Message::Text("nonsense".into()), true)]
fn non_frame_traffic_does_not_count_as_liveness(
#[case] filler: Message,
#[case] rejected_as_parse_error: bool,
) {
let (connections, snapshot) = record_async(async {
let fake =
FakeTitan::spawn(frame_then_repeat(frame(), filler, Duration::from_millis(10)))
.await;
let stream = messages(fake.url(), fast_settings());
tokio::pin!(stream);
next_frame(&mut stream).await;
next_frame(&mut stream).await;
fake.connections.load(Ordering::SeqCst)
});
assert!(connections >= 2, "no reconnect on non-frame traffic");
assert!(counter_value(&snapshot, RECONNECTS, &[("reason", "idle_timeout")]) >= 1);
let parse_errors = counter_value(&snapshot, FRAMES_REJECTED, &[("reason", "parse_error")]);
assert_eq!(parse_errors > 0, rejected_as_parse_error);
}
#[test]
fn server_close_frame_reconnects() {
let (connections, snapshot) = record_async(async {
let fake = FakeTitan::spawn(|_, mut socket| async move {
if socket.send(frame()).await.is_err() {
return;
}
let _ = socket.send(Message::Close(None)).await;
})
.await;
let stream = messages(fake.url(), fast_settings());
tokio::pin!(stream);
next_frame(&mut stream).await;
next_frame(&mut stream).await;
fake.connections.load(Ordering::SeqCst)
});
assert_eq!(connections, 2);
assert_eq!(counter_value(&snapshot, RECONNECTS, &[("reason", "closed")]), 1);
}
#[test]
fn slow_consumer_does_not_trigger_the_idle_timeout() {
let (connections, snapshot) = record_async(async {
let fake =
FakeTitan::spawn(frame_then_repeat(frame(), frame(), Duration::from_millis(20)))
.await;
let stream = messages(fake.url(), fast_settings());
tokio::pin!(stream);
next_frame(&mut stream).await;
tokio::time::sleep(Duration::from_millis(300)).await;
next_frame(&mut stream).await;
fake.connections.load(Ordering::SeqCst)
});
assert_eq!(connections, 1, "reconnected while the consumer was not polling");
assert_eq!(counter_value(&snapshot, RECONNECTS, &[("reason", "idle_timeout")]), 0);
}
#[tokio::test]
async fn parsed_frames_keep_the_connection_alive() {
let fake =
FakeTitan::spawn(frame_then_repeat(frame(), frame(), Duration::from_millis(30))).await;
let settings =
ConnectionSettings { read_idle_timeout: Duration::from_millis(150), ..fast_settings() };
let stream = messages(fake.url(), settings);
tokio::pin!(stream);
for _ in 0..10 {
next_frame(&mut stream).await;
}
assert_eq!(fake.connections.load(Ordering::SeqCst), 1);
}
}