Skip to main content

nautilus_hyperliquid/
data.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::{
17    str::FromStr,
18    sync::{
19        Arc,
20        atomic::{AtomicBool, Ordering},
21    },
22    time::{Duration, Instant},
23};
24
25use ahash::{AHashMap, AHashSet};
26use anyhow::Context;
27use jiff::Timestamp;
28use nautilus_common::{
29    cache::InstrumentLookupError,
30    clients::DataClient,
31    live::runner::get_data_event_sender,
32    messages::{
33        DataEvent,
34        data::{
35            BarsResponse, BookResponse, CustomDataResponse, DataResponse, FundingRatesResponse,
36            InstrumentResponse, InstrumentsResponse, RequestBars, RequestBookSnapshot,
37            RequestCustomData, RequestFundingRates, RequestInstrument, RequestInstruments,
38            RequestTrades, SubscribeBars, SubscribeBookDeltas, SubscribeBookDepth10,
39            SubscribeCustomData, SubscribeFundingRates, SubscribeIndexPrices, SubscribeInstrument,
40            SubscribeMarkPrices, SubscribeQuotes, SubscribeTrades, TradesResponse, UnsubscribeBars,
41            UnsubscribeBookDeltas, UnsubscribeBookDepth10, UnsubscribeCustomData,
42            UnsubscribeFundingRates, UnsubscribeIndexPrices, UnsubscribeInstrument,
43            UnsubscribeInstruments, UnsubscribeMarkPrices, UnsubscribeQuotes, UnsubscribeTrades,
44        },
45    },
46};
47use nautilus_core::{
48    AtomicMap, Params, UnixNanos,
49    datetime::{datetime_to_unix_nanos, unix_nanos_to_iso8601},
50    time::{AtomicTime, get_atomic_clock_realtime},
51};
52use nautilus_live::{
53    SocketControl,
54    task::{TaskGroup, TaskGroupGuard},
55};
56use nautilus_model::{
57    data::{Bar, BarType, BookOrder, CustomData, Data, DataType, FundingRateUpdate, TradeTick},
58    enums::{BarAggregation, BookType, OrderSide},
59    identifiers::{ClientId, InstrumentId, Venue},
60    instruments::{Instrument, InstrumentAny},
61    orderbook::OrderBook,
62    types::{Price, Quantity},
63};
64use parking_lot::Mutex;
65use rust_decimal::Decimal;
66use tokio_util::sync::CancellationToken;
67use ustr::Ustr;
68
69use crate::{
70    common::{
71        consts::HYPERLIQUID_VENUE,
72        credential::{Secrets, credential_env_vars},
73        parse::{bar_type_to_interval, millis_to_nanos},
74    },
75    config::HyperliquidDataClientConfig,
76    data_types::register_hyperliquid_custom_data,
77    http::{
78        client::HyperliquidHttpClient,
79        models::{HyperliquidCandle, HyperliquidFundingHistoryEntry, HyperliquidL2Book},
80        parse::parse_recent_trade,
81    },
82    websocket::{
83        DATA_STREAMS_ENDPOINT, client::HyperliquidWebSocketClient, messages::NautilusWsMessage,
84    },
85};
86
87#[derive(Debug)]
88pub struct HyperliquidDataClient {
89    clock: &'static AtomicTime,
90    client_id: ClientId,
91    config: HyperliquidDataClientConfig,
92    http_client: HyperliquidHttpClient,
93    ws_client: HyperliquidWebSocketClient,
94    is_connected: AtomicBool,
95    cancellation_token: CancellationToken,
96    session_tasks: TaskGroup,
97    pending_tasks: TaskGroup,
98    shutdown_errors: Vec<String>,
99    data_sender: tokio::sync::mpsc::UnboundedSender<DataEvent>,
100    instruments: Arc<AtomicMap<InstrumentId, InstrumentAny>>,
101    coin_to_instrument_id: Arc<AtomicMap<Ustr, InstrumentId>>,
102    // serializes instrument fetch-and-apply passes, see `refresh_instruments`
103    instrument_update_lock: Arc<tokio::sync::Mutex<()>>,
104    stream_health: Arc<Mutex<MarketDataStreamHealthMonitor>>,
105}
106
107impl HyperliquidDataClient {
108    /// Creates a new [`HyperliquidDataClient`] instance.
109    ///
110    /// # Errors
111    ///
112    /// Returns an error if the HTTP client fails to initialize.
113    pub fn new(client_id: ClientId, config: HyperliquidDataClientConfig) -> anyhow::Result<Self> {
114        let clock = get_atomic_clock_realtime();
115        let data_sender = get_data_event_sender();
116
117        // Only fall back to unauthenticated when credentials are absent,
118        // not when they're invalid (fail fast on malformed keys)
119        let (pk_var, _) = credential_env_vars(config.environment);
120        let has_credentials = config.has_credentials() || std::env::var(pk_var).is_ok();
121        let proxy_url = config
122            .proxy_url
123            .as_ref()
124            .map(|value| value.expose_secret().to_owned());
125
126        let mut http_client = if has_credentials {
127            let secrets = Secrets::resolve(
128                config
129                    .private_key
130                    .as_ref()
131                    .map(|value| value.expose_secret()),
132                None,
133                config.environment,
134            )?;
135            HyperliquidHttpClient::with_secrets(
136                &secrets,
137                config.http_timeout_secs,
138                proxy_url.clone(),
139            )?
140        } else {
141            HyperliquidHttpClient::new(
142                config.environment,
143                config.http_timeout_secs,
144                proxy_url.clone(),
145            )?
146        };
147
148        if let Some(url) = &config.base_url_http {
149            http_client.set_base_info_url(url.clone());
150        }
151
152        let ws_url = config.base_url_ws.clone();
153        let ws_client = HyperliquidWebSocketClient::new(
154            ws_url,
155            config.environment,
156            None,
157            config.transport_backend,
158            proxy_url,
159        );
160        let ws_client = ws_client.with_socket_control(SocketControl::new(
161            client_id,
162            Some(*HYPERLIQUID_VENUE),
163            DATA_STREAMS_ENDPOINT,
164        ));
165        let mut stream_health_monitor = MarketDataStreamHealthMonitor::new(
166            Duration::from_secs(config.stale_stream_receive_timeout_secs),
167            Duration::from_secs(config.stale_stream_warning_cooldown_secs),
168        );
169
170        if config.stale_stream_recovery_enabled {
171            if config.stale_stream_recovery_cooldown_secs > 0 {
172                stream_health_monitor = stream_health_monitor.with_recovery(
173                    Duration::from_secs(config.stale_stream_recovery_cooldown_secs),
174                    config.stale_stream_max_targeted_resubscribes,
175                );
176            } else {
177                log::warn!(
178                    "Hyperliquid stale stream recovery disabled: \
179                     stale_stream_recovery_cooldown_secs must be positive"
180                );
181            }
182        }
183
184        let stream_health = Arc::new(Mutex::new(stream_health_monitor));
185
186        let session_tasks = TaskGroup::new();
187        let pending_tasks = TaskGroup::new();
188
189        Ok(Self {
190            clock,
191            client_id,
192            config,
193            http_client,
194            ws_client,
195            is_connected: AtomicBool::new(false),
196            cancellation_token: CancellationToken::new(),
197            session_tasks,
198            pending_tasks,
199            shutdown_errors: Vec::new(),
200            data_sender,
201            instruments: Arc::new(AtomicMap::new()),
202            coin_to_instrument_id: Arc::new(AtomicMap::new()),
203            instrument_update_lock: Arc::new(tokio::sync::Mutex::new(())),
204            stream_health,
205        })
206    }
207
208    fn spawn_task<F>(&self, description: &'static str, fut: F)
209    where
210        F: std::future::Future<Output = anyhow::Result<()>> + Send + 'static,
211    {
212        let future = async move {
213            if let Err(e) = fut.await {
214                log::warn!("{description} failed: {e:?}");
215            }
216        };
217
218        if let Err(e) = self.pending_tasks.spawn(future) {
219            log::warn!("Skipping Hyperliquid {description} after shutdown began: {e}");
220        }
221    }
222
223    fn abort_pending_tasks(&self) {
224        self.pending_tasks.begin_shutdown();
225    }
226
227    fn abort_session_tasks(&self) {
228        self.session_tasks.begin_shutdown();
229        self.ws_client.begin_shutdown();
230    }
231
232    async fn teardown_partial_connect(&mut self) -> anyhow::Result<()> {
233        self.cancellation_token.cancel();
234        self.abort_session_tasks();
235        self.abort_pending_tasks();
236
237        if let Err(e) = self.ws_client.disconnect().await {
238            self.shutdown_errors
239                .push(format!("Hyperliquid WebSocket shutdown failed: {e}"));
240        }
241
242        if let Err(e) = self.await_session_tasks().await {
243            self.shutdown_errors.push(e.to_string());
244        }
245
246        if let Err(e) = self.await_pending_tasks().await {
247            self.shutdown_errors.push(e.to_string());
248        }
249        self.clear_stream_health();
250        self.is_connected.store(false, Ordering::Release);
251
252        if !self.shutdown_errors.is_empty() {
253            anyhow::bail!(std::mem::take(&mut self.shutdown_errors).join("; "));
254        }
255        Ok(())
256    }
257
258    async fn await_pending_tasks(&self) -> anyhow::Result<()> {
259        self.pending_tasks.begin_shutdown();
260        self.pending_tasks
261            .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
262            .await
263            .map_err(|e| anyhow::anyhow!("Failed to terminate Hyperliquid data tasks: {e}"))?;
264        Ok(())
265    }
266
267    async fn await_session_tasks(&self) -> anyhow::Result<()> {
268        self.session_tasks.begin_shutdown();
269        self.session_tasks
270            .finish_shutdown(Duration::from_secs(1), Duration::from_secs(2))
271            .await
272            .map_err(|e| {
273                anyhow::anyhow!("Failed to terminate Hyperliquid data session tasks: {e}")
274            })?;
275        Ok(())
276    }
277
278    fn clear_stream_health(&self) {
279        self.stream_health.lock().clear();
280    }
281
282    fn register_stream_health(&self, channel: MarketDataChannel, instrument_id: InstrumentId) {
283        if !self.stream_health_monitor_enabled() {
284            return;
285        }
286
287        self.stream_health
288            .lock()
289            .subscribe(channel, instrument_id, Instant::now());
290    }
291
292    fn remove_stream_health(&self, channel: MarketDataChannel, instrument_id: InstrumentId) {
293        self.stream_health
294            .lock()
295            .unsubscribe(channel, instrument_id);
296    }
297
298    fn stream_health_monitor_enabled(&self) -> bool {
299        self.config.stale_stream_receive_timeout_secs > 0
300            && self.config.stream_health_check_interval_secs > 0
301    }
302
303    fn spawn_stream_health_monitor(&self) -> anyhow::Result<()> {
304        if !self.stream_health_monitor_enabled() {
305            return Ok(());
306        }
307
308        let stream_health = Arc::clone(&self.stream_health);
309        let cancellation_token = self.cancellation_token.clone();
310        let interval = Duration::from_secs(self.config.stream_health_check_interval_secs);
311        let clock = self.clock;
312        let ws_client = self.ws_client.clone();
313
314        self.session_tasks.spawn(async move {
315            log::debug!("Hyperliquid stream health monitor started");
316
317            loop {
318                tokio::select! {
319                    () = cancellation_token.cancelled() => {
320                        log::debug!("Hyperliquid stream health monitor cancelled");
321                        break;
322                    }
323                    () = tokio::time::sleep(interval) => {
324                        let events = stream_health
325                            .lock()
326                            .check_stale(Instant::now(), clock.get_time_ns());
327
328                        handle_stream_health_events(&ws_client, &events).await;
329                    }
330                }
331            }
332
333            log::debug!("Hyperliquid stream health monitor stopped");
334        })?;
335
336        Ok(())
337    }
338
339    fn venue(&self) -> Venue {
340        *HYPERLIQUID_VENUE
341    }
342
343    fn custom_instrument_id(data_type: &DataType) -> anyhow::Result<Option<InstrumentId>> {
344        let Some(raw_instrument_id) = data_type
345            .metadata()
346            .and_then(|m| m.get("instrument_id"))
347            .and_then(|v| v.as_str())
348            .map(str::trim)
349            .filter(|value| !value.is_empty())
350        else {
351            return Ok(None);
352        };
353
354        let instrument_id = InstrumentId::from_str(raw_instrument_id)
355            .with_context(|| format!("invalid instrument_id metadata `{raw_instrument_id}`"))?;
356
357        Ok(Some(instrument_id))
358    }
359
360    fn custom_user(data_type: &DataType) -> anyhow::Result<Option<String>> {
361        let Some(user) = data_type
362            .metadata()
363            .and_then(|m| m.get("user"))
364            .and_then(|v| v.as_str())
365            .filter(|value| !value.is_empty())
366        else {
367            return Ok(None);
368        };
369
370        anyhow::ensure!(
371            user == user.trim(),
372            "metadata['user'] must not contain surrounding whitespace",
373        );
374
375        Ok(Some(user.to_string()))
376    }
377
378    async fn bootstrap_instruments(&self) -> anyhow::Result<Vec<InstrumentAny>> {
379        // a pass that fetched before the bootstrap must not apply over its universe
380        let _update_guard = self.instrument_update_lock.lock().await;
381
382        let instruments = self
383            .http_client
384            .request_instruments()
385            .await
386            .context("failed to fetch instruments during bootstrap")?;
387
388        cache_instruments(
389            &instruments,
390            &self.instruments,
391            &self.coin_to_instrument_id,
392            &self.http_client,
393            &self.ws_client,
394        );
395        rebuild_all_dex_asset_ctxs_mapping(&self.http_client, &self.ws_client).await;
396
397        log::debug!(
398            "Bootstrapped {} instruments with {} coin mappings",
399            self.instruments.len(),
400            self.coin_to_instrument_id.len()
401        );
402        Ok(instruments)
403    }
404
405    /// Spawns the periodic instrument refresh task, disabled when the configured
406    /// interval is zero. The task is tracked in `self.session_tasks`, so the
407    /// cancellation token stops it on disconnect, failed connect, and dispose.
408    fn spawn_instrument_refresh(&self) -> anyhow::Result<()> {
409        let minutes = self.config.update_instruments_interval_mins;
410
411        if minutes == 0 {
412            log::debug!(
413                "Hyperliquid instrument refresh disabled (update_instruments_interval_mins=0)"
414            );
415            return Ok(());
416        }
417
418        let interval = Duration::from_secs(minutes.saturating_mul(60));
419        let cancellation_token = self.cancellation_token.clone();
420        let http_client = self.http_client.clone();
421        let ws_client = self.ws_client.clone();
422        let instruments = Arc::clone(&self.instruments);
423        let coin_to_instrument_id = Arc::clone(&self.coin_to_instrument_id);
424        let instrument_update_lock = Arc::clone(&self.instrument_update_lock);
425        let data_sender = self.data_sender.clone();
426        let client_id = self.client_id;
427
428        self.session_tasks.spawn(async move {
429            log::info!("Hyperliquid instrument refresh started, interval={interval:?}");
430
431            loop {
432                tokio::select! {
433                    () = cancellation_token.cancelled() => {
434                        log::debug!("Hyperliquid instrument refresh cancelled");
435                        break;
436                    }
437                    () = tokio::time::sleep(interval) => {}
438                }
439
440                // The refresh performs several REST calls, so cancellation is
441                // also raced against the pass itself rather than only the sleep
442                let result = tokio::select! {
443                    () = cancellation_token.cancelled() => {
444                        log::debug!("Hyperliquid instrument refresh cancelled");
445                        break;
446                    }
447                    result = refresh_instruments(
448                        &instrument_update_lock,
449                        &http_client,
450                        &ws_client,
451                        &instruments,
452                        &coin_to_instrument_id,
453                        &data_sender,
454                    ) => result,
455                };
456
457                match result {
458                    Ok(summary) => summary.log(client_id),
459                    Err(e) => log::warn!(
460                        "Failed to refresh Hyperliquid instruments: client_id={client_id}, error={e:?}"
461                    ),
462                }
463            }
464
465            log::debug!("Hyperliquid instrument refresh stopped");
466        })?;
467
468        Ok(())
469    }
470
471    async fn spawn_ws(&self) -> anyhow::Result<()> {
472        // Clone client before connecting so the clone can have out_rx set
473        let mut ws_client = self.ws_client.clone();
474
475        ws_client
476            .connect()
477            .await
478            .context("failed to connect to Hyperliquid WebSocket")?;
479
480        let data_sender = self.data_sender.clone();
481        let cancellation_token = self.cancellation_token.clone();
482        let stream_health = Arc::clone(&self.stream_health);
483
484        self.session_tasks.spawn(async move {
485            log::debug!("Hyperliquid WebSocket consumption loop started");
486
487            loop {
488                tokio::select! {
489                    () = cancellation_token.cancelled() => {
490                        log::debug!("WebSocket consumption loop cancelled");
491                        break;
492                    }
493                    msg_opt = ws_client.next_event() => {
494                        if let Some(msg) = msg_opt {
495                            if let Some((channel, instrument_id, ts_event)) =
496                                stream_health_update(&msg)
497                            {
498                                record_stream_receive(
499                                    &stream_health,
500                                    channel,
501                                    instrument_id,
502                                    ts_event,
503                                );
504                            }
505
506                            match msg {
507                                NautilusWsMessage::Trades(trades) => {
508                                    for trade in trades {
509                                        if let Err(e) = data_sender
510                                            .send(DataEvent::Data(Data::Trade(trade)))
511                                        {
512                                            log::error!("Failed to send trade tick: {e}");
513                                        }
514                                    }
515                                }
516                                NautilusWsMessage::Quote(quote) => {
517                                    if let Err(e) = data_sender
518                                        .send(DataEvent::Data(Data::Quote(quote)))
519                                    {
520                                        log::error!("Failed to send quote tick: {e}");
521                                    }
522                                }
523                                NautilusWsMessage::Deltas(deltas) => {
524                                    if let Err(e) = data_sender
525                                        .send(DataEvent::Data(Data::BookDeltas(
526                                            Box::new(deltas),
527                                        )))
528                                    {
529                                        log::error!("Failed to send order book deltas: {e}");
530                                    }
531                                }
532                                NautilusWsMessage::Depth10(depth) => {
533                                    if let Err(e) =
534                                        data_sender.send(DataEvent::Data(Data::BookDepth10(depth)))
535                                    {
536                                        log::error!("Failed to send order book depth10: {e}");
537                                    }
538                                }
539                                NautilusWsMessage::Candle(bar) => {
540                                    if let Err(e) = data_sender
541                                        .send(DataEvent::Data(Data::Bar(bar)))
542                                    {
543                                        log::error!("Failed to send bar: {e}");
544                                    }
545                                }
546                                NautilusWsMessage::MarkPrice(update) => {
547                                    if let Err(e) = data_sender
548                                        .send(DataEvent::Data(Data::MarkPrice(update)))
549                                    {
550                                        log::error!("Failed to send mark price update: {e}");
551                                    }
552                                }
553                                NautilusWsMessage::IndexPrice(update) => {
554                                    if let Err(e) = data_sender
555                                        .send(DataEvent::Data(Data::IndexPrice(update)))
556                                    {
557                                        log::error!("Failed to send index price update: {e}");
558                                    }
559                                }
560                                NautilusWsMessage::FundingRate(update) => {
561                                    if let Err(e) = data_sender
562                                        .send(DataEvent::FundingRate(update))
563                                    {
564                                        log::error!("Failed to send funding rate update: {e}");
565                                    }
566                                }
567                                NautilusWsMessage::CustomData(data) => {
568                                    if let Err(e) = data_sender.send(DataEvent::Data(data)) {
569                                        log::error!("Failed to send custom data: {e}");
570                                    }
571                                }
572                                NautilusWsMessage::Reconnected => {
573                                    log::info!("WebSocket reconnected");
574                                }
575                                NautilusWsMessage::Error(e) => {
576                                    log::warn!("WebSocket error: {e}");
577                                }
578                                NautilusWsMessage::ExecutionReports(_) => {
579                                    // Handled by execution client
580                                }
581                            }
582                        } else {
583                            // Connection closed or error
584                            log::debug!("WebSocket next_event returned None, stream closed");
585                            tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
586                        }
587                    }
588                }
589            }
590
591            log::debug!("Hyperliquid WebSocket consumption loop finished");
592        })?;
593
594        log::debug!("WebSocket consumption task spawned");
595
596        Ok(())
597    }
598}
599
600#[async_trait::async_trait(?Send)]
601impl DataClient for HyperliquidDataClient {
602    fn client_id(&self) -> ClientId {
603        self.client_id
604    }
605
606    fn venue(&self) -> Option<Venue> {
607        Some(self.venue())
608    }
609
610    fn start(&mut self) -> anyhow::Result<()> {
611        log::info!(
612            "Starting Hyperliquid data client: client_id={}, environment={:?}, proxy_url={:?}",
613            self.client_id,
614            self.config.environment,
615            self.config.proxy_url,
616        );
617        Ok(())
618    }
619
620    fn stop(&mut self) -> anyhow::Result<()> {
621        log::info!("Stopping Hyperliquid data client {}", self.client_id);
622        self.cancellation_token.cancel();
623        self.abort_session_tasks();
624        self.abort_pending_tasks();
625        self.is_connected.store(false, Ordering::Relaxed);
626        Ok(())
627    }
628
629    fn reset(&mut self) -> anyhow::Result<()> {
630        log::debug!("Resetting Hyperliquid data client {}", self.client_id);
631        // Keep this generation cancelled until `connect()` has torn down the
632        // inner WebSocket client. Replacing it here would allow the next
633        // connection to reuse an active old-generation handler.
634        self.cancellation_token.cancel();
635        self.abort_session_tasks();
636        self.abort_pending_tasks();
637        self.is_connected.store(false, Ordering::Relaxed);
638        self.instruments.store(AHashMap::new());
639        self.coin_to_instrument_id.store(AHashMap::new());
640        Ok(())
641    }
642
643    fn dispose(&mut self) -> anyhow::Result<()> {
644        log::debug!("Disposing Hyperliquid data client {}", self.client_id);
645        self.stop()
646    }
647
648    fn is_connected(&self) -> bool {
649        self.is_connected.load(Ordering::Acquire)
650    }
651
652    fn is_disconnected(&self) -> bool {
653        !self.is_connected()
654    }
655
656    async fn connect(&mut self) -> anyhow::Result<()> {
657        if self.is_connected()
658            && !self.cancellation_token.is_cancelled()
659            && self.session_tasks.is_open()
660            && self.pending_tasks.is_open()
661        {
662            return Ok(());
663        }
664
665        if self.cancellation_token.is_cancelled()
666            || !self.session_tasks.is_open()
667            || !self.pending_tasks.is_open()
668        {
669            // `reset()` is synchronous, while shutting down the inner socket
670            // is async. Complete that teardown before creating any new stream
671            // task so its receiver and subscription registries cannot belong
672            // to the previous generation.
673            self.ws_client.begin_shutdown();
674            self.ws_client
675                .disconnect()
676                .await
677                .context("failed to tear down Hyperliquid WebSocket before reconnect")?;
678            self.ws_client.reset_runtime_state();
679            self.abort_session_tasks();
680            self.abort_pending_tasks();
681            let (session_result, pending_result) =
682                tokio::join!(self.await_session_tasks(), self.await_pending_tasks());
683            session_result?;
684            pending_result?;
685            self.session_tasks.start_generation().map_err(|e| {
686                anyhow::anyhow!("Failed to start Hyperliquid data session generation: {e}")
687            })?;
688            self.pending_tasks.start_generation().map_err(|e| {
689                anyhow::anyhow!("Failed to start Hyperliquid data task generation: {e}")
690            })?;
691            self.cancellation_token = CancellationToken::new();
692        }
693        let cancellation_token = self.cancellation_token.clone();
694        let ws_client = self.ws_client.clone();
695        let setup_guard =
696            TaskGroupGuard::new(&[&self.session_tasks, &self.pending_tasks], move || {
697                cancellation_token.cancel();
698                ws_client.begin_shutdown();
699            });
700
701        register_hyperliquid_custom_data();
702
703        let instruments = self
704            .bootstrap_instruments()
705            .await
706            .context("failed to bootstrap instruments")?;
707
708        for instrument in instruments {
709            if let Err(e) = self.data_sender.send(DataEvent::Instrument(instrument)) {
710                log::warn!("Failed to send instrument: {e}");
711            }
712        }
713
714        let session_result = async {
715            self.spawn_ws()
716                .await
717                .context("failed to spawn WebSocket client")?;
718            self.spawn_stream_health_monitor()?;
719            self.spawn_instrument_refresh()?;
720            Ok::<(), anyhow::Error>(())
721        }
722        .await;
723
724        if let Err(e) = session_result {
725            if let Err(teardown_error) = self.teardown_partial_connect().await {
726                return Err(e.context(format!(
727                    "Hyperliquid data startup teardown failed: {teardown_error}"
728                )));
729            }
730            return Err(e);
731        }
732
733        self.is_connected.store(true, Ordering::Relaxed);
734        setup_guard.disarm();
735        log::info!("Connected: client_id={}", self.client_id);
736
737        Ok(())
738    }
739
740    async fn disconnect(&mut self) -> anyhow::Result<()> {
741        self.teardown_partial_connect().await?;
742        self.instruments.store(AHashMap::new());
743        log::info!("Disconnected: client_id={}", self.client_id);
744
745        Ok(())
746    }
747
748    fn subscribe(&mut self, cmd: SubscribeCustomData) -> anyhow::Result<()> {
749        let data_type = cmd.data_type.type_name();
750
751        if data_type == "HyperliquidAllMids" {
752            let ws = self.ws_client.clone();
753            let dex = cmd
754                .data_type
755                .metadata()
756                .as_ref()
757                .and_then(|m| m.get("dex"))
758                .and_then(|v| v.as_str())
759                .map(str::trim)
760                .filter(|value| !value.is_empty())
761                .map(ToString::to_string);
762
763            log::debug!("Subscribing to all mids (dex: {:?})", dex.as_deref());
764
765            self.spawn_task("subscribe_all_mids", async move {
766                ws.subscribe_all_mids_with_dex(dex.as_deref()).await
767            });
768
769            return Ok(());
770        }
771
772        if data_type == "HyperliquidAllDexsAssetCtxs" {
773            let ws = self.ws_client.clone();
774
775            self.spawn_task("subscribe_all_dexs_asset_ctxs", async move {
776                ws.subscribe_all_dexs_asset_ctxs().await
777            });
778
779            return Ok(());
780        }
781
782        if data_type == "HyperliquidOpenInterest" {
783            let ws = self.ws_client.clone();
784            let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
785                "HyperliquidOpenInterest subscriptions require metadata['instrument_id']",
786            )?;
787
788            self.spawn_task("subscribe_open_interest", async move {
789                ws.subscribe_open_interest(instrument_id).await
790            });
791
792            return Ok(());
793        }
794
795        if data_type == "HyperliquidPublicTrade" {
796            let ws = self.ws_client.clone();
797            let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
798                "HyperliquidPublicTrade subscriptions require metadata['instrument_id']",
799            )?;
800
801            self.spawn_task("subscribe_public_trades", async move {
802                ws.subscribe_public_trades(instrument_id).await
803            });
804
805            return Ok(());
806        }
807
808        if data_type == "HyperliquidTwapHistory" {
809            let ws = self.ws_client.clone();
810            let user = Self::custom_user(&cmd.data_type)?
811                .context("HyperliquidTwapHistory subscriptions require metadata['user']")?;
812
813            self.spawn_task("subscribe_user_twap_history", async move {
814                ws.subscribe_user_twap_history(&user).await
815            });
816
817            return Ok(());
818        }
819
820        if data_type == "HyperliquidTwapSliceFill" {
821            let ws = self.ws_client.clone();
822            let user = Self::custom_user(&cmd.data_type)?
823                .context("HyperliquidTwapSliceFill subscriptions require metadata['user']")?;
824
825            self.spawn_task("subscribe_user_twap_slice_fills", async move {
826                ws.subscribe_user_twap_slice_fills(&user).await
827            });
828
829            return Ok(());
830        }
831
832        log::warn!("Unsupported custom data subscription: {data_type}");
833        Ok(())
834    }
835
836    fn unsubscribe(&mut self, cmd: &UnsubscribeCustomData) -> anyhow::Result<()> {
837        let data_type = cmd.data_type.type_name();
838
839        if data_type == "HyperliquidAllMids" {
840            let ws = self.ws_client.clone();
841            let dex = cmd
842                .data_type
843                .metadata()
844                .as_ref()
845                .and_then(|m| m.get("dex"))
846                .and_then(|v| v.as_str())
847                .map(str::trim)
848                .filter(|value| !value.is_empty())
849                .map(ToString::to_string);
850
851            log::debug!("Unsubscribing from all mids (dex: {:?})", dex.as_deref());
852
853            self.spawn_task("unsubscribe_all_mids", async move {
854                ws.unsubscribe_all_mids_with_dex(dex.as_deref()).await
855            });
856
857            return Ok(());
858        }
859
860        if data_type == "HyperliquidAllDexsAssetCtxs" {
861            let ws = self.ws_client.clone();
862
863            self.spawn_task("unsubscribe_all_dexs_asset_ctxs", async move {
864                ws.unsubscribe_all_dexs_asset_ctxs().await
865            });
866
867            return Ok(());
868        }
869
870        if data_type == "HyperliquidOpenInterest" {
871            let ws = self.ws_client.clone();
872            let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
873                "HyperliquidOpenInterest unsubscriptions require metadata['instrument_id']",
874            )?;
875
876            self.spawn_task("unsubscribe_open_interest", async move {
877                ws.unsubscribe_open_interest(instrument_id).await
878            });
879
880            return Ok(());
881        }
882
883        if data_type == "HyperliquidPublicTrade" {
884            let ws = self.ws_client.clone();
885            let instrument_id = Self::custom_instrument_id(&cmd.data_type)?.context(
886                "HyperliquidPublicTrade unsubscriptions require metadata['instrument_id']",
887            )?;
888
889            self.spawn_task("unsubscribe_public_trades", async move {
890                ws.unsubscribe_public_trades(instrument_id).await
891            });
892
893            return Ok(());
894        }
895
896        if data_type == "HyperliquidTwapHistory" {
897            let ws = self.ws_client.clone();
898            let user = Self::custom_user(&cmd.data_type)?
899                .context("HyperliquidTwapHistory unsubscriptions require metadata['user']")?;
900
901            self.spawn_task("unsubscribe_user_twap_history", async move {
902                ws.unsubscribe_user_twap_history(&user).await
903            });
904
905            return Ok(());
906        }
907
908        if data_type == "HyperliquidTwapSliceFill" {
909            let ws = self.ws_client.clone();
910            let user = Self::custom_user(&cmd.data_type)?
911                .context("HyperliquidTwapSliceFill unsubscriptions require metadata['user']")?;
912
913            self.spawn_task("unsubscribe_user_twap_slice_fills", async move {
914                ws.unsubscribe_user_twap_slice_fills(&user).await
915            });
916
917            return Ok(());
918        }
919
920        log::warn!("Unsupported custom data unsubscription: {data_type}");
921        Ok(())
922    }
923
924    fn subscribe_instrument(&mut self, cmd: SubscribeInstrument) -> anyhow::Result<()> {
925        let instruments = self.instruments.load();
926        if let Some(instrument) = instruments.get(&cmd.instrument_id) {
927            if let Err(e) = self
928                .data_sender
929                .send(DataEvent::Instrument(instrument.clone()))
930            {
931                log::error!("Failed to send instrument {}: {e}", cmd.instrument_id);
932            }
933        } else {
934            log::warn!("Instrument {} not found in cache", cmd.instrument_id);
935        }
936        Ok(())
937    }
938
939    fn subscribe_book_deltas(&mut self, subscription: SubscribeBookDeltas) -> anyhow::Result<()> {
940        if subscription.book_type != BookType::L2_MBP {
941            anyhow::bail!("Hyperliquid only supports L2_MBP order book deltas");
942        }
943
944        let ws = self.ws_client.clone();
945        let instrument_id = subscription.instrument_id;
946        let (n_sig_figs, mantissa) = parse_book_precision_params(subscription.params.as_ref())?;
947        self.register_stream_health(MarketDataChannel::Deltas, instrument_id);
948
949        self.spawn_task("subscribe_book_deltas", async move {
950            ws.subscribe_book_with_options(instrument_id, n_sig_figs, mantissa)
951                .await
952        });
953
954        Ok(())
955    }
956
957    fn subscribe_book_depth10(&mut self, subscription: SubscribeBookDepth10) -> anyhow::Result<()> {
958        log::debug!(
959            "Subscribing to book depth10: {}",
960            subscription.instrument_id
961        );
962
963        if subscription.book_type != BookType::L2_MBP {
964            anyhow::bail!("Hyperliquid only supports L2_MBP order book depth10");
965        }
966
967        let ws = self.ws_client.clone();
968        let instrument_id = subscription.instrument_id;
969        let (n_sig_figs, mantissa) = parse_book_precision_params(subscription.params.as_ref())?;
970        self.register_stream_health(MarketDataChannel::Depth10, instrument_id);
971
972        self.spawn_task("subscribe_book_depth10", async move {
973            ws.subscribe_book_depth10_with_options(instrument_id, n_sig_figs, mantissa)
974                .await
975        });
976
977        Ok(())
978    }
979
980    fn subscribe_quotes(&mut self, subscription: SubscribeQuotes) -> anyhow::Result<()> {
981        let ws = self.ws_client.clone();
982        let instrument_id = subscription.instrument_id;
983        self.register_stream_health(MarketDataChannel::Quote, instrument_id);
984
985        self.spawn_task("subscribe_quotes", async move {
986            ws.subscribe_quotes(instrument_id).await
987        });
988
989        Ok(())
990    }
991
992    fn subscribe_trades(&mut self, subscription: SubscribeTrades) -> anyhow::Result<()> {
993        let ws = self.ws_client.clone();
994        let instrument_id = subscription.instrument_id;
995
996        self.spawn_task("subscribe_trades", async move {
997            ws.subscribe_trades(instrument_id).await
998        });
999
1000        Ok(())
1001    }
1002
1003    fn subscribe_mark_prices(&mut self, cmd: SubscribeMarkPrices) -> anyhow::Result<()> {
1004        let ws = self.ws_client.clone();
1005        let instrument_id = cmd.instrument_id;
1006
1007        self.spawn_task("subscribe_mark_prices", async move {
1008            ws.subscribe_mark_prices(instrument_id).await
1009        });
1010
1011        Ok(())
1012    }
1013
1014    fn subscribe_index_prices(&mut self, cmd: SubscribeIndexPrices) -> anyhow::Result<()> {
1015        let ws = self.ws_client.clone();
1016        let instrument_id = cmd.instrument_id;
1017
1018        self.spawn_task("subscribe_index_prices", async move {
1019            ws.subscribe_index_prices(instrument_id).await
1020        });
1021
1022        Ok(())
1023    }
1024
1025    fn subscribe_funding_rates(&mut self, cmd: SubscribeFundingRates) -> anyhow::Result<()> {
1026        let ws = self.ws_client.clone();
1027        let instrument_id = cmd.instrument_id;
1028
1029        self.spawn_task("subscribe_funding_rates", async move {
1030            ws.subscribe_funding_rates(instrument_id).await
1031        });
1032
1033        Ok(())
1034    }
1035
1036    fn subscribe_bars(&mut self, subscription: SubscribeBars) -> anyhow::Result<()> {
1037        let instrument_id = subscription.bar_type.instrument_id();
1038        if !self.instruments.contains_key(&instrument_id) {
1039            anyhow::bail!(InstrumentLookupError::not_found(instrument_id));
1040        }
1041
1042        let bar_type = subscription.bar_type;
1043        let ws = self.ws_client.clone();
1044
1045        self.spawn_task("subscribe_bars", async move {
1046            ws.subscribe_bars(bar_type).await
1047        });
1048
1049        Ok(())
1050    }
1051
1052    fn unsubscribe_instrument(&mut self, _cmd: &UnsubscribeInstrument) -> anyhow::Result<()> {
1053        // `subscribe_instrument` only emits the cached instrument; it opens no
1054        // venue channel, so there is nothing to tear down here.
1055        Ok(())
1056    }
1057
1058    fn unsubscribe_instruments(&mut self, _cmd: &UnsubscribeInstruments) -> anyhow::Result<()> {
1059        // See `unsubscribe_instrument`: instrument subscriptions carry no
1060        // venue-side state to unsubscribe from.
1061        Ok(())
1062    }
1063
1064    fn unsubscribe_book_deltas(
1065        &mut self,
1066        unsubscription: &UnsubscribeBookDeltas,
1067    ) -> anyhow::Result<()> {
1068        log::debug!(
1069            "Unsubscribing from book deltas: {}",
1070            unsubscription.instrument_id
1071        );
1072
1073        let ws = self.ws_client.clone();
1074        let instrument_id = unsubscription.instrument_id;
1075        self.remove_stream_health(MarketDataChannel::Deltas, instrument_id);
1076
1077        self.spawn_task("unsubscribe_book_deltas", async move {
1078            ws.unsubscribe_book(instrument_id).await
1079        });
1080
1081        Ok(())
1082    }
1083
1084    fn unsubscribe_book_depth10(
1085        &mut self,
1086        unsubscription: &UnsubscribeBookDepth10,
1087    ) -> anyhow::Result<()> {
1088        log::debug!(
1089            "Unsubscribing from book depth10: {}",
1090            unsubscription.instrument_id
1091        );
1092
1093        let ws = self.ws_client.clone();
1094        let instrument_id = unsubscription.instrument_id;
1095        self.remove_stream_health(MarketDataChannel::Depth10, instrument_id);
1096
1097        self.spawn_task("unsubscribe_book_depth10", async move {
1098            ws.unsubscribe_book_depth10(instrument_id).await
1099        });
1100
1101        Ok(())
1102    }
1103
1104    fn unsubscribe_quotes(&mut self, unsubscription: &UnsubscribeQuotes) -> anyhow::Result<()> {
1105        log::debug!(
1106            "Unsubscribing from quotes: {}",
1107            unsubscription.instrument_id
1108        );
1109
1110        let ws = self.ws_client.clone();
1111        let instrument_id = unsubscription.instrument_id;
1112        self.remove_stream_health(MarketDataChannel::Quote, instrument_id);
1113
1114        self.spawn_task("unsubscribe_quotes", async move {
1115            ws.unsubscribe_quotes(instrument_id).await
1116        });
1117
1118        Ok(())
1119    }
1120
1121    fn unsubscribe_trades(&mut self, unsubscription: &UnsubscribeTrades) -> anyhow::Result<()> {
1122        log::debug!(
1123            "Unsubscribing from trades: {}",
1124            unsubscription.instrument_id
1125        );
1126
1127        let ws = self.ws_client.clone();
1128        let instrument_id = unsubscription.instrument_id;
1129
1130        self.spawn_task("unsubscribe_trades", async move {
1131            ws.unsubscribe_trades(instrument_id).await
1132        });
1133
1134        Ok(())
1135    }
1136
1137    fn unsubscribe_mark_prices(&mut self, cmd: &UnsubscribeMarkPrices) -> anyhow::Result<()> {
1138        let ws = self.ws_client.clone();
1139        let instrument_id = cmd.instrument_id;
1140
1141        self.spawn_task("unsubscribe_mark_prices", async move {
1142            ws.unsubscribe_mark_prices(instrument_id).await
1143        });
1144
1145        Ok(())
1146    }
1147
1148    fn unsubscribe_index_prices(&mut self, cmd: &UnsubscribeIndexPrices) -> anyhow::Result<()> {
1149        let ws = self.ws_client.clone();
1150        let instrument_id = cmd.instrument_id;
1151
1152        self.spawn_task("unsubscribe_index_prices", async move {
1153            ws.unsubscribe_index_prices(instrument_id).await
1154        });
1155
1156        Ok(())
1157    }
1158
1159    fn unsubscribe_funding_rates(&mut self, cmd: &UnsubscribeFundingRates) -> anyhow::Result<()> {
1160        let ws = self.ws_client.clone();
1161        let instrument_id = cmd.instrument_id;
1162
1163        self.spawn_task("unsubscribe_funding_rates", async move {
1164            ws.unsubscribe_funding_rates(instrument_id).await
1165        });
1166
1167        Ok(())
1168    }
1169
1170    fn unsubscribe_bars(&mut self, unsubscription: &UnsubscribeBars) -> anyhow::Result<()> {
1171        let bar_type = unsubscription.bar_type;
1172        let ws = self.ws_client.clone();
1173
1174        self.spawn_task("unsubscribe_bars", async move {
1175            ws.unsubscribe_bars(bar_type).await
1176        });
1177
1178        Ok(())
1179    }
1180
1181    fn request_instruments(&self, request: RequestInstruments) -> anyhow::Result<()> {
1182        log::debug!("Requesting all instruments");
1183
1184        let http = self.http_client.clone();
1185        let ws = self.ws_client.clone();
1186        let sender = self.data_sender.clone();
1187        let instruments_cache = self.instruments.clone();
1188        let coin_map = self.coin_to_instrument_id.clone();
1189        let update_lock = Arc::clone(&self.instrument_update_lock);
1190        let request_id = request.request_id;
1191        let client_id = request.client_id.unwrap_or(self.client_id);
1192        let venue = self.venue();
1193        let start_nanos = datetime_to_unix_nanos(request.start);
1194        let end_nanos = datetime_to_unix_nanos(request.end);
1195        let params = request.params;
1196        let clock = self.clock;
1197
1198        self.spawn_task("request_instruments", async move {
1199            // the fetched universe also feeds the cache the periodic refresh diffs
1200            // against, so it must be published like a refresh pass or a market first
1201            // seen here would never reach execution
1202            let refresh = refresh_instruments(
1203                &update_lock,
1204                &http,
1205                &ws,
1206                &instruments_cache,
1207                &coin_map,
1208                &sender,
1209            )
1210            .await?;
1211            refresh.log(client_id);
1212            let instruments = refresh.fetched;
1213
1214            let response = DataResponse::Instruments(InstrumentsResponse::new(
1215                request_id,
1216                client_id,
1217                venue,
1218                instruments,
1219                start_nanos,
1220                end_nanos,
1221                clock.get_time_ns(),
1222                params,
1223            ));
1224
1225            if let Err(e) = sender.send(DataEvent::Response(response)) {
1226                log::error!("Failed to send instruments response: {e}");
1227            }
1228            Ok(())
1229        });
1230
1231        Ok(())
1232    }
1233
1234    fn request_instrument(&self, request: RequestInstrument) -> anyhow::Result<()> {
1235        log::debug!("Requesting instrument: {}", request.instrument_id);
1236
1237        let http = self.http_client.clone();
1238        let ws = self.ws_client.clone();
1239        let sender = self.data_sender.clone();
1240        let instruments_cache = self.instruments.clone();
1241        let coin_map = self.coin_to_instrument_id.clone();
1242        let update_lock = Arc::clone(&self.instrument_update_lock);
1243        let instrument_id = request.instrument_id;
1244        let request_id = request.request_id;
1245        let client_id = request.client_id.unwrap_or(self.client_id);
1246        let start_nanos = datetime_to_unix_nanos(request.start);
1247        let end_nanos = datetime_to_unix_nanos(request.end);
1248        let params = request.params;
1249        let clock = self.clock;
1250
1251        self.spawn_task("request_instrument", async move {
1252            // the venue only serves the whole universe, so a single-instrument request
1253            // can discover other new markets and must publish them like a refresh pass
1254            let refresh = refresh_instruments(
1255                &update_lock,
1256                &http,
1257                &ws,
1258                &instruments_cache,
1259                &coin_map,
1260                &sender,
1261            )
1262            .await?;
1263            refresh.log(client_id);
1264            let all_instruments = refresh.fetched;
1265
1266            if let Some(instrument) = all_instruments
1267                .into_iter()
1268                .find(|i| i.id() == instrument_id)
1269            {
1270                let response = DataResponse::Instrument(Box::new(InstrumentResponse::new(
1271                    request_id,
1272                    client_id,
1273                    instrument.id(),
1274                    instrument,
1275                    start_nanos,
1276                    end_nanos,
1277                    clock.get_time_ns(),
1278                    params,
1279                )));
1280
1281                if let Err(e) = sender.send(DataEvent::Response(response)) {
1282                    log::error!("Failed to send instrument response: {e}");
1283                }
1284            } else {
1285                log::error!("Instrument not found: {instrument_id}");
1286            }
1287            Ok(())
1288        });
1289
1290        Ok(())
1291    }
1292
1293    fn request_bars(&self, request: RequestBars) -> anyhow::Result<()> {
1294        log::debug!("Requesting bars for {}", request.bar_type);
1295
1296        let http = self.http_client.clone();
1297        let sender = self.data_sender.clone();
1298        let bar_type = request.bar_type;
1299        let start = request.start;
1300        let end = request.end;
1301        let limit = request.limit.map(|n| n.get() as u32);
1302        let request_id = request.request_id;
1303        let client_id = request.client_id.unwrap_or(self.client_id);
1304        let params = request.params;
1305        let clock = self.clock;
1306        let start_nanos = datetime_to_unix_nanos(start);
1307        let end_nanos = datetime_to_unix_nanos(end);
1308        let instruments = Arc::clone(&self.instruments);
1309
1310        self.spawn_task("request_bars", async move {
1311            let bars = request_bars_from_http(http, bar_type, start, end, limit, instruments)
1312                .await
1313                .context("bar request failed")?;
1314
1315            let response = DataResponse::Bars(BarsResponse::new(
1316                request_id,
1317                client_id,
1318                bar_type,
1319                bars,
1320                start_nanos,
1321                end_nanos,
1322                clock.get_time_ns(),
1323                params,
1324            ));
1325
1326            if let Err(e) = sender.send(DataEvent::Response(response)) {
1327                log::error!("Failed to send bars response: {e}");
1328            }
1329            Ok(())
1330        });
1331
1332        Ok(())
1333    }
1334
1335    fn request_trades(&self, request: RequestTrades) -> anyhow::Result<()> {
1336        let instrument_id = request.instrument_id;
1337        log::debug!("Requesting trades for {instrument_id}");
1338
1339        let instruments = self.instruments.load();
1340        let instrument = instruments
1341            .get(&instrument_id)
1342            .cloned()
1343            .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1344
1345        let coin = instrument.raw_symbol().to_string();
1346        let http = self.http_client.clone();
1347        let sender = self.data_sender.clone();
1348        let client_id = request.client_id.unwrap_or(self.client_id);
1349        let request_id = request.request_id;
1350        let params = request.params;
1351        let clock = self.clock;
1352        let limit = request.limit.map(|n| n.get());
1353        let start_nanos = datetime_to_unix_nanos(request.start);
1354        let end_nanos = datetime_to_unix_nanos(request.end);
1355
1356        self.spawn_task("request_trades", async move {
1357            // `recentTrades` depends on the Hyperliquid indexer; nodes without it
1358            // return HTTP 422. Treat that as "no coverage" and serve an empty
1359            // response so the awaiting caller still completes.
1360            let raw_trades = match http.info_recent_trades(&coin).await {
1361                Ok(trades) => trades,
1362                Err(e) if e.is_unprocessable_entity() => {
1363                    log::warn!(
1364                        "Recent trades endpoint unavailable for {instrument_id} \
1365                         (requires the Hyperliquid indexer); sending empty response"
1366                    );
1367                    Vec::new()
1368                }
1369                Err(e) => {
1370                    return Err(anyhow::Error::new(e))
1371                        .with_context(|| format!("trades request failed for {instrument_id}"));
1372                }
1373            };
1374
1375            let mut trades: Vec<TradeTick> = Vec::with_capacity(raw_trades.len());
1376            for raw in &raw_trades {
1377                match parse_recent_trade(raw, &instrument) {
1378                    Ok(trade) => trades.push(trade),
1379                    Err(e) => log::warn!("Skipping recent trade for {instrument_id}: {e}"),
1380                }
1381            }
1382            trades.sort_by_key(|trade| trade.ts_event);
1383
1384            let trades = filter_recent_trades(trades, start_nanos, end_nanos, limit, instrument_id);
1385
1386            log::debug!("Fetched {} trades for {instrument_id}", trades.len());
1387
1388            let response = DataResponse::Trades(TradesResponse::new(
1389                request_id,
1390                client_id,
1391                instrument_id,
1392                trades,
1393                start_nanos,
1394                end_nanos,
1395                clock.get_time_ns(),
1396                params,
1397            ));
1398
1399            if let Err(e) = sender.send(DataEvent::Response(response)) {
1400                log::error!("Failed to send trades response: {e}");
1401            }
1402            Ok(())
1403        });
1404
1405        Ok(())
1406    }
1407
1408    fn request_data(&self, request: RequestCustomData) -> anyhow::Result<()> {
1409        if request.data_type.type_name() != "HyperliquidPublicTrade" {
1410            log::warn!(
1411                "Unsupported custom data request: {}",
1412                request.data_type.type_name()
1413            );
1414            return Ok(());
1415        }
1416
1417        let instrument_id = Self::custom_instrument_id(&request.data_type)?
1418            .context("HyperliquidPublicTrade requests require metadata['instrument_id']")?;
1419        let data_type = DataType::new(
1420            request.data_type.type_name(),
1421            request.data_type.metadata().cloned(),
1422            Some(instrument_id.to_string()),
1423        );
1424        let http = self.http_client.clone();
1425        let sender = self.data_sender.clone();
1426        let request_id = request.request_id;
1427        let client_id = request.client_id;
1428        let params = request.params;
1429        let clock = self.clock;
1430        let limit = request.limit.map(|limit| limit.get());
1431        let start = request.start;
1432        let end = request.end;
1433        let start_nanos = datetime_to_unix_nanos(start);
1434        let end_nanos = datetime_to_unix_nanos(end);
1435        let venue = self.venue();
1436
1437        self.spawn_task("request_public_trades", async move {
1438            let trades = http
1439                .request_public_trades(instrument_id, start, end, limit)
1440                .await
1441                .map_err(anyhow::Error::new)
1442                .with_context(|| format!("public trades request failed for {instrument_id}"))?;
1443            let data: Vec<CustomData> = trades
1444                .into_iter()
1445                .map(|trade| CustomData::new(Arc::new(trade), data_type.clone()))
1446                .collect();
1447
1448            let response = DataResponse::Data(CustomDataResponse::new(
1449                request_id,
1450                client_id,
1451                Some(venue),
1452                data_type,
1453                data,
1454                start_nanos,
1455                end_nanos,
1456                clock.get_time_ns(),
1457                params,
1458            ));
1459
1460            if let Err(e) = sender.send(DataEvent::Response(response)) {
1461                log::error!("Failed to send public trades response: {e}");
1462            }
1463            Ok(())
1464        });
1465
1466        Ok(())
1467    }
1468
1469    fn request_funding_rates(&self, request: RequestFundingRates) -> anyhow::Result<()> {
1470        let instrument_id = request.instrument_id;
1471        log::debug!("Requesting funding rates for {instrument_id}");
1472
1473        let instruments = self.instruments.load();
1474        let instrument = instruments
1475            .get(&instrument_id)
1476            .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1477
1478        if !matches!(instrument, InstrumentAny::CryptoPerpetual(_)) {
1479            anyhow::bail!("Funding rates are only available for perpetual instruments");
1480        }
1481
1482        let coin = instrument.raw_symbol().to_string();
1483        let http = self.http_client.clone();
1484        let sender = self.data_sender.clone();
1485        let client_id = request.client_id.unwrap_or(self.client_id);
1486        let request_id = request.request_id;
1487        let params = request.params;
1488        let clock = self.clock;
1489        let limit = request.limit.map(|n| n.get());
1490        let start_dt = request.start;
1491        let end_dt = request.end;
1492        let start_nanos = datetime_to_unix_nanos(start_dt);
1493        let end_nanos = datetime_to_unix_nanos(end_dt);
1494
1495        let now_ms = Timestamp::now().as_millisecond() as u64;
1496
1497        // Hyperliquid requires a startTime; default to a 7-day lookback when none given
1498        let default_lookback_ms: u64 = 7 * 86_400_000;
1499        let start_ms = match start_dt {
1500            Some(dt) => dt.as_millisecond().max(0) as u64,
1501            None => now_ms.saturating_sub(default_lookback_ms),
1502        };
1503        let end_ms = end_dt.map(|dt| dt.as_millisecond().max(0) as u64);
1504
1505        self.spawn_task("request_funding_rates", async move {
1506            let entries = http
1507                .info_funding_history(&coin, start_ms, end_ms)
1508                .await
1509                .with_context(|| format!("funding rates request failed for {instrument_id}"))?;
1510
1511            let mut funding_rates: Vec<FundingRateUpdate> = entries
1512                .iter()
1513                .map(|entry| funding_entry_to_update(entry, instrument_id))
1514                .collect();
1515
1516            if let Some(limit) = limit
1517                && funding_rates.len() > limit
1518            {
1519                funding_rates.truncate(limit);
1520            }
1521
1522            log::debug!(
1523                "Fetched {} funding rates for {instrument_id}",
1524                funding_rates.len(),
1525            );
1526
1527            let response = DataResponse::FundingRates(FundingRatesResponse::new(
1528                request_id,
1529                client_id,
1530                instrument_id,
1531                funding_rates,
1532                start_nanos,
1533                end_nanos,
1534                clock.get_time_ns(),
1535                params,
1536            ));
1537
1538            if let Err(e) = sender.send(DataEvent::Response(response)) {
1539                log::error!("Failed to send funding rates response: {e}");
1540            }
1541            Ok(())
1542        });
1543
1544        Ok(())
1545    }
1546
1547    fn request_book_snapshot(&self, request: RequestBookSnapshot) -> anyhow::Result<()> {
1548        let instrument_id = request.instrument_id;
1549        let instruments = self.instruments.load();
1550        let instrument = instruments
1551            .get(&instrument_id)
1552            .ok_or_else(|| InstrumentLookupError::not_found(instrument_id))?;
1553
1554        let raw_symbol = instrument.raw_symbol().to_string();
1555        let price_precision = instrument.price_precision();
1556        let size_precision = instrument.size_precision();
1557        let depth = request.depth.map(|d| d.get());
1558
1559        let http = self.http_client.clone();
1560        let sender = self.data_sender.clone();
1561        let client_id = request.client_id.unwrap_or(self.client_id);
1562        let request_id = request.request_id;
1563        let params = request.params;
1564        let clock = self.clock;
1565
1566        self.spawn_task("request_book_snapshot", async move {
1567            let l2_book = http
1568                .info_l2_book(&raw_symbol)
1569                .await
1570                .with_context(|| format!("book snapshot request failed for {instrument_id}"))?;
1571
1572            let book = parse_l2_book_snapshot(
1573                &l2_book,
1574                instrument_id,
1575                price_precision,
1576                size_precision,
1577                depth,
1578            );
1579
1580            let response = DataResponse::Book(BookResponse::new(
1581                request_id,
1582                client_id,
1583                instrument_id,
1584                book,
1585                None,
1586                None,
1587                clock.get_time_ns(),
1588                params,
1589            ));
1590
1591            if let Err(e) = sender.send(DataEvent::Response(response)) {
1592                log::error!("Failed to send book snapshot response: {e}");
1593            }
1594            Ok(())
1595        });
1596
1597        Ok(())
1598    }
1599}
1600
1601/// Applies fetched instruments to the client caches and both transports.
1602fn cache_instruments(
1603    instruments: &[InstrumentAny],
1604    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1605    coin_to_instrument_id: &Arc<AtomicMap<Ustr, InstrumentId>>,
1606    http_client: &HyperliquidHttpClient,
1607    ws_client: &HyperliquidWebSocketClient,
1608) {
1609    instruments_by_id.rcu(|m| {
1610        for instrument in instruments {
1611            m.insert(instrument.id(), instrument.clone());
1612        }
1613    });
1614
1615    coin_to_instrument_id.rcu(|m| {
1616        for instrument in instruments {
1617            m.insert(instrument.raw_symbol().inner(), instrument.id());
1618        }
1619    });
1620
1621    for instrument in instruments {
1622        http_client.cache_instrument(instrument);
1623        ws_client.cache_instrument(instrument.clone());
1624    }
1625}
1626
1627/// Rebuilds the `allDexsAssetCtxs` mapping from the HTTP client's cached instruments.
1628///
1629/// Callers rebuild on every pass rather than only when a definition changed. The
1630/// mapping is positional over each perp dex universe, so a listing or delisting
1631/// shifts the entries of coins whose own definitions are unchanged, and a stale
1632/// mapping would misattribute incoming `ctxs` arrays. Rebuilding unconditionally
1633/// also retries a build that failed or fell back on an earlier pass.
1634async fn rebuild_all_dex_asset_ctxs_mapping(
1635    http_client: &HyperliquidHttpClient,
1636    ws_client: &HyperliquidWebSocketClient,
1637) {
1638    match http_client.build_all_dex_asset_ctxs_instrument_ids().await {
1639        Ok(mapping) => {
1640            let mapping = mapping
1641                .into_iter()
1642                .map(|(dex, instrument_ids)| (Ustr::from(dex.as_str()), instrument_ids))
1643                .collect();
1644            ws_client.cache_all_dex_asset_ctxs_instrument_ids(mapping);
1645        }
1646        Err(e) => {
1647            log::warn!("Failed to build Hyperliquid allDexsAssetCtxs mapping: {e}");
1648        }
1649    }
1650}
1651
1652/// Summary of a single instrument refresh pass.
1653#[derive(Debug)]
1654struct InstrumentRefresh {
1655    /// Instruments returned by the venue.
1656    fetched: Vec<InstrumentAny>,
1657    /// Symbols of the definitions that were new to the cache.
1658    added: Vec<Ustr>,
1659    /// New or materially changed definitions published downstream.
1660    changed: usize,
1661}
1662
1663impl InstrumentRefresh {
1664    fn log(&self, client_id: ClientId) {
1665        // a quiet pass every interval would be noise, but a market becoming
1666        // tradable mid-session is the event an operator needs to see
1667        if self.added.is_empty() {
1668            log::debug!(
1669                "Hyperliquid instruments refreshed: client_id={client_id}, fetched={}, changed={}",
1670                self.fetched.len(),
1671                self.changed,
1672            );
1673        } else {
1674            log::info!(
1675                "Hyperliquid instruments refreshed: client_id={client_id}, fetched={}, changed={}, added={:?}",
1676                self.fetched.len(),
1677                self.changed,
1678                self.added,
1679            );
1680        }
1681    }
1682}
1683
1684/// Fetches the instrument universe and reconciles it against the caches.
1685///
1686/// Holds `update_lock` across the fetch and the reconcile so concurrent passes
1687/// apply in the order they fetched. The diff only asks whether a definition
1688/// differs from the cached one, not whether it is newer, so without the lock a
1689/// pass that fetched before another but finished after it would republish the
1690/// older definition.
1691async fn refresh_instruments(
1692    update_lock: &tokio::sync::Mutex<()>,
1693    http_client: &HyperliquidHttpClient,
1694    ws_client: &HyperliquidWebSocketClient,
1695    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1696    coin_to_instrument_id: &Arc<AtomicMap<Ustr, InstrumentId>>,
1697    data_sender: &tokio::sync::mpsc::UnboundedSender<DataEvent>,
1698) -> anyhow::Result<InstrumentRefresh> {
1699    let _update_guard = update_lock.lock().await;
1700
1701    let fetched = http_client
1702        .request_instruments()
1703        .await
1704        .context("failed to fetch Hyperliquid instruments")?;
1705
1706    Ok(reconcile_instruments(
1707        fetched,
1708        http_client,
1709        ws_client,
1710        instruments_by_id,
1711        coin_to_instrument_id,
1712        data_sender,
1713    )
1714    .await)
1715}
1716
1717/// Reconciles the instrument caches against a fetched instrument universe.
1718///
1719/// Caches and publishes new or materially changed definitions as
1720/// [`DataEvent::Instrument`]. Unchanged definitions are not republished. Cached
1721/// instruments absent from `fetched` are retained because they may still back
1722/// open subscriptions.
1723///
1724/// `instruments_by_id` is the baseline later passes diff against, so every path
1725/// that writes fetched instruments into it must come through here, under the
1726/// update lock taken by [`refresh_instruments`]. A market cached without being
1727/// published would compare as unchanged on every later pass, and the execution
1728/// client would never receive its asset index.
1729async fn reconcile_instruments(
1730    fetched: Vec<InstrumentAny>,
1731    http_client: &HyperliquidHttpClient,
1732    ws_client: &HyperliquidWebSocketClient,
1733    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1734    coin_to_instrument_id: &Arc<AtomicMap<Ustr, InstrumentId>>,
1735    data_sender: &tokio::sync::mpsc::UnboundedSender<DataEvent>,
1736) -> InstrumentRefresh {
1737    let changed = changed_definitions(&fetched, instruments_by_id);
1738    let added = added_symbols(&changed, instruments_by_id);
1739
1740    cache_instruments(
1741        &changed,
1742        instruments_by_id,
1743        coin_to_instrument_id,
1744        http_client,
1745        ws_client,
1746    );
1747
1748    // publish before awaiting the mapping rebuild, so a pass cancelled during the
1749    // rebuild cannot leave these definitions cached but unpublished
1750    for instrument in &changed {
1751        if let Err(e) = data_sender.send(DataEvent::Instrument(instrument.clone())) {
1752            log::warn!("Failed to send instrument: {e}");
1753        }
1754    }
1755
1756    rebuild_all_dex_asset_ctxs_mapping(http_client, ws_client).await;
1757
1758    InstrumentRefresh {
1759        added,
1760        changed: changed.len(),
1761        fetched,
1762    }
1763}
1764
1765/// Returns the fetched instruments that are new or materially changed.
1766fn changed_definitions(
1767    fetched: &[InstrumentAny],
1768    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1769) -> Vec<InstrumentAny> {
1770    fetched
1771        .iter()
1772        .filter(|instrument| {
1773            instruments_by_id
1774                .get_cloned(&instrument.id())
1775                .is_none_or(|cached| !instrument_definitions_match(&cached, instrument))
1776        })
1777        .cloned()
1778        .collect()
1779}
1780
1781/// Returns the symbols of the changed definitions the cache has never held.
1782///
1783/// A market listed after startup is the case the refresh exists for, so a pass
1784/// reports which symbols became tradable rather than only how many definitions
1785/// moved. Callers pass the changed set, so a definition that merely moved its
1786/// tick size is not reported as new.
1787fn added_symbols(
1788    changed: &[InstrumentAny],
1789    instruments_by_id: &Arc<AtomicMap<InstrumentId, InstrumentAny>>,
1790) -> Vec<Ustr> {
1791    changed
1792        .iter()
1793        .filter(|instrument| instruments_by_id.get_cloned(&instrument.id()).is_none())
1794        .map(|instrument| instrument.symbol().inner())
1795        .collect()
1796}
1797
1798/// Returns `true` when two instruments carry the same tradable definition,
1799/// ignoring event timestamps.
1800///
1801/// Comparison runs on the serialized form so every venue field, including the
1802/// free-form `info` metadata, participates without listing each field.
1803fn instrument_definitions_match(a: &InstrumentAny, b: &InstrumentAny) -> bool {
1804    fn normalized(instrument: &InstrumentAny) -> Option<serde_json::Value> {
1805        let mut value = serde_json::to_value(instrument).ok()?;
1806
1807        if let Some(definition) = value
1808            .as_object_mut()
1809            .and_then(|obj| obj.values_mut().next())
1810            .and_then(serde_json::Value::as_object_mut)
1811        {
1812            definition.remove("ts_event");
1813            definition.remove("ts_init");
1814        }
1815
1816        Some(value)
1817    }
1818
1819    // A serialization failure compares as changed so updates are never suppressed
1820    match (normalized(a), normalized(b)) {
1821        (Some(a), Some(b)) => a == b,
1822        _ => false,
1823    }
1824}
1825
1826#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1827enum MarketDataChannel {
1828    Deltas,
1829    Depth10,
1830    Quote,
1831}
1832
1833impl MarketDataChannel {
1834    const fn as_str(self) -> &'static str {
1835        match self {
1836            Self::Deltas => "deltas",
1837            Self::Depth10 => "depth10",
1838            Self::Quote => "quote",
1839        }
1840    }
1841}
1842
1843type MarketDataStreamKey = (MarketDataChannel, InstrumentId);
1844
1845#[derive(Debug, Clone)]
1846struct MarketDataStreamHealth {
1847    last_receive_at: Instant,
1848    last_venue_ts_event: Option<UnixNanos>,
1849    consecutive_stale_count: u32,
1850    last_warning_at: Option<Instant>,
1851    last_recovery_at: Option<Instant>,
1852    resubscribe_attempts: u32,
1853}
1854
1855impl MarketDataStreamHealth {
1856    fn new(receive_at: Instant) -> Self {
1857        Self {
1858            last_receive_at: receive_at,
1859            last_venue_ts_event: None,
1860            consecutive_stale_count: 0,
1861            last_warning_at: None,
1862            last_recovery_at: None,
1863            resubscribe_attempts: 0,
1864        }
1865    }
1866
1867    fn record_receive(&mut self, receive_at: Instant, venue_ts_event: UnixNanos) {
1868        self.last_receive_at = receive_at;
1869        self.last_venue_ts_event = Some(venue_ts_event);
1870        self.consecutive_stale_count = 0;
1871        self.last_warning_at = None;
1872        self.last_recovery_at = None;
1873        self.resubscribe_attempts = 0;
1874    }
1875}
1876
1877#[derive(Debug, Clone, Copy)]
1878struct StreamRecoveryConfig {
1879    cooldown: Duration,
1880    max_targeted_resubscribes: u32,
1881}
1882
1883#[derive(Debug)]
1884struct MarketDataStreamHealthMonitor {
1885    stale_receive_threshold: Duration,
1886    warning_cooldown: Duration,
1887    recovery: Option<StreamRecoveryConfig>,
1888    streams: AHashMap<MarketDataStreamKey, MarketDataStreamHealth>,
1889}
1890
1891impl MarketDataStreamHealthMonitor {
1892    fn new(stale_receive_threshold: Duration, warning_cooldown: Duration) -> Self {
1893        Self {
1894            stale_receive_threshold,
1895            warning_cooldown,
1896            recovery: None,
1897            streams: AHashMap::new(),
1898        }
1899    }
1900
1901    fn with_recovery(mut self, cooldown: Duration, max_targeted_resubscribes: u32) -> Self {
1902        self.recovery = Some(StreamRecoveryConfig {
1903            cooldown,
1904            max_targeted_resubscribes,
1905        });
1906        self
1907    }
1908
1909    fn subscribe(
1910        &mut self,
1911        channel: MarketDataChannel,
1912        instrument_id: InstrumentId,
1913        receive_at: Instant,
1914    ) {
1915        self.streams.insert(
1916            (channel, instrument_id),
1917            MarketDataStreamHealth::new(receive_at),
1918        );
1919    }
1920
1921    fn unsubscribe(&mut self, channel: MarketDataChannel, instrument_id: InstrumentId) {
1922        self.streams.remove(&(channel, instrument_id));
1923    }
1924
1925    fn clear(&mut self) {
1926        self.streams.clear();
1927    }
1928
1929    fn record_receive(
1930        &mut self,
1931        channel: MarketDataChannel,
1932        instrument_id: InstrumentId,
1933        receive_at: Instant,
1934        venue_ts_event: UnixNanos,
1935    ) {
1936        if let Some(stream) = self.streams.get_mut(&(channel, instrument_id)) {
1937            stream.record_receive(receive_at, venue_ts_event);
1938        }
1939    }
1940
1941    fn check_stale(
1942        &mut self,
1943        now: Instant,
1944        wall_clock_now: UnixNanos,
1945    ) -> Vec<MarketDataStaleEvent> {
1946        // Fresh BBO makes stale book streams relative-stale, not transport-stale
1947        let fresh_quote_instruments: AHashSet<InstrumentId> = self
1948            .streams
1949            .iter()
1950            .filter(|((channel, _), stream)| {
1951                *channel == MarketDataChannel::Quote
1952                    && now.saturating_duration_since(stream.last_receive_at)
1953                        < self.stale_receive_threshold
1954            })
1955            .map(|((_, instrument_id), _)| *instrument_id)
1956            .collect();
1957
1958        let mut events = Vec::new();
1959
1960        for ((channel, instrument_id), stream) in &mut self.streams {
1961            let receive_age = now.saturating_duration_since(stream.last_receive_at);
1962            if receive_age < self.stale_receive_threshold {
1963                stream.consecutive_stale_count = 0;
1964                continue;
1965            }
1966
1967            stream.consecutive_stale_count = stream.consecutive_stale_count.saturating_add(1);
1968
1969            let quote_is_fresh = matches!(
1970                channel,
1971                MarketDataChannel::Deltas | MarketDataChannel::Depth10
1972            ) && fresh_quote_instruments.contains(instrument_id);
1973
1974            let venue_age = stream.last_venue_ts_event.map(|ts_event| {
1975                Duration::from_nanos(wall_clock_now.as_u64().saturating_sub(ts_event.as_u64()))
1976            });
1977
1978            if let Some(recovery) = self.recovery {
1979                // Recovery requires one prior warning, even after long check lag
1980                let stale_since = stream.last_receive_at + self.stale_receive_threshold;
1981                let anchor = stream.last_recovery_at.unwrap_or(stale_since);
1982
1983                if stream.last_warning_at.is_some()
1984                    && now.saturating_duration_since(anchor) >= recovery.cooldown
1985                {
1986                    let action = if stream.resubscribe_attempts < recovery.max_targeted_resubscribes
1987                    {
1988                        stream.resubscribe_attempts += 1;
1989                        StaleStreamAction::Resubscribe
1990                    } else {
1991                        // Reconnect replays all active subscriptions
1992                        stream.resubscribe_attempts = 0;
1993                        StaleStreamAction::Reconnect
1994                    };
1995                    stream.last_recovery_at = Some(now);
1996                    stream.last_warning_at = Some(now);
1997
1998                    events.push(MarketDataStaleEvent {
1999                        channel: *channel,
2000                        instrument_id: *instrument_id,
2001                        receive_age,
2002                        venue_age,
2003                        stale_count: stream.consecutive_stale_count,
2004                        action,
2005                        cooldown: recovery.cooldown,
2006                        quote_is_fresh,
2007                    });
2008                    continue;
2009                }
2010            }
2011
2012            let should_warn = stream.last_warning_at.is_none_or(|last_warning_at| {
2013                now.saturating_duration_since(last_warning_at) >= self.warning_cooldown
2014            });
2015
2016            if !should_warn {
2017                continue;
2018            }
2019
2020            stream.last_warning_at = Some(now);
2021            events.push(MarketDataStaleEvent {
2022                channel: *channel,
2023                instrument_id: *instrument_id,
2024                receive_age,
2025                venue_age,
2026                stale_count: stream.consecutive_stale_count,
2027                action: StaleStreamAction::Warn,
2028                cooldown: self.warning_cooldown,
2029                quote_is_fresh,
2030            });
2031        }
2032
2033        events
2034    }
2035}
2036
2037#[derive(Debug, Clone, Copy, PartialEq, Eq)]
2038enum StaleStreamAction {
2039    Warn,
2040    Resubscribe,
2041    Reconnect,
2042}
2043
2044impl StaleStreamAction {
2045    const fn as_str(self) -> &'static str {
2046        match self {
2047            Self::Warn => "warn",
2048            Self::Resubscribe => "resubscribe",
2049            Self::Reconnect => "reconnect",
2050        }
2051    }
2052}
2053
2054#[derive(Debug, Clone, PartialEq, Eq)]
2055struct MarketDataStaleEvent {
2056    channel: MarketDataChannel,
2057    instrument_id: InstrumentId,
2058    receive_age: Duration,
2059    venue_age: Option<Duration>,
2060    stale_count: u32,
2061    action: StaleStreamAction,
2062    cooldown: Duration,
2063    quote_is_fresh: bool,
2064}
2065
2066fn stream_health_update(
2067    msg: &NautilusWsMessage,
2068) -> Option<(MarketDataChannel, InstrumentId, UnixNanos)> {
2069    match msg {
2070        NautilusWsMessage::Quote(quote) => Some((
2071            MarketDataChannel::Quote,
2072            quote.instrument_id,
2073            quote.ts_event,
2074        )),
2075        NautilusWsMessage::Deltas(deltas) => Some((
2076            MarketDataChannel::Deltas,
2077            deltas.instrument_id,
2078            deltas.ts_event,
2079        )),
2080        NautilusWsMessage::Depth10(depth) => Some((
2081            MarketDataChannel::Depth10,
2082            depth.instrument_id,
2083            depth.ts_event,
2084        )),
2085        _ => None,
2086    }
2087}
2088
2089fn record_stream_receive(
2090    stream_health: &Arc<Mutex<MarketDataStreamHealthMonitor>>,
2091    channel: MarketDataChannel,
2092    instrument_id: InstrumentId,
2093    venue_ts_event: UnixNanos,
2094) {
2095    stream_health
2096        .lock()
2097        .record_receive(channel, instrument_id, Instant::now(), venue_ts_event);
2098}
2099
2100fn log_stream_health_event(event: &MarketDataStaleEvent) {
2101    let venue_age_ms = event
2102        .venue_age
2103        .map_or_else(|| "n/a".to_string(), |age| age.as_millis().to_string());
2104    let prefix = if event.quote_is_fresh {
2105        "Hyperliquid book stream stale while bbo advances"
2106    } else {
2107        "Hyperliquid market data stream stale"
2108    };
2109
2110    log::warn!(
2111        "{prefix}: channel={}, instrument_id={}, receive_age_ms={}, venue_age_ms={}, \
2112         stale_count={}, action={}, cooldown_secs={}",
2113        event.channel.as_str(),
2114        event.instrument_id,
2115        event.receive_age.as_millis(),
2116        venue_age_ms,
2117        event.stale_count,
2118        event.action.as_str(),
2119        event.cooldown.as_secs(),
2120    );
2121}
2122
2123async fn handle_stream_health_events(
2124    ws_client: &HyperliquidWebSocketClient,
2125    events: &[MarketDataStaleEvent],
2126) {
2127    // Deltas and depth10 share one venue `l2Book` stream
2128    let mut resubscribed_books: AHashSet<InstrumentId> = AHashSet::new();
2129    let mut reconnect_requested = false;
2130
2131    for event in events {
2132        log_stream_health_event(event);
2133
2134        match event.action {
2135            StaleStreamAction::Warn => {}
2136            StaleStreamAction::Resubscribe => match event.channel {
2137                MarketDataChannel::Deltas | MarketDataChannel::Depth10 => {
2138                    if resubscribed_books.insert(event.instrument_id)
2139                        && let Err(e) = ws_client.resubscribe_book(event.instrument_id).await
2140                    {
2141                        log::warn!(
2142                            "Failed targeted l2Book resubscribe for {}: {e}",
2143                            event.instrument_id,
2144                        );
2145                    }
2146                }
2147                MarketDataChannel::Quote => {
2148                    if let Err(e) = ws_client.resubscribe_quotes(event.instrument_id).await {
2149                        log::warn!(
2150                            "Failed targeted bbo resubscribe for {}: {e}",
2151                            event.instrument_id,
2152                        );
2153                    }
2154                }
2155            },
2156            StaleStreamAction::Reconnect => reconnect_requested = true,
2157        }
2158    }
2159
2160    if reconnect_requested {
2161        if ws_client.request_reconnect() {
2162            log::warn!("Requested full WebSocket reconnect after failed targeted stream recovery");
2163        } else {
2164            log::debug!("Skipping reconnect request: connection not active");
2165        }
2166    }
2167}
2168
2169// Applies the request window and limit to a snapshot of recent trades. `trades`
2170// must be sorted ascending by `ts_event`. Returns the subset within `[start, end]`
2171// (each bound unbounded when `None`), keeping at most the most recent `limit`
2172// trades. Because `recentTrades` exposes only a recent snapshot with no historical
2173// depth, a warning is logged when the request reaches below the snapshot's
2174// coverage floor (its oldest trade).
2175fn filter_recent_trades(
2176    trades: Vec<TradeTick>,
2177    start: Option<UnixNanos>,
2178    end: Option<UnixNanos>,
2179    limit: Option<usize>,
2180    instrument_id: InstrumentId,
2181) -> Vec<TradeTick> {
2182    let Some(floor) = trades.first().map(|trade| trade.ts_event) else {
2183        return Vec::new();
2184    };
2185
2186    if let Some(end) = end
2187        && end < floor
2188    {
2189        log::warn!(
2190            "Recent trades for {instrument_id} are entirely older than the requested window; \
2191             snapshot only covers back to {}",
2192            unix_nanos_to_iso8601(floor),
2193        );
2194        return Vec::new();
2195    }
2196
2197    if let Some(start) = start
2198        && start < floor
2199    {
2200        log::warn!(
2201            "Recent trades for {instrument_id} only cover back to {}; \
2202             the requested start is earlier and cannot be served",
2203            unix_nanos_to_iso8601(floor),
2204        );
2205    }
2206
2207    let mut filtered: Vec<TradeTick> = trades
2208        .into_iter()
2209        .filter(|trade| start.is_none_or(|s| trade.ts_event >= s))
2210        .filter(|trade| end.is_none_or(|e| trade.ts_event <= e))
2211        .collect();
2212
2213    if let Some(limit) = limit
2214        && filtered.len() > limit
2215    {
2216        // Keep the most recent `limit` trades; ascending order is preserved
2217        filtered.drain(0..filtered.len() - limit);
2218    }
2219
2220    filtered
2221}
2222
2223// Levels with unparsable px/sz or non-positive size are skipped rather than
2224// erroring; the snapshot's `time` field (ms) becomes `ts_event` after the
2225// ms->ns conversion.
2226pub(crate) fn parse_l2_book_snapshot(
2227    l2_book: &HyperliquidL2Book,
2228    instrument_id: InstrumentId,
2229    price_precision: u8,
2230    size_precision: u8,
2231    depth: Option<usize>,
2232) -> OrderBook {
2233    let mut book = OrderBook::new(instrument_id, BookType::L2_MBP);
2234    let ts_event = UnixNanos::from(l2_book.time * 1_000_000);
2235
2236    let all_bids = l2_book
2237        .levels
2238        .first()
2239        .map_or([].as_slice(), |v| v.as_slice());
2240    let all_asks = l2_book
2241        .levels
2242        .get(1)
2243        .map_or([].as_slice(), |v| v.as_slice());
2244
2245    let bids = match depth {
2246        Some(d) if d < all_bids.len() => &all_bids[..d],
2247        _ => all_bids,
2248    };
2249    let asks = match depth {
2250        Some(d) if d < all_asks.len() => &all_asks[..d],
2251        _ => all_asks,
2252    };
2253
2254    for (i, level) in bids.iter().enumerate() {
2255        if level.sz <= Decimal::ZERO {
2256            continue;
2257        }
2258        let Ok(price) = Price::from_decimal_dp(level.px, price_precision) else {
2259            continue;
2260        };
2261        let Ok(size) = Quantity::from_decimal_dp(level.sz, size_precision) else {
2262            continue;
2263        };
2264
2265        let order = BookOrder::new(OrderSide::Buy, price, size, i as u64);
2266        book.add(order, 0, i as u64, ts_event);
2267    }
2268
2269    let bids_len = bids.len();
2270
2271    for (i, level) in asks.iter().enumerate() {
2272        if level.sz <= Decimal::ZERO {
2273            continue;
2274        }
2275        let Ok(price) = Price::from_decimal_dp(level.px, price_precision) else {
2276            continue;
2277        };
2278        let Ok(size) = Quantity::from_decimal_dp(level.sz, size_precision) else {
2279            continue;
2280        };
2281
2282        let order = BookOrder::new(OrderSide::Sell, price, size, (bids_len + i) as u64);
2283        book.add(order, 0, (bids_len + i) as u64, ts_event);
2284    }
2285
2286    log::debug!(
2287        "Built order book for {instrument_id} with {} bids and {} asks",
2288        bids.len(),
2289        asks.len(),
2290    );
2291
2292    book
2293}
2294
2295// Reads optional `nSigFigs` / `mantissa` L2 precision controls from
2296// `subscribe_params`; bails on non-positive integer values.
2297pub(crate) fn parse_book_precision_params(
2298    params: Option<&Params>,
2299) -> anyhow::Result<(Option<u32>, Option<u32>)> {
2300    let Some(params) = params else {
2301        return Ok((None, None));
2302    };
2303
2304    let read_u32 = |key: &str| -> anyhow::Result<Option<u32>> {
2305        match params.get(key) {
2306            None => Ok(None),
2307            Some(v) => v
2308                .as_u64()
2309                .and_then(|n| u32::try_from(n).ok())
2310                .ok_or_else(|| anyhow::anyhow!("`{key}` must be a positive u32"))
2311                .map(Some),
2312        }
2313    };
2314
2315    Ok((read_u32("n_sig_figs")?, read_u32("mantissa")?))
2316}
2317
2318// Hyperliquid funds perpetuals hourly, so `interval` is fixed at 60 mins;
2319// `time` from the venue marks the end of the funding interval in ms.
2320pub(crate) fn funding_entry_to_update(
2321    entry: &HyperliquidFundingHistoryEntry,
2322    instrument_id: InstrumentId,
2323) -> FundingRateUpdate {
2324    let rate = entry.funding_rate;
2325    let ts = UnixNanos::from(entry.time * 1_000_000);
2326    FundingRateUpdate::new(instrument_id, rate, Some(60), None, ts, ts)
2327}
2328
2329pub(crate) fn candle_to_bar(
2330    candle: &HyperliquidCandle,
2331    bar_type: BarType,
2332    price_precision: u8,
2333    size_precision: u8,
2334) -> anyhow::Result<Bar> {
2335    let ts_event = millis_to_nanos(candle.timestamp)?;
2336    let close_boundary = candle
2337        .end_timestamp
2338        .checked_add(1)
2339        .context("candle close boundary overflow")?;
2340    let ts_init = millis_to_nanos(close_boundary)?;
2341
2342    let open = Price::from_decimal_dp(candle.open, price_precision)
2343        .map_err(|e| anyhow::anyhow!("invalid open price: {e}"))?;
2344    let high = Price::from_decimal_dp(candle.high, price_precision)
2345        .map_err(|e| anyhow::anyhow!("invalid high price: {e}"))?;
2346    let low = Price::from_decimal_dp(candle.low, price_precision)
2347        .map_err(|e| anyhow::anyhow!("invalid low price: {e}"))?;
2348    let close = Price::from_decimal_dp(candle.close, price_precision)
2349        .map_err(|e| anyhow::anyhow!("invalid close price: {e}"))?;
2350    let volume = Quantity::from_decimal_dp(candle.volume, size_precision)
2351        .map_err(|e| anyhow::anyhow!("invalid volume: {e}"))?;
2352
2353    Ok(Bar::new(
2354        bar_type, open, high, low, close, volume, ts_event, ts_init,
2355    ))
2356}
2357
2358/// Request bars from HTTP API.
2359async fn request_bars_from_http(
2360    http_client: HyperliquidHttpClient,
2361    bar_type: BarType,
2362    start: Option<Timestamp>,
2363    end: Option<Timestamp>,
2364    limit: Option<u32>,
2365    instruments: Arc<AtomicMap<InstrumentId, InstrumentAny>>,
2366) -> anyhow::Result<Vec<Bar>> {
2367    // Get instrument details for precision
2368    let instrument_id = bar_type.instrument_id();
2369    let instrument = instruments
2370        .load()
2371        .get(&instrument_id)
2372        .cloned()
2373        .context("instrument not found in cache")?;
2374
2375    let price_precision = instrument.price_precision();
2376    let size_precision = instrument.size_precision();
2377    let raw_symbol = instrument.raw_symbol();
2378    let coin = raw_symbol.as_str();
2379
2380    let interval = bar_type_to_interval(&bar_type)?;
2381
2382    // Hyperliquid uses millisecond timestamps
2383    let now = Timestamp::now();
2384    let end_time = end.unwrap_or(now).as_millisecond() as u64;
2385    let start_time = if let Some(start) = start {
2386        start.as_millisecond() as u64
2387    } else {
2388        // Default to 1000 bars before end_time
2389        let spec = bar_type.spec();
2390        let step_ms = match spec.aggregation {
2391            BarAggregation::Minute => spec.step.get() as u64 * 60_000,
2392            BarAggregation::Hour => spec.step.get() as u64 * 3_600_000,
2393            BarAggregation::Day => spec.step.get() as u64 * 86_400_000,
2394            _ => 60_000,
2395        };
2396        end_time.saturating_sub(1000 * step_ms)
2397    };
2398
2399    let candles = http_client
2400        .info_candle_snapshot(coin, interval, start_time, end_time)
2401        .await
2402        .context("failed to fetch candle snapshot from Hyperliquid")?;
2403
2404    let now_ms = now.as_millisecond() as u64;
2405    let mut bars: Vec<Bar> = candles
2406        .iter()
2407        .filter(|candle| candle.end_timestamp < now_ms)
2408        .filter_map(|candle| {
2409            candle_to_bar(candle, bar_type, price_precision, size_precision)
2410                .map_err(|e| {
2411                    log::warn!("Failed to convert candle to bar: {e}");
2412                    e
2413                })
2414                .ok()
2415        })
2416        .collect();
2417
2418    if let Some(limit) = limit
2419        && bars.len() > limit as usize
2420    {
2421        bars = bars.into_iter().take(limit as usize).collect();
2422    }
2423
2424    log::debug!("Fetched {} bars for {}", bars.len(), bar_type);
2425    Ok(bars)
2426}
2427
2428#[cfg(test)]
2429mod tests {
2430    use nautilus_common::live::runner::set_data_event_sender;
2431    use nautilus_model::{
2432        data::{
2433            QuoteTick,
2434            stubs::{stub_deltas, stub_depth10},
2435        },
2436        enums::{AggressorSide, CurrencyType},
2437        identifiers::{Symbol, TradeId},
2438        instruments::CryptoPerpetual,
2439        types::Currency,
2440    };
2441    use rstest::rstest;
2442    use rust_decimal_macros::dec;
2443    use ustr::Ustr;
2444
2445    use super::*;
2446    use crate::common::{consts::HYPERLIQUID_CLIENT_ID, testing::load_test_data};
2447
2448    fn btc_perp_id() -> InstrumentId {
2449        InstrumentId::from("BTC-PERP.HYPERLIQUID")
2450    }
2451
2452    #[rstest]
2453    fn test_candle_to_bar_uses_causal_initialization_timestamp() {
2454        let candle = HyperliquidCandle {
2455            timestamp: 1_700_000_000_000,
2456            end_timestamp: 1_700_000_059_999,
2457            open: dec!(100.0),
2458            high: dec!(101.0),
2459            low: dec!(99.0),
2460            close: dec!(100.5),
2461            volume: dec!(10.0),
2462            num_trades: Some(42),
2463        };
2464        let bar_type = BarType::from("BTC-USD-PERP.HYPERLIQUID-1-MINUTE-LAST-EXTERNAL");
2465
2466        let bar = candle_to_bar(&candle, bar_type, 1, 1).unwrap();
2467
2468        assert_eq!(candle.end_timestamp - candle.timestamp, 59_999);
2469        assert_eq!(bar.ts_event, millis_to_nanos(candle.timestamp).unwrap());
2470        assert_eq!(
2471            bar.ts_init,
2472            millis_to_nanos(candle.end_timestamp + 1).unwrap()
2473        );
2474        assert!(bar.ts_init > bar.ts_event);
2475    }
2476
2477    #[rstest]
2478    fn test_candle_to_bar_rejects_close_boundary_overflow() {
2479        let candle = HyperliquidCandle {
2480            timestamp: 1_700_000_000_000,
2481            end_timestamp: u64::MAX,
2482            open: dec!(100.0),
2483            high: dec!(101.0),
2484            low: dec!(99.0),
2485            close: dec!(100.5),
2486            volume: dec!(10.0),
2487            num_trades: Some(42),
2488        };
2489        let bar_type = BarType::from("BTC-USD-PERP.HYPERLIQUID-1-MINUTE-LAST-EXTERNAL");
2490
2491        let err = candle_to_bar(&candle, bar_type, 1, 1).unwrap_err();
2492
2493        assert!(err.to_string().contains("close boundary overflow"));
2494    }
2495
2496    #[rstest]
2497    fn test_stream_health_monitor_fresh_stream_does_not_warn() {
2498        let mut monitor =
2499            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2500        let instrument_id = btc_perp_id();
2501        let start = Instant::now();
2502
2503        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2504
2505        let warnings = monitor.check_stale(
2506            start + Duration::from_secs(4),
2507            UnixNanos::from(4_000_000_000),
2508        );
2509        assert!(warnings.is_empty());
2510    }
2511
2512    #[rstest]
2513    fn test_stream_health_monitor_warns_once_after_threshold() {
2514        let mut monitor =
2515            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2516        let instrument_id = btc_perp_id();
2517        let start = Instant::now();
2518
2519        monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
2520        monitor.record_receive(
2521            MarketDataChannel::Quote,
2522            instrument_id,
2523            start + Duration::from_secs(1),
2524            UnixNanos::from(1_000_000_000),
2525        );
2526
2527        let warnings = monitor.check_stale(
2528            start + Duration::from_secs(7),
2529            UnixNanos::from(9_000_000_000),
2530        );
2531
2532        assert_eq!(
2533            warnings,
2534            vec![MarketDataStaleEvent {
2535                channel: MarketDataChannel::Quote,
2536                instrument_id,
2537                receive_age: Duration::from_secs(6),
2538                venue_age: Some(Duration::from_secs(8)),
2539                stale_count: 1,
2540                action: StaleStreamAction::Warn,
2541                cooldown: Duration::from_secs(30),
2542                quote_is_fresh: false,
2543            }]
2544        );
2545    }
2546
2547    #[rstest]
2548    fn test_stream_health_monitor_warns_at_receive_threshold() {
2549        let mut monitor =
2550            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2551        let instrument_id = btc_perp_id();
2552        let start = Instant::now();
2553
2554        monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
2555
2556        let warnings = monitor.check_stale(
2557            start + Duration::from_secs(5),
2558            UnixNanos::from(5_000_000_000),
2559        );
2560
2561        assert_eq!(warnings.len(), 1);
2562        assert_eq!(warnings[0].receive_age, Duration::from_secs(5));
2563        assert_eq!(warnings[0].stale_count, 1);
2564    }
2565
2566    #[rstest]
2567    fn test_stream_health_monitor_new_update_resets_age_and_stale_count() {
2568        let mut monitor =
2569            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2570        let instrument_id = btc_perp_id();
2571        let start = Instant::now();
2572
2573        monitor.subscribe(MarketDataChannel::Depth10, instrument_id, start);
2574        assert_eq!(
2575            monitor
2576                .check_stale(
2577                    start + Duration::from_secs(6),
2578                    UnixNanos::from(6_000_000_000),
2579                )
2580                .len(),
2581            1,
2582        );
2583
2584        monitor.record_receive(
2585            MarketDataChannel::Depth10,
2586            instrument_id,
2587            start + Duration::from_secs(7),
2588            UnixNanos::from(7_000_000_000),
2589        );
2590
2591        assert!(
2592            monitor
2593                .check_stale(
2594                    start + Duration::from_secs(11),
2595                    UnixNanos::from(11_000_000_000),
2596                )
2597                .is_empty()
2598        );
2599
2600        let warnings = monitor.check_stale(
2601            start + Duration::from_secs(13),
2602            UnixNanos::from(13_000_000_000),
2603        );
2604        assert_eq!(warnings.len(), 1);
2605        assert_eq!(warnings[0].stale_count, 1);
2606        assert_eq!(warnings[0].receive_age, Duration::from_secs(6));
2607    }
2608
2609    #[rstest]
2610    fn test_stream_health_monitor_unsubscribe_removes_stream() {
2611        let mut monitor =
2612            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2613        let instrument_id = btc_perp_id();
2614        let start = Instant::now();
2615
2616        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2617        monitor.unsubscribe(MarketDataChannel::Deltas, instrument_id);
2618
2619        let warnings = monitor.check_stale(
2620            start + Duration::from_secs(6),
2621            UnixNanos::from(6_000_000_000),
2622        );
2623
2624        assert!(warnings.is_empty());
2625    }
2626
2627    #[rstest]
2628    #[case(0, 15)]
2629    #[case(120, 0)]
2630    fn test_data_client_stream_health_config_zero_disables_monitor(
2631        #[case] stale_receive_timeout_secs: u64,
2632        #[case] check_interval_secs: u64,
2633    ) {
2634        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
2635        set_data_event_sender(tx);
2636        let client = HyperliquidDataClient::new(
2637            *crate::common::consts::HYPERLIQUID_CLIENT_ID,
2638            HyperliquidDataClientConfig {
2639                stale_stream_receive_timeout_secs: stale_receive_timeout_secs,
2640                stream_health_check_interval_secs: check_interval_secs,
2641                ..HyperliquidDataClientConfig::default()
2642            },
2643        )
2644        .unwrap();
2645        let instrument_id = btc_perp_id();
2646        let start = Instant::now();
2647
2648        assert!(!client.stream_health_monitor_enabled());
2649        client.register_stream_health(MarketDataChannel::Deltas, instrument_id);
2650
2651        let warnings = client.stream_health.lock().check_stale(
2652            start + Duration::from_secs(121),
2653            UnixNanos::from(121_000_000_000),
2654        );
2655
2656        assert!(warnings.is_empty());
2657    }
2658
2659    #[rstest]
2660    fn test_data_client_recovery_requires_positive_cooldown() {
2661        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
2662        set_data_event_sender(tx);
2663        let client = HyperliquidDataClient::new(
2664            *crate::common::consts::HYPERLIQUID_CLIENT_ID,
2665            HyperliquidDataClientConfig {
2666                stale_stream_recovery_enabled: true,
2667                stale_stream_recovery_cooldown_secs: 0,
2668                ..HyperliquidDataClientConfig::default()
2669            },
2670        )
2671        .unwrap();
2672
2673        assert!(
2674            client.stream_health.lock().recovery.is_none(),
2675            "a zero recovery cooldown must leave the monitor observability-only",
2676        );
2677    }
2678
2679    #[rstest]
2680    fn test_stream_health_monitor_warning_cooldown_prevents_repeated_logs() {
2681        let mut monitor =
2682            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10));
2683        let instrument_id = btc_perp_id();
2684        let start = Instant::now();
2685
2686        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2687
2688        let first = monitor.check_stale(
2689            start + Duration::from_secs(6),
2690            UnixNanos::from(6_000_000_000),
2691        );
2692        let inside_cooldown = monitor.check_stale(
2693            start + Duration::from_secs(7),
2694            UnixNanos::from(7_000_000_000),
2695        );
2696        let second = monitor.check_stale(
2697            start + Duration::from_secs(16),
2698            UnixNanos::from(16_000_000_000),
2699        );
2700
2701        assert_eq!(first.len(), 1);
2702        assert!(inside_cooldown.is_empty());
2703        assert_eq!(second.len(), 1);
2704        assert_eq!(second[0].stale_count, 3);
2705    }
2706
2707    fn check_at(
2708        monitor: &mut MarketDataStreamHealthMonitor,
2709        start: Instant,
2710        secs: u64,
2711    ) -> Vec<MarketDataStaleEvent> {
2712        monitor.check_stale(
2713            start + Duration::from_secs(secs),
2714            UnixNanos::from(secs * 1_000_000_000),
2715        )
2716    }
2717
2718    #[rstest]
2719    fn test_stream_health_recovery_ladder_escalates_and_resets() {
2720        let mut monitor =
2721            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
2722                .with_recovery(Duration::from_secs(30), 2);
2723        let instrument_id = btc_perp_id();
2724        let start = Instant::now();
2725
2726        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2727
2728        let events = check_at(&mut monitor, start, 5);
2729        assert_eq!(events.len(), 1);
2730        assert_eq!(events[0].action, StaleStreamAction::Warn);
2731
2732        let events = check_at(&mut monitor, start, 20);
2733        assert_eq!(events[0].action, StaleStreamAction::Warn);
2734
2735        let events = check_at(&mut monitor, start, 35);
2736        assert_eq!(
2737            events,
2738            vec![MarketDataStaleEvent {
2739                channel: MarketDataChannel::Deltas,
2740                instrument_id,
2741                receive_age: Duration::from_secs(35),
2742                venue_age: None,
2743                stale_count: 3,
2744                action: StaleStreamAction::Resubscribe,
2745                cooldown: Duration::from_secs(30),
2746                quote_is_fresh: false,
2747            }],
2748        );
2749
2750        let events = check_at(&mut monitor, start, 50);
2751        assert_eq!(events[0].action, StaleStreamAction::Warn);
2752
2753        let events = check_at(&mut monitor, start, 65);
2754        assert_eq!(events[0].action, StaleStreamAction::Resubscribe);
2755
2756        let events = check_at(&mut monitor, start, 95);
2757        assert_eq!(events[0].action, StaleStreamAction::Reconnect);
2758
2759        let events = check_at(&mut monitor, start, 125);
2760        assert_eq!(events[0].action, StaleStreamAction::Resubscribe);
2761    }
2762
2763    #[rstest]
2764    fn test_stream_health_recovery_first_breach_warns_even_past_cooldown() {
2765        let mut monitor =
2766            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
2767                .with_recovery(Duration::from_secs(1), 1);
2768        let instrument_id = btc_perp_id();
2769        let start = Instant::now();
2770
2771        monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
2772
2773        // First stale checks must stay observability-only, even after cooldown
2774        let events = check_at(&mut monitor, start, 40);
2775        assert_eq!(events.len(), 1);
2776        assert_eq!(events[0].action, StaleStreamAction::Warn);
2777
2778        let events = check_at(&mut monitor, start, 41);
2779        assert_eq!(events.len(), 1);
2780        assert_eq!(events[0].action, StaleStreamAction::Resubscribe);
2781    }
2782
2783    #[rstest]
2784    fn test_stream_health_receive_resets_recovery_state() {
2785        let mut monitor =
2786            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(10))
2787                .with_recovery(Duration::from_secs(10), 1);
2788        let instrument_id = btc_perp_id();
2789        let start = Instant::now();
2790
2791        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2792        assert_eq!(
2793            check_at(&mut monitor, start, 5)[0].action,
2794            StaleStreamAction::Warn
2795        );
2796        assert_eq!(
2797            check_at(&mut monitor, start, 15)[0].action,
2798            StaleStreamAction::Resubscribe,
2799        );
2800
2801        monitor.record_receive(
2802            MarketDataChannel::Deltas,
2803            instrument_id,
2804            start + Duration::from_secs(16),
2805            UnixNanos::from(16_000_000_000),
2806        );
2807
2808        assert!(check_at(&mut monitor, start, 20).is_empty());
2809
2810        let events = check_at(&mut monitor, start, 21);
2811        assert_eq!(events[0].action, StaleStreamAction::Warn);
2812        assert_eq!(events[0].stale_count, 1);
2813
2814        assert_eq!(
2815            check_at(&mut monitor, start, 31)[0].action,
2816            StaleStreamAction::Resubscribe,
2817        );
2818        assert_eq!(
2819            check_at(&mut monitor, start, 41)[0].action,
2820            StaleStreamAction::Reconnect,
2821        );
2822    }
2823
2824    #[rstest]
2825    fn test_check_stale_book_with_fresh_quote_flags_relative_staleness() {
2826        let mut monitor =
2827            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2828        let instrument_id = btc_perp_id();
2829        let start = Instant::now();
2830
2831        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2832        monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
2833        monitor.record_receive(
2834            MarketDataChannel::Quote,
2835            instrument_id,
2836            start + Duration::from_secs(8),
2837            UnixNanos::from(8_000_000_000),
2838        );
2839
2840        let events = check_at(&mut monitor, start, 10);
2841
2842        assert_eq!(events.len(), 1, "fresh quote stream must not be reported");
2843        assert_eq!(events[0].channel, MarketDataChannel::Deltas);
2844        assert!(events[0].quote_is_fresh);
2845    }
2846
2847    #[rstest]
2848    #[case(true)]
2849    #[case(false)]
2850    fn test_check_stale_book_without_fresh_quote_is_not_flagged(#[case] quote_subscribed: bool) {
2851        let mut monitor =
2852            MarketDataStreamHealthMonitor::new(Duration::from_secs(5), Duration::from_secs(30));
2853        let instrument_id = btc_perp_id();
2854        let start = Instant::now();
2855
2856        monitor.subscribe(MarketDataChannel::Deltas, instrument_id, start);
2857        if quote_subscribed {
2858            monitor.subscribe(MarketDataChannel::Quote, instrument_id, start);
2859        }
2860
2861        let events = check_at(&mut monitor, start, 10);
2862
2863        let deltas_event = events
2864            .iter()
2865            .find(|event| event.channel == MarketDataChannel::Deltas)
2866            .expect("deltas event");
2867        assert!(
2868            !deltas_event.quote_is_fresh,
2869            "a stale or absent quote stream must not flag relative staleness",
2870        );
2871
2872        if quote_subscribed {
2873            let quote_event = events
2874                .iter()
2875                .find(|event| event.channel == MarketDataChannel::Quote)
2876                .expect("quote event");
2877            assert!(!quote_event.quote_is_fresh);
2878        }
2879    }
2880
2881    #[rstest]
2882    fn test_stream_health_update_extracts_tracked_market_data_messages() {
2883        let quote = QuoteTick {
2884            instrument_id: btc_perp_id(),
2885            ts_event: UnixNanos::from(1),
2886            ..QuoteTick::default()
2887        };
2888        let deltas = stub_deltas();
2889        let depth = stub_depth10();
2890
2891        assert_eq!(
2892            stream_health_update(&NautilusWsMessage::Quote(quote)),
2893            Some((
2894                MarketDataChannel::Quote,
2895                quote.instrument_id,
2896                quote.ts_event
2897            )),
2898        );
2899        assert_eq!(
2900            stream_health_update(&NautilusWsMessage::Deltas(deltas.clone())),
2901            Some((
2902                MarketDataChannel::Deltas,
2903                deltas.instrument_id,
2904                deltas.ts_event
2905            )),
2906        );
2907        assert_eq!(
2908            stream_health_update(&NautilusWsMessage::Depth10(Box::new(depth))),
2909            Some((
2910                MarketDataChannel::Depth10,
2911                depth.instrument_id,
2912                depth.ts_event
2913            )),
2914        );
2915        assert_eq!(stream_health_update(&NautilusWsMessage::Reconnected), None,);
2916    }
2917
2918    #[rstest]
2919    fn test_funding_entry_to_update_parses_positive_rate() {
2920        let entry = HyperliquidFundingHistoryEntry {
2921            coin: Ustr::from("BTC"),
2922            funding_rate: dec!(0.0000125),
2923            premium: Some(dec!(0.00029005)),
2924            time: 1769908800000,
2925        };
2926        let instrument_id = btc_perp_id();
2927
2928        let update = funding_entry_to_update(&entry, instrument_id);
2929
2930        assert_eq!(update.instrument_id, instrument_id);
2931        assert_eq!(update.rate, dec!(0.0000125));
2932        assert_eq!(update.interval, Some(60));
2933        assert!(update.next_funding_ns.is_none());
2934        assert_eq!(update.ts_event, UnixNanos::from(1769908800000 * 1_000_000));
2935        assert_eq!(update.ts_init, update.ts_event);
2936    }
2937
2938    #[rstest]
2939    fn test_funding_entry_to_update_handles_negative_rate() {
2940        let entry = HyperliquidFundingHistoryEntry {
2941            coin: Ustr::from("BTC"),
2942            funding_rate: dec!(-0.0000081),
2943            premium: None,
2944            time: 1769912400000,
2945        };
2946        let update = funding_entry_to_update(&entry, btc_perp_id());
2947        assert_eq!(update.rate, dec!(-0.0000081));
2948    }
2949
2950    #[rstest]
2951    fn test_funding_history_entry_rejects_invalid_rate() {
2952        // The funding rate is now a Decimal field, so an invalid value is
2953        // rejected at deserialization rather than by funding_entry_to_update.
2954        let json = r#"{"coin":"BTC","fundingRate":"not-a-number","time":1769912400000}"#;
2955        assert!(serde_json::from_str::<HyperliquidFundingHistoryEntry>(json).is_err());
2956    }
2957
2958    #[rstest]
2959    fn test_parse_book_precision_params_none() {
2960        let (n, m) = parse_book_precision_params(None).unwrap();
2961        assert_eq!(n, None);
2962        assert_eq!(m, None);
2963    }
2964
2965    fn make_params(json: serde_json::Value) -> Params {
2966        serde_json::from_value(json).expect("valid params payload")
2967    }
2968
2969    #[rstest]
2970    fn test_parse_book_precision_params_only_n_sig_figs() {
2971        let params = make_params(serde_json::json!({"n_sig_figs": 4}));
2972        let (n, m) = parse_book_precision_params(Some(&params)).unwrap();
2973        assert_eq!(n, Some(4));
2974        assert_eq!(m, None);
2975    }
2976
2977    #[rstest]
2978    fn test_parse_book_precision_params_both() {
2979        let params = make_params(serde_json::json!({"n_sig_figs": 5, "mantissa": 2}));
2980        let (n, m) = parse_book_precision_params(Some(&params)).unwrap();
2981        assert_eq!(n, Some(5));
2982        assert_eq!(m, Some(2));
2983    }
2984
2985    #[rstest]
2986    fn test_parse_book_precision_params_rejects_negative() {
2987        let params = make_params(serde_json::json!({"n_sig_figs": -1}));
2988        let err = parse_book_precision_params(Some(&params)).unwrap_err();
2989        assert!(err.to_string().contains("n_sig_figs"));
2990    }
2991
2992    #[rstest]
2993    fn test_funding_history_fixture_parses() {
2994        let entries: Vec<HyperliquidFundingHistoryEntry> =
2995            load_test_data("http_funding_history.json");
2996        assert_eq!(entries.len(), 3);
2997        assert_eq!(entries[0].coin, "BTC");
2998        assert_eq!(entries[0].funding_rate, dec!(0.0000125));
2999        assert_eq!(entries[0].premium, Some(dec!(0.00029005)));
3000        assert!(entries[2].premium.is_none());
3001
3002        let updates: Vec<FundingRateUpdate> = entries
3003            .iter()
3004            .map(|e| funding_entry_to_update(e, btc_perp_id()))
3005            .collect();
3006        assert_eq!(updates.len(), 3);
3007        assert_eq!(updates[0].rate, dec!(0.0000125));
3008        assert_eq!(updates[1].rate, dec!(-0.0000081));
3009        assert_eq!(updates[2].rate, dec!(0.0000033));
3010    }
3011
3012    fn level(px: &str, sz: &str) -> crate::http::models::HyperliquidLevel {
3013        crate::http::models::HyperliquidLevel {
3014            px: px.parse().unwrap(),
3015            sz: sz.parse().unwrap(),
3016        }
3017    }
3018
3019    fn sample_l2_book() -> HyperliquidL2Book {
3020        HyperliquidL2Book {
3021            coin: Ustr::from("BTC"),
3022            levels: vec![
3023                vec![
3024                    level("98450.50", "2.5"),
3025                    level("98449.00", "1.2"),
3026                    level("98448.00", "0.8"),
3027                ],
3028                vec![
3029                    level("98451.00", "1.5"),
3030                    level("98452.00", "2.0"),
3031                    level("98453.00", "0.5"),
3032                ],
3033            ],
3034            time: 1769908800000,
3035        }
3036    }
3037
3038    #[rstest]
3039    fn test_parse_l2_book_snapshot_populates_both_sides() {
3040        let book_data = sample_l2_book();
3041        let instrument_id = btc_perp_id();
3042        let book = parse_l2_book_snapshot(&book_data, instrument_id, 2, 4, None);
3043
3044        assert_eq!(book.instrument_id, instrument_id);
3045        assert_eq!(book.book_type, BookType::L2_MBP);
3046        assert_eq!(book.best_bid_price(), Some(Price::new(98450.50, 2)));
3047        assert_eq!(book.best_ask_price(), Some(Price::new(98451.00, 2)));
3048        assert_eq!(book.best_bid_size(), Some(Quantity::new(2.5, 4)));
3049        assert_eq!(book.best_ask_size(), Some(Quantity::new(1.5, 4)));
3050        assert_eq!(book.update_count, 6);
3051    }
3052
3053    #[rstest]
3054    fn test_parse_l2_book_snapshot_truncates_to_depth() {
3055        let book_data = sample_l2_book();
3056        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, Some(1));
3057
3058        // depth=1 keeps the top of book on each side, drops the rest.
3059        assert_eq!(book.update_count, 2);
3060        assert_eq!(book.best_bid_price(), Some(Price::new(98450.50, 2)));
3061        assert_eq!(book.best_ask_price(), Some(Price::new(98451.00, 2)));
3062    }
3063
3064    #[rstest]
3065    fn test_parse_l2_book_snapshot_uses_venue_time_as_ts_event() {
3066        let book_data = sample_l2_book();
3067        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3068        let expected_ts = UnixNanos::from(1769908800000_u64 * 1_000_000);
3069
3070        // ts_last reflects the last applied delta; every added order
3071        // carries the venue time after the ms->ns conversion.
3072        assert_eq!(book.ts_last, expected_ts);
3073    }
3074
3075    #[rstest]
3076    fn test_parse_l2_book_snapshot_skips_non_positive_size() {
3077        let book_data = HyperliquidL2Book {
3078            coin: Ustr::from("BTC"),
3079            levels: vec![
3080                vec![level("98450.50", "2.5"), level("98449.00", "0")],
3081                vec![level("98451.00", "0"), level("98452.00", "1.5")],
3082            ],
3083            time: 1769908800000,
3084        };
3085        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3086
3087        assert_eq!(book.update_count, 2, "zero-sized levels must be skipped");
3088        assert_eq!(book.best_bid_price(), Some(Price::new(98450.50, 2)));
3089        assert_eq!(book.best_ask_price(), Some(Price::new(98452.00, 2)));
3090    }
3091
3092    #[rstest]
3093    fn test_parse_l2_book_snapshot_skips_zero_size_levels() {
3094        let book_data = HyperliquidL2Book {
3095            coin: Ustr::from("BTC"),
3096            levels: vec![
3097                vec![level("98448.00", "0.0"), level("98449.00", "1.2")],
3098                vec![level("98451.00", "0.0"), level("98452.00", "1.5")],
3099            ],
3100            time: 1769908800000,
3101        };
3102        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3103
3104        // Zero-size levels are skipped; one priced level remains per side.
3105        assert_eq!(book.update_count, 2);
3106        assert_eq!(book.best_bid_price(), Some(Price::new(98449.00, 2)));
3107        assert_eq!(book.best_ask_price(), Some(Price::new(98452.00, 2)));
3108    }
3109
3110    #[rstest]
3111    fn test_parse_l2_book_snapshot_empty_levels_yields_empty_book() {
3112        let book_data = HyperliquidL2Book {
3113            coin: Ustr::from("BTC"),
3114            levels: vec![],
3115            time: 1769908800000,
3116        };
3117        let book = parse_l2_book_snapshot(&book_data, btc_perp_id(), 2, 4, None);
3118
3119        assert_eq!(book.update_count, 0);
3120        assert!(book.best_bid_price().is_none());
3121        assert!(book.best_ask_price().is_none());
3122    }
3123
3124    fn trade_at(ts_ns: u64, tid: u64) -> TradeTick {
3125        TradeTick::new(
3126            btc_perp_id(),
3127            Price::from("104300.0"),
3128            Quantity::from("0.01000"),
3129            AggressorSide::Buy,
3130            TradeId::new(tid.to_string()),
3131            UnixNanos::from(ts_ns),
3132            UnixNanos::from(ts_ns),
3133        )
3134    }
3135
3136    // A snapshot of three trades at 1000/2000/3000ns, sorted ascending. The
3137    // coverage floor (oldest) is 1000ns.
3138    fn sample_trades() -> Vec<TradeTick> {
3139        vec![trade_at(1000, 1), trade_at(2000, 2), trade_at(3000, 3)]
3140    }
3141
3142    #[rstest]
3143    fn test_recent_trades_fixture_parses_and_sorts() {
3144        let raw: Vec<crate::http::models::HyperliquidRecentTrade> =
3145            load_test_data("http_recent_trades_btc.json");
3146        assert_eq!(raw.len(), 3);
3147        // Fixture is newest-first as the venue returns it.
3148        assert_eq!(raw[0].tid, 300003);
3149
3150        let meta: crate::http::models::PerpMeta = load_test_data("http_meta_perp_sample.json");
3151        let defs = crate::http::parse::parse_perp_instruments(&meta, 0).unwrap();
3152        let instrument =
3153            crate::http::parse::create_instrument_from_def(&defs[0], UnixNanos::default()).unwrap();
3154
3155        let mut trades: Vec<TradeTick> = raw
3156            .iter()
3157            .map(|t| parse_recent_trade(t, &instrument).unwrap())
3158            .collect();
3159        trades.sort_by_key(|trade| trade.ts_event);
3160
3161        // Ascending after sort: oldest tid first.
3162        assert_eq!(trades[0].trade_id.to_string(), "300001");
3163        assert_eq!(trades[2].trade_id.to_string(), "300003");
3164        assert!(trades[0].ts_event <= trades[2].ts_event);
3165        // Historical ticks carry ts_init == ts_event.
3166        assert_eq!(trades[0].ts_init, trades[0].ts_event);
3167    }
3168
3169    #[rstest]
3170    fn test_filter_recent_trades_full_window_returns_all() {
3171        let filtered = filter_recent_trades(sample_trades(), None, None, None, btc_perp_id());
3172
3173        assert_eq!(filtered.len(), 3);
3174    }
3175
3176    #[rstest]
3177    fn test_filter_recent_trades_empty_snapshot_returns_empty() {
3178        let filtered = filter_recent_trades(
3179            Vec::new(),
3180            Some(UnixNanos::from(500)),
3181            Some(UnixNanos::from(2500)),
3182            None,
3183            btc_perp_id(),
3184        );
3185
3186        assert!(filtered.is_empty());
3187    }
3188
3189    #[rstest]
3190    fn test_filter_recent_trades_entirely_older_returns_empty() {
3191        // Requested window ends before the snapshot floor (1000ns).
3192        let filtered = filter_recent_trades(
3193            sample_trades(),
3194            Some(UnixNanos::from(100)),
3195            Some(UnixNanos::from(500)),
3196            None,
3197            btc_perp_id(),
3198        );
3199
3200        assert!(filtered.is_empty());
3201    }
3202
3203    #[rstest]
3204    fn test_filter_recent_trades_partial_keeps_in_range_subset() {
3205        // Start (500ns) is below the floor; end (2500ns) drops the 3000ns trade.
3206        let filtered = filter_recent_trades(
3207            sample_trades(),
3208            Some(UnixNanos::from(500)),
3209            Some(UnixNanos::from(2500)),
3210            None,
3211            btc_perp_id(),
3212        );
3213
3214        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3215        assert_eq!(ts, vec![1000, 2000]);
3216    }
3217
3218    #[rstest]
3219    fn test_filter_recent_trades_within_window_filters_bounds() {
3220        let filtered = filter_recent_trades(
3221            sample_trades(),
3222            Some(UnixNanos::from(1500)),
3223            Some(UnixNanos::from(3000)),
3224            None,
3225            btc_perp_id(),
3226        );
3227
3228        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3229        assert_eq!(ts, vec![2000, 3000]);
3230    }
3231
3232    #[rstest]
3233    fn test_filter_recent_trades_limit_keeps_most_recent() {
3234        let filtered = filter_recent_trades(sample_trades(), None, None, Some(2), btc_perp_id());
3235
3236        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3237        assert_eq!(ts, vec![2000, 3000]);
3238    }
3239
3240    #[rstest]
3241    fn test_filter_recent_trades_end_equal_to_floor_keeps_floor_trade() {
3242        // `end` exactly on the floor (1000ns) is inclusive: not "entirely
3243        // older". Distinguishes `end < floor` from `end <= floor`.
3244        let filtered = filter_recent_trades(
3245            sample_trades(),
3246            None,
3247            Some(UnixNanos::from(1000)),
3248            None,
3249            btc_perp_id(),
3250        );
3251
3252        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3253        assert_eq!(ts, vec![1000]);
3254    }
3255
3256    #[rstest]
3257    fn test_filter_recent_trades_bounds_are_inclusive() {
3258        // `start`/`end` landing exactly on a trade's ts_event keep that trade.
3259        // Distinguishes `>=`/`<=` from strict `>`/`<`.
3260        let filtered = filter_recent_trades(
3261            sample_trades(),
3262            Some(UnixNanos::from(2000)),
3263            Some(UnixNanos::from(3000)),
3264            None,
3265            btc_perp_id(),
3266        );
3267
3268        let ts: Vec<u64> = filtered.iter().map(|t| t.ts_event.as_u64()).collect();
3269        assert_eq!(ts, vec![2000, 3000]);
3270    }
3271
3272    fn perp_instrument(symbol: &str, tick_size: &str, ts_init: UnixNanos) -> InstrumentAny {
3273        let base = Currency::new("BTC", 8, 0, "BTC", CurrencyType::Crypto);
3274        let usd = Currency::new("USD", 8, 0, "USD", CurrencyType::Crypto);
3275        let usdc = Currency::new("USDC", 6, 0, "USDC", CurrencyType::Crypto);
3276
3277        InstrumentAny::CryptoPerpetual(
3278            CryptoPerpetual::builder()
3279                .instrument_id(InstrumentId::new(Symbol::new(symbol), *HYPERLIQUID_VENUE))
3280                .raw_symbol(Symbol::new("BTC"))
3281                .base_currency(base)
3282                .quote_currency(usd)
3283                .settlement_currency(usdc)
3284                .is_inverse(false)
3285                .price_precision(1)
3286                .size_precision(3)
3287                .price_increment(Price::from(tick_size))
3288                .size_increment(Quantity::from("0.001"))
3289                .ts_event(ts_init)
3290                .ts_init(ts_init)
3291                .build()
3292                .unwrap(),
3293        )
3294    }
3295
3296    fn cached(instruments: &[InstrumentAny]) -> Arc<AtomicMap<InstrumentId, InstrumentAny>> {
3297        let map = AtomicMap::new();
3298        map.rcu(|m| {
3299            for instrument in instruments {
3300                m.insert(instrument.id(), instrument.clone());
3301            }
3302        });
3303        Arc::new(map)
3304    }
3305
3306    fn data_client_with_refresh_interval(minutes: u64) -> HyperliquidDataClient {
3307        let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();
3308        set_data_event_sender(tx);
3309
3310        HyperliquidDataClient::new(
3311            *HYPERLIQUID_CLIENT_ID,
3312            HyperliquidDataClientConfig {
3313                update_instruments_interval_mins: minutes,
3314                ..HyperliquidDataClientConfig::default()
3315            },
3316        )
3317        .unwrap()
3318    }
3319
3320    #[tokio::test]
3321    async fn test_spawn_instrument_refresh_skipped_when_interval_zero() {
3322        let client = data_client_with_refresh_interval(0);
3323
3324        client.spawn_instrument_refresh().unwrap();
3325
3326        assert!(client.session_tasks.is_empty());
3327    }
3328
3329    #[tokio::test]
3330    async fn test_spawn_instrument_refresh_registers_task() {
3331        let client = data_client_with_refresh_interval(60);
3332
3333        client.spawn_instrument_refresh().unwrap();
3334
3335        assert_eq!(client.session_tasks.len(), 1);
3336
3337        client.cancellation_token.cancel();
3338        client.await_session_tasks().await.unwrap();
3339    }
3340
3341    #[rstest]
3342    fn test_changed_definitions_reports_a_newly_listed_market() {
3343        let cached_instruments =
3344            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3345        let fetched = vec![
3346            perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1)),
3347            perp_instrument("NEW-USD-PERP", "0.1", UnixNanos::from(1)),
3348        ];
3349
3350        let changed = changed_definitions(&fetched, &cached_instruments);
3351
3352        assert_eq!(changed.len(), 1);
3353        assert_eq!(changed[0].id().symbol.as_str(), "NEW-USD-PERP");
3354    }
3355
3356    #[rstest]
3357    fn test_added_symbols_names_only_the_market_the_cache_never_held() {
3358        let cached_instruments =
3359            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3360        // BTC moved its tick size, so it is changed but not new
3361        let changed = vec![
3362            perp_instrument("BTC-USD-PERP", "0.5", UnixNanos::from(1)),
3363            perp_instrument("NEW-USD-PERP", "0.1", UnixNanos::from(1)),
3364        ];
3365
3366        let added = added_symbols(&changed, &cached_instruments);
3367
3368        assert_eq!(added, vec![Ustr::from("NEW-USD-PERP")]);
3369    }
3370
3371    #[rstest]
3372    fn test_added_symbols_is_empty_when_every_change_is_a_known_market() {
3373        let cached_instruments =
3374            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3375        let changed = vec![perp_instrument("BTC-USD-PERP", "0.5", UnixNanos::from(1))];
3376
3377        assert!(added_symbols(&changed, &cached_instruments).is_empty());
3378    }
3379
3380    #[rstest]
3381    fn test_changed_definitions_ignores_a_later_ts_init_alone() {
3382        // Every pass restamps `ts_init` from the clock, so comparing it would
3383        // republish the whole universe on every tick.
3384        let cached_instruments =
3385            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3386        let fetched = vec![perp_instrument(
3387            "BTC-USD-PERP",
3388            "0.1",
3389            UnixNanos::from(2_000_000_000),
3390        )];
3391
3392        assert!(changed_definitions(&fetched, &cached_instruments).is_empty());
3393    }
3394
3395    #[rstest]
3396    fn test_changed_definitions_reports_a_changed_tick_size() {
3397        let cached_instruments =
3398            cached(&[perp_instrument("BTC-USD-PERP", "0.1", UnixNanos::from(1))]);
3399        let fetched = vec![perp_instrument("BTC-USD-PERP", "0.5", UnixNanos::from(1))];
3400
3401        let changed = changed_definitions(&fetched, &cached_instruments);
3402
3403        assert_eq!(changed.len(), 1);
3404        assert_eq!(changed[0].price_increment(), Price::from("0.5"));
3405    }
3406}