nautilus_testkit/
common.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::{
17    fs::File,
18    path::{Path, PathBuf},
19    sync::OnceLock,
20};
21
22use nautilus_core::paths::get_test_data_path;
23use nautilus_model::data::OrderBookDelta;
24use nautilus_serialization::arrow::DecodeFromRecordBatch;
25use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
26
27use crate::files::ensure_file_exists_or_download_http;
28
29/// Returns the full path to the test data file at the specified relative `path` within the standard test data directory.
30///
31/// # Panics
32///
33/// Panics if the computed path cannot be represented as a valid UTF-8 string.
34#[must_use]
35pub fn get_test_data_file_path(path: &str) -> String {
36    get_test_data_path()
37        .join(path)
38        .to_str()
39        .unwrap()
40        .to_string()
41}
42
43/// Returns the full path to the Nautilus-specific test data file given by `filename`, within the configured precision directory ("64-bit" or "128-bit").
44///
45/// # Panics
46///
47/// Panics if the computed path cannot be represented as a valid UTF-8 string.
48#[must_use]
49#[allow(unused_mut)]
50pub fn get_nautilus_test_data_file_path(filename: &str) -> String {
51    let mut path = get_test_data_path().join("nautilus");
52
53    #[cfg(feature = "high-precision")]
54    {
55        path = path.join("128-bit");
56    }
57    #[cfg(not(feature = "high-precision"))]
58    {
59        path = path.join("64-bit");
60    }
61
62    path.join(filename).to_str().unwrap().to_string()
63}
64
65/// Returns the path to the checksums file for large test data files.
66#[must_use]
67pub fn get_test_data_large_checksums_filepath() -> PathBuf {
68    get_test_data_path().join("large").join("checksums.json")
69}
70
71/// Ensures that the specified test data file exists locally by downloading it if necessary, using the provided `url`.
72///
73/// # Panics
74///
75/// Panics if the download or checksum verification fails, or if the resulting path cannot be represented as a valid UTF-8 string.
76#[must_use]
77pub fn ensure_test_data_exists(filename: &str, url: &str) -> PathBuf {
78    let filepath = get_test_data_path().join("large").join(filename);
79    let checksums_filepath = get_test_data_large_checksums_filepath();
80    ensure_file_exists_or_download_http(&filepath, url, Some(&checksums_filepath), None).unwrap();
81    filepath
82}
83
84/// Ensures the NASDAQ ITCH AAPL deltas Parquet file exists locally, downloading from R2 if necessary.
85///
86/// # Panics
87///
88/// Panics if the download or checksum verification fails.
89#[must_use]
90pub fn ensure_itch_aapl_deltas_parquet() -> PathBuf {
91    ensure_test_data_exists(
92        "itch_AAPL.XNAS_2019-01-30_deltas.parquet",
93        "https://test-data.nautechsystems.io/large/itch_AAPL.XNAS_2019-01-30_deltas.parquet",
94    )
95}
96
97/// Ensures the Tardis Deribit BTC-PERPETUAL deltas Parquet file exists locally, downloading from R2 if necessary.
98///
99/// # Panics
100///
101/// Panics if the download or checksum verification fails.
102#[must_use]
103pub fn ensure_tardis_deribit_deltas_parquet() -> PathBuf {
104    ensure_test_data_exists(
105        "tardis_BTC-PERPETUAL.DERIBIT_2020-04-01_deltas.parquet",
106        "https://test-data.nautechsystems.io/large/tardis_BTC-PERPETUAL.DERIBIT_2020-04-01_deltas.parquet",
107    )
108}
109
110/// Returns the path to the Tardis Deribit incremental book L2 test data.
111#[must_use]
112pub fn get_tardis_deribit_book_l2_path() -> PathBuf {
113    get_test_data_path()
114        .join("tardis")
115        .join("deribit_incremental_book_L2_BTC-PERPETUAL.csv")
116}
117
118/// Returns the path to the Tardis Binance Futures book snapshot (depth 5) test data.
119#[must_use]
120pub fn get_tardis_binance_snapshot5_path() -> PathBuf {
121    get_test_data_path()
122        .join("tardis")
123        .join("binance-futures_book_snapshot_5_BTCUSDT.csv")
124}
125
126/// Returns the path to the Tardis Binance Futures book snapshot (depth 25) test data.
127#[must_use]
128pub fn get_tardis_binance_snapshot25_path() -> PathBuf {
129    get_test_data_path()
130        .join("tardis")
131        .join("binance-futures_book_snapshot_25_BTCUSDT.csv")
132}
133
134/// Returns the path to the Tardis Huobi quotes test data.
135#[must_use]
136pub fn get_tardis_huobi_quotes_path() -> PathBuf {
137    get_test_data_path()
138        .join("tardis")
139        .join("huobi-dm-swap_quotes_BTC-USD.csv")
140}
141
142/// Returns the path to the Tardis Bitmex trades test data.
143#[must_use]
144pub fn get_tardis_bitmex_trades_path() -> PathBuf {
145    get_test_data_path()
146        .join("tardis")
147        .join("bitmex_trades_XBTUSD.csv")
148}
149
150/// Loads ITCH AAPL order book deltas from the parquet test dataset.
151///
152/// Downloads the file on first access. Pass `limit` to subsample.
153#[must_use]
154pub fn load_itch_aapl_deltas(limit: Option<usize>) -> Vec<OrderBookDelta> {
155    static PATH: OnceLock<PathBuf> = OnceLock::new();
156    let filepath = PATH.get_or_init(ensure_itch_aapl_deltas_parquet);
157    load_deltas_from_parquet(filepath, limit)
158}
159
160/// Loads Tardis Deribit BTC-PERPETUAL order book deltas from the parquet test dataset.
161///
162/// Downloads the file on first access. Pass `limit` to subsample.
163#[must_use]
164pub fn load_tardis_deribit_deltas(limit: Option<usize>) -> Vec<OrderBookDelta> {
165    static PATH: OnceLock<PathBuf> = OnceLock::new();
166    let filepath = PATH.get_or_init(ensure_tardis_deribit_deltas_parquet);
167    load_deltas_from_parquet(filepath, limit)
168}
169
170fn load_deltas_from_parquet(filepath: &Path, limit: Option<usize>) -> Vec<OrderBookDelta> {
171    let file = File::open(filepath).unwrap();
172    let mut builder = ParquetRecordBatchReaderBuilder::try_new(file).unwrap();
173    let metadata = builder.schema().metadata().clone();
174    if let Some(limit) = limit {
175        builder = builder.with_limit(limit);
176    }
177    let reader = builder.build().unwrap();
178
179    let mut deltas = Vec::new();
180    for batch_result in reader {
181        let batch = batch_result.unwrap();
182        let batch_deltas = OrderBookDelta::decode_batch(&metadata, batch).unwrap();
183        deltas.extend(batch_deltas);
184    }
185    deltas
186}
nautilus_testkit/common.rs

nautilus_testkit/
common.rs
