nautilus_testkit/
common.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::{
17    fs::File,
18    path::{Path, PathBuf},
19    sync::OnceLock,
20};
21
22use nautilus_core::paths::get_test_data_path;
23use nautilus_model::{
24    data::OrderBookDelta,
25    instruments::{InstrumentAny, stubs::equity_aapl_itch},
26};
27use nautilus_serialization::arrow::DecodeFromRecordBatch;
28use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
29
30use crate::files::ensure_file_exists_or_download_http;
31
32/// Returns the full path to the test data file at the specified relative `path` within the standard test data directory.
33///
34/// # Panics
35///
36/// Panics if the computed path cannot be represented as a valid UTF-8 string.
37#[must_use]
38pub fn get_test_data_file_path(path: &str) -> String {
39    get_test_data_path()
40        .join(path)
41        .to_str()
42        .unwrap()
43        .to_string()
44}
45
46/// Returns the full path to the Nautilus-specific test data file given by `filename`, within the configured precision directory ("64-bit" or "128-bit").
47///
48/// # Panics
49///
50/// Panics if the computed path cannot be represented as a valid UTF-8 string.
51#[must_use]
52#[allow(unused_mut)]
53pub fn get_nautilus_test_data_file_path(filename: &str) -> String {
54    let mut path = get_test_data_path().join("nautilus");
55
56    #[cfg(feature = "high-precision")]
57    {
58        path = path.join("128-bit");
59    }
60    #[cfg(not(feature = "high-precision"))]
61    {
62        path = path.join("64-bit");
63    }
64
65    path.join(filename).to_str().unwrap().to_string()
66}
67
68/// Returns the path to the checksums file for large test data files.
69#[must_use]
70pub fn get_test_data_large_checksums_filepath() -> PathBuf {
71    get_test_data_path().join("large").join("checksums.json")
72}
73
74/// Ensures that the specified test data file exists locally by downloading it if necessary, using the provided `url`.
75///
76/// # Panics
77///
78/// Panics if the download or checksum verification fails, or if the resulting path cannot be represented as a valid UTF-8 string.
79#[must_use]
80pub fn ensure_test_data_exists(filename: &str, url: &str) -> PathBuf {
81    let filepath = get_test_data_path().join("large").join(filename);
82    let checksums_filepath = get_test_data_large_checksums_filepath();
83    ensure_file_exists_or_download_http(&filepath, url, Some(&checksums_filepath), None).unwrap();
84    filepath
85}
86
87/// Ensures the NASDAQ ITCH AAPL deltas Parquet file exists locally, downloading from R2 if necessary.
88///
89/// # Panics
90///
91/// Panics if the download or checksum verification fails.
92#[must_use]
93pub fn ensure_itch_aapl_deltas_parquet() -> PathBuf {
94    ensure_test_data_exists(
95        "itch_AAPL.XNAS_2019-01-30_deltas.parquet",
96        "https://test-data.nautechsystems.io/large/itch_AAPL.XNAS_2019-01-30_deltas.parquet",
97    )
98}
99
100/// Ensures the Tardis Deribit BTC-PERPETUAL deltas Parquet file exists locally, downloading from R2 if necessary.
101///
102/// # Panics
103///
104/// Panics if the download or checksum verification fails.
105#[must_use]
106pub fn ensure_tardis_deribit_deltas_parquet() -> PathBuf {
107    ensure_test_data_exists(
108        "tardis_BTC-PERPETUAL.DERIBIT_2020-04-01_deltas.parquet",
109        "https://test-data.nautechsystems.io/large/tardis_BTC-PERPETUAL.DERIBIT_2020-04-01_deltas.parquet",
110    )
111}
112
113/// Ensures the HISTDATA EURUSD.SIM quotes Parquet file exists locally, downloading from R2
114/// if necessary.
115///
116/// # Panics
117///
118/// Panics if the download or checksum verification fails.
119#[must_use]
120pub fn ensure_histdata_eurusd_quotes_parquet() -> PathBuf {
121    ensure_test_data_exists(
122        "histdata_EURUSD.SIM_2020-01_quotes.parquet",
123        "https://test-data.nautechsystems.io/large/histdata_EURUSD.SIM_2020-01_quotes.parquet",
124    )
125}
126
127/// Ensures the HISTDATA EURUSD.SIM instrument Parquet file exists locally, downloading from R2
128/// if necessary.
129///
130/// # Panics
131///
132/// Panics if the download or checksum verification fails.
133#[must_use]
134pub fn ensure_histdata_eurusd_instrument_parquet() -> PathBuf {
135    ensure_test_data_exists(
136        "histdata_EURUSD.SIM_2020-01_instrument.parquet",
137        "https://test-data.nautechsystems.io/large/histdata_EURUSD.SIM_2020-01_instrument.parquet",
138    )
139}
140
141/// Returns the path to the Tardis Deribit incremental book L2 test data.
142#[must_use]
143pub fn get_tardis_deribit_book_l2_path() -> PathBuf {
144    get_test_data_path()
145        .join("tardis")
146        .join("deribit_incremental_book_L2_BTC-PERPETUAL.csv")
147}
148
149/// Returns the path to the Tardis Binance Futures book snapshot (depth 5) test data.
150#[must_use]
151pub fn get_tardis_binance_snapshot5_path() -> PathBuf {
152    get_test_data_path()
153        .join("tardis")
154        .join("binance-futures_book_snapshot_5_BTCUSDT.csv")
155}
156
157/// Returns the path to the Tardis Binance Futures book snapshot (depth 25) test data.
158#[must_use]
159pub fn get_tardis_binance_snapshot25_path() -> PathBuf {
160    get_test_data_path()
161        .join("tardis")
162        .join("binance-futures_book_snapshot_25_BTCUSDT.csv")
163}
164
165/// Returns the path to the Tardis Huobi quotes test data.
166#[must_use]
167pub fn get_tardis_huobi_quotes_path() -> PathBuf {
168    get_test_data_path()
169        .join("tardis")
170        .join("huobi-dm-swap_quotes_BTC-USD.csv")
171}
172
173/// Returns the path to the Tardis Bitmex trades test data.
174#[must_use]
175pub fn get_tardis_bitmex_trades_path() -> PathBuf {
176    get_test_data_path()
177        .join("tardis")
178        .join("bitmex_trades_XBTUSD.csv")
179}
180
181/// Returns an AAPL equity instrument with ITCH-compatible precision
182/// (price_precision=4, price_increment=0.0001).
183#[must_use]
184pub fn itch_aapl_equity() -> InstrumentAny {
185    InstrumentAny::Equity(equity_aapl_itch())
186}
187
188/// Loads ITCH AAPL order book deltas from the parquet test dataset.
189///
190/// Downloads the file on first access. Pass `limit` to subsample.
191#[must_use]
192pub fn load_itch_aapl_deltas(limit: Option<usize>) -> Vec<OrderBookDelta> {
193    static PATH: OnceLock<PathBuf> = OnceLock::new();
194    let filepath = PATH.get_or_init(ensure_itch_aapl_deltas_parquet);
195    load_deltas_from_parquet(filepath, limit)
196}
197
198/// Loads Tardis Deribit BTC-PERPETUAL order book deltas from the parquet test dataset.
199///
200/// Downloads the file on first access. Pass `limit` to subsample.
201#[must_use]
202pub fn load_tardis_deribit_deltas(limit: Option<usize>) -> Vec<OrderBookDelta> {
203    static PATH: OnceLock<PathBuf> = OnceLock::new();
204    let filepath = PATH.get_or_init(ensure_tardis_deribit_deltas_parquet);
205    load_deltas_from_parquet(filepath, limit)
206}
207
208fn load_deltas_from_parquet(filepath: &Path, limit: Option<usize>) -> Vec<OrderBookDelta> {
209    let file = File::open(filepath).unwrap();
210    let mut builder = ParquetRecordBatchReaderBuilder::try_new(file).unwrap();
211    let metadata = builder.schema().metadata().clone();
212
213    if let Some(limit) = limit {
214        builder = builder.with_limit(limit);
215    }
216    let reader = builder.build().unwrap();
217
218    let mut deltas = Vec::new();
219
220    for batch_result in reader {
221        let batch = batch_result.unwrap();
222        let batch_deltas = OrderBookDelta::decode_batch(&metadata, batch).unwrap();
223        deltas.extend(batch_deltas);
224    }
225    deltas
226}
nautilus_testkit/common.rs

nautilus_testkit/
common.rs
