nautilus_testkit/
common.rs

1// -------------------------------------------------------------------------------------------------
2//  Copyright (C) 2015-2026 Nautech Systems Pty Ltd. All rights reserved.
3//  https://nautechsystems.io
4//
5//  Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
6//  You may not use this file except in compliance with the License.
7//  You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
8//
9//  Unless required by applicable law or agreed to in writing, software
10//  distributed under the License is distributed on an "AS IS" BASIS,
11//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12//  See the License for the specific language governing permissions and
13//  limitations under the License.
14// -------------------------------------------------------------------------------------------------
15
16use std::{
17    fs::File,
18    path::{Path, PathBuf},
19    sync::OnceLock,
20};
21
22use nautilus_core::paths::get_test_data_path;
23use nautilus_model::{
24    data::OrderBookDelta,
25    instruments::{InstrumentAny, stubs::equity_aapl_itch},
26    types::fixed::PRECISION_BYTES,
27};
28use nautilus_serialization::arrow::DecodeFromRecordBatch;
29use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
30
31use crate::files::ensure_file_exists_or_download_http;
32
33/// Returns the full path to the test data file at the specified relative `path` within the standard test data directory.
34///
35/// # Panics
36///
37/// Panics if the computed path cannot be represented as a valid UTF-8 string.
38#[must_use]
39pub fn get_test_data_file_path(path: &str) -> String {
40    get_test_data_path()
41        .join(path)
42        .to_str()
43        .unwrap()
44        .to_string()
45}
46
47/// Returns the full path to the Nautilus-specific test data file given by `filename`, within the configured precision directory ("64-bit" or "128-bit").
48///
49/// # Panics
50///
51/// Panics if the computed path cannot be represented as a valid UTF-8 string.
52#[must_use]
53pub fn get_nautilus_test_data_file_path(filename: &str) -> String {
54    let precision_directory = format!("{}-bit", PRECISION_BYTES * 8);
55    let path = get_test_data_path()
56        .join("nautilus")
57        .join(precision_directory);
58
59    path.join(filename).to_str().unwrap().to_string()
60}
61
62/// Returns the path to the checksums file for large test data files.
63#[must_use]
64pub fn get_test_data_large_checksums_filepath() -> PathBuf {
65    get_test_data_path().join("large").join("checksums.json")
66}
67
68/// Ensures that the specified test data file exists locally by downloading it if necessary, using the provided `url`.
69///
70/// # Panics
71///
72/// Panics if the download or checksum verification fails, or if the resulting path cannot be represented as a valid UTF-8 string.
73#[must_use]
74pub fn ensure_test_data_exists(filename: &str, url: &str) -> PathBuf {
75    let filepath = get_test_data_path().join("large").join(filename);
76    let checksums_filepath = get_test_data_large_checksums_filepath();
77    ensure_file_exists_or_download_http(&filepath, url, Some(&checksums_filepath), None).unwrap();
78    filepath
79}
80
81/// Ensures the NASDAQ ITCH AAPL deltas Parquet file exists locally, downloading from R2 if necessary.
82///
83/// # Panics
84///
85/// Panics if the download or checksum verification fails.
86#[must_use]
87pub fn ensure_itch_aapl_deltas_parquet() -> PathBuf {
88    ensure_test_data_exists(
89        "itch_AAPL.XNAS_2019-01-30_deltas.parquet",
90        "https://test-data.nautechsystems.io/large/itch_AAPL.XNAS_2019-01-30_deltas.parquet",
91    )
92}
93
94/// Ensures the Tardis Deribit BTC-PERPETUAL deltas Parquet file exists locally, downloading from R2 if necessary.
95///
96/// # Panics
97///
98/// Panics if the download or checksum verification fails.
99#[must_use]
100pub fn ensure_tardis_deribit_deltas_parquet() -> PathBuf {
101    ensure_test_data_exists(
102        "tardis_BTC-PERPETUAL.DERIBIT_2020-04-01_deltas.parquet",
103        "https://test-data.nautechsystems.io/large/tardis_BTC-PERPETUAL.DERIBIT_2020-04-01_deltas.parquet",
104    )
105}
106
107/// Ensures the HISTDATA EURUSD.SIM quotes Parquet file exists locally, downloading from R2
108/// if necessary.
109///
110/// # Panics
111///
112/// Panics if the download or checksum verification fails.
113#[must_use]
114pub fn ensure_histdata_eurusd_quotes_parquet() -> PathBuf {
115    ensure_test_data_exists(
116        "histdata_EURUSD.SIM_2020-01_quotes.parquet",
117        "https://test-data.nautechsystems.io/large/histdata_EURUSD.SIM_2020-01_quotes.parquet",
118    )
119}
120
121/// Ensures the HISTDATA EURUSD.SIM instrument Parquet file exists locally, downloading from R2
122/// if necessary.
123///
124/// # Panics
125///
126/// Panics if the download or checksum verification fails.
127#[must_use]
128pub fn ensure_histdata_eurusd_instrument_parquet() -> PathBuf {
129    ensure_test_data_exists(
130        "histdata_EURUSD.SIM_2020-01_instrument.parquet",
131        "https://test-data.nautechsystems.io/large/histdata_EURUSD.SIM_2020-01_instrument.parquet",
132    )
133}
134
135/// Returns the path to the Tardis Deribit incremental book L2 test data.
136#[must_use]
137pub fn get_tardis_deribit_book_l2_path() -> PathBuf {
138    get_test_data_path()
139        .join("tardis")
140        .join("deribit_incremental_book_L2_BTC-PERPETUAL.csv")
141}
142
143/// Returns the path to the Tardis Binance Futures book snapshot (depth 5) test data.
144#[must_use]
145pub fn get_tardis_binance_snapshot5_path() -> PathBuf {
146    get_test_data_path()
147        .join("tardis")
148        .join("binance-futures_book_snapshot_5_BTCUSDT.csv")
149}
150
151/// Returns the path to the Tardis Binance Futures book snapshot (depth 25) test data.
152#[must_use]
153pub fn get_tardis_binance_snapshot25_path() -> PathBuf {
154    get_test_data_path()
155        .join("tardis")
156        .join("binance-futures_book_snapshot_25_BTCUSDT.csv")
157}
158
159/// Returns the path to the Tardis Huobi quotes test data.
160#[must_use]
161pub fn get_tardis_huobi_quotes_path() -> PathBuf {
162    get_test_data_path()
163        .join("tardis")
164        .join("huobi-dm-swap_quotes_BTC-USD.csv")
165}
166
167/// Returns the path to the Tardis Bitmex trades test data.
168#[must_use]
169pub fn get_tardis_bitmex_trades_path() -> PathBuf {
170    get_test_data_path()
171        .join("tardis")
172        .join("bitmex_trades_XBTUSD.csv")
173}
174
175/// Returns an AAPL equity instrument with ITCH-compatible precision
176/// (`price_precision=4`, `price_increment=0.0001`).
177#[must_use]
178pub fn itch_aapl_equity() -> InstrumentAny {
179    InstrumentAny::Equity(equity_aapl_itch())
180}
181
182/// Loads ITCH AAPL order book deltas from the parquet test dataset.
183///
184/// Downloads the file on first access. Pass `limit` to subsample.
185#[must_use]
186pub fn load_itch_aapl_deltas(limit: Option<usize>) -> Vec<OrderBookDelta> {
187    static PATH: OnceLock<PathBuf> = OnceLock::new();
188    let filepath = PATH.get_or_init(ensure_itch_aapl_deltas_parquet);
189    load_deltas_from_parquet(filepath, limit)
190}
191
192/// Loads Tardis Deribit BTC-PERPETUAL order book deltas from the parquet test dataset.
193///
194/// Downloads the file on first access. Pass `limit` to subsample.
195#[must_use]
196pub fn load_tardis_deribit_deltas(limit: Option<usize>) -> Vec<OrderBookDelta> {
197    static PATH: OnceLock<PathBuf> = OnceLock::new();
198    let filepath = PATH.get_or_init(ensure_tardis_deribit_deltas_parquet);
199    load_deltas_from_parquet(filepath, limit)
200}
201
202fn load_deltas_from_parquet(filepath: &Path, limit: Option<usize>) -> Vec<OrderBookDelta> {
203    let file = File::open(filepath).unwrap();
204    let mut builder = ParquetRecordBatchReaderBuilder::try_new(file).unwrap();
205    let metadata = builder.schema().metadata().clone();
206
207    if let Some(limit) = limit {
208        builder = builder.with_limit(limit);
209    }
210    let reader = builder.build().unwrap();
211
212    let mut deltas = Vec::new();
213
214    for batch_result in reader {
215        let batch = batch_result.unwrap();
216        let batch_deltas = OrderBookDelta::decode_batch(&metadata, batch).unwrap();
217        deltas.extend(batch_deltas);
218    }
219    deltas
220}
nautilus_testkit/common.rs

nautilus_testkit/
common.rs
