atelier_data 0.0.15

Data Artifacts and I/O for the atelier-rs engine
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173

//! # atelier-data
//!
//! Market data infrastructure for the **atelier-rs** trading engine.
//!
//! This crate provides everything needed to connect to cryptocurrency
//! exchanges, normalise their heterogeneous WebSocket feeds into a
//! common data model, and persist the result to Parquet, CSV, or JSON.
//!
//! ## Data types
//!
//! - Market : Centralized Exchanges (CEX), Decentralized Exchanges (DEX).
//! - Protocols: Blockchains
//!
//! ### Market : CEX
//!
//! Market related datatypes used throughout the `atelier-data` crate.
//!
//! | Type | Description |
//! |------|-------------|
//! | [`Orderbook`](orderbooks::Orderbook)| Full-depth limit order book snapshot (bid/ask levels). |
//! | [`OrderbookDelta`](orderbooks::OrderbookDelta) | Incremental order book maintained via `NormalizedDelta` updates. |
//! | [`Trade`] | A single public trade (price, size, side, timestamp). |
//! | [`FundingRate`] | Perpetual futures funding rate observation. |
//! | [`Liquidation`] | Forced liquidation event. |
//! | [`OpenInterest`] | Aggregate open interest snapshot. |
//!
//! ## Connectivity
//!
//! ### Centralized Exchanges
//!
//! Each supported exchange has a dedicated WebSocket client that
//! decodes exchange-specific JSON into [`NormalizedDelta`] and
//! [`Trade`] objects:
//!
//! <br>
//!
//! | Exchange | Client |
//! |----------|--------|
//! | Bybit | [`BybitWssClient`] |
//! | Coinbase | [`CoinbaseWssClient`] |
//! | Kraken | [`KrakenWssClient`] |
//! | Binance | [`BinanceWssClient`] |
//!
//! <br>
//!
//! ### Composition
//!
//! Usage wrappers with the purpose to deliver aggregation ergonomics.
//!
//! | Type | Description |
//! |------|-------------|
//! | [`MarketSnapshot`] | Time-aligned bundle of all market data for one grid period. |
//! | [`MarketAggregate`] | 15-scalar feature vector derived from a `MarketSnapshot`. |
//!
//! ### Synchronisation
//!
//! [`MarketSynchronizer`] bins heterogeneous exchange events onto a
//! uniform nanosecond grid, producing [`MarketSnapshot`] objects at
//! each tick.  Multiple [`ClockMode`] strategies are supported
//! (orderbook-driven, trade-driven, external clock).
//!
//! ### Persistence
//!
//! The following requires `parquet` feature.
//!
//! Orderbooks, trades, funding rates, liquidations, and open interest
//! can be written to and read from Apache Parquet files.  See the
//! `io` sub-modules of each data type (e.g.
//! [`orderbooks::io`], [`trades::io`], [`open_interest::io`],
//! [`liquidations::io`], [`funding::io`]).
//!
//! ## Feature flags
//!
//! | Flag | Effect |
//! |------|--------|
//! | `parquet` | Enables Apache Parquet I/O (adds `arrow` + `parquet` deps). |
//! | `torch` | Enables `tch`-based tensor conversion in the `datasets` module. |

#![allow(clippy::large_enum_variant)]
#![allow(clippy::too_many_arguments)]

/// Configuration templates
pub mod config;

/// Communication clients
pub mod clients;

/// Exchanges and properties
pub mod exchanges;

/// Dataset definition and tools
pub mod datasets;

/// Result and error handling
pub mod errors;

/// Funding rate data for perpetual futures
pub mod funding;

/// Orders-Price-Volume levels for Orderbooks.
pub mod levels;

/// Liquidations of position in CEX
pub mod liquidations;

/// Open interest tracking for derivatives
pub mod open_interest;

/// Single thread Orderbook structure.
pub mod orderbooks;

/// Implementation of orders
pub mod orders;

/// Multi-source market snapshot aggregation
pub mod snapshots;

/// Multi-source time synchronizer
pub mod synchronizers;

/// Configurations and experiments
pub mod templates;

/// Public Trades
pub mod trades;

/// Temporal data treatments
pub mod temporal;

/// Mappings for data sources
pub mod sources;

/// General utilities
pub mod utils;

/// Multi-worker collection orchestration
pub mod workers;

// Re-export common types for convenience

pub use clients::{
    connection_manager::{ConnectionManager, ConnectionManagerConfig},
    connection_state::ConnectionState,
    disconnect::{DisconnectReason, classify_close_frame, classify_tungstenite_error},
    reconnect::{
        CircuitState, ConnectionHealth, HealthMonitor, ReconnectAction, ReconnectPolicy,
    },
    wss::{WssClient, WssClientBuilder, WssDecoder},
};
pub use funding::FundingRate;
pub use levels::Level;
pub use liquidations::Liquidation;
pub use open_interest::OpenInterest;
pub use orderbooks::NormalizedDelta;
pub use orders::{Order, OrderSide, OrderType};
pub use snapshots::{MarketAggregate, MarketSnapshot};
pub use sources::{
    ExchangeEvent,
    binance::client::BinanceWssClient,
    bybit::{client::BybitWssClient, responses::BybitLiquidationData},
    coinbase::client::CoinbaseWssClient,
    kraken::client::KrakenWssClient,
};
pub use synchronizers::{
    ClockMode, EventSynchronizer, FlushResult, MarketSynchronizer, ReferenceEventType,
};
pub use trades::Trade;
pub use workers::{
    DataWorker, DataWorkerReport, EventPipeline, GapDetector, GapDetectorSet, GapStats,
    IngestionCore, IngestionReport, MarketWorker, MarketWorkerReport, OutputSink,
    OutputSinkSet, PassthroughPipeline, PublishError, TopicMessage, TopicPublisher,
    TopicRegistry, build_pipeline, build_sinks,
};