tickvault 0.1.1

Full-depth crypto order book recorder and archive that publishes its own gap statistics
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
//! Decoded archive rows, and a stream of them.
//!
//! The stream exists so a consumer never has to hold a day in memory. It opens
//! one file at a time and pulls one Parquet batch at a time, so its footprint
//! is a batch rather than an archive, whatever the range asked for.

use std::collections::VecDeque;
use std::path::{Path, PathBuf};

use arrow::array::{
    Array, BooleanArray, Int64Array, RecordBatch, StringArray, TimestampNanosecondArray,
    UInt8Array, UInt32Array, UInt64Array,
};
use parquet::arrow::arrow_reader::{ParquetRecordBatchReader, ParquetRecordBatchReaderBuilder};

use crate::error::{Error, Result};
use crate::fixed::Fixed;
use crate::store::manifest::FileRecord;
use crate::store::schema::EventKind;
use crate::types::{BookLevel, Side};

/// One archived row.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Row {
    pub event: EventKind,
    /// Groups the levels that arrived in one message. Applying a message's
    /// levels one at a time is not the same as applying them together on a
    /// depth-limited feed, so this is load bearing rather than metadata.
    pub msg_index: u64,
    pub recv_wall: i64,
    pub venue_ts: Option<i64>,
    pub side: Side,
    pub price: Fixed,
    pub qty: Fixed,
    pub suspect: bool,
    pub book_level: BookLevel,
    pub order_id: Option<String>,
    pub action: Option<String>,
    pub queue_index: Option<u32>,
    pub queue_certainty: Option<String>,
    /// Quantity this event traded, where the feed reports executions.
    ///
    /// `None` on every aggregated feed: a level shrinking might be a fill or a
    /// cancellation and the venue never says which. Zero means the feed could
    /// answer and the answer was that nothing traded.
    pub traded_qty: Option<Fixed>,
}

fn column<'a, T: 'static>(batch: &'a RecordBatch, name: &str) -> Result<&'a T> {
    batch
        .column_by_name(name)
        .and_then(|c| c.as_any().downcast_ref::<T>())
        .ok_or_else(|| Error::Other(format!("archive row is missing column {name}")))
}

/// Decode a whole batch.
pub fn decode(batch: &RecordBatch) -> Result<Vec<Row>> {
    let event = column::<UInt8Array>(batch, "event")?;
    let msg_index = column::<UInt64Array>(batch, "msg_index")?;
    let recv_wall = column::<TimestampNanosecondArray>(batch, "recv_wall")?;
    let venue_ts = column::<TimestampNanosecondArray>(batch, "venue_ts")?;
    let side = column::<UInt8Array>(batch, "side")?;
    let price = column::<Int64Array>(batch, "price")?;
    let qty = column::<Int64Array>(batch, "qty")?;
    let suspect = column::<BooleanArray>(batch, "suspect")?;
    // Archives written before the L3 columns existed simply lack them.
    let level = batch
        .column_by_name("book_level")
        .and_then(|c| c.as_any().downcast_ref::<UInt8Array>());
    let order_id = batch
        .column_by_name("order_id")
        .and_then(|c| c.as_any().downcast_ref::<StringArray>());
    let action = batch
        .column_by_name("action")
        .and_then(|c| c.as_any().downcast_ref::<StringArray>());
    let queue_index = batch
        .column_by_name("queue_index")
        .and_then(|c| c.as_any().downcast_ref::<UInt32Array>());
    let queue_certainty = batch
        .column_by_name("queue_certainty")
        .and_then(|c| c.as_any().downcast_ref::<StringArray>());
    let traded_qty = batch
        .column_by_name("traded_qty")
        .and_then(|c| c.as_any().downcast_ref::<Int64Array>());

    let mut rows = Vec::with_capacity(batch.num_rows());
    for i in 0..batch.num_rows() {
        rows.push(Row {
            event: EventKind::from_u8(event.value(i))
                .ok_or_else(|| Error::Other(format!("unknown event kind {}", event.value(i))))?,
            msg_index: msg_index.value(i),
            recv_wall: recv_wall.value(i),
            venue_ts: (!venue_ts.is_null(i)).then(|| venue_ts.value(i)),
            side: if side.value(i) == 0 {
                Side::Bid
            } else {
                Side::Ask
            },
            price: Fixed::from_mantissa(price.value(i)),
            qty: Fixed::from_mantissa(qty.value(i)),
            suspect: suspect.value(i),
            book_level: match level.map(|l| l.value(i)) {
                Some(3) => BookLevel::L3,
                _ => BookLevel::L2,
            },
            order_id: order_id
                .filter(|c| !c.is_null(i))
                .map(|c| c.value(i).to_string()),
            action: action
                .filter(|c| !c.is_null(i))
                .map(|c| c.value(i).to_string()),
            queue_index: queue_index.filter(|c| !c.is_null(i)).map(|c| c.value(i)),
            queue_certainty: queue_certainty
                .filter(|c| !c.is_null(i))
                .map(|c| c.value(i).to_string()),
            traded_qty: traded_qty
                .filter(|c| !c.is_null(i))
                .map(|c| Fixed::from_mantissa(c.value(i))),
        });
    }
    Ok(rows)
}

/// A lazy stream of rows across a set of files.
///
/// Bounded memory by construction: one open file, one decoded batch. A query
/// over a whole day costs the same resident memory as a query over a second.
pub struct RowStream {
    root: PathBuf,
    files: VecDeque<FileRecord>,
    current: Option<ParquetRecordBatchReader>,
    buffer: VecDeque<Row>,
    /// Stop once rows pass this instant. Rows are in arrival order, so the
    /// first one past it ends the stream rather than being skipped over.
    until_wall: i64,
    /// Skip rows at or before this instant without decoding further files.
    after_wall: Option<i64>,
    finished: bool,
    files_opened: usize,
    rows_yielded: u64,
}

impl RowStream {
    /// Stream `files` in the order given, up to and including `until_wall`.
    pub fn new(root: impl Into<PathBuf>, files: Vec<FileRecord>, until_wall: i64) -> Self {
        RowStream {
            root: root.into(),
            files: files.into(),
            current: None,
            buffer: VecDeque::new(),
            until_wall,
            after_wall: None,
            finished: false,
            files_opened: 0,
            rows_yielded: 0,
        }
    }

    /// Skip everything at or before `wall`.
    pub fn after(mut self, wall: i64) -> Self {
        self.after_wall = Some(wall);
        self
    }

    pub fn files_opened(&self) -> usize {
        self.files_opened
    }

    pub fn rows_yielded(&self) -> u64 {
        self.rows_yielded
    }

    /// Rows currently decoded and waiting, for the memory-bound test.
    pub fn buffered_rows(&self) -> usize {
        self.buffer.len()
    }

    fn open_next(&mut self) -> Result<bool> {
        let Some(record) = self.files.pop_front() else {
            return Ok(false);
        };
        let path = self.root.join(&record.path);
        let file = std::fs::File::open(&path)?;
        let reader = ParquetRecordBatchReaderBuilder::try_new(file)
            .map_err(|e| Error::Other(format!("{}: {e}", path.display())))?
            .build()
            .map_err(|e| Error::Other(format!("{}: {e}", path.display())))?;
        self.current = Some(reader);
        self.files_opened += 1;
        Ok(true)
    }

    fn fill(&mut self) -> Result<bool> {
        loop {
            if let Some(reader) = self.current.as_mut() {
                match reader.next() {
                    Some(Ok(batch)) => {
                        let rows = decode(&batch)?;
                        if rows.is_empty() {
                            continue;
                        }
                        self.buffer.extend(rows);
                        return Ok(true);
                    }
                    Some(Err(e)) => return Err(Error::Other(e.to_string())),
                    None => self.current = None,
                }
            }
            if self.current.is_none() && !self.open_next()? {
                return Ok(false);
            }
        }
    }
}

impl Iterator for RowStream {
    type Item = Result<Row>;

    fn next(&mut self) -> Option<Self::Item> {
        loop {
            if self.finished {
                return None;
            }
            let Some(row) = self.buffer.pop_front() else {
                match self.fill() {
                    Ok(true) => continue,
                    Ok(false) => {
                        self.finished = true;
                        return None;
                    }
                    Err(e) => {
                        self.finished = true;
                        return Some(Err(e));
                    }
                }
            };
            if row.recv_wall > self.until_wall {
                // Arrival order, so nothing after this can be in range either.
                self.finished = true;
                return None;
            }
            if self.after_wall.is_some_and(|w| row.recv_wall <= w) {
                continue;
            }
            self.rows_yielded += 1;
            return Some(Ok(row));
        }
    }
}

/// Files for one partition, oldest first.
pub fn partition_files(
    reader: &crate::store::reader::ArchiveReader,
    venue: crate::types::VenueId,
    symbol: &crate::types::Symbol,
    date: &str,
) -> Vec<FileRecord> {
    let mut files: Vec<FileRecord> = reader
        .files()
        .iter()
        .filter(|f| f.venue == venue && f.symbol == *symbol && f.date == date)
        .cloned()
        .collect();
    files.sort_by_key(|f| (f.first_recv_wall, f.path.clone()));
    files
}

/// Path of a file relative to a root.
pub fn relative(root: &Path, record: &FileRecord) -> PathBuf {
    root.join(&record.path)
}

/// A stream of whole messages, each one the rows that arrived together.
///
/// Grouping is not a convenience. Applying a message's levels one at a time is
/// a different operation from applying them together on a depth-limited feed:
/// one truncates per level, the other per message, and a message that adds a
/// new best while removing the old worst gives a different book each way. This
/// makes the message the unit, so nothing downstream can get it wrong.
///
/// The group key is `(msg_index, event, recv_wall)` rather than `msg_index`
/// alone. That index is a per-recorder-run counter and restarts at zero when
/// the recorder does, so two genuinely different messages either side of a
/// restart can carry the same one.
pub struct MessageStream {
    rows: RowStream,
    held: Option<Row>,
    finished: bool,
}

impl MessageStream {
    pub fn new(rows: RowStream) -> Self {
        MessageStream {
            rows,
            held: None,
            finished: false,
        }
    }

    pub fn files_opened(&self) -> usize {
        self.rows.files_opened()
    }

    pub fn rows_yielded(&self) -> u64 {
        self.rows.rows_yielded()
    }

    pub fn buffered_rows(&self) -> usize {
        self.rows.buffered_rows() + usize::from(self.held.is_some())
    }
}

fn same_message(a: &Row, b: &Row) -> bool {
    a.msg_index == b.msg_index && a.event == b.event && a.recv_wall == b.recv_wall
}

/// Split rows already in hand into messages, the same way the stream does.
///
/// The unit of application is the message, never the row. Applying a message's
/// levels one at a time truncates once per *level* instead of once per message,
/// which on a depth-limited feed rebuilds a book that differs from the one the
/// recorder held; that was a real bug, and it survived a passing determinism
/// gate. So there is one definition of where a message ends, and this is how a
/// consumer holding rows from somewhere other than a file gets at it rather
/// than writing the rule out a second time.
pub fn split_messages(rows: &[Row]) -> Messages<'_> {
    Messages { rows }
}

/// Iterator returned by [`split_messages`].
pub struct Messages<'a> {
    rows: &'a [Row],
}

impl<'a> Iterator for Messages<'a> {
    type Item = &'a [Row];

    fn next(&mut self) -> Option<&'a [Row]> {
        let first = self.rows.first()?;
        let mut n = 1;
        while n < self.rows.len() && same_message(first, &self.rows[n]) {
            n += 1;
        }
        let (message, rest) = self.rows.split_at(n);
        self.rows = rest;
        Some(message)
    }
}

impl Iterator for MessageStream {
    type Item = Result<Vec<Row>>;

    fn next(&mut self) -> Option<Self::Item> {
        if self.finished {
            return None;
        }
        let mut message: Vec<Row> = Vec::new();
        if let Some(row) = self.held.take() {
            message.push(row);
        }
        loop {
            match self.rows.next() {
                Some(Ok(row)) => {
                    if message
                        .first()
                        .is_some_and(|first| !same_message(first, &row))
                    {
                        self.held = Some(row);
                        return Some(Ok(message));
                    }
                    message.push(row);
                }
                Some(Err(e)) => {
                    self.finished = true;
                    return Some(Err(e));
                }
                None => {
                    self.finished = true;
                    return (!message.is_empty()).then_some(Ok(message));
                }
            }
        }
    }
}

#[cfg(test)]
mod message_split_tests {
    use super::*;
    use crate::types::{BookLevel, Side};

    fn row(msg_index: u64, event: EventKind, recv_wall: i64, price: i64) -> Row {
        Row {
            event,
            msg_index,
            recv_wall,
            venue_ts: None,
            side: Side::Bid,
            price: Fixed::from_mantissa(price),
            qty: Fixed::from_mantissa(1),
            suspect: false,
            book_level: BookLevel::L2,
            order_id: None,
            action: None,
            queue_index: None,
            queue_certainty: None,
            traded_qty: None,
        }
    }

    #[test]
    fn levels_of_one_message_stay_together() {
        let rows = vec![
            row(0, EventKind::Delta, 100, 1),
            row(0, EventKind::Delta, 100, 2),
            row(1, EventKind::Delta, 200, 3),
        ];
        let split: Vec<usize> = split_messages(&rows).map(|m| m.len()).collect();
        assert_eq!(split, vec![2, 1]);
    }

    #[test]
    fn all_three_fields_end_a_message() {
        // A shared index is not enough on its own. A snapshot and a delta that
        // happen to carry the same index are different messages, and so are two
        // messages that arrived at different instants.
        let by_index = vec![
            row(0, EventKind::Delta, 100, 1),
            row(1, EventKind::Delta, 100, 2),
        ];
        let by_kind = vec![
            row(0, EventKind::Snapshot, 100, 1),
            row(0, EventKind::Delta, 100, 2),
        ];
        let by_time = vec![
            row(0, EventKind::Delta, 100, 1),
            row(0, EventKind::Delta, 101, 2),
        ];
        for rows in [by_index, by_kind, by_time] {
            assert_eq!(split_messages(&rows).count(), 2);
        }
    }

    #[test]
    fn every_row_lands_in_exactly_one_message() {
        let rows = vec![
            row(0, EventKind::Snapshot, 10, 1),
            row(0, EventKind::Snapshot, 10, 2),
            row(1, EventKind::Delta, 20, 3),
            row(2, EventKind::Delta, 30, 4),
            row(2, EventKind::Delta, 30, 5),
        ];
        let regrouped: Vec<Row> = split_messages(&rows).flatten().cloned().collect();
        assert_eq!(regrouped.len(), rows.len());
        for (a, b) in regrouped.iter().zip(rows.iter()) {
            assert_eq!(a.msg_index, b.msg_index);
            assert_eq!(a.price, b.price);
        }
    }

    #[test]
    fn an_empty_run_yields_no_messages() {
        assert_eq!(split_messages(&[]).count(), 0);
    }
}