1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
/// this module handles orderbook operations

use indexmap::IndexMap;
use crate::postprocessing::histogram::{Histogram, BinCount};
use crate::dtf::update::Update;
use std::fmt;
use std::f64;

type PriceBits = u64;
type Size = f32;
type Time = u64;
type OrderbookSide = IndexMap<PriceBits, Size>;

/// data structure for orderbook
#[derive(Clone)]
pub struct Orderbook {
    /// bids side
    pub bids: OrderbookSide,
    /// asks side
    pub asks: OrderbookSide,
}

impl Orderbook {
    /// Create empty orderbook
    pub fn new() -> Orderbook {
        Orderbook {
            bids: IndexMap::new(),
            asks: IndexMap::new(),
        }
    }

    /// Remove zero levels from books
    pub fn clean(&mut self) {
        self.bids = self.bids.iter()
                .map(|(&a,&b)| (a,b))
                .filter(|&(_p,s)|s != 0.)
                .collect::<IndexMap<PriceBits, Size>>();
        self.asks = self.asks.iter()
                .map(|(&a,&b)| (a,b))
                .filter(|&(_p,s)|s != 0.)
                .collect::<IndexMap<PriceBits, Size>>();
    }
}

impl fmt::Debug for Orderbook {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        let _ = write!(f, "bids:\n");
        for (&price, size) in self.bids.iter() {
            let _ = write!(
                f,
                "- price: {} \t - size: {}\n",
                f64::from_bits(price),
                size
            );
        }
        let _ = write!(f, "\n");

        let _ = write!(f, "asks:\n");
        for (&price, size) in self.asks.iter() {
            let _ = write!(
                f,
                "- price: {} \t - size: {}\n",
                f64::from_bits(price),
                size
            );
        }
        write!(f, "\n")
    }
}


/// Data structure for rebinning orderbooks
///
/// If you think of an order as a 2D image, rebinning is lowering the resolution
/// For example, the raw orderbook is 1 nano second apart, you can "zoom out" to 1 second
///
/// Price rebinning is similar.
pub struct RebinnedOrderbook {
    /// a map from time to orderbook
    pub book: IndexMap<u64, Orderbook>,
    /// histogram of price
    pub price_hist: Histogram,
}

impl RebinnedOrderbook {
    /// convert a list of updates to rebinned orderbook with fixed number of time steps bins and ticks bins
    pub fn from(ups: &[Update], step_bins: BinCount, tick_bins: BinCount, m: f64) -> RebinnedOrderbook {

        // build histogram so later can put price and time into bins
        let (price_hist, step_hist) = Histogram::from(&ups, step_bins, tick_bins, m);

        // raw_price -> size
        // using a fine_level to track individual price level instead of a batched one
        let mut fine_level = Orderbook::new();
        // coarse grained books, temp_ob keeps track of current level
        // coarse means rebinned(like snap to grid)
        let mut temp_ob = Orderbook::new();
        // coarse price orderbook across coarse time
        let mut ob_across_time = IndexMap::<Time, Orderbook>::new();

        // iterate over each update
        for up in ups.iter() {
            // ignore trades, since there should be an accompanying level update
            if up.is_trade {
                continue;
            }

            // rebinned ts, price
            let ts = step_hist.to_bin((up.ts / 1000) as f64);
            let price = price_hist.to_bin(up.price as f64);

            // if is an outlier, don't update orderbook
            if ts == None || price == None {
                continue;
            }
            let coarse_time = ts.unwrap().to_bits();
            let coarse_price = price.unwrap().to_bits();

            // get coarse_size and update local book
            let coarse_size = {
                // get fine-grained size
                // if the fine price does not exist in the dict, insert the current size
                // returns a mutable reference
                fine_level.clean();
                let fine_book = if up.is_bid {
                    &mut fine_level.bids
                } else {
                    &mut fine_level.asks
                };
                let fine_size = fine_book.entry((up.price as f64).to_bits()).or_insert(
                    up.size,
                );

                // coarse_size is the size at coarse_price
                let local_side = if up.is_bid {
                    &mut temp_ob.bids
                } else {
                    &mut temp_ob.asks
                };
                let coarse_size = (*local_side).entry(coarse_price).or_insert(up.size);

                if (*fine_size) == up.size {
                    // if level was 0, fine_size == coarse_size == up.size
                    () // do nothing
                } else if (*fine_size) > up.size {
                    // if size shrinks
                    *coarse_size -= (*fine_size) - up.size; // shrink the coarse size
                } else
                /* if (*fine_size) < up.size */
                {
                    // if size grows
                    *coarse_size += up.size - (*fine_size); // grow the coarse size
                }

                *fine_size = up.size;

                // XXX: important
                // there might be orders before the first cancellation
                // we simply ignore those by setting the size to 0
                if *coarse_size < 0. {
                    *coarse_size = 0.;
                }

                *coarse_size
            };

            // if the current coarse_time is not in orderbook
            if !ob_across_time.contains_key(&coarse_time) {
                // insert a copy of current book
                // temp_ob.clean();
                ob_across_time.insert(coarse_time, temp_ob.clone());
            } else {
                // if already in global, modify the orderbook at ts
                let ob_at_time = ob_across_time.get_mut(&coarse_time).unwrap();
                let global_side = if up.is_bid {
                    &mut ob_at_time.bids
                } else {
                    &mut ob_at_time.asks
                };
                (*global_side).insert(coarse_price, coarse_size);
            }
        }

        for v in ob_across_time.values_mut() {
            v.clean();
        }

        RebinnedOrderbook {
            book: ob_across_time,
            price_hist: price_hist,
        }
    }
}

impl fmt::Debug for RebinnedOrderbook {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        for (&ts, ob) in self.book.iter() {
            let _ = write!(f, "ts: {}\n", f64::from_bits(ts));
            let _ = write!(f, "{:?}\n", ob);
        }
        write!(f, "")
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::dtf;
    static FNAME: &str = "test/test-data/bt_btcnav.dtf";

    #[test]
    fn test_level_orderbook() {
        let step_bins = 100;
        let tick_bins = 100;

        let ups = dtf::file_format::decode(FNAME, Some(1000)).unwrap();
        let ob = RebinnedOrderbook::from(ups.as_slice(), step_bins, tick_bins, 2.);

        assert_eq!(ob.book.len(), step_bins - 1);
        for v in ob.book.values() {
            assert!(v.bids.values().len() < tick_bins);
            assert!(v.asks.values().len() < tick_bins);
        }

        assert_eq!(
            format!("{:?}", ob.book.values().next_back().unwrap()),
            "".to_owned()+
"bids:
- price: 0.00010943656738475906 	 - size: 266.0109
- price: 0.00011011131470576117 	 - size: 600
- price: 0.00010923414318845842 	 - size: 157.5617
- price: 0.00010936909265265885 	 - size: 269.10645
- price: 0.00010754727488595314 	 - size: 587.0855
- price: 0.00010997636524156074 	 - size: 69.96973
- price: 0.00011132585988356497 	 - size: 247.91327
- price: 0.00011004383997366096 	 - size: 149.29991

asks:
- price: 0.00011173070827616624 	 - size: 1768.4296
- price: 0.00011098848622306392 	 - size: 0.00012207031

"
,
        );
    }
}