1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
//! The run log: everything that crossed into the engine, in order.
//!
//! # Why a seed is not enough
//!
//! "Same seed, same market" holds only when nothing else varies. But the
//! market an agent trades in depends on the agent's own orders — they apply
//! pressure and move the price — so two runs with one seed and different order
//! flow are different markets, correctly.
//!
//! A seed therefore reproduces a run only if you also reproduce every input:
//! the sessions, the news, the flow, the macro pins, the roster edits, and any
//! draws the embedder took from the external stream. That whole sequence is
//! the specification of a run, and this is it.
//!
//! # What it is for
//!
//! Three things a seed alone cannot do. Replaying someone else's result
//! without their code. Bisecting a run to find the step where it diverged.
//! And archiving a published experiment as data rather than as a script that
//! may not run next year.
//!
//! # What it deliberately does not record
//!
//! Anything the engine can recompute. Prices, attribution, draw counts and
//! book state are all consequences, and logging them would create a second
//! source of truth that could disagree with the first. The log holds INPUTS
//! only; the outputs are whatever replaying them produces.
#![allow(unexpected_cfgs, clippy::useless_conversion)]
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyList};
/// One recorded call.
#[derive(Debug, Clone)]
pub enum LogEntry {
/// `day` is the label the day was opened under, present only when it
/// is not the engine's own counter (`run_days(first_day=...)`,
/// `open_market(day=...)`), so every log of a run that numbered its
/// days from the counter reads as it always did. A label moves no
/// price; it is logged because the book stamps fills with it.
OpenMarket {
day: Option<i64>,
},
CloseMarket,
/// `set_day`: the label the draws and fills from here on carry.
SetDay {
day: i64,
},
Tick {
hour: i64,
minute: i64,
day_of_week: i64,
volatility: f64,
news: Vec<(Option<String>, Option<String>, f64)>,
flow: Vec<(String, f64, f64)>,
},
RunSession {
hour: i64,
minute: i64,
day_of_week: i64,
ticks: usize,
volatility: f64,
close_at_end: bool,
news: Vec<(Option<String>, Option<String>, f64)>,
/// `flow_per_tick`: held on every tick of the session.
flow: Vec<(String, f64, f64)>,
/// `fills`: applied once, on the session's first tick.
fills: Vec<(String, f64, f64)>,
},
PinMacro {
fields: Vec<(String, f64)>,
cycle: Option<String>,
/// The pinned crisis epicentre: a sector key, or `"none"`. Carried
/// for the reason `cycle` is -- it is an input to the run, and a
/// replay that dropped it would draw its own epicentre and call the
/// result the same experiment.
epicentre: Option<String>,
/// The forced-VIX mark (`pin_macro(vix_sets_variance=True)`). An
/// input like the pins beside it; written into `fields` only when
/// true, so every log of a run that never set it is the one it was.
vix_sets_variance: bool,
},
/// The whole `avg_volume` column, one value per instrument.
///
/// A liquidity intervention is an INPUT, not a consequence: nothing in
/// the engine writes this column, so a replay that did not carry it would
/// rebuild a market with its original depth and call it the same run.
SetAvgVolume {
values: Vec<f64>,
},
/// Every company's fair-value inputs, as `Engine.set_fundamentals` wrote
/// them: reported earnings, book value and revenue growth per share, one
/// value each in roster order, NaN where absent.
///
/// An input for the reason `SetAvgVolume` is: the `market.earnings`
/// scenario target writes through it, nothing in the engine writes these
/// figures back, and a replay that did not carry them rebuilt the market
/// with its original earnings and failed its digest. NaN goes out as
/// `None`, which JSON can carry.
SetFundamentals {
eps: Vec<f64>,
book_value_per_share: Vec<f64>,
revenue_growth: Vec<f64>,
},
/// Every company's share count, as `Engine.set_shares_outstanding`
/// wrote it, in roster order. An input for the reason `SetFundamentals`
/// is: nothing in the engine writes the counts back, and a replay that
/// did not carry them rebuilt the market on its original counts.
SetSharesOutstanding {
shares: Vec<f64>,
},
/// The host's premium on the VIX target, as
/// `Engine.set_vix_target_premium` wrote it. An input for the reason
/// `SetFundamentals` is: nothing in the engine writes it, and the VIX
/// reverts toward the target it raises.
SetVixTargetPremium {
points: f64,
half_life_sessions: f64,
},
/// The host's floor under the VIX target (`Engine.set_vix_target_floor`),
/// `None` to clear it.
SetVixTargetFloor {
floor: Option<f64>,
},
/// A `market.earnings` write's common factor, in logs, as
/// `Engine.note_market_earnings` took it. An input for the reason
/// `SetFundamentals` is: the look-through's written level is held from
/// it. Logged only while `earnings_lookthrough_half_life` is set, so every
/// log of a run with the dial off is the one it was.
NoteMarketEarnings {
log_factor: f64,
},
ListInstrument {
ticker: String,
sector: String,
initial_price: f64,
shares_outstanding: f64,
eps: Option<f64>,
book_value_per_share: Option<f64>,
revenue_growth: Option<f64>,
avg_volume: f64,
beta: f64,
short_interest: f64,
},
Delist {
index: usize,
},
/// A draw the embedder took from the EXTERNAL stream.
///
/// Recorded because it MOVES THAT STREAM. Since the stream split it can
/// no longer move the market — that is the cutover guarantee — but the
/// embedder's own randomness is still part of the run's history: a
/// replay that skipped a recorded draw would hand the embedder different
/// values afterwards, and any decision built on them would diverge from
/// the run the log claims to reproduce.
Draw {
normal: bool,
},
Record {
day: u32,
},
/// An agent's order sent to the book. `quantity` is signed, positive to
/// buy. The id is logged only when the caller chose it: an id the
/// engine assigned is a consequence of the log, and replaying the log
/// assigns it again.
Submit {
agent: String,
ticker: String,
quantity: f64,
limit_price: Option<f64>,
order_id: Option<String>,
},
Cancel {
order_id: String,
agent: Option<String>,
},
/// Fills collected by `take_fills`. Logged because collecting them is a
/// change of state (the book no longer owes them), so a replay that did
/// not collect at the same point would hash differently.
TakeFills {
agent: Option<String>,
},
TakeImpacts {
agent: Option<String>,
},
/// Steps of the night session for index futures (`run_night`), between
/// a close and the next open. Logged because each step draws on the
/// derivatives' stream and moves the futures' books; `u32::MAX` is
/// "the rest of the night".
RunNight {
steps: u32,
},
/// An agent's instruction to exercise a single-name option at the next
/// close (`options_names_listed`). Logged because it moves the close's
/// deliveries and positions.
Exercise {
agent: String,
symbol: String,
quantity: f64,
},
/// An agent's exercise instructions on a symbol withdrawn.
CancelExercise {
agent: String,
symbol: String,
},
}
fn news_to_py(py: Python<'_>, news: &[(Option<String>, Option<String>, f64)]) -> PyResult<PyObject> {
let out = PyList::empty_bound(py);
for (ticker, sector, impact) in news {
let d = PyDict::new_bound(py);
d.set_item("ticker", ticker)?;
d.set_item("sector", sector)?;
d.set_item("price_impact", impact)?;
out.append(d)?;
}
Ok(out.into())
}
fn flow_to_py(py: Python<'_>, flow: &[(String, f64, f64)]) -> PyResult<PyObject> {
let d = PyDict::new_bound(py);
for (ticker, buy, sell) in flow {
// A LIST, not a tuple. The log's whole purpose is to be archived and
// replayed as data, and a tuple does not survive a JSON round trip --
// it comes back a list, so `log == json.loads(json.dumps(log))` was
// false and an archived experiment did not compare equal to the run
// that produced it. Replay worked either way, which is exactly why
// this would have gone unnoticed until someone diffed two logs.
d.set_item(ticker, PyList::new_bound(py, [*buy, *sell]))?;
}
Ok(d.into())
}
impl LogEntry {
/// Render as a plain dict, so a log is JSON-serialisable without this
/// crate taking an opinion about how it should be stored.
pub fn to_py(&self, py: Python<'_>) -> PyResult<PyObject> {
let d = PyDict::new_bound(py);
match self {
LogEntry::OpenMarket { day } => {
d.set_item("op", "open_market")?;
if let Some(day) = day {
d.set_item("day", day)?;
}
}
LogEntry::CloseMarket => {
d.set_item("op", "close_market")?;
}
LogEntry::SetDay { day } => {
d.set_item("op", "set_day")?;
d.set_item("day", day)?;
}
LogEntry::Tick {
hour,
minute,
day_of_week,
volatility,
news,
flow,
} => {
d.set_item("op", "tick")?;
d.set_item("hour", hour)?;
d.set_item("minute", minute)?;
d.set_item("day_of_week", day_of_week)?;
d.set_item("volatility", volatility)?;
d.set_item("news", news_to_py(py, news)?)?;
d.set_item("order_flow", flow_to_py(py, flow)?)?;
}
LogEntry::RunSession {
hour,
minute,
day_of_week,
ticks,
volatility,
close_at_end,
news,
flow,
fills,
} => {
d.set_item("op", "run_session")?;
d.set_item("hour", hour)?;
d.set_item("minute", minute)?;
d.set_item("day_of_week", day_of_week)?;
d.set_item("ticks", ticks)?;
d.set_item("volatility", volatility)?;
d.set_item("close_at_end", close_at_end)?;
d.set_item("news", news_to_py(py, news)?)?;
// Named after the arguments that carried them. A log written
// before 0.8.5 has `order_flow` here instead, which meant the
// per-tick flow, and the replays read it as `flow_per_tick`,
// so an archived run replays into the market it described.
d.set_item("flow_per_tick", flow_to_py(py, flow)?)?;
d.set_item("fills", flow_to_py(py, fills)?)?;
}
LogEntry::PinMacro { fields, cycle, epicentre, vix_sets_variance } => {
d.set_item("op", "pin_macro")?;
let f = PyDict::new_bound(py);
for (name, value) in fields {
f.set_item(name, value)?;
}
if let Some(c) = cycle {
f.set_item("cycle", c)?;
}
if let Some(x) = epicentre {
f.set_item("epicentre", x)?;
}
if *vix_sets_variance {
f.set_item("vix_sets_variance", true)?;
}
d.set_item("fields", f)?;
}
LogEntry::ListInstrument {
ticker,
sector,
initial_price,
shares_outstanding,
eps,
book_value_per_share,
revenue_growth,
avg_volume,
beta,
short_interest,
} => {
d.set_item("op", "list_instrument")?;
d.set_item("ticker", ticker)?;
d.set_item("sector", sector)?;
d.set_item("initial_price", initial_price)?;
d.set_item("shares_outstanding", shares_outstanding)?;
d.set_item("eps", eps)?;
d.set_item("book_value_per_share", book_value_per_share)?;
d.set_item("revenue_growth", revenue_growth)?;
d.set_item("avg_volume", avg_volume)?;
d.set_item("beta", beta)?;
d.set_item("short_interest", short_interest)?;
}
LogEntry::SetAvgVolume { values } => {
d.set_item("op", "set_avg_volume")?;
d.set_item("values", values.to_vec())?;
}
LogEntry::SetFundamentals { eps, book_value_per_share, revenue_growth } => {
let absent = |v: &Vec<f64>| -> Vec<Option<f64>> {
v.iter().map(|x| if x.is_nan() { None } else { Some(*x) }).collect()
};
d.set_item("op", "set_fundamentals")?;
d.set_item("eps", absent(eps))?;
d.set_item("book_value_per_share", absent(book_value_per_share))?;
d.set_item("revenue_growth", absent(revenue_growth))?;
}
LogEntry::SetSharesOutstanding { shares } => {
d.set_item("op", "set_shares_outstanding")?;
d.set_item("shares", shares.to_vec())?;
}
LogEntry::SetVixTargetPremium { points, half_life_sessions } => {
d.set_item("op", "set_vix_target_premium")?;
d.set_item("points", points)?;
d.set_item("half_life_sessions", half_life_sessions)?;
}
LogEntry::SetVixTargetFloor { floor } => {
d.set_item("op", "set_vix_target_floor")?;
d.set_item("floor", floor)?;
}
LogEntry::NoteMarketEarnings { log_factor } => {
d.set_item("op", "note_market_earnings")?;
d.set_item("log_factor", log_factor)?;
}
LogEntry::Delist { index } => {
d.set_item("op", "delist")?;
d.set_item("index", index)?;
}
LogEntry::Draw { normal } => {
d.set_item("op", if *normal { "draw_normal" } else { "draw_uniform" })?;
}
LogEntry::Record { day } => {
d.set_item("op", "record")?;
d.set_item("day", day)?;
}
LogEntry::Submit { agent, ticker, quantity, limit_price, order_id } => {
d.set_item("op", "submit")?;
d.set_item("agent", agent)?;
d.set_item("ticker", ticker)?;
d.set_item("quantity", quantity)?;
d.set_item("limit_price", limit_price)?;
d.set_item("order_id", order_id)?;
}
LogEntry::Cancel { order_id, agent } => {
d.set_item("op", "cancel")?;
d.set_item("order_id", order_id)?;
d.set_item("agent", agent)?;
}
LogEntry::TakeFills { agent } => {
d.set_item("op", "take_fills")?;
d.set_item("agent", agent)?;
}
LogEntry::TakeImpacts { agent } => {
d.set_item("op", "take_impacts")?;
d.set_item("agent", agent)?;
}
LogEntry::RunNight { steps } => {
d.set_item("op", "run_night")?;
// The rest of the night is None, which JSON can carry.
d.set_item("steps", if *steps == u32::MAX { None } else { Some(*steps) })?;
}
LogEntry::Exercise { agent, symbol, quantity } => {
d.set_item("op", "exercise")?;
d.set_item("agent", agent)?;
d.set_item("symbol", symbol)?;
d.set_item("quantity", quantity)?;
}
LogEntry::CancelExercise { agent, symbol } => {
d.set_item("op", "cancel_exercise")?;
d.set_item("agent", agent)?;
d.set_item("symbol", symbol)?;
}
}
Ok(d.into())
}
}