#![allow(unexpected_cfgs, clippy::useless_conversion)]
use std::sync::Arc;
use arrow::array::{ArrayRef, Float64Array, RecordBatch, UInt32Array};
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use arrow::ffi_stream::FFI_ArrowArrayStream;
use arrow::record_batch::RecordBatchReader;
use pyo3::prelude::*;
use pyo3::types::PyCapsule;
use crate::python::ValidationError;
pub fn bars_schema() -> SchemaRef {
Arc::new(Schema::new(vec![
Field::new("day", DataType::UInt32, false),
Field::new("tick", DataType::UInt32, false),
Field::new("instrument_id", DataType::UInt32, false),
Field::new("close", DataType::Float64, false),
Field::new("volume", DataType::Float64, false),
]))
}
pub fn truth_schema() -> SchemaRef {
let mut fields = vec![
Field::new("day", DataType::UInt32, false),
Field::new("tick", DataType::UInt32, false),
Field::new("instrument_id", DataType::UInt32, false),
Field::new("mispricing_s", DataType::Float64, false),
Field::new("fundamental_value", DataType::Float64, false),
Field::new("anchor_price", DataType::Float64, false),
];
for key in crate::market::factors::S_COMPONENT_KEYS {
fields.push(Field::new(key, DataType::Float64, false));
}
fields.push(Field::new(
crate::market::factors::JUMP_COMPONENT_KEY,
DataType::Float64,
false,
));
fields.push(Field::new(
crate::market::factors::OVERNIGHT_COMPONENT_KEY,
DataType::Float64,
false,
));
fields.push(Field::new(
crate::market::factors::FAIR_VALUE_COMPONENT_KEY,
DataType::Float64,
false,
));
fields.push(Field::new(
crate::market::factors::DIVIDEND_COMPONENT_KEY,
DataType::Float64,
false,
));
Arc::new(Schema::new(fields))
}
pub fn bars_batch(
day: u32,
ticks: usize,
instruments: usize,
prices: &[f64],
volumes: &[f64],
volume_base: &[f64],
) -> Result<RecordBatch, String> {
let rows = ticks * instruments;
if prices.len() < rows || volumes.len() < rows {
return Err(format!(
"buffer shorter than {ticks} ticks x {instruments} instruments"
));
}
let mut volume_col = Vec::with_capacity(rows);
for t in 0..ticks {
for i in 0..instruments {
volume_col.push(bar_volume(volumes, volume_base, instruments, i, t, t + 1));
}
}
let mut day_col = Vec::with_capacity(rows);
let mut tick_col = Vec::with_capacity(rows);
let mut id_col = Vec::with_capacity(rows);
for t in 0..ticks {
for i in 0..instruments {
day_col.push(day);
tick_col.push(t as u32);
id_col.push(i as u32);
}
}
let columns: Vec<ArrayRef> = vec![
Arc::new(UInt32Array::from(day_col)),
Arc::new(UInt32Array::from(tick_col)),
Arc::new(UInt32Array::from(id_col)),
Arc::new(Float64Array::from(prices[..rows].to_vec())),
Arc::new(Float64Array::from(volume_col)),
];
RecordBatch::try_new(bars_schema(), columns).map_err(|e| e.to_string())
}
pub fn bar_volume(
volumes: &[f64],
volume_base: &[f64],
instruments: usize,
i: usize,
first: usize,
last: usize,
) -> f64 {
let before = if first == 0 {
volume_base.get(i).copied().unwrap_or(0.0)
} else {
volumes[(first - 1) * instruments + i]
};
volumes[(last - 1) * instruments + i] - before
}
pub fn truth_batch(
day: u32,
ticks: usize,
instruments: usize,
mispricing: &[f64],
fundamental: &[f64],
anchor: &[f64],
components: &[Vec<f64>; crate::market::factors::COMPONENT_COUNT],
) -> Result<RecordBatch, String> {
let rows = ticks * instruments;
if mispricing.len() < rows || fundamental.len() < rows || anchor.len() < rows {
return Err(format!(
"buffer shorter than {ticks} ticks x {instruments} instruments"
));
}
for column in components.iter() {
if column.len() < rows {
return Err(format!(
"component buffer shorter than {ticks} ticks x {instruments} instruments"
));
}
}
let mut day_col = Vec::with_capacity(rows);
let mut tick_col = Vec::with_capacity(rows);
let mut id_col = Vec::with_capacity(rows);
for t in 0..ticks {
for i in 0..instruments {
day_col.push(day);
tick_col.push(t as u32);
id_col.push(i as u32);
}
}
let mut columns: Vec<ArrayRef> = vec![
Arc::new(UInt32Array::from(day_col)),
Arc::new(UInt32Array::from(tick_col)),
Arc::new(UInt32Array::from(id_col)),
Arc::new(Float64Array::from(mispricing[..rows].to_vec())),
Arc::new(Float64Array::from(fundamental[..rows].to_vec())),
Arc::new(Float64Array::from(anchor[..rows].to_vec())),
];
for column in components.iter() {
columns.push(Arc::new(Float64Array::from(column[..rows].to_vec())));
}
RecordBatch::try_new(truth_schema(), columns).map_err(|e| e.to_string())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DepthColumns {
Present,
Absent,
PartialDay,
}
pub fn prints_schema(depth: DepthColumns) -> SchemaRef {
prints_schema_with(depth, false)
}
pub fn prints_schema_with(depth: DepthColumns, distribution: bool) -> SchemaRef {
let mut fields = vec![
Field::new("day", DataType::UInt32, false),
Field::new("tick", DataType::UInt32, false),
Field::new("instrument_id", DataType::UInt32, false),
Field::new("print", DataType::Float64, false),
Field::new("model_price", DataType::Float64, false),
Field::new("shock", DataType::Float64, false),
Field::new("absorbed", DataType::Float64, false),
Field::new("clamp", DataType::Float64, false),
Field::new("repriced", DataType::Float64, false),
];
if depth == DepthColumns::Present {
fields.push(Field::new("unbounded_print", DataType::Float64, false));
fields.push(Field::new("liquidity_share", DataType::Float64, false));
}
if distribution {
fields.push(Field::new("distribution", DataType::Float64, false));
}
let absorbed = "absorbed is measured to the printed price, so it carries \
the circuit breaker as well as the book. clamp is the \
breaker's own part and book = absorbed - clamp is the \
book's. Read them apart: on a halted print the two pull \
opposite ways and often cancel exactly, so absorbed alone \
reads zero on a name the breaker had just moved. \
repriced is what was written to the price between the \
last print and the tick: the close's re-mark to a \
published macro figure, a pin_macro's re-mark and the \
overnight opening print. repriced + shock + absorbed is \
the print's log move from the last print; it is NaN \
where not known, on the first print after a restore on a \
model that can write a price between prints. ";
let caveat = match depth {
DepthColumns::Present => format!(
"{absorbed}unbounded_print is the same tick settled against every \
resting level, under the same draws and from the same book \
state; it is one tick from the real state and says nothing \
about the next one. liquidity_share is NEGATIVE where the depth \
bound truncated a walk, which is most rows that carry one, it \
is not bounded by one, and it is NaN where the print did not \
move. The unbounded move is (1 - liquidity_share) times the \
printed move."
),
DepthColumns::Absent => format!(
"{absorbed}The depth counterfactual did not run, so \
unbounded_print and liquidity_share are absent. Set \
Engine.settle_depth_counterfactual(True) before the run to get \
them."
),
DepthColumns::PartialDay => format!(
"{absorbed}The depth counterfactual was switched part way \
through a day, so that day recorded fewer counterfactual values \
than it has rows. unbounded_print and liquidity_share are \
dropped for the whole day rather than served with a gap in \
them. Set Engine.settle_depth_counterfactual before the first \
session of a day."
),
};
let metadata = std::collections::HashMap::from([("caveat".to_string(), caveat)]);
Arc::new(Schema::new_with_metadata(fields, metadata))
}
#[allow(clippy::too_many_arguments)]
pub fn prints_batch_with(
day: u32,
ticks: usize,
instruments: usize,
prints: &[f64],
model_price: &[f64],
shock: &[f64],
absorbed: &[f64],
clamp: &[f64],
repriced: &[f64],
unbounded_print: &[f64],
liquidity_share: &[f64],
depth: DepthColumns,
distribution: Option<&[f64]>,
) -> Result<RecordBatch, String> {
let rows = ticks * instruments;
for (name, len) in [
("print", prints.len()),
("model_price", model_price.len()),
("shock", shock.len()),
("absorbed", absorbed.len()),
("clamp", clamp.len()),
("repriced", repriced.len()),
] {
if len < rows {
return Err(format!(
"{name} buffer shorter than {ticks} ticks x {instruments} instruments"
));
}
}
if depth == DepthColumns::Present
&& (unbounded_print.len() < rows || liquidity_share.len() < rows)
{
return Err(format!(
"counterfactual buffer shorter than {ticks} ticks x {instruments} instruments"
));
}
let mut day_col = Vec::with_capacity(rows);
let mut tick_col = Vec::with_capacity(rows);
let mut id_col = Vec::with_capacity(rows);
for t in 0..ticks {
for i in 0..instruments {
day_col.push(day);
tick_col.push(t as u32);
id_col.push(i as u32);
}
}
let mut columns: Vec<ArrayRef> = vec![
Arc::new(UInt32Array::from(day_col)),
Arc::new(UInt32Array::from(tick_col)),
Arc::new(UInt32Array::from(id_col)),
Arc::new(Float64Array::from(prints[..rows].to_vec())),
Arc::new(Float64Array::from(model_price[..rows].to_vec())),
Arc::new(Float64Array::from(shock[..rows].to_vec())),
Arc::new(Float64Array::from(absorbed[..rows].to_vec())),
Arc::new(Float64Array::from(clamp[..rows].to_vec())),
Arc::new(Float64Array::from(repriced[..rows].to_vec())),
];
if depth == DepthColumns::Present {
columns.push(Arc::new(Float64Array::from(unbounded_print[..rows].to_vec())));
columns.push(Arc::new(Float64Array::from(liquidity_share[..rows].to_vec())));
}
if let Some(amounts) = distribution {
let mut col = vec![0.0; rows];
if ticks > 0 {
for (i, slot) in col.iter_mut().take(instruments).enumerate() {
*slot = amounts.get(i).copied().unwrap_or(0.0);
}
}
columns.push(Arc::new(Float64Array::from(col)));
}
let schema = if distribution.is_some() {
prints_schema_with(depth, true)
} else {
prints_schema(depth)
};
RecordBatch::try_new(schema, columns).map_err(|e| e.to_string())
}
struct BatchReader {
schema: SchemaRef,
batches: std::vec::IntoIter<RecordBatch>,
}
impl Iterator for BatchReader {
type Item = Result<RecordBatch, arrow::error::ArrowError>;
fn next(&mut self) -> Option<Self::Item> {
self.batches.next().map(Ok)
}
}
impl RecordBatchReader for BatchReader {
fn schema(&self) -> SchemaRef {
self.schema.clone()
}
}
#[pyclass(name = "ArrowStream", module = "tradefloor._core")]
pub struct PyArrowStream {
schema: SchemaRef,
batches: Vec<RecordBatch>,
name: String,
}
impl PyArrowStream {
pub fn new(name: &str, schema: SchemaRef, batches: Vec<RecordBatch>) -> Self {
Self { schema, batches, name: name.to_string() }
}
}
#[pymethods]
impl PyArrowStream {
#[pyo3(signature = (requested_schema = None))]
fn __arrow_c_stream__<'py>(
&self,
py: Python<'py>,
requested_schema: Option<PyObject>,
) -> PyResult<Bound<'py, PyCapsule>> {
let _ = requested_schema;
let reader = BatchReader {
schema: self.schema.clone(),
batches: self.batches.clone().into_iter(),
};
let stream = FFI_ArrowArrayStream::new(Box::new(reader));
let name = std::ffi::CString::new("arrow_array_stream").unwrap();
PyCapsule::new_bound(py, stream, Some(name))
}
#[getter]
fn num_rows(&self) -> usize {
self.batches.iter().map(|b| b.num_rows()).sum()
}
#[getter]
fn num_batches(&self) -> usize {
self.batches.len()
}
#[getter]
fn columns(&self) -> Vec<String> {
self.schema.fields().iter().map(|f| f.name().clone()).collect()
}
fn __repr__(&self) -> String {
format!(
"ArrowStream({:?}, {} rows in {} batch(es)). Read it with \
pyarrow.table(stream) or polars.from_arrow(stream); pip install \
\"tradefloor[arrow]\" for pyarrow.",
self.name,
self.num_rows(),
self.batches.len()
)
}
}
pub fn arrow_err(e: String) -> PyErr {
ValidationError::new_err(e)
}
pub fn macro_schema() -> SchemaRef {
Arc::new(Schema::new(vec![
Field::new("day", DataType::UInt32, false),
Field::new("vix", DataType::Float64, false),
Field::new("federal_funds_rate", DataType::Float64, false),
Field::new("corporate_bond_yield", DataType::Float64, false),
Field::new("inflation_rate", DataType::Float64, false),
Field::new("unemployment_rate", DataType::Float64, false),
Field::new("gdp_growth", DataType::Float64, false),
Field::new("qe_pe_boost", DataType::Float64, false),
Field::new("fear_greed_index", DataType::Float64, false),
Field::new("universe_stress", DataType::Float64, false),
]))
}
#[derive(Debug, Clone, Copy)]
pub struct MacroRow {
pub day: u32,
pub vix: f64,
pub federal_funds_rate: f64,
pub corporate_bond_yield: f64,
pub inflation_rate: f64,
pub unemployment_rate: f64,
pub gdp_growth: f64,
pub qe_pe_boost: f64,
pub fear_greed_index: f64,
pub universe_stress: f64,
}
pub fn macro_batch(rows: &[MacroRow]) -> Result<RecordBatch, String> {
let columns: Vec<ArrayRef> = vec![
Arc::new(UInt32Array::from(rows.iter().map(|r| r.day).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.vix).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.federal_funds_rate).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.corporate_bond_yield).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.inflation_rate).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.unemployment_rate).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.gdp_growth).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.qe_pe_boost).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.fear_greed_index).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.universe_stress).collect::<Vec<_>>())),
];
RecordBatch::try_new(macro_schema(), columns).map_err(|e| e.to_string())
}
pub fn distributions_schema() -> SchemaRef {
Arc::new(Schema::new(vec![
Field::new("day", DataType::UInt32, false),
Field::new("instrument_id", DataType::UInt32, false),
Field::new("declared_day", DataType::UInt32, false),
Field::new("ex_day", DataType::UInt32, false),
Field::new("amount", DataType::Float64, false),
Field::new("kind", DataType::Utf8, false),
]))
}
pub fn distributions_batch(rows: &[&crate::engine::Distribution]) -> Result<RecordBatch, String> {
let u = |x: i64| if x < 0 { 0u32 } else { x as u32 };
let columns: Vec<ArrayRef> = vec![
Arc::new(UInt32Array::from(rows.iter().map(|r| u(r.ex_day)).collect::<Vec<_>>())),
Arc::new(UInt32Array::from(rows.iter().map(|r| r.company as u32).collect::<Vec<_>>())),
Arc::new(UInt32Array::from(rows.iter().map(|r| u(r.declared_day)).collect::<Vec<_>>())),
Arc::new(UInt32Array::from(rows.iter().map(|r| u(r.ex_day)).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.amount).collect::<Vec<_>>())),
Arc::new(arrow::array::StringArray::from(rows.iter().map(|_| "cash").collect::<Vec<_>>())),
];
RecordBatch::try_new(distributions_schema(), columns).map_err(|e| e.to_string())
}
pub const FUTURES_TABLES_SCHEMA_VERSION: &str = "1";
fn versioned(fields: Vec<Field>) -> SchemaRef {
let metadata = std::collections::HashMap::from([(
"schema_version".to_string(),
FUTURES_TABLES_SCHEMA_VERSION.to_string(),
)]);
Arc::new(Schema::new_with_metadata(fields, metadata))
}
pub fn futures_bars_schema() -> SchemaRef {
versioned(vec![
Field::new("day", DataType::UInt32, false),
Field::new("symbol", DataType::Utf8, false),
Field::new("root", DataType::Utf8, false),
Field::new("expiry", DataType::Int64, false),
Field::new("front", DataType::Boolean, false),
Field::new("price", DataType::Float64, false),
Field::new("fair", DataType::Float64, false),
Field::new("basis_bp", DataType::Float64, false),
Field::new("bid", DataType::Float64, true),
Field::new("ask", DataType::Float64, true),
Field::new("mark", DataType::Float64, true),
Field::new("index", DataType::Float64, false),
Field::new("sessions_to_expiry", DataType::Float64, false),
Field::new("initial_margin", DataType::Float64, true),
])
}
#[derive(Debug, Clone)]
pub struct FuturesBarRow {
pub day: u32,
pub symbol: String,
pub root: String,
pub expiry: i64,
pub front: bool,
pub price: f64,
pub fair: f64,
pub basis_bp: f64,
pub bid: Option<f64>,
pub ask: Option<f64>,
pub mark: Option<f64>,
pub index: f64,
pub sessions_to_expiry: f64,
pub initial_margin: Option<f64>,
}
pub fn futures_bars_batch(rows: &[&FuturesBarRow]) -> Result<RecordBatch, String> {
use arrow::array::{BooleanArray, Int64Array, StringArray};
let f = |g: fn(&FuturesBarRow) -> f64| -> ArrayRef {
Arc::new(Float64Array::from(rows.iter().map(|r| g(r)).collect::<Vec<_>>()))
};
let o = |g: fn(&FuturesBarRow) -> Option<f64>| -> ArrayRef {
Arc::new(Float64Array::from(rows.iter().map(|r| g(r)).collect::<Vec<_>>()))
};
let columns: Vec<ArrayRef> = vec![
Arc::new(UInt32Array::from(rows.iter().map(|r| r.day).collect::<Vec<_>>())),
Arc::new(StringArray::from(rows.iter().map(|r| r.symbol.as_str()).collect::<Vec<_>>())),
Arc::new(StringArray::from(rows.iter().map(|r| r.root.as_str()).collect::<Vec<_>>())),
Arc::new(Int64Array::from(rows.iter().map(|r| r.expiry).collect::<Vec<_>>())),
Arc::new(BooleanArray::from(rows.iter().map(|r| r.front).collect::<Vec<_>>())),
f(|r| r.price),
f(|r| r.fair),
f(|r| r.basis_bp),
o(|r| r.bid),
o(|r| r.ask),
o(|r| r.mark),
f(|r| r.index),
f(|r| r.sessions_to_expiry),
o(|r| r.initial_margin),
];
RecordBatch::try_new(futures_bars_schema(), columns).map_err(|e| e.to_string())
}
pub fn settlements_schema() -> SchemaRef {
versioned(vec![
Field::new("session", DataType::Int64, false),
Field::new("symbol", DataType::Utf8, false),
Field::new("root", DataType::Utf8, false),
Field::new("kind", DataType::Utf8, false),
Field::new("value", DataType::Float64, false),
Field::new("reference", DataType::Float64, false),
Field::new("event", DataType::Utf8, false),
Field::new("agent", DataType::Utf8, true),
Field::new("quantity", DataType::Float64, false),
Field::new("shares", DataType::Float64, false),
Field::new("cash", DataType::Float64, false),
])
}
pub fn settlements_batch(rows: &[crate::derivatives::Settlement]) -> Result<RecordBatch, String> {
use arrow::array::{Int64Array, StringArray};
let columns: Vec<ArrayRef> = vec![
Arc::new(Int64Array::from(rows.iter().map(|r| r.session).collect::<Vec<_>>())),
Arc::new(StringArray::from(rows.iter().map(|r| r.symbol.as_str()).collect::<Vec<_>>())),
Arc::new(StringArray::from(rows.iter().map(|r| r.root.as_str()).collect::<Vec<_>>())),
Arc::new(StringArray::from(rows.iter().map(|r| r.kind.family()).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.value).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.reference).collect::<Vec<_>>())),
Arc::new(StringArray::from(rows.iter().map(|r| r.event.as_str()).collect::<Vec<_>>())),
Arc::new(StringArray::from(rows.iter().map(|r| r.agent.as_deref()).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.quantity).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.shares).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.cash).collect::<Vec<_>>())),
];
RecordBatch::try_new(settlements_schema(), columns).map_err(|e| e.to_string())
}
pub fn option_quotes_schema() -> SchemaRef {
versioned(vec![
Field::new("day", DataType::UInt32, false),
Field::new("symbol", DataType::Utf8, false),
Field::new("expiry", DataType::Int64, false),
Field::new("right", DataType::Utf8, false),
Field::new("strike", DataType::Float64, false),
Field::new("bid", DataType::Float64, true),
Field::new("ask", DataType::Float64, true),
Field::new("bid_size", DataType::Float64, false),
Field::new("ask_size", DataType::Float64, false),
Field::new("mid", DataType::Float64, false),
Field::new("iv", DataType::Float64, false),
Field::new("fair", DataType::Float64, false),
Field::new("surface_iv", DataType::Float64, false),
Field::new("delta", DataType::Float64, false),
Field::new("gamma", DataType::Float64, false),
Field::new("vega", DataType::Float64, false),
Field::new("theta", DataType::Float64, false),
Field::new("forward", DataType::Float64, false),
Field::new("index", DataType::Float64, false),
Field::new("rate", DataType::Float64, false),
Field::new("dividends", DataType::Float64, false),
Field::new("sessions_to_expiry", DataType::Float64, false),
Field::new("open_interest", DataType::Float64, false),
])
}
pub fn option_quotes_batch(day: u32, rows: &[crate::derivatives::OptionQuote]) -> Result<RecordBatch, String> {
use crate::derivatives::OptionQuote;
use arrow::array::{Int64Array, StringArray};
let f = |g: fn(&OptionQuote) -> f64| -> ArrayRef {
Arc::new(Float64Array::from(rows.iter().map(g).collect::<Vec<_>>()))
};
let o = |g: fn(&OptionQuote) -> Option<f64>| -> ArrayRef {
Arc::new(Float64Array::from(rows.iter().map(g).collect::<Vec<_>>()))
};
let columns: Vec<ArrayRef> = vec![
Arc::new(UInt32Array::from(vec![day; rows.len()])),
Arc::new(StringArray::from(rows.iter().map(|r| r.symbol.as_str()).collect::<Vec<_>>())),
Arc::new(Int64Array::from(rows.iter().map(|r| r.expiry).collect::<Vec<_>>())),
Arc::new(StringArray::from(rows.iter().map(|r| r.right.letter().to_string()).collect::<Vec<_>>())),
f(|r| r.strike),
o(|r| r.bid),
o(|r| r.ask),
f(|r| r.bid_size),
f(|r| r.ask_size),
f(|r| r.mid),
f(|r| r.iv),
f(|r| r.fair),
f(|r| r.surface_iv),
f(|r| r.greeks.delta),
f(|r| r.greeks.gamma),
f(|r| r.greeks.vega),
f(|r| r.greeks.theta),
f(|r| r.forward),
f(|r| r.index),
f(|r| r.rate),
f(|r| r.dividends),
f(|r| r.sessions_to_expiry),
f(|r| r.open_interest),
];
RecordBatch::try_new(option_quotes_schema(), columns).map_err(|e| e.to_string())
}
pub fn fills_schema() -> SchemaRef {
Arc::new(Schema::new(vec![
Field::new("day", DataType::UInt32, false),
Field::new("step", DataType::UInt32, false),
Field::new("tick", DataType::UInt32, false),
Field::new("instrument_id", DataType::UInt32, false),
Field::new("quantity", DataType::Float64, false),
Field::new("price", DataType::Float64, false),
Field::new("worst_price", DataType::Float64, false),
Field::new("notional", DataType::Float64, false),
]))
}
#[allow(clippy::too_many_arguments)]
pub fn fills_batch(
day: Vec<u32>,
step: Vec<u32>,
tick: Vec<u32>,
instrument_id: Vec<u32>,
quantity: Vec<f64>,
price: Vec<f64>,
worst_price: Vec<f64>,
notional: Vec<f64>,
) -> Result<RecordBatch, String> {
let n = day.len();
for (name, len) in [
("step", step.len()),
("tick", tick.len()),
("instrument_id", instrument_id.len()),
("quantity", quantity.len()),
("price", price.len()),
("worst_price", worst_price.len()),
("notional", notional.len()),
] {
if len != n {
return Err(format!("{name} has {len} rows, expected {n}"));
}
}
let columns: Vec<ArrayRef> = vec![
Arc::new(UInt32Array::from(day)),
Arc::new(UInt32Array::from(step)),
Arc::new(UInt32Array::from(tick)),
Arc::new(UInt32Array::from(instrument_id)),
Arc::new(Float64Array::from(quantity)),
Arc::new(Float64Array::from(price)),
Arc::new(Float64Array::from(worst_price)),
Arc::new(Float64Array::from(notional)),
];
RecordBatch::try_new(fills_schema(), columns).map_err(|e| e.to_string())
}
#[pyfunction]
#[allow(clippy::too_many_arguments)]
pub fn fills_stream(
day: Vec<u32>,
step: Vec<u32>,
tick: Vec<u32>,
instrument_id: Vec<u32>,
quantity: Vec<f64>,
price: Vec<f64>,
worst_price: Vec<f64>,
notional: Vec<f64>,
) -> PyResult<PyArrowStream> {
let batch = fills_batch(
day, step, tick, instrument_id, quantity, price, worst_price, notional,
)
.map_err(arrow_err)?;
Ok(PyArrowStream::new("fills", fills_schema(), vec![batch]))
}
#[derive(Debug, Clone)]
pub struct RecordedDay {
pub day: u32,
pub ticks: usize,
pub instruments: usize,
pub prices: Vec<f64>,
pub volumes: Vec<f64>,
pub volume_base: Vec<f64>,
pub opens: Vec<f64>,
pub mispricing: Vec<f64>,
pub fundamental: Vec<f64>,
pub anchor: Vec<f64>,
pub components: [Vec<f64>; crate::market::factors::COMPONENT_COUNT],
pub shock: Vec<f64>,
pub absorbed: Vec<f64>,
pub clamp: Vec<f64>,
pub repriced: Vec<f64>,
pub unbounded_print: Vec<f64>,
pub liquidity_share: Vec<f64>,
pub distribution: Vec<f64>,
}
pub fn ohlc_schema() -> SchemaRef {
Arc::new(Schema::new(vec![
Field::new("day", DataType::UInt32, false),
Field::new("bar", DataType::UInt32, false),
Field::new("instrument_id", DataType::UInt32, false),
Field::new("open", DataType::Float64, false),
Field::new("high", DataType::Float64, false),
Field::new("low", DataType::Float64, false),
Field::new("close", DataType::Float64, false),
Field::new("volume", DataType::Float64, false),
]))
}
pub fn ohlc_batch(day: &RecordedDay, bucket: usize) -> Result<RecordBatch, String> {
if bucket == 0 {
return Err("bucket must be at least one tick".to_string());
}
let n = day.instruments;
if n == 0 || day.ticks == 0 {
return RecordBatch::try_new(
ohlc_schema(),
vec![
Arc::new(UInt32Array::from(Vec::<u32>::new())),
Arc::new(UInt32Array::from(Vec::<u32>::new())),
Arc::new(UInt32Array::from(Vec::<u32>::new())),
Arc::new(Float64Array::from(Vec::<f64>::new())),
Arc::new(Float64Array::from(Vec::<f64>::new())),
Arc::new(Float64Array::from(Vec::<f64>::new())),
Arc::new(Float64Array::from(Vec::<f64>::new())),
Arc::new(Float64Array::from(Vec::<f64>::new())),
],
)
.map_err(|e| e.to_string());
}
let bars = day.ticks.div_ceil(bucket);
let rows = bars * n;
let (mut day_c, mut bar_c, mut id_c) = (
Vec::with_capacity(rows),
Vec::with_capacity(rows),
Vec::with_capacity(rows),
);
let (mut o, mut h, mut l, mut c, mut v) = (
Vec::with_capacity(rows),
Vec::with_capacity(rows),
Vec::with_capacity(rows),
Vec::with_capacity(rows),
Vec::with_capacity(rows),
);
for bar in 0..bars {
let first = bar * bucket;
let last = core::cmp::min(first + bucket, day.ticks);
for i in 0..n {
let session_open = if first == 0 && day.opens.len() == n {
day.opens[i]
} else {
f64::NAN
};
let mut open = session_open;
let mut high = if session_open.is_nan() { f64::NEG_INFINITY } else { session_open };
let mut low = if session_open.is_nan() { f64::INFINITY } else { session_open };
let mut close = f64::NAN;
for t in first..last {
let price = day.prices[t * n + i];
if t == first && open.is_nan() {
open = price;
}
if price > high {
high = price;
}
if price < low {
low = price;
}
close = price;
}
let volume = bar_volume(&day.volumes, &day.volume_base, n, i, first, last);
day_c.push(day.day);
bar_c.push(bar as u32);
id_c.push(i as u32);
o.push(open);
h.push(high);
l.push(low);
c.push(close);
v.push(volume);
}
}
RecordBatch::try_new(
ohlc_schema(),
vec![
Arc::new(UInt32Array::from(day_c)),
Arc::new(UInt32Array::from(bar_c)),
Arc::new(UInt32Array::from(id_c)),
Arc::new(Float64Array::from(o)),
Arc::new(Float64Array::from(h)),
Arc::new(Float64Array::from(l)),
Arc::new(Float64Array::from(c)),
Arc::new(Float64Array::from(v)),
],
)
.map_err(|e| e.to_string())
}
pub fn book_schema() -> SchemaRef {
Arc::new(Schema::new(vec![
Field::new("day", DataType::UInt32, false),
Field::new("tick", DataType::UInt32, false),
Field::new("instrument_id", DataType::UInt32, false),
Field::new("side", DataType::UInt32, false),
Field::new("level", DataType::UInt32, false),
Field::new("price", DataType::Float64, false),
Field::new("size", DataType::Float64, false),
]))
}
#[derive(Debug, Clone, Copy)]
pub struct BookRow {
pub day: u32,
pub tick: u32,
pub instrument_id: u32,
pub side: u32,
pub level: u32,
pub price: f64,
pub size: f64,
}
pub fn book_batch(rows: &[BookRow]) -> Result<RecordBatch, String> {
let columns: Vec<ArrayRef> = vec![
Arc::new(UInt32Array::from(rows.iter().map(|r| r.day).collect::<Vec<_>>())),
Arc::new(UInt32Array::from(rows.iter().map(|r| r.tick).collect::<Vec<_>>())),
Arc::new(UInt32Array::from(rows.iter().map(|r| r.instrument_id).collect::<Vec<_>>())),
Arc::new(UInt32Array::from(rows.iter().map(|r| r.side).collect::<Vec<_>>())),
Arc::new(UInt32Array::from(rows.iter().map(|r| r.level).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.price).collect::<Vec<_>>())),
Arc::new(Float64Array::from(rows.iter().map(|r| r.size).collect::<Vec<_>>())),
];
RecordBatch::try_new(book_schema(), columns).map_err(|e| e.to_string())
}