use std::{
collections::{HashSet, VecDeque},
fmt::Write as _,
io::{BufReader, BufWriter, Read as _, Write as _},
path::Path,
sync::{
Arc, Condvar, Mutex, MutexGuard,
atomic::{AtomicUsize, Ordering},
mpsc::{Receiver, SyncSender, TrySendError, sync_channel},
},
thread::JoinHandle,
time::Instant,
};
use serde::{Deserialize, Serialize};
use trading_data_dag::{Fidelity, Fire, Ink, Observer, Plot, Want};
use crate::{
api_types::{Activation, ActivationFrame, DayOut, ExactBlock, FidelityOut, GuideOut, InkOut, PlotOut, PointOut, SeriesOut, TopoNode},
cost::{Clock, Cost, TICK_STRIDE},
};
const QUEUE: usize = 1024;
const BATCH: usize = 64;
const TAPE_MAGIC: [u8; 8] = *b"EXECVIZT";
const TAPE_SCHEMA: u32 = 2;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Backpressure {
Block,
Drop,
}
pub struct Tape {
viz: Viz,
join: JoinHandle<()>,
}
impl Tape {
pub fn new(price_node: Option<&'static str>, capacity: usize, bucket_ms: i64, mode: Backpressure) -> (Self, Recorder) {
assert!(capacity > 3 && bucket_ms > 0);
let dropped = Arc::new(AtomicUsize::new(0));
let viz = Viz {
inner: Arc::new(Mutex::new(Inner {
price_node: price_node.map(str::to_string),
capacity,
bucket_ms,
topology: Vec::new(),
cost: Vec::new(),
ticks: VecDeque::new(),
opened: 0,
fires: Vec::new(),
keep: Vec::new(),
fired: Vec::new(),
sealed: false,
dropped: dropped.clone(),
series: Vec::new(),
cursor: 0,
})),
done: Arc::new(Condvar::new()),
};
let (tx, rx) = sync_channel(QUEUE);
let (back, recycle) = sync_channel(capacity / 2);
let tape = viz.clone();
let batch_cap = BATCH.min(capacity / 4).max(1);
let join = std::thread::Builder::new()
.name("exec_viz tape".into())
.spawn(move || {
let mut batch = Vec::with_capacity(batch_cap);
let mut freed = Vec::new();
while let Ok(first) = rx.recv() {
batch.push(first);
batch.extend(rx.try_iter().take(batch_cap - 1));
{
let mut t = tape.lock();
for msg in batch.drain(..) {
freed.append(&mut t.absorb(msg));
}
}
for acts in freed.drain(..) {
if back.try_send(acts).is_err() {
break;
}
}
}
tape.lock().sealed = true;
tape.done.notify_all();
})
.expect("spawn the tape thread");
(
Tape { viz: viz.clone(), join },
Recorder {
tx,
recycle,
viz,
mode,
raw: Vec::new(),
meta: None,
acts: Acts::default(),
spans: Vec::new(),
clock: Clock::new(),
opened: 0,
dropped,
ts_ns: 0,
idx: 0,
timed: false,
},
)
}
pub fn viz(&self) -> Viz {
self.viz.clone()
}
pub fn save(self, path: impl AsRef<Path>) -> std::io::Result<()> {
self.viz.wait_sealed();
self.join.join().expect("the tape thread only ends by running out of ticks");
self.viz.write(path)
}
}
#[derive(Clone)]
pub struct Viz {
inner: Arc<Mutex<Inner>>,
done: Arc<Condvar>,
}
impl Viz {
pub fn bytes(&self) -> usize {
let t = self.lock();
t.ticks.iter().map(Tick::bytes).sum()
}
fn wait_sealed(&self) {
let mut t = self.lock();
while !t.sealed {
t = self.done.wait(t).unwrap_or_else(std::sync::PoisonError::into_inner);
}
}
fn write(&self, path: impl AsRef<Path>) -> std::io::Result<()> {
let t = self.lock();
let last = t.last();
let file = TapeFile {
price_node: t.price_node.clone(),
capacity: t.capacity,
bucket_ms: t.bucket_ms,
topology: t.topology.clone(),
cost: t.cost.clone(),
ticks: t.ticks.iter().take(last.map_or(0, |l| l + 1)).cloned().collect(),
opened: t.head(),
fires: t.fires.clone(),
keep: t.keep.clone(),
dropped: t.dropped.load(Ordering::Relaxed),
series: t.series.clone(),
cursor: t.cursor,
};
let mut w = BufWriter::new(std::fs::File::create(path)?);
w.write_all(&TAPE_MAGIC)?;
w.write_all(&TAPE_SCHEMA.to_le_bytes())?;
rmp_serde::encode::write(&mut w, &file).map_err(std::io::Error::other)?;
w.flush()
}
pub fn load(path: impl AsRef<Path>) -> std::io::Result<Self> {
let path = path.as_ref();
let mut r = BufReader::new(std::fs::File::open(path)?);
let mut magic = [0u8; 8];
r.read_exact(&mut magic)?;
if magic != TAPE_MAGIC {
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{} is not an exec_viz tape", path.display())));
}
let mut schema = [0u8; 4];
r.read_exact(&mut schema)?;
let schema = u32::from_le_bytes(schema);
if schema != TAPE_SCHEMA {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("{} is a schema-{schema} tape and this build reads schema {TAPE_SCHEMA} — re-record it", path.display()),
));
}
let file: TapeFile = rmp_serde::decode::from_read(&mut r).map_err(std::io::Error::other)?;
let ticks: VecDeque<Tick> = file.ticks.into();
Ok(Self {
inner: Arc::new(Mutex::new(Inner {
price_node: file.price_node,
capacity: file.capacity,
bucket_ms: file.bucket_ms,
topology: file.topology,
cost: file.cost,
opened: file.opened,
fires: file.fires,
keep: file.keep,
fired: index(&ticks),
ticks,
sealed: true,
dropped: Arc::new(AtomicUsize::new(file.dropped)),
series: file.series,
cursor: file.cursor,
})),
done: Arc::new(Condvar::new()),
})
}
pub(crate) fn lock(&self) -> MutexGuard<'_, Inner> {
self.inner.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
}
pub struct Recorder {
tx: SyncSender<TickMsg>,
recycle: Receiver<Acts>,
viz: Viz,
mode: Backpressure,
raw: Vec<&'static str>,
meta: Option<Vec<Meta>>,
acts: Acts,
spans: Vec<f64>,
clock: Clock,
opened: usize,
dropped: Arc<AtomicUsize>,
ts_ns: i64,
idx: usize,
timed: bool,
}
impl Recorder {
pub fn at(&mut self, ts_ns: i64) -> Rec<'_> {
self.ts_ns = ts_ns;
self.idx = 0;
self.timed = self.opened % TICK_STRIDE == 0;
self.opened += 1;
if self.acts.ends.is_empty() {
self.acts = self.recycle.try_recv().unwrap_or_default();
}
self.acts.clear();
if self.timed {
self.spans.clear();
self.clock.mark(Instant::now());
}
Rec(self)
}
pub fn seal(self) {
let Recorder { tx, viz, .. } = self;
drop(tx);
viz.wait_sealed();
}
}
pub struct Rec<'a>(&'a mut Recorder);
#[derive(Deserialize, Serialize)]
struct TapeFile {
price_node: Option<String>,
capacity: usize,
bucket_ms: i64,
topology: Vec<TopoNode>,
cost: Vec<Cost>,
ticks: Vec<Tick>,
opened: usize,
fires: Vec<usize>,
keep: Vec<u8>,
dropped: usize,
series: Vec<SeriesOut>,
cursor: usize,
}
impl Observer for Rec<'_> {
fn want(&self, _: &'static str) -> Want {
Want::Exact
}
fn on(&mut self, node: &'static str, deps: &'static [&'static str], gates: &'static [bool], fire: Fire<'_>) {
let entry = self.0.timed.then(Instant::now);
let r = &mut *self.0;
let i = r.idx;
r.idx += 1;
if r.raw.len() == i {
r.raw.push(node);
r.meta.get_or_insert_default().push(Meta {
node,
deps,
gates,
dims: fire.dims,
plots: fire.plots,
clock_ms: fire.clock.map(|tf| tf.duration().as_millis() as i64),
fidelity: fire.fidelity,
deriv: fire.deriv.map(|d| d.to_string()),
});
} else {
assert_eq!(r.raw[i], node, "step order shifted between ticks");
}
if let Some(entry) = entry {
let ns = r.clock.mark(entry);
r.spans.push(ns);
}
assert!(fire.dims.iter().product::<usize>() > 0, "{node} flattens to nothing");
assert!(fire.jac.is_none_or(|j| !j.is_empty()), "{node}'s Jacobian ran and came back empty");
if let Some(vals) = fire.vals {
write!(r.acts.outs, "{}", fire.glance).expect("`String`'s `Write` is infallible");
r.acts.vals.extend_from_slice(vals);
}
if let Some(jac) = fire.jac {
r.acts.jac.extend_from_slice(jac);
}
if let Some((block, widths)) = fire.exact_block {
r.acts.block.extend_from_slice(block);
r.acts.widths.extend_from_slice(widths);
}
r.acts.exact.push(fire.exact);
r.acts.ran.push(fire.ran);
r.acts.close();
if r.timed {
r.clock.mark(Instant::now());
}
}
}
impl Drop for Rec<'_> {
fn drop(&mut self) {
if std::thread::panicking() {
return;
}
let r = &mut *self.0;
assert_eq!(r.idx, r.acts.ends.len(), "every node reports on every tick");
let msg = TickMsg {
ts_ns: r.ts_ns,
acts: std::mem::take(&mut r.acts),
meta: r.meta.take(),
spans: r.timed.then(|| r.spans.clone()),
};
let gone = "the tape thread outlives every recorder but a sealed one";
match r.mode {
Backpressure::Block => r.tx.send(msg).expect(gone),
Backpressure::Drop => match r.tx.try_send(msg) {
Ok(()) => {}
Err(TrySendError::Full(msg)) => {
r.dropped.fetch_add(1, Ordering::Relaxed);
r.acts = msg.acts;
r.meta = msg.meta;
}
Err(TrySendError::Disconnected(_)) => panic!("{gone}"),
},
}
}
}
struct Meta {
node: &'static str,
deps: &'static [&'static str],
gates: &'static [bool],
dims: &'static [usize],
plots: &'static [Plot],
clock_ms: Option<i64>,
fidelity: Fidelity,
deriv: Option<String>,
}
struct TickMsg {
ts_ns: i64,
acts: Acts,
meta: Option<Vec<Meta>>,
spans: Option<Vec<f64>>,
}
#[derive(Clone, Default, Deserialize, Serialize)]
struct Acts {
outs: String,
vals: Vec<f64>,
jac: Vec<f64>,
block: Vec<f64>,
widths: Vec<usize>,
exact: Vec<bool>,
ran: Vec<bool>,
ends: Vec<Ends>,
}
impl Acts {
fn get(&self, i: usize) -> Option<ActRef<'_>> {
let end = *self.ends.get(i)?;
let start = if i == 0 { Ends::default() } else { self.ends[i - 1] };
let span = |a: u32, b: u32| (b > a).then_some(a as usize..b as usize);
Some(ActRef {
out: &self.outs[start.out as usize..end.out as usize],
ran: self.ran[i],
vals: span(start.vals, end.vals).map(|r| &self.vals[r]),
jac: span(start.jac, end.jac).map(|r| &self.jac[r]),
block: span(start.block, end.block).map(|r| (&self.block[r], &self.widths[start.widths as usize..end.widths as usize])),
exact: self.exact[i],
})
}
fn close(&mut self) {
let end = |n: usize| u32::try_from(n).expect("one tick's columns are far under 4G");
self.ends.push(Ends {
out: end(self.outs.len()),
vals: end(self.vals.len()),
jac: end(self.jac.len()),
block: end(self.block.len()),
widths: end(self.widths.len()),
});
}
fn clear(&mut self) {
self.outs.clear();
self.vals.clear();
self.jac.clear();
self.block.clear();
self.widths.clear();
self.exact.clear();
self.ran.clear();
self.ends.clear();
}
}
#[derive(Clone, Copy, Default, Deserialize, Serialize)]
struct Ends {
out: u32,
vals: u32,
jac: u32,
block: u32,
widths: u32,
}
#[derive(Clone, Copy)]
struct ActRef<'a> {
out: &'a str,
ran: bool,
vals: Option<&'a [f64]>,
jac: Option<&'a [f64]>,
block: Option<(&'a [f64], &'a [usize])>,
exact: bool,
}
#[derive(Clone, Deserialize, Serialize)]
struct Tick {
abs: usize,
ts_ns: i64,
acts: Acts,
ranks: Vec<u8>,
}
impl Tick {
fn bytes(&self) -> usize {
let a = &self.acts;
let col = a.outs.capacity() + (a.vals.capacity() + a.jac.capacity() + a.block.capacity()) * size_of::<f64>();
size_of::<Self>() + col + a.widths.capacity() * size_of::<usize>() + a.exact.capacity() + a.ends.capacity() * size_of::<Ends>() + self.ranks.capacity()
}
}
pub(crate) struct Inner {
price_node: Option<String>,
capacity: usize,
bucket_ms: i64,
topology: Vec<TopoNode>,
cost: Vec<Cost>,
ticks: VecDeque<Tick>,
opened: usize,
fires: Vec<usize>,
keep: Vec<u8>,
fired: Vec<Vec<usize>>,
sealed: bool,
dropped: Arc<AtomicUsize>,
series: Vec<SeriesOut>,
cursor: usize,
}
impl Inner {
fn absorb(&mut self, msg: TickMsg) -> Vec<Acts> {
let TickMsg { ts_ns, acts, meta, spans } = msg;
for m in meta.into_iter().flatten() {
let gates: Vec<String> = m.deps.iter().zip(m.gates).filter(|(_, g)| **g).map(|(d, _)| trim(d)).collect();
let deps: Vec<String> = m.deps.iter().map(|d| trim(d)).collect();
let topo = TopoNode {
node: trim(m.node),
deps,
gates,
dims: m.dims.to_vec(),
plots: m.plots.iter().map(PlotOut::from).collect(),
cost_ns: None,
fidelity: m.fidelity.into(),
deriv: m.deriv,
};
self.series.push(SeriesOut {
node: topo.node.clone(),
deps: topo.deps.clone(),
gates: topo.gates.clone(),
dims: topo.dims.clone(),
plots: topo.plots.clone(),
points: Vec::new(),
clock_ms: m.clock_ms,
});
self.topology.push(topo);
self.cost.push(Cost::default());
self.fires.push(0);
self.keep.push(0);
}
if let Some(spans) = spans {
assert_eq!(spans.len(), self.cost.len(), "a clocked tick clocks every node in it");
for (c, ns) in self.cost.iter_mut().zip(spans) {
c.sample(ns);
}
}
let ms = ts_ns / 1_000_000;
let bucket = ms - ms.rem_euclid(self.bucket_ms);
let mut ranks = vec![0u8; self.series.len()];
for (i, s) in self.series.iter_mut().enumerate() {
let Some(vals) = acts.get(i).and_then(|a| a.vals) else { continue };
ranks[i] = 1 + self.fires[i].trailing_zeros().min(63) as u8;
self.fires[i] += 1;
match s.points.last_mut() {
Some(p) if p.ts_ms >= bucket => {
p.vals.clear();
p.vals.extend_from_slice(vals);
}
_ => s.points.push(PointOut { ts_ms: bucket, vals: vals.to_vec() }),
}
}
let freed = if self.ticks.len() == self.capacity { self.thin() } else { Vec::new() };
debug_assert!(self.ticks.len() < self.capacity);
let abs = self.opened;
self.opened += 1;
let p = self.ticks.len();
self.fired.resize(ranks.len(), Vec::new());
for (i, r) in ranks.iter().enumerate() {
if *r > 0 {
self.fired[i].push(p);
}
}
self.ticks.push_back(Tick { abs, ts_ns, acts, ranks });
freed
}
fn thin(&mut self) -> Vec<Acts> {
let whole = self.opened.saturating_sub(self.capacity / 2);
let pre = self.ticks.iter().take_while(|t| t.abs < whole);
let mut counts = vec![0usize; self.keep.len()];
loop {
counts.iter_mut().for_each(|c| *c = 0);
let mut union = 0;
for t in pre.clone() {
let mut any = false;
for (i, &r) in t.ranks.iter().enumerate() {
if r > self.keep[i] {
counts[i] += 1;
any = true;
}
}
union += usize::from(any);
}
if union <= self.capacity / 4 {
break;
}
let (i, n) = counts.iter().enumerate().max_by_key(|(_, n)| **n).expect("a non-empty union came from some node");
assert!(*n > 0, "the union counts ticks no node claims");
self.keep[i] += 1;
}
let mut freed = Vec::new();
let mut kept = VecDeque::with_capacity(self.ticks.len());
for t in self.ticks.drain(..) {
match t.abs >= whole || t.ranks.iter().zip(&self.keep).any(|(r, k)| r > k) {
true => kept.push_back(t),
false => freed.push(t.acts),
}
}
self.fired = index(kept.iter());
self.ticks = kept;
freed
}
fn head(&self) -> usize {
if self.sealed { self.opened } else { self.opened.saturating_sub(1) }
}
fn last(&self) -> Option<usize> {
let n = if self.sealed { self.ticks.len() } else { self.ticks.len().saturating_sub(1) };
n.checked_sub(1)
}
fn pos(&self) -> Option<usize> {
let last = self.last()?;
Some(self.ticks.partition_point(|t| t.abs < self.cursor).saturating_sub(1).min(last))
}
fn park(&mut self, p: usize) {
let Some(last) = self.last() else { return };
self.cursor = self.ticks[p.min(last)].abs + 1;
}
pub(crate) fn topology(&self) -> Vec<TopoNode> {
if self.head() < 1 {
return Vec::new();
}
self.topology.iter().zip(&self.cost).map(|(n, c)| TopoNode { cost_ns: c.ns(), ..n.clone() }).collect()
}
pub(crate) fn day(&self) -> DayOut {
let buffers: HashSet<&str> = self.series.iter().map(|s| s.node.as_str()).filter(|n| n.starts_with("Buffer<")).collect();
if let Some(p) = &self.price_node {
match self.series.iter().find(|s| &s.node == p) {
Some(s) => assert_eq!(s.dims.iter().product::<usize>(), 5, "price_node `{p}` must be an OHLCV bar"),
None => assert!(self.head() < 1, "price_node `{p}` names no node in the graph"),
}
}
DayOut {
series: self.series.iter().filter(|s| !buffers.contains(s.node.as_str())).cloned().collect(),
price_node: self.price_node.clone(),
}
}
pub(crate) fn frame(&self) -> ActivationFrame {
let tick = self.pos().map(|p| (p, &self.ticks[p]));
ActivationFrame {
tick: tick.map_or(0, |(_, t)| t.abs + 1),
total: self.head(),
sealed: self.sealed,
dropped: self.dropped.load(Ordering::Relaxed),
pending: false,
found: true,
gap: tick.map_or(1, |(p, t)| t.abs + 1 - p.checked_sub(1).map_or(0, |q| self.ticks[q].abs + 1)),
ts_ns: tick.map_or(0, |(_, t)| t.ts_ns),
activations: tick.map_or_else(Vec::new, |(p, t)| {
self.topology
.iter()
.enumerate()
.filter_map(|(i, n)| {
let a = t.acts.get(i)?;
let held = if a.vals.is_some() { a } else { self.held(i, p).unwrap_or(a) };
Some(Activation {
node: n.node.clone(),
deps: n.deps.clone(),
gates: n.gates.clone(),
out: held.out.to_string(),
ran: a.ran,
fired: a.vals.is_some(),
dims: n.dims.clone(),
vals: held.vals.map(|v| v.iter().map(|x| x.is_finite().then_some(*x)).collect()),
jac: a.jac.map(|j| j.iter().map(|w| (!w.is_nan()).then_some(*w)).collect()),
exact_block: a.block.map(|(cols, widths)| ExactBlock {
cols: cols.iter().map(|w| (!w.is_nan()).then_some(*w)).collect(),
widths: widths.to_vec(),
}),
cost_ns: self.cost[i].ns(),
exact: a.exact,
})
})
.collect()
}),
}
}
fn held(&self, i: usize, p: usize) -> Option<ActRef<'_>> {
let fired = self.fired.get(i)?;
let q = fired[..fired.partition_point(|&q| q <= p)].last()?;
self.ticks[*q].acts.get(i).filter(|a| a.vals.is_some())
}
pub(crate) fn step(&mut self, n: usize) -> ActivationFrame {
let target = self.pos().map_or(0, |p| p.saturating_add(n));
self.park(target);
let mut f = self.frame();
f.pending = !self.sealed && self.last().is_none_or(|l| target > l);
f
}
pub(crate) fn seek(&mut self, tick: usize) -> ActivationFrame {
self.park(self.ticks.partition_point(|t| t.abs < tick).saturating_sub(1));
let mut f = self.frame();
f.pending = !self.sealed && tick > self.head();
f
}
pub(crate) fn seek_ts(&mut self, ts_ns: i64) -> ActivationFrame {
self.park(self.ticks.iter().rposition(|t| t.ts_ns <= ts_ns).unwrap_or(0));
let mut f = self.frame();
f.pending = !self.sealed && self.ticks.back().is_some_and(|t| t.ts_ns < ts_ns);
f
}
pub(crate) fn step_until(&mut self, node: &str) -> ActivationFrame {
match self.topology.iter().position(|n| n.node == node) {
Some(i) => self.scan(|t| t.acts.get(i).is_some_and(|a| a.vals.is_some())),
None => ActivationFrame { found: false, ..self.frame() },
}
}
pub(crate) fn step_until_change(&mut self, nodes: &[String]) -> ActivationFrame {
let watched: Vec<usize> = self.topology.iter().enumerate().filter(|(_, n)| nodes.contains(&n.node)).map(|(i, _)| i).collect();
let baseline: Vec<Option<String>> = {
let now = self.frame();
watched.iter().map(|&i| now.activations.get(i).map(|a| a.out.clone())).collect()
};
self.scan(|t| {
watched
.iter()
.zip(&baseline)
.any(|(&i, was)| t.acts.get(i).is_some_and(|a| a.vals.is_some() && was.as_deref() != Some(a.out)))
})
}
fn scan(&mut self, hit: impl Fn(&Tick) -> bool) -> ActivationFrame {
let mut found = false;
if let (Some(mut p), Some(last)) = (self.pos(), self.last()) {
while p < last {
p += 1;
if hit(&self.ticks[p]) {
found = true;
break;
}
}
if found || !self.sealed {
self.park(p);
}
}
let mut f = self.frame();
f.pending = !self.sealed && !found;
f.found = found;
f
}
}
impl From<Fidelity> for FidelityOut {
fn from(f: Fidelity) -> Self {
match f {
Fidelity::Exact => FidelityOut::Exact,
Fidelity::Partial(omits) => FidelityOut::Partial { omits: omits.to_string() },
Fidelity::Opaque(why) => FidelityOut::Opaque { why: why.to_string() },
}
}
}
impl From<&Plot> for PlotOut {
fn from(p: &Plot) -> Self {
let ink = |i: &Ink| InkOut { l: i.l, c: i.c, a: i.a };
PlotOut {
slots: p.slots.to_vec(),
range: p.range,
guides: p
.guides
.iter()
.map(|g| GuideOut {
label: g.label.to_string(),
value: g.value,
ink: ink(&g.ink),
})
.collect(),
labels: p.labels.iter().map(|axis| axis.iter().map(|l| l.to_string()).collect()).collect(),
inks: p.inks.iter().map(ink).collect(),
overlay: p.overlay,
solo: p.solo,
bars: p.bars,
candles: p.candles,
}
}
}
fn index<'a>(ticks: impl IntoIterator<Item = &'a Tick>) -> Vec<Vec<usize>> {
let mut fired: Vec<Vec<usize>> = Vec::new();
for (p, t) in ticks.into_iter().enumerate() {
fired.resize(fired.len().max(t.ranks.len()), Vec::new());
for (i, r) in t.ranks.iter().enumerate() {
if *r > 0 {
fired[i].push(p);
}
}
}
fired
}
fn trim(name: &str) -> String {
let mut out = String::with_capacity(name.len());
let mut seg = 0;
let mut rest = name;
while let Some(c) = rest.chars().next() {
if let Some(after) = rest.strip_prefix("::") {
match out[seg..].starts_with(|c: char| c.is_lowercase() || c == '_') {
true => out.truncate(seg),
false => {
out.push_str("::");
seg = out.len();
}
}
rest = after;
continue;
}
out.push(c);
if !(c.is_alphanumeric() || c == '_') {
seg = out.len();
}
rest = &rest[c.len_utf8()..];
}
out
}
#[cfg(test)]
mod tests {
use trading_data_dag::Plot;
use super::*;
fn fire(vals: &[f64]) -> Fire<'_> {
Fire {
glance: &f64::NAN,
dims: &[1],
plots: &[Plot::DEFAULT],
clock: None,
fidelity: Fidelity::Exact,
ran: true,
fires: 1,
vals: Some(vals),
dep_dims: &[],
jac: None,
exact_block: None,
exact: false,
formula: None,
deriv: None,
trace: None,
}
}
#[test]
fn a_backwards_tick_leaves_the_series_ascending() {
let (tape, mut rec) = Tape::new(None, 8, 60_000, Backpressure::Block);
let viz = tape.viz();
for min in [2, 3, 2, 4] {
let ts_ns = min * 60 * 1_000_000_000;
rec.at(ts_ns).on("N", &[], &[], fire(&[min as f64]));
}
rec.seal();
let day = viz.lock().day();
let ts: Vec<i64> = day.series[0].points.iter().map(|p| p.ts_ms).collect();
assert!(ts.windows(2).all(|w| w[0] < w[1]), "{ts:?}");
}
#[test]
fn a_slow_step_is_charged_to_the_node_that_took_it() {
let (tape, mut rec) = Tape::new(None, 4096, 60_000, Backpressure::Block);
let viz = tape.viz();
for i in 0..TICK_STRIDE * 32 {
let mut r = rec.at(i as i64 * 60 * 1_000_000_000);
r.on("Fast", &[], &[], fire(&[0.0]));
std::thread::sleep(std::time::Duration::from_micros(100));
r.on("Slow", &["Fast"], &[false], fire(&[1.0]));
}
rec.seal();
let cost: Vec<f64> = viz.lock().topology().iter().map(|n| n.cost_ns.expect("blocks closed")).collect();
assert!(cost[1] > 50_000.0, "the slept step reads as {}ns", cost[1]);
assert!(cost[0] * 4.0 < cost[1], "the fast step reads as {}ns next to {}ns", cost[0], cost[1]);
}
#[test]
fn a_dropped_tick_is_counted_rather_than_lost() {
let (tape, mut rec) = Tape::new(None, 8, 60_000, Backpressure::Drop);
let viz = tape.viz();
let stepped = QUEUE + 64;
let held = viz.lock();
for i in 0..stepped as i64 {
rec.at(i * 60 * 1_000_000_000).on("N", &[], &[], fire(&[i as f64]));
}
drop(held);
rec.seal();
let f = viz.lock().frame();
assert!(f.dropped > 0, "the queue never filled, so this measured nothing");
assert_eq!(f.dropped + f.total, stepped, "every stepped tick is either on the tape or counted");
}
#[test]
fn the_whole_run_stays_walkable_past_the_capacity() {
let (tape, mut rec) = Tape::new(None, 64, 60_000, Backpressure::Block);
let viz = tape.viz();
for i in 0..5000 {
rec.at(i * 60 * 1_000_000_000).on("N", &[], &[], fire(&[i as f64]));
}
rec.seal();
let walk = walk(&viz);
assert_eq!(walk.first(), Some(&1), "the recording's first tick is addressable");
assert_eq!(walk.last(), Some(&5000), "and so is its last: {walk:?}");
assert!(walk.windows(2).all(|w| w[0] < w[1]), "no step stands still: {walk:?}");
assert!(walk.windows(2).rev().take(8).all(|w| w[1] - w[0] == 1), "{walk:?}");
}
#[test]
fn a_rare_node_keeps_its_fires_past_the_capacity() {
const RARE: usize = 500;
const TICKS: usize = 200_000;
let (tape, mut rec) = Tape::new(None, 4096, 60_000, Backpressure::Block);
let viz = tape.viz();
for i in 0..TICKS {
let mut r = rec.at(i as i64 * 60 * 1_000_000_000);
r.on("A", &[], &[], fire(&[i as f64]));
r.on(
"B",
&["A"],
&[false],
Fire {
vals: (i % RARE == 0).then_some(&[1.0]),
..fire(&[1.0])
},
);
}
rec.seal();
let mut t = viz.lock();
t.seek(0);
let mut reached = vec![t.frame().tick - 1];
loop {
let at = t.step_until("B").tick - 1;
if at == *reached.last().expect("seeded") {
break;
}
reached.push(at);
}
let want: Vec<usize> = (0..TICKS).step_by(RARE).collect();
assert_eq!(reached, want, "every one of B's {} fires is addressable", want.len());
}
#[test]
fn a_scan_that_finds_nothing_stays_put() {
let (tape, mut rec) = Tape::new(None, 64, 60_000, Backpressure::Block);
let viz = tape.viz();
for i in 0..500 {
rec.at(i * 60 * 1_000_000_000).on("N", &[], &[], fire(&[i as f64]));
}
rec.seal();
let mut t = viz.lock();
let was = t.seek(300).tick;
let f = t.step_until("Absent");
assert_eq!(f.tick, was, "a search for a node the graph does not carry moved the cursor");
assert!(!f.found, "and it reported as though it had reached one");
t.seek(usize::MAX);
let end = t.frame().tick;
let f = t.step_until("N");
assert_eq!(f.tick, end);
assert!(!f.found && !f.pending, "a sealed tape with no hit is an answer, not a wait: {f:?}");
}
#[test]
fn a_saved_tape_reads_back_as_the_one_recorded() {
let (tape, mut rec) = Tape::new(Some("P"), 64, 60_000, Backpressure::Block);
let viz = tape.viz();
for i in 0..5000 {
let mut r = rec.at(i * 60 * 1_000_000_000);
r.on(
"P",
&[],
&[],
Fire {
dims: &[5],
..fire(&[1.0, 2.0, 0.5, i as f64, 10.0])
},
);
r.on("N", &["P"], &[false], fire(&[i as f64]));
}
rec.seal();
let path = std::env::temp_dir().join("exec_viz_round_trip.bin");
tape.save(&path).expect("save");
let back = Viz::load(&path).expect("load");
std::fs::remove_file(&path).expect("written above");
assert_eq!(back.lock().topology(), viz.lock().topology());
let (was, is) = (viz.lock().day(), back.lock().day());
assert_eq!(is.price_node, was.price_node);
assert_eq!(is.series, was.series);
assert_eq!(walk(&back), walk(&viz));
}
fn walk(viz: &Viz) -> Vec<usize> {
let mut t = viz.lock();
let mut walk = vec![t.seek(0).tick];
loop {
let tick = t.step(1).tick;
if tick == *walk.last().expect("seeded") {
return walk;
}
walk.push(tick);
}
}
}