#[cfg(doctest)]
#[doc = include_str!("../README.md")]
pub struct ReadmeDoctests;
use std::collections::HashMap;
use fv_value::Value;
mod acc;
pub mod batch_join;
pub mod batch_window;
mod join;
pub mod lookup_join;
mod rank;
mod session;
pub mod sharded_window;
mod snapshot;
mod watermark;
use acc::Acc;
pub use batch_join::{FrozenFile, StreamJoinBatch};
pub use batch_window::{BatchOperator, WindowAggBatch};
pub use join::{Side, StreamJoin};
pub use lookup_join::LookupJoinBatch;
pub use rank::{LastN, TopN};
pub use session::SessionAgg;
pub use sharded_window::ShardedWindow;
pub mod error;
pub use error::{state_fault, StateError};
use watermark::Watermark;
pub trait WindowOperator {
fn expect_partitions(&mut self, partitions: &[i32], now_ms: i64);
fn push(&mut self, partition: i32, event_ms: i64, row: &[(String, Value)]) -> bool;
fn push_batch(&mut self, _partition: i32, _batch: &arrow::array::RecordBatch, _event_ms: &[i64]) -> Option<usize> {
None
}
fn advance_batch(&mut self, _now_ms: i64) -> Option<Option<arrow::array::RecordBatch>> {
None
}
fn flush_batch(&mut self) -> Option<Option<arrow::array::RecordBatch>> {
None
}
fn advance(&mut self, now_ms: i64) -> Vec<WindowRow>;
fn advance_to(&mut self, watermark_ms: i64) -> Vec<WindowRow>;
fn advance_batch_to(&mut self, _watermark_ms: i64) -> Option<Option<arrow::array::RecordBatch>> {
None
}
fn flush(&mut self) -> Vec<WindowRow>;
fn state_bytes(&self) -> usize {
0
}
fn open_windows(&self) -> usize {
0
}
fn snapshot(&mut self) -> Result<Vec<u8>, StateError>;
fn load_snapshot(&mut self, bytes: &[u8]) -> Result<(), String>;
fn shard_files_supported(&self) -> bool {
false
}
fn snapshot_shard_files(
&mut self,
_dir: &std::path::Path,
_tag: &str,
) -> Result<Vec<(usize, std::path::PathBuf)>, StateError> {
unreachable!("snapshot_shard_files without shard_files_supported")
}
fn load_shard_files(&mut self, _files: &[(usize, std::path::PathBuf)]) -> Result<(), String> {
Err("this operator does not support file-backed restore".into())
}
}
impl WindowOperator for WindowAgg {
fn expect_partitions(&mut self, partitions: &[i32], now_ms: i64) {
WindowAgg::expect_partitions(self, partitions, now_ms)
}
fn push(&mut self, partition: i32, event_ms: i64, row: &[(String, Value)]) -> bool {
WindowAgg::push(self, partition, event_ms, row)
}
fn advance(&mut self, now_ms: i64) -> Vec<WindowRow> {
WindowAgg::advance(self, now_ms)
}
fn advance_to(&mut self, watermark_ms: i64) -> Vec<WindowRow> {
WindowAgg::advance_to(self, watermark_ms)
}
fn flush(&mut self) -> Vec<WindowRow> {
WindowAgg::flush(self)
}
fn snapshot(&mut self) -> Result<Vec<u8>, StateError> {
Ok(WindowAgg::snapshot(self))
}
fn load_snapshot(&mut self, bytes: &[u8]) -> Result<(), String> {
WindowAgg::load_snapshot(self, bytes)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AggOp {
Count,
Sum,
Avg,
Min,
Max,
CountDistinct,
VarPop,
VarSamp,
StddevPop,
StddevSamp,
BoolAnd,
BoolOr,
BitAnd,
BitOr,
BitXor,
}
impl AggOp {
pub(crate) fn is_variance(self) -> bool {
matches!(
self,
AggOp::VarPop | AggOp::VarSamp | AggOp::StddevPop | AggOp::StddevSamp
)
}
}
#[derive(Debug, Clone)]
pub struct Agg {
pub op: AggOp,
pub column: String,
pub alias: String,
}
impl Agg {
pub fn count(alias: impl Into<String>) -> Self {
Self {
op: AggOp::Count,
column: String::new(),
alias: alias.into(),
}
}
pub fn of(op: AggOp, column: impl Into<String>, alias: impl Into<String>) -> Self {
Self {
op,
column: column.into(),
alias: alias.into(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WindowKind {
Tumbling { size_ms: i64 },
Sliding { size_ms: i64, slide_ms: i64 },
}
impl WindowKind {
pub(crate) fn size(&self) -> i64 {
match self {
WindowKind::Tumbling { size_ms } | WindowKind::Sliding { size_ms, .. } => *size_ms,
}
}
pub fn starts_for(&self, t: i64) -> Vec<i64> {
match self {
WindowKind::Tumbling { size_ms } => vec![t.div_euclid(*size_ms) * size_ms],
WindowKind::Sliding { size_ms, slide_ms } => {
let mut starts = Vec::new();
let mut s = t.div_euclid(*slide_ms) * slide_ms; while s > t - size_ms {
starts.push(s);
s -= slide_ms;
}
starts
}
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum Trigger {
#[default]
OnWatermark,
EveryRows(u64),
EveryMs(i64),
}
impl Trigger {
pub(crate) fn fingerprint(&self) -> String {
match self {
Trigger::OnWatermark => String::new(),
Trigger::EveryRows(n) => format!(";trigger=rows({n})"),
Trigger::EveryMs(ms) => format!(";trigger=ms({ms})"),
}
}
pub(crate) fn early_fires(&self) -> bool {
!matches!(self, Trigger::OnWatermark)
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct WindowRow {
pub window_start: i64,
pub window_end: i64,
pub key: Vec<Value>,
pub aggs: Vec<(String, Value)>,
}
pub struct WindowAgg {
kind: WindowKind,
group_by: Vec<String>,
aggs: Vec<Agg>,
key_ids: HashMap<String, u32>,
key_data: Vec<(String, Vec<Value>)>,
state: HashMap<(i64, u32), Acc>,
with_sets: bool,
with_welford: bool,
wm: Watermark,
fired_through: i64,
late_dropped: u64,
}
impl WindowAgg {
pub fn tumbling(size_ms: i64, allowed_lateness_ms: i64, group_by: Vec<String>, aggs: Vec<Agg>) -> Self {
assert!(size_ms > 0, "window size must be > 0");
Self::new(WindowKind::Tumbling { size_ms }, allowed_lateness_ms, group_by, aggs)
}
pub fn sliding(
size_ms: i64,
slide_ms: i64,
allowed_lateness_ms: i64,
group_by: Vec<String>,
aggs: Vec<Agg>,
) -> Self {
assert!(
size_ms > 0 && slide_ms > 0 && slide_ms <= size_ms,
"sliding needs 0 < slide ≤ size"
);
Self::new(
WindowKind::Sliding { size_ms, slide_ms },
allowed_lateness_ms,
group_by,
aggs,
)
}
fn new(kind: WindowKind, allowed_lateness_ms: i64, group_by: Vec<String>, aggs: Vec<Agg>) -> Self {
let with_sets = acc::needs_sets(&aggs);
let with_welford = acc::needs_welford(&aggs);
Self {
kind,
group_by,
aggs,
key_ids: HashMap::new(),
key_data: Vec::new(),
state: HashMap::new(),
with_sets,
with_welford,
wm: Watermark::new(allowed_lateness_ms),
fired_through: i64::MIN,
late_dropped: 0,
}
}
pub fn with_idle_timeout(mut self, idle_timeout_ms: i64) -> Self {
self.wm = self.wm.with_idle_timeout(idle_timeout_ms);
self
}
pub fn watermark(&self, now_ms: i64) -> i64 {
self.wm.current(now_ms)
}
pub fn late_dropped(&self) -> u64 {
self.late_dropped
}
pub fn expect_partitions(&mut self, partitions: &[i32], now_ms: i64) {
self.wm.expect(partitions, now_ms);
}
pub fn push(&mut self, partition: i32, event_ms: i64, row: &[(String, Value)]) -> bool {
self.wm.observe(partition, event_ms);
let size = self.kind.size();
let composite = acc::composite_of(row, &self.group_by);
let id = match self.key_ids.get(&composite) {
Some(&id) => id,
None => {
let id = self.key_data.len() as u32;
self.key_data
.push((composite.clone(), acc::vals_of(row, &self.group_by)));
self.key_ids.insert(composite, id);
id
}
};
let (n_aggs, with_sets, with_welford) = (self.aggs.len(), self.with_sets, self.with_welford);
let mut added = false;
for start in self.kind.starts_for(event_ms) {
if start + size <= self.fired_through {
continue; }
added = true;
self.state
.entry((start, id))
.or_insert_with(|| Acc::new(n_aggs, with_sets, with_welford))
.add(&self.aggs, row);
}
if !added {
self.late_dropped += 1;
}
added
}
pub fn advance(&mut self, now_ms: i64) -> Vec<WindowRow> {
self.wm.tick(now_ms);
let wm = self.wm.current(now_ms);
self.fire_through(wm)
}
pub fn advance_to(&mut self, watermark_ms: i64) -> Vec<WindowRow> {
let wm = self.wm.after_lateness(watermark_ms);
self.fire_through(wm)
}
fn fire_through(&mut self, wm: i64) -> Vec<WindowRow> {
if wm == i64::MIN {
return Vec::new();
}
self.fire(|start, size| start + size <= wm, Some(wm))
}
pub fn flush(&mut self) -> Vec<WindowRow> {
self.fired_through = i64::MAX;
self.fire(|_, _| true, None)
}
fn fire(&mut self, ready: impl Fn(i64, i64) -> bool, set_fired_through: Option<i64>) -> Vec<WindowRow> {
let size = self.kind.size();
let keys: Vec<(i64, u32)> = self
.state
.keys()
.filter(|(start, _)| ready(*start, size))
.copied()
.collect();
let mut fired: Vec<((i64, u32), Acc)> = Vec::with_capacity(keys.len());
for k in keys {
if let Some(acc) = self.state.remove(&k) {
fired.push((k, acc));
}
}
if let Some(wm) = set_fired_through {
self.fired_through = self.fired_through.max(wm);
}
fired.sort_by(|a, b| {
a.0 .0
.cmp(&b.0 .0)
.then_with(|| self.key_data[a.0 .1 as usize].0.cmp(&self.key_data[b.0 .1 as usize].0))
});
fired
.into_iter()
.map(|((start, id), acc)| WindowRow {
window_start: start,
window_end: start + size,
aggs: acc.finalize_aggs(&self.aggs),
key: self.key_data[id as usize].1.clone(),
})
.collect()
}
fn cfg_fingerprint(&self) -> String {
format!(
"kind={:?};group_by={:?};aggs={:?};{}",
self.kind,
self.group_by,
self.aggs,
self.wm.cfg_fingerprint()
)
}
pub fn snapshot(&self) -> Vec<u8> {
use serde_json::json;
let mut snap_idx: HashMap<u32, usize> = HashMap::new();
let mut keys: Vec<serde_json::Value> = Vec::new();
let mut state: Vec<serde_json::Value> = Vec::with_capacity(self.state.len());
for ((start, id), acc) in &self.state {
let idx = *snap_idx.entry(*id).or_insert_with(|| {
let (composite, vals) = &self.key_data[*id as usize];
keys.push(json!([
composite,
vals.iter().map(fv_value::value_to_json).collect::<Vec<_>>()
]));
keys.len() - 1
});
state.push(json!([start, idx, snapshot::acc_to_json_v2(acc)]));
}
snapshot::envelope(
"window",
&self.cfg_fingerprint(),
json!({
"format": 2,
"keys": keys,
"state": state,
"wm": self.wm.state_to_json(),
"fired_through": self.fired_through,
"late_dropped": self.late_dropped,
}),
)
}
fn intern(&mut self, composite: String, vals: Vec<Value>) -> u32 {
match self.key_ids.get(&composite) {
Some(&id) => id,
None => {
let id = self.key_data.len() as u32;
self.key_data.push((composite.clone(), vals));
self.key_ids.insert(composite, id);
id
}
}
}
pub fn load_snapshot(&mut self, bytes: &[u8]) -> Result<(), String> {
let s = snapshot::open_envelope("window", &self.cfg_fingerprint(), bytes)?;
let mut map = HashMap::new();
if s["format"].as_i64() == Some(2) {
let key_entries = s["keys"].as_array().ok_or("window snapshot v2: keys not an array")?;
let mut ids = Vec::with_capacity(key_entries.len());
for e in key_entries {
let t = e
.as_array()
.filter(|a| a.len() == 2)
.ok_or("window snapshot v2: bad key entry")?;
let composite = t[0].as_str().ok_or("window snapshot v2: bad composite")?.to_string();
let vals: Vec<Value> = t[1]
.as_array()
.ok_or("window snapshot v2: bad key vals")?
.iter()
.map(fv_value::json_to_value)
.collect();
ids.push(self.intern(composite, vals));
}
for e in s["state"].as_array().ok_or("window snapshot v2: state not an array")? {
let a = e
.as_array()
.filter(|a| a.len() == 3)
.ok_or("window snapshot v2: bad state entry")?;
let start = a[0].as_i64().ok_or("window snapshot v2: bad window start")?;
let idx = a[1].as_u64().ok_or("window snapshot v2: bad key index")? as usize;
let id = *ids.get(idx).ok_or("window snapshot v2: key index out of range")?;
map.insert((start, id), snapshot::acc_from_json_v2(&a[2])?);
}
} else {
for e in s["state"].as_array().ok_or("window snapshot: state not an array")? {
let a = e
.as_array()
.filter(|a| a.len() == 3)
.ok_or("window snapshot: bad state entry")?;
let start = a[0].as_i64().ok_or("window snapshot: bad window start")?;
let gk: Vec<String> = a[1]
.as_array()
.ok_or("window snapshot: bad group key")?
.iter()
.map(|s| {
s.as_str()
.map(String::from)
.ok_or_else(|| "window snapshot: bad group key part".to_string())
})
.collect::<Result<_, _>>()?;
let (vals, acc) = snapshot::acc_from_json_v1(&a[2])?;
let id = self.intern(gk.join("\u{1}"), vals);
map.insert((start, id), acc);
}
}
self.state = map;
self.wm.load_state(&s["wm"])?;
self.fired_through = s["fired_through"]
.as_i64()
.ok_or("window snapshot: missing fired_through")?;
self.late_dropped = s["late_dropped"]
.as_u64()
.ok_or("window snapshot: missing late_dropped")?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn row(pairs: &[(&str, Value)]) -> Vec<(String, Value)> {
pairs.iter().map(|(k, v)| (k.to_string(), v.clone())).collect()
}
fn num(w: &WindowRow, alias: &str) -> f64 {
match &w.aggs.iter().find(|(a, _)| a == alias).unwrap().1 {
Value::Num(n) => *n,
other => panic!("expected num for {alias}, got {other:?}"),
}
}
#[test]
fn v1_snapshot_loads_and_fires_identically_after_the_t2_upgrade() {
let mk = || {
WindowAgg::tumbling(
1000,
0,
vec!["k".into()],
vec![Agg::count("n"), Agg::of(AggOp::Sum, "amount", "total")],
)
};
let op = mk();
let v1 = serde_json::json!({
"v": 1,
"kind": "window",
"cfg": op.cfg_fingerprint(),
"state": {
"state": [
[0i64, ["sa"], {"key": ["a"], "count": 2, "slots": [null, 7.0]}],
[0i64, ["sb"], {"key": ["b"], "count": 1, "slots": [null, 3.0]}],
[1000i64, ["sa"], {"key": ["a"], "count": 5, "slots": [null, 50.0]}],
],
"wm": {"part_max": {"0": 1500}, "last_activity": {"0": 0}, "last_seen_max": {"0": 1500}},
"fired_through": i64::MIN,
"late_dropped": 0,
},
});
let mut restored = mk();
restored
.load_snapshot(&serde_json::to_vec(&v1).unwrap())
.expect("a v1 snapshot must load");
let fired = restored.flush();
assert_eq!(fired.len(), 3);
let w0a = fired
.iter()
.find(|w| w.window_start == 0 && w.key == vec![Value::Str("a".into())])
.unwrap();
assert_eq!(num(w0a, "n"), 2.0);
assert_eq!(num(w0a, "total"), 7.0);
let w1a = fired
.iter()
.find(|w| w.window_start == 1000 && w.key == vec![Value::Str("a".into())])
.unwrap();
assert_eq!(num(w1a, "n"), 5.0);
assert_eq!(num(w1a, "total"), 50.0);
}
#[test]
fn v2_snapshot_stores_each_key_once_across_windows() {
let mut op = WindowAgg::tumbling(1000, 1_000_000, vec!["k".into()], vec![Agg::count("n")]);
op.expect_partitions(&[0], 0);
for w in 0..100i64 {
op.push(
0,
w * 1000,
&row(&[
("k", Value::Str("the-only-key-string".into())),
("amount", Value::Num(1.0)),
]),
);
}
let snap = op.snapshot();
let text = String::from_utf8(snap).unwrap();
assert_eq!(
text.matches("the-only-key-string").count(),
2,
"once in the composite, once in the vals — never per window"
);
assert!(!text.contains("\"sets\""), "v2 tuples carry null, not a sets field");
}
#[test]
fn tumbling_window_assignment_is_by_event_time() {
let k = WindowKind::Tumbling { size_ms: 1000 };
assert_eq!(k.starts_for(0), vec![0]);
assert_eq!(k.starts_for(999), vec![0]);
assert_eq!(k.starts_for(1000), vec![1000]);
assert_eq!(k.starts_for(2500), vec![2000]);
}
#[test]
fn sliding_window_assignment_covers_overlapping_windows() {
let k = WindowKind::Sliding {
size_ms: 1000,
slide_ms: 500,
};
assert_eq!(k.starts_for(750), vec![500, 0]);
assert_eq!(k.starts_for(1200), vec![1000, 500]);
assert_eq!(k.starts_for(0), vec![0, -500]);
}
#[test]
fn aggregates_count_sum_avg_min_max_within_a_tumbling_window() {
let aggs = vec![
Agg::count("n"),
Agg::of(AggOp::Sum, "amount", "total"),
Agg::of(AggOp::Avg, "amount", "mean"),
Agg::of(AggOp::Min, "amount", "lo"),
Agg::of(AggOp::Max, "amount", "hi"),
];
let mut w = WindowAgg::tumbling(1000, 0, vec![], aggs);
for (t, a) in [(10, 2.0), (20, 4.0), (30, 6.0)] {
w.push(0, t, &row(&[("amount", Value::Num(a))]));
}
w.push(0, 1500, &row(&[("amount", Value::Num(99.0))])); let out = w.advance(0);
assert_eq!(out.len(), 1);
let r = &out[0];
assert_eq!((r.window_start, r.window_end), (0, 1000));
assert_eq!(num(r, "n"), 3.0);
assert_eq!(num(r, "total"), 12.0);
assert_eq!(num(r, "mean"), 4.0);
assert_eq!(num(r, "lo"), 2.0);
assert_eq!(num(r, "hi"), 6.0);
}
#[test]
fn sliding_event_lands_in_every_overlapping_window() {
let mut w = WindowAgg::sliding(1000, 500, 0, vec![], vec![Agg::count("n")]);
for t in [100, 300, 700] {
w.push(0, t, &row(&[]));
}
w.push(0, 2200, &row(&[])); let mut out = w.advance(0);
out.sort_by_key(|r| r.window_start);
let w0 = out.iter().find(|r| r.window_start == 0).unwrap();
let w500 = out.iter().find(|r| r.window_start == 500).unwrap();
assert_eq!(num(w0, "n"), 3.0, "[0,1000) saw 100,300,700");
assert_eq!(num(w500, "n"), 1.0, "[500,1500) saw only 700");
}
#[test]
fn separate_windows_and_keys_do_not_mix() {
let mut w = WindowAgg::tumbling(1000, 0, vec!["k".into()], vec![Agg::count("n")]);
w.push(0, 100, &row(&[("k", Value::Str("a".into()))]));
w.push(0, 200, &row(&[("k", Value::Str("b".into()))]));
w.push(0, 300, &row(&[("k", Value::Str("a".into()))]));
w.push(0, 1100, &row(&[("k", Value::Str("a".into()))]));
let out = w.advance(0);
assert_eq!(out.len(), 2);
let a = out.iter().find(|r| r.key == vec![Value::Str("a".into())]).unwrap();
let b = out.iter().find(|r| r.key == vec![Value::Str("b".into())]).unwrap();
assert_eq!(num(a, "n"), 2.0);
assert_eq!(num(b, "n"), 1.0);
}
#[test]
fn allowed_lateness_holds_a_window_open() {
let mut w = WindowAgg::tumbling(1000, 500, vec![], vec![Agg::count("n")]);
w.push(0, 100, &row(&[]));
w.push(0, 1200, &row(&[])); assert!(w.advance(0).is_empty());
w.push(0, 1600, &row(&[])); assert_eq!(w.advance(0).len(), 1);
}
#[test]
fn genuinely_late_events_are_dropped_and_counted() {
let mut w = WindowAgg::tumbling(1000, 0, vec![], vec![Agg::count("n")]);
w.push(0, 100, &row(&[]));
w.push(0, 1500, &row(&[]));
assert_eq!(w.advance(0).len(), 1);
assert!(!w.push(0, 200, &row(&[])), "late event rejected");
assert_eq!(w.late_dropped(), 1);
assert!(w.push(0, 1600, &row(&[])));
}
#[test]
fn sliding_streaming_scenario_counts_every_window_exactly() {
let mut w = WindowAgg::sliding(1000, 500, 0, vec![], vec![Agg::count("n")]);
let mut fired: std::collections::HashMap<i64, f64> = std::collections::HashMap::new();
let mut t = 0i64;
for _batch in 0..16 {
for _ in 0..250 {
w.push(0, t, &[]);
t += 1;
}
for r in w.advance(0) {
fired.insert(r.window_start, num(&r, "n"));
}
}
for r in w.flush() {
fired.insert(r.window_start, num(&r, "n"));
}
for s in [1000_i64, 1500, 2000, 2500, 3000] {
assert_eq!(
fired.get(&s),
Some(&1000.0),
"window [{s},{}) should have 1000 events",
s + 1000
);
}
}
#[test]
fn multi_partition_watermark_is_the_min_so_a_lagging_partition_holds_firing() {
let mut w = WindowAgg::tumbling(1000, 0, vec![], vec![Agg::count("n")]);
w.push(0, 100, &row(&[]));
w.push(0, 5000, &row(&[])); w.push(1, 100, &row(&[])); assert!(w.advance(0).is_empty(), "the lagging partition holds the watermark");
w.push(1, 1500, &row(&[])); let out = w.advance(0); assert_eq!(out.len(), 1);
assert_eq!(
num(&out[0], "n"),
2.0,
"both partitions' ts-100 events counted, none dropped"
);
}
#[test]
fn expected_but_unseen_partition_holds_the_watermark() {
let mut w = WindowAgg::tumbling(1000, 0, vec![], vec![Agg::count("n")]);
w.expect_partitions(&[0, 1], 0);
w.push(0, 100, &row(&[])); w.push(0, 5000, &row(&[])); assert!(
w.advance(0).is_empty(),
"unseen assigned partition 1 pins the watermark → nothing fires"
);
w.push(1, 5000, &row(&[])); let out = w.advance(0);
assert_eq!(out.len(), 1, "now both seen → window [0,1000) fires");
assert_eq!(num(&out[0], "n"), 1.0, "with its one event, not dropped");
}
#[test]
fn flush_fires_all_remaining_windows() {
let mut w = WindowAgg::tumbling(1000, 10_000, vec![], vec![Agg::count("n")]);
w.push(0, 100, &row(&[]));
w.push(0, 5000, &row(&[]));
assert!(w.advance(0).is_empty());
assert_eq!(w.flush().len(), 2);
}
#[test]
fn snapshot_restore_fires_identically_to_the_uninterrupted_run() {
let mk = || {
let mut w = WindowAgg::tumbling(
1000,
100,
vec!["k".into()],
vec![Agg::count("n"), Agg::of(AggOp::Sum, "amount", "total")],
);
w.expect_partitions(&[0, 1], 0);
w
};
let ev = |i: i64| {
(
(i % 2) as i32,
i * 7, row(&[
("k", Value::Str(format!("g{}", i % 5))),
("amount", Value::Num(i as f64)),
]),
)
};
let head: Vec<_> = (0..500).map(ev).collect();
let tail: Vec<_> = (500..1000).map(ev).collect();
let mut a = mk();
let mut fired_a = Vec::new();
for (p, t, r) in &head {
a.push(*p, *t, r);
}
fired_a.extend(a.advance(0));
for (p, t, r) in &tail {
a.push(*p, *t, r);
}
assert!(
!a.push(0, 0, &row(&[("k", Value::Str("g0".into()))])),
"late event dropped"
);
fired_a.extend(a.advance(0));
fired_a.extend(a.flush());
let mut b1 = mk();
let mut fired_b = Vec::new();
for (p, t, r) in &head {
b1.push(*p, *t, r);
}
fired_b.extend(b1.advance(0));
let snap = b1.snapshot();
drop(b1); let mut b2 = mk();
b2.load_snapshot(&snap).expect("restore");
for (p, t, r) in &tail {
b2.push(*p, *t, r);
}
assert!(
!b2.push(0, 0, &row(&[("k", Value::Str("g0".into()))])),
"late event still dropped after restore"
);
fired_b.extend(b2.advance(0));
fired_b.extend(b2.flush());
assert_eq!(fired_a, fired_b, "restored run must fire byte-identically");
assert_eq!(a.late_dropped(), b2.late_dropped(), "late counts preserved");
}
#[test]
fn snapshot_refuses_config_and_kind_mismatch() {
let mut w = WindowAgg::tumbling(1000, 0, vec![], vec![Agg::count("n")]);
w.push(0, 100, &row(&[]));
let snap = w.snapshot();
let mut other = WindowAgg::tumbling(2000, 0, vec![], vec![Agg::count("n")]);
assert!(other.load_snapshot(&snap).unwrap_err().contains("config changed"));
let mut s = SessionAgg::new(1000, 0, vec![], vec![Agg::count("n")]);
let mut w2 = WindowAgg::tumbling(1000, 0, vec![], vec![Agg::count("n")]);
assert!(w2
.load_snapshot(&WindowOperator::snapshot(&mut s).unwrap())
.unwrap_err()
.contains("kind"));
}
#[test]
fn snapshot_preserves_bounded_idleness_timers() {
let mk = || {
let mut w = WindowAgg::tumbling(1000, 0, vec![], vec![Agg::count("n")]).with_idle_timeout(1000);
w.expect_partitions(&[0, 1], 0);
w.push(0, 100, &row(&[]));
w.push(1, 200, &row(&[]));
w.advance(0); w.push(0, 1500, &row(&[]));
w
};
let mut orig = mk();
let mut restored = WindowAgg::tumbling(1000, 0, vec![], vec![Agg::count("n")]).with_idle_timeout(1000);
restored.load_snapshot(&mk().snapshot()).expect("restore");
let a = orig.advance(5000); let b = restored.advance(5000);
assert_eq!(a, b, "idle-release behavior identical after restore");
assert!(a.iter().any(|r| r.window_start == 0));
}
#[test]
fn bounded_idleness_releases_windows_held_by_a_silent_partition() {
let mk = |idle_ms: i64| {
let mut w = WindowAgg::tumbling(1000, 0, vec![], vec![Agg::count("n")]);
if idle_ms > 0 {
w = w.with_idle_timeout(idle_ms);
}
w.expect_partitions(&[0, 1], 0);
w.push(0, 100, &row(&[]));
w.push(1, 200, &row(&[])); w.advance(0); w.push(0, 1500, &row(&[])); w
};
let mut no_idle = mk(0);
assert!(
no_idle.advance(5000).is_empty(),
"no idle timeout → the silent partition holds forever"
);
let mut idle = mk(1000);
let out = idle.advance(5000); assert!(
out.iter().any(|r| r.window_start == 0),
"idle partition released → window [0,1000) fires"
);
}
}