fv_streams_ops/types.rs
1//! The types every windowed operator shares: what to aggregate, how to window, when to fire, and
2//! the rows a window emits.
3
4use fv_value::Value;
5
6/// The aggregation op applied to a column within each window+key.
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub enum AggOp {
9 Count,
10 Sum,
11 Avg,
12 Min,
13 Max,
14 /// Exact distinct count of a column's non-null values per (window, group) — a per-accumulator
15 /// value-set (unlocks Nexmark q15-style reports). Memory bound: distinct values per group
16 /// per open window — document the cardinality when using it.
17 CountDistinct,
18 // FUNC-2: incremental aggregates, all O(1)/event and mergeable (session bridging).
19 /// Population / sample variance + standard deviation over a numeric column, via Welford's
20 /// online algorithm (numerically stable, parallel-mergeable). Samp needs ≥2 values (else null).
21 VarPop,
22 VarSamp,
23 StddevPop,
24 StddevSamp,
25 /// Boolean AND / OR fold over a boolean column (`true` until a `false` for And; mirror for Or).
26 BoolAnd,
27 BoolOr,
28 /// Bitwise AND / OR / XOR fold over an integer column (values must be whole numbers).
29 BitAnd,
30 BitOr,
31 BitXor,
32}
33
34impl AggOp {
35 /// The variance/stddev family — the ops that need Welford side-state.
36 pub(crate) fn is_variance(self) -> bool {
37 matches!(
38 self,
39 AggOp::VarPop | AggOp::VarSamp | AggOp::StddevPop | AggOp::StddevSamp
40 )
41 }
42}
43
44/// One named aggregation: `op` over `column` (ignored for `Count`), output under `alias`.
45#[derive(Debug, Clone)]
46pub struct Agg {
47 pub op: AggOp,
48 pub column: String,
49 pub alias: String,
50}
51
52impl Agg {
53 pub fn count(alias: impl Into<String>) -> Self {
54 Self {
55 op: AggOp::Count,
56 column: String::new(),
57 alias: alias.into(),
58 }
59 }
60 pub fn of(op: AggOp, column: impl Into<String>, alias: impl Into<String>) -> Self {
61 Self {
62 op,
63 column: column.into(),
64 alias: alias.into(),
65 }
66 }
67}
68
69/// How an event time maps to windows.
70#[derive(Debug, Clone, Copy, PartialEq, Eq)]
71pub enum WindowKind {
72 /// Fixed, non-overlapping `[start, start+size)`.
73 Tumbling { size_ms: i64 },
74 /// Overlapping: windows of width `size` stepping by `slide` (`slide ≤ size`; `slide == size` is
75 /// tumbling).
76 Sliding { size_ms: i64, slide_ms: i64 },
77}
78
79impl WindowKind {
80 pub(crate) fn size(&self) -> i64 {
81 match self {
82 WindowKind::Tumbling { size_ms } | WindowKind::Sliding { size_ms, .. } => *size_ms,
83 }
84 }
85
86 /// The starts of every window an event time belongs to (one for tumbling; `ceil(size/slide)` for
87 /// sliding — the slide-aligned starts `s` with `s ≤ t < s+size`).
88 pub fn starts_for(&self, t: i64) -> Vec<i64> {
89 match self {
90 WindowKind::Tumbling { size_ms } => vec![t.div_euclid(*size_ms) * size_ms],
91 WindowKind::Sliding { size_ms, slide_ms } => {
92 let mut starts = Vec::new();
93 let mut s = t.div_euclid(*slide_ms) * slide_ms; // latest slide-aligned start ≤ t
94 while s > t - size_ms {
95 starts.push(s);
96 s -= slide_ms;
97 }
98 starts
99 }
100 }
101 }
102}
103
104/// When an **updating** windowed aggregate EARLY-FIRES: emits the current (partial) value of its
105/// still-open windows as upserts, before the watermark closes them. An early emission carries the
106/// same `window_start`/`window_end`/group as the eventual closing fire, so its emission key
107/// (`w|start|end|group`) is identical and downstream last-write-wins converges to the exact closing
108/// value. The contract is therefore **eventual correctness between fires, exact at close** — the
109/// closing fire is always the last write for a `(window, group)`.
110///
111/// `OnWatermark` (the default) never early-fires: the operator is byte-identical to a plain windowed
112/// aggregate, so an existing pipeline (and its checkpoints) are untouched.
113#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
114pub enum Trigger {
115 /// No early firing — windows emit once, when the watermark closes them (today's behaviour).
116 #[default]
117 OnWatermark,
118 /// Early-fire after every `n` ingested rows since the last fire. DATA-DRIVEN: the final value is
119 /// exact and the emission sequence replays identically whenever batch boundaries are stable (the
120 /// partition's row order is its log order). The cheap, deterministic default for early firing.
121 EveryRows(u64),
122 /// Early-fire once `ms` of PROCESSING time have passed since the last fire. Wall-clock, so the
123 /// *intermediate* emissions are not byte-replayable (only the closing value is), exactly like an
124 /// ingest-time window. Fires on a processing-time advance/tick, so it is best-effort under an
125 /// in-band (watermark-driven) task.
126 EveryMs(i64),
127}
128
129impl Trigger {
130 /// A stable fingerprint fragment for the operator config. **Empty for [`OnWatermark`]** so an
131 /// existing (on-watermark) checkpoint's fingerprint is byte-identical — a restore never regresses.
132 ///
133 /// [`OnWatermark`]: Trigger::OnWatermark
134 pub(crate) fn fingerprint(&self) -> String {
135 match self {
136 Trigger::OnWatermark => String::new(),
137 Trigger::EveryRows(n) => format!(";trigger=rows({n})"),
138 Trigger::EveryMs(ms) => format!(";trigger=ms({ms})"),
139 }
140 }
141 /// Whether this trigger ever emits before a window closes (drives the zero-cost fast path: when
142 /// `false`, no dirty-key tracking and no early-fire pass run at all).
143 pub(crate) fn early_fires(&self) -> bool {
144 !matches!(self, Trigger::OnWatermark)
145 }
146}
147
148/// One fired window's aggregated output for one group key.
149#[derive(Debug, Clone, PartialEq)]
150pub struct WindowRow {
151 /// Window bounds (epoch-ms), half-open `[start, end)`.
152 pub window_start: i64,
153 pub window_end: i64,
154 /// The group-by column values, in `group_by` order.
155 pub key: Vec<Value>,
156 /// The aggregated outputs: `(alias, value)`, in `aggs` order.
157 pub aggs: Vec<(String, Value)>,
158}