Skip to main content

fv_streams_ops/
types.rs

1//! The types every windowed operator shares: what to aggregate, how to window, when to fire, and
2//! the rows a window emits.
3
4use fv_value::Value;
5
6/// The aggregation op applied to a column within each window+key.
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub enum AggOp {
9    Count,
10    Sum,
11    Avg,
12    Min,
13    Max,
14    /// Exact distinct count of a column's non-null values per (window, group) — a per-accumulator
15    /// value-set (unlocks Nexmark q15-style reports). Memory bound: distinct values per group
16    /// per open window — document the cardinality when using it.
17    CountDistinct,
18    // FUNC-2: incremental aggregates, all O(1)/event and mergeable (session bridging).
19    /// Population / sample variance + standard deviation over a numeric column, via Welford's
20    /// online algorithm (numerically stable, parallel-mergeable). Samp needs ≥2 values (else null).
21    VarPop,
22    VarSamp,
23    StddevPop,
24    StddevSamp,
25    /// Boolean AND / OR fold over a boolean column (`true` until a `false` for And; mirror for Or).
26    BoolAnd,
27    BoolOr,
28    /// Bitwise AND / OR / XOR fold over an integer column (values must be whole numbers).
29    BitAnd,
30    BitOr,
31    BitXor,
32}
33
34impl AggOp {
35    /// The variance/stddev family — the ops that need Welford side-state.
36    pub(crate) fn is_variance(self) -> bool {
37        matches!(
38            self,
39            AggOp::VarPop | AggOp::VarSamp | AggOp::StddevPop | AggOp::StddevSamp
40        )
41    }
42}
43
44/// One named aggregation: `op` over `column` (ignored for `Count`), output under `alias`.
45#[derive(Debug, Clone)]
46pub struct Agg {
47    pub op: AggOp,
48    pub column: String,
49    pub alias: String,
50}
51
52impl Agg {
53    pub fn count(alias: impl Into<String>) -> Self {
54        Self {
55            op: AggOp::Count,
56            column: String::new(),
57            alias: alias.into(),
58        }
59    }
60    pub fn of(op: AggOp, column: impl Into<String>, alias: impl Into<String>) -> Self {
61        Self {
62            op,
63            column: column.into(),
64            alias: alias.into(),
65        }
66    }
67}
68
69/// How an event time maps to windows.
70#[derive(Debug, Clone, Copy, PartialEq, Eq)]
71pub enum WindowKind {
72    /// Fixed, non-overlapping `[start, start+size)`.
73    Tumbling { size_ms: i64 },
74    /// Overlapping: windows of width `size` stepping by `slide` (`slide ≤ size`; `slide == size` is
75    /// tumbling).
76    Sliding { size_ms: i64, slide_ms: i64 },
77}
78
79impl WindowKind {
80    pub(crate) fn size(&self) -> i64 {
81        match self {
82            WindowKind::Tumbling { size_ms } | WindowKind::Sliding { size_ms, .. } => *size_ms,
83        }
84    }
85
86    /// The starts of every window an event time belongs to (one for tumbling; `ceil(size/slide)` for
87    /// sliding — the slide-aligned starts `s` with `s ≤ t < s+size`).
88    pub fn starts_for(&self, t: i64) -> Vec<i64> {
89        match self {
90            WindowKind::Tumbling { size_ms } => vec![t.div_euclid(*size_ms) * size_ms],
91            WindowKind::Sliding { size_ms, slide_ms } => {
92                let mut starts = Vec::new();
93                let mut s = t.div_euclid(*slide_ms) * slide_ms; // latest slide-aligned start ≤ t
94                while s > t - size_ms {
95                    starts.push(s);
96                    s -= slide_ms;
97                }
98                starts
99            }
100        }
101    }
102}
103
104/// When an **updating** windowed aggregate EARLY-FIRES: emits the current (partial) value of its
105/// still-open windows as upserts, before the watermark closes them. An early emission carries the
106/// same `window_start`/`window_end`/group as the eventual closing fire, so its emission key
107/// (`w|start|end|group`) is identical and downstream last-write-wins converges to the exact closing
108/// value. The contract is therefore **eventual correctness between fires, exact at close** — the
109/// closing fire is always the last write for a `(window, group)`.
110///
111/// `OnWatermark` (the default) never early-fires: the operator is byte-identical to a plain windowed
112/// aggregate, so an existing pipeline (and its checkpoints) are untouched.
113#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
114pub enum Trigger {
115    /// No early firing — windows emit once, when the watermark closes them (today's behaviour).
116    #[default]
117    OnWatermark,
118    /// Early-fire after every `n` ingested rows since the last fire. DATA-DRIVEN: the final value is
119    /// exact and the emission sequence replays identically whenever batch boundaries are stable (the
120    /// partition's row order is its log order). The cheap, deterministic default for early firing.
121    EveryRows(u64),
122    /// Early-fire once `ms` of PROCESSING time have passed since the last fire. Wall-clock, so the
123    /// *intermediate* emissions are not byte-replayable (only the closing value is), exactly like an
124    /// ingest-time window. Fires on a processing-time advance/tick, so it is best-effort under an
125    /// in-band (watermark-driven) task.
126    EveryMs(i64),
127}
128
129impl Trigger {
130    /// A stable fingerprint fragment for the operator config. **Empty for [`OnWatermark`]** so an
131    /// existing (on-watermark) checkpoint's fingerprint is byte-identical — a restore never regresses.
132    ///
133    /// [`OnWatermark`]: Trigger::OnWatermark
134    pub(crate) fn fingerprint(&self) -> String {
135        match self {
136            Trigger::OnWatermark => String::new(),
137            Trigger::EveryRows(n) => format!(";trigger=rows({n})"),
138            Trigger::EveryMs(ms) => format!(";trigger=ms({ms})"),
139        }
140    }
141    /// Whether this trigger ever emits before a window closes (drives the zero-cost fast path: when
142    /// `false`, no dirty-key tracking and no early-fire pass run at all).
143    pub(crate) fn early_fires(&self) -> bool {
144        !matches!(self, Trigger::OnWatermark)
145    }
146}
147
148/// One fired window's aggregated output for one group key.
149#[derive(Debug, Clone, PartialEq)]
150pub struct WindowRow {
151    /// Window bounds (epoch-ms), half-open `[start, end)`.
152    pub window_start: i64,
153    pub window_end: i64,
154    /// The group-by column values, in `group_by` order.
155    pub key: Vec<Value>,
156    /// The aggregated outputs: `(alias, value)`, in `aggs` order.
157    pub aggs: Vec<(String, Value)>,
158}