use std::collections::HashMap;
use fv_value::Value;
use crate::acc::Acc;
use crate::watermark::Watermark;
use crate::{Agg, WindowOperator, WindowRow};
struct Session {
first: i64,
last: i64,
acc: Acc,
}
struct Bucket {
vals: Vec<Value>,
sessions: Vec<Session>,
}
pub struct SessionAgg {
gap_ms: i64,
group_by: Vec<String>,
aggs: Vec<Agg>,
sessions: HashMap<String, Bucket>,
with_sets: bool,
with_welford: bool,
wm: Watermark,
fired_through: i64,
late_dropped: u64,
}
impl SessionAgg {
pub fn new(gap_ms: i64, allowed_lateness_ms: i64, group_by: Vec<String>, aggs: Vec<Agg>) -> Self {
assert!(gap_ms > 0, "session gap must be > 0");
let with_sets = crate::acc::needs_sets(&aggs);
let with_welford = crate::acc::needs_welford(&aggs);
Self {
gap_ms,
group_by,
aggs,
sessions: HashMap::new(),
with_sets,
with_welford,
wm: Watermark::new(allowed_lateness_ms),
fired_through: i64::MIN,
late_dropped: 0,
}
}
pub fn with_idle_timeout(mut self, idle_timeout_ms: i64) -> Self {
self.wm = self.wm.with_idle_timeout(idle_timeout_ms);
self
}
pub fn watermark(&self, now_ms: i64) -> i64 {
self.wm.current(now_ms)
}
pub fn late_dropped(&self) -> u64 {
self.late_dropped
}
fn fire_through(&mut self, wm: i64) -> Vec<WindowRow> {
if wm == i64::MIN {
return Vec::new();
}
let out = self.fire(|end| end <= wm);
self.fired_through = self.fired_through.max(wm);
out
}
fn fire(&mut self, closed: impl Fn(i64) -> bool) -> Vec<WindowRow> {
let gap = self.gap_ms;
let aggs = &self.aggs;
let mut out = Vec::new();
for bucket in self.sessions.values_mut() {
let mut i = 0;
while i < bucket.sessions.len() {
if closed(bucket.sessions[i].last + gap) {
let s = bucket.sessions.remove(i);
out.push(WindowRow {
window_start: s.first,
window_end: s.last + gap,
aggs: s.acc.finalize_aggs(aggs),
key: bucket.vals.clone(),
});
} else {
i += 1;
}
}
}
self.sessions.retain(|_, b| !b.sessions.is_empty());
out.sort_by_key(|w| w.window_start);
out
}
fn cfg_fingerprint(&self) -> String {
format!(
"gap={};group_by={:?};aggs={:?};{}",
self.gap_ms,
self.group_by,
self.aggs,
self.wm.cfg_fingerprint()
)
}
pub fn snapshot(&self) -> Vec<u8> {
use serde_json::json;
let sessions: Vec<serde_json::Value> = self
.sessions
.iter()
.map(|(composite, bucket)| {
let b: Vec<serde_json::Value> = bucket
.sessions
.iter()
.map(|s| json!([s.first, s.last, crate::snapshot::acc_to_json_v2(&s.acc)]))
.collect();
json!([
composite,
bucket.vals.iter().map(fv_value::value_to_json).collect::<Vec<_>>(),
b
])
})
.collect();
crate::snapshot::envelope(
"session",
&self.cfg_fingerprint(),
json!({
"format": 2,
"sessions": sessions,
"wm": self.wm.state_to_json(),
"fired_through": self.fired_through,
"late_dropped": self.late_dropped,
}),
)
}
pub fn load_snapshot(&mut self, bytes: &[u8]) -> Result<(), String> {
let s = crate::snapshot::open_envelope("session", &self.cfg_fingerprint(), bytes)?;
let v2 = s["format"].as_i64() == Some(2);
let mut map = HashMap::new();
for e in s["sessions"]
.as_array()
.ok_or("session snapshot: sessions not an array")?
{
if v2 {
let a = e
.as_array()
.filter(|a| a.len() == 3)
.ok_or("session snapshot v2: bad entry")?;
let composite = a[0].as_str().ok_or("session snapshot v2: bad composite")?.to_string();
let vals: Vec<Value> = a[1]
.as_array()
.ok_or("session snapshot v2: bad key vals")?
.iter()
.map(fv_value::json_to_value)
.collect();
let sessions: Vec<Session> = a[2]
.as_array()
.ok_or("session snapshot v2: bad bucket")?
.iter()
.map(|se| {
let t = se
.as_array()
.filter(|t| t.len() == 3)
.ok_or("session snapshot v2: bad session")?;
Ok(Session {
first: t[0].as_i64().ok_or("session snapshot v2: bad first")?,
last: t[1].as_i64().ok_or("session snapshot v2: bad last")?,
acc: crate::snapshot::acc_from_json_v2(&t[2])?,
})
})
.collect::<Result<_, String>>()?;
map.insert(composite, Bucket { vals, sessions });
} else {
let a = e
.as_array()
.filter(|a| a.len() == 2)
.ok_or("session snapshot: bad entry")?;
let gk: Vec<String> = a[0]
.as_array()
.ok_or("session snapshot: bad group key")?
.iter()
.map(|s| {
s.as_str()
.map(String::from)
.ok_or_else(|| "session snapshot: bad group key part".to_string())
})
.collect::<Result<_, _>>()?;
let mut vals: Vec<Value> = Vec::new();
let sessions: Vec<Session> = a[1]
.as_array()
.ok_or("session snapshot: bad bucket")?
.iter()
.map(|se| {
let t = se
.as_array()
.filter(|t| t.len() == 3)
.ok_or("session snapshot: bad session")?;
let (key, acc) = crate::snapshot::acc_from_json_v1(&t[2])?;
if vals.is_empty() {
vals = key;
}
Ok(Session {
first: t[0].as_i64().ok_or("session snapshot: bad first")?,
last: t[1].as_i64().ok_or("session snapshot: bad last")?,
acc,
})
})
.collect::<Result<_, String>>()?;
map.insert(gk.join("\u{1}"), Bucket { vals, sessions });
}
}
self.sessions = map;
self.wm.load_state(&s["wm"])?;
self.fired_through = s["fired_through"]
.as_i64()
.ok_or("session snapshot: missing fired_through")?;
self.late_dropped = s["late_dropped"]
.as_u64()
.ok_or("session snapshot: missing late_dropped")?;
Ok(())
}
}
impl WindowOperator for SessionAgg {
fn expect_partitions(&mut self, partitions: &[i32], now_ms: i64) {
self.wm.expect(partitions, now_ms);
}
fn push(&mut self, partition: i32, event_ms: i64, row: &[(String, Value)]) -> bool {
self.wm.observe(partition, event_ms);
if self.fired_through != i64::MIN && event_ms + self.gap_ms <= self.fired_through {
self.late_dropped += 1;
return false;
}
let gap = self.gap_ms;
let (n_aggs, with_sets, with_welford) = (self.aggs.len(), self.with_sets, self.with_welford);
let composite = crate::acc::composite_of(row, &self.group_by);
let mut bucket = self.sessions.remove(&composite).unwrap_or_else(|| Bucket {
vals: crate::acc::vals_of(row, &self.group_by),
sessions: Vec::new(),
});
let hit: Vec<usize> = bucket
.sessions
.iter()
.enumerate()
.filter(|(_, s)| s.first - gap <= event_ms && event_ms <= s.last + gap)
.map(|(i, _)| i)
.collect();
let mut merged = Session {
first: event_ms,
last: event_ms,
acc: Acc::new(n_aggs, with_sets, with_welford),
};
for &i in hit.iter().rev() {
let s = bucket.sessions.remove(i); merged.first = merged.first.min(s.first);
merged.last = merged.last.max(s.last);
merged.acc.merge(&self.aggs, &s.acc);
}
merged.acc.add(&self.aggs, row);
bucket.sessions.push(merged);
self.sessions.insert(composite, bucket);
true
}
fn advance(&mut self, now_ms: i64) -> Vec<WindowRow> {
self.wm.tick(now_ms);
let wm = self.wm.current(now_ms);
self.fire_through(wm)
}
fn advance_to(&mut self, watermark_ms: i64) -> Vec<WindowRow> {
let wm = self.wm.after_lateness(watermark_ms);
self.fire_through(wm)
}
fn flush(&mut self) -> Vec<WindowRow> {
self.fired_through = i64::MAX;
self.fire(|_| true)
}
fn snapshot(&mut self) -> Result<Vec<u8>, crate::StateError> {
Ok(SessionAgg::snapshot(self))
}
fn load_snapshot(&mut self, bytes: &[u8]) -> Result<(), String> {
SessionAgg::load_snapshot(self, bytes)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::AggOp;
fn row(pairs: &[(&str, Value)]) -> Vec<(String, Value)> {
pairs.iter().map(|(k, v)| (k.to_string(), v.clone())).collect()
}
fn num(w: &WindowRow, alias: &str) -> f64 {
match &w.aggs.iter().find(|(a, _)| a == alias).unwrap().1 {
Value::Num(n) => *n,
other => panic!("expected num for {alias}, got {other:?}"),
}
}
#[test]
fn events_within_gap_form_one_session() {
let mut s = SessionAgg::new(
1000,
0,
vec![],
vec![Agg::count("n"), Agg::of(AggOp::Sum, "amount", "total")],
);
for (t, a) in [(100, 1.0), (500, 2.0), (1400, 3.0)] {
s.push(0, t, &row(&[("amount", Value::Num(a))])); }
s.push(0, 5000, &row(&[("amount", Value::Num(9.0))])); let out = s.advance(0); assert_eq!(out.len(), 1);
assert_eq!((out[0].window_start, out[0].window_end), (100, 2400));
assert_eq!(num(&out[0], "n"), 3.0);
assert_eq!(num(&out[0], "total"), 6.0);
}
#[test]
fn a_gap_starts_a_new_session() {
let mut s = SessionAgg::new(1000, 0, vec![], vec![Agg::count("n")]);
s.push(0, 100, &row(&[]));
s.push(0, 2000, &row(&[])); s.push(0, 10_000, &row(&[])); let mut out = s.advance(0);
out.sort_by_key(|w| w.window_start);
assert_eq!(out.len(), 2, "two distinct sessions");
assert_eq!(out[0].window_start, 100);
assert_eq!(out[1].window_start, 2000);
}
#[test]
fn a_bridging_event_merges_two_sessions() {
let mut s = SessionAgg::new(1000, 0, vec![], vec![Agg::count("n")]);
s.push(0, 100, &row(&[])); s.push(0, 2500, &row(&[])); s.push(0, 1000, &row(&[])); s.push(0, 1900, &row(&[])); s.push(0, 10_000, &row(&[])); let out = s.advance(0);
assert_eq!(out.len(), 1, "all merged into one session");
assert_eq!((out[0].window_start, out[0].window_end), (100, 3500)); assert_eq!(num(&out[0], "n"), 4.0, "all four events");
}
#[test]
fn separate_keys_have_separate_sessions() {
let mut s = SessionAgg::new(1000, 0, vec!["k".into()], vec![Agg::count("n")]);
s.push(0, 100, &row(&[("k", Value::Str("a".into()))]));
s.push(0, 200, &row(&[("k", Value::Str("b".into()))]));
s.push(0, 300, &row(&[("k", Value::Str("a".into()))]));
s.push(0, 10_000, &row(&[("k", Value::Str("a".into()))])); let out = s.advance(0);
let a = out
.iter()
.find(|w| w.key == vec![Value::Str("a".into())] && w.window_start == 100)
.unwrap();
let b = out.iter().find(|w| w.key == vec![Value::Str("b".into())]).unwrap();
assert_eq!(num(a, "n"), 2.0);
assert_eq!(num(b, "n"), 1.0);
}
#[test]
fn multi_partition_watermark_holds_firing() {
let mut s = SessionAgg::new(1000, 0, vec![], vec![Agg::count("n")]);
s.expect_partitions(&[0, 1], 0);
s.push(0, 100, &row(&[]));
s.push(0, 9000, &row(&[])); assert!(s.advance(0).is_empty(), "unseen partition 1 pins the watermark");
s.push(1, 9000, &row(&[]));
assert!(!s.advance(0).is_empty(), "both seen → the first session fires");
}
#[test]
fn flush_fires_all_open_sessions() {
let mut s = SessionAgg::new(1000, 10_000, vec![], vec![Agg::count("n")]);
s.push(0, 100, &row(&[]));
s.push(0, 5000, &row(&[]));
assert!(s.advance(0).is_empty(), "long lateness holds them open");
assert_eq!(s.flush().len(), 2);
}
#[test]
fn snapshot_restore_survives_a_bridging_merge_across_the_boundary() {
let mk = || {
let mut s = SessionAgg::new(1000, 0, vec![], vec![Agg::count("n")]);
s.expect_partitions(&[0], 0);
s.push(0, 100, &row(&[]));
s.push(0, 200, &row(&[])); s.push(0, 2000, &row(&[])); s
};
let drive = |s: &mut SessionAgg| {
s.push(0, 1100, &row(&[])); s.push(0, 9000, &row(&[])); s.advance(0)
};
let mut a = mk();
let fired_a = drive(&mut a);
let snap = mk().snapshot();
let mut b = SessionAgg::new(1000, 0, vec![], vec![Agg::count("n")]);
b.load_snapshot(&snap).expect("restore");
let fired_b = drive(&mut b);
assert_eq!(fired_a, fired_b, "restored run must fire byte-identically");
let m = fired_a
.iter()
.find(|w| w.window_start == 100)
.expect("merged session fired");
assert_eq!(m.window_end, 3000);
assert_eq!(num(m, "n"), 4.0);
}
#[test]
fn bounded_idleness_releases_sessions_held_by_a_silent_partition() {
let mk = |idle_ms: i64| {
let mut s = SessionAgg::new(1000, 0, vec![], vec![Agg::count("n")]);
if idle_ms > 0 {
s = s.with_idle_timeout(idle_ms);
}
s.expect_partitions(&[0, 1], 0);
s.push(0, 100, &row(&[]));
s.push(1, 200, &row(&[])); s.advance(0); s.push(0, 5000, &row(&[])); s
};
assert!(
mk(0).advance(6000).is_empty(),
"no idle timeout → silent partition holds the session shut"
);
let out = mk(1000).advance(6000); assert!(
out.iter().any(|w| w.window_start == 100),
"idle partition released → session [100,1100) fires"
);
}
}