use std::collections::HashMap;
pub(crate) struct Watermark {
allowed_lateness_ms: i64,
idle_timeout_ms: Option<i64>,
part_max: HashMap<i32, i64>,
last_activity: HashMap<i32, i64>,
last_seen_max: HashMap<i32, i64>,
}
impl Watermark {
pub fn new(allowed_lateness_ms: i64) -> Self {
Self {
allowed_lateness_ms: allowed_lateness_ms.max(0),
idle_timeout_ms: None,
part_max: HashMap::new(),
last_activity: HashMap::new(),
last_seen_max: HashMap::new(),
}
}
pub fn with_idle_timeout(mut self, idle_timeout_ms: i64) -> Self {
self.idle_timeout_ms = (idle_timeout_ms > 0).then_some(idle_timeout_ms);
self
}
pub fn expect(&mut self, partitions: &[i32], now_ms: i64) {
for &p in partitions {
self.part_max.entry(p).or_insert(i64::MIN);
self.last_activity.entry(p).or_insert(now_ms);
self.last_seen_max.entry(p).or_insert(i64::MIN);
}
}
pub fn observe(&mut self, partition: i32, event_ms: i64) {
let pm = self.part_max.entry(partition).or_insert(i64::MIN);
if event_ms > *pm {
*pm = event_ms;
}
}
pub fn tick(&mut self, now_ms: i64) {
let snapshot: Vec<(i32, i64)> = self.part_max.iter().map(|(&p, &m)| (p, m)).collect();
for (p, pm) in snapshot {
let seen = self.last_seen_max.entry(p).or_insert(i64::MIN);
if pm > *seen {
*seen = pm;
self.last_activity.insert(p, now_ms);
} else {
self.last_activity.entry(p).or_insert(now_ms);
}
}
}
pub fn current(&self, now_ms: i64) -> i64 {
let active_min = self
.part_max
.iter()
.filter(|(p, _)| self.is_active(**p, now_ms))
.map(|(_, &m)| m)
.min();
match active_min {
None => i64::MIN,
Some(m) => m.saturating_sub(self.allowed_lateness_ms),
}
}
pub fn after_lateness(&self, watermark_ms: i64) -> i64 {
if watermark_ms == i64::MIN {
i64::MIN
} else {
watermark_ms.saturating_sub(self.allowed_lateness_ms)
}
}
fn is_active(&self, partition: i32, now_ms: i64) -> bool {
match self.idle_timeout_ms {
None => true,
Some(timeout) => {
let last = self.last_activity.get(&partition).copied().unwrap_or(now_ms);
now_ms - last <= timeout
}
}
}
pub(crate) fn cfg_fingerprint(&self) -> String {
format!("lateness={};idle={:?}", self.allowed_lateness_ms, self.idle_timeout_ms)
}
pub(crate) fn state_to_json(&self) -> serde_json::Value {
serde_json::json!({
"part_max": crate::snapshot::i32map_to_json(&self.part_max),
"last_activity": crate::snapshot::i32map_to_json(&self.last_activity),
"last_seen_max": crate::snapshot::i32map_to_json(&self.last_seen_max),
})
}
pub(crate) fn load_state(&mut self, j: &serde_json::Value) -> Result<(), String> {
self.part_max =
crate::snapshot::i32map_from_json(&j["part_max"]).map_err(|e| format!("watermark part_max: {e}"))?;
self.last_activity = crate::snapshot::i32map_from_json(&j["last_activity"])
.map_err(|e| format!("watermark last_activity: {e}"))?;
self.last_seen_max = crate::snapshot::i32map_from_json(&j["last_seen_max"])
.map_err(|e| format!("watermark last_seen_max: {e}"))?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn min_across_partitions_holds_back_the_faster_one() {
let mut w = Watermark::new(0);
w.observe(0, 1000);
w.observe(1, 400);
assert_eq!(w.current(0), 400, "the lagging partition sets the watermark");
w.observe(1, 1200);
assert_eq!(w.current(0), 1000, "now partition 0 is the laggard");
}
#[test]
fn expected_but_unseen_partition_pins_at_min() {
let mut w = Watermark::new(0);
w.expect(&[0, 1], 0);
w.observe(0, 5000);
assert_eq!(w.current(0), i64::MIN, "partition 1 assigned but silent → no progress");
w.observe(1, 100);
assert_eq!(w.current(0), 100);
}
#[test]
fn allowed_lateness_shifts_the_watermark_back() {
let mut w = Watermark::new(250);
w.observe(0, 1000);
assert_eq!(w.current(0), 750);
}
#[test]
fn empty_is_min() {
assert_eq!(Watermark::new(0).current(0), i64::MIN);
}
#[test]
fn idleness_disabled_ignores_the_clock() {
let mut w = Watermark::new(0);
w.expect(&[0, 1], 0);
w.observe(0, 5000);
w.tick(1_000_000); assert_eq!(w.current(1_000_000), i64::MIN, "silent partition 1 still holds");
}
#[test]
fn a_silent_partition_goes_idle_and_releases_the_watermark() {
let mut w = Watermark::new(0).with_idle_timeout(1000);
w.expect(&[0, 1], 0);
w.observe(0, 5000);
w.observe(1, 400);
w.tick(0);
assert_eq!(w.current(0), 400, "both active → min holds at the laggard");
w.observe(0, 6000);
w.tick(500);
assert_eq!(w.current(500), 400, "within timeout, partition 1 still holds");
w.observe(0, 7000); w.tick(1600); assert_eq!(
w.current(1600),
7000,
"idle partition 1 dropped; partition 0 (still producing) drives"
);
}
#[test]
fn an_assigned_but_never_seen_partition_goes_idle() {
let mut w = Watermark::new(0).with_idle_timeout(1000);
w.expect(&[0, 1], 0); w.observe(0, 5000);
w.tick(0);
assert_eq!(
w.current(0),
i64::MIN,
"partition 1 assigned, timer not yet expired → holds"
);
w.observe(0, 5001); w.tick(1600); assert_eq!(
w.current(1600),
5001,
"never-seen partition released after its idle timeout"
);
}
#[test]
fn an_idle_partition_rejoins_when_it_produces_again() {
let mut w = Watermark::new(0).with_idle_timeout(1000);
w.expect(&[0, 1], 0);
w.observe(0, 5000);
w.tick(2000); assert_eq!(w.current(2000), 5000);
w.observe(1, 300); w.tick(2001);
assert_eq!(
w.current(2001),
300,
"rejoined partition holds the watermark back again"
);
}
#[test]
fn all_idle_holds_at_min() {
let mut w = Watermark::new(0).with_idle_timeout(1000);
w.expect(&[0, 1], 0);
w.observe(0, 5000);
w.observe(1, 6000);
w.tick(0);
w.tick(5000); assert_eq!(w.current(5000), i64::MIN, "no active partition → no safe progress");
}
}