fv-streams-ops 0.3.0

Stateful streaming operators for FusionVault Streams: event-time tumbling and sliding windows with keyed incremental aggregation, session windows, stream joins, top-N and last-N, watermarks and snapshots. Pure and synchronous — no I/O, no runtime.
Documentation
//! versioned SNAPSHOT/RESTORE of operator state, pure and I/O-free.
//!
//! The correctness rule: source offsets may only be committed together with the state
//! that consumed them. The worker checkpoints `(operator snapshot, offsets)` per partition; this
//! module is the snapshot half — each operator serializes its DYNAMIC state (accumulators, watermark
//! timers, fired-through, late counts) into a versioned JSON envelope and can restore it into a
//! freshly-configured operator. Restore into a *differently-configured* operator (window size changed,
//! different aggs…) is refused LOUDLY via a config fingerprint — stale state must never be silently
//! reinterpreted.
//!
//! Codec: JSON via the same `fv_value::value_to_json`/`json_to_value` codec the wire already trusts —
//! human-debuggable snapshots, zero new dependencies. The envelope carries a schema version (`v`) so a
//! binary codec can replace it later without a flag day. Determinism contract (tested): a restored
//! operator fed the same tail of events fires byte-identically to the uninterrupted original —
//! Kafka's immutable log + pure operators ⇒ replay rebuilds identical results.

use fv_value::{json_to_value, value_to_json, Value};
use serde_json::{json, Value as J};

use crate::acc::Acc;

pub(crate) const SNAPSHOT_VERSION: i64 = 1;

/// Wrap an operator's dynamic state in the versioned envelope.
pub(crate) fn envelope(kind: &str, cfg_fingerprint: &str, state: J) -> Vec<u8> {
    serde_json::to_vec(&json!({
        "v": SNAPSHOT_VERSION,
        "kind": kind,
        "cfg": cfg_fingerprint,
        "state": state,
    }))
    .expect("snapshot envelope serializes")
}

/// Open an envelope, refusing (loudly) a version, kind, or config mismatch.
pub(crate) fn open_envelope(kind: &str, cfg_fingerprint: &str, bytes: &[u8]) -> Result<J, String> {
    let v: J = serde_json::from_slice(bytes).map_err(|e| format!("snapshot: bad envelope json: {e}"))?;
    let got_v = v["v"].as_i64().ok_or("snapshot: missing version")?;
    if got_v != SNAPSHOT_VERSION {
        return Err(format!("snapshot: version {got_v} != supported {SNAPSHOT_VERSION}"));
    }
    let got_kind = v["kind"].as_str().unwrap_or("");
    if got_kind != kind {
        return Err(format!("snapshot: kind '{got_kind}' != expected '{kind}'"));
    }
    let got_cfg = v["cfg"].as_str().unwrap_or("");
    if got_cfg != cfg_fingerprint {
        return Err(format!(
            "snapshot: operator config changed — refusing to reinterpret state (snapshot cfg '{got_cfg}' vs current '{cfg_fingerprint}')"
        ));
    }
    Ok(v["state"].clone())
}

// ── Shared piece serializers ─────────────────────────────────────────────────────────────────────

/// `Acc` → compact JSON (T2, format 2): `[count, [slots...], sets|null]` — flat arrays, no per-acc
/// object keys, no embedded group key (operators store keys once per distinct key). `count` rides as
/// a native JSON u64; BTreeSet iteration keeps `sets` serialization deterministic.
pub(crate) fn acc_to_json_v2(a: &Acc) -> J {
    let slots: Vec<J> = a
        .slots
        .iter()
        .map(|s| match s {
            Some(f) => json!(f),
            None => J::Null,
        })
        .collect();
    let sets: J = match &a.sets {
        None => J::Null,
        Some(sets) => J::Array(
            sets.iter()
                .map(|s| match s {
                    Some(set) => J::Array(set.iter().map(|v| json!(v)).collect()),
                    None => J::Null,
                })
                .collect(),
        ),
    };
    let welford: J = match &a.welford {
        None => J::Null,
        Some(ws) => J::Array(ws.iter().map(|w| json!([w.n, w.mean, w.m2])).collect()),
    };
    json!([a.count, slots, sets, welford])
}

pub(crate) fn acc_from_json_v2(j: &J) -> Result<Acc, String> {
    let t = j
        .as_array()
        .filter(|a| a.len() == 3 || a.len() == 4)
        .ok_or("acc v2: not a 3/4-tuple")?;
    let count = t[0].as_u64().ok_or("acc v2: bad count")?;
    let slots: Box<[Option<f64>]> = t[1]
        .as_array()
        .ok_or("acc v2: bad slots")?
        .iter()
        .map(|s| s.as_f64())
        .collect();
    let sets = match &t[2] {
        J::Null => None,
        J::Array(arr) => Some(
            arr.iter()
                .map(|s| {
                    s.as_array()
                        .map(|vals| vals.iter().filter_map(|v| v.as_str().map(String::from)).collect())
                })
                .collect(),
        ),
        _ => return Err("acc v2: bad sets".into()),
    };
    // welford is the FUNC-2 4th element; a 3-tuple (pre-FUNC-2, or no variance agg) restores as None.
    let welford = match t.get(3) {
        None | Some(J::Null) => None,
        Some(J::Array(arr)) => Some(
            arr.iter()
                .map(|w| {
                    let a = w
                        .as_array()
                        .filter(|a| a.len() == 3)
                        .ok_or("acc v2: bad welford entry")?;
                    Ok(crate::acc::Welford {
                        n: a[0].as_u64().ok_or("acc v2: bad welford n")?,
                        mean: a[1].as_f64().ok_or("acc v2: bad welford mean")?,
                        m2: a[2].as_f64().ok_or("acc v2: bad welford m2")?,
                    })
                })
                .collect::<Result<_, String>>()?,
        ),
        _ => return Err("acc v2: bad welford".into()),
    };
    Ok(Acc {
        count,
        slots,
        sets,
        welford,
    })
}

/// Read the PRE-T2 accumulator object form `{ "key", "count", "slots", "sets"? }`, returning the
/// embedded group-key values alongside — v1 snapshots stored the key inside every acc; the dual-read
/// loaders lift it into the operator's per-key storage. Old snapshots must keep loading across the
/// upgrade (refusing them would silently lose consumed-but-unfired rows).
pub(crate) fn acc_from_json_v1(j: &J) -> Result<(Vec<Value>, Acc), String> {
    let key: Vec<Value> = j["key"]
        .as_array()
        .ok_or("acc v1: missing key")?
        .iter()
        .map(json_to_value)
        .collect();
    let count = j["count"].as_u64().ok_or("acc v1: missing count")?;
    let slots: Box<[Option<f64>]> = j["slots"]
        .as_array()
        .ok_or("acc v1: missing slots")?
        .iter()
        .map(|s| s.as_f64())
        .collect();
    let has_any_set = j["sets"].as_array().is_some_and(|a| a.iter().any(|s| !s.is_null()));
    let sets = if has_any_set {
        Some(
            j["sets"]
                .as_array()
                .unwrap()
                .iter()
                .map(|s| {
                    s.as_array()
                        .map(|vals| vals.iter().filter_map(|v| v.as_str().map(String::from)).collect())
                })
                .collect(),
        )
    } else {
        None
    };
    Ok((
        key,
        Acc {
            count,
            slots,
            sets,
            welford: None,
        },
    ))
}

/// A `Vec<(String, Value)>` row (buffered join rows) → JSON pair-array (preserves order exactly).
pub(crate) fn row_to_json(row: &[(String, Value)]) -> J {
    J::Array(row.iter().map(|(k, v)| json!([k, value_to_json(v)])).collect())
}

pub(crate) fn row_from_json(j: &J) -> Result<Vec<(String, Value)>, String> {
    j.as_array()
        .ok_or("row: not an array")?
        .iter()
        .map(|p| {
            let pair = p.as_array().filter(|a| a.len() == 2).ok_or("row: bad pair")?;
            Ok((
                pair[0].as_str().ok_or("row: bad key")?.to_string(),
                json_to_value(&pair[1]),
            ))
        })
        .collect()
}

/// An `i32 → i64` map (watermark internals) → JSON object with stringified keys.
pub(crate) fn i32map_to_json(m: &std::collections::HashMap<i32, i64>) -> J {
    J::Object(m.iter().map(|(k, v)| (k.to_string(), json!(v))).collect())
}

pub(crate) fn i32map_from_json(j: &J) -> Result<std::collections::HashMap<i32, i64>, String> {
    j.as_object()
        .ok_or("i32map: not an object")?
        .iter()
        .map(|(k, v)| {
            Ok((
                k.parse::<i32>().map_err(|e| format!("i32map key '{k}': {e}"))?,
                v.as_i64().ok_or_else(|| format!("i32map value for '{k}' not i64"))?,
            ))
        })
        .collect()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn envelope_round_trips_and_refuses_mismatches() {
        let b = envelope("window", "cfg-A", json!({"x": 1}));
        assert_eq!(open_envelope("window", "cfg-A", &b).unwrap()["x"], 1);
        assert!(open_envelope("session", "cfg-A", &b).unwrap_err().contains("kind"));
        assert!(open_envelope("window", "cfg-B", &b)
            .unwrap_err()
            .contains("config changed"));
        let bad_v = serde_json::to_vec(&json!({"v": 99, "kind": "window", "cfg": "cfg-A", "state": {}})).unwrap();
        assert!(open_envelope("window", "cfg-A", &bad_v)
            .unwrap_err()
            .contains("version"));
    }

    #[test]
    fn acc_v2_round_trips_exactly_including_u64_count() {
        let mut a = Acc::new(3, false, false);
        a.count = (1u64 << 60) + 7; // beyond f64-exact range — must survive as a native u64
        a.slots = vec![Some(3.25), None, Some(-0.0)].into_boxed_slice();
        let j = acc_to_json_v2(&a);
        let b = acc_from_json_v2(&j).unwrap();
        assert_eq!(b.count, a.count);
        assert_eq!(b.slots, a.slots);
        assert!(b.sets.is_none(), "scalar-only acc stays set-free through the codec");
    }

    #[test]
    fn acc_v2_round_trips_count_distinct_sets() {
        let mut a = Acc::new(2, true, false);
        a.sets.as_mut().unwrap()[1] = Some(
            ["sb1".to_string(), "sb2".to_string(), "n1".to_string()]
                .into_iter()
                .collect(),
        );
        let b = acc_from_json_v2(&acc_to_json_v2(&a)).unwrap();
        assert_eq!(b.sets, a.sets);
    }

    #[test]
    fn acc_v1_reader_lifts_the_embedded_key_and_defaults_sets() {
        // A PRE-T2 snapshot fragment: the upgrade contract is load-correctly, never refuse-silently.
        let v1 = json!({"key": ["k9", 1.5], "count": 41, "slots": [7.0, null]});
        let (key, acc) = acc_from_json_v1(&v1).unwrap();
        assert_eq!(key, vec![Value::Str("k9".into()), Value::Num(1.5)]);
        assert_eq!(acc.count, 41);
        assert_eq!(acc.slots.as_ref(), &[Some(7.0), None]);
        assert!(acc.sets.is_none());
        let v1_sets = json!({"key": [], "count": 2, "slots": [null], "sets": [["sa", "sb"]]});
        let (_, acc) = acc_from_json_v1(&v1_sets).unwrap();
        assert_eq!(acc.sets.as_ref().unwrap()[0].as_ref().unwrap().len(), 2);
    }

    #[test]
    fn row_pairs_preserve_order() {
        let row = vec![
            ("z".to_string(), Value::Num(1.0)),
            ("a".to_string(), Value::Str("x".into())),
        ];
        assert_eq!(row_from_json(&row_to_json(&row)).unwrap(), row);
    }

    #[test]
    fn i32map_round_trips_negative_and_min_sentinels() {
        let mut m = std::collections::HashMap::new();
        m.insert(0, i64::MIN);
        m.insert(7, 12345);
        m.insert(-1, -9);
        assert_eq!(i32map_from_json(&i32map_to_json(&m)).unwrap(), m);
    }
}