use std::collections::BTreeMap;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_json::Value;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MergePolicy {
LastWrite,
HighestPriority,
MostSpecific,
MergeArray,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Observation {
pub observed_at: DateTime<Utc>,
#[serde(default)]
pub source_priority: i32,
#[serde(default)]
pub specificity_score: i32,
pub source_event_id: String,
pub fields: BTreeMap<String, Value>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ProjectionDef {
pub entity_type: String,
pub field_policies: BTreeMap<String, MergePolicy>,
}
impl ProjectionDef {
pub fn new<I>(entity_type: impl Into<String>, fields: I) -> Self
where
I: IntoIterator<Item = (String, MergePolicy)>,
{
Self {
entity_type: entity_type.into(),
field_policies: fields.into_iter().collect(),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct EntitySnapshot {
pub entity_type: String,
pub fields: BTreeMap<String, Value>,
}
pub fn fold(observations: &[Observation], def: &ProjectionDef) -> EntitySnapshot {
let mut out: BTreeMap<String, Value> = BTreeMap::new();
for (field, policy) in &def.field_policies {
if let Some(value) = apply_policy(*policy, field, observations) {
out.insert(field.clone(), value);
}
}
EntitySnapshot {
entity_type: def.entity_type.clone(),
fields: out,
}
}
fn apply_policy(policy: MergePolicy, field: &str, observations: &[Observation]) -> Option<Value> {
apply_policy_with_source(policy, field, observations).map(|(value, _source)| value)
}
fn apply_policy_with_source<'a>(
policy: MergePolicy,
field: &str,
observations: &'a [Observation],
) -> Option<(Value, Option<&'a Observation>)> {
match policy {
MergePolicy::LastWrite => pick_observation(observations, field, |a, b| {
a.observed_at
.cmp(&b.observed_at)
.then_with(|| b.source_event_id.cmp(&a.source_event_id))
})
.map(|o| (o.fields[field].clone(), Some(o))),
MergePolicy::HighestPriority => pick_observation(observations, field, |a, b| {
a.source_priority
.cmp(&b.source_priority)
.then_with(|| a.observed_at.cmp(&b.observed_at))
.then_with(|| b.source_event_id.cmp(&a.source_event_id))
})
.map(|o| (o.fields[field].clone(), Some(o))),
MergePolicy::MostSpecific => pick_observation(observations, field, |a, b| {
a.specificity_score
.cmp(&b.specificity_score)
.then_with(|| a.observed_at.cmp(&b.observed_at))
.then_with(|| b.source_event_id.cmp(&a.source_event_id))
})
.map(|o| (o.fields[field].clone(), Some(o))),
MergePolicy::MergeArray => merge_array(observations, field).map(|value| (value, None)),
}
}
fn pick_observation<'a, F>(
observations: &'a [Observation],
field: &str,
mut cmp: F,
) -> Option<&'a Observation>
where
F: FnMut(&Observation, &Observation) -> std::cmp::Ordering,
{
observations
.iter()
.filter(|o| o.fields.contains_key(field))
.max_by(|a, b| cmp(a, b))
}
fn merge_array(observations: &[Observation], field: &str) -> Option<Value> {
let mut collected: Vec<Value> = Vec::new();
for obs in observations {
if let Some(v) = obs.fields.get(field) {
match v {
Value::Array(items) => {
for item in items {
if !collected.contains(item) {
collected.push(item.clone());
}
}
}
other => {
if !collected.contains(other) {
collected.push(other.clone());
}
}
}
}
}
if collected.is_empty() {
None
} else {
collected.sort_by(|a, b| {
serde_json::to_string(a)
.unwrap_or_default()
.cmp(&serde_json::to_string(b).unwrap_or_default())
});
Some(Value::Array(collected))
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct Provenance {
pub field: String,
pub value: Value,
pub source_event_id: String,
pub source_event_at: DateTime<Utc>,
pub merge_policy_applied: MergePolicy,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct ProvenanceSnapshot {
pub entity_type: String,
pub fields: BTreeMap<String, Provenance>,
}
pub fn provenance_for_field(
observations: &[Observation],
def: &ProjectionDef,
field: &str,
) -> Option<Provenance> {
let policy = *def.field_policies.get(field)?;
let (value, source) = apply_policy_with_source(policy, field, observations)?;
let source = source?; Some(Provenance {
field: field.to_string(),
value,
source_event_id: source.source_event_id.clone(),
source_event_at: source.observed_at,
merge_policy_applied: policy,
})
}
pub fn fold_with_provenance(
observations: &[Observation],
def: &ProjectionDef,
) -> ProvenanceSnapshot {
let mut fields: BTreeMap<String, Provenance> = BTreeMap::new();
for field in def.field_policies.keys() {
if let Some(prov) = provenance_for_field(observations, def, field) {
fields.insert(field.clone(), prov);
}
}
ProvenanceSnapshot {
entity_type: def.entity_type.clone(),
fields,
}
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::TimeZone;
use serde_json::json;
#[allow(clippy::needless_pass_by_value)]
fn obs(
ts: i64,
priority: i32,
specificity: i32,
id: &str,
fields: serde_json::Value,
) -> Observation {
let map = fields
.as_object()
.expect("test obs needs an object")
.iter()
.map(|(k, v)| (k.clone(), v.clone()))
.collect();
Observation {
observed_at: Utc.timestamp_opt(ts, 0).single().unwrap(),
source_priority: priority,
specificity_score: specificity,
source_event_id: id.to_string(),
fields: map,
}
}
fn def(field: &str, policy: MergePolicy) -> ProjectionDef {
ProjectionDef::new("contact", vec![(field.to_string(), policy)])
}
#[test]
fn last_write_picks_latest_observed_at() {
let obs = vec![
obs(100, 0, 0, "e1", json!({"status": "cold"})),
obs(200, 0, 0, "e2", json!({"status": "warm"})),
obs(150, 0, 0, "e3", json!({"status": "lukewarm"})),
];
let snap = fold(&obs, &def("status", MergePolicy::LastWrite));
assert_eq!(snap.fields.get("status"), Some(&json!("warm")));
}
#[test]
fn last_write_with_tied_timestamps_breaks_by_event_id() {
let obs = vec![
obs(100, 0, 0, "evt-b", json!({"status": "B"})),
obs(100, 0, 0, "evt-a", json!({"status": "A"})),
];
let snap = fold(&obs, &def("status", MergePolicy::LastWrite));
assert_eq!(snap.fields.get("status"), Some(&json!("A")));
}
#[test]
fn highest_priority_user_correction_beats_ai_extraction() {
let obs = vec![
obs(100, 0, 0, "ai1", json!({"name": "alice chen"})), obs(150, 100, 0, "agent1", json!({"name": "Alice C."})), obs(200, 1000, 0, "user1", json!({"name": "Alice Chen"})), ];
let snap = fold(&obs, &def("name", MergePolicy::HighestPriority));
assert_eq!(snap.fields.get("name"), Some(&json!("Alice Chen")));
}
#[test]
fn most_specific_picks_densest_observation() {
let obs = vec![
obs(100, 0, 1, "shallow", json!({"role": "engineer"})),
obs(
100,
0,
10,
"dense",
json!({"role": "Staff Engineer, Platform"}),
),
];
let snap = fold(&obs, &def("role", MergePolicy::MostSpecific));
assert_eq!(
snap.fields.get("role"),
Some(&json!("Staff Engineer, Platform"))
);
}
#[test]
fn merge_array_unions_scalars_into_a_sorted_set() {
let obs = vec![
obs(100, 0, 0, "e1", json!({"tags": "rust"})),
obs(200, 0, 0, "e2", json!({"tags": "agents"})),
obs(300, 0, 0, "e3", json!({"tags": "rust"})), ];
let snap = fold(&obs, &def("tags", MergePolicy::MergeArray));
assert_eq!(snap.fields.get("tags"), Some(&json!(["agents", "rust"])));
}
#[test]
fn merge_array_flattens_array_inputs() {
let obs = vec![
obs(100, 0, 0, "e1", json!({"tags": ["a", "b"]})),
obs(200, 0, 0, "e2", json!({"tags": ["b", "c"]})),
];
let snap = fold(&obs, &def("tags", MergePolicy::MergeArray));
assert_eq!(snap.fields.get("tags"), Some(&json!(["a", "b", "c"])));
}
#[test]
fn fields_not_in_definition_are_dropped() {
let obs = vec![obs(
100,
0,
0,
"e1",
json!({"name": "Alice", "secret": "x"}),
)];
let snap = fold(&obs, &def("name", MergePolicy::LastWrite));
assert_eq!(snap.fields.len(), 1);
assert!(snap.fields.contains_key("name"));
assert!(!snap.fields.contains_key("secret"));
}
#[test]
fn missing_field_yields_empty_snapshot_field() {
let obs = vec![obs(100, 0, 0, "e1", json!({"other": "x"}))];
let snap = fold(&obs, &def("name", MergePolicy::LastWrite));
assert!(snap.fields.is_empty());
}
#[test]
fn fold_is_order_independent_for_each_policy() {
let policies = [
MergePolicy::LastWrite,
MergePolicy::HighestPriority,
MergePolicy::MostSpecific,
MergePolicy::MergeArray,
];
for &policy in &policies {
let a = obs(100, 1, 1, "e1", json!({"f": 1}));
let b = obs(200, 2, 2, "e2", json!({"f": 2}));
let c = obs(150, 3, 3, "e3", json!({"f": 3}));
let def = def("f", policy);
let s1 = fold(&[a.clone(), b.clone(), c.clone()], &def);
let s2 = fold(&[c, b, a], &def);
assert_eq!(s1, s2, "policy {policy:?} is order-dependent — bug");
}
}
#[test]
fn provenance_for_last_write_picks_winning_event() {
let obs = vec![
obs(100, 0, 0, "evt-a", json!({"status": "cold"})),
obs(200, 0, 0, "evt-b", json!({"status": "warm"})),
obs(150, 0, 0, "evt-c", json!({"status": "lukewarm"})),
];
let d = def("status", MergePolicy::LastWrite);
let p = provenance_for_field(&obs, &d, "status").expect("should have provenance");
assert_eq!(p.field, "status");
assert_eq!(p.value, json!("warm"));
assert_eq!(p.source_event_id, "evt-b");
assert_eq!(p.merge_policy_applied, MergePolicy::LastWrite);
}
#[test]
fn provenance_for_highest_priority_credits_the_user_correction() {
let obs = vec![
obs(100, 0, 0, "ai-evt", json!({"name": "alice chen"})),
obs(200, 1000, 0, "user-evt", json!({"name": "Alice Chen"})),
];
let d = def("name", MergePolicy::HighestPriority);
let p = provenance_for_field(&obs, &d, "name").unwrap();
assert_eq!(p.source_event_id, "user-evt");
assert_eq!(p.value, json!("Alice Chen"));
}
#[test]
fn provenance_skips_merge_array_fields() {
let obs = vec![
obs(100, 0, 0, "e1", json!({"tags": "rust"})),
obs(200, 0, 0, "e2", json!({"tags": "agents"})),
];
let d = def("tags", MergePolicy::MergeArray);
assert!(provenance_for_field(&obs, &d, "tags").is_none());
}
#[test]
fn provenance_for_unknown_field_returns_none() {
let obs = vec![obs(100, 0, 0, "e1", json!({"status": "x"}))];
let d = def("name", MergePolicy::LastWrite);
assert!(provenance_for_field(&obs, &d, "name").is_none());
}
#[test]
fn provenance_for_field_not_in_def_returns_none() {
let obs = vec![obs(100, 0, 0, "e1", json!({"status": "x"}))];
let d = def("name", MergePolicy::LastWrite); assert!(provenance_for_field(&obs, &d, "status").is_none());
}
#[test]
fn fold_with_provenance_covers_every_single_source_field() {
let mut policies = BTreeMap::new();
policies.insert("status".to_string(), MergePolicy::LastWrite);
policies.insert("name".to_string(), MergePolicy::HighestPriority);
policies.insert("tags".to_string(), MergePolicy::MergeArray);
let def = ProjectionDef {
entity_type: "contact".to_string(),
field_policies: policies,
};
let obs = vec![
obs(100, 0, 0, "e1", json!({"status": "cold", "tags": "a"})),
obs(200, 1000, 0, "e2", json!({"name": "Alice", "tags": "b"})),
obs(150, 0, 0, "e3", json!({"status": "warm"})),
];
let snap = fold_with_provenance(&obs, &def);
assert_eq!(snap.fields.len(), 2);
assert_eq!(snap.fields["status"].source_event_id, "e3");
assert_eq!(snap.fields["status"].value, json!("warm"));
assert_eq!(snap.fields["name"].source_event_id, "e2");
assert!(!snap.fields.contains_key("tags"));
}
#[test]
fn provenance_is_order_independent() {
let a = obs(100, 1, 1, "e-a", json!({"f": "A"}));
let b = obs(200, 2, 2, "e-b", json!({"f": "B"}));
let c = obs(150, 3, 3, "e-c", json!({"f": "C"}));
for &policy in &[
MergePolicy::LastWrite,
MergePolicy::HighestPriority,
MergePolicy::MostSpecific,
] {
let d = def("f", policy);
let p1 = provenance_for_field(&[a.clone(), b.clone(), c.clone()], &d, "f").unwrap();
let p2 = provenance_for_field(&[c.clone(), b.clone(), a.clone()], &d, "f").unwrap();
assert_eq!(
p1, p2,
"policy {policy:?} provenance is order-dependent — bug"
);
}
}
use proptest::prelude::*;
fn arb_observation() -> impl Strategy<Value = Observation> {
(
0i64..1_000_i64, -100i32..1000i32, 0i32..10i32, "[a-z]{1,8}".prop_map(String::from), prop_oneof![
Just(json!("A")),
Just(json!("B")),
Just(json!(42)),
Just(json!(["x", "y"])),
Just(json!(["y", "z"])),
],
)
.prop_map(|(ts, prio, spec, id, val)| {
let mut fields = BTreeMap::new();
fields.insert("f".to_string(), val);
Observation {
observed_at: Utc.timestamp_opt(ts, 0).single().unwrap(),
source_priority: prio,
specificity_score: spec,
source_event_id: id,
fields,
}
})
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(64))]
#[test]
fn fold_determinism_under_shuffling(
mut obs in prop::collection::vec(arb_observation(), 1..6),
policy_idx in 0u32..4u32,
) {
let policy = match policy_idx {
0 => MergePolicy::LastWrite,
1 => MergePolicy::HighestPriority,
2 => MergePolicy::MostSpecific,
_ => MergePolicy::MergeArray,
};
let def = def("f", policy);
let baseline = fold(&obs, &def);
obs.reverse();
prop_assert_eq!(&baseline, &fold(&obs, &def));
if obs.len() > 1 {
obs.rotate_left(1);
prop_assert_eq!(&baseline, &fold(&obs, &def));
let last = obs.len() - 1;
obs.swap(0, last);
prop_assert_eq!(&baseline, &fold(&obs, &def));
}
}
}
}