use std::collections::BTreeMap;
use polyc_eventlog_model::Event;
use polyc_proto::events_decode::try_decode_event_payload;
use polyc_proto::kinds;
use polyc_proto::proto::polychrome::events::v1::{
ConversationSummaryEvent, MemoryAddedEvent, MemoryCorroboratedEvent, MemoryDurability,
MemoryExtractedEvent, MemoryInvalidatedEvent, MemoryProvenance, MemoryScope,
ProfileDocUpdatedEvent,
};
pub const SCOPE_DIRECT: &str = "direct";
pub const SCOPE_PORTABLE: &str = "portable";
pub const SCOPE_UNSPECIFIED: &str = "unspecified";
pub const DURABILITY_DURABLE: &str = "durable";
pub const DURABILITY_SESSION: &str = "session";
pub const DURABILITY_UNSPECIFIED: &str = "unspecified";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum FactScope {
Direct,
Portable,
Unspecified,
}
impl FactScope {
const fn as_str(self) -> &'static str {
match self {
Self::Direct => SCOPE_DIRECT,
Self::Portable => SCOPE_PORTABLE,
Self::Unspecified => SCOPE_UNSPECIFIED,
}
}
fn of(scope: buffa::EnumValue<MemoryScope>) -> Self {
match scope.as_known() {
Some(MemoryScope::Direct) => Self::Direct,
Some(MemoryScope::Portable) => Self::Portable,
Some(MemoryScope::Unspecified) | None => Self::Unspecified,
}
}
}
fn durability_str(durability: buffa::EnumValue<MemoryDurability>) -> &'static str {
match durability.as_known() {
Some(MemoryDurability::Durable) => DURABILITY_DURABLE,
Some(MemoryDurability::Session) => DURABILITY_SESSION,
Some(MemoryDurability::Unspecified) | None => DURABILITY_UNSPECIFIED,
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
#[allow(
clippy::struct_field_names,
reason = "conversation_id/turn_exec_id/turn_id are the wire and column names this triple mirrors"
)]
pub struct ProvenanceTriple {
pub conversation_id: String,
pub turn_exec_id: String,
pub turn_id: String,
}
fn normalize_turn_exec_id(raw: &str) -> String {
uuid::Uuid::parse_str(raw).map_or_else(|_| String::new(), |id| id.as_simple().to_string())
}
fn provenance_triple(provenance: Option<&MemoryProvenance>) -> ProvenanceTriple {
let Some(provenance) = provenance else {
return ProvenanceTriple::default();
};
ProvenanceTriple {
conversation_id: provenance.conversation_id.clone(),
turn_id: normalize_turn_exec_id(&provenance.turn_exec_id),
turn_exec_id: provenance.turn_exec_id.clone(),
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct ProvenanceIdentity {
pub provider: String,
pub scope: String,
pub external_id: String,
pub display_name: String,
}
fn provenance_identity(provenance: Option<&MemoryProvenance>) -> Option<ProvenanceIdentity> {
let identity = provenance?.identity.as_option()?;
Some(ProvenanceIdentity {
provider: identity.provider.clone(),
scope: identity.scope.clone(),
external_id: identity.external_id.clone(),
display_name: identity.display_name.clone(),
})
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemoryFactRow {
pub position: u64,
pub fact_id: String,
pub text: String,
pub entities: Vec<String>,
pub provenance: ProvenanceTriple,
pub event_time_ms: u64,
pub ingested_at_ms: u64,
pub valid_from_ms: u64,
pub confidence_bps: u32,
scope: FactScope,
pub durability: &'static str,
}
impl MemoryFactRow {
#[must_use]
pub const fn scope(&self) -> &'static str {
self.scope.as_str()
}
#[must_use]
const fn is_portable(&self) -> bool {
matches!(self.scope, FactScope::Portable)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemoryInvalidationRow {
pub position: u64,
pub fact_id: String,
pub invalid_at_ms: u64,
pub reason: String,
pub superseded_by_fact_id: String,
pub provenance: ProvenanceTriple,
fact_scope: FactScope,
}
impl MemoryInvalidationRow {
#[must_use]
pub const fn fact_scope(&self) -> &'static str {
self.fact_scope.as_str()
}
#[must_use]
const fn is_portable(&self) -> bool {
matches!(self.fact_scope, FactScope::Portable)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemoryListRow {
pub position: u64,
pub owner_kind: &'static str,
pub owner_id: String,
pub list_kind: &'static str,
pub index: u64,
pub text_value: String,
portable: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemoryCorroborationRow {
pub position: u64,
pub fact_id: String,
pub origin: ProvenanceTriple,
pub at_ms: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemoryExtractionRow {
pub position: u64,
pub conversation_id: String,
pub turn_exec_id: String,
pub extracted_at_ms: u64,
pub facts_added: u32,
pub facts_invalidated: u32,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemoryProfileRewriteRow {
pub position: u64,
pub document: String,
pub updated_at_ms: u64,
pub source_fact_ids: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemorySummaryRow {
pub position: u64,
pub conversation_id: String,
pub summary_text: String,
pub event_time_ms: u64,
pub scope: &'static str,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemoryProvenanceIdentityRow {
pub position: u64,
pub record_kind: &'static str,
pub record_id: String,
pub identity: ProvenanceIdentity,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct MemoryFenceRow {
pub position: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemoryUnknownRow {
pub position: u64,
pub kind: String,
}
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum PersonaMemoryFoldError {
#[error("position {position} carries a {kind} payload that will not decode: {reason}")]
UndecodableRecord {
position: u64,
kind: &'static str,
reason: String,
},
#[error("position {position} appears more than once in one source prefix")]
RepeatedPosition {
position: u64,
},
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct PersonaMemoryFacts {
pub facts: Vec<MemoryFactRow>,
pub invalidations: Vec<MemoryInvalidationRow>,
pub lists: Vec<MemoryListRow>,
pub corroborations: Vec<MemoryCorroborationRow>,
pub extractions: Vec<MemoryExtractionRow>,
pub profile_rewrites: Vec<MemoryProfileRewriteRow>,
pub summaries: Vec<MemorySummaryRow>,
pub provenance_identities: Vec<MemoryProvenanceIdentityRow>,
pub fences: Vec<MemoryFenceRow>,
pub unknown: Vec<MemoryUnknownRow>,
}
impl PersonaMemoryFacts {
#[must_use]
pub fn portable_facts(&self) -> Vec<&MemoryFactRow> {
self.facts.iter().filter(|row| row.is_portable()).collect()
}
#[must_use]
pub fn portable_invalidations(&self) -> Vec<&MemoryInvalidationRow> {
self.invalidations
.iter()
.filter(|row| row.is_portable())
.collect()
}
#[must_use]
pub fn portable_lists(&self) -> Vec<&MemoryListRow> {
self.lists.iter().filter(|row| row.portable).collect()
}
}
const KIND_MEMORY_FENCE_BARRIER: &str = "memory_fence_barrier";
const KNOWN_KINDS: [&str; 7] = [
kinds::MEMORY_ADDED,
kinds::MEMORY_INVALIDATED,
kinds::MEMORY_CORROBORATED,
kinds::PROFILE_DOC_UPDATED,
kinds::CONVERSATION_SUMMARY,
kinds::MEMORY_EXTRACTED,
KIND_MEMORY_FENCE_BARRIER,
];
#[allow(clippy::too_many_lines)]
pub fn fold_persona_memory(
events: &[(u64, Event)],
) -> Result<PersonaMemoryFacts, PersonaMemoryFoldError> {
let mut seen = std::collections::BTreeSet::new();
for (position, _) in events {
if !seen.insert(*position) {
return Err(PersonaMemoryFoldError::RepeatedPosition {
position: *position,
});
}
}
let mut fact_scopes: BTreeMap<String, FactScope> = BTreeMap::new();
let mut facts = Vec::new();
let mut invalidations = Vec::new();
let mut lists = Vec::new();
let mut corroborations = Vec::new();
let mut extractions = Vec::new();
let mut profile_rewrites = Vec::new();
let mut summaries = Vec::new();
let mut provenance_identities = Vec::new();
let mut fences = Vec::new();
let mut unknown = Vec::new();
for (position, event) in events {
let position = *position;
let base = kinds::base(&event.kind);
if !KNOWN_KINDS.contains(&base) {
unknown.push(MemoryUnknownRow {
position,
kind: event.kind.clone(),
});
continue;
}
if base == kinds::MEMORY_ADDED {
let added = decode::<MemoryAddedEvent>(position, "memory_added", &event.payload)?;
let provenance = added.provenance.as_option();
let scope = FactScope::of(added.scope);
fact_scopes.insert(added.fact_id.clone(), scope);
if let Some(identity) = provenance_identity(provenance) {
provenance_identities.push(MemoryProvenanceIdentityRow {
position,
record_kind: "fact",
record_id: added.fact_id.clone(),
identity,
});
}
for (index, entity) in added.entities.iter().enumerate() {
lists.push(MemoryListRow {
position,
owner_kind: "fact",
owner_id: added.fact_id.clone(),
list_kind: "entities",
index: index as u64,
text_value: entity.clone(),
portable: matches!(scope, FactScope::Portable),
});
}
facts.push(MemoryFactRow {
position,
fact_id: added.fact_id,
text: added.text,
entities: added.entities,
provenance: provenance_triple(provenance),
event_time_ms: added.event_time_ms,
ingested_at_ms: added.ingested_at_ms,
valid_from_ms: added.valid_from_ms,
confidence_bps: added.confidence_bps,
scope,
durability: durability_str(added.durability),
});
} else if base == kinds::MEMORY_INVALIDATED {
let invalidated =
decode::<MemoryInvalidatedEvent>(position, "memory_invalidated", &event.payload)?;
let provenance = invalidated.provenance.as_option();
if let Some(identity) = provenance_identity(provenance) {
provenance_identities.push(MemoryProvenanceIdentityRow {
position,
record_kind: "invalidation",
record_id: invalidated.fact_id.clone(),
identity,
});
}
let fact_scope = fact_scopes
.get(&invalidated.fact_id)
.copied()
.unwrap_or(FactScope::Unspecified);
invalidations.push(MemoryInvalidationRow {
position,
fact_id: invalidated.fact_id,
invalid_at_ms: invalidated.invalid_at_ms,
reason: invalidated.reason,
superseded_by_fact_id: invalidated.superseded_by_fact_id,
provenance: provenance_triple(provenance),
fact_scope,
});
} else if base == kinds::MEMORY_CORROBORATED {
let corroborated =
decode::<MemoryCorroboratedEvent>(position, "memory_corroborated", &event.payload)?;
for (index, corroborator) in corroborated.corroborating.iter().enumerate() {
lists.push(MemoryListRow {
position,
owner_kind: "corroboration",
owner_id: corroborated.fact_id.clone(),
list_kind: "corroborators",
index: index as u64,
text_value: corroborator.conversation_id.clone(),
portable: false,
});
}
corroborations.push(MemoryCorroborationRow {
position,
fact_id: corroborated.fact_id,
origin: provenance_triple(corroborated.origin.as_option()),
at_ms: corroborated.at_ms,
});
} else if base == kinds::PROFILE_DOC_UPDATED {
let rewrite =
decode::<ProfileDocUpdatedEvent>(position, "profile_doc_updated", &event.payload)?;
for (index, fact_id) in rewrite.source_fact_ids.iter().enumerate() {
lists.push(MemoryListRow {
position,
owner_kind: "profile_rewrite",
owner_id: rewrite.updated_at_ms.to_string(),
list_kind: "source_fact_ids",
index: index as u64,
text_value: fact_id.clone(),
portable: false,
});
}
profile_rewrites.push(MemoryProfileRewriteRow {
position,
document: rewrite.document,
updated_at_ms: rewrite.updated_at_ms,
source_fact_ids: rewrite.source_fact_ids,
});
} else if base == kinds::CONVERSATION_SUMMARY {
let summary = decode::<ConversationSummaryEvent>(
position,
"conversation_summary",
&event.payload,
)?;
summaries.push(MemorySummaryRow {
position,
conversation_id: summary.conversation_id,
summary_text: summary.summary_text,
event_time_ms: summary.event_time_ms,
scope: FactScope::of(summary.scope).as_str(),
});
} else if base == kinds::MEMORY_EXTRACTED {
let extracted =
decode::<MemoryExtractedEvent>(position, "memory_extracted", &event.payload)?;
extractions.push(MemoryExtractionRow {
position,
conversation_id: extracted.conversation_id,
turn_exec_id: extracted.turn_exec_id,
extracted_at_ms: extracted.extracted_at_ms,
facts_added: extracted.facts_added,
facts_invalidated: extracted.facts_invalidated,
});
} else {
debug_assert_eq!(base, KIND_MEMORY_FENCE_BARRIER);
fences.push(MemoryFenceRow { position });
}
}
Ok(PersonaMemoryFacts {
facts,
invalidations,
lists,
corroborations,
extractions,
profile_rewrites,
summaries,
provenance_identities,
fences,
unknown,
})
}
fn decode<T: buffa::Message + Default>(
position: u64,
kind: &'static str,
payload: &[u8],
) -> Result<T, PersonaMemoryFoldError> {
if payload.is_empty() {
return Ok(T::default());
}
try_decode_event_payload::<T>(payload).map_err(|error| {
PersonaMemoryFoldError::UndecodableRecord {
position,
kind,
reason: error.to_string(),
}
})
}
#[cfg(test)]
mod tests {
#![allow(clippy::pedantic, clippy::nursery, missing_docs, clippy::unwrap_used)]
use buffa::Message as _;
use polyc_proto::proto::polychrome::persona::v1::ExternalIdentity;
use super::*;
fn provenance(conversation_id: &str, turn_exec_id: &str) -> MemoryProvenance {
MemoryProvenance {
conversation_id: conversation_id.to_owned(),
turn_exec_id: turn_exec_id.to_owned(),
identity: buffa::MessageField::some(ExternalIdentity {
provider: "slack".to_owned(),
external_id: "U123".to_owned(),
display_name: "Alex".to_owned(),
..Default::default()
}),
..Default::default()
}
}
fn added(fact_id: &str, scope: MemoryScope, entities: Vec<String>) -> Event {
let event = MemoryAddedEvent {
fact_id: fact_id.to_owned(),
text: "likes tea".to_owned(),
entities,
provenance: buffa::MessageField::some(provenance(
"conv-a",
"01950000-0000-7000-8000-00000000aaaa",
)),
event_time_ms: 1,
ingested_at_ms: 2,
valid_from_ms: 1,
confidence_bps: 9000,
scope: buffa::EnumValue::from(scope),
durability: buffa::EnumValue::from(MemoryDurability::Durable),
..Default::default()
};
Event::new(kinds::MEMORY_ADDED, event.encode_to_vec())
}
fn invalidated(fact_id: &str) -> Event {
let event = MemoryInvalidatedEvent {
fact_id: fact_id.to_owned(),
invalid_at_ms: 5,
reason: "user-corrected".to_owned(),
provenance: buffa::MessageField::some(provenance(
"conv-a",
"01950000-0000-7000-8000-00000000aaaa",
)),
..Default::default()
};
Event::new(kinds::MEMORY_INVALIDATED, event.encode_to_vec())
}
#[test]
fn a_direct_fact_never_reaches_the_portable_tables() {
let events = vec![(1, added("f1", MemoryScope::Direct, vec!["tea".to_owned()]))];
let facts = fold_persona_memory(&events).unwrap();
assert_eq!(facts.facts.len(), 1);
assert_eq!(facts.facts[0].scope(), SCOPE_DIRECT);
assert!(facts.portable_facts().is_empty());
assert!(facts.portable_lists().is_empty());
}
#[test]
fn a_portable_fact_is_duplicated_into_the_portable_table_and_no_other() {
let events = vec![(
1,
added("f1", MemoryScope::Portable, vec!["tea".to_owned()]),
)];
let facts = fold_persona_memory(&events).unwrap();
assert_eq!(facts.facts.len(), 1);
let portable = facts.portable_facts();
assert_eq!(portable.len(), 1);
assert_eq!(portable[0].fact_id, "f1");
let portable_lists = facts.portable_lists();
assert_eq!(portable_lists.len(), 1);
assert_eq!(portable_lists[0].text_value, "tea");
}
#[test]
fn an_invalidation_of_a_portable_fact_is_also_portable() {
let events = vec![
(1, added("f1", MemoryScope::Portable, vec![])),
(2, invalidated("f1")),
];
let facts = fold_persona_memory(&events).unwrap();
assert_eq!(facts.portable_invalidations().len(), 1);
}
#[test]
fn an_invalidation_of_an_unresolved_fact_is_never_portable() {
let events = vec![(1, invalidated("ghost"))];
let facts = fold_persona_memory(&events).unwrap();
assert_eq!(facts.invalidations.len(), 1);
assert_eq!(facts.invalidations[0].fact_scope(), SCOPE_UNSPECIFIED);
assert!(facts.portable_invalidations().is_empty());
}
#[test]
fn an_invalidation_of_a_direct_fact_stays_owner_only() {
let events = vec![
(1, added("f1", MemoryScope::Direct, vec![])),
(2, invalidated("f1")),
];
let facts = fold_persona_memory(&events).unwrap();
assert!(facts.portable_invalidations().is_empty());
}
#[test]
fn provenance_identity_never_reaches_a_fact_or_invalidation_row() {
let events = vec![(1, added("f1", MemoryScope::Portable, vec![]))];
let facts = fold_persona_memory(&events).unwrap();
assert_eq!(facts.provenance_identities.len(), 1);
assert_eq!(facts.provenance_identities[0].identity.provider, "slack");
let _: MemoryFactRow = facts.facts.into_iter().next().unwrap();
}
#[test]
fn an_unrecognized_kind_folds_to_memory_unknown_not_an_error() {
let events = vec![(1, Event::new("some_future_kind", Vec::new()))];
let facts = fold_persona_memory(&events).unwrap();
assert_eq!(facts.unknown.len(), 1);
assert_eq!(facts.unknown[0].kind, "some_future_kind");
}
#[test]
fn a_fence_barrier_folds_with_no_payload() {
let events = vec![(1, Event::new(KIND_MEMORY_FENCE_BARRIER, Vec::new()))];
let facts = fold_persona_memory(&events).unwrap();
assert_eq!(facts.fences.len(), 1);
}
#[test]
fn a_repeated_position_refuses_the_fold() {
let events = vec![
(1, added("f1", MemoryScope::Direct, vec![])),
(1, invalidated("f1")),
];
assert!(matches!(
fold_persona_memory(&events).unwrap_err(),
PersonaMemoryFoldError::RepeatedPosition { position: 1 }
));
}
#[test]
fn an_undecodable_known_kind_payload_refuses_the_fold() {
let events = vec![(1, Event::new(kinds::MEMORY_ADDED, vec![0xff; 8]))];
let error = fold_persona_memory(&events).unwrap_err();
assert!(matches!(
error,
PersonaMemoryFoldError::UndecodableRecord { position: 1, .. }
));
}
#[test]
fn the_turn_exec_id_normalizes_to_the_conversation_partitions_simple_hex_form() {
let events = vec![(1, added("f1", MemoryScope::Direct, vec![]))];
let facts = fold_persona_memory(&events).unwrap();
assert_eq!(
facts.facts[0].provenance.turn_id,
"0195000000007000800000000000aaaa"
);
assert_eq!(
facts.facts[0].provenance.turn_exec_id,
"01950000-0000-7000-8000-00000000aaaa"
);
}
#[test]
fn the_same_prefix_folds_to_the_same_rows() {
let events = vec![(
1,
added("f1", MemoryScope::Portable, vec!["tea".to_owned()]),
)];
assert_eq!(
fold_persona_memory(&events).unwrap(),
fold_persona_memory(&events).unwrap()
);
}
}