use std::collections::{BTreeMap, HashMap};
use std::str::FromStr;
use serde_json::Value;
use tracing::debug;
use crate::dedup::FUZZY_RECENT_POOL_SIZE;
use crate::error::{MemdError, Result};
use crate::store::metadata::MetadataStore;
use crate::store::persistent::PersistentStore;
use crate::store::supersession::{canonicalize_for_type, is_near_duplicate};
use crate::types::lifecycle::{LifecycleDelta, MemoryTier};
use crate::types::{
ChunkId, ChunkStatus, ChunkType, IngestionMode, MemoryChunk, ProjectId, TenantId,
};
use super::time::now_utc_ms;
#[allow(unused_imports)]
use super::{validate_omf, OmfDocument, OmfItem, MEMD_EXT_VERSION, MEMD_SOURCE_APP};
#[derive(Debug, Clone)]
pub struct ImportOptions {
pub include_archived: bool,
pub fuzzy_threshold: Option<f32>,
}
impl Default for ImportOptions {
fn default() -> Self {
Self {
include_archived: true,
fuzzy_threshold: None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ImportResult {
pub total: usize,
pub imported: usize,
pub duplicates: usize,
pub skipped: usize,
}
#[derive(Debug, Clone)]
pub struct ImportedChunk {
pub chunk_id: ChunkId,
pub chunk_type: ChunkType,
pub project_id: Option<String>,
pub text: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct PreviewResult {
pub total: usize,
pub to_import: usize,
pub duplicates: usize,
pub filtered: usize,
pub unscoped: usize,
pub by_project: BTreeMap<String, usize>,
}
pub async fn import_omf(
store: &PersistentStore,
tenant_id: &TenantId,
doc: &OmfDocument,
opts: ImportOptions,
) -> Result<ImportResult> {
let (result, _events) = import_omf_with_events(store, tenant_id, doc, opts).await?;
Ok(result)
}
pub async fn import_omf_with_events(
store: &PersistentStore,
tenant_id: &TenantId,
doc: &OmfDocument,
opts: ImportOptions,
) -> Result<(ImportResult, Vec<ImportedChunk>)> {
validate_omf(doc)?;
let trusted = is_trusted(doc);
if trusted {
validate_trusted_supersession_invariants(doc)?;
}
let mut result = ImportResult {
total: doc.memories.len(),
imported: 0,
duplicates: 0,
skipped: 0,
};
let mut events: Vec<ImportedChunk> = Vec::new();
let mut source_to_target: HashMap<ChunkId, ChunkId> = HashMap::new();
let mut pending_edges: Vec<PendingSupersede> = Vec::new();
for item in &doc.memories {
if !opts.include_archived
&& matches!(item.status.as_deref(), Some("archived") | Some("expired"))
{
result.skipped += 1;
continue;
}
let mut project_id = extract_project_id(&item.extensions).or_else(|| item.category.clone());
if ProjectId::validate_opt(project_id.as_deref()).is_err() {
project_id = None;
}
let chunk_type = extract_chunk_type(&item.extensions).unwrap_or(ChunkType::Doc);
let canonical = canonicalize_for_type(&item.content, chunk_type);
if is_exact_duplicate(store, tenant_id, project_id.as_deref(), &canonical)? {
result.duplicates += 1;
continue;
}
if let Some(thr) = opts.fuzzy_threshold {
if is_fuzzy_duplicate(store, tenant_id, project_id.as_deref(), &canonical, thr)? {
result.duplicates += 1;
continue;
}
}
let trusted_item = trusted && ext_version_supported(&item.extensions);
let mut initial = if trusted_item {
extract_lifecycle_strict(&item.extensions)?
} else {
LifecycleDelta::default()
};
let ingestion_mode =
extract_ingestion_mode(&item.extensions).unwrap_or(IngestionMode::Document);
let prepared =
crate::write_service::prepare_write(crate::write_service::PrepareWriteRequest {
chunk_type,
text: &item.content,
tags: &item.tags,
ingestion_mode,
expires_at_ms: initial.expires_at_ms.flatten(),
review_after_ms: initial.review_after_ms.flatten(),
});
if prepared.is_rejected() {
result.skipped += 1;
continue;
}
let prepared_lifecycle = prepared.lifecycle_delta();
if initial.tier.is_none() {
initial.tier = prepared_lifecycle.tier;
}
if initial.expires_at_ms.is_none() {
initial.expires_at_ms = prepared_lifecycle.expires_at_ms;
}
if initial.review_after_ms.is_none() {
initial.review_after_ms = prepared_lifecycle.review_after_ms;
}
let mut chunk = MemoryChunk::new(tenant_id.clone(), item.content.clone(), chunk_type)
.with_ingestion_mode(ingestion_mode);
if let Some(ref p) = project_id {
chunk = chunk.with_project(ProjectId::new(Some(p.clone())));
}
chunk = prepared.apply_to_chunk(chunk);
let text = chunk.text.clone();
let chunk_id = store.add_chunk_with_lifecycle(chunk, initial).await?;
if trusted_item {
if let Some(src_id) = extract_source_chunk_id(&item.extensions)? {
source_to_target.insert(src_id, chunk_id.clone());
}
if let Some(supersedes_src) = extract_supersession_ref(&item.extensions, "supersedes")?
{
pending_edges.push(PendingSupersede {
old_source_id: supersedes_src,
new_target_id: chunk_id.clone(),
});
}
}
events.push(ImportedChunk {
chunk_id,
chunk_type,
project_id,
text,
});
result.imported += 1;
}
replay_supersession_edges(store, tenant_id, &source_to_target, &pending_edges)?;
Ok((result, events))
}
#[derive(Debug)]
struct PendingSupersede {
old_source_id: ChunkId,
new_target_id: ChunkId,
}
fn validate_trusted_supersession_invariants(doc: &OmfDocument) -> Result<()> {
use std::collections::HashSet;
let mut seen_source_ids: HashSet<ChunkId> = HashSet::new();
let mut seen_supersedes: HashSet<ChunkId> = HashSet::new();
for (i, item) in doc.memories.iter().enumerate() {
if !ext_version_supported(&item.extensions) {
continue;
}
if let Some(src_id) = extract_source_chunk_id(&item.extensions)? {
if !seen_source_ids.insert(src_id.clone()) {
return Err(MemdError::ValidationError(format!(
"memories[{i}].extensions.memd.chunk_id duplicates an earlier item: {src_id}"
)));
}
}
if let Some(old_src) = extract_supersession_ref(&item.extensions, "supersedes")? {
if !seen_supersedes.insert(old_src.clone()) {
return Err(MemdError::ValidationError(format!(
"memories[{i}].extensions.memd.lifecycle.supersedes forks: {old_src} is named by multiple successors"
)));
}
}
let _ = extract_supersession_ref(&item.extensions, "superseded_by")?;
}
Ok(())
}
fn replay_supersession_edges(
store: &PersistentStore,
tenant_id: &TenantId,
source_to_target: &HashMap<ChunkId, ChunkId>,
pending: &[PendingSupersede],
) -> Result<()> {
if pending.is_empty() {
return Ok(());
}
let now_ms = now_utc_ms();
for edge in pending {
let Some(old_target) = source_to_target.get(&edge.old_source_id) else {
debug!(
tenant_id = %tenant_id,
old_source_id = %edge.old_source_id,
new_target_id = %edge.new_target_id,
"import_omf: supersession edge dropped — sibling chunk not in document"
);
continue;
};
store
.metadata()
.atomic_supersede(tenant_id, old_target, &edge.new_target_id, now_ms)?;
}
Ok(())
}
pub async fn preview_omf_import(
store: &PersistentStore,
tenant_id: &TenantId,
doc: &OmfDocument,
opts: ImportOptions,
) -> Result<PreviewResult> {
validate_omf(doc)?;
if is_trusted(doc) {
validate_trusted_supersession_invariants(doc)?;
}
let mut result = PreviewResult {
total: doc.memories.len(),
..Default::default()
};
for item in &doc.memories {
if !opts.include_archived
&& matches!(item.status.as_deref(), Some("archived") | Some("expired"))
{
result.filtered += 1;
continue;
}
let mut project_id = extract_project_id(&item.extensions).or_else(|| item.category.clone());
if ProjectId::validate_opt(project_id.as_deref()).is_err() {
project_id = None;
}
let chunk_type = extract_chunk_type(&item.extensions).unwrap_or(ChunkType::Doc);
let canonical = canonicalize_for_type(&item.content, chunk_type);
if is_exact_duplicate(store, tenant_id, project_id.as_deref(), &canonical)? {
result.duplicates += 1;
continue;
}
if let Some(thr) = opts.fuzzy_threshold {
if is_fuzzy_duplicate(store, tenant_id, project_id.as_deref(), &canonical, thr)? {
result.duplicates += 1;
continue;
}
}
let trusted = is_trusted(doc) && ext_version_supported(&item.extensions);
let initial = if trusted {
extract_lifecycle_strict(&item.extensions)?
} else {
LifecycleDelta::default()
};
let ingestion_mode =
extract_ingestion_mode(&item.extensions).unwrap_or(IngestionMode::Document);
let prepared =
crate::write_service::prepare_write(crate::write_service::PrepareWriteRequest {
chunk_type,
text: &item.content,
tags: &item.tags,
ingestion_mode,
expires_at_ms: initial.expires_at_ms.flatten(),
review_after_ms: initial.review_after_ms.flatten(),
});
if prepared.is_rejected() {
result.filtered += 1;
continue;
}
result.to_import += 1;
match project_id {
Some(p) => *result.by_project.entry(p).or_default() += 1,
None => result.unscoped += 1,
}
}
Ok(result)
}
fn is_trusted(doc: &OmfDocument) -> bool {
doc.source
.as_ref()
.is_some_and(|s| s.app == MEMD_SOURCE_APP)
}
fn is_exact_duplicate(
store: &PersistentStore,
tenant_id: &TenantId,
project_id: Option<&str>,
canonical: &str,
) -> Result<bool> {
let matches = store
.metadata()
.list_by_canonical_text(tenant_id, project_id, canonical)?;
let hit = match project_id {
Some(_) => !matches.is_empty(),
None => matches.iter().any(|m| m.project_id.is_none()),
};
Ok(hit)
}
fn is_fuzzy_duplicate(
store: &PersistentStore,
tenant_id: &TenantId,
project_id: Option<&str>,
canonical: &str,
threshold: f32,
) -> Result<bool> {
let recent = match project_id {
Some(p) => {
store
.metadata()
.list_recent_for_project(tenant_id, Some(p), FUZZY_RECENT_POOL_SIZE)?
}
None => store
.metadata()
.list_recent_with_null_project(tenant_id, FUZZY_RECENT_POOL_SIZE)?,
};
Ok(recent.iter().any(|m| {
m.canonical_text
.as_deref()
.is_some_and(|existing_canon| is_near_duplicate(canonical, existing_canon, threshold))
}))
}
fn extract_project_id(ext: &Value) -> Option<String> {
ext.get("memd")?
.get("project_id")?
.as_str()
.map(|s| s.to_string())
}
fn extract_chunk_type(ext: &Value) -> Option<ChunkType> {
ext.get("memd")?.get("chunk_type")?.as_str()?.parse().ok()
}
fn extract_ingestion_mode(ext: &Value) -> Option<IngestionMode> {
ext.get("memd")?
.get("ingestion_mode")?
.as_str()?
.parse()
.ok()
}
fn ext_version_supported(ext: &Value) -> bool {
ext.get("memd")
.and_then(|m| m.get("v"))
.and_then(|v| v.as_u64())
== Some(MEMD_EXT_VERSION as u64)
}
fn extract_source_chunk_id(ext: &Value) -> Result<Option<ChunkId>> {
let memd = match ext.get("memd") {
Some(v) => v,
None => return Ok(None),
};
match memd.get("chunk_id") {
None | Some(Value::Null) => Ok(None),
Some(Value::String(s)) => ChunkId::parse(s).map(Some),
Some(_) => Err(MemdError::ValidationError(
"extensions.memd.chunk_id must be a string".into(),
)),
}
}
fn extract_supersession_ref(ext: &Value, field: &str) -> Result<Option<ChunkId>> {
let lc = match ext.get("memd").and_then(|m| m.get("lifecycle")) {
Some(v) if v.is_object() => v,
Some(_) => {
return Err(MemdError::ValidationError(
"extensions.memd.lifecycle must be an object".into(),
));
}
None => return Ok(None),
};
match lc.get(field) {
None | Some(Value::Null) => Ok(None),
Some(Value::String(s)) => ChunkId::parse(s).map(Some),
Some(_) => Err(MemdError::ValidationError(format!(
"extensions.memd.lifecycle.{field} must be a string"
))),
}
}
fn extract_lifecycle_strict(ext: &Value) -> Result<LifecycleDelta> {
let lc = match ext.get("memd").and_then(|m| m.get("lifecycle")) {
Some(v) if v.is_object() => v,
Some(_) => {
return Err(MemdError::ValidationError(
"extensions.memd.lifecycle must be an object".into(),
));
}
None => return Ok(LifecycleDelta::default()),
};
let status = parse_optional_string_field(lc, "status", ChunkStatus::from_str)?;
if status == Some(ChunkStatus::Candidate) {
return Err(MemdError::ValidationError(
"extensions.memd.lifecycle.status=candidate requires a local consolidation journal"
.to_string(),
));
}
let tier = parse_optional_string_field(lc, "tier", MemoryTier::from_str)?;
let review_after_ms = parse_optional_i64_field(lc, "review_after_ms")?.map(Some);
let lifecycle_updated_at_ms = parse_optional_i64_field(lc, "lifecycle_updated_at_ms")?;
let expires_at_ms = parse_optional_i64_field(lc, "expires_at_ms")?.map(Some);
Ok(LifecycleDelta {
status,
tier,
supersedes: None,
superseded_by: None,
expires_at_ms,
review_after_ms,
lifecycle_updated_at_ms,
})
}
fn parse_optional_string_field<T, F>(obj: &Value, key: &str, parse: F) -> Result<Option<T>>
where
F: FnOnce(&str) -> Result<T>,
{
match obj.get(key) {
None | Some(Value::Null) => Ok(None),
Some(Value::String(s)) => parse(s).map(Some),
Some(_) => Err(MemdError::ValidationError(format!(
"extensions.memd.lifecycle.{key} must be a string"
))),
}
}
fn parse_optional_i64_field(obj: &Value, key: &str) -> Result<Option<i64>> {
match obj.get(key) {
None | Some(Value::Null) => Ok(None),
Some(v) => v.as_i64().map(Some).ok_or_else(|| {
MemdError::ValidationError(format!(
"extensions.memd.lifecycle.{key} must be an integer"
))
}),
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn is_trusted_requires_source_app_memd() {
let mut doc = OmfDocument {
omf: "1.0".into(),
exported_at: "x".into(),
source: Some(super::super::OmfSource {
app: "nanomem".into(),
}),
memories: vec![],
};
assert!(!is_trusted(&doc));
doc.source = Some(super::super::OmfSource {
app: MEMD_SOURCE_APP.into(),
});
assert!(is_trusted(&doc));
doc.source = None;
assert!(!is_trusted(&doc));
}
#[test]
fn ext_version_supported_matches_exact_u64() {
assert!(ext_version_supported(&json!({"memd": {"v": 1}})));
assert!(!ext_version_supported(&json!({"memd": {"v": 2}})));
assert!(!ext_version_supported(&json!({"memd": {}})));
assert!(!ext_version_supported(&json!({})));
}
#[test]
fn extract_lifecycle_strict_parses_known_fields() {
let ext = json!({
"memd": {
"lifecycle": {
"status": "superseded",
"tier": "working",
"expires_at_ms": 123i64,
"review_after_ms": 456i64,
"lifecycle_updated_at_ms": 789i64,
}
}
});
let delta = extract_lifecycle_strict(&ext).unwrap();
assert_eq!(delta.status, Some(ChunkStatus::Superseded));
assert_eq!(delta.tier, Some(MemoryTier::Working));
assert_eq!(delta.expires_at_ms, Some(Some(123)));
assert_eq!(delta.review_after_ms, Some(Some(456)));
assert_eq!(delta.lifecycle_updated_at_ms, Some(789));
}
#[test]
fn extract_lifecycle_strict_fails_closed_on_malformed_fields() {
let bad_status = json!({"memd": {"lifecycle": {"status": 42}}});
assert!(matches!(
extract_lifecycle_strict(&bad_status),
Err(MemdError::ValidationError(_))
));
let unjournaled_candidate = json!({"memd": {"lifecycle": {"status": "candidate"}}});
assert!(matches!(
extract_lifecycle_strict(&unjournaled_candidate),
Err(MemdError::ValidationError(_))
));
let bad_tier = json!({"memd": {"lifecycle": {"tier": "galaxy_brain"}}});
assert!(matches!(
extract_lifecycle_strict(&bad_tier),
Err(MemdError::ValidationError(_))
));
let bad_ms = json!({"memd": {"lifecycle": {"expires_at_ms": "tomorrow"}}});
assert!(matches!(
extract_lifecycle_strict(&bad_ms),
Err(MemdError::ValidationError(_))
));
let bad_shape = json!({"memd": {"lifecycle": []}});
assert!(matches!(
extract_lifecycle_strict(&bad_shape),
Err(MemdError::ValidationError(_))
));
}
#[test]
fn extract_lifecycle_strict_missing_block_returns_default() {
let no_lifecycle = json!({"memd": {"v": 1}});
let delta = extract_lifecycle_strict(&no_lifecycle).unwrap();
assert!(delta.is_empty());
}
#[test]
fn extract_project_id_and_chunk_type_and_mode_roundtrip() {
let ext = json!({
"memd": {
"project_id": "p1",
"chunk_type": "decision",
"ingestion_mode": "conversation",
}
});
assert_eq!(extract_project_id(&ext), Some("p1".to_string()));
assert_eq!(extract_chunk_type(&ext), Some(ChunkType::Decision));
assert_eq!(
extract_ingestion_mode(&ext),
Some(IngestionMode::Conversation)
);
}
#[test]
fn omf_document_source_app_is_directly_accessible() {
let doc = OmfDocument {
omf: "1.0".into(),
exported_at: "x".into(),
source: Some(super::super::OmfSource {
app: MEMD_SOURCE_APP.into(),
}),
memories: vec![OmfItem {
content: "x".into(),
..Default::default()
}],
};
assert_eq!(doc.source.as_ref().unwrap().app, MEMD_SOURCE_APP);
}
#[test]
fn extract_source_chunk_id_accepts_valid_uuid() {
let id = ChunkId::new();
let ext = json!({ "memd": { "chunk_id": id.to_string() } });
assert_eq!(extract_source_chunk_id(&ext).unwrap(), Some(id));
}
#[test]
fn extract_source_chunk_id_missing_returns_none() {
assert_eq!(extract_source_chunk_id(&json!({})).unwrap(), None);
assert_eq!(
extract_source_chunk_id(&json!({"memd": {"v": 1}})).unwrap(),
None
);
assert_eq!(
extract_source_chunk_id(&json!({"memd": {"chunk_id": null}})).unwrap(),
None
);
}
#[test]
fn extract_source_chunk_id_fails_closed_on_bad_input() {
let bad_type = json!({"memd": {"chunk_id": 42}});
assert!(matches!(
extract_source_chunk_id(&bad_type),
Err(MemdError::ValidationError(_))
));
let bad_uuid = json!({"memd": {"chunk_id": "not-a-uuid"}});
assert!(matches!(
extract_source_chunk_id(&bad_uuid),
Err(MemdError::ValidationError(_))
));
}
#[test]
fn extract_supersession_ref_reads_both_fields() {
let old = ChunkId::new();
let new = ChunkId::new();
let ext = json!({
"memd": {
"lifecycle": {
"supersedes": old.to_string(),
"superseded_by": new.to_string(),
}
}
});
assert_eq!(
extract_supersession_ref(&ext, "supersedes").unwrap(),
Some(old)
);
assert_eq!(
extract_supersession_ref(&ext, "superseded_by").unwrap(),
Some(new)
);
}
#[test]
fn extract_supersession_ref_missing_lifecycle_returns_none() {
assert_eq!(
extract_supersession_ref(&json!({"memd": {"v": 1}}), "supersedes").unwrap(),
None
);
let null_field = json!({"memd": {"lifecycle": {"supersedes": null}}});
assert_eq!(
extract_supersession_ref(&null_field, "supersedes").unwrap(),
None
);
}
#[test]
fn extract_supersession_ref_fails_closed_on_bad_shape() {
let bad = json!({"memd": {"lifecycle": []}});
assert!(matches!(
extract_supersession_ref(&bad, "supersedes"),
Err(MemdError::ValidationError(_))
));
let bad = json!({"memd": {"lifecycle": {"supersedes": 7}}});
assert!(matches!(
extract_supersession_ref(&bad, "supersedes"),
Err(MemdError::ValidationError(_))
));
let bad = json!({"memd": {"lifecycle": {"supersedes": "not-a-uuid"}}});
assert!(matches!(
extract_supersession_ref(&bad, "supersedes"),
Err(MemdError::ValidationError(_))
));
}
}