use std::collections::{HashMap, HashSet};
use anyhow::Context as _;
use rusqlite::{OptionalExtension as _, Transaction, params};
use crate::engine::memory::entity::{extract_entities, normalize_entity_value};
use crate::engine::memory::external_id;
use crate::engine::memory::extract::{
ExtractedMemory, KnownClaim, PendingSources, SourceCandidate, validate_candidate,
};
use crate::engine::memory::types::{RelationshipKind, RememberedMemory, SupersededMemory};
use super::util::{claim_id, display_id_in_tx, memory_is_tombstoned, memory_type_and_text};
pub(super) fn ensure_project(tx: &Transaction<'_>, project: &str, now: i64) -> anyhow::Result<i64> {
tx.execute(
"INSERT OR IGNORE INTO projects(path, created_at) VALUES (?1, ?2)",
params![project, now],
)?;
Ok(tx.query_row(
"SELECT id FROM projects WHERE path = ?1",
params![project],
|row| row.get(0),
)?)
}
pub(super) struct InsertMemoriesResult {
pub(super) claims_added: usize,
pub(super) claims_superseded: usize,
pub(super) evidence_added: usize,
pub(super) relations_added: usize,
pub(super) duplicates_merged: usize,
pub(super) contradictions_found: usize,
pub(super) added: Vec<RememberedMemory>,
pub(super) superseded: Vec<SupersededMemory>,
}
pub(super) fn insert_claims(
tx: &Transaction<'_>,
pending: &PendingSources,
evidence_ids: &HashMap<String, i64>,
extracted: Vec<ExtractedMemory>,
known: &[KnownClaim],
project_id: i64,
now: i64,
) -> anyhow::Result<InsertMemoriesResult> {
let source_by_prompt: HashMap<&str, &SourceCandidate> = pending
.evidence
.iter()
.map(|source| (source.prompt_id.as_str(), source))
.collect();
let known_by_prompt = known
.iter()
.map(|claim| (claim.prompt_id.as_str(), claim))
.collect::<HashMap<_, _>>();
let mut result = InsertMemoriesResult {
claims_added: 0,
claims_superseded: 0,
evidence_added: 0,
relations_added: 0,
duplicates_merged: 0,
contradictions_found: 0,
added: Vec::new(),
superseded: Vec::new(),
};
let mut seen_claims = HashSet::new();
for candidate in extracted {
let candidate = match validate_candidate(candidate, &source_by_prompt, &known_by_prompt) {
Ok(candidate) => candidate,
Err(error) => {
eprintln!("goosedump: warning: ignored invalid extracted memory: {error}");
continue;
}
};
let generated_id = claim_id(&pending.project, candidate.memory_type, &candidate.text);
if memory_is_tombstoned(tx, &generated_id)? {
continue;
}
let id = existing_claim_id(tx, project_id, &candidate)?.unwrap_or(generated_id);
let first_occurrence = seen_claims.insert(id.clone());
let valid_from = candidate
.evidence_ids
.iter()
.filter_map(|prompt_id| source_by_prompt.get(prompt_id.as_str()))
.map(|source| source.observed_at)
.max();
let inserted = if first_occurrence {
insert_memory_row(tx, &id, project_id, &candidate, valid_from, now)?
} else {
0
};
result.claims_added += inserted;
result.evidence_added += link_claim_evidence(tx, &id, &candidate, evidence_ids)?;
let evidence_texts = candidate
.evidence_ids
.iter()
.filter_map(|prompt_id| source_by_prompt.get(prompt_id.as_str()))
.map(|source| source.extraction_text.as_str())
.collect::<Vec<_>>();
attach_entities(
tx,
project_id,
&id,
&candidate.text,
&candidate.keywords,
&evidence_texts,
)?;
if inserted == 0 {
reactivate_reasserted_claim(tx, &id, &candidate, valid_from.unwrap_or(now))?;
}
let applied = apply_relationships(
tx,
&id,
&candidate,
&known_by_prompt,
valid_from.unwrap_or(now),
now,
)?;
record_relationship_results(tx, &mut result, &id, &candidate, &applied)?;
if inserted > 0 {
result.added.push(RememberedMemory {
id: external_id(&id),
display_id: display_id_in_tx(tx, &id)?,
memory_type: candidate.memory_type,
text: candidate.text,
supersedes: applied
.superseded
.iter()
.map(|related| external_id(related))
.collect(),
});
}
}
Ok(result)
}
fn existing_claim_id(
tx: &Transaction<'_>,
project_id: i64,
candidate: &ExtractedMemory,
) -> anyhow::Result<Option<String>> {
Ok(tx
.query_row(
"SELECT id FROM claims
WHERE project_id = ?1 AND memory_type = ?2 AND statement = ?3 AND status = 'active'
ORDER BY created_at, id
LIMIT 1",
params![project_id, candidate.memory_type.as_str(), candidate.text],
|row| row.get(0),
)
.optional()?)
}
pub(super) fn insert_memory_row(
tx: &Transaction<'_>,
id: &str,
project_id: i64,
candidate: &ExtractedMemory,
valid_from: Option<i64>,
now: i64,
) -> anyhow::Result<usize> {
Ok(tx.execute(
"INSERT OR IGNORE INTO claims(
id, project_id, memory_type, statement, keywords, status,
valid_from, valid_until, created_at
) VALUES (?1, ?2, ?3, ?4, ?5, 'active', ?6, NULL, ?7)",
params![
id,
project_id,
candidate.memory_type.as_str(),
candidate.text,
candidate.keywords.join("\n"),
valid_from,
now,
],
)?)
}
pub(super) fn reactivate_reasserted_claim(
tx: &Transaction<'_>,
id: &str,
candidate: &ExtractedMemory,
valid_from: i64,
) -> anyhow::Result<()> {
let is_reasserted = candidate
.relationships
.iter()
.any(|relationship| relationship.kind != RelationshipKind::Duplicates);
if is_reasserted {
tx.execute(
"UPDATE claims
SET status = 'active', superseded_by = NULL, valid_from = ?2, valid_until = NULL
WHERE id = ?1 AND status = 'superseded'",
params![id, valid_from],
)?;
}
Ok(())
}
pub(super) fn link_claim_evidence(
tx: &Transaction<'_>,
id: &str,
candidate: &ExtractedMemory,
evidence_ids: &HashMap<String, i64>,
) -> anyhow::Result<usize> {
let mut evidence_added = 0;
for prompt_id in &candidate.evidence_ids {
if let Some(evidence_id) = evidence_ids.get(prompt_id) {
evidence_added += tx.execute(
"INSERT OR IGNORE INTO claim_evidence(claim_id, evidence_id)
VALUES (?1, ?2)",
params![id, evidence_id],
)?;
}
}
Ok(evidence_added)
}
#[derive(Default)]
pub(super) struct AppliedRelationships {
pub(super) superseded: Vec<String>,
pub(super) duplicate_target: Option<String>,
pub(super) evidence_added: usize,
pub(super) relations_added: usize,
pub(super) contradictions_found: usize,
}
pub(super) fn record_relationship_results(
tx: &Transaction<'_>,
result: &mut InsertMemoriesResult,
claim_id: &str,
candidate: &ExtractedMemory,
applied: &AppliedRelationships,
) -> anyhow::Result<()> {
result.claims_superseded +=
applied.superseded.len() + usize::from(applied.duplicate_target.is_some());
result.evidence_added += applied.evidence_added;
result.relations_added += applied.relations_added;
result.duplicates_merged += usize::from(applied.duplicate_target.is_some());
result.contradictions_found += applied.contradictions_found;
for related_id in &applied.superseded {
let (memory_type, text) = memory_type_and_text(tx, related_id)?;
result.superseded.push(SupersededMemory {
id: external_id(related_id),
display_id: display_id_in_tx(tx, related_id)?,
memory_type,
text,
superseded_by: external_id(claim_id),
});
}
if let Some(target) = applied.duplicate_target.as_deref() {
result.superseded.push(SupersededMemory {
id: external_id(claim_id),
display_id: display_id_in_tx(tx, claim_id)?,
memory_type: candidate.memory_type,
text: candidate.text.clone(),
superseded_by: external_id(target),
});
}
Ok(())
}
pub(super) fn apply_relationships(
tx: &Transaction<'_>,
claim_id: &str,
candidate: &ExtractedMemory,
known: &HashMap<&str, &KnownClaim>,
valid_from: i64,
now: i64,
) -> anyhow::Result<AppliedRelationships> {
let mut applied = AppliedRelationships {
superseded: Vec::new(),
duplicate_target: None,
evidence_added: 0,
relations_added: 0,
contradictions_found: 0,
};
for relationship in &candidate.relationships {
let related = known
.get(relationship.claim_id.as_str())
.context("validated memory relationship disappeared")?;
if related.id == claim_id {
continue;
}
let inserted = tx.execute(
"INSERT OR IGNORE INTO relations(
subject_claim_id, predicate, object_claim_id, rationale, created_at
) VALUES (?1, ?2, ?3, ?4, ?5)",
params![
claim_id,
relationship.kind.as_str(),
related.id,
relationship.rationale,
now,
],
)?;
applied.relations_added += inserted;
match relationship.kind {
RelationshipKind::Duplicates => {
if applied.duplicate_target.is_none() {
let changed = tx.execute(
"UPDATE claims
SET status = 'superseded', superseded_by = ?1, valid_until = ?2
WHERE id = ?3 AND status = 'active'",
params![related.id, valid_from, claim_id],
)?;
if changed > 0 {
applied.evidence_added += tx.execute(
"INSERT OR IGNORE INTO claim_evidence(claim_id, evidence_id)
SELECT ?1, evidence_id FROM claim_evidence WHERE claim_id = ?2",
params![related.id, claim_id],
)?;
applied.duplicate_target = Some(related.id.clone());
}
}
}
RelationshipKind::Revises => {
let changed = tx.execute(
"UPDATE claims
SET status = 'superseded', superseded_by = ?1, valid_until = ?2
WHERE id = ?3 AND status = 'active'",
params![claim_id, valid_from, related.id],
)?;
if changed > 0 {
applied.superseded.push(related.id.clone());
}
}
RelationshipKind::Contradicts => {
applied.contradictions_found += inserted;
}
RelationshipKind::Supports => {}
}
}
Ok(applied)
}
pub(super) fn attach_entities(
tx: &Transaction<'_>,
project_id: i64,
claim_id: &str,
statement: &str,
keywords: &[String],
evidence_texts: &[&str],
) -> anyhow::Result<()> {
let candidates = extract_entities(statement, keywords, evidence_texts);
for candidate in candidates {
let entity_id = ensure_entity(tx, project_id, candidate.kind, &candidate.value)?;
tx.execute(
"INSERT OR IGNORE INTO claim_entities(claim_id, entity_id, origin)
VALUES (?1, ?2, ?3)",
params![claim_id, entity_id, candidate.origin],
)?;
}
Ok(())
}
pub(super) fn ensure_entity(
tx: &Transaction<'_>,
project_id: i64,
kind: &str,
value: &str,
) -> anyhow::Result<i64> {
let normalized = normalize_entity_value(value);
tx.execute(
"INSERT OR IGNORE INTO entities(project_id, kind, value, normalized)
VALUES (?1, ?2, ?3, ?4)",
params![project_id, kind, value, normalized],
)?;
Ok(tx.query_row(
"SELECT id FROM entities
WHERE project_id = ?1 AND kind = ?2 AND normalized = ?3",
params![project_id, kind, normalized],
|row| row.get(0),
)?)
}