second-brain-adapters 0.13.4

Conversation parsers for second-brain: Claude Code and Cursor JSONL adapters
Documentation
use std::collections::HashSet;

use anyhow::Result;
use second_brain_core::embedding::DocumentEmbedder;
use second_brain_core::redact::{redact_for_store, redact_memory};
use second_brain_core::schema::{Conversation, Entity, Memory, Relation, RelationType};
use second_brain_core::store::{Store, StoreAdmin};
use uuid::Uuid;

use crate::IngestResult;

/// The store surface `store_ingest_result` writes through. Every
/// `StoreAdmin` has it through the blanket impl below; tests implement it
/// directly to inject failing writes.
pub trait IngestStore {
    fn store_memory(&self, memory: &Memory) -> Result<()>;
    fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
    fn store_relation(&self, relation: &Relation) -> Result<()>;
    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
    /// True when it wrote. A store that cannot check existence writes.
    fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
        self.store_memory(memory)?;
        Ok(true)
    }
    /// True when it wrote. A store that cannot check existence writes.
    fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
        self.store_conversation(conversation)?;
        Ok(true)
    }
    /// Whether a memory holds this id in any state or it is tombstoned.
    fn memory_id_taken(&self, _id: Uuid) -> Result<bool> {
        Ok(false)
    }
    /// Whether a delete removed this memory's text from its source.
    fn memory_blocked(&self, _memory: &Memory) -> Result<bool> {
        Ok(false)
    }
}

impl<T: StoreAdmin + ?Sized> IngestStore for T {
    fn store_memory(&self, memory: &Memory) -> Result<()> {
        Store::store_memory(self, memory)
    }
    fn store_conversation(&self, conversation: &Conversation) -> Result<()> {
        Store::store_conversation(self, conversation)
    }
    fn store_relation(&self, relation: &Relation) -> Result<()> {
        Store::store_relation(self, relation)
    }
    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity> {
        StoreAdmin::find_or_create_entity(self, name, entity_type)
    }
    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool> {
        StoreAdmin::memory_content_exists(self, content_prefix)
    }
    fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
        StoreAdmin::ingest_memory_if_absent(self, memory)
    }
    fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
        StoreAdmin::store_conversation_if_absent(self, conversation)
    }
    fn memory_id_taken(&self, id: Uuid) -> Result<bool> {
        StoreAdmin::memory_id_taken(self, id)
    }
    fn memory_blocked(&self, memory: &Memory) -> Result<bool> {
        StoreAdmin::ingest_blocked(self, memory)
    }
}

#[derive(Debug, Default, PartialEq, Eq)]
pub struct StoreOutcome {
    pub stored: usize,
    pub dupes: usize,
    pub entities: usize,
    pub errors: usize,
}

/// Persists one adapter result: every memory whose id is not already stored
/// or tombstoned, whose text no delete removed from its source, and whose
/// first 128 characters are not already stored
/// (embedded first when an embedder is given), its conversation once one of
/// them lands and only if absent, a derived_from edge per stored memory, and
/// a mentions edge per entity whose memory actually landed. Write failures
/// are counted in the outcome rather than returned, so one bad row does not
/// cost the rest.
pub fn store_ingest_result<S: IngestStore + ?Sized>(
    store: &S,
    embedder: Option<&dyn DocumentEmbedder>,
    result: &IngestResult,
    machine_id: &str,
) -> StoreOutcome {
    store_result(store, embedder, result, machine_id, ContentDedup::Prefix)
}

/// `store_ingest_result` for versioned documents, whose memories carry ids
/// derived from their content: a memory is skipped only when its id is
/// taken, since an edited document keeps most of its opening text and must
/// still land as a new version.
pub fn store_document_result<S: IngestStore + ?Sized>(
    store: &S,
    embedder: Option<&dyn DocumentEmbedder>,
    result: &IngestResult,
    machine_id: &str,
) -> StoreOutcome {
    store_result(store, embedder, result, machine_id, ContentDedup::IdOnly)
}

#[derive(Clone, Copy, PartialEq, Eq)]
enum ContentDedup {
    Prefix,
    IdOnly,
}

fn store_result<S: IngestStore + ?Sized>(
    store: &S,
    embedder: Option<&dyn DocumentEmbedder>,
    result: &IngestResult,
    machine_id: &str,
    dedup: ContentDedup,
) -> StoreOutcome {
    let mut outcome = StoreOutcome::default();

    if result.memories.is_empty() {
        return outcome;
    }

    // Only memories that actually landed can anchor an edge. A deduped or failed
    // write still has an id in result.entities, and a mentions relation built from
    // it dangles: the local store has no foreign key to reject it, but the sync
    // importer does, so the edge can never replicate.
    let mut stored_ids: HashSet<Uuid> = HashSet::new();
    let mut conversation_written = false;

    for memory in &result.memories {
        match store.memory_id_taken(memory.id) {
            Ok(true) => {
                outcome.dupes += 1;
                continue;
            }
            Ok(false) => {}
            Err(e) => {
                tracing::warn!("ingest: id check failed: {e}");
                outcome.errors += 1;
            }
        }
        let mut mem = memory.clone();
        redact_memory(&mut mem);
        match store.memory_blocked(&mem) {
            Ok(true) => {
                outcome.dupes += 1;
                continue;
            }
            Ok(false) => {}
            Err(e) => {
                tracing::warn!("ingest: block list check failed: {e}");
                outcome.errors += 1;
            }
        }
        if dedup == ContentDedup::Prefix {
            let dedup_prefix: String = mem
                .content
                .chars()
                .take(second_brain_core::store::CONTENT_PREFIX_CHARS)
                .collect();
            match store.memory_content_exists(&dedup_prefix) {
                Ok(true) => {
                    outcome.dupes += 1;
                    continue;
                }
                Ok(false) => {}
                Err(e) => {
                    // store the memory anyway: a duplicate is recoverable, a dropped memory is not
                    tracing::warn!("ingest: dedup check failed: {e}");
                    outcome.errors += 1;
                }
            }
        }

        mem.machine_id = machine_id.to_string();
        if let Some(emb) = embedder {
            match emb.embed_document(&mem.content) {
                Ok(embedding) => mem.embedding = embedding,
                Err(e) => {
                    // stored without an embedding the memory is invisible to vector
                    // recall, so this must be loud
                    tracing::warn!("ingest: embedding failed, storing without vector: {e}");
                    outcome.errors += 1;
                }
            }
        }

        if !conversation_written {
            conversation_written = true;
            if let Err(e) = store.store_conversation_if_absent(&result.conversation) {
                tracing::warn!("ingest: store_conversation failed: {e}");
                outcome.errors += 1;
            }
        }

        match store.store_memory_if_absent(&mem) {
            Ok(false) => outcome.dupes += 1,
            Ok(true) => {
                stored_ids.insert(mem.id);
                let relation = Relation {
                    from_id: mem.id,
                    to_id: result.conversation.id,
                    relation_type: RelationType::DerivedFrom,
                    strength: 1.0,
                    context: None,
                };
                if let Err(e) = store.store_relation(&relation) {
                    tracing::warn!("ingest: store_relation (derived_from) failed: {e}");
                    outcome.errors += 1;
                }
                outcome.stored += 1;
            }
            Err(e) => {
                tracing::warn!("ingest: store_memory failed: {e}");
                outcome.errors += 1;
            }
        }
    }

    for (entity_info, memory_id) in &result.entities {
        if !stored_ids.contains(memory_id) {
            continue;
        }
        let name = redact_for_store(&entity_info.name);
        match store.find_or_create_entity(&name, &entity_info.entity_type) {
            Ok(entity) => {
                let relation = Relation {
                    from_id: *memory_id,
                    to_id: entity.id,
                    relation_type: RelationType::Mentions,
                    strength: 1.0,
                    context: None,
                };
                if let Err(e) = store.store_relation(&relation) {
                    tracing::warn!("ingest: store_relation (mentions) failed: {e}");
                    outcome.errors += 1;
                } else {
                    outcome.entities += 1;
                }
            }
            Err(e) => {
                tracing::warn!("ingest: find_or_create_entity failed for {name}: {e}");
                outcome.errors += 1;
            }
        }
    }

    outcome
}