second-brain-api 0.10.0

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::sync::Arc;

use anyhow::Result;

use second_brain_adapters::claude_code::ClaudeCodeAdapter;
use second_brain_adapters::claude_memory::ClaudeMemoryAdapter;
use second_brain_adapters::cursor::CursorAdapter;
use second_brain_adapters::{Adapter, ConversationSource, store_ingest_result};
use second_brain_core::embedding::{DocumentEmbedder, Embedder};
use second_brain_core::store::StoreAdmin;

pub struct IngestStats {
    pub new_memories: usize,
    pub new_conversations: usize,
    pub new_entities: usize,
    pub skipped_dupes: usize,
    pub unchanged: usize,
    pub errors: usize,
    pub files_processed: usize,
}

/// Every adapter an ingest can run, by the `source` name it stores.
pub const SOURCES: [&str; 3] = ["claude-code", "claude-memory", "cursor"];

fn adapter_named(source: &str, machine_id: &str) -> Option<Box<dyn Adapter>> {
    let machine_id = machine_id.to_string();
    match source {
        "claude-code" => Some(Box::new(ClaudeCodeAdapter::new(machine_id))),
        "claude-memory" => Some(Box::new(ClaudeMemoryAdapter::new(machine_id))),
        "cursor" => Some(Box::new(CursorAdapter::new(machine_id))),
        _ => None,
    }
}

/// The adapters an ingest of `source` runs: the one named, or all of them
/// when no source is given. An unknown name is the caller's mistake and is
/// answered with a message listing the sources that exist.
pub fn adapters_for(
    source: Option<&str>,
    machine_id: &str,
) -> Result<Vec<Box<dyn Adapter>>, String> {
    match source {
        None => Ok(SOURCES
            .iter()
            .filter_map(|name| adapter_named(name, machine_id))
            .collect()),
        Some(name) => adapter_named(name, machine_id)
            .map(|a| vec![a])
            .ok_or_else(|| {
                format!(
                    "unknown ingest source '{name}'; valid values: {}",
                    SOURCES.join(", ")
                )
            }),
    }
}

pub fn run_ingest(
    store: &Arc<dyn StoreAdmin>,
    embedder: Option<&Embedder>,
    machine_id: &str,
    source: Option<&str>,
) -> Result<IngestStats> {
    let adapters = adapters_for(source, machine_id).map_err(anyhow::Error::msg)?;
    let embedder = embedder.map(|e| e as &dyn DocumentEmbedder);

    let mut total = IngestStats {
        new_memories: 0,
        new_conversations: 0,
        new_entities: 0,
        skipped_dupes: 0,
        unchanged: 0,
        errors: 0,
        files_processed: 0,
    };

    for adapter in &adapters {
        let sources = match adapter.discover_conversations() {
            Ok(s) => s,
            Err(e) => {
                tracing::warn!("ingest: {} discovery failed: {e}", adapter.name());
                continue;
            }
        };

        let source_name = adapter.name().to_string();
        total.files_processed += sources.len();
        ingest_sources(
            adapter.as_ref(),
            &sources,
            store.as_ref(),
            embedder,
            &source_name,
            machine_id,
            &mut total,
        );
    }

    Ok(total)
}

pub fn ingest_file(
    store: &Arc<dyn StoreAdmin>,
    embedder: Option<&Embedder>,
    path: &std::path::Path,
    machine_id: &str,
) {
    let path_str = path.to_string_lossy().to_string();

    let hash = match file_hash(path) {
        Some(h) => h,
        None => return,
    };

    if let Ok(false) = store.is_file_changed(&path_str, &hash) {
        return;
    }

    let adapter = ClaudeCodeAdapter::new(machine_id.to_string());
    let source = ConversationSource {
        id: path_str.clone(),
        path: path_str.clone(),
        source_name: "claude-code".to_string(),
        machine_id: machine_id.to_string(),
    };

    let result = match adapter.ingest_conversation(&source) {
        Ok(r) => r,
        Err(e) => {
            tracing::debug!("watcher: failed to parse {}: {e}", path_str);
            return;
        }
    };

    let outcome = store_ingest_result(
        store.as_ref(),
        embedder.map(|e| e as &dyn DocumentEmbedder),
        &result,
        machine_id,
    );

    if let Err(e) = store.mark_ingested(&path_str, &hash, outcome.stored, "claude-code") {
        tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
    }
    if outcome.errors > 0 {
        tracing::warn!(
            "watcher: ingested {} with {} write errors ({} memories stored)",
            path_str,
            outcome.errors,
            outcome.stored
        );
    } else if outcome.stored > 0 {
        tracing::info!(
            "watcher: ingested {} ({} memories)",
            path_str,
            outcome.stored
        );
    }
}

pub fn ingest_memory_file(
    store: &Arc<dyn StoreAdmin>,
    embedder: Option<&Embedder>,
    path: &std::path::Path,
    machine_id: &str,
) {
    let path_str = path.to_string_lossy().to_string();

    let hash = match file_hash(path) {
        Some(h) => h,
        None => return,
    };

    if let Ok(false) = store.is_file_changed(&path_str, &hash) {
        return;
    }

    // remove the previous memory for this file before re-ingesting so stale
    // content does not accumulate alongside the updated version
    if let Err(e) = store.delete_memories_by_source_id(&path_str) {
        tracing::warn!(
            "watcher: delete_memories_by_source_id failed for {}: {e}",
            path_str
        );
    }

    let adapter = ClaudeMemoryAdapter::new(machine_id.to_string());
    let source = ConversationSource {
        id: path_str.clone(),
        path: path_str.clone(),
        source_name: "claude-memory".to_string(),
        machine_id: machine_id.to_string(),
    };

    let result = match adapter.ingest_conversation(&source) {
        Ok(r) => r,
        Err(e) => {
            tracing::debug!("watcher: failed to parse memory file {}: {e}", path_str);
            return;
        }
    };

    let outcome = store_ingest_result(
        store.as_ref(),
        embedder.map(|e| e as &dyn DocumentEmbedder),
        &result,
        machine_id,
    );

    if let Err(e) = store.mark_ingested(&path_str, &hash, outcome.stored, "claude-memory") {
        tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
    }
    if outcome.stored > 0 || outcome.errors > 0 {
        tracing::info!(
            "watcher: ingested memory file {} ({} memories, {} errors)",
            path_str,
            outcome.stored,
            outcome.errors
        );
    }
}

fn ingest_sources(
    adapter: &dyn Adapter,
    sources: &[ConversationSource],
    store: &dyn StoreAdmin,
    embedder: Option<&dyn DocumentEmbedder>,
    source_name: &str,
    machine_id: &str,
    stats: &mut IngestStats,
) {
    for source in sources {
        let path = std::path::Path::new(&source.path);
        let hash = match file_hash(path) {
            Some(h) => h,
            None => {
                stats.errors += 1;
                continue;
            }
        };

        if let Ok(false) = store.is_file_changed(&source.path, &hash) {
            stats.unchanged += 1;
            continue;
        }

        let result = match adapter.ingest_conversation(source) {
            Ok(r) => r,
            Err(e) => {
                tracing::warn!("ingest: failed to parse {}: {e}", source.path);
                stats.errors += 1;
                continue;
            }
        };

        let outcome = store_ingest_result(store, embedder, &result, machine_id);
        if !result.memories.is_empty() {
            stats.new_conversations += 1;
        }
        stats.new_memories += outcome.stored;
        stats.skipped_dupes += outcome.dupes;
        stats.new_entities += outcome.entities;
        stats.errors += outcome.errors;

        if let Err(e) = store.mark_ingested(&source.path, &hash, outcome.stored, source_name) {
            tracing::warn!("ingest: mark_ingested failed for {}: {e}", source.path);
            stats.errors += 1;
        }
    }
}

fn file_hash(path: &std::path::Path) -> Option<String> {
    use std::hash::Hasher;
    use std::io::Read;
    let mut file = std::fs::File::open(path).ok()?;
    let mut hasher = std::collections::hash_map::DefaultHasher::new();
    let mut buf = [0u8; 8192];
    loop {
        let n = file.read(&mut buf).ok()?;
        if n == 0 {
            break;
        }
        hasher.write(&buf[..n]);
    }
    Some(format!("{:016x}", hasher.finish()))
}

#[cfg(test)]
mod tests {
    use super::*;

    fn names(source: Option<&str>) -> Result<Vec<String>, String> {
        adapters_for(source, "m")
            .map(|adapters| adapters.iter().map(|a| a.name().to_string()).collect())
    }

    #[test]
    fn a_named_source_runs_only_that_adapter() {
        for source in SOURCES {
            assert_eq!(names(Some(source)).unwrap(), vec![source.to_string()]);
        }
    }

    #[test]
    fn no_source_runs_every_adapter() {
        assert_eq!(names(None).unwrap(), SOURCES.map(str::to_string).to_vec());
    }

    #[test]
    fn an_unknown_source_is_refused_with_the_valid_names() {
        let error = names(Some("bogus")).unwrap_err();
        assert!(error.contains("bogus"), "{error}");
        for source in SOURCES {
            assert!(error.contains(source), "{error}");
        }
    }
}