use std::collections::HashSet;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};
use anyhow::{Context, Result};
use notify::{EventKind, RecursiveMode, Watcher};
use second_brain_adapters::claude_code::ClaudeCodeAdapter;
use second_brain_adapters::cursor::CursorAdapter;
use second_brain_adapters::{Adapter, ConversationSource};
use second_brain_core::embedding::Embedder;
use second_brain_core::kuzu_store::KuzuStore;
use second_brain_core::query::{QueryEngine, QueryFilters, QueryRequest};
use second_brain_core::schema::{Memory, MemoryType, Relation, RelationType};
use second_brain_core::store::Store;
fn main() {
tracing_subscriber::fmt()
.with_writer(std::io::stderr)
.with_env_filter(
tracing_subscriber::EnvFilter::from_default_env()
.add_directive("second_brain=info".parse().unwrap()),
)
.init();
let args: Vec<String> = std::env::args().collect();
let cmd = args.get(1).map(|s| s.as_str()).unwrap_or("help");
if let Err(e) = run(cmd, &args) {
eprintln!("error: {e:#}");
std::process::exit(1);
}
}
fn run(cmd: &str, args: &[String]) -> Result<()> {
match cmd {
"ingest" => {
let source = args.get(2).map(|s| s.as_str()).unwrap_or("claude-code");
cmd_ingest(source)
}
"recall" => {
let query = args.get(2).map(|s| s.as_str()).unwrap_or("");
cmd_recall(query)
}
"remember" => {
let content = args[2..].join(" ");
cmd_remember(&content)
}
"status" => cmd_status(),
"watch" => cmd_watch(),
"sync" => {
let peer = args
.get(2)
.ok_or_else(|| anyhow::anyhow!("usage: sb sync <peer-host>"))?;
cmd_sync(peer, args)
}
"sync-export" => {
let after_seq = parse_after_seq(args);
cmd_sync_export(after_seq)
}
"sync-import" => cmd_sync_import(),
"sync-status" => cmd_sync_status(),
"sync-backfill" => cmd_sync_backfill(),
"reingest" => {
let clean = args.iter().any(|a| a == "--clean");
cmd_reingest(clean)
}
"consolidate" => {
let dry_run = args.iter().any(|a| a == "--dry-run");
let limit = args
.windows(2)
.find(|w| w[0] == "--limit")
.and_then(|w| w[1].parse().ok())
.unwrap_or(50);
let model = args
.windows(2)
.find(|w| w[0] == "--model")
.map(|w| w[1].clone())
.unwrap_or_else(|| "gemma4:latest".to_string());
let ollama_url = args
.windows(2)
.find(|w| w[0] == "--ollama-url")
.map(|w| w[1].clone())
.unwrap_or_else(|| "http://localhost:11434".to_string());
cmd_consolidate(dry_run, limit, &model, &ollama_url)
}
_ => {
println!("second-brain CLI");
println!();
println!("Usage: sb <command> [args]");
println!();
println!("Commands:");
println!(
" ingest [source] Ingest conversation history (claude-code, cursor, all)"
);
println!(" recall <query> Search the brain");
println!(" remember <text> Store new knowledge");
println!(" status Show brain stats");
println!(" watch Watch for new conversations and auto-ingest");
println!(" sync <peer> Bidirectional sync with a peer");
println!(" sync-export Export changes as JSONL to stdout");
println!(" sync-import Import JSONL changes from stdin");
println!(" sync-status Show sync state for all peers");
println!(" sync-backfill Backfill sync log for existing data");
println!(" reingest [--clean] Re-ingest all conversations (--clean removes old memories first)");
println!(" consolidate Distill raw memories via local LLM");
println!(" --dry-run Preview without writing");
println!(" --limit N Process N memories (default: 50)");
println!(" --model NAME Ollama model (default: gemma4:latest)");
println!(" --ollama-url URL Ollama endpoint (default: http://localhost:11434)");
Ok(())
}
}
}
fn open_store() -> Result<Arc<KuzuStore>> {
let db_path = std::env::var("SECOND_BRAIN_DB")
.map(PathBuf::from)
.unwrap_or_else(|_| {
let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
PathBuf::from(home).join(".second-brain").join("graph.kuzu")
});
if let Some(parent) = db_path.parent() {
std::fs::create_dir_all(parent).ok();
}
let store = KuzuStore::open(&db_path, machine_id())
.with_context(|| format!("opening database at {}", db_path.display()))?;
eprintln!("db: {}", db_path.display());
Ok(Arc::new(store))
}
fn load_embedder() -> Option<Embedder> {
eprintln!("loading embedding model...");
match Embedder::new() {
Ok(e) => {
eprintln!("embedder ready (BGE-small-en-v1.5, 384-dim)");
Some(e)
}
Err(e) => {
eprintln!("embedder failed, proceeding without embeddings: {e}");
None
}
}
}
fn machine_id() -> String {
hostname::get()
.map(|h| h.to_string_lossy().to_string())
.unwrap_or_else(|_| "unknown".to_string())
}
fn cmd_ingest(source: &str) -> Result<()> {
let store = open_store()?;
let embedder = load_embedder();
let machine = machine_id();
match source {
"claude-code" => ingest_claude_code(&store, embedder.as_ref(), &machine)?,
"cursor" => ingest_cursor(&store, embedder.as_ref(), &machine)?,
"all" => {
ingest_claude_code(&store, embedder.as_ref(), &machine)?;
ingest_cursor(&store, embedder.as_ref(), &machine)?;
}
other => anyhow::bail!("unknown source: {other}. supported: claude-code, cursor, all"),
}
Ok(())
}
fn file_hash(path: &std::path::Path) -> Result<String> {
use std::io::Read;
let mut file = std::fs::File::open(path)?;
let mut hasher = std::collections::hash_map::DefaultHasher::new();
let mut buf = [0u8; 8192];
use std::hash::Hasher;
loop {
let n = file.read(&mut buf)?;
if n == 0 {
break;
}
hasher.write(&buf[..n]);
}
Ok(format!("{:016x}", hasher.finish()))
}
fn ingest_claude_code(
store: &Arc<KuzuStore>,
embedder: Option<&Embedder>,
machine: &str,
) -> Result<()> {
let adapter = ClaudeCodeAdapter::new(machine.to_string());
let sources = adapter.discover_conversations()?;
eprintln!("discovered {} conversation files", sources.len());
if sources.is_empty() {
eprintln!("nothing to ingest");
return Ok(());
}
let already_ingested = store.ingested_file_count().unwrap_or(0);
eprintln!("already ingested: {} files", already_ingested);
let start = Instant::now();
let mut total_memories = 0usize;
let mut total_entities = 0usize;
let mut total_conversations = 0usize;
let mut total_skipped = 0usize;
let mut total_unchanged = 0usize;
let mut total_errors = 0usize;
let batch_size = 32;
for (i, source) in sources.iter().enumerate() {
let path = std::path::Path::new(&source.path);
let hash = match file_hash(path) {
Ok(h) => h,
Err(_) => {
total_errors += 1;
continue;
}
};
match store.is_file_changed(&source.path, &hash) {
Ok(false) => {
total_unchanged += 1;
continue;
}
Ok(true) => {}
Err(_) => {}
}
let result = match adapter.ingest_conversation(source) {
Ok(r) => r,
Err(e) => {
if total_errors < 5 {
eprintln!(" error parsing {}: {e}", source.path);
}
total_errors += 1;
continue;
}
};
if result.memories.is_empty() {
store
.mark_ingested(&source.path, &hash, 0, "claude-code")
.ok();
total_skipped += 1;
continue;
}
if let Err(e) = store.store_conversation(&result.conversation) {
tracing::debug!("store conversation: {e}");
}
total_conversations += 1;
let mut pending_memories = Vec::new();
let mut pending_texts: Vec<String> = Vec::new();
let mut batch_stored = 0usize;
for memory in &result.memories {
pending_memories.push(memory.clone());
pending_texts.push(memory.content.clone());
if pending_memories.len() >= batch_size {
batch_stored += store_batch(
store,
embedder,
&mut pending_memories,
&pending_texts,
&result.conversation.id,
)?;
pending_texts.clear();
}
}
if !pending_memories.is_empty() {
batch_stored += store_batch(
store,
embedder,
&mut pending_memories,
&pending_texts,
&result.conversation.id,
)?;
}
total_entities += store_entities(store, &result.entities);
store
.mark_ingested(&source.path, &hash, batch_stored, "claude-code")
.ok();
total_memories += batch_stored;
if (i + 1) % 100 == 0 || i + 1 == sources.len() {
let elapsed = start.elapsed().as_secs();
eprintln!(
" [{}/{}] {} memories, {} entities, {} unchanged, {} skipped, {} errors ({}s)",
i + 1,
sources.len(),
total_memories,
total_entities,
total_unchanged,
total_skipped,
total_errors,
elapsed,
);
}
}
let elapsed = start.elapsed();
eprintln!();
eprintln!("ingestion complete in {:.1}s", elapsed.as_secs_f64());
eprintln!(" new conversations: {total_conversations}");
eprintln!(" new memories: {total_memories}");
eprintln!(" new entities: {total_entities}");
eprintln!(" unchanged (skipped): {total_unchanged}");
eprintln!(" no memorable content: {total_skipped}");
eprintln!(" errors: {total_errors}");
Ok(())
}
fn ingest_cursor(store: &Arc<KuzuStore>, embedder: Option<&Embedder>, machine: &str) -> Result<()> {
let adapter = CursorAdapter::new(machine.to_string());
let sources = adapter.discover_conversations()?;
eprintln!("cursor: discovered {} conversation files", sources.len());
if sources.is_empty() {
eprintln!("cursor: nothing to ingest");
return Ok(());
}
let start = Instant::now();
let mut total_memories = 0usize;
let mut total_conversations = 0usize;
let mut total_skipped = 0usize;
let mut total_unchanged = 0usize;
let mut total_errors = 0usize;
let batch_size = 32;
for (i, source) in sources.iter().enumerate() {
let path = std::path::Path::new(&source.path);
let hash = match file_hash(path) {
Ok(h) => h,
Err(_) => {
total_errors += 1;
continue;
}
};
match store.is_file_changed(&source.path, &hash) {
Ok(false) => {
total_unchanged += 1;
continue;
}
Ok(true) => {}
Err(_) => {}
}
let result = match adapter.ingest_conversation(source) {
Ok(r) => r,
Err(e) => {
if total_errors < 5 {
eprintln!(" error parsing {}: {e}", source.path);
}
total_errors += 1;
continue;
}
};
if result.memories.is_empty() {
store.mark_ingested(&source.path, &hash, 0, "cursor").ok();
total_skipped += 1;
continue;
}
if let Err(e) = store.store_conversation(&result.conversation) {
tracing::debug!("store conversation: {e}");
}
total_conversations += 1;
let mut pending_memories = Vec::new();
let mut pending_texts: Vec<String> = Vec::new();
let mut batch_stored = 0usize;
for memory in &result.memories {
pending_memories.push(memory.clone());
pending_texts.push(memory.content.clone());
if pending_memories.len() >= batch_size {
batch_stored += store_batch(
store,
embedder,
&mut pending_memories,
&pending_texts,
&result.conversation.id,
)?;
pending_texts.clear();
}
}
if !pending_memories.is_empty() {
batch_stored += store_batch(
store,
embedder,
&mut pending_memories,
&pending_texts,
&result.conversation.id,
)?;
}
store
.mark_ingested(&source.path, &hash, batch_stored, "cursor")
.ok();
total_memories += batch_stored;
if (i + 1) % 50 == 0 || i + 1 == sources.len() {
let elapsed = start.elapsed().as_secs();
eprintln!(
" cursor [{}/{}] {} new memories, {} unchanged ({}s)",
i + 1,
sources.len(),
total_memories,
total_unchanged,
elapsed,
);
}
}
let elapsed = start.elapsed();
eprintln!();
eprintln!("cursor ingestion complete in {:.1}s", elapsed.as_secs_f64());
eprintln!(" new conversations: {total_conversations}");
eprintln!(" new memories: {total_memories}");
eprintln!(" unchanged (skipped): {total_unchanged}");
eprintln!(" no memorable content: {total_skipped}");
eprintln!(" errors: {total_errors}");
Ok(())
}
fn store_batch(
store: &Arc<KuzuStore>,
embedder: Option<&Embedder>,
memories: &mut Vec<second_brain_core::schema::Memory>,
texts: &[String],
conversation_id: &uuid::Uuid,
) -> Result<usize> {
if let Some(emb) = embedder {
let refs: Vec<&str> = texts.iter().map(|s| s.as_str()).collect();
match emb.embed_batch(&refs) {
Ok(embeddings) => {
for (mem, embedding) in memories.iter_mut().zip(embeddings) {
mem.embedding = embedding;
}
}
Err(e) => {
tracing::warn!("batch embed failed, storing without vectors: {e}");
}
}
}
let mut stored = 0;
for memory in memories.iter() {
let dedup_prefix: String = memory.content.chars().take(128).collect();
if store.memory_content_exists(&dedup_prefix).unwrap_or(false) {
continue;
}
if let Err(e) = store.store_memory(memory) {
tracing::debug!("store memory: {e}");
continue;
}
let relation = Relation {
from_id: memory.id,
to_id: *conversation_id,
relation_type: RelationType::DerivedFrom,
strength: 1.0,
context: None,
};
store.store_relation(&relation).ok();
stored += 1;
}
memories.clear();
Ok(stored)
}
fn store_entities(
store: &Arc<KuzuStore>,
entities: &[(second_brain_adapters::ExtractedEntity, uuid::Uuid)],
) -> usize {
let mut count = 0;
for (entity_info, memory_id) in entities {
if let Ok(entity) = store.find_or_create_entity(&entity_info.name, &entity_info.entity_type)
{
let relation = Relation {
from_id: *memory_id,
to_id: entity.id,
relation_type: RelationType::Mentions,
strength: 1.0,
context: None,
};
store.store_relation(&relation).ok();
count += 1;
}
}
count
}
fn cmd_reingest(clean: bool) -> Result<()> {
let store = open_store()?;
if clean {
let deleted = store.delete_memories_by_source("claude-code")?;
eprintln!("cleaned {} claude-code memories", deleted);
let deleted = store.delete_memories_by_source("cursor")?;
eprintln!("cleaned {} cursor memories", deleted);
}
let cleared = store.clear_ingest_log()?;
eprintln!("cleared {} ingest log entries", cleared);
eprintln!();
cmd_ingest("all")
}
fn cmd_watch() -> Result<()> {
let store = open_store()?;
let embedder = load_embedder();
let machine = machine_id();
let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
let watch_dir = PathBuf::from(&home).join(".claude").join("projects");
if !watch_dir.exists() {
std::fs::create_dir_all(&watch_dir)
.with_context(|| format!("creating watch directory: {}", watch_dir.display()))?;
}
eprintln!("watching {} for new .jsonl files", watch_dir.display());
eprintln!("press Ctrl+C to stop");
let mut ingested: HashSet<PathBuf> = HashSet::new();
let (tx, rx) = std::sync::mpsc::channel::<PathBuf>();
let mut watcher = notify::recommended_watcher(move |event: notify::Result<notify::Event>| {
let event = match event {
Ok(e) => e,
Err(_) => return,
};
let dominated_by_write = matches!(event.kind, EventKind::Create(_) | EventKind::Modify(_));
if !dominated_by_write {
return;
}
for path in event.paths {
if path.extension().is_some_and(|e| e == "jsonl") {
tx.send(path).ok();
}
}
})
.context("initializing filesystem watcher")?;
watcher
.watch(&watch_dir, RecursiveMode::Recursive)
.with_context(|| format!("watching {}", watch_dir.display()))?;
let settle = Duration::from_secs(5);
let mut pending: std::collections::HashMap<PathBuf, Instant> = std::collections::HashMap::new();
loop {
match rx.recv_timeout(Duration::from_millis(500)) {
Ok(path) => {
pending.insert(path, Instant::now());
}
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {}
Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break,
}
let now = Instant::now();
let ready: Vec<PathBuf> = pending
.iter()
.filter(|(_, ts)| now.duration_since(**ts) >= settle)
.map(|(p, _)| p.clone())
.collect();
for path in ready {
pending.remove(&path);
if ingested.contains(&path) {
continue;
}
match ingest_single_file(&store, embedder.as_ref(), &machine, &path) {
Ok(count) => {
eprintln!("ingested {}: {} memories", path.display(), count);
ingested.insert(path);
}
Err(e) => {
eprintln!("error ingesting {}: {e}", path.display());
}
}
}
}
Ok(())
}
fn ingest_single_file(
store: &Arc<KuzuStore>,
embedder: Option<&Embedder>,
machine: &str,
path: &PathBuf,
) -> Result<usize> {
let source = ConversationSource {
id: path.to_string_lossy().to_string(),
path: path.to_string_lossy().to_string(),
source_name: "claude-code".to_string(),
machine_id: machine.to_string(),
};
let adapter = ClaudeCodeAdapter::new(machine.to_string());
let result = adapter.ingest_conversation(&source)?;
if result.memories.is_empty() {
return Ok(0);
}
if let Err(e) = store.store_conversation(&result.conversation) {
tracing::debug!("store conversation: {e}");
}
let batch_size = 32;
let mut pending_memories = Vec::new();
let mut pending_texts: Vec<String> = Vec::new();
let total = result.memories.len();
for memory in &result.memories {
pending_memories.push(memory.clone());
pending_texts.push(memory.content.clone());
if pending_memories.len() >= batch_size {
store_batch(
store,
embedder,
&mut pending_memories,
&pending_texts,
&result.conversation.id,
)?;
pending_texts.clear();
}
}
if !pending_memories.is_empty() {
store_batch(
store,
embedder,
&mut pending_memories,
&pending_texts,
&result.conversation.id,
)?;
}
Ok(total)
}
fn cmd_recall(query: &str) -> Result<()> {
if query.is_empty() {
anyhow::bail!("usage: sb recall <query>");
}
let store = open_store()?;
let embedder = load_embedder();
let results = if let Some(ref emb) = embedder {
if let Ok(embedding) = emb.embed(query) {
let engine = QueryEngine::new(store.as_ref());
let request = QueryRequest {
text: query.to_string(),
embedding,
limit: 10,
filters: QueryFilters::default(),
};
match engine.recall(&request) {
Ok(results) if !results.is_empty() => results,
other => {
if let Err(e) = &other {
eprintln!("warning: vector recall failed: {e}");
}
let text_results = store.text_search(query, 10)?;
if text_results.is_empty() {
println!("no memories found for: \"{query}\"");
return Ok(());
}
println!("(text search fallback — no vector matches)\n");
for (i, memory) in text_results.iter().enumerate() {
println!(
"{}. [{:?}] (source: {})",
i + 1,
memory.memory_type,
memory.source,
);
let preview: String = memory.content.chars().take(200).collect();
println!(" {preview}\n");
}
return Ok(());
}
}
} else {
eprintln!("warning: embedding failed, falling back to text search");
let text_results = store.text_search(query, 10)?;
if text_results.is_empty() {
println!("no memories found for: \"{query}\"");
return Ok(());
}
for (i, memory) in text_results.iter().enumerate() {
println!(
"{}. [{:?}] (source: {})",
i + 1,
memory.memory_type,
memory.source,
);
let preview: String = memory.content.chars().take(200).collect();
println!(" {preview}\n");
}
return Ok(());
}
} else {
eprintln!("warning: no embedder available, falling back to text search");
let text_results = store.text_search(query, 10)?;
if text_results.is_empty() {
println!("no memories found for: \"{query}\"");
return Ok(());
}
for (i, memory) in text_results.iter().enumerate() {
println!(
"{}. [{:?}] (source: {})",
i + 1,
memory.memory_type,
memory.source,
);
let preview: String = memory.content.chars().take(200).collect();
println!(" {preview}\n");
}
return Ok(());
};
for (i, result) in results.iter().enumerate() {
println!(
"{}. [{:?}] (score: {:.4}, source: {})",
i + 1,
result.memory.memory_type,
result.score,
result.memory.source,
);
let preview: String = result.memory.content.chars().take(200).collect();
println!(" {preview}\n");
}
Ok(())
}
fn cmd_remember(content: &str) -> Result<()> {
if content.is_empty() {
anyhow::bail!("usage: sb remember <text>");
}
let store = open_store()?;
let embedder = load_embedder();
let mut memory = second_brain_core::schema::Memory::new(
content.to_string(),
second_brain_core::schema::MemoryType::Semantic,
"cli".to_string(),
String::new(),
);
if let Some(ref emb) = embedder {
if let Ok(embedding) = emb.embed(content) {
memory.embedding = embedding;
}
}
store.store_memory(&memory)?;
println!("remembered [{}]: {}", memory.id, content);
Ok(())
}
fn cmd_status() -> Result<()> {
let store = open_store()?;
let memories = store.memory_count()?;
let files = store.ingested_file_count().unwrap_or(0);
println!("memories: {memories}");
println!("ingested files: {files}");
println!();
match store.diagnostic() {
Ok(diag) => println!("{diag}"),
Err(e) => eprintln!("diagnostic error: {e}"),
}
Ok(())
}
fn parse_after_seq(args: &[String]) -> u64 {
for (i, arg) in args.iter().enumerate() {
if arg == "--after-seq" {
if let Some(val) = args.get(i + 1) {
return val.parse().unwrap_or(0);
}
}
}
0
}
fn cmd_sync(peer_host: &str, args: &[String]) -> Result<()> {
let api_base = api_base_url();
let mut sb_binary = "sb".to_string();
for (i, arg) in args.iter().enumerate() {
if arg == "--sb-path" {
if let Some(val) = args.get(i + 1) {
sb_binary = val.clone();
}
}
}
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(600))
.build()?;
eprintln!("pulling from {peer_host}...");
let pull_stats = sync_pull_via_daemon(&client, &api_base, peer_host, &sb_binary)?;
eprintln!(
" {} created, {} updated, {} deleted, {} skipped",
pull_stats.created, pull_stats.updated, pull_stats.deleted, pull_stats.skipped
);
eprintln!("pushing to {peer_host}...");
let max_seq = sync_push_via_daemon(&client, &api_base, peer_host, &sb_binary)?;
eprintln!(" pushed through seq {max_seq}");
eprintln!("sync complete.");
Ok(())
}
fn api_base_url() -> String {
std::env::var("SECOND_BRAIN_API").unwrap_or_else(|_| "http://127.0.0.1:7200".to_string())
}
fn sync_pull_via_daemon(
client: &reqwest::blocking::Client,
api_base: &str,
peer_host: &str,
sb_binary: &str,
) -> Result<SyncImportStats> {
#[derive(serde::Deserialize)]
struct PeerInfo {
peers: Vec<PeerState>,
}
#[derive(serde::Deserialize)]
struct PeerState {
peer_id: String,
last_seq: u64,
}
let status: PeerInfo = client
.get(format!("{api_base}/v1/sync/status"))
.send()
.context("getting sync status")?
.json()?;
let after_seq = status
.peers
.iter()
.find(|p| p.peer_id == peer_host)
.map(|p| p.last_seq)
.unwrap_or(0);
let mut child = std::process::Command::new("ssh")
.arg("-o")
.arg("StrictHostKeyChecking=no")
.arg("-o")
.arg("ServerAliveInterval=30")
.arg(peer_host)
.arg(sb_binary)
.arg("sync-export")
.arg("--after-seq")
.arg(after_seq.to_string())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::inherit())
.spawn()
.context("spawning ssh for pull")?;
let stdout = child.stdout.take().unwrap();
let reader = std::io::BufReader::new(stdout);
let mut records = Vec::new();
for line in std::io::BufRead::lines(reader) {
let line = line?;
if line.trim().is_empty() {
continue;
}
let val: serde_json::Value = serde_json::from_str(&line)?;
records.push(val);
}
let status = child.wait()?;
if !status.success() {
anyhow::bail!("ssh pull exited with {}", status);
}
#[derive(serde::Deserialize)]
struct ImportResp {
created: u64,
updated: u64,
deleted: u64,
skipped: u64,
#[allow(dead_code)]
errors: u64,
max_seq: u64,
}
let resp: ImportResp = client
.post(format!("{api_base}/v1/sync/import"))
.json(&serde_json::json!({"records": records}))
.send()
.context("posting import to daemon")?
.error_for_status()
.context("import returned error")?
.json()?;
if resp.max_seq > after_seq {
client
.post(format!("{api_base}/v1/sync/state"))
.json(&serde_json::json!({"peer_id": peer_host, "last_seq": resp.max_seq}))
.send()
.context("updating sync state")?;
}
Ok(SyncImportStats {
created: resp.created,
updated: resp.updated,
deleted: resp.deleted,
skipped: resp.skipped,
})
}
struct SyncImportStats {
created: u64,
updated: u64,
deleted: u64,
skipped: u64,
}
fn sync_push_via_daemon(
client: &reqwest::blocking::Client,
api_base: &str,
peer_host: &str,
sb_binary: &str,
) -> Result<u64> {
let push_peer = format!("push:{}", peer_host);
const BATCH_SIZE: usize = 500;
#[derive(serde::Deserialize)]
struct PeerInfo {
peers: Vec<PeerState>,
}
#[derive(serde::Deserialize)]
struct PeerState {
peer_id: String,
last_seq: u64,
}
let status: PeerInfo = client
.get(format!("{api_base}/v1/sync/status"))
.send()
.context("getting sync status")?
.json()?;
let mut cursor = status
.peers
.iter()
.find(|p| p.peer_id == push_peer)
.map(|p| p.last_seq)
.unwrap_or(0);
#[derive(serde::Deserialize)]
struct ExportResp {
records: Vec<serde_json::Value>,
max_seq: u64,
}
let mut total_pushed = 0u64;
loop {
let export: ExportResp = client
.post(format!("{api_base}/v1/sync/export"))
.json(&serde_json::json!({"after_seq": cursor, "limit": BATCH_SIZE}))
.send()
.context("getting export from daemon")?
.error_for_status()
.context("export returned error")?
.json()?;
if export.records.is_empty() {
break;
}
let batch_count = export.records.len();
let mut child = std::process::Command::new("ssh")
.arg("-o")
.arg("StrictHostKeyChecking=no")
.arg("-o")
.arg("ServerAliveInterval=60")
.arg("-o")
.arg("ServerAliveCountMax=20")
.arg(peer_host)
.arg(sb_binary)
.arg("sync-import")
.stdin(std::process::Stdio::piped())
.stderr(std::process::Stdio::inherit())
.spawn()
.context("spawning ssh for push")?;
{
let stdin = child.stdin.take().unwrap();
let mut writer = std::io::BufWriter::new(stdin);
for record in &export.records {
use std::io::Write;
serde_json::to_writer(&mut writer, record)?;
writer.write_all(b"\n")?;
}
}
let exit = child.wait()?;
if !exit.success() {
anyhow::bail!("ssh push exited with {}", exit);
}
total_pushed += batch_count as u64;
cursor = export.max_seq;
client
.post(format!("{api_base}/v1/sync/state"))
.json(&serde_json::json!({"peer_id": push_peer, "last_seq": cursor}))
.send()
.context("updating push sync state")?;
eprintln!(" batch: {} records (through seq {})", batch_count, cursor);
if batch_count < BATCH_SIZE {
break;
}
}
eprintln!(" total: {} records pushed", total_pushed);
Ok(cursor)
}
fn cmd_sync_export(after_seq: u64) -> Result<()> {
let api_base = api_base_url();
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(120))
.build()?;
#[derive(serde::Deserialize)]
struct ExportResp {
records: Vec<serde_json::Value>,
max_seq: u64,
}
let resp: ExportResp = client
.post(format!("{api_base}/v1/sync/export"))
.json(&serde_json::json!({"after_seq": after_seq}))
.send()
.context("calling sync/export on daemon")?
.error_for_status()
.context("sync/export returned error")?
.json()?;
let stdout = std::io::stdout();
let mut writer = std::io::BufWriter::new(stdout.lock());
for record in &resp.records {
use std::io::Write;
serde_json::to_writer(&mut writer, record)?;
writer.write_all(b"\n")?;
}
eprintln!("exported through seq {}", resp.max_seq);
Ok(())
}
fn cmd_sync_import() -> Result<()> {
let api_base = api_base_url();
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(600))
.build()?;
const BATCH_SIZE: usize = 500;
let stdin = std::io::stdin();
let reader = std::io::BufReader::new(stdin.lock());
#[derive(serde::Deserialize)]
struct ImportResp {
created: u64,
updated: u64,
deleted: u64,
skipped: u64,
errors: u64,
max_seq: u64,
}
let mut totals = (0u64, 0u64, 0u64, 0u64, 0u64, 0u64);
let mut batch: Vec<serde_json::Value> = Vec::with_capacity(BATCH_SIZE);
for line in std::io::BufRead::lines(reader) {
let line = line?;
if line.trim().is_empty() {
continue;
}
batch.push(serde_json::from_str(&line)?);
if batch.len() >= BATCH_SIZE {
let resp: ImportResp = client
.post(format!("{api_base}/v1/sync/import"))
.json(&serde_json::json!({"records": batch}))
.send()
.context("calling sync/import on daemon")?
.error_for_status()
.context("sync/import returned error")?
.json()?;
totals.0 += resp.created;
totals.1 += resp.updated;
totals.2 += resp.deleted;
totals.3 += resp.skipped;
totals.4 += resp.errors;
totals.5 = resp.max_seq;
batch = Vec::with_capacity(BATCH_SIZE);
}
}
if !batch.is_empty() {
let resp: ImportResp = client
.post(format!("{api_base}/v1/sync/import"))
.json(&serde_json::json!({"records": batch}))
.send()
.context("calling sync/import on daemon")?
.error_for_status()
.context("sync/import returned error")?
.json()?;
totals.0 += resp.created;
totals.1 += resp.updated;
totals.2 += resp.deleted;
totals.3 += resp.skipped;
totals.4 += resp.errors;
totals.5 = resp.max_seq;
}
eprintln!(
"imported: {} created, {} updated, {} deleted, {} skipped, {} errors (through seq {})",
totals.0, totals.1, totals.2, totals.3, totals.4, totals.5
);
Ok(())
}
fn cmd_sync_status() -> Result<()> {
let api_base = api_base_url();
let client = reqwest::blocking::Client::new();
#[derive(serde::Deserialize)]
struct PeerInfo {
peers: Vec<PeerState>,
}
#[derive(serde::Deserialize)]
struct PeerState {
peer_id: String,
last_seq: u64,
last_sync_at: String,
}
let resp: PeerInfo = client
.get(format!("{api_base}/v1/sync/status"))
.send()
.context("getting sync status from daemon")?
.error_for_status()?
.json()?;
if resp.peers.is_empty() {
println!("no sync peers configured");
} else {
for state in &resp.peers {
println!(
"peer: {}, last_seq: {}, last_sync: {}",
state.peer_id, state.last_seq, state.last_sync_at
);
}
}
Ok(())
}
fn cmd_sync_backfill() -> Result<()> {
let api_base = api_base_url();
let client = reqwest::blocking::Client::new();
#[derive(serde::Deserialize)]
struct BackfillResp {
backfilled: usize,
}
let resp: BackfillResp = client
.post(format!("{api_base}/v1/sync/backfill"))
.send()
.context("calling backfill on daemon")?
.error_for_status()?
.json()?;
eprintln!("backfilled {} sync log entries", resp.backfilled);
Ok(())
}
fn cmd_consolidate(dry_run: bool, limit: usize, model: &str, ollama_url: &str) -> Result<()> {
let store = open_store()?;
let embedder = load_embedder();
let consolidated = store.consolidation_count()?;
let total = store.memory_count()?;
eprintln!("consolidated: {consolidated}/{total} memories");
let memories = store.unconsolidated_memories(limit)?;
if memories.is_empty() {
eprintln!("nothing to consolidate");
return Ok(());
}
eprintln!("processing {} memories with {model}", memories.len());
let rt = tokio::runtime::Runtime::new()?;
let generate_url = format!("{ollama_url}/api/generate");
let mut success = 0;
let mut skipped = 0;
let mut errors = 0;
for (i, memory) in memories.iter().enumerate() {
let prompt = build_consolidation_prompt(&memory.content);
let response = match rt.block_on(call_ollama(&generate_url, model, &prompt)) {
Ok(r) => r,
Err(e) => {
eprintln!(" [{}/{}] error: {e}", i + 1, memories.len());
errors += 1;
continue;
}
};
let distilled = response.trim().to_string();
if distilled.is_empty() || distilled.len() < 20 {
eprintln!(" [{}/{}] skipped (empty/too short)", i + 1, memories.len());
skipped += 1;
store.mark_consolidated(memory.id, memory.id, model).ok();
continue;
}
if dry_run {
println!(
"--- [{}/{}] raw ({} chars) ---",
i + 1,
memories.len(),
memory.content.len()
);
let preview: String = memory.content.chars().take(100).collect();
println!(" {preview}...");
println!("--- distilled ({} chars) ---", distilled.len());
println!(" {distilled}");
println!();
success += 1;
continue;
}
let mut distilled_memory = Memory::new(
distilled.clone(),
MemoryType::Semantic,
"consolidation".to_string(),
memory.id.to_string(),
);
if let Some(ref emb) = embedder {
if let Ok(embedding) = emb.embed(&distilled) {
distilled_memory.embedding = embedding;
}
}
if let Err(e) = store.store_memory(&distilled_memory) {
eprintln!(" [{}/{}] store error: {e}", i + 1, memories.len());
errors += 1;
continue;
}
let link = Relation {
from_id: distilled_memory.id,
to_id: memory.id,
relation_type: RelationType::DistilledFrom,
strength: 1.0,
context: Some(model.to_string()),
};
store.store_relation(&link).ok();
store.mark_consolidated(memory.id, distilled_memory.id, model)?;
success += 1;
if (i + 1) % 10 == 0 {
eprintln!(
" [{}/{}] {success} distilled, {skipped} skipped, {errors} errors",
i + 1,
memories.len()
);
}
}
eprintln!();
eprintln!("consolidation complete: {success} distilled, {skipped} skipped, {errors} errors");
if dry_run {
eprintln!("(dry run — nothing written)");
}
Ok(())
}
fn build_consolidation_prompt(raw_content: &str) -> String {
format!(
r#"Extract the key knowledge facts from this conversation fragment. Return ONLY the distilled facts as a concise paragraph. No preamble, no explanation, no bullet points unless the content is a list of items.
Rules:
- Keep technical details exact (names, versions, commands, file paths, config values)
- Remove conversational noise ("Let me check...", "Perfect!", "Now let me...")
- Preserve decisions and their reasoning
- Preserve architecture choices and trade-offs
- If the fragment contains no extractable knowledge, respond with just "SKIP"
- Keep it under 200 words
Fragment:
{raw_content}"#
)
}
async fn call_ollama(url: &str, model: &str, prompt: &str) -> Result<String> {
let client = reqwest::Client::new();
let body = serde_json::json!({
"model": model,
"prompt": prompt,
"stream": false,
"options": {
"temperature": 0.1,
"num_predict": 512
}
});
let resp = client
.post(url)
.json(&body)
.timeout(std::time::Duration::from_secs(120))
.send()
.await
.context("connecting to Ollama")?;
if !resp.status().is_success() {
anyhow::bail!("Ollama returned {}", resp.status());
}
let json: serde_json::Value = resp.json().await.context("parsing Ollama response")?;
Ok(json["response"].as_str().unwrap_or("").to_string())
}