use std::path::PathBuf;
use anyhow::{Context, Result};
mod consolidate;
mod tokens;
fn main() {
tracing_subscriber::fmt()
.with_writer(std::io::stderr)
.with_env_filter(
tracing_subscriber::EnvFilter::from_default_env()
.add_directive("second_brain=info".parse().unwrap()),
)
.init();
let args: Vec<String> = std::env::args().collect();
let cmd = args.get(1).map(|s| s.as_str()).unwrap_or("help");
if let Err(e) = run(cmd, &args) {
eprintln!("error: {e:#}");
std::process::exit(1);
}
}
fn run(cmd: &str, args: &[String]) -> Result<()> {
match cmd {
"ingest" => {
let source = args.get(2).map(|s| s.as_str()).unwrap_or("claude-code");
cmd_ingest(source)
}
"recall" => {
let query = args.get(2).map(|s| s.as_str()).unwrap_or("");
cmd_recall(query)
}
"remember" => {
let content = args[2..].join(" ");
cmd_remember(&content)
}
"status" => cmd_status(),
"whoami" => cmd_whoami(),
"setup" => cmd_setup(),
"watch" => {
eprintln!("watch is deprecated — the daemon handles file watching automatically");
Ok(())
}
"sync" => {
let peer = args
.get(2)
.ok_or_else(|| anyhow::anyhow!("usage: sb sync <peer-host>"))?;
cmd_sync(peer, args)
}
"sync-export" => {
let after_seq = parse_after_seq(args);
cmd_sync_export(after_seq)
}
"sync-import" => cmd_sync_import(),
"sync-status" => cmd_sync_status(),
"sync-key" => cmd_sync_key(),
"sync-backfill" => cmd_sync_backfill(),
"backfill-session-dates" => {
let dry_run = args.iter().any(|a| a == "--dry-run");
let reemit = args.iter().any(|a| a == "--reemit");
cmd_backfill_session_dates(dry_run, reemit)
}
"sync-compact" => {
let dry_run = args.iter().any(|a| a == "--dry-run");
let ops: Vec<String> = args
.windows(2)
.find(|w| w[0] == "--ops")
.map(|w| w[1].split(',').map(|s| s.trim().to_string()).collect())
.unwrap_or_else(|| vec!["delete".to_string()]);
cmd_sync_compact(&ops, dry_run)
}
"prune-relations" => cmd_prune_relations(),
"doctor" => {
let repair = args.iter().any(|a| a == "--repair");
let dry_run = args.iter().any(|a| a == "--dry-run");
cmd_doctor(repair, dry_run)
}
"vacuum" => cmd_vacuum(),
"sync-trust" => {
let host = args
.get(2)
.ok_or_else(|| anyhow::anyhow!("usage: sb sync-trust <host>"))?;
cmd_sync_trust(host)
}
"start" => cmd_daemon("start"),
"stop" => cmd_daemon("stop"),
"restart" => cmd_daemon("restart"),
"reingest" => {
let clean = args.iter().any(|a| a == "--clean");
cmd_reingest(clean)
}
"consolidate" => {
let dry_run = args.iter().any(|a| a == "--dry-run");
let limit = args
.windows(2)
.find(|w| w[0] == "--limit")
.and_then(|w| w[1].parse().ok())
.unwrap_or(50);
let model = args
.windows(2)
.find(|w| w[0] == "--model")
.map(|w| w[1].clone())
.unwrap_or_else(|| "gemma-4-E4B-it-Q4_K_M".to_string());
let llm_url = args
.windows(2)
.find(|w| w[0] == "--llm-url")
.map(|w| w[1].clone())
.unwrap_or_else(|| "http://localhost:8081".to_string());
let concurrency: usize = args
.windows(2)
.find(|w| w[0] == "--concurrency")
.and_then(|w| w[1].parse().ok())
.unwrap_or(3);
let guard = args
.windows(2)
.find(|w| w[0] == "--guard")
.map(|w| consolidate::Guard::parse(&w[1]))
.transpose()?
.unwrap_or(consolidate::Guard::Retry);
consolidate::cmd_consolidate(dry_run, limit, &model, &llm_url, concurrency, guard)
}
"daily-pipeline" => {
let llm_url = args
.windows(2)
.find(|w| w[0] == "--llm-url")
.map(|w| w[1].clone())
.unwrap_or_else(|| "http://localhost:8081".to_string());
let model = args
.windows(2)
.find(|w| w[0] == "--model")
.map(|w| w[1].clone())
.unwrap_or_else(|| "gemma-4-E4B-it-Q4_K_M".to_string());
let consolidate_limit: usize = args
.windows(2)
.find(|w| w[0] == "--consolidate-limit")
.and_then(|w| w[1].parse().ok())
.unwrap_or(200);
let enhance = args.iter().any(|a| a == "--enhance");
cmd_daily_pipeline(&llm_url, &model, consolidate_limit, enhance)
}
_ => {
println!("second-brain CLI");
println!();
println!("Usage: sb <command> [args]");
println!();
println!("Commands:");
println!(
" ingest [source] Ingest conversation history (claude-code, cursor, all)"
);
println!(" recall <query> Search the brain");
println!(" remember <text> Store new knowledge");
println!(
" setup One-command install: systemd service, MCP config, first ingest"
);
println!(" start Start the daemon");
println!(" stop Stop the daemon");
println!(" restart Restart the daemon");
println!(" status Show brain stats");
println!(" sync <peer> Bidirectional sync with a peer");
println!(" sync-export Export changes as JSONL to stdout");
println!(" sync-import Import JSONL changes from stdin");
println!(" sync-status Show sync state for all peers");
println!(" sync-key Print the base64 sync key to copy to paired devices");
println!(" sync-backfill Backfill sync log for existing data");
println!(
" backfill-session-dates Date memories from this machine's transcripts (--dry-run, --reemit)"
);
println!(
" sync-compact Collapse duplicate sync_log rows (--dry-run, --ops delete,create)"
);
println!(" prune-relations Delete relations whose endpoints no longer resolve");
println!(
" doctor Report store integrity (--repair to fix, --dry-run to preview)"
);
println!(" vacuum Reclaim free pages in brain.db after a repair");
println!(
" sync-trust <host> Fetch and pin a host's SSH key after fingerprint review"
);
println!(
" reingest [--clean] Re-ingest all conversations (--clean removes old memories first)"
);
println!(" consolidate Distill raw memories via local LLM");
println!(" daily-pipeline End-of-day: consolidate → daily → export → enhance");
Ok(())
}
}
}
fn cmd_ingest(_source: &str) -> Result<()> {
let api = api_base_url();
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(600))
.build()?;
eprintln!("ingesting via daemon...");
let resp = client
.post(format!("{api}/v1/ingest"))
.json(&serde_json::json!({}))
.send()?;
if !resp.status().is_success() {
anyhow::bail!(
"ingest failed: {} {}",
resp.status(),
resp.text().unwrap_or_default()
);
}
let body: serde_json::Value = resp.json()?;
eprintln!(
"ingested: {} memories, {} entities, {} conversations ({} dupes, {} unchanged, {} errors)",
body["new_memories"],
body["new_entities"],
body["new_conversations"],
body["skipped_dupes"],
body["unchanged"],
body["errors"],
);
Ok(())
}
fn cmd_reingest(clean: bool) -> Result<()> {
let api = api_base_url();
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(600))
.build()?;
eprintln!("reingest via daemon (clean={clean})...");
let resp = client
.post(format!("{api}/v1/reingest"))
.json(&serde_json::json!({"clean": clean}))
.send()?;
if !resp.status().is_success() {
anyhow::bail!(
"reingest failed: {} {}",
resp.status(),
resp.text().unwrap_or_default()
);
}
let body: serde_json::Value = resp.json()?;
if clean {
eprintln!(
"cleaned {} claude-code + {} cursor memories",
body["cleaned_claude"], body["cleaned_cursor"]
);
}
eprintln!("cleared {} ingest log entries", body["cleared_log"]);
eprintln!(
"ingested: {} memories, {} entities, {} conversations ({} dupes, {} unchanged, {} errors)",
body["new_memories"],
body["new_entities"],
body["new_conversations"],
body["skipped_dupes"],
body["unchanged"],
body["errors"],
);
Ok(())
}
fn cmd_recall(query: &str) -> Result<()> {
if query.is_empty() {
anyhow::bail!("usage: sb recall <query>");
}
let api = api_base_url();
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(30))
.build()?;
let resp = client
.post(format!("{api}/v1/recall"))
.json(&serde_json::json!({ "query": query, "limit": 10 }))
.send()
.context("daemon not reachable — is it running?")?
.error_for_status()
.context("recall request failed")?;
let body: serde_json::Value = resp.json()?;
let memories = body["memories"].as_array();
if memories.is_none() || memories.unwrap().is_empty() {
println!("no memories found for: \"{query}\"");
return Ok(());
}
for (i, m) in memories.unwrap().iter().enumerate() {
let score = m["score"]
.as_f64()
.map(|s| format!("score: {:.4}, ", s))
.unwrap_or_default();
let mtype = m["memory_type"].as_str().unwrap_or("unknown");
let source = m["source"].as_str().unwrap_or("unknown");
let machine = m["machine_name"].as_str().unwrap_or("");
let project = m["project_path"].as_str().unwrap_or("");
let content = m["content"].as_str().unwrap_or("");
let preview: String = content.chars().take(200).collect();
let mut meta = format!("{score}type: {mtype}, source: {source}");
if !project.is_empty() {
meta.push_str(&format!(", project: {project}"));
}
if !machine.is_empty() {
meta.push_str(&format!(", machine: {machine}"));
}
println!("{}. ({meta})", i + 1);
println!(" {preview}\n");
}
Ok(())
}
fn cmd_remember(content: &str) -> Result<()> {
if content.is_empty() {
anyhow::bail!("usage: sb remember <text>");
}
let api = api_base_url();
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(30))
.build()?;
let resp = client
.post(format!("{api}/v1/remember"))
.json(&serde_json::json!({
"content": content,
"source": "cli",
}))
.send()
.context("daemon not reachable — is it running?")?
.error_for_status()
.context("remember request failed")?;
let body: serde_json::Value = resp.json()?;
let id = body["id"].as_str().unwrap_or("unknown");
println!("remembered [{id}]: {content}");
Ok(())
}
fn cmd_daemon(action: &str) -> Result<()> {
let status = std::process::Command::new("systemctl")
.args(["--user", action, "second-brain"])
.status()
.context("failed to run systemctl")?;
if !status.success() {
anyhow::bail!("systemctl --user {action} second-brain failed");
}
Ok(())
}
fn cmd_status() -> Result<()> {
let api = api_base_url();
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(5))
.build()?;
let resp = client.get(format!("{api}/v1/status")).send();
match resp {
Ok(r) if r.status().is_success() => {
let body: serde_json::Value = r.json()?;
println!("memories: {}", body["memory_count"]);
println!("ingested files: {}", body["ingested_file_count"]);
println!("version: {}", body["version"]);
println!("embedder: {}", body["embedder_available"]);
let c = &body["consolidation"];
if c.is_object() {
println!(
"consolidation: {} distilled, {} raw pending, {} retired, {} awaiting retirement",
c["distilled_live"],
c["raw_pending"],
c["raw_retired"],
c["raw_retire_pending"]
);
}
match &body["maintenance"] {
m if m.is_object() => {
println!("last repair: {}", m["ran_at"].as_str().unwrap_or("?"));
for (name, value) in drift_fields(&m["report_after"]) {
println!("WARN {name}: {value}");
}
}
_ => println!("last repair: none since the daemon started"),
}
if let Some(diag) = body["diagnostic"].as_str()
&& !diag.is_empty()
{
println!("\n{diag}");
}
}
Ok(r) => anyhow::bail!("daemon returned {}", r.status()),
Err(_) => anyhow::bail!("daemon not reachable at {api} — is it running?"),
}
Ok(())
}
fn cmd_whoami() -> Result<()> {
let api = api_base_url();
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(5))
.build()?;
let body: serde_json::Value = client
.get(format!("{api}/v1/status"))
.send()
.context("daemon not reachable")?
.error_for_status()?
.json()?;
let id = body["machine_id"]
.as_str()
.context("daemon response missing machine_id")?;
println!("{id}");
Ok(())
}
fn cmd_setup() -> Result<()> {
let home = std::env::var("HOME").context("HOME not set")?;
let home = PathBuf::from(&home);
eprintln!("=== second-brain setup ===\n");
let db_dir = home.join(".second-brain");
std::fs::create_dir_all(&db_dir)?;
eprintln!("[1/5] data dir: {}", db_dir.display());
let (api_bin, mcp_bin, cli_bin) = locate_binaries()?;
eprintln!("[2/5] binaries:");
eprintln!(" api: {}", api_bin.display());
eprintln!(" mcp: {}", mcp_bin.display());
eprintln!(" cli: {}", cli_bin.display());
let systemd_dir = home.join(".config/systemd/user");
std::fs::create_dir_all(&systemd_dir)?;
let service_path = systemd_dir.join("second-brain.service");
let db_path = db_dir.join("brain.db");
let service_content = format!(
"[Unit]\n\
Description=Second Brain daemon (KuzuDB, embedder, file watcher)\n\
After=network.target\n\
\n\
[Service]\n\
Type=simple\n\
ExecStart={api}\n\
Environment=SECOND_BRAIN_DB={db}\n\
Environment=SECOND_BRAIN_BIND=127.0.0.1:7200\n\
Restart=on-failure\n\
RestartSec=5\n\
TimeoutStartSec=60\n\
\n\
[Install]\n\
WantedBy=default.target\n",
api = api_bin.display(),
db = db_path.display(),
);
std::fs::write(&service_path, &service_content)?;
eprintln!("[3/5] systemd service: {}", service_path.display());
let reload = std::process::Command::new("systemctl")
.args(["--user", "daemon-reload"])
.status();
if reload.is_ok_and(|s| s.success()) {
let enable = std::process::Command::new("systemctl")
.args(["--user", "enable", "--now", "second-brain.service"])
.status();
match enable {
Ok(s) if s.success() => eprintln!(" enabled and started"),
_ => eprintln!(" warning: could not enable service (systemd not available?)"),
}
} else {
eprintln!(
" warning: systemctl not available, start daemon manually: {}",
api_bin.display()
);
}
let claude_dir = home.join(".claude");
std::fs::create_dir_all(&claude_dir)?;
let settings_path = claude_dir.join("settings.json");
let mut settings: serde_json::Value = if settings_path.exists() {
let content = std::fs::read_to_string(&settings_path)?;
serde_json::from_str(&content).unwrap_or_else(|_| serde_json::json!({}))
} else {
serde_json::json!({})
};
let mcp_servers = settings
.as_object_mut()
.unwrap()
.entry("mcpServers")
.or_insert_with(|| serde_json::json!({}));
mcp_servers["second-brain"] = serde_json::json!({
"command": mcp_bin.to_string_lossy(),
"env": {
"SECOND_BRAIN_API": "http://127.0.0.1:7200",
"SECOND_BRAIN_DAEMON_BIN": api_bin.to_string_lossy(),
}
});
std::fs::write(&settings_path, serde_json::to_string_pretty(&settings)?)?;
eprintln!("[4/5] claude code MCP: {}", settings_path.display());
eprintln!("[5/5] running first ingest...");
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(600))
.build()?;
let api = api_base_url();
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);
let mut ready = false;
while std::time::Instant::now() < deadline {
if client
.get(format!("{api}/health"))
.timeout(std::time::Duration::from_secs(2))
.send()
.is_ok_and(|r| r.status().is_success())
{
ready = true;
break;
}
std::thread::sleep(std::time::Duration::from_millis(500));
}
if !ready {
eprintln!(" daemon not ready after 30s — run 'second-brain-cli ingest' later");
return Ok(());
}
match client
.post(format!("{api}/v1/ingest"))
.json(&serde_json::json!({}))
.send()
{
Ok(resp) if resp.status().is_success() => {
let body: serde_json::Value = resp.json().unwrap_or_default();
eprintln!(
" ingested: {} memories, {} conversations",
body["new_memories"], body["new_conversations"]
);
}
Ok(resp) => eprintln!(" ingest warning: {}", resp.status()),
Err(e) => eprintln!(" ingest warning: {e}"),
}
eprintln!("\n=== setup complete ===");
eprintln!("daemon: systemctl --user status second-brain");
eprintln!("recall: sb recall \"your query\"");
eprintln!("status: sb status");
eprintln!("\nRun your bootstrapper (harness) for wiki vault + daily timer setup.");
Ok(())
}
fn locate_binaries() -> Result<(PathBuf, PathBuf, PathBuf)> {
let cli_exe = std::env::current_exe().context("cannot determine own binary path")?;
let bin_dir = cli_exe.parent().unwrap();
let api_candidate = bin_dir.join("second-brain-api");
let mcp_candidate = bin_dir.join("second-brain-mcp");
let api_bin = if api_candidate.exists() {
api_candidate
} else {
which_bin("second-brain-api")?
};
let mcp_bin = if mcp_candidate.exists() {
mcp_candidate
} else {
which_bin("second-brain-mcp")?
};
Ok((api_bin, mcp_bin, cli_exe))
}
fn which_bin(name: &str) -> Result<PathBuf> {
let home = std::env::var("HOME").unwrap_or_default();
let cargo_bin = PathBuf::from(&home).join(".cargo/bin").join(name);
if cargo_bin.exists() {
return Ok(cargo_bin);
}
let output = std::process::Command::new("which")
.arg(name)
.output()
.context(format!("{name} not found"))?;
if output.status.success() {
let path = String::from_utf8_lossy(&output.stdout).trim().to_string();
Ok(PathBuf::from(path))
} else {
anyhow::bail!(
"{name} not found in PATH or ~/.cargo/bin — install with: cargo install {name}"
)
}
}
fn parse_after_seq(args: &[String]) -> u64 {
for (i, arg) in args.iter().enumerate() {
if arg == "--after-seq"
&& let Some(val) = args.get(i + 1)
{
return val.parse().unwrap_or(0);
}
}
0
}
fn cmd_sync_trust(host: &str) -> Result<()> {
use std::io::Write;
let scan = std::process::Command::new("ssh-keyscan")
.arg("-T")
.arg("10")
.arg(host)
.output()
.context("running ssh-keyscan (is openssh-client installed?)")?;
if !scan.status.success() {
let stderr = String::from_utf8_lossy(&scan.stderr);
anyhow::bail!(
"ssh-keyscan {host} failed (exit {}): {}",
scan.status,
stderr.trim()
);
}
let raw = String::from_utf8(scan.stdout).context("ssh-keyscan returned non-UTF8 output")?;
let lines = second_brain_sync::ssh::parse_keyscan_output(&raw);
if lines.is_empty() {
anyhow::bail!("ssh-keyscan {host} returned no host keys — host unreachable or filtered");
}
let canonical: String = lines
.iter()
.map(second_brain_sync::ssh::compose_known_host_entry)
.collect();
let mut fp_child = std::process::Command::new("ssh-keygen")
.arg("-lf")
.arg("-")
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::inherit())
.spawn()
.context("running ssh-keygen (is openssh-client installed?)")?;
{
let stdin = fp_child.stdin.take().unwrap();
let mut writer = std::io::BufWriter::new(stdin);
writer.write_all(canonical.as_bytes())?;
}
let fp_out = fp_child.wait_with_output()?;
if !fp_out.status.success() {
anyhow::bail!("ssh-keygen -lf failed: exit {}", fp_out.status);
}
let fingerprints = String::from_utf8_lossy(&fp_out.stdout);
eprintln!("Host: {host}");
eprintln!("Fetched {} key(s). Fingerprints:", lines.len());
for fp_line in fingerprints.lines() {
eprintln!(" {fp_line}");
}
eprintln!();
eprintln!("Verify these fingerprints out-of-band before trusting them.");
eprint!("Pin to ~/.second-brain/known_hosts? [y/N] ");
std::io::stderr().flush().ok();
let mut answer = String::new();
std::io::stdin()
.read_line(&mut answer)
.context("reading confirmation")?;
let trimmed = answer.trim();
if !trimmed.eq_ignore_ascii_case("y") && !trimmed.eq_ignore_ascii_case("yes") {
eprintln!("aborted; nothing written.");
return Ok(());
}
let path = second_brain_sync::ssh::known_hosts_path()?;
second_brain_sync::ssh::append_known_host(&path, &lines)?;
eprintln!(
"pinned {} key(s) for {host} -> {}",
lines.len(),
path.display()
);
Ok(())
}
fn cmd_sync(peer_host: &str, args: &[String]) -> Result<()> {
let api_base = api_base_url();
let mut sb_binary = "sb".to_string();
for (i, arg) in args.iter().enumerate() {
if arg == "--sb-path"
&& let Some(val) = args.get(i + 1)
{
sb_binary = val.clone();
}
}
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(600))
.build()?;
let peer_machine_id = resolve_peer_machine_id(peer_host, &sb_binary)
.context("resolving peer machine id (is the peer's daemon running and reachable?)")?;
eprintln!("pulling from {peer_host}...");
let pull_stats =
sync_pull_via_daemon(&client, &api_base, peer_host, &peer_machine_id, &sb_binary)?;
eprintln!(
" {} created, {} updated, {} deleted, {} skipped",
pull_stats.created, pull_stats.updated, pull_stats.deleted, pull_stats.skipped
);
if pull_stats.dangling > 0 {
eprintln!(
" warning: {} records dropped with unresolvable endpoints; \
run `sb prune-relations` on the peer if this persists",
pull_stats.dangling
);
}
eprintln!("pushing to {peer_host}...");
let max_seq =
sync_push_via_daemon(&client, &api_base, peer_host, &peer_machine_id, &sb_binary)?;
eprintln!(" pushed through seq {max_seq}");
eprintln!("sync complete.");
Ok(())
}
fn resolve_peer_machine_id(peer_host: &str, sb_binary: &str) -> Result<String> {
let known_hosts = second_brain_sync::ssh::known_hosts_path()?;
let output = std::process::Command::new("ssh")
.args(second_brain_sync::ssh::ssh_args(&known_hosts))
.arg(peer_host)
.arg(sb_binary)
.arg("whoami")
.output()
.context("spawning ssh to resolve peer machine id")?;
if !output.status.success() {
anyhow::bail!(
"ssh whoami on {peer_host} exited with {}: {}",
output.status,
String::from_utf8_lossy(&output.stderr).trim()
);
}
let id = String::from_utf8_lossy(&output.stdout).trim().to_string();
if id.is_empty() {
anyhow::bail!("peer {peer_host} returned an empty machine id");
}
Ok(id)
}
fn api_base_url() -> String {
std::env::var("SECOND_BRAIN_API").unwrap_or_else(|_| "http://127.0.0.1:7200".to_string())
}
fn sync_pull_via_daemon(
client: &reqwest::blocking::Client,
api_base: &str,
peer_host: &str,
peer_machine_id: &str,
sb_binary: &str,
) -> Result<SyncImportStats> {
#[derive(serde::Deserialize)]
struct PeerInfo {
peers: Vec<PeerState>,
}
#[derive(serde::Deserialize)]
struct PeerState {
peer_id: String,
last_seq: u64,
}
let status: PeerInfo = client
.get(format!("{api_base}/v1/sync/status"))
.send()
.context("getting sync status")?
.json()?;
let after_seq = status
.peers
.iter()
.find(|p| p.peer_id == peer_machine_id)
.or_else(|| status.peers.iter().find(|p| p.peer_id == peer_host))
.map(|p| p.last_seq)
.unwrap_or(0);
let known_hosts = second_brain_sync::ssh::known_hosts_path()?;
let mut child = std::process::Command::new("ssh")
.args(second_brain_sync::ssh::ssh_args(&known_hosts))
.arg(peer_host)
.arg(sb_binary)
.arg("sync-export")
.arg("--after-seq")
.arg(after_seq.to_string())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::inherit())
.spawn()
.context("spawning ssh for pull")?;
let stdout = child.stdout.take().unwrap();
let reader = std::io::BufReader::new(stdout);
#[derive(serde::Deserialize)]
struct ImportResp {
created: u64,
updated: u64,
deleted: u64,
skipped: u64,
#[allow(dead_code)]
errors: u64,
max_seq: u64,
#[serde(default)]
skipped_by_reason: std::collections::BTreeMap<String, u64>,
}
const IMPORT_CHUNK: usize = 1000;
let encryptor = default_sync_encryptor()?;
let mut totals = SyncImportStats {
created: 0,
updated: 0,
deleted: 0,
skipped: 0,
dangling: 0,
};
let mut overall_max_seq = 0u64;
let mut chunk = Vec::with_capacity(IMPORT_CHUNK);
let post_chunk = |chunk: &[serde_json::Value],
client: &reqwest::blocking::Client,
totals: &mut SyncImportStats,
overall_max_seq: &mut u64|
-> Result<()> {
if chunk.is_empty() {
return Ok(());
}
let resp: ImportResp = client
.post(format!("{api_base}/v1/sync/import"))
.json(&serde_json::json!({"records": chunk}))
.send()
.context("posting import to daemon")?
.error_for_status()
.context("import returned error")?
.json()?;
totals.created += resp.created;
totals.updated += resp.updated;
totals.deleted += resp.deleted;
totals.skipped += resp.skipped;
totals.dangling += resp
.skipped_by_reason
.get("dangling_endpoint")
.copied()
.unwrap_or(0);
if resp.max_seq > *overall_max_seq {
*overall_max_seq = resp.max_seq;
}
Ok(())
};
for line in std::io::BufRead::lines(reader) {
let line = line?;
if line.trim().is_empty() {
continue;
}
let plaintext = second_brain_sync::crypto::open_line(encryptor.as_ref(), &line)?;
let val: serde_json::Value = serde_json::from_slice(&plaintext)?;
chunk.push(val);
if chunk.len() >= IMPORT_CHUNK {
post_chunk(&chunk, client, &mut totals, &mut overall_max_seq)?;
chunk.clear();
}
}
post_chunk(&chunk, client, &mut totals, &mut overall_max_seq)?;
chunk.clear();
let status = child.wait()?;
if !status.success() {
anyhow::bail!(
"ssh pull from {peer_host} exited with {status}.\n hint: {}",
second_brain_sync::ssh::untrusted_host_hint(peer_host)
);
}
if overall_max_seq > after_seq {
client
.post(format!("{api_base}/v1/sync/state"))
.json(&serde_json::json!({"peer_id": peer_machine_id, "last_seq": overall_max_seq}))
.send()
.context("updating sync state")?;
}
Ok(totals)
}
struct SyncImportStats {
created: u64,
updated: u64,
deleted: u64,
skipped: u64,
dangling: u64,
}
fn sync_push_via_daemon(
client: &reqwest::blocking::Client,
api_base: &str,
peer_host: &str,
peer_machine_id: &str,
sb_binary: &str,
) -> Result<u64> {
let push_peer = format!("push:{}", peer_machine_id);
let legacy_push_peer = format!("push:{}", peer_host);
const BATCH_SIZE: usize = 2000;
#[derive(serde::Deserialize)]
struct PeerInfo {
peers: Vec<PeerState>,
}
#[derive(serde::Deserialize)]
struct PeerState {
peer_id: String,
last_seq: u64,
}
let status: PeerInfo = client
.get(format!("{api_base}/v1/sync/status"))
.send()
.context("getting sync status")?
.json()?;
let mut cursor = status
.peers
.iter()
.find(|p| p.peer_id == push_peer)
.or_else(|| status.peers.iter().find(|p| p.peer_id == legacy_push_peer))
.map(|p| p.last_seq)
.unwrap_or(0);
#[derive(serde::Deserialize)]
struct ExportResp {
records: Vec<serde_json::Value>,
max_seq: u64,
}
let known_hosts = second_brain_sync::ssh::known_hosts_path()?;
let mut child = std::process::Command::new("ssh")
.args(second_brain_sync::ssh::ssh_args(&known_hosts))
.arg("-o")
.arg("ServerAliveInterval=60")
.arg("-o")
.arg("ServerAliveCountMax=60")
.arg(peer_host)
.arg(sb_binary)
.arg("sync-import")
.stdin(std::process::Stdio::piped())
.stderr(std::process::Stdio::inherit())
.spawn()
.context("spawning ssh for push")?;
let mut total_pushed = 0u64;
let encryptor = default_sync_encryptor()?;
{
let stdin = child.stdin.take().unwrap();
let mut writer = std::io::BufWriter::new(stdin);
loop {
let export: ExportResp = client
.post(format!("{api_base}/v1/sync/export"))
.json(&serde_json::json!({"after_seq": cursor, "limit": BATCH_SIZE}))
.send()
.context("getting export from daemon")?
.error_for_status()
.context("export returned error")?
.json()?;
if export.records.is_empty() {
break;
}
let batch_count = export.records.len();
for record in &export.records {
use std::io::Write;
let plaintext = serde_json::to_vec(record)?;
let line = second_brain_sync::crypto::seal_line(encryptor.as_ref(), &plaintext)?;
writer.write_all(line.as_bytes())?;
writer.write_all(b"\n")?;
}
use std::io::Write;
writer.flush()?;
total_pushed += batch_count as u64;
cursor = export.max_seq;
eprintln!(" batch: {} records (through seq {})", batch_count, cursor);
if batch_count < BATCH_SIZE {
break;
}
}
}
let exit = child.wait()?;
if !exit.success() {
anyhow::bail!(
"ssh push to {peer_host} exited with {exit}.\n hint: {}",
second_brain_sync::ssh::untrusted_host_hint(peer_host)
);
}
if total_pushed > 0 {
client
.post(format!("{api_base}/v1/sync/state"))
.json(&serde_json::json!({"peer_id": push_peer, "last_seq": cursor}))
.send()
.context("updating push sync state")?;
}
eprintln!(" total: {} records pushed", total_pushed);
Ok(cursor)
}
fn cmd_sync_export(after_seq: u64) -> Result<()> {
let api_base = api_base_url();
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(120))
.build()?;
#[derive(serde::Deserialize)]
struct ExportResp {
records: Vec<serde_json::Value>,
max_seq: u64,
}
const BATCH_SIZE: usize = 2000;
let encryptor = default_sync_encryptor()?;
let stdout = std::io::stdout();
let mut writer = std::io::BufWriter::new(stdout.lock());
let mut cursor = after_seq;
let mut total = 0u64;
loop {
let resp: ExportResp = client
.post(format!("{api_base}/v1/sync/export"))
.json(&serde_json::json!({"after_seq": cursor, "limit": BATCH_SIZE}))
.send()
.context("calling sync/export on daemon")?
.error_for_status()
.context("sync/export returned error")?
.json()?;
if resp.records.is_empty() {
break;
}
let batch_count = resp.records.len();
for record in &resp.records {
use std::io::Write;
let plaintext = serde_json::to_vec(record)?;
let line = second_brain_sync::crypto::seal_line(encryptor.as_ref(), &plaintext)?;
writer.write_all(line.as_bytes())?;
writer.write_all(b"\n")?;
}
use std::io::Write;
writer.flush()?;
total += batch_count as u64;
cursor = resp.max_seq;
if batch_count < BATCH_SIZE {
break;
}
}
eprintln!("exported {total} records through seq {cursor}");
Ok(())
}
fn cmd_prune_relations() -> Result<()> {
let api_base = api_base_url();
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(1800))
.build()?;
#[derive(serde::Deserialize)]
struct PruneResp {
pruned: u64,
}
let resp: PruneResp = client
.post(format!("{api_base}/v1/sync/prune-relations"))
.send()
.context("calling prune-relations on daemon")?
.error_for_status()?
.json()?;
eprintln!("pruned {} dangling relations", resp.pruned);
Ok(())
}
fn drift_fields(report: &serde_json::Value) -> Vec<(&'static str, u64)> {
let mut out = Vec::new();
for name in [
"raw_retire_pending",
"duplicate_hot_groups",
"orphaned_distillation_markers",
"dead_vec_keys",
"dangling_relations",
"sync_log_reclaimable_rows",
"stale_peer_cursors",
"missing_embeddings",
] {
let value = report[name].as_u64().unwrap_or(0);
if value > 0 {
out.push((name, value));
}
}
if report["index_consistent"] == serde_json::Value::Bool(false) {
out.push(("index_inconsistent", 1));
}
out
}
fn print_report(report: &serde_json::Value) {
let Some(map) = report.as_object() else {
println!("{report}");
return;
};
let width = map.keys().map(|k| k.len()).max().unwrap_or(0);
for (key, value) in map {
let shown = match value {
serde_json::Value::Null => "n/a".to_string(),
v => v.to_string(),
};
println!("{key:width$} {shown}");
}
let drift = drift_fields(report);
if drift.is_empty() {
println!("\nno drift");
} else {
for (name, value) in drift {
println!("WARN {name}: {value}");
}
}
}
fn cmd_doctor(repair: bool, dry_run: bool) -> Result<()> {
let api = api_base_url();
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(3600))
.build()?;
if !repair {
let report: serde_json::Value = client
.get(format!("{api}/v1/health"))
.send()
.context("calling health on daemon")?
.error_for_status()?
.json()?;
print_report(&report);
return Ok(());
}
let summary: serde_json::Value = client
.post(format!("{api}/v1/maintenance/repair"))
.json(&serde_json::json!({"dry_run": dry_run}))
.send()
.context("calling repair on daemon")?
.error_for_status()?
.json()?;
println!(
"{}",
if dry_run {
"repair (dry run):"
} else {
"repair:"
}
);
if let Some(stats) = summary["stats"].as_object() {
for (key, value) in stats {
if key != "dry_run" {
println!(" {key}: {value}");
}
}
}
println!("\nreport after:");
print_report(&summary["report_after"]);
Ok(())
}
fn cmd_vacuum() -> Result<()> {
let api = api_base_url();
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(3600))
.build()?;
let resp: serde_json::Value = client
.post(format!("{api}/v1/maintenance/vacuum"))
.send()
.context("calling vacuum on daemon")?
.error_for_status()?
.json()?;
let before = resp["bytes_before"].as_u64().unwrap_or(0);
let after = resp["bytes_after"].as_u64().unwrap_or(0);
println!(
"vacuumed brain.db: {:.1} MB -> {:.1} MB",
before as f64 / 1e6,
after as f64 / 1e6
);
Ok(())
}
fn cmd_sync_key() -> Result<()> {
let path = second_brain_sync::crypto::default_key_path()?;
let key = second_brain_sync::crypto::load_or_create_key(&path)?;
let enc = second_brain_sync::crypto::XChaChaEncryptor::new(key);
println!("{}", enc.key_base64());
eprintln!("key file: {}", path.display());
eprintln!(
"copy this base64 value into ~/.second-brain/sync.key on each paired device \
(run `sb sync-key` once on a device to generate, then replicate it everywhere)."
);
Ok(())
}
fn default_sync_encryptor() -> Result<Box<dyn second_brain_sync::crypto::SyncEncryptor>> {
let path = second_brain_sync::crypto::default_key_path()?;
let key = second_brain_sync::crypto::load_or_create_key(&path)?;
Ok(Box::new(second_brain_sync::crypto::XChaChaEncryptor::new(
key,
)))
}
fn cmd_sync_import() -> Result<()> {
let api_base = api_base_url();
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(600))
.build()?;
const BATCH_SIZE: usize = 500;
let stdin = std::io::stdin();
let reader = std::io::BufReader::new(stdin.lock());
#[derive(serde::Deserialize)]
struct ImportResp {
created: u64,
updated: u64,
deleted: u64,
skipped: u64,
errors: u64,
max_seq: u64,
}
let mut totals = (0u64, 0u64, 0u64, 0u64, 0u64, 0u64);
let mut batch: Vec<serde_json::Value> = Vec::with_capacity(BATCH_SIZE);
let mut min_error_watermark: Option<u64> = None;
let flush_batch = |batch: &mut Vec<serde_json::Value>,
totals: &mut (u64, u64, u64, u64, u64, u64),
min_error_watermark: &mut Option<u64>|
-> Result<()> {
if batch.is_empty() {
return Ok(());
}
let resp: ImportResp = client
.post(format!("{api_base}/v1/sync/import"))
.json(&serde_json::json!({"records": *batch}))
.send()
.context("calling sync/import on daemon")?
.error_for_status()
.context("sync/import returned error")?
.json()?;
totals.0 += resp.created;
totals.1 += resp.updated;
totals.2 += resp.deleted;
totals.3 += resp.skipped;
totals.4 += resp.errors;
if resp.errors > 0 {
let candidate = resp.max_seq.saturating_sub(1);
*min_error_watermark = Some(
min_error_watermark
.map(|prev| prev.min(candidate))
.unwrap_or(candidate),
);
} else {
totals.5 = totals.5.max(resp.max_seq);
}
batch.clear();
Ok(())
};
let encryptor = default_sync_encryptor()?;
for line in std::io::BufRead::lines(reader) {
let line = line?;
if line.trim().is_empty() {
continue;
}
let plaintext = second_brain_sync::crypto::open_line(encryptor.as_ref(), &line)?;
batch.push(serde_json::from_slice(&plaintext)?);
if batch.len() >= BATCH_SIZE {
flush_batch(&mut batch, &mut totals, &mut min_error_watermark)?;
}
}
flush_batch(&mut batch, &mut totals, &mut min_error_watermark)?;
if let Some(err_wm) = min_error_watermark {
totals.5 = totals.5.min(err_wm);
}
eprintln!(
"imported: {} created, {} updated, {} deleted, {} skipped, {} errors (through seq {})",
totals.0, totals.1, totals.2, totals.3, totals.4, totals.5
);
Ok(())
}
fn cmd_sync_status() -> Result<()> {
let api_base = api_base_url();
let client = reqwest::blocking::Client::new();
#[derive(serde::Deserialize)]
struct PeerInfo {
peers: Vec<PeerState>,
}
#[derive(serde::Deserialize)]
struct PeerState {
peer_id: String,
last_seq: u64,
last_sync_at: String,
}
let resp: PeerInfo = client
.get(format!("{api_base}/v1/sync/status"))
.send()
.context("getting sync status from daemon")?
.error_for_status()?
.json()?;
if resp.peers.is_empty() {
println!("no sync peers configured");
} else {
for state in &resp.peers {
println!(
"peer: {}, last_seq: {}, last_sync: {}",
state.peer_id, state.last_seq, state.last_sync_at
);
}
}
Ok(())
}
fn cmd_sync_backfill() -> Result<()> {
let api_base = api_base_url();
let client = reqwest::blocking::Client::new();
#[derive(serde::Deserialize)]
struct BackfillResp {
backfilled: usize,
#[serde(default)]
tombstones_backfilled: usize,
}
let resp: BackfillResp = client
.post(format!("{api_base}/v1/sync/backfill"))
.send()
.context("calling backfill on daemon")?
.error_for_status()?
.json()?;
eprintln!(
"backfilled {} create entries, {} delete entries",
resp.backfilled, resp.tombstones_backfilled
);
Ok(())
}
fn cmd_backfill_session_dates(dry_run: bool, reemit: bool) -> Result<()> {
let api_base = api_base_url();
let client = reqwest::blocking::Client::builder()
.timeout(std::time::Duration::from_secs(1800))
.build()?;
let report: serde_json::Value = client
.post(format!("{api_base}/v1/backfill-session-at"))
.json(&serde_json::json!({"dry_run": dry_run, "reemit": reemit}))
.send()
.context("calling backfill-session-at on daemon")?
.error_for_status()?
.json()?;
let tiers = [
"transcript_line",
"conversation_start",
"transcript_start",
"file_mtime",
"remembered_at",
"unknown",
];
println!(
"{}undated rows: {}",
if dry_run { "dry run, " } else { "" },
report["candidates"]
);
println!("{:<20} {:>8} {:>8}", "evidence", "all", "live");
for tier in tiers {
println!(
"{:<20} {:>8} {:>8}",
tier,
report["all"][tier].as_u64().unwrap_or(0),
report["live"][tier].as_u64().unwrap_or(0)
);
}
println!(
"fallback conversations skipped: {}",
report["fallback_conversations_skipped"]
);
println!(
"written: {}, reemitted: {}",
report["written"], report["reemitted"]
);
for s in report["sample"].as_array().into_iter().flatten() {
let content: String = s["content"]
.as_str()
.unwrap_or("")
.chars()
.take(80)
.map(|c| if c == '\n' { ' ' } else { c })
.collect();
println!(
" {} {:<18} created {} {}: {}",
s["session_at"]
.as_str()
.unwrap_or("")
.get(..10)
.unwrap_or(""),
s["tier"].as_str().unwrap_or(""),
s["created_at"]
.as_str()
.unwrap_or("")
.get(..10)
.unwrap_or(""),
s["source"].as_str().unwrap_or(""),
content
);
}
Ok(())
}
fn cmd_sync_compact(ops: &[String], dry_run: bool) -> Result<()> {
let api_base = api_base_url();
let client = reqwest::blocking::Client::new();
#[derive(serde::Deserialize)]
struct CompactResp {
groups_with_duplicates: u64,
rows_deleted: u64,
max_local_seq_before: u64,
max_local_seq_after: u64,
}
let resp: CompactResp = client
.post(format!("{api_base}/v1/sync/compact"))
.json(&serde_json::json!({"ops": ops, "dry_run": dry_run}))
.send()
.context("calling compact on daemon")?
.error_for_status()?
.json()?;
if resp.max_local_seq_before != resp.max_local_seq_after {
anyhow::bail!(
"compact reported MAX(local_seq) moved from {} to {} -- aborting, this must never happen",
resp.max_local_seq_before,
resp.max_local_seq_after
);
}
if dry_run {
eprintln!(
"dry run: {} groups have duplicates, {} rows would be deleted (max_local_seq {} unaffected)",
resp.groups_with_duplicates, resp.rows_deleted, resp.max_local_seq_after
);
} else {
eprintln!(
"compacted {} groups, deleted {} duplicate rows (max_local_seq {} unchanged)",
resp.groups_with_duplicates, resp.rows_deleted, resp.max_local_seq_after
);
}
Ok(())
}
fn cmd_daily_pipeline(
llm_url: &str,
model: &str,
consolidate_limit: usize,
_enhance: bool,
) -> Result<()> {
println!("=== daily pipeline ===\n");
println!(
"[1/1] consolidating up to {} memories...",
consolidate_limit
);
match consolidate::cmd_consolidate(
false,
consolidate_limit,
model,
llm_url,
3,
consolidate::Guard::Retry,
) {
Ok(()) => println!(" consolidation complete"),
Err(e) => println!(" consolidation failed (non-fatal): {}", e),
}
println!("\n=== pipeline complete ===");
Ok(())
}