use std::path::{Path, PathBuf};
use std::sync::{Arc, RwLock};
use std::time::Duration;
use tokio::net::TcpListener;
use chrono::Utc;
use second_brain_api::health::{Health, Windows};
use second_brain_api::maintenance::DrainLoop;
use second_brain_api::progress::ProgressCheck;
use second_brain_api::store_compaction::Compaction;
use second_brain_api::write_gate::WriteGate;
use second_brain_api::write_queue::{self, WriteQueue};
use second_brain_api::{AppState, alert, file_modes, stored_redaction, watcher};
use second_brain_core::api_token;
use second_brain_core::config::Config;
use second_brain_core::embedding::Embedder;
use second_brain_core::index::Indexes;
use second_brain_core::machine::{self, MachineIdentity};
use second_brain_core::retrieval::Reranker;
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::StoreAdmin;
struct IndexHandles {
reader: Arc<RwLock<Indexes>>,
writer: Indexes,
}
fn open_index_handles(dir: &Path, store: &SqliteStore) -> anyhow::Result<IndexHandles> {
let mut writer = Indexes::open(dir)?;
if writer.consistent_with(store)? {
tracing::info!("search indexes consistent at {}", dir.display());
writer.persist()?;
} else {
tracing::info!("rebuilding search indexes at {}", dir.display());
let n = writer.rebuild(store)?;
tracing::info!("search indexes rebuilt: {n} memories");
}
let reader = Indexes::open_readonly(dir)?;
Ok(IndexHandles {
reader: Arc::new(RwLock::new(reader)),
writer,
})
}
fn rerank_pool_size() -> usize {
match std::env::var("SECOND_BRAIN_RERANK_POOL") {
Ok(v) => v
.parse::<usize>()
.unwrap_or_else(|_| {
panic!("invalid SECOND_BRAIN_RERANK_POOL '{v}': expected a positive integer")
})
.max(1),
Err(_) => {
let cores = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1);
cores.clamp(1, 4)
}
}
}
#[tokio::main]
async fn main() {
unsafe { libc::umask(0o077) };
second_brain_api::logging::init();
let bind = std::env::var("SECOND_BRAIN_BIND").unwrap_or_else(|_| "127.0.0.1:7200".to_string());
let listener = match TcpListener::bind(&bind).await {
Ok(l) => l,
Err(e) => {
tracing::error!(
"cannot listen on {bind}: {e}; another second-brain daemon probably holds it"
);
std::process::exit(1);
}
};
let db_path = std::env::var("SECOND_BRAIN_DB")
.map(PathBuf::from)
.unwrap_or_else(|_| {
let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
PathBuf::from(home).join(".second-brain").join("brain.db")
});
if let Some(parent) = db_path.parent() {
std::fs::create_dir_all(parent).ok();
}
let data_dir = machine::data_dir().expect("cannot determine home directory");
let api_token = match api_token::ensure(&data_dir) {
Ok(token) => token,
Err(e) => {
tracing::error!(
"cannot create the API token in {}: {e:#}",
data_dir.display()
);
std::process::exit(1);
}
};
let config_dir = MachineIdentity::config_dir().expect("cannot determine home directory");
let machine = MachineIdentity::load_or_create(&config_dir)
.expect("failed to load or create machine identity");
tracing::info!("machine identity: {} ({})", machine.name, machine.id);
let sqlite = match SqliteStore::open(&db_path, machine.id.clone()) {
Ok(s) => Arc::new(s),
Err(e) => {
tracing::error!("failed to open database at {}: {e}", db_path.display());
std::process::exit(1);
}
};
let store: Arc<dyn StoreAdmin> = sqlite.clone();
store
.register_machine(&machine)
.expect("failed to register machine identity");
tracing::info!("database opened at {}", db_path.display());
let config = match Config::load_default() {
Ok(c) => c,
Err(e) => {
tracing::warn!("config.toml unusable, running with defaults: {e:#}");
Config::default()
}
};
let health = Arc::new(
Health::new(
Utc::now(),
alert::start(&config.alert, machine.name.clone()),
)
.with_windows(Windows::from_config(&config.alert)),
);
let embedder = match Embedder::new() {
Ok(e) => {
tracing::info!("embedding model loaded");
Some(Arc::new(e))
}
Err(e) => {
tracing::warn!("embedding model unavailable: {e}");
None
}
};
let index_dir = db_path.with_extension("indexes");
let index_handles = match open_index_handles(&index_dir, &sqlite) {
Ok(h) => Some(h),
Err(e) => {
tracing::warn!(
"search indexes unavailable at {}: {e}; recall falls back to the legacy engine",
index_dir.display()
);
health.indexes_unavailable(&format!("{e:#}"), Utc::now());
None
}
};
let indexes = index_handles.as_ref().map(|h| h.reader.clone());
for (path, e) in file_modes::make_private(&data_dir, &db_path, &index_dir) {
tracing::warn!("could not make {} private: {e}", path.display());
}
let rerank_model = std::env::var("SECOND_BRAIN_RERANK_MODEL")
.unwrap_or_else(|_| "jina-reranker-v1-turbo".to_string());
let reranker = if indexes.is_none() || rerank_model == "off" {
None
} else {
let pool_size = rerank_pool_size();
match Reranker::with_sessions(&rerank_model, pool_size) {
Ok(r) => {
tracing::info!("rerank model {rerank_model} loaded ({pool_size} sessions)");
Some(Arc::new(r))
}
Err(e) => {
tracing::warn!("rerank model {rerank_model} unavailable: {e:#}");
None
}
}
};
let maintenance = Arc::new(RwLock::new(None));
let near_duplicates = Arc::new(RwLock::new(None));
let mut repair_tx = None;
if let Some(IndexHandles { reader, writer }) = index_handles {
let (tx, requests) = std::sync::mpsc::channel();
repair_tx = Some(tx);
DrainLoop {
store: sqlite.clone(),
reader,
writer,
embedder: embedder.clone(),
summary: maintenance.clone(),
near_duplicates: near_duplicates.clone(),
near_duplicate_threshold: config.distill.near_duplicate_threshold,
config: config.maintenance.clone(),
requests,
health: health.clone(),
}
.spawn();
}
watcher::spawn(
store.clone(),
embedder.clone(),
machine.id.clone(),
health.clone(),
);
let gate = Arc::new(WriteGate::new(data_dir.join(write_queue::DIR_NAME)));
let compaction = Compaction::new(
sqlite.clone(),
gate.clone(),
repair_tx.clone(),
health.clone(),
);
if let Err(e) = stored_redaction::spawn_pass(sqlite.clone(), embedder.clone(), compaction) {
tracing::error!("cannot start the stored-content redaction pass: {e}");
}
if let Some(emb) = embedder.clone() {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(60));
loop {
interval.tick().await;
emb.drop_if_idle(300);
}
});
}
if let Some(r) = reranker.clone() {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(60));
loop {
interval.tick().await;
r.drop_if_idle(300);
}
});
}
let progress = ProgressCheck {
store: sqlite.clone(),
data_dir: data_dir.clone(),
health: health.clone(),
};
let state = Arc::new(AppState {
store,
sqlite,
indexes,
reranker,
tunables: config.retrieval.tunables(),
embedder,
machine,
maintenance,
repair_tx,
near_duplicate_threshold: config.distill.near_duplicate_threshold,
near_duplicates,
health: health.clone(),
gate: gate.clone(),
});
let app = second_brain_api::app(state, api_token.clone());
tracing::info!("listening on {bind}");
let announced = health.clone();
tokio::spawn(async move {
progress.start().await;
announced.announce(Utc::now());
});
WriteQueue::new(
data_dir.join(write_queue::DIR_NAME),
app.clone(),
api_token,
health,
gate,
)
.spawn();
axum::serve(listener, app).await.unwrap();
}