use std::path::PathBuf;
use std::sync::{Arc, RwLock};
use tokio::net::TcpListener;
use chrono::Utc;
use second_brain_api::backfill::{Backfill, LiveIngest, PAUSE_BETWEEN_FILES};
use second_brain_api::footprint::{FootprintCheck, StorePaths};
use second_brain_api::health::{Health, Windows};
use second_brain_api::maintenance::{DrainLoop, WriterSlot};
use second_brain_api::models::{self, IndexHandles};
use second_brain_api::note_cover::NoteCover;
use second_brain_api::progress::ProgressCheck;
use second_brain_api::raw_index::{PAUSE_AFTER_EMBED, RawCoverage, RawIndexer, open_or_rebuild};
use second_brain_api::sources::{GitHub, PullRequests};
use second_brain_api::startup::Startup;
use second_brain_api::store_compaction::Compaction;
use second_brain_api::sync_loop::SyncLoop;
use second_brain_api::write_gate::WriteGate;
use second_brain_api::write_queue::{self, WriteQueue};
use second_brain_api::{AppState, alert, file_modes, sources, stored_redaction, vaults, watcher};
use second_brain_core::api_token;
use second_brain_core::config::Config;
use second_brain_core::embedding::DocumentEmbedder;
use second_brain_core::index::raw;
use second_brain_core::machine::{self, MachineIdentity};
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::StoreAdmin;
use second_brain_core::transcript_archive::TranscriptArchive;
#[tokio::main]
async fn main() {
unsafe { libc::umask(0o077) };
second_brain_api::logging::init();
let bind = std::env::var("SECOND_BRAIN_BIND").unwrap_or_else(|_| "127.0.0.1:7200".to_string());
let listener = match TcpListener::bind(&bind).await {
Ok(l) => l,
Err(e) => {
tracing::error!(
"cannot listen on {bind}: {e}; another second-brain daemon probably holds it"
);
std::process::exit(1);
}
};
let startup = Startup::new();
let server = tokio::spawn(startup.clone().serve(listener));
tracing::info!("listening on {bind}");
let db_path = std::env::var("SECOND_BRAIN_DB")
.map(PathBuf::from)
.unwrap_or_else(|_| {
let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
PathBuf::from(home).join(".second-brain").join("brain.db")
});
if let Some(parent) = db_path.parent() {
std::fs::create_dir_all(parent).ok();
}
let data_dir = machine::data_dir().expect("cannot determine home directory");
let api_token = match api_token::ensure(&data_dir) {
Ok(token) => token,
Err(e) => {
tracing::error!(
"cannot create the API token in {}: {e:#}",
data_dir.display()
);
std::process::exit(1);
}
};
let config_dir = MachineIdentity::config_dir().expect("cannot determine home directory");
let machine = MachineIdentity::load_or_create(&config_dir)
.expect("failed to load or create machine identity");
tracing::info!("machine identity: {} ({})", machine.name, machine.id);
startup.phase("opening the database");
let sqlite = match SqliteStore::open(&db_path, machine.id.clone()) {
Ok(s) => Arc::new(s),
Err(e) => {
tracing::error!("failed to open database at {}: {e}", db_path.display());
std::process::exit(1);
}
};
let store: Arc<dyn StoreAdmin> = sqlite.clone();
store
.register_machine(&machine)
.expect("failed to register machine identity");
tracing::info!("database opened at {}", db_path.display());
let (config, config_loaded) = match Config::load_default() {
Ok(c) => (c, true),
Err(e) => {
tracing::warn!("config.toml unusable, running with defaults: {e:#}");
(Config::default(), false)
}
};
let llm_configured = !config_loaded || config.llm.is_some();
let health = Arc::new(
Health::new(
Utc::now(),
alert::start(&config.alert, machine.name.clone()),
)
.with_windows(Windows::from_config(&config.alert)),
);
startup.phase("loading the embedding model");
let embedder = models::load_embedder();
startup.phase("opening the search indexes");
let index_dir = db_path.with_extension("indexes");
let index_handles = match models::open_index_handles(&index_dir, &sqlite, |n| {
startup.phase(format!("rebuilding search indexes for {n} memories"))
}) {
Ok(h) => Some(h),
Err(e) => {
tracing::warn!(
"search indexes unavailable at {}: {e}; recall falls back to the legacy engine",
index_dir.display()
);
health.indexes_unavailable(&format!("{e:#}"), Utc::now());
None
}
};
let indexes = index_handles.as_ref().map(|h| h.reader.clone());
let archive_dir = data_dir.join("transcripts");
let archive = match TranscriptArchive::open(&archive_dir) {
Ok(a) => Some(Arc::new(a)),
Err(e) => {
tracing::error!(
"cannot open the transcript archive at {}: {e:#}",
archive_dir.display()
);
None
}
};
startup.phase("opening the raw transcript index");
let raw_dir = raw::dir_for_db(&db_path);
let raw_index = match open_or_rebuild(&raw_dir) {
Ok(index) => Some(Arc::new(RwLock::new(index))),
Err(e) => {
tracing::warn!(
"raw transcript index unavailable at {}: {e:#}",
raw_dir.display()
);
None
}
};
for (path, e) in file_modes::make_private(&data_dir, &db_path, &index_dir) {
tracing::warn!("could not make {} private: {e}", path.display());
}
startup.phase("loading the reranker");
let reranker = models::load_reranker(indexes.is_some());
startup.phase("starting background tasks");
let maintenance = Arc::new(RwLock::new(None));
let near_duplicates = Arc::new(RwLock::new(None));
let mut repair_tx = None;
if let Some(IndexHandles {
reader,
writer,
claim,
}) = index_handles
{
let (tx, requests) = std::sync::mpsc::channel();
repair_tx = Some(tx);
DrainLoop {
store: sqlite.clone(),
reader,
writer: WriterSlot::new(writer, claim),
embedder: embedder.clone(),
summary: maintenance.clone(),
near_duplicates: near_duplicates.clone(),
near_duplicate_threshold: config.distill.near_duplicate_threshold,
config: config.maintenance.clone(),
requests,
health: health.clone(),
}
.spawn();
}
let raw_coverage = raw_index
.as_ref()
.map(|_| Arc::new(RawCoverage::new(config.raw_index.max_age_days)));
if let (Some(index), Some(coverage), Some(embedder)) = (&raw_index, &raw_coverage, &embedder) {
let embedder = embedder.clone();
RawIndexer {
index: index.clone(),
archive: archive.clone(),
store: sqlite.clone(),
embed: Arc::new(move |texts: &[&str]| embedder.embed_batch(texts)),
pause: PAUSE_AFTER_EMBED,
coverage: coverage.clone(),
}
.spawn();
}
let live = Arc::new(LiveIngest::default());
watcher::spawn(
store.clone(),
embedder.clone(),
machine.id.clone(),
health.clone(),
archive.clone(),
live.clone(),
);
vaults::spawn(
sqlite.clone(),
embedder.clone(),
config.sources.obsidian_vaults.clone(),
machine.id.clone(),
);
Backfill {
store: sqlite.clone(),
embedder: embedder.clone(),
machine_id: machine.id.clone(),
projects_dir: watcher::projects_dir(),
archive,
live,
health: health.clone(),
pause: PAUSE_BETWEEN_FILES,
}
.spawn();
sources::Sources {
store: sqlite.clone(),
embedder: embedder.clone().map(|e| e as Arc<dyn DocumentEmbedder>),
machine_id: machine.id.clone(),
pull_requests: GitHub::from_env(tokio::runtime::Handle::current())
.map(|g| Arc::new(g) as Arc<dyn PullRequests>),
}
.spawn();
if !llm_configured {
NoteCover {
store: sqlite.clone(),
}
.spawn();
}
let gate = Arc::new(WriteGate::new(data_dir.join(write_queue::DIR_NAME)));
let compaction = Compaction::new(
sqlite.clone(),
gate.clone(),
repair_tx.clone(),
health.clone(),
);
if let Err(e) = stored_redaction::spawn_pass(sqlite.clone(), embedder.clone(), compaction) {
tracing::error!("cannot start the stored-content redaction pass: {e}");
}
models::spawn_idle_unload(embedder.clone(), reranker.clone());
let progress = ProgressCheck {
store: sqlite.clone(),
data_dir: data_dir.clone(),
health: health.clone(),
llm_configured,
};
SyncLoop::over_ssh(data_dir.clone(), sqlite.clone(), gate.clone())
.with_health(health.clone())
.spawn();
let state = Arc::new(AppState {
store,
sqlite,
indexes,
raw_index,
raw_coverage,
reranker,
tunables: second_brain_api::daemon_tunables(config.retrieval.tunables()),
embedder,
machine,
maintenance,
repair_tx,
near_duplicate_threshold: config.distill.near_duplicate_threshold,
near_duplicates,
health: health.clone(),
gate: gate.clone(),
});
FootprintCheck {
paths: StorePaths {
raw_index: raw_dir.clone(),
db: db_path,
indexes: index_dir,
archive: archive_dir,
},
state: state.clone(),
health: health.clone(),
}
.spawn();
let app = second_brain_api::app(state, api_token.clone());
let announced = health.clone();
tokio::spawn(async move {
progress.start().await;
announced.announce(Utc::now());
});
WriteQueue::new(
data_dir.join(write_queue::DIR_NAME),
app.clone(),
api_token,
health,
gate,
)
.spawn();
second_brain_api::self_update::spawn();
startup.ready(app);
tracing::info!("startup finished; serving every route on {bind}");
match server.await {
Ok(Ok(())) => {}
Ok(Err(e)) => {
tracing::error!("serving on {bind} stopped: {e}");
std::process::exit(1);
}
Err(e) => {
tracing::error!("the server task on {bind} ended abnormally: {e}");
std::process::exit(1);
}
}
}