use anyhow::{Result, bail};
use clap::{Parser, Subcommand};
use fastembed::TextEmbedding;
use semlith::home;
use semlith::{Semlith, embed, embed::Model, filter::Filter, fleet::Fleet};
use std::io::{IsTerminal, Write};
use std::path::{Path, PathBuf};
use std::time::Instant;
#[derive(Parser)]
#[command(name = "semlith", version, about, long_about = None)]
struct Cli {
#[arg(long, short, global = true)]
store: Vec<PathBuf>,
#[command(subcommand)]
command: Command,
}
#[derive(Subcommand)]
enum Command {
Index {
paths: Vec<PathBuf>,
#[arg(long, short)]
model: Option<String>,
#[arg(long)]
name: Option<String>,
#[arg(long, short)]
quiet: bool,
#[arg(long)]
airgap: bool,
},
Start {
paths: Vec<PathBuf>,
#[arg(long)]
port: Option<u16>,
#[arg(long, default_value_t = semlith::watch::DEBOUNCE.as_millis() as u64)]
debounce: u64,
#[arg(long)]
airgap: bool,
},
Adopt {
store_dir: PathBuf,
#[arg(long)]
root: Option<PathBuf>,
#[arg(long)]
name: Option<String>,
},
Watch {
paths: Vec<PathBuf>,
#[arg(long, default_value_t = semlith::watch::DEBOUNCE.as_millis() as u64)]
debounce: u64,
#[arg(long, short)]
quiet: bool,
},
Search {
query: String,
#[arg(long, short, default_value_t = 8)]
k: usize,
#[arg(long, short)]
path: Vec<String>,
#[arg(long, short)]
ext: Vec<String>,
#[arg(long, short)]
lang: Vec<String>,
#[arg(long)]
json: bool,
},
Stats,
Files,
Forget { path: PathBuf },
Add {
url: String,
#[arg(long)]
name: Option<String>,
#[arg(long)]
airgap: bool,
},
Mcp,
Setup {
#[arg(long, short)]
yes: bool,
#[arg(long)]
airgap: bool,
},
Upgrade {
#[arg(long)]
check: bool,
#[arg(long)]
version: Option<String>,
#[arg(long)]
airgap: bool,
},
Models,
Languages,
}
fn main() -> Result<()> {
let cli = Cli::parse();
let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
match cli.command {
Command::Setup { yes, airgap } => {
arm_airgap(airgap);
semlith::setup::run(yes, airgap)?;
}
Command::Upgrade {
check,
version,
airgap,
} => {
arm_airgap(airgap);
if check {
if let Some(reason) = semlith::upgrade::offline() {
bail!("{reason}");
}
let found = semlith::upgrade::check()?;
println!("installed {}", found.installed);
println!("latest {}", found.latest.trim_start_matches('v'));
if let Some(reason) = &found.blocked {
println!("note: {reason}");
}
if found.available {
println!("run `semlith upgrade` to install it");
std::process::exit(semlith::upgrade::UPGRADE_AVAILABLE);
}
println!("already current");
} else {
semlith::upgrade::apply(version)?;
}
}
Command::Models => {
println!(
"{:<44} {:>5} dim default. IBM Granite R2 small, int8, English",
embed::GRANITE_NAME,
384
);
for info in TextEmbedding::list_supported_models() {
println!(
"{:<44} {:>5} dim {}",
info.model, info.dim, info.description
);
}
}
Command::Index {
paths,
model,
name,
quiet,
airgap,
} => {
arm_airgap(airgap);
let model = model
.map(|m| m.parse::<Model>().map_err(anyhow::Error::msg))
.transpose()?;
let roots = if paths.is_empty() {
vec![PathBuf::from(".")]
} else {
paths
};
let choice = home::resolve(&cli.store, &roots[0], name.as_deref())?;
if let Some(hint) = choice.hint() {
eprintln!("{hint}");
}
let dir = choice.one()?;
let mut store = Semlith::open(&dir, model)?;
store.quiet = quiet;
let started = Instant::now();
let mut spoke = Instant::now();
let report = store.index_paths(&roots, |path, p| {
if quiet {
return;
}
eprintln!(" + {}", display(path));
if spoke.elapsed() >= PROGRESS_INTERVAL {
spoke = Instant::now();
eprintln!(" {}", predict(p, started.elapsed()));
}
})?;
let model_name = store.model().to_string();
home::record(&choice, &roots, &model_name)?;
let (files, chunks, bytes) = store.stats()?;
eprintln!(
"indexed {} files ({} chunks) in {:.1}s — {} already indexed, {} skipped, {} removed",
report.indexed,
report.chunks,
started.elapsed().as_secs_f32(),
report.unchanged,
report.skipped,
report.removed,
);
eprintln!(
"store: {files} files, {chunks} chunks, {} at {}",
semlith::human_bytes(bytes),
dir.display()
);
}
Command::Watch {
paths,
debounce,
quiet,
} => {
let roots = if paths.is_empty() {
vec![PathBuf::from(".")]
} else {
paths
};
let choice = home::resolve(&cli.store, &roots[0], None)?;
if let Some(hint) = choice.hint() {
eprintln!("{hint}");
}
let dir = choice.one()?;
let mut store = Semlith::open(&dir, None)?;
store.quiet = quiet;
home::record(&choice, &roots, &store.model().to_string())?;
semlith::watch::stop_on_signal();
let shown: Vec<String> = roots.iter().map(|r| display(r)).collect();
semlith::watch::run(
&mut store,
&roots,
std::time::Duration::from_millis(debounce),
&semlith::watch::STOP,
|progress| {
use semlith::watch::Progress;
match progress {
Progress::Ready {
catch_up,
files,
chunks,
} => eprintln!(
"watching {} — {files} files, {chunks} chunks \
({} indexed at startup, {} unchanged)",
shown.join(", "),
catch_up.indexed,
catch_up.unchanged,
),
Progress::File(path) if !quiet => eprintln!(" ~ {}", display(path)),
Progress::Batch(report, elapsed) if !quiet => eprintln!(
" {} re-embedded, {} removed, {} chunks in {:.1}s",
report.indexed,
report.removed,
report.chunks,
elapsed.as_secs_f32(),
),
Progress::Error(e) => eprintln!("semlith: watch error: {e}"),
_ => {}
}
},
)?;
}
Command::Search {
query,
k,
path,
ext,
lang,
json,
} => {
let filter = Filter::new(&path, &ext, &lang)?;
let mut fleet = read_fleet(&cli.store, &cwd, false)?;
fleet.quiet = json;
let selected = (!filter.is_empty())
.then(|| fleet.matching_files(&filter))
.transpose()?;
if selected == Some(0) {
if json {
println!("[]");
} else {
eprintln!(
"no files match the filter (store has {} chunks)",
fleet.chunks()
);
}
return Ok(());
}
let started = Instant::now();
let hits = fleet.search_filtered(&query, k, &filter)?;
let elapsed = started.elapsed();
if json {
println!("{}", serde_json::to_string_pretty(&hits)?);
} else if hits.is_empty() {
eprintln!("no matches (store has {} chunks)", fleet.chunks());
} else {
let mut out = std::io::stdout().lock();
for (i, h) in hits.iter().enumerate() {
let from = match &h.store {
Some(label) => format!("[{label}] "),
None => String::new(),
};
writeln!(
out,
"{}{}. {:.3} {from}{}:{}-{}{}",
bold(),
i + 1,
h.score,
display(std::path::Path::new(&h.path)),
h.start_line,
h.end_line,
reset()
)?;
for line in h.text.lines() {
writeln!(out, " {line}")?;
}
writeln!(out)?;
}
let across = if fleet.len() > 1 {
let breakdown: Vec<String> = fleet
.labels()
.iter()
.map(|label| {
let n = hits
.iter()
.filter(|h| h.store.as_deref() == Some(*label))
.count();
format!("{label} {n}")
})
.collect();
format!(" across {} stores: {}", fleet.len(), breakdown.join(", "))
} else {
String::new()
};
match selected {
Some(n) => eprintln!(
"{} hits in {:?} (filter selected {n} of {} files){across}",
hits.len(),
elapsed,
fleet.files()?,
),
None => eprintln!("{} hits in {:?}{across}", hits.len(), elapsed),
}
let evicted: u64 = fleet.each().map(|(_, s)| s.evictions()).sum();
if evicted > 0 {
eprintln!(
"semlith: put down {evicted} shard(s) to stay inside the {} MB index \
budget; raise {} to hold more",
semlith::index::budget_mb(),
semlith::index::INDEX_MEMORY_ENV,
);
}
}
}
Command::Languages => {
for entry in semlith::filter::LANGUAGES {
let what = entry
.extensions
.iter()
.map(|e| format!(".{e}"))
.chain(entry.filenames.iter().map(|f| f.to_string()))
.collect::<Vec<_>>()
.join(" ");
println!("{:<12} {what}", entry.name);
}
}
Command::Stats => {
let fleet = read_fleet(&cli.store, &cwd, false)?;
let many = fleet.len() > 1;
let mut totals = (0, 0, 0);
for (label, store) in fleet.each() {
let (files, chunks, bytes) = store.stats()?;
totals = (totals.0 + files, totals.1 + chunks, totals.2 + bytes);
if many {
println!("{label}");
}
println!("store {}", store.dir().display());
println!("model {} ({} dim)", store.model(), store.dim());
println!("files {files}");
println!("chunks {chunks}");
println!("vectors {}", store.len());
if let Some((shards, max)) = store.shards() {
println!(
"shards {shards}, up to {max} resident ({} MB budget, {})",
semlith::index::budget_mb(),
semlith::index::INDEX_MEMORY_ENV,
);
}
println!("indexed {}", semlith::human_bytes(bytes));
if many {
println!();
}
}
if many {
println!(
"total {} stores, {} files, {} chunks, {}",
fleet.len(),
totals.0,
totals.1,
semlith::human_bytes(totals.2),
);
}
}
Command::Files => {
let fleet = read_fleet(&cli.store, &cwd, false)?;
let many = fleet.len() > 1;
for (label, store) in fleet.each() {
if many {
println!("{label}");
}
for path in semlith::store::all_paths(store.db())? {
println!("{}", display(std::path::Path::new(&path)));
}
if many {
println!();
}
}
}
Command::Add { url, name, airgap } => {
arm_airgap(airgap);
let choice = home::resolve(&cli.store, &cwd, name.as_deref())?;
let choice = resolve_for_add(choice, &cli.store, name.as_deref())?;
if let Some(hint) = choice.hint() {
eprintln!("{hint}");
}
let dir = choice.one()?;
let spinner = cliclack::spinner();
spinner.start(format!("fetching {url}"));
let fetched = match semlith::add::fetch(&url, &dir) {
Ok(fetched) => {
spinner.stop(format!(
"fetched {}",
semlith::human_bytes(fetched.bytes as i64)
));
fetched
}
Err(e) => {
spinner.error("fetch failed");
return Err(e);
}
};
let mut store = Semlith::open(&dir, None)?;
store.quiet = true;
let report = store.index_paths(std::slice::from_ref(&fetched.path), |_, _| {})?;
let model_name = store.model().to_string();
home::record(&choice, &[semlith::add::downloads_dir(&dir)], &model_name)?;
eprintln!("{} -> {}", fetched.url, display(&fetched.path));
eprintln!("indexed {} chunks into {}", report.chunks, dir.display());
}
Command::Forget { path } => {
let choice = home::resolve(&cli.store, &cwd, None)?;
if let Some(hint) = choice.hint() {
eprintln!("{hint}");
}
let mut store = Semlith::open(&choice.one()?, None)?;
let n = store.forget(&path)?;
eprintln!("removed {n} chunks for {}", path.display());
}
Command::Start {
paths,
port,
debounce,
airgap,
} => {
arm_airgap(airgap);
let dirs = semlith::daemon::stores_to_open(&cli.store, &paths, &cwd)?;
if dirs.is_empty() {
eprintln!("semlith: no store registered yet — the portal will offer to make one");
}
semlith::daemon::run(
&dirs,
semlith::daemon::port_of(port),
std::time::Duration::from_millis(debounce),
semlith::embed::airgap(),
|line| eprintln!("semlith: {line}"),
)?;
}
Command::Adopt {
store_dir,
root,
name,
} => {
let registered = home::Registry::load()?
.name_of(&store_dir)
.map(str::to_string)
.or_else(|| {
name.as_deref()
.filter(|n| home::Registry::load().is_ok_and(|r| r.stores.contains_key(*n)))
.map(str::to_string)
});
match (registered, &root) {
(Some(name), Some(root)) => {
home::repoint(&name, root)?;
eprintln!("{name} now indexes {}", root.display());
}
_ => {
let (name, target) = home::adopt(&store_dir, root.as_deref(), name.as_deref())?;
let store = Semlith::open_existing(&target)?;
let (files, chunks, bytes) = store.stats()?;
eprintln!(
"adopted {} as {name} — {files} files, {chunks} chunks, {} at {}",
store_dir.display(),
semlith::human_bytes(bytes),
target.display(),
);
}
}
}
Command::Mcp => {
let dirs = semlith::home::all_dirs(&cli.store, &cwd)?;
if let Some(upstream) = semlith::proxy::find(&semlith::proxy::candidates(&dirs)) {
eprintln!(
"semlith {}: forwarding to the daemon on 127.0.0.1:{} (found via {})",
env!("CARGO_PKG_VERSION"),
upstream.port,
upstream.via.display(),
);
return semlith::proxy::serve(
&upstream,
std::io::stdin().lock(),
std::io::stdout().lock(),
);
}
let mut fleet = read_fleet(&cli.store, &cwd, true)?;
fleet.quiet = true;
fleet.warm()?;
eprintln!(
"semlith {}: serving {} on {}",
env!("CARGO_PKG_VERSION"),
match fleet.len() {
1 => "1 store".to_string(),
n => format!("{n} stores"),
},
fleet.labels().join(", "),
);
semlith::mcp::serve(
&mut fleet,
std::io::stdin().lock(),
std::io::stdout().lock(),
)?;
}
}
Ok(())
}
fn arm_airgap(on: bool) {
if on {
unsafe { std::env::set_var(semlith::embed::AIRGAP_ENV, "1") };
}
}
fn read_fleet(flags: &[PathBuf], cwd: &Path, all: bool) -> Result<Fleet> {
let dirs = if all {
home::all_dirs(flags, cwd)?
} else {
home::read_dirs(flags, cwd)?
};
if dirs.is_empty() {
bail!(
"no semlith store covers {} and none is registered — \
run `semlith index` in a directory to make one, \
or `semlith adopt ./.semlith` to move an existing one into {}",
cwd.display(),
semlith::home::stores_root().display(),
);
}
Fleet::open(&dirs)
}
const PROGRESS_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2);
fn predict(p: semlith::IndexProgress, elapsed: std::time::Duration) -> String {
let secs = elapsed.as_secs_f32().max(0.001);
let rate = p.chunks as f32 / secs;
let left = p.total.saturating_sub(p.scanned);
let per_file = secs / p.scanned.max(1) as f32;
format!(
"{}/{} files, {} chunks, {rate:.0} chunks/s, ~{} left",
p.scanned,
p.total,
p.chunks,
human_duration(left as f32 * per_file),
)
}
fn human_duration(secs: f32) -> String {
if secs >= 3600.0 {
format!("{:.1}h", secs / 3600.0)
} else if secs >= 60.0 {
format!("{:.0}m", secs / 60.0)
} else {
format!("{secs:.0}s")
}
}
fn resolve_for_add(
choice: home::Choice,
flags: &[PathBuf],
name: Option<&str>,
) -> Result<home::Choice> {
if !matches!(choice, home::Choice::New { .. }) || !flags.is_empty() || name.is_some() {
return Ok(choice);
}
let registry = home::Registry::load()?;
let mut names = registry.stores.keys();
match (names.next(), names.next()) {
(Some(only), None) => {
let only = only.clone();
Ok(home::Choice::Registered {
dir: home::Registry::dir_of(&only),
name: only,
})
}
(None, _) => bail!(
"there is no store to add to yet — run `semlith index <path>` first, \
or give this one a name with `semlith add <url> --name <store>`"
),
_ => bail!(
"nothing indexes this directory, and there are {} stores to choose from: {}. \
Name one with `--store <dir>` or `--name <store>`, or run this from a \
directory one of them covers",
registry.stores.len(),
registry
.stores
.keys()
.cloned()
.collect::<Vec<_>>()
.join(", ")
),
}
}
fn display(path: &std::path::Path) -> String {
let cwd = std::env::current_dir().unwrap_or_default();
path.strip_prefix(&cwd)
.unwrap_or(path)
.display()
.to_string()
}
fn bold() -> &'static str {
if std::io::stdout().is_terminal() {
"\x1b[1m"
} else {
""
}
}
fn reset() -> &'static str {
if std::io::stdout().is_terminal() {
"\x1b[0m"
} else {
""
}
}