use crate::fleet::Fleet;
use crate::home::{self, Registry};
use crate::http::{self, Refusal, Server};
use crate::lock::StoreLock;
use crate::{Semlith, watch};
use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, VecDeque};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, RwLock, mpsc};
use std::time::{Duration, SystemTime};
pub const DISCOVERY_FILE: &str = "daemon.json";
const EVENT_HISTORY: usize = 20;
const SLICE: Duration = Duration::from_secs(45);
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Discovery {
pub pid: u32,
pub port: u16,
pub token: String,
pub version: String,
}
impl Discovery {
pub fn read(store_dir: &Path) -> Option<Self> {
if !home::Registry::load().ok()?.trusts(store_dir) {
return None;
}
let path = store_dir.join(DISCOVERY_FILE);
if !owner_only(&path) {
eprintln!(
"semlith: ignoring {} — it is not a file this user wrote privately",
path.display()
);
return None;
}
let text = std::fs::read_to_string(&path).ok()?;
let found: Self = serde_json::from_str(&text).ok()?;
if found.token.len() != 64 || !found.token.bytes().all(|b| b.is_ascii_hexdigit()) {
return None;
}
if !alive(found.pid) {
return None;
}
Some(found)
}
fn write(&self, store_dir: &Path) -> Result<()> {
let path = store_dir.join(DISCOVERY_FILE);
let mut options = std::fs::OpenOptions::new();
options.write(true).create(true).truncate(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(home::FILE_MODE);
}
let mut file = options
.open(&path)
.with_context(|| format!("writing {}", path.display()))?;
use std::io::Write;
file.write_all((serde_json::to_string_pretty(self)? + "\n").as_bytes())
.with_context(|| format!("writing {}", path.display()))?;
home::tighten_file(&path);
Ok(())
}
fn remove(store_dir: &Path) {
let _ = std::fs::remove_file(store_dir.join(DISCOVERY_FILE));
}
}
#[cfg(unix)]
fn owner_only(path: &Path) -> bool {
use std::os::unix::fs::MetadataExt;
use std::os::unix::fs::PermissionsExt;
let Ok(meta) = std::fs::metadata(path) else {
return false;
};
let me = unsafe { libc::getuid() };
meta.uid() == me && meta.permissions().mode() & 0o177 == 0
}
#[cfg(not(unix))]
fn owner_only(path: &Path) -> bool {
path.exists()
}
#[cfg(unix)]
fn alive(pid: u32) -> bool {
unsafe {
libc::kill(pid as i32, 0) == 0
|| std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)
}
}
#[cfg(not(unix))]
fn alive(_pid: u32) -> bool {
true
}
enum Job {
Index(Vec<PathBuf>, Vec<String>),
Forget(PathBuf),
}
struct Queued {
job: Job,
report: mpsc::Sender<serde_json::Value>,
}
#[derive(Debug, Clone, Serialize)]
pub struct Event {
pub at: u64,
pub text: String,
}
pub struct Store {
pub name: String,
pub dir: PathBuf,
pub roots: Vec<PathBuf>,
pub watched: Vec<PathBuf>,
queue: Mutex<VecDeque<Queued>>,
events: Mutex<VecDeque<Event>>,
pub watching: AtomicBool,
pub stop: AtomicBool,
pub paused: AtomicBool,
pub cancelled: AtomicBool,
pub last_write: AtomicUsize,
}
impl Store {
fn note(&self, text: String) {
let mut events = self.events.lock().unwrap_or_else(|e| e.into_inner());
if events.len() == EVENT_HISTORY {
events.pop_front();
}
events.push_back(Event { at: now(), text });
}
pub fn events(&self) -> Vec<Event> {
self.events
.lock()
.unwrap_or_else(|e| e.into_inner())
.iter()
.cloned()
.collect()
}
fn submit(
&self,
job: Job,
notice: Option<serde_json::Value>,
) -> mpsc::Receiver<serde_json::Value> {
let (report, progress) = mpsc::channel();
let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner());
if let Some(mut notice) = notice {
if let Some(object) = notice.as_object_mut() {
object.insert("store".into(), serde_json::json!(self.name));
object.insert("ahead".into(), serde_json::json!(queue.len()));
}
let _ = report.send(notice);
}
queue.push_back(Queued { job, report });
progress
}
pub fn queue_depth(&self) -> usize {
self.queue.lock().unwrap_or_else(|e| e.into_inner()).len()
}
fn requeue(&self, queued: Queued) {
self.queue
.lock()
.unwrap_or_else(|e| e.into_inner())
.push_back(queued);
}
pub fn cancel_queued(&self) -> usize {
let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner());
let mut dropped = 0;
queue.retain(|queued| {
if !matches!(queued.job, Job::Index(..)) {
return true;
}
let _ = queued.report.send(serde_json::json!({
"event": "done",
"indexed": 0,
"unchanged": 0,
"skipped": 0,
"removed": 0,
"chunks": 0,
"images": 0,
"remaining": 0,
"stopped": true,
}));
dropped += 1;
false
});
dropped
}
}
pub struct State {
pub server: Arc<Server>,
pub fleet: Mutex<Option<Fleet>>,
stores: RwLock<Vec<Arc<Store>>>,
pub airgap: bool,
pub started: SystemTime,
debounce: Duration,
report: Arc<dyn Fn(&str) + Send + Sync>,
pub refusals: Mutex<BTreeMap<&'static str, u64>>,
pub proxies: Mutex<BTreeMap<u32, u64>>,
pub clients: Mutex<BTreeMap<String, Client>>,
pub mcp_fleet: Mutex<Option<Fleet>>,
pub ledger: bool,
}
const PROXY_FRESH: u64 = 120;
fn named(stores: &[Arc<Store>]) -> Vec<(PathBuf, String)> {
stores
.iter()
.map(|s| (s.dir.clone(), s.name.clone()))
.collect()
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct Client {
pub name: String,
pub transport: String,
pub revision: String,
pub queries: u64,
pub seen: u64,
}
impl State {
pub fn stores(&self) -> Vec<Arc<Store>> {
self.stores.read().expect("the stores lock").clone()
}
pub fn store(&self, name: &str) -> Option<Arc<Store>> {
self.stores().into_iter().find(|s| s.name == name)
}
pub fn writable(&self, name: Option<&str>) -> Result<Arc<Store>> {
let stores = self.stores();
match (name, stores.as_slice()) {
(Some(name), _) => self
.store(name)
.with_context(|| format!("no store called {name} is open")),
(None, [one]) => Ok(Arc::clone(one)),
(None, []) => bail!("no store is open"),
(None, many) => bail!(
"this daemon has {} stores open, so a write has to name one: {}",
many.len(),
many.iter()
.map(|s| s.name.clone())
.collect::<Vec<_>>()
.join(", ")
),
}
}
pub fn index(
&self,
store: &Arc<Store>,
paths: Vec<PathBuf>,
) -> Result<mpsc::Receiver<serde_json::Value>> {
Self::writer_alive(store)?;
Ok(store.submit(
Job::Index(paths, Vec::new()),
Some(serde_json::json!({ "event": "queued" })),
))
}
pub fn forget(
&self,
store: &Arc<Store>,
path: PathBuf,
) -> Result<mpsc::Receiver<serde_json::Value>> {
Self::writer_alive(store)?;
Ok(store.submit(Job::Forget(path), None))
}
pub fn delete_store(&self, name: &str) -> Result<PathBuf> {
let Some(store) = self.store(name) else {
anyhow::bail!("this daemon is not serving a store called {name}");
};
store.stop.store(true, Ordering::SeqCst);
let deadline = std::time::Instant::now() + Duration::from_secs(10);
while store.watching.load(Ordering::Relaxed) {
if std::time::Instant::now() > deadline {
anyhow::bail!(
"{name}'s watcher did not stop, so its lock is still held; nothing was deleted"
);
}
std::thread::sleep(Duration::from_millis(50));
}
self.stores
.write()
.expect("the stores lock")
.retain(|s| s.name != name);
self.reopen_readers();
Discovery::remove(&store.dir);
crate::home::delete_store(name)
}
fn writer_alive(store: &Arc<Store>) -> Result<()> {
if store.watching.load(Ordering::Relaxed) {
return Ok(());
}
bail!(
"the writer for {} is not running, so it cannot be written to; \
the daemon's stderr says why, and restarting it is the fix",
store.name
)
}
pub fn rotate(&self) -> String {
let fresh = self.server.rotate();
for store in self.stores() {
let _ = discovery(self.server.port(), &fresh).write(&store.dir);
}
fresh
}
pub fn client_name(&self, session: &str, transport: &str) -> Option<String> {
let clients = self.clients.lock().unwrap_or_else(|e| e.into_inner());
clients
.get(&format!("{transport}:{session}"))
.map(|client| client.name.clone())
.filter(|name| name != "unnamed client")
}
pub fn note_client(
&self,
session: &str,
transport: &str,
name: Option<&str>,
revision: Option<&str>,
query: bool,
) {
let now = now();
let mut clients = self.clients.lock().unwrap_or_else(|e| e.into_inner());
let entry = clients
.entry(format!("{transport}:{session}"))
.or_insert_with(|| Client {
name: name.unwrap_or("unnamed client").to_string(),
transport: transport.to_string(),
revision: revision.unwrap_or("—").to_string(),
queries: 0,
seen: now,
});
if let Some(name) = name {
entry.name = name.to_string();
}
if let Some(revision) = revision {
entry.revision = revision.to_string();
}
if query {
entry.queries += 1;
}
entry.seen = now;
clients.retain(|_, client| now.saturating_sub(client.seen) <= PROXY_FRESH);
}
pub fn clients(&self) -> Vec<Client> {
let now = now();
self.clients
.lock()
.unwrap_or_else(|e| e.into_inner())
.values()
.filter(|client| now.saturating_sub(client.seen) <= PROXY_FRESH)
.cloned()
.collect()
}
pub fn saw_proxy(&self, pid: u32) {
let mut proxies = self.proxies.lock().unwrap_or_else(|e| e.into_inner());
let now = now();
proxies.insert(pid, now);
proxies.retain(|_, seen| now.saturating_sub(*seen) <= PROXY_FRESH);
}
pub fn proxy_count(&self) -> usize {
let now = now();
self.proxies
.lock()
.unwrap_or_else(|e| e.into_inner())
.values()
.filter(|seen| now.saturating_sub(**seen) <= PROXY_FRESH)
.count()
}
pub fn open_mcp_fleet(&self) -> Result<()> {
let mut fleet = self.mcp_fleet.lock().unwrap_or_else(|e| e.into_inner());
if fleet.is_some() {
return Ok(());
}
let open = self.stores();
let dirs: Vec<PathBuf> = open.iter().map(|s| s.dir.clone()).collect();
if dirs.is_empty() {
bail!("this daemon has no store open");
}
let mut opened = Fleet::open(&dirs)?;
opened.name_from(&named(&open));
opened.quiet = true;
*fleet = Some(opened);
Ok(())
}
pub fn open_store(self: &Arc<Self>, dir: &Path) -> Result<Arc<Store>> {
let dir = crate::canonical(dir);
if let Some(open) = self.stores().into_iter().find(|s| s.dir == dir) {
return Ok(open);
}
let lock = StoreLock::acquire(&dir)
.with_context(|| format!("{} cannot be opened by the daemon", dir.display()))?;
let registry = Registry::load()?;
let (name, roots) = roots_for(&dir, ®istry);
let watched: Vec<PathBuf> = roots.iter().filter(|r| r.exists()).cloned().collect();
let store = Arc::new(Store {
name: name.clone(),
dir: dir.clone(),
roots,
watched,
queue: Mutex::new(VecDeque::new()),
events: Mutex::new(VecDeque::new()),
watching: AtomicBool::new(true),
stop: AtomicBool::new(false),
paused: AtomicBool::new(false),
cancelled: AtomicBool::new(false),
last_write: AtomicUsize::new(0),
});
discovery(self.server.port(), &self.server.token()).write(&store.dir)?;
self.stores
.write()
.expect("the stores lock")
.push(Arc::clone(&store));
let watching = Arc::clone(&store);
let debounce = self.debounce;
let report = Arc::clone(&self.report);
std::thread::spawn(move || {
let _lock = lock;
if let Err(e) = tend(&watching, debounce, &watching.stop, &*report) {
report(&format!("{}: watcher stopped: {e}", watching.name));
watching.note(format!("watcher stopped: {e}"));
}
watching.watching.store(false, Ordering::Relaxed);
});
(self.report)(&format!(
"opened {name} at {} — now serving it",
dir.display()
));
self.reopen_readers();
Ok(store)
}
fn reopen_readers(&self) {
*self.fleet.lock().unwrap_or_else(|e| e.into_inner()) = None;
*self.mcp_fleet.lock().unwrap_or_else(|e| e.into_inner()) = None;
}
pub fn open_fleet(&self) -> Result<()> {
let mut fleet = self.fleet.lock().unwrap_or_else(|e| e.into_inner());
if fleet.is_some() {
return Ok(());
}
let open = self.stores();
let dirs: Vec<PathBuf> = open.iter().map(|s| s.dir.clone()).collect();
if dirs.is_empty() {
return Ok(());
}
let mut opened = Fleet::open(&dirs)?;
opened.name_from(&named(&open));
opened.quiet = true;
*fleet = Some(opened);
Ok(())
}
pub fn refuse(&self, class: Refusal) {
*self
.refusals
.lock()
.unwrap_or_else(|e| e.into_inner())
.entry(class.as_str())
.or_insert(0) += 1;
}
}
fn discovery(port: u16, token: &str) -> Discovery {
Discovery {
pid: std::process::id(),
port,
token: token.to_string(),
version: env!("CARGO_PKG_VERSION").to_string(),
}
}
pub fn stores_to_open(flags: &[PathBuf], paths: &[PathBuf], cwd: &Path) -> Result<Vec<PathBuf>> {
if !flags.is_empty() {
return Ok(flags.to_vec());
}
if paths.is_empty() {
return home::all_dirs(&[], cwd);
}
let mut out = Vec::new();
for path in paths {
let dir = home::resolve(&[], path, None)?.one()?;
if !dir.join("store.db").exists() {
bail!(
"{} has no store yet — run `semlith index {}` first",
path.display(),
path.display()
);
}
if !out.contains(&dir) {
out.push(dir);
}
}
Ok(out)
}
fn roots_for(dir: &Path, registry: &Registry) -> (String, Vec<PathBuf>) {
if let Some(name) = registry.name_of(dir) {
let roots = registry.stores[name].roots.clone();
return (name.to_string(), roots);
}
let name = dir
.parent()
.and_then(Path::file_name)
.or_else(|| dir.file_name())
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| "store".to_string());
let root = dir
.parent()
.map(crate::canonical)
.unwrap_or_else(|| dir.to_path_buf());
(name, vec![root])
}
#[allow(clippy::too_many_arguments)]
pub fn run(
dirs: &[PathBuf],
port: u16,
debounce: Duration,
airgap: bool,
ledger: bool,
mcp_http: bool,
report: impl Fn(&str) + Send + Sync + 'static,
) -> Result<Arc<State>> {
let registry = Registry::load()?;
let mut locks = Vec::new();
let mut opening = Vec::new();
for dir in dirs {
let lock = StoreLock::acquire(dir)
.with_context(|| format!("{} cannot be opened by the daemon", dir.display()))?;
let (name, roots) = roots_for(dir, ®istry);
opening.push((name, dir.clone(), roots));
locks.push(lock);
}
let server = Arc::new(Server::bind(port)?);
report(&format!("listening on 127.0.0.1:{}", server.port()));
report(if ledger {
"ledger: recording (local only; --no-ledger to stop)"
} else {
"ledger: off for this session"
});
let key = crate::home::agent_key()?;
server.set_agent_key(&key);
server.set_mcp_open(mcp_http);
if mcp_http {
report(&format!(
"MCP over HTTP at http://127.0.0.1:{}{}",
server.port(),
crate::http::MCP_PATH
));
} else {
report("MCP over HTTP is closed (--no-mcp-http); `semlith mcp` over stdio is unaffected");
}
let mut stores = Vec::new();
for (name, dir, roots) in opening {
let watched: Vec<PathBuf> = roots.iter().filter(|r| r.exists()).cloned().collect();
for missing in roots.iter().filter(|r| !r.exists()) {
report(&format!(
"{name}: root {} is not there; not watching it",
missing.display()
));
}
report(&format!(
"opened {name} at {} — watching {} root(s)",
dir.display(),
watched.len()
));
stores.push(Arc::new(Store {
name,
dir,
roots,
watched,
queue: Mutex::new(VecDeque::new()),
events: Mutex::new(VecDeque::new()),
watching: AtomicBool::new(false),
stop: AtomicBool::new(false),
paused: AtomicBool::new(false),
cancelled: AtomicBool::new(false),
last_write: AtomicUsize::new(0),
}));
}
let fleet = if dirs.is_empty() {
None
} else {
let mut fleet = Fleet::open(dirs)?;
fleet.name_from(&named(&stores));
fleet.quiet = true;
Some(fleet)
};
let token = server.token();
for store in &stores {
discovery(server.port(), &token).write(&store.dir)?;
}
let report_line: Arc<dyn Fn(&str) + Send + Sync> = Arc::new(report);
let state = Arc::new(State {
server: Arc::clone(&server),
fleet: Mutex::new(fleet),
stores: RwLock::new(stores),
airgap,
started: SystemTime::now(),
debounce,
report: Arc::clone(&report_line),
refusals: Mutex::new(BTreeMap::new()),
proxies: Mutex::new(BTreeMap::new()),
clients: Mutex::new(BTreeMap::new()),
mcp_fleet: Mutex::new(None),
ledger,
});
watch::stop_on_signal();
let report = Arc::clone(&report_line);
let mut watchers = Vec::new();
for (store, lock) in state.stores().into_iter().zip(locks) {
let report = Arc::clone(&report);
store.watching.store(true, Ordering::Relaxed);
watchers.push(std::thread::spawn(move || {
let _lock = lock;
if let Err(e) = tend(&store, debounce, &store.stop, &*report) {
report(&format!("{}: watcher stopped: {e}", store.name));
store.note(format!("watcher stopped: {e}"));
}
store.watching.store(false, Ordering::Relaxed);
}));
}
{
let warming = Arc::clone(&state);
let report = report_line.clone();
std::thread::spawn(move || {
let mut fleet = warming.fleet.lock().unwrap_or_else(|e| e.into_inner());
if let Some(fleet) = fleet.as_mut()
&& let Err(e) = fleet.warm()
{
report(&format!("could not load the embedding model: {e}"));
}
});
}
println!("{}", server.url());
report("open the URL on stdout to reach the portal; Ctrl-C stops the daemon");
Ok(state.clone()).and_then(|state| {
let handler = crate::routes::handler(Arc::clone(&state));
let for_refusals = Arc::clone(&state);
let result = server.serve(handler, &watch::STOP, move |class| {
for_refusals.refuse(class);
});
watch::STOP.store(true, Ordering::SeqCst);
for store in state.stores() {
store.stop.store(true, Ordering::SeqCst);
}
for watcher in watchers {
let _ = watcher.join();
}
for store in state.stores() {
Discovery::remove(&store.dir);
}
let refusals = state
.refusals
.lock()
.unwrap_or_else(|e| e.into_inner())
.clone();
if !refusals.is_empty() {
let counts: Vec<String> = refusals
.iter()
.map(|(class, n)| format!("{class} {n}"))
.collect();
report(&format!("refused requests: {}", counts.join(", ")));
}
report("stopped");
result.map(|()| state)
})
}
fn tend(
store: &Arc<Store>,
debounce: Duration,
stop: &AtomicBool,
report: &(dyn Fn(&str) + Send + Sync),
) -> Result<()> {
let mut writer = Semlith::open(&store.dir, None)?;
writer.quiet = true;
let roots = store.watched.clone();
watch::run_held(
&mut writer,
&roots,
debounce,
stop,
&|| store.queue_depth() > 0,
|progress| {
use watch::Progress;
match progress {
Progress::Ready {
catch_up,
files,
chunks,
} => {
let text = format!(
"watching — {files} files, {chunks} chunks \
({} indexed at startup, {} unchanged)",
catch_up.indexed, catch_up.unchanged
);
report(&format!("{}: {text}", store.name));
store.note(text);
}
Progress::Batch(batch, elapsed) => {
let text = format!(
"{} re-embedded, {} removed, {} chunks in {:.1}s",
batch.indexed,
batch.removed,
batch.chunks,
elapsed.as_secs_f32()
);
report(&format!("{}: {text}", store.name));
store.note(text);
store.last_write.store(now() as usize, Ordering::Relaxed);
}
Progress::Error(e) => {
report(&format!("{}: watch error: {e}", store.name));
store.note(format!("error: {e}"));
}
Progress::File(_) => {}
}
},
|writer| {
loop {
let Some(next) = store
.queue
.lock()
.unwrap_or_else(|e| e.into_inner())
.pop_front()
else {
return Ok(());
};
perform(store, writer, next);
}
},
)
}
fn perform(store: &Arc<Store>, writer: &mut Semlith, queued: Queued) {
let Queued { job, report } = queued;
let back = report.clone();
let say = |value: serde_json::Value| {
let _ = report.send(value);
};
match job {
Job::Index(paths, already) => {
if already.is_empty() {
let names: Vec<String> = paths.iter().map(|p| p.display().to_string()).collect();
say(serde_json::json!({ "event": "started", "paths": names }));
}
let started_at = std::time::Instant::now();
writer.boundary = crate::Boundary::within(crate::home::index_roots(&store.dir));
store.paused.store(false, Ordering::Relaxed);
let control = {
let store = Arc::clone(store);
let told = std::sync::atomic::AtomicBool::new(false);
move || {
if store.cancelled.load(Ordering::Relaxed) {
return crate::Flow::Stop;
}
if store.paused.load(Ordering::Relaxed) {
if !told.swap(true, Ordering::Relaxed) {
say(serde_json::json!({ "event": "paused" }));
}
return crate::Flow::Pause;
}
if told.swap(false, Ordering::Relaxed) {
say(serde_json::json!({ "event": "resumed" }));
}
crate::Flow::Run
}
};
let outcome =
writer.index_within_held_under(&paths, SLICE, &control, |path, progress| {
say(serde_json::json!({
"event": "file",
"path": path.display().to_string(),
"outcome": progress.outcome.as_str(),
"scanned": progress.scanned,
"total": progress.total,
"indexed": progress.indexed,
"chunks": progress.chunks,
"symbols": progress.symbols,
"elapsed_ms": started_at.elapsed().as_millis() as u64,
}));
});
match outcome {
Ok(mut done) => {
store.last_write.store(now() as usize, Ordering::Relaxed);
let mut written = already;
written.append(&mut done.written);
if done.stopped {
let mut undone = 0;
for key in &written {
if writer.forget_held(Path::new(key)).is_ok() {
undone += 1;
}
}
store.note(format!(
"an index run was stopped; {undone} file(s) it had embedded were undone"
));
say(serde_json::json!({
"event": "done",
"indexed": 0,
"unchanged": 0,
"skipped": 0,
"removed": undone,
"chunks": 0,
"images": 0,
"remaining": 0,
"stopped": true,
}));
store.paused.store(false, Ordering::Relaxed);
store.cancelled.store(false, Ordering::Relaxed);
return;
}
if done.remaining > 0 {
store.requeue(Queued {
job: Job::Index(paths, written),
report: back,
});
say(serde_json::json!({
"event": "slice",
"remaining": done.remaining,
"indexed": done.indexed,
"chunks": done.chunks,
}));
return;
}
store.note(format!("{} indexed from the portal", done.indexed));
say(serde_json::json!({
"event": "done",
"indexed": done.indexed,
"unchanged": done.unchanged,
"skipped": done.skipped,
"removed": done.removed,
"chunks": done.chunks,
"images": done.images,
"remaining": done.remaining,
"stopped": done.stopped,
"refused": done.refused.iter().map(|(path, why)| {
serde_json::json!({ "path": path, "why": why })
}).collect::<Vec<_>>(),
}));
}
Err(e) => say(serde_json::json!({ "event": "error", "error": e.to_string() })),
}
store.paused.store(false, Ordering::Relaxed);
store.cancelled.store(false, Ordering::Relaxed);
}
Job::Forget(path) => match writer.forget_held(&path) {
Ok((chunks, images)) => {
store.last_write.store(now() as usize, Ordering::Relaxed);
store.note(format!("forgot {}", path.display()));
say(serde_json::json!({
"event": "done",
"forgot": chunks,
"images": images,
"path": path.display().to_string(),
}));
}
Err(e) => say(serde_json::json!({ "event": "error", "error": e.to_string() })),
},
}
}
pub struct Writer(pub Arc<State>);
impl crate::mcp::Writer for Writer {
fn index(&self, store: Option<&str>, paths: &[PathBuf]) -> Result<String, String> {
let store = self.0.writable(store).map_err(|e| e.to_string())?.clone();
let progress = self
.0
.index(&store, paths.to_vec())
.map_err(|e| e.to_string())?;
let mut last = None;
for event in progress {
match event["event"].as_str() {
Some("error") => {
return Err(event["error"]
.as_str()
.unwrap_or("indexing failed")
.to_string());
}
Some("done") => last = Some(event),
_ => {}
}
}
let done = last.ok_or_else(|| "the writer stopped before answering".to_string())?;
let count = |key: &str| done[key].as_u64().unwrap_or(0);
let mut text = format!(
"{} indexed, {} unchanged, {} skipped, {} removed ({} chunks)",
count("indexed"),
count("unchanged"),
count("skipped"),
count("removed"),
count("chunks"),
);
if count("remaining") > 0 {
text.push_str(&format!(
"\nStopped at the time limit with {} paths remaining. \
Call semlith_index again with the same arguments to continue; \
nothing already indexed is redone.",
count("remaining")
));
}
Ok(text)
}
fn add(&self, store: Option<&str>, url: &str) -> Result<String, String> {
let store = self.0.writable(store).map_err(|e| e.to_string())?.clone();
let fetched = crate::add::fetch(url, &store.dir).map_err(|e| format!("{e:#}"))?;
let indexed = self.index(Some(&store.name), std::slice::from_ref(&fetched.path))?;
Ok(format!(
"Fetched {} into {}. {indexed}",
fetched.url,
fetched.path.display()
))
}
fn forget(&self, store: Option<&str>, path: &str) -> Result<String, String> {
let store = self.0.writable(store).map_err(|e| e.to_string())?.clone();
let progress = self
.0
.forget(&store, PathBuf::from(path))
.map_err(|e| e.to_string())?;
let done = progress
.recv()
.map_err(|_| "the writer stopped before answering".to_string())?;
if let Some(error) = done["error"].as_str() {
return Err(error.to_string());
}
let chunks = done["forgot"].as_u64().unwrap_or(0);
let images = done["images"].as_u64().unwrap_or(0);
Ok(match (chunks, images) {
(0, 0) => format!("{path} was not indexed; nothing removed."),
(0, images) => format!("Removed {images} image vector(s) for {path}."),
(chunks, 0) => format!("Removed {chunks} chunks for {path}."),
(chunks, images) => {
format!("Removed {chunks} chunks and {images} image vector(s) for {path}.")
}
})
}
}
pub fn url(state: &State) -> String {
state.server.url()
}
pub fn uptime(state: &State) -> u64 {
state
.started
.elapsed()
.map(|d| d.as_secs())
.unwrap_or_default()
}
pub fn now() -> u64 {
SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0)
}
pub fn port_of(flag: Option<u16>) -> u16 {
flag.or_else(|| {
std::env::var(http::PORT_ENV)
.ok()
.and_then(|v| v.parse().ok())
})
.unwrap_or(http::DEFAULT_PORT)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn an_unregistered_store_is_named_after_the_directory_holding_it() {
let registry = Registry::default();
let (name, roots) = roots_for(Path::new("/work/api/.semlith"), ®istry);
assert_eq!(name, "api");
assert_eq!(roots.len(), 1);
assert!(roots[0].ends_with("api") || roots[0] == Path::new("/work/api"));
}
#[test]
fn an_unnamed_write_among_several_stores_is_refused() {
let state = State {
server: Arc::new(Server::bind(0).expect("an ephemeral port")),
fleet: Mutex::new(None),
stores: RwLock::new(
["api", "cli"]
.into_iter()
.map(|name| {
Arc::new(Store {
name: name.to_string(),
dir: PathBuf::from("/nowhere").join(name),
roots: Vec::new(),
watched: Vec::new(),
queue: Mutex::new(VecDeque::new()),
events: Mutex::new(VecDeque::new()),
watching: AtomicBool::new(false),
stop: AtomicBool::new(false),
paused: AtomicBool::new(false),
cancelled: AtomicBool::new(false),
last_write: AtomicUsize::new(0),
})
})
.collect(),
),
airgap: false,
started: SystemTime::now(),
debounce: Duration::from_millis(500),
report: Arc::new(|_| {}),
refusals: Mutex::new(BTreeMap::new()),
proxies: Mutex::new(BTreeMap::new()),
clients: Mutex::new(BTreeMap::new()),
mcp_fleet: Mutex::new(None),
ledger: false,
};
let err = match state.writable(None) {
Err(e) => e.to_string(),
Ok(store) => panic!("an unnamed write picked {}", store.name),
};
assert!(err.contains("api, cli"), "{err}");
assert_eq!(
state.writable(Some("cli")).map(|s| s.name.clone()).ok(),
Some("cli".to_string())
);
assert!(state.writable(Some("nope")).is_err());
}
}