use super::connection::Connection;
use super::schema::init_db;
use crate::core::extract;
use crate::core::fields;
use crate::core::search::serialize_f32_vec;
use crate::core::search::SharedEmbedder;
use crate::core::text::{fts_segment, same_tokens};
use anyhow::{Context, Result};
use std::fs;
use std::path::{Path, PathBuf};
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct IndexStats {
pub scanned: usize,
pub indexed: usize,
pub unchanged: usize,
pub skipped: usize,
pub chunks: usize,
pub warnings: usize,
}
impl IndexStats {
pub fn merge(&mut self, other: IndexStats) {
self.scanned += other.scanned;
self.indexed += other.indexed;
self.unchanged += other.unchanged;
self.skipped += other.skipped;
self.chunks += other.chunks;
self.warnings += other.warnings;
}
}
#[derive(Debug, Clone)]
pub struct Chunk {
pub page: i64,
pub line_start: i64,
pub line_end: i64,
pub anchor: Option<String>,
pub section: Option<String>,
pub content: String,
}
const INDEXER_VERSION: &str = concat!(env!("CARGO_PKG_VERSION"), "+extract3+chunk1");
pub struct Indexer {
conn: Connection,
embedder: Option<SharedEmbedder>,
max_chunk_words: usize,
overlap_words: usize,
ocr_enabled: bool,
}
impl Indexer {
pub fn new(conn: Connection, embedder: Option<SharedEmbedder>) -> Result<Self> {
Self::with_ocr(conn, embedder, false)
}
pub fn with_ocr(
conn: Connection,
embedder: Option<SharedEmbedder>,
ocr_enabled: bool,
) -> Result<Self> {
init_db(&conn)?;
invalidate_on_upgrade(&conn)?;
if let Some(embedder) = &embedder {
Self::record_embedding_model(&conn, embedder.model_id())?;
}
Ok(Self {
conn,
embedder,
max_chunk_words: 400,
overlap_words: 40,
ocr_enabled,
})
}
fn record_embedding_model(conn: &Connection, model_id: &str) -> Result<()> {
conn.inner().execute(
"INSERT INTO index_meta (key, value) VALUES ('embedding_model', ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
[model_id],
)?;
Ok(())
}
pub fn connection(&self) -> &Connection {
&self.conn
}
pub fn index_roots(&self, roots: &[PathBuf]) -> Result<IndexStats> {
let mut stats = IndexStats::default();
for root in roots {
if root.is_file() {
stats.merge(self.index_file(root)?);
continue;
}
for entry in walkdir::WalkDir::new(root)
.follow_links(false)
.into_iter()
.filter_entry(|e| !is_hidden(e.file_name()))
{
let entry = match entry {
Ok(e) => e,
Err(e) => {
tracing::warn!("walk error: {e}");
stats.skipped += 1;
continue;
}
};
if entry.file_type().is_file() {
match self.index_file(entry.path()) {
Ok(file_stats) => stats.merge(file_stats),
Err(err) => {
tracing::warn!("{}: {err}", entry.path().display());
stats.skipped += 1;
}
}
}
}
}
Ok(stats)
}
pub fn index_file(&self, path: &Path) -> Result<IndexStats> {
let mut stats = IndexStats {
scanned: 1,
..Default::default()
};
let path_string = path.to_string_lossy().to_string();
if !extract::is_supported(&path_string) {
stats.skipped = 1;
return Ok(stats);
}
let metadata = std::fs::symlink_metadata(path)?;
if metadata.file_type().is_symlink() || !metadata.is_file() {
stats.skipped = 1;
return Ok(stats);
}
let mut max_allowed: u64 = 0;
for extractor in extract::extractors() {
if extractor
.extensions()
.contains(&extract::extension_of(&path_string).as_str())
{
max_allowed = max_allowed.max(extractor.capabilities().max_file_bytes);
}
}
if max_allowed == 0 || metadata.len() > max_allowed {
stats.skipped = 1;
return Ok(stats);
}
let mtime = metadata
.modified()?
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_millis() as i64;
let size = metadata.len() as i64;
let unchanged: Option<i64> = self
.conn
.inner()
.query_row(
"SELECT 1 FROM chunks WHERE source_path = ? AND mtime = ? AND size = ? LIMIT 1",
rusqlite::params![path_string, mtime, size],
|row| row.get(0),
)
.optional()?;
if unchanged.is_some() {
stats.unchanged = 1;
return Ok(stats);
}
let bytes = std::fs::read(path).with_context(|| format!("reading {path_string}"))?;
let hash = blake3::hash(&bytes);
let stored_hash: Option<Vec<u8>> = self
.conn
.inner()
.query_row(
"SELECT content_hash FROM chunks WHERE source_path = ? LIMIT 1",
[&path_string],
|row| row.get(0),
)
.optional()?;
if stored_hash.as_deref() == Some(hash.as_bytes().as_slice()) {
self.conn.inner().execute(
"UPDATE chunks SET mtime = ?, size = ? WHERE source_path = ?",
rusqlite::params![mtime, size, path_string],
)?;
stats.unchanged = 1;
return Ok(stats);
}
let docs = match extract::extract_named(&path_string, &bytes, self.ocr_enabled)? {
Some(docs) => docs,
None => {
stats.skipped = 1;
return Ok(stats);
}
};
let tx = self.conn.inner().unchecked_transaction()?;
tx.execute(
"DELETE FROM chunks_vec WHERE rowid IN (SELECT id FROM chunks WHERE source_path = ?)",
[&path_string],
)?;
tx.execute("DELETE FROM chunks WHERE source_path = ?", [&path_string])?;
let mut inserted = 0usize;
let mut warnings = 0usize;
for doc in &docs {
for warning in &doc.warnings {
tracing::warn!("{path_string}: {warning}");
warnings += 1;
}
let chunks = chunk_blocks(&doc.blocks, self.max_chunk_words, self.overlap_words);
for chunk in &chunks {
let mut fts_text = fts_segment(&chunk.content);
if same_tokens(&fts_text, &chunk.content) {
fts_text.clear();
}
let section = chunk.section.as_deref().unwrap_or("");
let mut fts_section = fts_segment(section);
if same_tokens(&fts_section, section) {
fts_section.clear();
}
let metadata = fields::analyse(&chunk.content);
tx.execute(
"INSERT INTO chunks
(path, source_path, page, line_start, line_end, anchor, section, content,
fts_text, fts_section, content_hash, mtime, size, metadata)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
rusqlite::params![
doc.path,
path_string,
chunk.page,
chunk.line_start,
chunk.line_end,
chunk.anchor,
chunk.section,
chunk.content,
fts_text,
fts_section,
hash.as_bytes().as_slice(),
mtime,
size,
metadata,
],
)?;
let id = tx.last_insert_rowid();
if let Some(embedder) = &self.embedder {
let vector = embedder.embed(&chunk.content)?;
tx.execute(
"INSERT INTO chunks_vec (rowid, embedding) VALUES (?, ?)",
rusqlite::params![id, serialize_f32_vec(&vector)],
)?;
}
inserted += 1;
}
}
tx.commit()?;
stats.warnings = warnings;
if inserted == 0 {
stats.skipped = 1;
} else {
stats.indexed = 1;
stats.chunks = inserted;
}
Ok(stats)
}
pub fn reindex_file(&self, path: &Path) -> Result<IndexStats> {
self.index_file(path)
}
pub fn prune_missing(&self) -> Result<usize> {
let mut stmt = self
.conn
.inner()
.prepare("SELECT DISTINCT source_path FROM chunks")?;
let paths: Vec<String> = stmt
.query_map([], |row| row.get(0))?
.collect::<std::result::Result<_, _>>()?;
drop(stmt);
let missing: Vec<String> = paths
.into_iter()
.filter(|path| !path.is_empty() && !Path::new(path).exists())
.collect();
if missing.is_empty() {
return Ok(0);
}
let tx = self.conn.inner().unchecked_transaction()?;
{
let mut del_vec = tx.prepare(
"DELETE FROM chunks_vec WHERE rowid IN
(SELECT id FROM chunks WHERE source_path = ?1)",
)?;
let mut del_chunks = tx.prepare("DELETE FROM chunks WHERE source_path = ?1")?;
for path in &missing {
del_vec.execute([path])?;
del_chunks.execute([path])?;
}
}
tx.commit()?;
let removed = missing.len();
self.conn
.inner()
.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);")?;
Ok(removed)
}
}
use rusqlite::OptionalExtension;
fn invalidate_on_upgrade(conn: &Connection) -> Result<()> {
let stored: Option<String> = conn
.inner()
.query_row(
"SELECT value FROM index_meta WHERE key = 'indexer_version'",
[],
|row| row.get(0),
)
.optional()?;
if stored.as_deref() == Some(INDEXER_VERSION) {
return Ok(());
}
let tx = conn.inner().unchecked_transaction()?;
tx.execute("DELETE FROM chunks_vec", [])?;
tx.execute("DELETE FROM chunks", [])?;
tx.execute(
"INSERT INTO index_meta (key, value) VALUES ('indexer_version', ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
[INDEXER_VERSION],
)?;
tx.commit()?;
Ok(())
}
fn is_hidden(name: &std::ffi::OsStr) -> bool {
name.to_string_lossy().starts_with('.')
}
pub fn is_indexable(path: &Path) -> bool {
extract::is_supported(&path.to_string_lossy())
}
pub fn chunk_text(text: &str, max_words: usize, overlap_words: usize) -> Vec<Chunk> {
let max_words = max_words.max(1);
let overlap = overlap_words.min(max_words - 1);
let lines: Vec<&str> = text.lines().collect();
let mut chunks = Vec::new();
let mut current: Vec<&str> = Vec::new();
let mut current_words = 0usize;
let mut start_line = 1usize;
for (idx, line) in lines.iter().enumerate() {
let line_no = idx + 1;
let words = line.split_whitespace().count();
let heading = is_markdown_heading(line);
if !current.is_empty() && (current_words + words > max_words || heading) {
chunks.push(Chunk {
page: 0,
line_start: start_line as i64,
line_end: (line_no - 1) as i64,
anchor: None,
section: None,
content: current.join("\n"),
});
let mut kept: Vec<&str> = Vec::new();
let mut kept_words = 0usize;
if !heading {
for prev in current.iter().rev() {
let w = prev.split_whitespace().count();
if kept_words + w > overlap {
break;
}
kept.insert(0, prev);
kept_words += w;
}
}
start_line = line_no - kept.len();
current = kept;
current_words = kept_words;
}
if current.is_empty() {
start_line = line_no;
}
current.push(line);
current_words += words;
}
if current.iter().any(|l| !l.trim().is_empty()) {
chunks.push(Chunk {
page: 0,
line_start: start_line as i64,
line_end: lines.len().max(1) as i64,
anchor: None,
section: None,
content: current.join("\n"),
});
}
chunks
}
fn is_markdown_heading(line: &str) -> bool {
let trimmed = line.trim_start();
let hashes = trimmed.chars().take_while(|c| *c == '#').count();
(1..=6).contains(&hashes) && trimmed.chars().nth(hashes) == Some(' ')
}
pub fn chunk_blocks(
blocks: &[extract::Block],
max_words: usize,
overlap_words: usize,
) -> Vec<Chunk> {
let mut out = Vec::new();
for block in blocks {
for chunk in chunk_text(&block.text, max_words, overlap_words) {
let (line_start, line_end) = if block.line_start > 0 {
(
block.line_start + chunk.line_start - 1,
block.line_start + chunk.line_end - 1,
)
} else {
(0, 0)
};
out.push(Chunk {
page: block.page,
line_start,
line_end,
anchor: block.anchor.clone(),
section: block.section.clone(),
content: chunk.content,
});
}
}
out
}
pub fn index_with_defaults(
db_path: &Path,
roots: &[PathBuf],
embedder: Option<SharedEmbedder>,
) -> Result<IndexStats> {
index_with_defaults_atomic(db_path, roots, embedder)
}
fn index_with_defaults_atomic(
db_path: &Path,
roots: &[PathBuf],
embedder: Option<SharedEmbedder>,
) -> Result<IndexStats> {
let tmp_path = db_path.with_extension("db.tmp");
let _ = fs::remove_file(&tmp_path);
let conn = Connection::open_or_create(&tmp_path).context("opening temp index database")?;
let stats = Indexer::new(conn, embedder)?.index_roots(roots)?;
atomic_replace(&tmp_path, db_path)?;
Ok(stats)
}
fn atomic_replace(src: &Path, dst: &Path) -> Result<()> {
match fs::rename(src, dst) {
Ok(()) => {}
Err(e) if e.kind() == std::io::ErrorKind::CrossesDevices => {
fs::copy(src, dst).with_context(|| {
format!(
"copying {} -> {} (cross-device)",
src.display(),
dst.display()
)
})?;
fs::remove_file(src)
.with_context(|| format!("removing temp file {}", src.display()))?;
}
Err(e) => {
return Err(e)
.with_context(|| format!("renaming {} -> {}", src.display(), dst.display()))
}
}
Ok(())
}