use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use tracing::info;
use crate::pipeline::files::to_repo_relative;
const STATE_DIR: &str = ".knot";
const STATE_FILE: &str = "index_state.json";
const CURRENT_STATE_VERSION: u32 = 7;
const MIN_COMPATIBLE_STATE_VERSION: u32 = 6;
pub fn fastembed_cache_dir(repo_path: &str) -> PathBuf {
if let Ok(custom_dir) = std::env::var("KNOT_FASTEMBED_CACHE_DIR") {
return PathBuf::from(custom_dir);
}
Path::new(repo_path).join(STATE_DIR).join("fastembed_cache")
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum FileStatus {
Unchanged,
Modified,
Added,
Deleted,
}
pub type FileClassification = (Vec<PathBuf>, Vec<PathBuf>, Vec<PathBuf>, Vec<String>);
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IndexState {
#[serde(default)]
pub version: u32,
pub file_hashes: HashMap<String, String>,
#[serde(default)]
pub embed_model: Option<String>,
}
impl Default for IndexState {
fn default() -> Self {
Self {
version: CURRENT_STATE_VERSION,
file_hashes: HashMap::new(),
embed_model: None,
}
}
}
impl IndexState {
pub fn load_for_indexer(
repo_path: &str,
configured_embed_model: &str,
clean: bool,
) -> Result<Self> {
let state = Self::load(repo_path)?;
if let Some(model) = &state.embed_model
&& model != configured_embed_model
&& !clean
{
anyhow::bail!(
"The persisted index state was built with embedding model '{model}' \
but this run is configured with '{configured_embed_model}'. \
Run `knot-indexer --clean` to rebuild this repository with '{configured_embed_model}'."
);
}
Ok(state)
}
fn load(repo_path: &str) -> Result<Self> {
let state_path = Self::state_file_path(repo_path);
if !state_path.exists() {
info!("No existing index state found โ will perform full indexing");
return Ok(Self::default());
}
let content = fs::read_to_string(&state_path)
.with_context(|| format!("Failed to read state file: {}", state_path.display()))?;
let state: IndexState =
serde_json::from_str(&content).context("Failed to deserialize index state JSON")?;
if state.version < MIN_COMPATIBLE_STATE_VERSION {
anyhow::bail!(
"Detected index_state v{}; the minimum compatible version is v{} \
(current v{}). The on-disk index is incompatible.\n\
Run `knot-indexer --clean` to rebuild from scratch.",
state.version,
MIN_COMPATIBLE_STATE_VERSION,
CURRENT_STATE_VERSION
);
}
if state.version > CURRENT_STATE_VERSION {
anyhow::bail!(
"Detected index_state v{}, which is newer than this knot understands \
(current v{}). The index was written by a newer knot version.\n\
Upgrade knot, or run `knot-indexer --clean` to rebuild from scratch.",
state.version,
CURRENT_STATE_VERSION
);
}
info!(
"Loaded index state v{} with {} tracked files",
state.version,
state.file_hashes.len()
);
Ok(state)
}
pub fn save(&self, repo_path: &str) -> Result<()> {
let state_dir = Self::state_dir_path(repo_path);
let state_path = Self::state_file_path(repo_path);
fs::create_dir_all(&state_dir).with_context(|| {
format!("Failed to create state directory: {}", state_dir.display())
})?;
let to_persist = Self {
version: CURRENT_STATE_VERSION,
file_hashes: self.file_hashes.clone(),
embed_model: self.embed_model.clone(),
};
let content = serde_json::to_string_pretty(&to_persist)
.context("Failed to serialize index state to JSON")?;
fs::write(&state_path, content)
.with_context(|| format!("Failed to write state file: {}", state_path.display()))?;
info!(
"Saved index state v{} with {} tracked files",
CURRENT_STATE_VERSION,
self.file_hashes.len()
);
Ok(())
}
pub fn compute_file_hash(file_path: &Path) -> Result<String> {
use std::fmt::Write as _;
let content = fs::read(file_path)
.with_context(|| format!("Failed to read file for hashing: {}", file_path.display()))?;
let mut hasher = Sha256::new();
hasher.update(&content);
let hash = hasher.finalize();
let mut hex = String::with_capacity(hash.len() * 2);
for byte in hash {
let _ = write!(hex, "{byte:02x}");
}
Ok(hex)
}
pub fn classify_files(
&self,
current_files: &[PathBuf],
repo_root: &Path,
) -> Result<FileClassification> {
let mut unchanged = Vec::new();
let mut modified = Vec::new();
let mut added = Vec::new();
let current_rel_paths: std::collections::HashSet<String> = current_files
.iter()
.map(|p| to_repo_relative(p, repo_root))
.collect();
for file_path in current_files {
let key = to_repo_relative(file_path, repo_root);
let current_hash = Self::compute_file_hash(file_path)?;
match self.file_hashes.get(&key) {
Some(old_hash) if old_hash == ¤t_hash => {
unchanged.push(file_path.clone());
}
Some(_old_hash) => {
modified.push(file_path.clone());
}
None => {
added.push(file_path.clone());
}
}
}
let deleted: Vec<String> = self
.file_hashes
.keys()
.filter(|old_path| !current_rel_paths.contains(*old_path))
.cloned()
.collect();
info!(
"File classification: {} unchanged, {} modified, {} added, {} deleted",
unchanged.len(),
modified.len(),
added.len(),
deleted.len()
);
Ok((unchanged, modified, added, deleted))
}
pub fn update_files(&mut self, files: &[PathBuf], repo_root: &Path) -> Result<()> {
for file_path in files {
let key = to_repo_relative(file_path, repo_root);
let hash = Self::compute_file_hash(file_path)?;
self.file_hashes.insert(key, hash);
}
Ok(())
}
pub fn remove_files(&mut self, file_paths: &[String]) {
for path in file_paths {
self.file_hashes.remove(path);
}
}
fn state_dir_path(repo_path: &str) -> PathBuf {
Path::new(repo_path).join(STATE_DIR)
}
fn state_file_path(repo_path: &str) -> PathBuf {
Self::state_dir_path(repo_path).join(STATE_FILE)
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::fs;
use tempfile::tempdir;
#[test]
fn test_compute_file_hash() {
let dir = tempdir().unwrap();
let file_path = dir.path().join("test.txt");
fs::write(&file_path, "test content").unwrap();
let hash = IndexState::compute_file_hash(&file_path).unwrap();
assert_eq!(
hash,
"6ae8a75555209fd6c44157c0aed8016e763ff435a19cf186f76863140143ff72"
);
fs::write(&file_path, "updated content").unwrap();
let updated_hash = IndexState::compute_file_hash(&file_path).unwrap();
assert_ne!(hash, updated_hash);
}
#[test]
fn test_state_save_and_load() {
let dir = tempdir().unwrap();
let repo_path = dir.path().to_str().unwrap();
let mut state = IndexState::default();
state
.file_hashes
.insert("file1.ts".to_string(), "hash1".to_string());
state
.file_hashes
.insert("file2.java".to_string(), "hash2".to_string());
state.save(repo_path).unwrap();
let state_file = dir.path().join(".knot").join("index_state.json");
assert!(state_file.exists());
let loaded_state = IndexState::load(repo_path).unwrap();
assert_eq!(loaded_state.file_hashes.len(), 2);
assert_eq!(loaded_state.file_hashes.get("file1.ts").unwrap(), "hash1");
assert_eq!(loaded_state.file_hashes.get("file2.java").unwrap(), "hash2");
}
#[test]
fn test_classify_files() {
let dir = tempdir().unwrap();
let repo_root = dir.path();
let unchanged_file = repo_root.join("unchanged.ts");
let modified_file = repo_root.join("modified.java");
let added_file = repo_root.join("added.tsx");
fs::write(&unchanged_file, "unchanged").unwrap();
fs::write(&modified_file, "original content").unwrap();
fs::write(&added_file, "new file").unwrap();
let mut state = IndexState::default();
state.file_hashes.insert(
to_repo_relative(&unchanged_file, repo_root),
IndexState::compute_file_hash(&unchanged_file).unwrap(),
);
state.file_hashes.insert(
to_repo_relative(&modified_file, repo_root),
"fake_old_hash".to_string(),
);
state
.file_hashes
.insert("deleted.java".to_string(), "deleted_hash".to_string());
let current_files = vec![
unchanged_file.clone(),
modified_file.clone(),
added_file.clone(),
];
let (unchanged, modified, added, deleted) =
state.classify_files(¤t_files, repo_root).unwrap();
assert_eq!(unchanged.len(), 1);
assert_eq!(unchanged[0], unchanged_file);
assert_eq!(modified.len(), 1);
assert_eq!(modified[0], modified_file);
assert_eq!(added.len(), 1);
assert_eq!(added[0], added_file);
assert_eq!(deleted.len(), 1);
assert_eq!(deleted[0], "deleted.java");
}
#[test]
fn test_update_and_remove_files() {
let dir = tempdir().unwrap();
let repo_root = dir.path();
let file1 = repo_root.join("file1.ts");
let file2 = repo_root.join("file2.java");
fs::write(&file1, "content1").unwrap();
fs::write(&file2, "content2").unwrap();
let mut state = IndexState::default();
state
.update_files(&[file1.clone(), file2.clone()], repo_root)
.unwrap();
assert_eq!(state.file_hashes.len(), 2);
let key1 = to_repo_relative(&file1, repo_root);
let key2 = to_repo_relative(&file2, repo_root);
assert!(state.file_hashes.contains_key(&key1));
assert!(state.file_hashes.contains_key(&key2));
state.remove_files(std::slice::from_ref(&key1));
assert_eq!(state.file_hashes.len(), 1);
assert!(!state.file_hashes.contains_key(&key1));
assert!(state.file_hashes.contains_key(&key2));
}
#[test]
fn test_default_state_uses_current_version() {
let state = IndexState::default();
assert_eq!(state.version, CURRENT_STATE_VERSION);
assert!(state.file_hashes.is_empty());
}
#[test]
fn test_save_writes_current_version() {
let dir = tempdir().unwrap();
let repo_path = dir.path().to_str().unwrap();
let mut state = IndexState {
version: 0,
file_hashes: HashMap::new(),
embed_model: None,
};
state
.file_hashes
.insert("file1.rs".to_string(), "hash1".to_string());
state.save(repo_path).unwrap();
let state_file = dir.path().join(".knot").join("index_state.json");
let raw = fs::read_to_string(&state_file).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&raw).unwrap();
assert_eq!(
parsed.get("version").and_then(|v| v.as_u64()),
Some(CURRENT_STATE_VERSION as u64)
);
}
#[test]
fn test_load_v6_state_is_accepted_without_reindex() {
let dir = tempdir().unwrap();
let repo_path = dir.path().to_str().unwrap();
let state_dir = dir.path().join(".knot");
fs::create_dir_all(&state_dir).unwrap();
let state_file = state_dir.join("index_state.json");
let raw = r#"{
"version": 6,
"file_hashes": {
"src/lib.rs": "abc123"
}
}"#;
fs::write(&state_file, raw).unwrap();
let loaded = IndexState::load(repo_path).unwrap();
assert_eq!(loaded.version, 6);
assert_eq!(loaded.file_hashes.get("src/lib.rs").unwrap(), "abc123");
assert_eq!(loaded.embed_model, None);
}
#[test]
fn test_load_newer_version_rejected_as_downgrade() {
let dir = tempdir().unwrap();
let repo_path = dir.path().to_str().unwrap();
let state_dir = dir.path().join(".knot");
fs::create_dir_all(&state_dir).unwrap();
let state_file = state_dir.join("index_state.json");
let raw = r#"{
"version": 99,
"file_hashes": {}
}"#;
fs::write(&state_file, raw).unwrap();
let err = IndexState::load(repo_path).unwrap_err();
let msg = format!("{}", err);
assert!(
msg.contains("newer than this knot understands"),
"downgrade must be rejected with an Upgrade hint: {msg}"
);
assert!(msg.contains("--clean"), "must suggest --clean: {msg}");
}
#[test]
fn test_save_round_trips_embed_model() {
let dir = tempdir().unwrap();
let repo_path = dir.path().to_str().unwrap();
let mut state = IndexState {
embed_model: Some("BGEBaseENV15".to_string()),
..Default::default()
};
state
.file_hashes
.insert("file1.rs".to_string(), "h".to_string());
state.save(repo_path).unwrap();
let loaded = IndexState::load(repo_path).unwrap();
assert_eq!(
loaded.embed_model.as_deref(),
Some("BGEBaseENV15"),
"save() must round-trip the active model"
);
}
#[test]
fn test_load_for_indexer_rejects_model_mismatch_on_incremental_run() {
let dir = tempdir().unwrap();
let repo_path = dir.path().to_str().unwrap();
let state = IndexState {
embed_model: Some("BGEBaseENV15".to_string()),
..Default::default()
};
state.save(repo_path).unwrap();
let err = IndexState::load_for_indexer(repo_path, "AllMiniLML6V2", false).unwrap_err();
let msg = format!("{}", err);
assert!(msg.contains("BGEBaseENV15"), "{msg}");
assert!(msg.contains("AllMiniLML6V2"), "{msg}");
assert!(msg.contains("--clean"), "{msg}");
let loaded = IndexState::load_for_indexer(repo_path, "AllMiniLML6V2", true).unwrap();
assert_eq!(loaded.embed_model.as_deref(), Some("BGEBaseENV15"));
IndexState::load_for_indexer(repo_path, "BGEBaseENV15", false).unwrap();
}
#[test]
fn test_load_older_version_returns_error_with_instructions() {
let dir = tempdir().unwrap();
let repo_path = dir.path().to_str().unwrap();
let state_dir = dir.path().join(".knot");
fs::create_dir_all(&state_dir).unwrap();
let state_file = state_dir.join("index_state.json");
let raw = r#"{
"version": 1,
"file_hashes": {
"/tmp/stale.rs": "abc123"
}
}"#;
fs::write(&state_file, raw).unwrap();
let err = IndexState::load(repo_path).unwrap_err();
let msg = format!("{}", err);
assert!(
msg.contains("incompatible"),
"error should mention incompatibility: {msg}"
);
assert!(
msg.contains("--clean"),
"error should suggest --clean flag: {msg}"
);
}
#[test]
fn test_load_missing_version_treated_as_incompatible() {
let dir = tempdir().unwrap();
let repo_path = dir.path().to_str().unwrap();
let state_dir = dir.path().join(".knot");
fs::create_dir_all(&state_dir).unwrap();
let state_file = state_dir.join("index_state.json");
let raw = r#"{
"file_hashes": {
"/tmp/legacy.rs": "deadbeef"
}
}"#;
fs::write(&state_file, raw).unwrap();
let err = IndexState::load(repo_path).unwrap_err();
let msg = format!("{}", err);
assert!(
msg.contains("incompatible"),
"missing version should be treated as incompatible: {msg}"
);
}
#[test]
fn test_load_current_version_preserves_state() {
let dir = tempdir().unwrap();
let repo_path = dir.path().to_str().unwrap();
let mut state = IndexState::default();
state
.file_hashes
.insert("file1.rs".to_string(), "hash1".to_string());
state.save(repo_path).unwrap();
let loaded = IndexState::load(repo_path).unwrap();
assert_eq!(loaded.version, CURRENT_STATE_VERSION);
assert_eq!(loaded.file_hashes.len(), 1);
assert_eq!(loaded.file_hashes.get("file1.rs").unwrap(), "hash1");
}
#[test]
fn test_classify_files_uses_relative_keys() {
let dir = tempdir().unwrap();
let repo_root = dir.path();
let file = repo_root.join("src/lib.rs");
fs::create_dir_all(repo_root.join("src")).unwrap();
fs::write(&file, "fn foo() {}").unwrap();
let mut state = IndexState::default();
let key = to_repo_relative(&file, repo_root);
state
.file_hashes
.insert(key.clone(), IndexState::compute_file_hash(&file).unwrap());
let current = vec![file.clone()];
let (unchanged, modified, added, deleted) =
state.classify_files(¤t, repo_root).unwrap();
assert_eq!(unchanged, vec![file]);
assert!(modified.is_empty());
assert!(added.is_empty());
assert!(deleted.is_empty());
}
#[test]
fn test_update_files_stores_relative_keys() {
let dir = tempdir().unwrap();
let repo_root = dir.path();
let f = repo_root.join("a/b.rs");
fs::create_dir_all(repo_root.join("a")).unwrap();
fs::write(&f, "fn x() {}").unwrap();
let mut state = IndexState::default();
state
.update_files(std::slice::from_ref(&f), repo_root)
.unwrap();
let key = to_repo_relative(&f, repo_root);
assert_eq!(key, "a/b.rs");
assert!(state.file_hashes.contains_key(&key));
assert!(
!state.file_hashes.contains_key(f.to_str().unwrap()),
"absolute path must NOT be used as a key"
);
}
#[test]
fn test_deleted_files_reported_relative() {
let dir = tempdir().unwrap();
let repo_root = dir.path();
fs::create_dir_all(repo_root.join("src")).unwrap();
let present = repo_root.join("src/present.rs");
fs::write(&present, "x").unwrap();
let mut state = IndexState::default();
state
.file_hashes
.insert("src/present.rs".to_string(), "h".to_string());
state
.file_hashes
.insert("src/gone.rs".to_string(), "h2".to_string());
let current = vec![present.clone()];
let (_, _, _, deleted) = state.classify_files(¤t, repo_root).unwrap();
assert_eq!(deleted, vec!["src/gone.rs".to_string()]);
}
#[test]
fn test_load_rejects_v3_state() {
let dir = tempdir().unwrap();
let repo_path = dir.path().to_str().unwrap();
let state_dir = dir.path().join(".knot");
fs::create_dir_all(&state_dir).unwrap();
let state_file = state_dir.join("index_state.json");
let raw = r#"{
"version": 3,
"file_hashes": {
"/tmp/stale.rs": "abc123"
}
}"#;
fs::write(&state_file, raw).unwrap();
let err = IndexState::load(repo_path).unwrap_err();
let msg = format!("{}", err);
assert!(
msg.contains("incompatible"),
"error should mention incompatibility: {msg}"
);
assert!(
msg.contains("--clean"),
"error should suggest --clean flag: {msg}"
);
}
}