use std::collections::{HashMap, HashSet};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use khive_runtime::ann_registry::{self, CompactionScope, WatermarkAuthority};
use khive_runtime::{KhiveRuntime, Namespace, NamespaceToken, RuntimeError};
use khive_storage::types::{SqlStatement, SqlValue};
use khive_vamana::{
read_commit_fingerprint, read_commit_info, read_external_ids_sidecar, segment_commit_digest,
write_external_ids_sidecar, CorpusFingerprint, VamanaConfig, VamanaIndex, VamanaSnapshot,
};
use tokio::sync::RwLock;
use uuid::Uuid;
pub(crate) struct AnnBridge {
index: VamanaIndex,
id_map: Vec<Uuid>,
commit_digest: Option<[u8; 32]>,
generation: u64,
#[cfg(test)]
drop_probe: Option<Arc<()>>,
#[cfg(test)]
search_pause: Option<Arc<TestSearchPause>>,
}
#[cfg(test)]
#[derive(Default)]
struct TestSearchPause {
started: AtomicBool,
released: std::sync::Mutex<bool>,
wake: std::sync::Condvar,
}
#[cfg(test)]
impl TestSearchPause {
fn wait(&self) {
let mut released = self.released.lock().expect("search pause");
self.started.store(true, Ordering::SeqCst);
while !*released {
released = self.wake.wait(released).expect("search pause");
}
}
fn release(&self) {
*self.released.lock().expect("search pause") = true;
self.wake.notify_all();
}
}
#[derive(Clone, Debug, Eq, PartialEq, Hash)]
pub(crate) struct AnnKey {
namespace: String,
model: String,
}
impl AnnKey {
pub(crate) fn new(namespace: &str, model: &str) -> Self {
Self {
namespace: namespace.to_owned(),
model: model.to_owned(),
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum AnnWarmFailure {
EmptyCorpus,
Operational,
Interrupted,
NotWarmHost,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum AnnWarmOutcome {
Ready,
Empty,
Failed,
Declined,
}
#[derive(Debug)]
enum AnnWarmState {
Warming {
attempt_id: u64,
generation: u64,
started_at: std::time::Instant,
},
Ready {
generation: u64,
},
Failed {
generation: u64,
error: AnnWarmFailure,
},
}
struct AnnWarmPermit {
ann: SharedAnn,
key: AnnKey,
attempt_id: u64,
generation: u64,
finished: bool,
}
pub(crate) struct AnnState {
indexes: RwLock<HashMap<AnnKey, AnnBridge>>,
warm_states: std::sync::Mutex<HashMap<AnnKey, AnnWarmState>>,
next_warm_attempt_id: AtomicU64,
generations: std::sync::Mutex<HashMap<String, u64>>,
unavailable: std::sync::Mutex<HashMap<AnnKey, u64>>,
force_rebuild: std::sync::Mutex<HashSet<AnnKey>>,
checkpoint_locks: std::sync::Mutex<HashMap<AnnKey, std::sync::Weak<tokio::sync::Mutex<()>>>>,
rotation_watch_started: AtomicBool,
builds_corpus_indexes: bool,
#[cfg(test)]
test_finish_warm_ready_hook: std::sync::Mutex<Option<Arc<FinishWarmReadyTestHook>>>,
}
#[cfg(test)]
pub(crate) struct FinishWarmReadyTestHook {
pub(crate) incumbent_digest: [u8; 32],
pub(crate) generation: u64,
pub(crate) handle_tx: tokio::sync::mpsc::UnboundedSender<tokio::task::JoinHandle<()>>,
}
pub(crate) type SharedAnn = Arc<AnnState>;
pub(crate) fn new_shared() -> SharedAnn {
new_shared_for_role(true)
}
pub(crate) fn new_shared_for_role(builds_corpus_indexes: bool) -> SharedAnn {
Arc::new(AnnState {
builds_corpus_indexes,
indexes: RwLock::new(HashMap::new()),
warm_states: std::sync::Mutex::new(HashMap::new()),
next_warm_attempt_id: AtomicU64::new(1),
generations: std::sync::Mutex::new(HashMap::new()),
unavailable: std::sync::Mutex::new(HashMap::new()),
force_rebuild: std::sync::Mutex::new(HashSet::new()),
checkpoint_locks: std::sync::Mutex::new(HashMap::new()),
rotation_watch_started: AtomicBool::new(false),
#[cfg(test)]
test_finish_warm_ready_hook: std::sync::Mutex::new(None),
})
}
fn force_rebuild_guard(
m: &std::sync::Mutex<HashSet<AnnKey>>,
) -> std::sync::MutexGuard<'_, HashSet<AnnKey>> {
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn mark_force_rebuild(ann: &SharedAnn, key: &AnnKey) {
force_rebuild_guard(&ann.force_rebuild).insert(key.clone());
}
fn force_rebuild_required(ann: &SharedAnn, key: &AnnKey) -> bool {
force_rebuild_guard(&ann.force_rebuild).contains(key)
}
fn clear_force_rebuild_if_current(ann: &SharedAnn, key: &AnnKey, generation: u64) {
if current_generation(ann, &key.namespace) == generation {
clear_force_rebuild(ann, key);
}
}
fn clear_force_rebuild(ann: &SharedAnn, key: &AnnKey) {
force_rebuild_guard(&ann.force_rebuild).remove(key);
}
fn checkpoint_lock(ann: &SharedAnn, key: &AnnKey) -> Arc<tokio::sync::Mutex<()>> {
let mut locks = ann
.checkpoint_locks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
locks.retain(|_, lock| lock.strong_count() > 0);
if let Some(lock) = locks.get(key).and_then(std::sync::Weak::upgrade) {
return lock;
}
let lock = Arc::new(tokio::sync::Mutex::new(()));
locks.insert(key.clone(), Arc::downgrade(&lock));
lock
}
fn generations_guard(
m: &std::sync::Mutex<HashMap<String, u64>>,
) -> std::sync::MutexGuard<'_, HashMap<String, u64>> {
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn bump_generation(ann: &SharedAnn, namespace: &str) -> u64 {
let mut gens = generations_guard(&ann.generations);
let slot = gens.entry(namespace.to_owned()).or_insert(0);
*slot += 1;
*slot
}
pub(crate) fn current_generation(ann: &SharedAnn, namespace: &str) -> u64 {
generations_guard(&ann.generations)
.get(namespace)
.copied()
.unwrap_or(0)
}
pub(crate) async fn install_if_fresher(ann: &SharedAnn, key: &AnnKey, candidate: AnnBridge) {
let mut idxs = ann.indexes.write().await;
let ns_generation = current_generation(ann, &key.namespace);
if candidate.generation < ns_generation {
tracing::debug!(
key = ?key,
candidate_generation = candidate.generation,
namespace_generation = ns_generation,
"knowledge ANN install skipped: candidate predates namespace's current generation"
);
return;
}
match idxs.get(key) {
Some(existing) if existing.generation >= candidate.generation => {
tracing::debug!(
key = ?key,
existing_generation = existing.generation,
candidate_generation = candidate.generation,
"knowledge ANN install skipped: cached entry already >= this build's generation"
);
}
_ => {
idxs.insert(key.clone(), candidate);
unavailable_guard(&ann.unavailable).remove(key);
}
}
}
pub(crate) async fn install_replacing(ann: &SharedAnn, key: &AnnKey, candidate: AnnBridge) -> bool {
let mut idxs = ann.indexes.write().await;
let ns_generation = current_generation(ann, &key.namespace);
if candidate.generation < ns_generation {
tracing::debug!(
key = ?key,
candidate_generation = candidate.generation,
namespace_generation = ns_generation,
"knowledge ANN replace skipped: candidate predates namespace's current generation"
);
return false;
}
idxs.insert(key.clone(), candidate);
unavailable_guard(&ann.unavailable).remove(key);
true
}
async fn has_current_index(ann: &SharedAnn, key: &AnnKey) -> bool {
let idxs = ann.indexes.read().await;
let current = current_generation(ann, &key.namespace);
idxs.get(key)
.is_some_and(|bridge| bridge.generation >= current)
}
async fn has_current_index_at_watermark(ann: &SharedAnn, key: &AnnKey, watermark: u64) -> bool {
let idxs = ann.indexes.read().await;
let current = current_generation(ann, &key.namespace);
idxs.get(key).is_some_and(|bridge| {
bridge.generation >= current && bridge.index.last_applied_seq().unwrap_or(0) >= watermark
})
}
fn warm_states_guard(
m: &std::sync::Mutex<HashMap<AnnKey, AnnWarmState>>,
) -> std::sync::MutexGuard<'_, HashMap<AnnKey, AnnWarmState>> {
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn begin_warm(ann: &SharedAnn, key: AnnKey) -> Option<AnnWarmPermit> {
let mut states = warm_states_guard(&ann.warm_states);
let generation = current_generation(ann, &key.namespace);
match states.get(&key) {
Some(
AnnWarmState::Warming {
generation: state_generation,
..
}
| AnnWarmState::Ready {
generation: state_generation,
},
) if *state_generation >= generation => return None,
Some(AnnWarmState::Failed {
generation: failed_generation,
error,
}) => {
tracing::debug!(
key = ?key,
failed_generation,
error = ?error,
"retrying failed knowledge ANN warm"
);
}
_ => {}
}
let attempt_id = ann.next_warm_attempt_id.fetch_add(1, Ordering::Relaxed);
states.insert(
key.clone(),
AnnWarmState::Warming {
attempt_id,
generation,
started_at: std::time::Instant::now(),
},
);
Some(AnnWarmPermit {
ann: ann.clone(),
key,
attempt_id,
generation,
finished: false,
})
}
fn finish_warm_state(permit: &mut AnnWarmPermit, next: AnnWarmState) {
let mut states = warm_states_guard(&permit.ann.warm_states);
let started_at = match states.get(&permit.key) {
Some(AnnWarmState::Warming {
attempt_id,
generation,
started_at,
}) if *attempt_id == permit.attempt_id && *generation == permit.generation => *started_at,
_ => {
permit.finished = true;
return;
}
};
tracing::debug!(
key = ?permit.key,
attempt_id = permit.attempt_id,
elapsed_ms = started_at.elapsed().as_millis(),
state = ?next,
"knowledge ANN warm finished"
);
states.insert(permit.key.clone(), next);
permit.finished = true;
}
async fn finish_warm(mut permit: AnnWarmPermit, outcome: AnnWarmOutcome) {
match outcome {
AnnWarmOutcome::Ready => {
let ann = permit.ann.clone();
let idxs = ann.indexes.read().await;
let next = match idxs
.get(&permit.key)
.map(|bridge| bridge.generation)
.filter(|generation| *generation >= permit.generation)
{
Some(generation) => AnnWarmState::Ready { generation },
None => AnnWarmState::Failed {
generation: permit.generation,
error: AnnWarmFailure::Operational,
},
};
#[cfg(test)]
run_finish_warm_ready_test_hook(&ann, &permit.key).await;
finish_warm_state(&mut permit, next);
drop(idxs);
}
AnnWarmOutcome::Empty => {
let next = AnnWarmState::Failed {
generation: permit.generation,
error: AnnWarmFailure::EmptyCorpus,
};
finish_warm_state(&mut permit, next);
}
AnnWarmOutcome::Failed => {
let next = AnnWarmState::Failed {
generation: permit.generation,
error: AnnWarmFailure::Operational,
};
finish_warm_state(&mut permit, next);
}
AnnWarmOutcome::Declined => {
let next = AnnWarmState::Failed {
generation: permit.generation,
error: AnnWarmFailure::NotWarmHost,
};
finish_warm_state(&mut permit, next);
}
}
}
#[cfg(test)]
async fn run_finish_warm_ready_test_hook(ann: &SharedAnn, key: &AnnKey) {
let hook = ann
.test_finish_warm_ready_hook
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let Some(hook) = hook else {
return;
};
let ann = ann.clone();
let key = key.clone();
let incumbent_digest = hook.incumbent_digest;
let generation = hook.generation;
let handle = tokio::spawn(async move {
evict_bridge_and_ready_state(&ann, &key, incumbent_digest, generation).await;
});
let _ = hook.handle_tx.send(handle);
tokio::task::yield_now().await;
}
impl Drop for AnnWarmPermit {
fn drop(&mut self) {
if !self.finished {
let next = AnnWarmState::Failed {
generation: self.generation,
error: AnnWarmFailure::Interrupted,
};
finish_warm_state(self, next);
}
}
}
#[cfg(test)]
impl AnnWarmPermit {
fn leave_in_flight_for_test(mut self) {
self.finished = true;
}
}
fn unavailable_guard(
m: &std::sync::Mutex<HashMap<AnnKey, u64>>,
) -> std::sync::MutexGuard<'_, HashMap<AnnKey, u64>> {
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn mark_unavailable(ann: &SharedAnn, key: &AnnKey, generation: u64) {
unavailable_guard(&ann.unavailable).insert(key.clone(), generation);
}
fn is_terminally_unavailable(ann: &SharedAnn, key: &AnnKey) -> bool {
let current = current_generation(ann, &key.namespace);
let mut guard = unavailable_guard(&ann.unavailable);
match guard.get(key) {
Some(&marked_generation) if marked_generation >= current => true,
Some(_) => {
guard.remove(key);
false
}
None => false,
}
}
#[cfg(test)]
pub(crate) async fn insert_ann_if_absent(ann: &SharedAnn, key: AnnKey, bridge: AnnBridge) -> bool {
use std::collections::hash_map::Entry;
let mut guard = ann.indexes.write().await;
match guard.entry(key) {
Entry::Occupied(_) => false,
Entry::Vacant(e) => {
e.insert(bridge);
true
}
}
}
pub(crate) async fn clear_namespace(ann: &SharedAnn, namespace: &str) {
let mut idxs = ann.indexes.write().await;
let mut states = warm_states_guard(&ann.warm_states);
idxs.retain(|k, _| k.namespace != namespace);
states.retain(|k, _| k.namespace != namespace);
bump_generation(ann, namespace);
}
#[cfg(test)]
pub(crate) async fn search_loaded(
ann: &SharedAnn,
key: &AnnKey,
query: &[f32],
k: usize,
) -> Option<Vec<(Uuid, f32)>> {
let guard = ann.indexes.read().await;
guard.get(key).map(|bridge| bridge.search(query, k))
}
pub(crate) async fn search_loaded_with_seq(
ann: &SharedAnn,
key: &AnnKey,
query: &[f32],
k: usize,
) -> Option<(Vec<(Uuid, f32)>, u64)> {
if !ann.indexes.read().await.contains_key(key) {
return None;
}
let ann = Arc::clone(ann);
let key = key.clone();
let query = query.to_vec();
tokio::task::spawn_blocking(move || {
let guard = ann.indexes.blocking_read();
guard.get(&key).map(|bridge| {
(
bridge.search(&query, k),
bridge.index.last_applied_seq().unwrap_or(0),
)
})
})
.await
.expect("loaded ANN traversal task panicked")
}
pub(crate) fn is_warming_not_loaded(ann: &SharedAnn, key: &AnnKey) -> bool {
let in_warming = {
let states = warm_states_guard(&ann.warm_states);
let generation = current_generation(ann, &key.namespace);
matches!(
states.get(key),
Some(AnnWarmState::Warming {
generation: state_generation,
..
}) if *state_generation >= generation
)
};
if !in_warming {
return false;
}
match ann.indexes.try_read() {
Ok(guard) => !guard.contains_key(key),
Err(_) => true,
}
}
pub(crate) async fn wait_ready(
ann: &SharedAnn,
key: &AnnKey,
timeout_ms: u64,
poll_ms: u64,
) -> bool {
let deadline = std::time::Instant::now() + std::time::Duration::from_millis(timeout_ms);
loop {
if ann.indexes.read().await.contains_key(key) {
return true;
}
if is_terminally_unavailable(ann, key) {
return false;
}
if std::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(std::time::Duration::from_millis(poll_ms)).await;
}
}
pub(crate) const ANN_WARM_WAIT_TIMEOUT_MS: u64 = 5_000;
pub(crate) const ANN_WARM_WAIT_POLL_MS: u64 = 50;
const ROTATION_WATCH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5);
static ANN_WARM_WAIT_TIMEOUT_OVERRIDE_MS: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
pub(crate) fn warm_wait_timeout_ms() -> u64 {
let o = ANN_WARM_WAIT_TIMEOUT_OVERRIDE_MS.load(std::sync::atomic::Ordering::Relaxed);
if o > 0 {
o
} else {
ANN_WARM_WAIT_TIMEOUT_MS
}
}
#[cfg(test)]
pub(crate) fn set_warm_wait_timeout_override_ms(ms: u64) {
ANN_WARM_WAIT_TIMEOUT_OVERRIDE_MS.store(ms, std::sync::atomic::Ordering::Relaxed);
}
impl AnnBridge {
pub fn build(mut vectors: Vec<f32>, dim: usize, id_map: Vec<Uuid>) -> Result<Self, String> {
if dim == 0 {
return Err("dimension must be > 0".into());
}
if vectors.is_empty() || id_map.is_empty() {
return Err("no vectors to build ANN index from".into());
}
let n = vectors.len() / dim;
if n != id_map.len() {
return Err(format!(
"id_map length {} != vector count {}",
id_map.len(),
n
));
}
for row in vectors.chunks_exact_mut(dim) {
l2_normalize(row);
}
let cfg = VamanaConfig::with_dimensions(dim);
let index = VamanaIndex::build(&vectors, cfg).map_err(|e| format!("{e}"))?;
Ok(Self {
index,
id_map,
commit_digest: None,
generation: 0,
#[cfg(test)]
drop_probe: None,
#[cfg(test)]
search_pause: None,
})
}
pub(crate) fn with_generation(mut self, generation: u64) -> Self {
self.generation = generation;
self
}
pub(crate) fn set_applied_seq(&mut self, seq: u64) {
self.index.set_last_applied_seq(Some(seq));
}
pub(crate) fn reverse_map(&self) -> HashMap<Uuid, u32> {
let mut reverse: HashMap<Uuid, u32> = HashMap::with_capacity(self.index.live_count());
for (ordinal, uuid) in self.id_map.iter().enumerate() {
if self.index.is_tombstoned(ordinal as u32) {
continue;
}
reverse.insert(*uuid, ordinal as u32);
}
reverse
}
pub(crate) fn apply_final_op(
&mut self,
reverse: &mut HashMap<Uuid, u32>,
uuid: Uuid,
op: Option<Vec<f32>>,
) -> Result<(), String> {
match op {
None => {
if let Some(&ordinal) = reverse.get(&uuid) {
if self.id_map.get(ordinal as usize) != Some(&uuid) {
tracing::warn!(
subject = %uuid,
ordinal,
"replay delete: ordinal reassigned within batch, skipping tombstone"
);
reverse.remove(&uuid);
return Ok(());
}
self.index
.tombstone(ordinal)
.map_err(|e| format!("replay tombstone({ordinal}): {e}"))?;
reverse.remove(&uuid);
}
}
Some(mut embedding) => {
l2_normalize(&mut embedding);
if let Some(&old) = reverse.get(&uuid) {
self.index
.tombstone(old)
.map_err(|e| format!("replay tombstone({old}): {e}"))?;
}
let ordinal = self
.index
.insert(&embedding)
.map_err(|e| format!("replay insert: {e}"))?;
let slot = ordinal as usize;
match slot.cmp(&self.id_map.len()) {
std::cmp::Ordering::Less => self.id_map[slot] = uuid,
std::cmp::Ordering::Equal => self.id_map.push(uuid),
std::cmp::Ordering::Greater => {
return Err(format!(
"replay insert returned ordinal {ordinal} beyond id_map len {}",
self.id_map.len()
));
}
}
reverse.insert(uuid, ordinal);
}
}
Ok(())
}
pub fn search(&self, query: &[f32], k: usize) -> Vec<(Uuid, f32)> {
#[cfg(test)]
if let Some(pause) = &self.search_pause {
pause.wait();
}
let mut q = query.to_vec();
l2_normalize(&mut q);
match self.index.search(&q, k) {
Ok(results) => results
.into_iter()
.filter_map(|(idx, dist)| {
self.id_map.get(idx as usize).map(|uuid| {
let cosine = 1.0 - dist / 2.0;
(*uuid, cosine.max(0.0))
})
})
.collect(),
Err(e) => {
tracing::warn!(error = %e, "vamana ANN search failed");
Vec::new()
}
}
}
pub fn num_vectors(&self) -> usize {
self.index.num_vectors()
}
pub fn from_vamana_snapshot(snapshot: VamanaSnapshot) -> Result<Self, String> {
let id_map: Vec<Uuid> = snapshot
.external_ids
.iter()
.map(|s| Uuid::parse_str(s).map_err(|e| format!("bad UUID {s}: {e}")))
.collect::<Result<_, _>>()?;
let index =
VamanaIndex::from_snapshot(&snapshot).map_err(|e| format!("snapshot restore: {e}"))?;
Ok(Self {
index,
id_map,
commit_digest: None,
generation: 0,
#[cfg(test)]
drop_probe: None,
#[cfg(test)]
search_pause: None,
})
}
#[allow(dead_code)]
pub fn save_atomic(&self, dir: &std::path::Path) -> Result<(), String> {
let _publication_lock = acquire_bridge_checkpoint_lock(dir)?;
self.save_atomic_locked(dir)
}
fn save_atomic_locked(&self, dir: &std::path::Path) -> Result<(), String> {
let count = self.id_map.len();
if count != self.index.num_vectors() {
return Err(format!(
"id_map length {count} != index.num_vectors() {}",
self.index.num_vectors()
));
}
self.index
.save_atomic(dir)
.map_err(|e| format!("VamanaIndex::save_atomic: {e}"))?;
let digest = segment_commit_digest(dir)
.map_err(|e| format!("segment_commit_digest after save: {e}"))?
.ok_or_else(|| {
"save_atomic succeeded but metadata.bin is absent (torn commit)".to_string()
})?;
write_external_ids_sidecar(dir, &digest, &self.id_map).map_err(|e| e.to_string())
}
#[allow(dead_code)]
pub fn load(dir: &std::path::Path) -> Result<Self, String> {
read_commit_fingerprint(dir)
.map_err(|e| format!("read_commit_fingerprint: {e}"))?
.ok_or_else(|| {
"no v2 commit fingerprint: segment dir is absent, v1, or has a torn commit"
.to_string()
})?;
let index = VamanaIndex::load(dir).map_err(|e| format!("VamanaIndex::load: {e}"))?;
let (sidecar_digest, id_map) = read_external_ids_sidecar(dir)?;
let commit_digest = segment_commit_digest(dir)
.map_err(|e| format!("segment_commit_digest: {e}"))?
.ok_or_else(|| "metadata.bin vanished between fingerprint and digest".to_string())?;
if sidecar_digest != commit_digest {
return Err(
"external_ids.bin commit-digest mismatch: torn segment/sidecar pair".to_string(),
);
}
if id_map.len() != index.num_vectors() {
return Err(format!(
"external_ids.bin count {} != index.num_vectors() {}",
id_map.len(),
index.num_vectors()
));
}
Ok(Self {
index,
id_map,
commit_digest: Some(commit_digest),
generation: 0,
#[cfg(test)]
drop_probe: None,
#[cfg(test)]
search_pause: None,
})
}
}
fn acquire_bridge_checkpoint_lock(dir: &std::path::Path) -> Result<std::fs::File, String> {
std::fs::create_dir_all(dir).map_err(|error| {
format!(
"create ANN bridge checkpoint directory {}: {error}",
dir.display()
)
})?;
let lock_path = dir.join(".bridge-checkpoint.lock");
let lock = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.read(true)
.write(true)
.open(&lock_path)
.map_err(|error| format!("open ANN bridge lock {}: {error}", lock_path.display()))?;
lock.lock()
.map_err(|error| format!("acquire ANN bridge lock {}: {error}", lock_path.display()))?;
Ok(lock)
}
async fn acquire_bridge_checkpoint_lock_async(
dir: std::path::PathBuf,
) -> Result<std::fs::File, String> {
tokio::task::spawn_blocking(move || acquire_bridge_checkpoint_lock(&dir))
.await
.map_err(|error| format!("ANN bridge lock task failed: {error}"))?
}
fn l2_normalize(v: &mut [f32]) {
let norm: f32 = v.iter().map(|x| x * x).sum::<f32>().sqrt();
if norm > 1e-8 {
for x in v.iter_mut() {
*x /= norm;
}
}
}
pub(crate) fn snapshot_key(namespace: &str, model: &str) -> String {
format!("{namespace}::vamana::{model}")
}
fn ann_segment_dir(rt: &KhiveRuntime, ns: &str, model: &str) -> Option<std::path::PathBuf> {
let ann_root = rt.backend_ann_root()?;
Some(ann_segment_dir_from_root(&ann_root, ns, model))
}
fn ann_segment_dir_from_root(
ann_root: &std::path::Path,
ns: &str,
model: &str,
) -> std::path::PathBuf {
let key = snapshot_key(ns, model);
let hex: String = key.bytes().map(|b| format!("{b:02x}")).collect();
ann_root.join(hex)
}
fn decode_ann_dir_name(name: &str) -> Option<(String, String)> {
let raw = name.as_bytes();
if !raw.len().is_multiple_of(2) {
return None;
}
let mut bytes = Vec::with_capacity(raw.len() / 2);
#[allow(unknown_lints, clippy::chunks_exact_to_as_chunks)]
for pair in raw.chunks_exact(2) {
let hi = (pair[0] as char).to_digit(16)?;
let lo = (pair[1] as char).to_digit(16)?;
bytes.push((hi * 16 + lo) as u8);
}
let key = String::from_utf8(bytes).ok()?;
let (ns, model) = key.split_once("::vamana::")?;
if ns.is_empty() || model.is_empty() {
return None;
}
Some((ns.to_string(), model.to_string()))
}
pub(crate) fn sanitize_model_key(s: &str) -> String {
s.chars()
.map(|c| if c.is_ascii_alphanumeric() { c } else { '_' })
.collect()
}
#[cfg(test)]
pub(crate) fn persist_ann_v2(
rt: &KhiveRuntime,
ns: &str,
model: &str,
bridge: &AnnBridge,
) -> Result<(), String> {
match ann_segment_dir(rt, ns, model) {
Some(dir) => bridge.save_atomic(&dir),
None => Ok(()), }
}
fn persist_ann_v2_locked(
rt: &KhiveRuntime,
ns: &str,
model: &str,
bridge: &AnnBridge,
) -> Result<(), String> {
match ann_segment_dir(rt, ns, model) {
Some(dir) => bridge.save_atomic_locked(&dir),
None => Ok(()),
}
}
const ANN_CONSUMER: &str = "knowledge:knowledge.atom";
const ANN_REBUILD_THRESHOLD_DEFAULT: f64 = 0.20;
fn ann_rebuild_threshold() -> f64 {
std::env::var("KHIVE_ANN_REBUILD_THRESHOLD")
.ok()
.and_then(|v| v.parse::<f64>().ok())
.filter(|v| *v > 0.0 && *v <= 1.0)
.unwrap_or(ANN_REBUILD_THRESHOLD_DEFAULT)
}
#[cfg(test)]
async fn register_consumer(rt: &KhiveRuntime, ns: &str, model: &str) -> Result<(), String> {
let sql = rt.sql();
if ann_segment_dir(rt, ns, model).is_none() {
let mut writer = sql.writer().await.map_err(|error| error.to_string())?;
writer
.execute(SqlStatement {
sql: "INSERT OR IGNORE INTO ann_consumer_watermark \
(consumer, namespace, embedding_model, watermark) \
VALUES (?1, ?2, ?3, ?4)"
.into(),
params: vec![
SqlValue::Text(ANN_CONSUMER.into()),
SqlValue::Text(ns.to_owned()),
SqlValue::Text(model.to_owned()),
SqlValue::Integer(ann_registry::PENDING_WATERMARK),
],
label: Some("knowledge_ann_register_pathless_consumer".into()),
})
.await
.map_err(|error| error.to_string())?;
return Ok(());
}
ann_registry::register_pending(sql.as_ref(), ANN_CONSUMER, ns, model)
.await
.map_err(|e| e.to_string())
}
async fn write_force_rebuild_sentinel_row(rt: &KhiveRuntime, key: &AnnKey) -> Result<(), String> {
let sql = rt.sql();
if ann_segment_dir(rt, &key.namespace, &key.model).is_none() {
let mut writer = sql.writer().await.map_err(|error| error.to_string())?;
writer
.execute(SqlStatement {
sql: "INSERT INTO ann_consumer_watermark \
(consumer, namespace, embedding_model, watermark) \
VALUES (?1, ?2, ?3, ?4) \
ON CONFLICT(consumer, namespace, embedding_model) \
DO UPDATE SET watermark = excluded.watermark"
.into(),
params: vec![
SqlValue::Text(ANN_CONSUMER.into()),
SqlValue::Text(key.namespace.clone()),
SqlValue::Text(key.model.clone()),
SqlValue::Integer(ann_registry::RECOVERING_WATERMARK),
],
label: Some("knowledge_ann_mark_pathless_recovering".into()),
})
.await
.map_err(|error| error.to_string())?;
return Ok(());
}
ann_registry::mark_recovering(sql.as_ref(), ANN_CONSUMER, &key.namespace, &key.model)
.await
.map_err(|error| error.to_string())
}
async fn write_force_rebuild_sentinel(rt: &KhiveRuntime, key: &AnnKey) -> Result<(), String> {
let _publication_lock = match ann_segment_dir(rt, &key.namespace, &key.model) {
Some(dir) => Some(acquire_bridge_checkpoint_lock_async(dir).await?),
None => None,
};
if matches!(
read_own_watermark(rt, &key.namespace, &key.model).await?,
Some(watermark) if watermark >= 0
) {
return Ok(());
}
write_force_rebuild_sentinel_row(rt, key).await
}
async fn prepare_authoritative_rebuild(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
) -> Result<(), String> {
mark_force_rebuild(ann, key);
let local_publication_lock = checkpoint_lock(ann, key);
let _local_publication_guard = local_publication_lock.lock().await;
write_force_rebuild_sentinel(rt, key).await
}
async fn read_own_watermark(
rt: &KhiveRuntime,
ns: &str,
model: &str,
) -> Result<Option<i64>, String> {
let sql = rt.sql();
let mut reader = sql.reader().await.map_err(|e| e.to_string())?;
let rows = reader
.query_all(SqlStatement {
sql: "SELECT watermark FROM ann_consumer_watermark \
WHERE consumer = ?1 AND namespace = ?2 AND embedding_model = ?3"
.into(),
params: vec![
SqlValue::Text(ANN_CONSUMER.into()),
SqlValue::Text(ns.to_owned()),
SqlValue::Text(model.to_owned()),
],
label: Some("ann_read_own_watermark".into()),
})
.await
.map_err(|e| e.to_string())?;
Ok(rows
.into_iter()
.next()
.and_then(|row| match row.get("watermark") {
Some(SqlValue::Integer(n)) => Some(*n),
_ => None,
}))
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum CheckpointAuthority {
Incremental,
FullRegistered,
FullSentinel,
}
pub(crate) async fn prepare_full_corpus_scan(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
) -> Result<CheckpointAuthority, String> {
match read_own_watermark(rt, &key.namespace, &key.model).await? {
Some(watermark) if watermark >= 0 => Ok(CheckpointAuthority::FullRegistered),
Some(watermark) if watermark == ann_registry::RECOVERING_WATERMARK => {
mark_force_rebuild(ann, key);
Ok(CheckpointAuthority::FullSentinel)
}
Some(_) | None => {
prepare_authoritative_rebuild(rt, ann, key).await?;
Ok(CheckpointAuthority::FullSentinel)
}
}
}
async fn raise_watermark(
rt: &KhiveRuntime,
ns: &str,
model: &str,
s: u64,
authority: CheckpointAuthority,
) -> Result<(), String> {
let shared_authority = match authority {
CheckpointAuthority::FullSentinel => WatermarkAuthority::Recovering,
CheckpointAuthority::Incremental => WatermarkAuthority::Active,
CheckpointAuthority::FullRegistered => WatermarkAuthority::PendingOrActive,
};
let sql = rt.sql();
let raised = if ann_segment_dir(rt, ns, model).is_none() {
let watermark = i64::try_from(s)
.map_err(|_| format!("knowledge ANN watermark {s} exceeds SQLite INTEGER range"))?;
let predicate = match shared_authority {
WatermarkAuthority::PendingOrActive => {
"(watermark = -2 OR (watermark >= 0 AND watermark <= ?4))"
}
WatermarkAuthority::Active => "watermark >= 0 AND watermark <= ?4",
WatermarkAuthority::Recovering => "watermark = -1",
};
let mut writer = sql.writer().await.map_err(|error| error.to_string())?;
writer
.execute(SqlStatement {
sql: format!(
"UPDATE ann_consumer_watermark SET watermark = ?4 \
WHERE consumer = ?1 AND namespace = ?2 AND embedding_model = ?3 \
AND {predicate}"
),
params: vec![
SqlValue::Text(ANN_CONSUMER.into()),
SqlValue::Text(ns.to_owned()),
SqlValue::Text(model.to_owned()),
SqlValue::Integer(watermark),
],
label: Some("knowledge_ann_raise_pathless_watermark".into()),
})
.await
.map_err(|error| error.to_string())?
== 1
} else {
ann_registry::raise_watermark(sql.as_ref(), ANN_CONSUMER, ns, model, s, shared_authority)
.await
.map_err(|e| e.to_string())?
};
if !raised {
return Err(format!(
"ANN watermark publication fence rejected {authority:?}: affected 0 rows"
));
}
Ok(())
}
async fn compact_log(rt: &KhiveRuntime, ns: &str, model: &str) -> Result<(), String> {
let sql = rt.sql();
if ann_segment_dir(rt, ns, model).is_none() {
let mut writer = sql.writer().await.map_err(|error| error.to_string())?;
writer
.execute(SqlStatement {
sql: "DELETE FROM ann_write_log \
WHERE namespace = ?1 AND embedding_model = ?2 \
AND seq <= (SELECT MIN(watermark.watermark) \
FROM ann_consumer_watermark watermark \
WHERE (watermark.namespace = ?1 \
OR watermark.namespace = '*') \
AND watermark.embedding_model = ?2)"
.into(),
params: vec![
SqlValue::Text(ns.to_owned()),
SqlValue::Text(model.to_owned()),
],
label: Some("knowledge_ann_compact_pathless_log".into()),
})
.await
.map(|_| ())
.map_err(|error| error.to_string())?;
return Ok(());
}
ann_registry::compact_write_log(
sql.as_ref(),
CompactionScope::Namespace(ns.to_owned()),
model,
)
.await
.map(|_| ())
.map_err(|e| e.to_string())
}
async fn tail_exists(rt: &KhiveRuntime, ns: &str, model: &str, s: u64) -> Result<bool, String> {
let sql = rt.sql();
let mut reader = sql.reader().await.map_err(|e| e.to_string())?;
let rows = reader
.query_all(SqlStatement {
sql: "SELECT EXISTS(SELECT 1 FROM ann_write_log \
WHERE namespace = ?1 AND embedding_model = ?2 \
AND field = 'knowledge.atom' AND seq > ?3) AS has_tail"
.into(),
params: vec![
SqlValue::Text(ns.to_owned()),
SqlValue::Text(model.to_owned()),
SqlValue::Integer(s as i64),
],
label: Some("ann_tail_probe".into()),
})
.await
.map_err(|e| e.to_string())?;
match rows.first().and_then(|r| r.get("has_tail")) {
Some(SqlValue::Integer(n)) => Ok(*n != 0),
other => Err(format!("tail probe: unexpected value {other:?}")),
}
}
async fn scope_counts(
rt: &KhiveRuntime,
ns: &str,
model: &str,
s: u64,
) -> Result<(u64, u64), String> {
let table_name = format!("vec_{}", sanitize_model_key(model));
let sql = rt.sql();
let mut reader = sql.reader().await.map_err(|e| e.to_string())?;
let rows = reader
.query_all(SqlStatement {
sql: format!(
"SELECT \
(SELECT COUNT(*) FROM {table_name} \
WHERE namespace = ?1 AND embedding_model = ?2 \
AND field = 'knowledge.atom') AS live, \
(SELECT COUNT(*) FROM ann_write_log \
WHERE namespace = ?1 AND embedding_model = ?2 \
AND field = 'knowledge.atom' AND seq > ?3) AS tail"
),
params: vec![
SqlValue::Text(ns.to_owned()),
SqlValue::Text(model.to_owned()),
SqlValue::Integer(s as i64),
],
label: Some("ann_scope_counts".into()),
})
.await
.map_err(|e| e.to_string())?;
let row = rows
.into_iter()
.next()
.ok_or("scope_counts returned no row")?;
let get = |col: &str| match row.get(col) {
Some(SqlValue::Integer(n)) => u64::try_from(*n).map_err(|_| format!("negative {col}")),
other => Err(format!("scope_counts {col}: unexpected value {other:?}")),
};
Ok((get("live")?, get("tail")?))
}
async fn fetch_final_states(
rt: &KhiveRuntime,
ns: &str,
model: &str,
s: u64,
) -> Result<(Vec<(Uuid, bool)>, u64), String> {
let sql = rt.sql();
let mut reader = sql.reader().await.map_err(|e| e.to_string())?;
let rows = reader
.query_all(SqlStatement {
sql: "SELECT subject_id, op, MAX(seq) AS seq FROM ann_write_log \
WHERE namespace = ?1 AND embedding_model = ?2 \
AND field = 'knowledge.atom' AND seq > ?3 \
GROUP BY subject_id"
.into(),
params: vec![
SqlValue::Text(ns.to_owned()),
SqlValue::Text(model.to_owned()),
SqlValue::Integer(s as i64),
],
label: Some("ann_fetch_final_states".into()),
})
.await
.map_err(|e| e.to_string())?;
let mut new_s = s;
let mut finals: Vec<(Uuid, bool)> = Vec::with_capacity(rows.len());
for row in &rows {
let seq = match row.get("seq") {
Some(SqlValue::Integer(n)) => *n,
_ => return Err("ann_write_log.seq: unexpected value".into()),
};
new_s = new_s.max(u64::try_from(seq).map_err(|_| "negative seq")?);
let uuid = match row.get("subject_id") {
Some(SqlValue::Text(t)) => {
Uuid::parse_str(t).map_err(|e| format!("tail subject_id {t}: {e}"))?
}
_ => return Err("ann_write_log.subject_id: unexpected value".into()),
};
let is_delete = match row.get("op") {
Some(SqlValue::Text(t)) => t == "delete",
_ => return Err("ann_write_log.op: unexpected value".into()),
};
finals.push((uuid, is_delete));
}
Ok((finals, new_s))
}
const REPLAY_BATCH: usize = 500;
async fn replay_final_states(
rt: &KhiveRuntime,
bridge: &mut AnnBridge,
ns: &str,
model: &str,
finals: &[(Uuid, bool)],
) -> Result<(), String> {
let table_name = format!("vec_{}", sanitize_model_key(model));
let point_read_sql = format!(
"SELECT namespace, embedding_model, field, embedding \
FROM {table_name} WHERE subject_id = ?1"
);
let sql = rt.sql();
let mut reader = sql.reader().await.map_err(|e| e.to_string())?;
let mut reverse = bridge.reverse_map();
for batch in finals.chunks(REPLAY_BATCH) {
let mut embeddings: HashMap<Uuid, Vec<f32>> = HashMap::new();
for (uuid, is_delete) in batch {
if *is_delete {
continue;
}
let rows = reader
.query_all(SqlStatement {
sql: point_read_sql.clone(),
params: vec![SqlValue::Text(uuid.to_string())],
label: Some("ann_replay_point_read".into()),
})
.await
.map_err(|e| e.to_string())?;
let Some(row) = rows.first() else {
return Err(format!(
"final upsert for {uuid} has no source row (contradiction → Cold)"
));
};
let in_scope = matches!(row.get("namespace"), Some(SqlValue::Text(t)) if t == ns)
&& matches!(row.get("embedding_model"), Some(SqlValue::Text(t)) if t == model)
&& matches!(row.get("field"), Some(SqlValue::Text(t)) if t == "knowledge.atom");
if !in_scope {
return Err(format!(
"final upsert for {uuid}: source row left the consumer scope \
(contradiction → Cold)"
));
}
let Some(SqlValue::Blob(bytes)) = row.get("embedding") else {
return Err(format!("final upsert for {uuid}: embedding missing on row"));
};
#[allow(unknown_lints, clippy::chunks_exact_to_as_chunks)]
let vec: Vec<f32> = bytes
.chunks_exact(4)
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
.collect();
embeddings.insert(*uuid, vec);
}
for (uuid, is_delete) in batch {
let op =
if *is_delete {
None
} else {
Some(embeddings.remove(uuid).ok_or_else(|| {
format!("final upsert for {uuid}: embedding lost in batch")
})?)
};
bridge.apply_final_op(&mut reverse, *uuid, op)?;
}
}
Ok(())
}
struct FreshTailSnapshot {
own_watermark: Option<i64>,
registry_min: Option<i64>,
live_count: Option<u64>,
ops: Vec<(Uuid, Option<Vec<f32>>)>,
}
async fn fetch_fresh_tail_snapshot(
rt: &KhiveRuntime,
ns: &str,
model: &str,
watermark: u64,
live_threshold: Option<f64>,
) -> Result<FreshTailSnapshot, String> {
let watermark = i64::try_from(watermark)
.map_err(|_| "fresh-tail watermark exceeds SQLite INTEGER range".to_string())?;
let table_name = format!("vec_{}", sanitize_model_key(model));
let (live_cte, selected_order, live_join, live_column) = match live_threshold {
Some(_) => (
format!(
"live AS (\
SELECT COUNT(*) AS live_count FROM {table_name} \
WHERE namespace = ?1 AND embedding_model = ?2 \
AND field = 'knowledge.atom'\
),"
),
"ORDER BY seq DESC \
LIMIT (SELECT CAST(live_count * ?5 AS INTEGER) + \
CASE WHEN CAST(live_count * ?5 AS INTEGER) < live_count * ?5 \
THEN 1 ELSE 0 END FROM live)",
"CROSS JOIN live",
"live.live_count",
),
None => (String::new(), "ORDER BY seq", "", "NULL"),
};
let mut params = vec![
SqlValue::Text(ns.to_owned()),
SqlValue::Text(model.to_owned()),
SqlValue::Integer(watermark),
SqlValue::Text(ANN_CONSUMER.into()),
];
if let Some(threshold) = live_threshold {
params.push(SqlValue::Float(threshold));
}
let sql = rt.sql();
let mut reader = sql.reader().await.map_err(|error| error.to_string())?;
let rows = reader
.query_all(SqlStatement {
sql: format!(
"WITH \
registry AS (\
SELECT MIN(watermark) AS registry_min \
FROM ann_consumer_watermark \
WHERE (namespace = ?1 OR namespace = '*') \
AND embedding_model = ?2\
), \
own AS (\
SELECT (SELECT watermark FROM ann_consumer_watermark \
WHERE consumer = ?4 AND namespace = ?1 \
AND embedding_model = ?2) AS own_watermark\
), \
{live_cte} \
selected AS (\
SELECT seq, subject_id, op FROM ann_write_log \
WHERE namespace = ?1 AND embedding_model = ?2 \
AND field = 'knowledge.atom' \
AND seq > MAX(\
?3, COALESCE((SELECT registry_min FROM registry), ?3)\
) \
{selected_order}\
) \
SELECT selected.seq, selected.subject_id, selected.op, \
vectors.namespace AS vector_namespace, \
vectors.embedding_model AS vector_model, \
vectors.field AS vector_field, \
vectors.embedding, registry.registry_min, \
own.own_watermark, {live_column} AS live_count \
FROM registry CROSS JOIN own {live_join} \
LEFT JOIN selected ON 1 = 1 \
LEFT JOIN {table_name} AS vectors \
ON vectors.subject_id = selected.subject_id \
ORDER BY selected.seq"
),
params,
label: Some("knowledge_ann_fresh_tail_snapshot".into()),
})
.await
.map_err(|error| error.to_string())?;
let first = rows
.first()
.ok_or_else(|| "fresh-tail snapshot returned no registry row".to_string())?;
let read_optional_i64 = |column: &str| -> Result<Option<i64>, String> {
match first.get(column) {
Some(SqlValue::Integer(value)) => Ok(Some(*value)),
Some(SqlValue::Null) | None => Ok(None),
other => Err(format!("fresh-tail {column}: unexpected value {other:?}")),
}
};
let own_watermark = read_optional_i64("own_watermark")?;
let registry_min = read_optional_i64("registry_min")?;
let live_count = match read_optional_i64("live_count")? {
Some(value) => {
Some(u64::try_from(value).map_err(|_| "negative fresh-tail live_count".to_string())?)
}
None => None,
};
type RawVector = (
Option<String>,
Option<String>,
Option<String>,
Option<Vec<u8>>,
);
let mut finals: Vec<(Uuid, bool, RawVector)> = Vec::new();
let mut index_by_id: HashMap<Uuid, usize> = HashMap::new();
for row in &rows {
let Some(SqlValue::Integer(_)) = row.get("seq") else {
continue;
};
let subject = match row.get("subject_id") {
Some(SqlValue::Text(value)) => Uuid::parse_str(value)
.map_err(|error| format!("fresh-tail subject_id {value}: {error}"))?,
other => return Err(format!("fresh-tail subject_id: unexpected value {other:?}")),
};
let is_delete = match row.get("op") {
Some(SqlValue::Text(value)) if value == "delete" => true,
Some(SqlValue::Text(value)) if value == "upsert" => false,
other => return Err(format!("fresh-tail op: unexpected value {other:?}")),
};
let raw_vector = (
match row.get("vector_namespace") {
Some(SqlValue::Text(value)) => Some(value.clone()),
_ => None,
},
match row.get("vector_model") {
Some(SqlValue::Text(value)) => Some(value.clone()),
_ => None,
},
match row.get("vector_field") {
Some(SqlValue::Text(value)) => Some(value.clone()),
_ => None,
},
match row.get("embedding") {
Some(SqlValue::Blob(value)) => Some(value.clone()),
_ => None,
},
);
match index_by_id.get(&subject) {
Some(&index) => finals[index] = (subject, is_delete, raw_vector),
None => {
index_by_id.insert(subject, finals.len());
finals.push((subject, is_delete, raw_vector));
}
}
}
let mut ops = Vec::with_capacity(finals.len());
for (subject, is_delete, (vector_ns, vector_model, vector_field, embedding)) in finals {
if is_delete {
ops.push((subject, None));
continue;
}
let in_scope = vector_ns.as_deref() == Some(ns)
&& vector_model.as_deref() == Some(model)
&& vector_field.as_deref() == Some("knowledge.atom");
if !in_scope {
return Err(format!(
"fresh-tail upsert {subject}: vector row outside consumer scope"
));
}
let Some(bytes) = embedding else {
return Err(format!(
"fresh-tail upsert {subject}: embedding is not a blob"
));
};
if bytes.len() % std::mem::size_of::<f32>() != 0 {
return Err(format!(
"fresh-tail upsert {subject}: malformed embedding byte length {}",
bytes.len()
));
}
#[allow(unknown_lints, clippy::chunks_exact_to_as_chunks)]
let embedding = bytes
.chunks_exact(4)
.map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]))
.collect();
ops.push((subject, Some(embedding)));
}
Ok(FreshTailSnapshot {
own_watermark,
registry_min,
live_count,
ops,
})
}
fn exact_cosine(query: &[f32], embedding: &[f32]) -> f32 {
if query.len() != embedding.len() || query.is_empty() {
return 0.0;
}
let mut query = query.to_vec();
let mut embedding = embedding.to_vec();
l2_normalize(&mut query);
l2_normalize(&mut embedding);
query
.iter()
.zip(embedding.iter())
.map(|(left, right)| left * right)
.sum::<f32>()
.max(0.0)
}
pub(crate) fn merge_fresh_tail(
candidates: Vec<(Uuid, f32)>,
query: &[f32],
ops: Vec<(Uuid, Option<Vec<f32>>)>,
) -> Vec<(Uuid, f32)> {
if ops.is_empty() {
return candidates;
}
let mut deletes = HashSet::new();
let mut upserts = HashMap::new();
for (subject, op) in ops {
match op {
Some(embedding) => {
upserts.insert(subject, exact_cosine(query, &embedding));
}
None => {
deletes.insert(subject);
}
}
}
let mut merged: Vec<(Uuid, f32)> = candidates
.into_iter()
.filter(|(subject, _)| !deletes.contains(subject) && !upserts.contains_key(subject))
.collect();
merged.extend(upserts);
merged.sort_by(|left, right| {
right
.1
.partial_cmp(&left.1)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| left.0.cmp(&right.0))
});
merged
}
pub(crate) async fn merge_fresh_tail_off_thread(
candidates: Vec<(Uuid, f32)>,
query: &[f32],
ops: Vec<(Uuid, Option<Vec<f32>>)>,
) -> Vec<(Uuid, f32)> {
if ops.is_empty() {
return candidates;
}
let query = query.to_vec();
tokio::task::spawn_blocking(move || merge_fresh_tail(candidates, &query, ops))
.await
.expect("ANN fresh-tail merge task panicked")
}
pub(crate) enum FreshTailOutcome {
Ops(Vec<(Uuid, Option<Vec<f32>>)>),
Replace {
candidates: Vec<(Uuid, f32)>,
source_exhausted: bool,
},
Skipped,
}
pub(crate) async fn fresh_tail_leg(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
query: &[f32],
k: usize,
watermark: Option<u64>,
) -> FreshTailOutcome {
let ns = key.namespace.as_str();
let model = key.model.as_str();
if force_rebuild_required(ann, key) {
return FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
};
}
if !rt.ann_fresh_tail_enabled() {
return match read_own_watermark(rt, ns, model).await {
Ok(Some(watermark)) if watermark >= 0 => FreshTailOutcome::Skipped,
Ok(_) => force_cold_after_registry_loss(rt, ann, key).await,
Err(error) => {
tracing::warn!(
error = %error,
namespace = ns,
model,
"knowledge ANN registry read failed while fresh-tail was disabled"
);
FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
}
}
};
}
match watermark {
Some(watermark) => fresh_tail_serving(rt, ann, key, query, k, watermark).await,
None => fresh_tail_capped(rt, ann, key).await,
}
}
async fn force_cold_after_registry_loss(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
) -> FreshTailOutcome {
if rt.is_read_only() {
clear_namespace(ann, &key.namespace).await;
return FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
};
}
if let Err(error) = prepare_authoritative_rebuild(rt, ann, key).await {
tracing::warn!(
error = %error,
namespace = key.namespace,
model = key.model,
"knowledge ANN failed to publish registry-loss rebuild sentinel"
);
}
clear_namespace(ann, &key.namespace).await;
FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
}
}
fn ready_snapshot_watermark(snapshot: &FreshTailSnapshot) -> Option<u64> {
snapshot
.own_watermark
.filter(|watermark| *watermark >= 0)
.and_then(|watermark| u64::try_from(watermark).ok())
}
fn nonnegative_registry_min(snapshot: &FreshTailSnapshot) -> u64 {
snapshot
.registry_min
.filter(|watermark| *watermark >= 0)
.and_then(|watermark| u64::try_from(watermark).ok())
.unwrap_or(0)
}
async fn fresh_tail_serving(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
query: &[f32],
k: usize,
watermark: u64,
) -> FreshTailOutcome {
let ns = key.namespace.as_str();
let model = key.model.as_str();
let snapshot = match fetch_fresh_tail_snapshot(rt, ns, model, watermark, None).await {
Ok(snapshot) => snapshot,
Err(error) => {
tracing::warn!(
error = %error,
namespace = ns,
model,
"knowledge fresh-tail snapshot failed; dropping stale vector leg"
);
return FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
};
}
};
let Some(_own_watermark) = ready_snapshot_watermark(&snapshot) else {
return force_cold_after_registry_loss(rt, ann, key).await;
};
let registry_min = nonnegative_registry_min(&snapshot);
if registry_min > watermark {
let published_watermark = ann_segment_dir(rt, ns, model)
.and_then(|dir| read_commit_info(&dir).ok().flatten())
.and_then(|info| info.last_applied_seq)
.filter(|published| *published >= registry_min);
match published_watermark {
Some(published_watermark) => {
clear_namespace(ann, ns).await;
return fresh_tail_reresolve(rt, ann, key, query, k, published_watermark).await;
}
None => {
clear_namespace(ann, ns).await;
return FreshTailOutcome::Replace {
candidates: merge_fresh_tail_off_thread(Vec::new(), query, snapshot.ops).await,
source_exhausted: true,
};
}
}
}
FreshTailOutcome::Ops(snapshot.ops)
}
const FRESH_TAIL_RERESOLVE_MAX_ROUNDS: u32 = 3;
async fn fresh_tail_reresolve(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
query: &[f32],
k: usize,
published_watermark: u64,
) -> FreshTailOutcome {
let ns = key.namespace.as_str();
let model = key.model.as_str();
let mut expected_watermark = published_watermark;
for round in 1..=FRESH_TAIL_RERESOLVE_MAX_ROUNDS {
let Some(dir) = ann_segment_dir(rt, ns, model) else {
tracing::warn!(
key = ?key,
namespace = ns,
model,
"knowledge fresh-tail published segment disappeared during re-resolution"
);
return FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
};
};
let query_for_search = query.to_vec();
let loaded = tokio::task::spawn_blocking(move || {
let bridge = AnnBridge::load(&dir)?;
let loaded_watermark = bridge.index.last_applied_seq();
let candidates = bridge.search(&query_for_search, k);
Ok::<_, String>((candidates, loaded_watermark))
})
.await;
let (candidates, loaded_watermark) = match loaded {
Ok(Ok(loaded)) => loaded,
Ok(Err(error)) => {
tracing::warn!(
error = %error,
key = ?key,
namespace = ns,
model,
"knowledge fresh-tail published segment load failed"
);
return FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
};
}
Err(error) => {
tracing::warn!(
error = %error,
key = ?key,
namespace = ns,
model,
"knowledge fresh-tail segment worker failed during re-resolution"
);
return FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
};
}
};
let loaded_watermark = loaded_watermark.unwrap_or(expected_watermark);
let source_exhausted = candidates.len() < k;
let snapshot = match fetch_fresh_tail_snapshot(rt, ns, model, loaded_watermark, None).await
{
Ok(snapshot) => snapshot,
Err(error) => {
tracing::warn!(
error = %error,
key = ?key,
namespace = ns,
model,
"knowledge fresh-tail re-resolution snapshot failed"
);
return FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
};
}
};
let Some(_own_watermark) = ready_snapshot_watermark(&snapshot) else {
return force_cold_after_registry_loss(rt, ann, key).await;
};
let registry_min = nonnegative_registry_min(&snapshot);
if registry_min <= loaded_watermark {
return FreshTailOutcome::Replace {
candidates: merge_fresh_tail_off_thread(candidates, query, snapshot.ops).await,
source_exhausted,
};
}
if round == FRESH_TAIL_RERESOLVE_MAX_ROUNDS {
tracing::warn!(
key = ?key,
namespace = ns,
model,
rounds = round,
floor = registry_min,
"knowledge fresh-tail re-resolution did not converge; using exact-only floored suffix"
);
return FreshTailOutcome::Replace {
candidates: merge_fresh_tail_off_thread(Vec::new(), query, snapshot.ops).await,
source_exhausted: true,
};
}
expected_watermark = registry_min;
}
unreachable!("fresh-tail re-resolution loop returns within its bounded rounds")
}
async fn fresh_tail_capped(rt: &KhiveRuntime, ann: &SharedAnn, key: &AnnKey) -> FreshTailOutcome {
let snapshot = match fetch_fresh_tail_snapshot(
rt,
&key.namespace,
&key.model,
0,
Some(ann_rebuild_threshold()),
)
.await
{
Ok(snapshot) => snapshot,
Err(error) => {
tracing::warn!(
error = %error,
namespace = key.namespace,
model = key.model,
"knowledge capped fresh-tail fetch failed"
);
return FreshTailOutcome::Skipped;
}
};
if ready_snapshot_watermark(&snapshot).is_none() {
return force_cold_after_registry_loss(rt, ann, key).await;
}
debug_assert!(snapshot.live_count.is_some());
FreshTailOutcome::Ops(snapshot.ops)
}
async fn adopt_checkpoint_winner(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
generation: u64,
observed_watermark: u64,
) -> bool {
let installed = match ann_segment_dir(rt, &key.namespace, &key.model) {
Some(dir) => match AnnBridge::load(&dir) {
Ok(bridge) if bridge.index.last_applied_seq().unwrap_or(0) >= observed_watermark => {
ann.indexes.write().await.remove(key);
install_replacing(ann, key, bridge.with_generation(generation)).await
}
Ok(bridge) => {
tracing::warn!(
key = ?key,
bridge_watermark = bridge.index.last_applied_seq().unwrap_or(0),
observed_watermark,
"checkpoint winner segment trails its durable watermark"
);
ann.indexes.write().await.remove(key);
false
}
Err(error) => {
tracing::warn!(
error = %error,
key = ?key,
"failed to adopt checkpoint race winner"
);
ann.indexes.write().await.remove(key);
false
}
},
None => {
let current = has_current_index_at_watermark(ann, key, observed_watermark).await;
if !current {
ann.indexes.write().await.remove(key);
}
current
}
};
clear_force_rebuild(ann, key);
installed
}
pub(crate) async fn checkpoint_raise_compact_readopt(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
bridge: AnnBridge,
generation: u64,
authority: CheckpointAuthority,
) -> bool {
let ns = key.namespace.as_str();
let model = key.model.as_str();
let local_publication_lock = checkpoint_lock(ann, key);
let _local_publication_guard = local_publication_lock.lock().await;
let publication_lock = match ann_segment_dir(rt, ns, model) {
Some(dir) => match acquire_bridge_checkpoint_lock_async(dir).await {
Ok(lock) => Some(lock),
Err(error) => {
tracing::warn!(error = %error, "failed to acquire ANN checkpoint lock");
return false;
}
},
None => None,
};
let applied = bridge.index.last_applied_seq().unwrap_or(0);
let own_watermark = match read_own_watermark(rt, ns, model).await {
Ok(value) => value,
Err(error) => {
tracing::warn!(error = %error, "ann checkpoint registry read failed");
return false;
}
};
if authority == CheckpointAuthority::FullSentinel {
if let Some(observed) = own_watermark.filter(|watermark| *watermark >= 0) {
tracing::info!(
key = ?key,
observed_watermark = observed,
"authoritative ANN rebuild lost publication race; adopting winner"
);
let observed = u64::try_from(observed).unwrap_or(0);
return adopt_checkpoint_winner(rt, ann, key, generation, observed).await;
}
} else if let Some(observed) = own_watermark.filter(|watermark| *watermark >= 0) {
let observed = u64::try_from(observed).unwrap_or(0);
if observed > applied {
tracing::info!(
key = ?key,
candidate_watermark = applied,
observed_watermark = observed,
"stale ANN checkpoint lost publication race; adopting winner"
);
return adopt_checkpoint_winner(rt, ann, key, generation, observed).await;
}
}
let authorized = match authority {
CheckpointAuthority::FullSentinel => own_watermark == Some(-1),
CheckpointAuthority::Incremental | CheckpointAuthority::FullRegistered => {
own_watermark.is_some_and(|watermark| watermark >= 0)
}
};
if !authorized {
mark_force_rebuild(ann, key);
if let Err(error) = write_force_rebuild_sentinel_row(rt, key).await {
tracing::warn!(error = %error, "failed to fence unauthorized ANN checkpoint");
}
return false;
}
if let Err(e) = persist_ann_v2_locked(rt, ns, model, &bridge) {
tracing::error!(error = %e, "failed to persist v2 Vamana segment");
install_replacing(ann, key, bridge.with_generation(generation)).await;
return false;
}
let published = match ann_segment_dir(rt, ns, model) {
Some(dir) => match AnnBridge::load(&dir) {
Ok(mmap_bridge) => mmap_bridge,
Err(e) => {
tracing::warn!(error = %e, "mmap re-adoption failed; serving Owned build");
bridge
}
},
None => bridge,
};
let file_backed = publication_lock.is_some();
let mut pending_in_memory = Some(published.with_generation(generation));
let mut installed = false;
if file_backed {
installed = install_replacing(
ann,
key,
pending_in_memory.take().expect("published bridge"),
)
.await;
}
if let Err(e) = raise_watermark(rt, ns, model, applied, authority).await {
tracing::warn!(error = %e, "ann watermark raise failed (under-compacts; safe)");
match read_own_watermark(rt, ns, model).await {
Ok(Some(observed)) if observed >= 0 => {
let observed = u64::try_from(observed).unwrap_or(0);
if observed > applied {
return adopt_checkpoint_winner(rt, ann, key, generation, observed).await;
}
if let Some(published) = pending_in_memory.take() {
installed = install_replacing(ann, key, published).await;
}
clear_force_rebuild(ann, key);
return installed || has_current_index_at_watermark(ann, key, observed).await;
}
Ok(_) => {}
Err(read_error) => {
tracing::warn!(
error = %read_error,
"failed to reconcile ANN checkpoint race"
);
}
}
mark_force_rebuild(ann, key);
if let Err(error) = write_force_rebuild_sentinel_row(rt, key).await {
tracing::warn!(error = %error, "failed to restore ANN checkpoint fence");
}
return false;
}
if let Some(published) = pending_in_memory {
installed = install_replacing(ann, key, published).await;
}
if authority != CheckpointAuthority::Incremental {
clear_force_rebuild_if_current(ann, key, generation);
}
drop(publication_lock);
if let Err(e) = compact_log(rt, ns, model).await {
tracing::warn!(error = %e, "ann log compaction failed (retries next checkpoint)");
}
installed
}
async fn try_load_snapshot(
rt: &KhiveRuntime,
namespace: &str,
model: &str,
) -> Option<VamanaSnapshot> {
let key = snapshot_key(namespace, model);
let sql = rt.sql();
let mut reader = sql.reader().await.ok()?;
let rows = reader
.query_all(SqlStatement {
sql: "SELECT snapshot FROM retrieval_snapshots \
WHERE namespace = ?1 AND index_type = ?2"
.into(),
params: vec![SqlValue::Text(key), SqlValue::Text("vamana".into())],
label: None,
})
.await
.ok()?;
let row = rows.into_iter().next()?;
let blob = match row.get("snapshot")? {
SqlValue::Blob(b) => b.clone(),
_ => return None,
};
serde_json::from_slice::<VamanaSnapshot>(&blob).ok()
}
pub(crate) async fn compute_fingerprint(
rt: &KhiveRuntime,
token: &NamespaceToken,
model: &str,
) -> Option<CorpusFingerprint> {
let store = rt.vectors_for_model(token, model).ok()?;
let info = store.info().await.ok()?;
Some(CorpusFingerprint {
vector_count: info.entry_count,
dimensions: info.dimensions as u32,
})
}
async fn scan_corpus_raw(
rt: &KhiveRuntime,
token: &NamespaceToken,
model: &str,
) -> Result<Option<(Vec<f32>, Vec<Uuid>, u64)>, RuntimeError> {
let store = rt
.vectors_for_model(token, model)
.map_err(|e| RuntimeError::Internal(e.to_string()))?;
let info = store
.info()
.await
.map_err(|e| RuntimeError::Internal(e.to_string()))?;
let count = info.entry_count;
let dims = info.dimensions;
if count == 0 || dims == 0 {
return Ok(None);
}
let ns = token.namespace().as_str().to_owned();
let model_key = sanitize_model_key(model);
let table_name = format!("vec_{model_key}");
let model_str = model.to_owned();
let sql = rt.sql();
let mut reader = sql
.reader()
.await
.map_err(|e| RuntimeError::Internal(e.to_string()))?;
let rows = reader
.query_all(SqlStatement {
sql: format!(
"SELECT subject_id, embedding, \
(SELECT COALESCE(\
(SELECT seq FROM sqlite_sequence \
WHERE name = 'ann_write_log'), 0)) AS log_s \
FROM {table_name} \
WHERE namespace = ?1 AND embedding_model = ?2 \
AND field = 'knowledge.atom' \
ORDER BY subject_id"
),
params: vec![SqlValue::Text(ns), SqlValue::Text(model_str)],
label: Some("vamana_corpus_scan".into()),
})
.await
.map_err(|e| RuntimeError::Internal(e.to_string()))?;
if rows.is_empty() {
return Ok(None);
}
let mut id_map: Vec<Uuid> = Vec::with_capacity(rows.len());
let mut flat: Vec<f32> = Vec::with_capacity(rows.len() * dims);
let scan_watermark = rows
.first()
.and_then(|row| match row.get("log_s") {
Some(SqlValue::Integer(n)) => u64::try_from(*n).ok(),
_ => None,
})
.unwrap_or(0);
for row in &rows {
let id_str = match row.get("subject_id") {
Some(SqlValue::Text(s)) => s.as_str(),
_ => continue,
};
let uuid = match Uuid::parse_str(id_str) {
Ok(id) => id,
Err(_) => continue,
};
let bytes = match row.get("embedding") {
Some(SqlValue::Blob(b)) => b.as_slice(),
_ => continue,
};
if bytes.len() != dims * 4 {
continue;
}
#[allow(unknown_lints, clippy::chunks_exact_to_as_chunks)]
let vec: Vec<f32> = bytes
.chunks_exact(4)
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
.collect();
id_map.push(uuid);
flat.extend_from_slice(&vec);
}
if id_map.is_empty() {
return Ok(None);
}
Ok(Some((flat, id_map, scan_watermark)))
}
pub(crate) async fn load_and_build_from_vector_store(
rt: &KhiveRuntime,
token: &NamespaceToken,
model: &str,
) -> Result<Option<AnnBridge>, RuntimeError> {
let Some((flat, id_map, scan_watermark)) = scan_corpus_raw(rt, token, model).await? else {
return Ok(None);
};
let dims = flat.len() / id_map.len();
AnnBridge::build(flat, dims, id_map)
.map(|mut bridge| {
bridge.set_applied_seq(scan_watermark);
Some(bridge)
})
.map_err(RuntimeError::Internal)
}
fn escape_like(input: &str) -> String {
let mut out = String::with_capacity(input.len());
for c in input.chars() {
if matches!(c, '\\' | '%' | '_') {
out.push('\\');
}
out.push(c);
}
out
}
pub(crate) async fn invalidate_snapshot(rt: &KhiveRuntime, namespace: &str) {
let pattern = format!("{}::vamana::%", escape_like(namespace));
let sql = rt.sql();
let mut w = match sql.writer().await {
Ok(w) => w,
Err(e) => {
tracing::warn!(error = %e, "failed to open writer for Vamana snapshot invalidation");
return;
}
};
match w
.execute(SqlStatement {
sql: "DELETE FROM retrieval_snapshots WHERE namespace LIKE ?1 ESCAPE '\\'".into(),
params: vec![SqlValue::Text(pattern)],
label: Some("invalidate_vamana_snapshot".into()),
})
.await
{
Ok(_) => {}
Err(e) if e.to_string().contains("no such table") => {}
Err(e) => {
tracing::warn!(error = %e, "failed to invalidate Vamana snapshot");
}
}
}
async fn run_warm_attempt(
rt: &KhiveRuntime,
token: &NamespaceToken,
ann: &SharedAnn,
model: &str,
permit: AnnWarmPermit,
) {
let outcome = ensure_ann_for_model(rt, token, ann, model).await;
finish_warm(permit, outcome).await;
}
async fn warm_ann_for_model_once(
rt: &KhiveRuntime,
token: &NamespaceToken,
ann: &SharedAnn,
model: &str,
) {
if model.is_empty() {
return;
}
let key = AnnKey::new(token.namespace().as_str(), model);
let Some(permit) = begin_warm(ann, key) else {
return;
};
run_warm_attempt(rt, token, ann, model, permit).await;
}
pub(crate) async fn warm_known_snapshots(rt: &KhiveRuntime, ann: &SharedAnn) {
let rows = {
let sql = rt.sql();
match sql.reader().await {
Ok(mut reader) => reader
.query_all(SqlStatement {
sql:
"SELECT DISTINCT namespace FROM retrieval_snapshots WHERE namespace LIKE ?1"
.into(),
params: vec![SqlValue::Text("%::vamana::%".into())],
label: None,
})
.await
.unwrap_or_default(),
Err(_) => Vec::new(),
}
};
for row in &rows {
let ns_key = match row.get("namespace") {
Some(SqlValue::Text(s)) => s.as_str(),
_ => continue,
};
let Some((ns_str, model)) = ns_key.split_once("::vamana::") else {
continue;
};
if ns_str.is_empty() || model.is_empty() {
continue;
}
let ns = match Namespace::parse(ns_str) {
Ok(n) => n,
Err(_) => continue,
};
let token = match rt.authorize(ns) {
Ok(t) => t,
Err(_) => continue,
};
warm_ann_for_model_once(rt, &token, ann, model).await;
}
let ann_root = match rt.backend_ann_root() {
Some(d) => d,
None => return,
};
let read_dir = match std::fs::read_dir(&ann_root) {
Ok(rd) => rd,
Err(_) => return, };
for entry in read_dir.flatten() {
let name = entry.file_name();
let hex = name.to_string_lossy();
let Some((ns_str, model)) = decode_ann_dir_name(hex.as_ref()) else {
continue;
};
let ns = match Namespace::parse(&ns_str) {
Ok(n) => n,
Err(_) => continue,
};
let token = match rt.authorize(ns) {
Ok(t) => t,
Err(_) => continue,
};
warm_ann_for_model_once(rt, &token, ann, &model).await;
}
}
#[cfg(test)]
pub(crate) fn rotation_watch_started_for_test(ann: &SharedAnn) -> bool {
ann.rotation_watch_started.load(Ordering::Acquire)
}
pub(crate) fn start_rotation_watcher(rt: &KhiveRuntime, ann: &SharedAnn) {
drop(start_rotation_watcher_with_shutdown(
rt,
ann,
khive_runtime::daemon_shutdown_token(),
));
}
fn start_rotation_watcher_with_shutdown(
rt: &KhiveRuntime,
ann: &SharedAnn,
shutdown: tokio_util::sync::CancellationToken,
) -> Option<tokio::task::JoinHandle<()>> {
let ann_root = rt.backend_ann_root()?;
if ann
.rotation_watch_started
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.is_err()
{
return None;
}
let ann = Arc::downgrade(ann);
Some(khive_runtime::spawn_named_tracked_task(
"knowledge_ann_rotation_watch",
async move {
let start = tokio::time::Instant::now() + ROTATION_WATCH_INTERVAL;
let mut ticks = tokio::time::interval_at(start, ROTATION_WATCH_INTERVAL);
ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
tokio::select! {
_ = shutdown.cancelled() => break,
_ = ticks.tick() => {}
}
let Some(ann) = ann.upgrade() else {
break;
};
refresh_rotated_segments_in_root(&ann_root, &ann).await;
}
},
))
}
#[cfg(test)]
async fn refresh_rotated_segments_once(rt: &KhiveRuntime, ann: &SharedAnn) {
let Some(ann_root) = rt.backend_ann_root() else {
return;
};
refresh_rotated_segments_in_root(&ann_root, ann).await;
}
async fn refresh_rotated_segments_in_root(ann_root: &std::path::Path, ann: &SharedAnn) {
let installed: Vec<(AnnKey, [u8; 32])> = ann
.indexes
.read()
.await
.iter()
.filter_map(|(key, bridge)| bridge.commit_digest.map(|digest| (key.clone(), digest)))
.collect();
for (key, expected) in installed {
let dir = ann_segment_dir_from_root(ann_root, &key.namespace, &key.model);
match segment_commit_digest(&dir) {
Ok(Some(observed)) if observed != expected => {
refresh_rotated_segment(ann, &key, expected, dir).await;
}
Ok(_) => {}
Err(error) => {
tracing::warn!(
error = %error,
namespace = %key.namespace,
model = %key.model,
"knowledge ANN rotation check failed; retaining the installed generation"
);
}
}
}
}
async fn evict_bridge_and_ready_state(
ann: &SharedAnn,
key: &AnnKey,
incumbent_digest: [u8; 32],
generation: u64,
) -> bool {
let mut indexes = ann.indexes.write().await;
if indexes
.get(key)
.is_none_or(|bridge| bridge.commit_digest != Some(incumbent_digest))
{
return false;
}
indexes.remove(key);
let mut states = warm_states_guard(&ann.warm_states);
if matches!(
states.get(key),
Some(AnnWarmState::Ready { generation: state_generation }) if *state_generation == generation
) {
states.remove(key);
}
true
}
async fn refresh_rotated_segment(
ann: &SharedAnn,
key: &AnnKey,
expected: [u8; 32],
dir: std::path::PathBuf,
) {
let local_lock = checkpoint_lock(ann, key);
let _local_guard = local_lock.lock().await;
let incumbent = ann.indexes.read().await.get(key).map(|bridge| {
(
bridge.commit_digest,
bridge.generation,
bridge.index.last_applied_seq().unwrap_or(0),
)
});
let Some((Some(incumbent_digest), generation, incumbent_seq)) = incumbent else {
return;
};
if incumbent_digest != expected {
return;
}
let _publication_guard = match acquire_bridge_checkpoint_lock_async(dir.clone()).await {
Ok(lock) => lock,
Err(error) => {
tracing::warn!(
error = %error,
namespace = %key.namespace,
model = %key.model,
"knowledge ANN rotation reload could not acquire the publication lock"
);
return;
}
};
let observed = match segment_commit_digest(&dir) {
Ok(Some(digest)) if digest != incumbent_digest => digest,
Ok(_) => return,
Err(error) => {
tracing::warn!(
error = %error,
namespace = %key.namespace,
model = %key.model,
"knowledge ANN rotated commit identity could not be read"
);
return;
}
};
let replacement = AnnBridge::load(&dir).and_then(|bridge| {
if bridge.commit_digest != Some(observed) {
return Err("commit identity changed during locked rotation reload".to_string());
}
let replacement_seq = bridge.index.last_applied_seq().unwrap_or(0);
if replacement_seq < incumbent_seq {
return Err(format!(
"rotated segment watermark {replacement_seq} regressed below installed {incumbent_seq}"
));
}
Ok(bridge.with_generation(generation))
});
match replacement {
Ok(bridge) => {
if install_replacing(ann, key, bridge).await {
tracing::debug!(
namespace = %key.namespace,
model = %key.model,
"knowledge ANN adopted rotated mmap generation and released its predecessor"
);
}
}
Err(error) => {
evict_bridge_and_ready_state(ann, key, incumbent_digest, generation).await;
tracing::warn!(
error = %error,
namespace = %key.namespace,
model = %key.model,
"knowledge ANN rotated generation failed validation; released predecessor and will rebuild on demand"
);
}
}
}
pub(crate) fn ensure_ann_background(rt: &KhiveRuntime, token: &NamespaceToken, ann: &SharedAnn) {
if rt.is_read_only() {
return;
}
let model = rt.default_embedder_name().to_string();
if model.is_empty() {
return;
}
let ns = token.namespace().as_str().to_owned();
let key = AnnKey::new(&ns, &model);
let Some(permit) = begin_warm(ann, key) else {
return;
};
let rt = rt.clone();
let ann = ann.clone();
let token = token.clone();
tokio::spawn(async move {
run_warm_attempt(&rt, &token, &ann, &model, permit).await;
});
}
enum SegmentOutcome {
Installed,
Empty,
Cold,
ForceRebuild,
}
#[allow(clippy::too_many_arguments)]
async fn classify_and_adopt_segment(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
ns: &str,
model: &str,
seg_dir: &std::path::Path,
target_generation: u64,
) -> SegmentOutcome {
if force_rebuild_required(ann, key) {
return SegmentOutcome::ForceRebuild;
}
let info = match read_commit_info(seg_dir) {
Ok(Some(info)) => info,
Ok(None) => return SegmentOutcome::Cold,
Err(e) => {
tracing::warn!(error = %e, dir = %seg_dir.display(),
"error reading v2 commit record; Cold");
return SegmentOutcome::Cold;
}
};
let Some(s) = info.last_applied_seq else {
tracing::info!(namespace = %ns, model = %model,
"pre-amendment v2 segment (no watermark); Cold rebuild");
return SegmentOutcome::Cold;
};
match rt.embedder_dimensions(model) {
Some(dims) if dims as u64 == info.dimensions => {}
Some(dims) => {
tracing::info!(namespace = %ns, model = %model,
segment_dims = info.dimensions, live_dims = dims,
"v2 segment dimension mismatch; Cold rebuild");
return SegmentOutcome::Cold;
}
None => return SegmentOutcome::Cold,
}
match read_own_watermark(rt, ns, model).await {
Ok(Some(watermark)) if watermark >= 0 => {}
Ok(Some(_)) => {
mark_force_rebuild(ann, key);
return SegmentOutcome::ForceRebuild;
}
Ok(None) => {
tracing::info!(namespace = %ns, model = %model,
"ann consumer registry row absent; fencing for authoritative rebuild");
if let Err(error) = prepare_authoritative_rebuild(rt, ann, key).await {
tracing::warn!(error = %error, "ann consumer re-registration failed");
}
return SegmentOutcome::ForceRebuild;
}
Err(e) => {
tracing::warn!(error = %e, "ann registry read failed; Cold");
return SegmentOutcome::Cold;
}
}
match tail_exists(rt, ns, model, s).await {
Ok(false) => {
return match AnnBridge::load(seg_dir) {
Ok(bridge) => {
install_if_fresher(ann, key, bridge.with_generation(target_generation)).await;
SegmentOutcome::Installed
}
Err(e) => {
tracing::warn!(error = %e, dir = %seg_dir.display(),
"Hot segment load failed; Cold rebuild");
SegmentOutcome::Cold
}
};
}
Ok(true) => {}
Err(e) => {
tracing::warn!(error = %e, "ann tail probe failed; Cold");
return SegmentOutcome::Cold;
}
}
let (live, tail) = match scope_counts(rt, ns, model, s).await {
Ok(counts) => counts,
Err(e) => {
tracing::warn!(error = %e, "ann scope-count read failed; Cold");
return SegmentOutcome::Cold;
}
};
if live == 0 {
tracing::info!(namespace = %ns, model = %model,
"zero live corpus for scope; Empty (FTS/degraded path)");
return SegmentOutcome::Empty;
}
let threshold = (ann_rebuild_threshold() * live as f64).ceil() as u64;
if tail <= threshold {
let mut bridge = match AnnBridge::load(seg_dir) {
Ok(b) => b,
Err(e) => {
tracing::warn!(error = %e, dir = %seg_dir.display(),
"Stale-tail segment load failed; Cold rebuild");
return SegmentOutcome::Cold;
}
};
let (finals, new_s) = match fetch_final_states(rt, ns, model, s).await {
Ok(t) => t,
Err(e) => {
tracing::warn!(error = %e, "tail replay contradiction; Cold rebuild");
return SegmentOutcome::Cold;
}
};
if let Err(e) = replay_final_states(rt, &mut bridge, ns, model, &finals).await {
tracing::warn!(error = %e, "tail replay failed; Cold rebuild");
return SegmentOutcome::Cold;
}
bridge.set_applied_seq(new_s);
if !ann.builds_corpus_indexes {
install_if_fresher(ann, key, bridge.with_generation(target_generation)).await;
return SegmentOutcome::Installed;
}
let checkpointed = checkpoint_raise_compact_readopt(
rt,
ann,
key,
bridge,
target_generation,
CheckpointAuthority::Incremental,
)
.await;
if !checkpointed && force_rebuild_required(ann, key) {
return SegmentOutcome::ForceRebuild;
}
return SegmentOutcome::Installed;
}
match AnnBridge::load(seg_dir) {
Ok(bridge) => {
tracing::info!(namespace = %ns, model = %model, tail, live,
"tail above rebuild threshold; serving stale segment during rebuild");
install_if_fresher(ann, key, bridge.with_generation(target_generation)).await;
if !ann.builds_corpus_indexes {
return SegmentOutcome::Installed;
}
}
Err(e) => {
tracing::warn!(error = %e, dir = %seg_dir.display(),
"Stale-rebuild segment load failed; rebuilding without serve-stale");
}
}
SegmentOutcome::Cold
}
pub(crate) async fn ensure_ann_for_model(
rt: &KhiveRuntime,
token: &NamespaceToken,
ann: &SharedAnn,
model: &str,
) -> AnnWarmOutcome {
if model.is_empty() {
return AnnWarmOutcome::Empty;
}
let ns = token.namespace().as_str().to_owned();
let key = AnnKey::new(&ns, model);
let warm_host = ann.builds_corpus_indexes;
let mut force_rebuild = force_rebuild_required(ann, &key);
match read_own_watermark(rt, &ns, model).await {
Ok(Some(watermark)) if watermark < 0 => {
if !warm_host {
tracing::debug!(namespace = %ns, model = %model,
"ANN rebuild is pending and this process is not the warm index host; \
declining");
return AnnWarmOutcome::Declined;
}
mark_force_rebuild(ann, &key);
force_rebuild = true;
}
Ok(Some(_)) => {}
Ok(None) => {
if !warm_host {
tracing::debug!(namespace = %ns, model = %model,
"ANN registry row absent and this process is not the warm index host; \
declining rather than claiming the rebuild");
return AnnWarmOutcome::Declined;
}
if let Err(error) = prepare_authoritative_rebuild(rt, ann, &key).await {
tracing::warn!(error = %error, "failed to fence ANN registry loss");
return AnnWarmOutcome::Failed;
}
force_rebuild = true;
}
Err(error) => {
tracing::warn!(error = %error, "failed to read ANN registration before scan");
return AnnWarmOutcome::Failed;
}
}
if force_rebuild && !matches!(read_own_watermark(rt, &ns, model).await, Ok(Some(-1))) {
if let Err(error) = prepare_authoritative_rebuild(rt, ann, &key).await {
tracing::warn!(error = %error, "failed to establish ANN rebuild sentinel");
return AnnWarmOutcome::Failed;
}
}
let target_generation = current_generation(ann, &ns);
if !force_rebuild {
if let Some(loaded_generation) = ann
.indexes
.read()
.await
.get(&key)
.map(|bridge| bridge.generation)
{
if loaded_generation >= target_generation {
return AnnWarmOutcome::Ready;
}
tracing::debug!(
namespace = %ns,
model = %model,
loaded_generation,
target_generation,
"knowledge ANN fast path skipped: cached entry generation stale; rebuilding"
);
}
}
if !force_rebuild {
if let Some(seg_dir) = ann_segment_dir(rt, &ns, model) {
match classify_and_adopt_segment(rt, ann, &key, &ns, model, &seg_dir, target_generation)
.await
{
SegmentOutcome::Installed => return AnnWarmOutcome::Ready,
SegmentOutcome::Empty => {
mark_unavailable(ann, &key, target_generation);
return AnnWarmOutcome::Empty;
}
SegmentOutcome::Cold => {} SegmentOutcome::ForceRebuild => force_rebuild = true,
}
}
}
if !force_rebuild {
if let Some(snapshot) = try_load_snapshot(rt, &ns, model).await {
let current_fp = compute_fingerprint(rt, token, model).await;
if let Some(fp) = current_fp {
if snapshot.fingerprint == fp {
match AnnBridge::from_vamana_snapshot(snapshot) {
Ok(bridge) => {
install_if_fresher(
ann,
&key,
bridge.with_generation(target_generation),
)
.await;
return AnnWarmOutcome::Ready;
}
Err(e) => {
tracing::warn!(error = %e, "corrupt Vamana v1 snapshot; rebuilding");
}
}
} else {
tracing::info!(
namespace = %ns,
model = %model,
"stale Vamana v1 snapshot (fingerprint mismatch); rebuilding"
);
}
}
}
}
if !warm_host {
tracing::info!(namespace = %ns, model = %model,
"no adoptable ANN segment and this process is not the warm index host; \
serving degraded and leaving the corpus build to the daemon");
return AnnWarmOutcome::Declined;
}
let scan_authority = match prepare_full_corpus_scan(rt, ann, &key).await {
Ok(authority) => authority,
Err(error) => {
tracing::warn!(error = %error, "failed to establish ANN full-scan authority");
return AnnWarmOutcome::Failed;
}
};
match load_and_build_from_vector_store(rt, token, model).await {
Ok(Some(bridge)) => {
let checkpointed = checkpoint_raise_compact_readopt(
rt,
ann,
&key,
bridge,
target_generation,
scan_authority,
)
.await;
if checkpointed {
AnnWarmOutcome::Ready
} else if force_rebuild_required(ann, &key) {
AnnWarmOutcome::Failed
} else if has_current_index(ann, &key).await {
AnnWarmOutcome::Ready
} else {
AnnWarmOutcome::Failed
}
}
Ok(None) => {
mark_unavailable(ann, &key, target_generation);
AnnWarmOutcome::Empty
}
Err(e) => {
tracing::warn!(error = %e, "failed to rebuild Vamana ANN index");
AnnWarmOutcome::Failed
}
}
}
#[cfg(test)]
pub(crate) fn simulate_warming_in_flight(ann: &SharedAnn, key: AnnKey) {
begin_warm(ann, key)
.expect("fresh test ANN state must accept a warm")
.leave_in_flight_for_test();
}
#[cfg(test)]
mod tests {
use super::*;
use khive_runtime::KhiveRuntime;
use khive_storage::types::{SqlStatement, SqlValue};
use serde_json::json;
#[tokio::test(flavor = "current_thread")]
async fn loaded_traversal_leaves_the_lexical_executor_runnable() {
let ann = new_shared_for_role(false);
let key = AnnKey::new("local", "test-model");
let id = Uuid::new_v4();
let pause = Arc::new(TestSearchPause::default());
let mut bridge = AnnBridge::build(vec![1.0, 0.0, 0.0, 0.0], 4, vec![id])
.expect("build one-vector bridge");
bridge.search_pause = Some(Arc::clone(&pause));
assert!(insert_ann_if_absent(&ann, key.clone(), bridge).await);
let (cancel_watchdog, watchdog_cancelled) = std::sync::mpsc::channel();
let watchdog_pause = Arc::clone(&pause);
let watchdog = std::thread::spawn(move || {
if watchdog_cancelled
.recv_timeout(std::time::Duration::from_secs(2))
.is_err()
{
watchdog_pause.release();
}
});
let search = search_loaded_with_seq(&ann, &key, &[1.0, 0.0, 0.0, 0.0], 1);
let lexical_progress = async {
while !pause.started.load(Ordering::SeqCst) {
tokio::task::yield_now().await;
}
let reached_before_release = !*pause.released.lock().expect("search pause");
pause.release();
reached_before_release
};
let (hits, progressed) = tokio::join!(search, lexical_progress);
let _ = cancel_watchdog.send(());
watchdog.join().expect("watchdog thread");
assert!(progressed, "loaded traversal blocked the Tokio executor");
assert_eq!(hits.expect("loaded bridge").0[0].0, id);
}
#[tokio::test(start_paused = true)]
#[serial_test::serial(background_tasks)]
async fn rotation_watcher_exits_on_local_shutdown_without_advancing_time() {
let dir = tempfile::tempdir().expect("tempdir");
let rt = KhiveRuntime::new(khive_runtime::RuntimeConfig {
db_path: Some(dir.path().join("rotation-shutdown.db")),
..khive_runtime::RuntimeConfig::no_embeddings()
})
.expect("writable runtime");
let ann = new_shared();
let shutdown = tokio_util::sync::CancellationToken::new();
let before = khive_runtime::background_task_count();
let started_at = tokio::time::Instant::now();
let watcher = start_rotation_watcher_with_shutdown(&rt, &ann, shutdown.clone())
.expect("file-backed ANN state starts a watcher");
assert!(rotation_watch_started_for_test(&ann));
assert_eq!(khive_runtime::background_task_count(), before + 1);
for _ in 0..10 {
tokio::task::yield_now().await;
}
assert_eq!(khive_runtime::background_task_count(), before + 1);
assert!(
!watcher.is_finished(),
"uncancelled watcher must stay active"
);
shutdown.cancel();
for _ in 0..100 {
if watcher.is_finished() {
break;
}
tokio::task::yield_now().await;
}
if !watcher.is_finished() {
watcher.abort();
let _ = watcher.await;
panic!("local shutdown must stop the watcher while its ANN state is alive");
}
watcher
.await
.expect("rotation watcher must finish successfully after local shutdown");
assert_eq!(khive_runtime::background_task_count(), before);
assert_eq!(Arc::strong_count(&ann), 1);
assert_eq!(tokio::time::Instant::now(), started_at);
}
#[test]
fn fresh_tail_merge_deduplicates_and_applies_deletes() {
let deleted = Uuid::new_v4();
let updated = Uuid::new_v4();
let stable = Uuid::new_v4();
let candidates = vec![(deleted, 0.99), (updated, 0.1), (stable, 0.5)];
let ops = vec![(deleted, None), (updated, Some(vec![1.0, 0.0]))];
let merged = merge_fresh_tail(candidates, &[1.0, 0.0], ops);
assert!(
!merged.iter().any(|(subject, _)| *subject == deleted),
"a final tail delete must remove a stale ANN candidate"
);
assert_eq!(
merged
.iter()
.filter(|(subject, _)| *subject == updated)
.count(),
1,
"a tail upsert must replace, not duplicate, an ANN candidate"
);
assert_eq!(merged[0].0, updated, "the exact tail score must win");
assert!(merged.iter().any(|(subject, _)| *subject == stable));
}
#[test]
fn fresh_tail_merge_breaks_equal_scores_by_uuid() {
let low = Uuid::from_u128(1);
let middle = Uuid::from_u128(2);
let high = Uuid::from_u128(3);
let ops = vec![
(high, Some(vec![1.0, 0.0])),
(low, Some(vec![1.0, 0.0])),
(middle, Some(vec![1.0, 0.0])),
];
let merged = merge_fresh_tail(Vec::new(), &[1.0, 0.0], ops);
assert_eq!(
merged
.into_iter()
.map(|(subject, _)| subject)
.collect::<Vec<_>>(),
vec![low, middle, high],
"equal-cosine fresh hits must not inherit HashMap iteration order"
);
}
#[tokio::test]
async fn fresh_tail_missing_registration_publishes_force_rebuild_sentinel() {
let rt = KhiveRuntime::memory().expect("in-memory runtime");
let ann = new_shared();
let namespace = "local";
let model = "fresh-tail-registration-test";
let key = AnnKey::new(namespace, model);
let outcome = force_cold_after_registry_loss(&rt, &ann, &key).await;
assert!(matches!(
outcome,
FreshTailOutcome::Replace { ref candidates, source_exhausted: true }
if candidates.is_empty()
));
assert_eq!(
read_own_watermark(&rt, namespace, model)
.await
.expect("registry read"),
Some(-1),
"registry loss must publish the cross-process rebuild sentinel"
);
assert!(force_rebuild_required(&ann, &key));
}
#[tokio::test]
async fn read_only_fresh_tail_registry_loss_stays_process_local() {
let dir = tempfile::tempdir().expect("tempdir");
let config = khive_runtime::RuntimeConfig {
db_path: Some(dir.path().join("read-only-fresh-tail.db")),
..khive_runtime::RuntimeConfig::no_embeddings()
};
drop(KhiveRuntime::new(config.clone()).expect("migrate snapshot source"));
#[cfg(unix)]
{
let db_path = config.db_path.as_ref().expect("db path");
khive_storage::test_support::freeze_snapshot_sidecars(db_path);
}
let rt = KhiveRuntime::new_readonly(config).expect("open snapshot read-only");
let ann = new_shared();
let namespace = "local";
let model = "read-only-fresh-tail-model";
let key = AnnKey::new(namespace, model);
let before = rt.backend().pool().writer_acquisition_snapshot();
let outcome = force_cold_after_registry_loss(&rt, &ann, &key).await;
assert!(matches!(
outcome,
FreshTailOutcome::Replace { ref candidates, source_exhausted: true }
if candidates.is_empty()
));
assert_eq!(
read_own_watermark(&rt, namespace, model)
.await
.expect("registry read"),
None,
"read-only registry loss must not publish a rebuild sentinel"
);
assert!(
!force_rebuild_required(&ann, &key),
"read-only registry loss must not claim local rebuild authority"
);
assert_eq!(
rt.backend().pool().writer_acquisition_snapshot(),
before,
"read-only fresh-tail degradation must not acquire a writer"
);
}
#[tokio::test]
async fn force_rebuild_queries_do_not_retire_authoritative_warm() {
let rt = KhiveRuntime::memory().expect("in-memory runtime");
let ann = new_shared();
let key = AnnKey::new("local", "force-warm-test");
force_cold_after_registry_loss(&rt, &ann, &key).await;
simulate_warming_in_flight(&ann, key.clone());
let outcome = fresh_tail_leg(&rt, &ann, &key, &[1.0], 1, None).await;
assert!(matches!(
outcome,
FreshTailOutcome::Replace { ref candidates, source_exhausted: true }
if candidates.is_empty()
));
assert!(
is_warming_not_loaded(&ann, &key),
"a concurrent degraded query must not invalidate the authoritative warm"
);
}
#[test]
fn replay_does_not_tombstone_slot_reused_by_same_batch_upsert() {
let id_a = Uuid::new_v4();
let id_b = Uuid::new_v4();
let id_c = Uuid::new_v4();
let vectors = vec![
1.0f32, 0.0, 0.0, 0.0, 1.0, 0.0, ];
let mut bridge = AnnBridge::build(vectors, 3, vec![id_a, id_b]).expect("build");
bridge.index.tombstone(0).expect("tombstone id_a");
assert_eq!(
bridge.id_map[0], id_a,
"id-map entry stays stale after tombstone"
);
let mut reverse = bridge.reverse_map();
bridge
.apply_final_op(&mut reverse, id_c, Some(vec![0.0f32, 0.0, 1.0]))
.expect("apply upsert");
bridge
.apply_final_op(&mut reverse, id_a, None)
.expect("apply delete");
assert_eq!(
bridge.id_map[0], id_c,
"ordinal 0 must be owned by id_c after the replay"
);
assert!(
!bridge.index.is_tombstoned(0),
"id_a's stale delete must not tombstone the slot id_c now owns"
);
let hits = bridge.search(&[0.0, 0.0, 1.0], 2);
assert!(
hits.iter().any(|(id, score)| *id == id_c && *score > 0.9),
"id_c must remain live and searchable, got: {hits:?}"
);
assert!(
!hits.iter().any(|(id, _)| *id == id_a),
"id_a must not resurface as a search hit, got: {hits:?}"
);
}
#[tokio::test]
async fn test_invalidate_snapshot_removes_vamana_rows() {
let rt = KhiveRuntime::memory().expect("in-memory runtime");
let sql = rt.sql();
let mut w = sql.writer().await.expect("writer");
w.execute_script(
"CREATE TABLE IF NOT EXISTS retrieval_snapshots (\
namespace TEXT NOT NULL, index_type TEXT NOT NULL, \
snapshot BLOB NOT NULL, created_at INTEGER NOT NULL, \
PRIMARY KEY (namespace, index_type));"
.into(),
)
.await
.expect("create table");
for (ns, idx_type) in &[
("local::vamana::model-a", "vamana"),
("local::vamana::model-b", "vamana"),
("local::hnsw::model-a", "hnsw"),
] {
w.execute(SqlStatement {
sql: "INSERT INTO retrieval_snapshots (namespace, index_type, snapshot, created_at) VALUES (?1, ?2, ?3, 0)".into(),
params: vec![
SqlValue::Text(ns.to_string()),
SqlValue::Text(idx_type.to_string()),
SqlValue::Blob(b"{}".to_vec()),
],
label: None,
})
.await
.expect("insert");
}
drop(w);
invalidate_snapshot(&rt, "local").await;
let mut r = sql.reader().await.expect("reader");
let rows = r
.query_all(SqlStatement {
sql: "SELECT namespace FROM retrieval_snapshots ORDER BY namespace".into(),
params: vec![],
label: None,
})
.await
.expect("query");
let remaining: Vec<String> = rows
.iter()
.filter_map(|row| match row.get("namespace") {
Some(SqlValue::Text(s)) => Some(s.clone()),
_ => None,
})
.collect();
assert!(
remaining.contains(&"local::hnsw::model-a".to_string()),
"HNSW rows must survive: {remaining:?}"
);
assert!(
!remaining.contains(&"local::vamana::model-a".to_string()),
"vamana model-a must be deleted: {remaining:?}"
);
assert!(
!remaining.contains(&"local::vamana::model-b".to_string()),
"vamana model-b must be deleted: {remaining:?}"
);
}
#[tokio::test]
async fn test_invalidate_snapshot_does_not_cross_underscore_namespace() {
let rt = KhiveRuntime::memory().expect("in-memory runtime");
let sql = rt.sql();
let mut w = sql.writer().await.expect("writer");
w.execute_script(
"CREATE TABLE IF NOT EXISTS retrieval_snapshots (\
namespace TEXT NOT NULL, index_type TEXT NOT NULL, \
snapshot BLOB NOT NULL, created_at INTEGER NOT NULL, \
PRIMARY KEY (namespace, index_type));"
.into(),
)
.await
.expect("create table");
for ns in &["a_b::vamana::model-a", "aXb::vamana::model-a"] {
w.execute(SqlStatement {
sql: "INSERT INTO retrieval_snapshots (namespace, index_type, snapshot, created_at) VALUES (?1, ?2, ?3, 0)".into(),
params: vec![
SqlValue::Text(ns.to_string()),
SqlValue::Text("vamana".to_string()),
SqlValue::Blob(b"{}".to_vec()),
],
label: None,
})
.await
.expect("insert");
}
drop(w);
invalidate_snapshot(&rt, "a_b").await;
let mut r = sql.reader().await.expect("reader");
let rows = r
.query_all(SqlStatement {
sql: "SELECT namespace FROM retrieval_snapshots ORDER BY namespace".into(),
params: vec![],
label: None,
})
.await
.expect("query");
let remaining: Vec<String> = rows
.iter()
.filter_map(|row| match row.get("namespace") {
Some(SqlValue::Text(s)) => Some(s.clone()),
_ => None,
})
.collect();
assert!(
remaining.contains(&"aXb::vamana::model-a".to_string()),
"unrelated namespace 'aXb' must survive invalidating 'a_b': {remaining:?}"
);
assert!(
!remaining.contains(&"a_b::vamana::model-a".to_string()),
"'a_b' own snapshot must still be deleted: {remaining:?}"
);
}
#[tokio::test]
async fn test_invalidate_snapshot_tolerates_missing_table() {
let rt = KhiveRuntime::memory().expect("in-memory runtime");
invalidate_snapshot(&rt, "local").await;
}
#[tokio::test]
async fn test_invalidate_clears_in_memory_ann() {
let ann = new_shared();
let dim = 4;
let vectors = vec![1.0f32, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0];
let ids = vec![Uuid::new_v4(), Uuid::new_v4()];
let bridge = AnnBridge::build(vectors, dim, ids).expect("build");
let key = AnnKey::new("local", "test-model");
assert!(
insert_ann_if_absent(&ann, key.clone(), bridge).await,
"insert must succeed on empty cache"
);
assert!(
ann.indexes.read().await.contains_key(&key),
"pre-condition: ANN loaded"
);
clear_namespace(&ann, "local").await;
assert!(
!ann.indexes.read().await.contains_key(&key),
"clearing SharedAnn must remove the bridge"
);
}
#[tokio::test]
async fn shared_ann_is_keyed_by_namespace_and_model() {
let ann = new_shared();
let model = "test-model";
let id_a = Uuid::new_v4();
let id_b = Uuid::new_v4();
let bridge_a = AnnBridge::build(vec![1.0, 0.0, 0.0, 0.0], 4, vec![id_a])
.expect("build namespace A bridge");
let bridge_b = AnnBridge::build(vec![0.0, 1.0, 0.0, 0.0], 4, vec![id_b])
.expect("build namespace B bridge");
assert!(insert_ann_if_absent(&ann, AnnKey::new("ns:a", model), bridge_a).await);
assert!(insert_ann_if_absent(&ann, AnnKey::new("ns:b", model), bridge_b).await);
let hits_b = search_loaded(&ann, &AnnKey::new("ns:b", model), &[1.0, 0.0, 0.0, 0.0], 1)
.await
.expect("namespace B bridge exists");
assert_eq!(hits_b.len(), 1);
assert_eq!(
hits_b[0].0, id_b,
"namespace B query must not return namespace A neighbour"
);
}
#[tokio::test]
async fn install_if_fresher_rejects_late_stale_build() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
let fresh = AnnBridge::build(vec![1.0, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build fresh bridge")
.with_generation(2);
let stale = AnnBridge::build(vec![0.0, 1.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build stale bridge")
.with_generation(1);
install_if_fresher(&ann, &key, fresh).await;
install_if_fresher(&ann, &key, stale).await;
let installed_generation = ann
.indexes
.read()
.await
.get(&key)
.expect("entry present")
.generation;
assert_eq!(
installed_generation, 2,
"stale build (generation 1) must not replace fresher installed entry (generation 2)"
);
}
#[tokio::test]
async fn install_if_fresher_accepts_forward_progress() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
let old = AnnBridge::build(vec![1.0, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build old bridge")
.with_generation(1);
let newer = AnnBridge::build(vec![0.0, 1.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build newer bridge")
.with_generation(2);
install_if_fresher(&ann, &key, old).await;
install_if_fresher(&ann, &key, newer).await;
let installed_generation = ann
.indexes
.read()
.await
.get(&key)
.expect("entry present")
.generation;
assert_eq!(
installed_generation, 2,
"newer build must replace an older installed entry"
);
}
#[tokio::test]
async fn install_if_fresher_ties_keep_incumbent() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
let first = AnnBridge::build(vec![1.0, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build first bridge")
.with_generation(1);
let second_id = Uuid::new_v4();
let second = AnnBridge::build(vec![0.0, 1.0, 0.0, 0.0], 4, vec![second_id])
.expect("build second bridge")
.with_generation(1);
install_if_fresher(&ann, &key, first).await;
install_if_fresher(&ann, &key, second).await;
let hits = search_loaded(&ann, &key, &[0.0, 1.0, 0.0, 0.0], 1)
.await
.expect("entry present");
assert_ne!(
hits.first().map(|(id, _)| *id),
Some(second_id),
"equal-generation candidate must not replace the incumbent"
);
}
#[tokio::test]
async fn install_if_fresher_installs_into_empty_slot() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
let bridge = AnnBridge::build(vec![1.0, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build bridge")
.with_generation(0);
install_if_fresher(&ann, &key, bridge).await;
assert!(
ann.indexes.read().await.contains_key(&key),
"first successful build must always install into an empty slot"
);
}
#[tokio::test]
async fn clear_namespace_bumps_generation_scoped_to_namespace() {
let ann = new_shared();
assert_eq!(current_generation(&ann, "ns:a"), 0);
assert_eq!(current_generation(&ann, "ns:b"), 0);
clear_namespace(&ann, "ns:a").await;
assert_eq!(
current_generation(&ann, "ns:a"),
1,
"clear_namespace must bump the invalidated namespace's generation"
);
assert_eq!(
current_generation(&ann, "ns:b"),
0,
"clear_namespace must not affect a different namespace's generation"
);
}
#[tokio::test]
async fn stale_build_installs_before_invalidation_race_is_rejected_after() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
let build_a_generation = current_generation(&ann, "local");
assert_eq!(build_a_generation, 0);
clear_namespace(&ann, "local").await;
assert_eq!(current_generation(&ann, "local"), 1);
let build_b_generation = current_generation(&ann, "local");
let build_b_id = Uuid::new_v4();
let build_b_bridge = AnnBridge::build(vec![0.0, 1.0, 0.0, 0.0], 4, vec![build_b_id])
.expect("build fresh bridge")
.with_generation(build_b_generation);
install_if_fresher(&ann, &key, build_b_bridge).await;
assert!(
ann.indexes.read().await.contains_key(&key),
"build B (post-invalidation generation) must install"
);
let build_a_bridge = AnnBridge::build(vec![1.0, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build stale bridge")
.with_generation(build_a_generation);
install_if_fresher(&ann, &key, build_a_bridge).await;
let hits = search_loaded(&ann, &key, &[0.0, 1.0, 0.0, 0.0], 1)
.await
.expect("entry present");
assert_eq!(
hits.first().map(|(id, _)| *id),
Some(build_b_id),
"build A's late, stale install must not clobber build B's fresher result"
);
}
#[tokio::test]
async fn stale_build_rejected_installing_into_still_empty_post_invalidation_slot() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
let build_a_generation = current_generation(&ann, "local");
assert_eq!(build_a_generation, 0);
clear_namespace(&ann, "local").await;
assert_eq!(current_generation(&ann, "local"), 1);
assert!(
!ann.indexes.read().await.contains_key(&key),
"precondition: slot must still be empty after clear_namespace"
);
let build_a_bridge = AnnBridge::build(vec![1.0, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build stale bridge")
.with_generation(build_a_generation);
install_if_fresher(&ann, &key, build_a_bridge).await;
assert!(
!ann.indexes.read().await.contains_key(&key),
"stale pre-invalidation build must not install into the emptied slot, \
even with no incumbent to compare against"
);
assert!(
search_loaded(&ann, &key, &[1.0, 0.0, 0.0, 0.0], 1)
.await
.is_none(),
"the fast path must not serve a stale index that was correctly rejected at install"
);
}
#[test]
fn is_warming_false_when_neither_warming_nor_loaded() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
assert!(
!is_warming_not_loaded(&ann, &key),
"key absent from both sets must return false"
);
}
#[test]
fn is_warming_true_when_in_warming_but_not_indexes() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
simulate_warming_in_flight(&ann, key.clone());
assert!(
is_warming_not_loaded(&ann, &key),
"key in warming but not indexes must return true"
);
}
#[tokio::test]
async fn is_warming_false_when_both_warming_and_loaded() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
simulate_warming_in_flight(&ann, key.clone());
let bridge =
AnnBridge::build(vec![1.0f32, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()]).expect("build");
insert_ann_if_absent(&ann, key.clone(), bridge).await;
assert!(
!is_warming_not_loaded(&ann, &key),
"key in both warming and indexes must return false (warm is done)"
);
}
#[tokio::test]
async fn wait_ready_returns_true_immediately_when_already_loaded() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
let bridge =
AnnBridge::build(vec![1.0f32, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()]).expect("build");
insert_ann_if_absent(&ann, key.clone(), bridge).await;
let ready = wait_ready(&ann, &key, 100, 10).await;
assert!(ready, "must return true when index is already in the map");
}
#[tokio::test]
async fn wait_ready_returns_false_on_timeout_when_never_loaded() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
let ready = wait_ready(&ann, &key, 60, 10).await;
assert!(
!ready,
"must return false when index never appears within timeout"
);
}
#[tokio::test]
async fn wait_ready_returns_true_when_index_appears_mid_poll() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
let ann2 = ann.clone();
let key2 = key.clone();
tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(40)).await;
let bridge = AnnBridge::build(vec![1.0f32, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build");
insert_ann_if_absent(&ann2, key2, bridge).await;
});
let ready = wait_ready(&ann, &key, 500, 10).await;
assert!(ready, "must return true when index appears before timeout");
}
fn assert_terminal_wait_latency(elapsed: std::time::Duration) {
let bound_ms = if std::env::var_os("LLVM_PROFILE_FILE").is_some() {
ANN_WARM_WAIT_TIMEOUT_MS / 2
} else {
ANN_WARM_WAIT_POLL_MS * 10
};
assert!(
elapsed < std::time::Duration::from_millis(bound_ms),
"terminal unavailable outcome must short-circuit within {bound_ms}ms: {elapsed:?}"
);
}
#[tokio::test]
async fn wait_ready_returns_false_immediately_when_marked_unavailable() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
mark_unavailable(&ann, &key, current_generation(&ann, "local"));
let start = std::time::Instant::now();
let ready = wait_ready(&ann, &key, ANN_WARM_WAIT_TIMEOUT_MS, ANN_WARM_WAIT_POLL_MS).await;
let elapsed = start.elapsed();
assert!(
!ready,
"must return false for a key marked unavailable at the current generation"
);
assert_terminal_wait_latency(elapsed);
}
#[tokio::test]
async fn wait_ready_resumes_polling_when_unavailable_marker_is_stale() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
mark_unavailable(&ann, &key, 0);
clear_namespace(&ann, "local").await;
assert_eq!(current_generation(&ann, "local"), 1);
let ann2 = ann.clone();
let key2 = key.clone();
tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(40)).await;
let bridge = AnnBridge::build(vec![1.0f32, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build")
.with_generation(1);
install_if_fresher(&ann2, &key2, bridge).await;
});
let ready = wait_ready(&ann, &key, 500, 10).await;
assert!(
ready,
"a stale unavailable marker must not block polling; the index installed \
mid-poll must still be observed"
);
}
#[tokio::test]
async fn install_if_fresher_clears_unavailable_marker_on_successful_install() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
mark_unavailable(&ann, &key, 0);
let bridge = AnnBridge::build(vec![1.0f32, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build")
.with_generation(0);
install_if_fresher(&ann, &key, bridge).await;
assert!(
!unavailable_guard(&ann.unavailable).contains_key(&key),
"a successful install must clear the unavailable marker for its key"
);
}
#[tokio::test]
async fn install_if_fresher_stale_reject_does_not_clear_unavailable_marker() {
let ann = new_shared();
let key = AnnKey::new("local", "test-model");
clear_namespace(&ann, "local").await;
mark_unavailable(&ann, &key, current_generation(&ann, "local"));
let stale = AnnBridge::build(vec![1.0f32, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build")
.with_generation(0);
install_if_fresher(&ann, &key, stale).await;
assert!(
!ann.indexes.read().await.contains_key(&key),
"stale candidate must not install"
);
assert!(
unavailable_guard(&ann.unavailable).contains_key(&key),
"a rejected (non-installed) candidate must not clear the unavailable marker"
);
}
#[test]
fn warm_states_guard_recovers_from_poison() {
let ann = new_shared();
let key = AnnKey::new("poison-ns", "poison-model");
let ann2 = ann.clone();
let join_result = std::thread::spawn(move || {
let _guard = ann2.warm_states.lock().expect("pre-poison lock");
panic!("deliberate poison");
})
.join();
assert!(join_result.is_err(), "poison thread must have panicked");
assert!(
ann.warm_states.is_poisoned(),
"mutex must be poisoned before recovery"
);
let guard = warm_states_guard(&ann.warm_states);
assert!(
!guard.contains_key(&key),
"recovered guard must report key absent"
);
drop(guard);
assert!(
!is_warming_not_loaded(&ann, &key),
"is_warming_not_loaded must not panic on poisoned Mutex"
);
}
#[tokio::test]
async fn warm_state_success_becomes_ready_and_suppresses_duplicates() {
let ann = new_shared();
let key = AnnKey::new("local", "warm-unify-model");
let permit = begin_warm(&ann, key.clone()).expect("Absent -> Warming");
assert!(
is_warming_not_loaded(&ann, &key),
"owned Warming state without an index must report in flight"
);
assert!(
begin_warm(&ann, key.clone()).is_none(),
"a current Warming owner must singleflight duplicate callers"
);
let bridge = AnnBridge::build(vec![1.0f32, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build bridge for warm-path test");
insert_ann_if_absent(&ann, key.clone(), bridge).await;
finish_warm(permit, AnnWarmOutcome::Ready).await;
assert!(
!is_warming_not_loaded(&ann, &key),
"Ready state must not report warming in flight"
);
assert!(
matches!(
warm_states_guard(&ann.warm_states).get(&key),
Some(AnnWarmState::Ready { generation: 0 })
),
"a published index must finish in Ready"
);
assert!(
begin_warm(&ann, key).is_none(),
"Ready at the current generation must suppress redundant loads"
);
}
#[tokio::test]
async fn warm_state_failed_and_empty_outcomes_remain_retryable() {
let ann = new_shared();
let key = AnnKey::new("local", "warm-unify-fail-model");
let failed = begin_warm(&ann, key.clone()).expect("first warm");
finish_warm(failed, AnnWarmOutcome::Failed).await;
assert!(
matches!(
warm_states_guard(&ann.warm_states).get(&key),
Some(AnnWarmState::Failed {
error: AnnWarmFailure::Operational,
..
})
),
"no index and no empty marker is an operational failure"
);
let empty_retry = begin_warm(&ann, key.clone()).expect("Failed -> Warming retry");
mark_unavailable(&ann, &key, current_generation(&ann, "local"));
finish_warm(empty_retry, AnnWarmOutcome::Empty).await;
assert!(
matches!(
warm_states_guard(&ann.warm_states).get(&key),
Some(AnnWarmState::Failed {
error: AnnWarmFailure::EmptyCorpus,
..
})
),
"current-generation empty scan must retain its distinct failure reason"
);
assert!(
!is_warming_not_loaded(&ann, &key),
"Failed must not masquerade as an in-flight warm"
);
let retry = begin_warm(&ann, key).expect("empty failures must remain retryable");
drop(retry);
}
#[tokio::test]
async fn failed_replacement_stays_retryable_with_servable_stale_index() {
let ann = new_shared();
let key = AnnKey::new("local", "warm-stale-retry-model");
let permit = begin_warm(&ann, key.clone()).expect("replacement warm");
let stale = AnnBridge::build(vec![1.0f32, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build stale bridge");
insert_ann_if_absent(&ann, key.clone(), stale).await;
finish_warm(permit, AnnWarmOutcome::Failed).await;
assert!(
search_loaded(&ann, &key, &[1.0, 0.0, 0.0, 0.0], 1)
.await
.is_some(),
"the stale fallback must remain available to search"
);
assert!(
begin_warm(&ann, key).is_some(),
"a failed replacement must retry despite the servable stale fallback"
);
}
#[tokio::test]
async fn stale_finish_cannot_steal_new_post_invalidation_owner() {
let ann = new_shared();
let key = AnnKey::new("local", "warm-owner-model");
let stale = begin_warm(&ann, key.clone()).expect("warm A");
clear_namespace(&ann, "local").await;
let current = begin_warm(&ann, key.clone()).expect("warm B after invalidation");
let current_id = current.attempt_id;
finish_warm(stale, AnnWarmOutcome::Failed).await;
assert!(
matches!(
warm_states_guard(&ann.warm_states).get(&key),
Some(AnnWarmState::Warming { attempt_id, .. }) if *attempt_id == current_id
),
"warm A's late cleanup must not erase or complete warm B's ownership"
);
assert!(
begin_warm(&ann, key.clone()).is_none(),
"warm B must remain the only current singleflight owner"
);
finish_warm(current, AnnWarmOutcome::Failed).await;
assert!(
begin_warm(&ann, key).is_some(),
"warm B's failed completion must make the slot retryable"
);
}
#[test]
fn dropped_warm_permit_cannot_leave_stale_warming_ownership() {
let ann = new_shared();
let key = AnnKey::new("local", "warm-cancel-model");
let permit = begin_warm(&ann, key.clone()).expect("warm attempt");
drop(permit);
assert!(
matches!(
warm_states_guard(&ann.warm_states).get(&key),
Some(AnnWarmState::Failed {
error: AnnWarmFailure::Interrupted,
..
})
),
"cancellation must transition the owned slot out of Warming"
);
assert!(
begin_warm(&ann, key).is_some(),
"an interrupted warm must be retryable"
);
}
fn build_test_bridge(dim: usize, n: usize) -> (AnnBridge, Vec<Uuid>) {
let ids: Vec<Uuid> = (0..n).map(|_| Uuid::new_v4()).collect();
let mut vectors: Vec<f32> = Vec::with_capacity(n * dim);
for i in 0..n {
for d in 0..dim {
vectors.push(if d == i % dim { 1.0 } else { 0.0 });
}
}
let bridge = AnnBridge::build(vectors, dim, ids.clone()).expect("build test bridge");
(bridge, ids)
}
#[tokio::test]
async fn rotation_tick_releases_predecessor_and_adopts_identical_peer_checkpoint() {
let temp = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(temp.path().join("rotation.db"));
let ann = new_shared();
let key = AnnKey::new("local", WARM_TEST_MODEL);
let segment_dir =
ann_segment_dir(&rt, "local", WARM_TEST_MODEL).expect("file-backed segment directory");
let (first, first_ids) = build_test_bridge(4, 2);
first
.save_atomic(&segment_dir)
.expect("persist first generation");
let mut loaded = AnnBridge::load(&segment_dir)
.expect("load first mmap generation")
.with_generation(7);
let probe = Arc::new(());
let dropped = Arc::downgrade(&probe);
loaded.drop_probe = Some(probe);
assert!(install_replacing(&ann, &key, loaded).await);
let first_digest = ann
.indexes
.read()
.await
.get(&key)
.and_then(|bridge| bridge.commit_digest)
.expect("loaded bridge identity");
let (second, second_ids) = build_test_bridge(4, 2);
assert_ne!(first_ids, second_ids, "fixture sidecars must differ");
second
.save_atomic(&segment_dir)
.expect("rotate identical checkpoint");
refresh_rotated_segments_once(&rt, &ann).await;
assert!(
dropped.upgrade().is_none(),
"replacing the cache entry must drop the predecessor mmap owner"
);
let installed = ann.indexes.read().await;
let bridge = installed.get(&key).expect("rotated bridge installed");
assert_ne!(bridge.commit_digest, Some(first_digest));
assert_eq!(
bridge.search(&[1.0, 0.0, 0.0, 0.0], 1)[0].0,
second_ids[0],
"the rotated UUID sidecar must be the served mapping"
);
assert_eq!(bridge.generation, 7, "local generation fence is preserved");
}
async fn install_generation_then_publish_invalid_rotation(
temp: &TempDir,
) -> (KhiveRuntime, SharedAnn, AnnKey) {
let rt = file_rt_with_embedder(temp.path().join("rotation.db"));
let ann = new_shared();
let key = AnnKey::new("local", WARM_TEST_MODEL);
let segment_dir =
ann_segment_dir(&rt, "local", WARM_TEST_MODEL).expect("file-backed segment directory");
let (first, _) = build_test_bridge(4, 2);
first
.save_atomic(&segment_dir)
.expect("persist first generation");
let loaded = AnnBridge::load(&segment_dir)
.expect("load first mmap generation")
.with_generation(7);
assert!(install_replacing(&ann, &key, loaded).await);
let (second, _) = build_test_bridge(4, 2);
second.save_atomic(&segment_dir).expect("rotate checkpoint");
std::fs::remove_file(segment_dir.join("external_ids.bin")).expect("remove sidecar");
(rt, ann, key)
}
#[tokio::test]
async fn invalid_rotation_keeps_in_flight_warming_state() {
let temp = TempDir::new().expect("tempdir");
let (rt, ann, key) = install_generation_then_publish_invalid_rotation(&temp).await;
warm_states_guard(&ann.warm_states).insert(
key.clone(),
AnnWarmState::Warming {
attempt_id: 41,
generation: 7,
started_at: std::time::Instant::now(),
},
);
refresh_rotated_segments_once(&rt, &ann).await;
assert!(
ann.indexes.read().await.get(&key).is_none(),
"an invalid rotated generation must evict the incumbent"
);
assert!(
matches!(
warm_states_guard(&ann.warm_states).get(&key),
Some(AnnWarmState::Warming { attempt_id: 41, .. })
),
"eviction must not clear a warm state owned by an in-flight permit"
);
}
#[tokio::test]
async fn invalid_rotation_clears_ready_state_of_evicted_bridge() {
let temp = TempDir::new().expect("tempdir");
let (rt, ann, key) = install_generation_then_publish_invalid_rotation(&temp).await;
warm_states_guard(&ann.warm_states)
.insert(key.clone(), AnnWarmState::Ready { generation: 7 });
refresh_rotated_segments_once(&rt, &ann).await;
assert!(
ann.indexes.read().await.get(&key).is_none(),
"an invalid rotated generation must evict the incumbent"
);
assert!(
warm_states_guard(&ann.warm_states).get(&key).is_none(),
"the Ready state described the evicted bridge and must go with it"
);
}
#[tokio::test]
async fn invalid_rotation_does_not_clear_a_newer_builds_ready_state() {
let temp = TempDir::new().expect("tempdir");
let (rt, ann, key) = install_generation_then_publish_invalid_rotation(&temp).await;
warm_states_guard(&ann.warm_states)
.insert(key.clone(), AnnWarmState::Ready { generation: 8 });
refresh_rotated_segments_once(&rt, &ann).await;
assert!(
ann.indexes.read().await.get(&key).is_none(),
"an invalid rotated generation must still evict the incumbent"
);
assert!(
matches!(
warm_states_guard(&ann.warm_states).get(&key),
Some(AnnWarmState::Ready { generation: 8 })
),
"eviction must not clear a later build's Ready state sharing the key"
);
}
#[tokio::test]
async fn finish_warm_ready_publish_is_atomic_with_concurrent_eviction() {
let temp = TempDir::new().expect("tempdir");
let (_rt, ann, key) = install_generation_then_publish_invalid_rotation(&temp).await;
let incumbent_digest = ann
.indexes
.read()
.await
.get(&key)
.and_then(|bridge| bridge.commit_digest)
.expect("incumbent bridge installed with a commit identity");
let permit = begin_warm(&ann, key.clone()).expect("a fresh key grants a warm permit");
let (handle_tx, mut handle_rx) = tokio::sync::mpsc::unbounded_channel();
*ann.test_finish_warm_ready_hook.lock().unwrap() =
Some(Arc::new(FinishWarmReadyTestHook {
incumbent_digest,
generation: 7,
handle_tx,
}));
finish_warm(permit, AnnWarmOutcome::Ready).await;
let evict_handle = handle_rx
.recv()
.await
.expect("the ready hook spawns the eviction task");
evict_handle.await.expect("eviction task completes");
let is_ready = matches!(
warm_states_guard(&ann.warm_states).get(&key),
Some(AnnWarmState::Ready { .. })
);
if is_ready {
assert!(
ann.indexes.read().await.get(&key).is_some(),
"a Ready warm state must not describe a bridge the rotation watcher evicted"
);
}
}
#[test]
fn ann_bridge_save_atomic_load_round_trip() {
let dir = tempfile::TempDir::new().expect("tempdir");
let dim = 4;
let (bridge, ids) = build_test_bridge(dim, 4);
let first_id = ids[0];
bridge.save_atomic(dir.path()).expect("save_atomic");
let loaded = AnnBridge::load(dir.path()).expect("load");
assert_eq!(
loaded.num_vectors(),
bridge.num_vectors(),
"loaded vector count must match saved"
);
let query = vec![1.0f32, 0.0, 0.0, 0.0];
let hits = loaded.search(&query, 1);
assert_eq!(hits.len(), 1, "must return 1 hit");
assert_eq!(hits[0].0, first_id, "top hit must be the first UUID");
}
#[test]
fn ann_bridge_load_missing_sidecar_err() {
let dir = tempfile::TempDir::new().expect("tempdir");
let (bridge, _) = build_test_bridge(4, 2);
bridge.save_atomic(dir.path()).expect("save_atomic");
std::fs::remove_file(dir.path().join("external_ids.bin")).expect("remove sidecar");
let result = AnnBridge::load(dir.path());
assert!(
result.is_err(),
"load must fail when external_ids.bin is missing"
);
}
#[test]
fn ann_bridge_load_torn_pair_err() {
let dir = tempfile::TempDir::new().expect("tempdir");
let dim = 4;
let (bridge_a, _) = build_test_bridge(dim, 2);
bridge_a.save_atomic(dir.path()).expect("save_atomic A");
let (bridge_b, _) = build_test_bridge(dim, 3);
bridge_b
.index
.save_atomic(dir.path())
.expect("save_atomic B segments");
let result = AnnBridge::load(dir.path());
assert!(
result.is_err(),
"load must fail when sidecar commit digest != on-disk commit record (torn pair)"
);
let err = result.err().expect("already asserted is_err");
assert!(
err.contains("commit-digest mismatch") || err.contains("torn"),
"error message must mention digest mismatch or torn pair, got: {err}"
);
}
#[test]
fn ann_bridge_load_count_mismatch_err() {
let dir = tempfile::TempDir::new().expect("tempdir");
let (bridge, _) = build_test_bridge(4, 2);
bridge.save_atomic(dir.path()).expect("save_atomic");
let digest = segment_commit_digest(dir.path())
.expect("digest ok")
.expect("commit record present");
let wrong_ids: Vec<uuid::Uuid> = (0..99).map(|_| uuid::Uuid::new_v4()).collect();
write_external_ids_sidecar(dir.path(), &digest, &wrong_ids).expect("write patched sidecar");
let result = AnnBridge::load(dir.path());
assert!(
result.is_err(),
"load must fail when sidecar count != index.num_vectors()"
);
}
#[test]
fn ann_bridge_load_bad_magic_err() {
let dir = tempfile::TempDir::new().expect("tempdir");
let (bridge, _) = build_test_bridge(4, 2);
bridge.save_atomic(dir.path()).expect("save_atomic");
let mut sidecar_bytes =
std::fs::read(dir.path().join("external_ids.bin")).expect("read sidecar");
sidecar_bytes[0..8].copy_from_slice(b"WRONGMAG");
std::fs::write(dir.path().join("external_ids.bin"), &sidecar_bytes)
.expect("write bad-magic sidecar");
let result = AnnBridge::load(dir.path());
assert!(
result.is_err(),
"load must fail when external_ids.bin has wrong magic"
);
let err = result.err().expect("already asserted is_err");
assert!(
err.contains("magic"),
"error must mention magic mismatch, got: {err}"
);
}
use async_trait::async_trait;
use khive_runtime::{AllowAllGate, BackendId, EmbedderProvider, RuntimeConfig};
use lattice_embed::{EmbedError, EmbeddingModel, EmbeddingService};
use tempfile::TempDir;
const WARM_TEST_MODEL: &str = "all-minilm-l6-v2";
const WARM_DIMS: usize = 384;
struct ConstVecService;
#[async_trait]
impl EmbeddingService for ConstVecService {
async fn embed(
&self,
texts: &[String],
_model: EmbeddingModel,
) -> std::result::Result<Vec<Vec<f32>>, EmbedError> {
Ok(texts.iter().map(|_| vec![1.0_f32; WARM_DIMS]).collect())
}
fn supports_model(&self, _: EmbeddingModel) -> bool {
true
}
fn name(&self) -> &'static str {
"const-vec"
}
}
struct TestEmbedderProvider;
#[async_trait]
impl EmbedderProvider for TestEmbedderProvider {
fn name(&self) -> &str {
WARM_TEST_MODEL
}
fn dimensions(&self) -> usize {
WARM_DIMS
}
async fn build(&self) -> khive_runtime::RuntimeResult<Arc<dyn EmbeddingService>> {
Ok(Arc::new(ConstVecService))
}
}
fn rt_with_embedder(db_path: Option<std::path::PathBuf>) -> KhiveRuntime {
let rt = KhiveRuntime::new(RuntimeConfig {
web: Default::default(),
telemetry: Default::default(),
mounts: Vec::new(),
brain: Default::default(),
git_write: Default::default(),
display_timezone: khive_runtime::config::resolve_default_display_timezone(),
events_split: None,
db_path,
blob_hydration_bytes: khive_runtime::DEFAULT_BLOB_HYDRATION_BYTES,
default_namespace: Namespace::local(),
embedding_model: Some(EmbeddingModel::AllMiniLmL6V2),
additional_embedding_models: vec![],
gate: Arc::new(AllowAllGate),
packs: vec!["kg".to_string(), "knowledge".to_string()],
backend_id: BackendId::main(),
brain_profile: None,
visible_namespaces: vec![],
allowed_outbound_namespaces: vec![],
actor_id: None,
exec: Default::default(),
})
.expect("test runtime");
rt.register_embedder(TestEmbedderProvider);
rt
}
fn file_rt_with_embedder(db_path: std::path::PathBuf) -> KhiveRuntime {
rt_with_embedder(Some(db_path))
}
fn memory_rt_with_embedder() -> KhiveRuntime {
rt_with_embedder(None)
}
async fn seed_warm_corpus(rt: &KhiveRuntime, token: &NamespaceToken, n: usize) {
seed_warm_corpus_opts(rt, token, n, true).await;
}
async fn seed_warm_corpus_opts(rt: &KhiveRuntime, token: &NamespaceToken, n: usize, log: bool) {
let _store = rt
.vectors_for_model(token, WARM_TEST_MODEL)
.expect("vec store");
let model_key = sanitize_model_key(WARM_TEST_MODEL);
let table = format!("vec_{model_key}");
let ns = token.namespace().as_str().to_owned();
let sql = rt.sql();
let mut w = sql.writer().await.expect("writer");
for i in 0..n {
let id = Uuid::new_v4();
let mut v = [0.0_f32; WARM_DIMS];
v[i % WARM_DIMS] = 1.0;
let bytes: Vec<u8> = v.iter().flat_map(|f| f.to_le_bytes()).collect();
w.execute(SqlStatement {
sql: format!(
"INSERT INTO {table} \
(subject_id, namespace, kind, field, embedding_model, embedding) \
VALUES (?1, ?2, ?3, ?4, ?5, ?6)"
),
params: vec![
SqlValue::Text(id.to_string()),
SqlValue::Text(ns.clone()),
SqlValue::Text("concept".to_string()),
SqlValue::Text("knowledge.atom".to_string()),
SqlValue::Text(WARM_TEST_MODEL.to_string()),
SqlValue::Blob(bytes),
],
label: None,
})
.await
.expect("insert corpus row");
if !log {
continue;
}
w.execute(SqlStatement {
sql: "INSERT INTO ann_write_log \
(namespace, embedding_model, kind, field, subject_id, op) \
VALUES (?1, ?2, ?3, ?4, ?5, 'upsert')"
.into(),
params: vec![
SqlValue::Text(ns.clone()),
SqlValue::Text(WARM_TEST_MODEL.to_string()),
SqlValue::Text("concept".to_string()),
SqlValue::Text("knowledge.atom".to_string()),
SqlValue::Text(id.to_string()),
],
label: None,
})
.await
.expect("append write-log row");
}
}
async fn append_warm_vector(
rt: &KhiveRuntime,
token: &NamespaceToken,
embedding: [f32; WARM_DIMS],
) -> Uuid {
let _store = rt
.vectors_for_model(token, WARM_TEST_MODEL)
.expect("vec store");
let table = format!("vec_{}", sanitize_model_key(WARM_TEST_MODEL));
let namespace = token.namespace().as_str().to_owned();
let subject = Uuid::new_v4();
let bytes: Vec<u8> = embedding
.iter()
.flat_map(|value| value.to_le_bytes())
.collect();
let sql = rt.sql();
let mut writer = sql.writer().await.expect("writer");
writer
.execute(SqlStatement {
sql: format!(
"INSERT INTO {table} \
(subject_id, namespace, kind, field, embedding_model, embedding) \
VALUES (?1, ?2, 'concept', 'knowledge.atom', ?3, ?4)"
),
params: vec![
SqlValue::Text(subject.to_string()),
SqlValue::Text(namespace.clone()),
SqlValue::Text(WARM_TEST_MODEL.to_string()),
SqlValue::Blob(bytes),
],
label: None,
})
.await
.expect("insert fresh vector");
writer
.execute(SqlStatement {
sql: "INSERT INTO ann_write_log \
(namespace, embedding_model, kind, field, subject_id, op) \
VALUES (?1, ?2, 'concept', 'knowledge.atom', ?3, 'upsert')"
.into(),
params: vec![
SqlValue::Text(namespace),
SqlValue::Text(WARM_TEST_MODEL.to_string()),
SqlValue::Text(subject.to_string()),
],
label: None,
})
.await
.expect("append fresh-tail log row");
subject
}
#[test]
fn threshold_sized_tail_cap_scales_with_live_corpus() {
let cap = |live: u64, threshold: f64| (live as f64 * threshold).ceil() as u64;
assert_eq!(cap(3, 0.20), 1, "small corpora retain one newest row");
assert_eq!(cap(5, 0.21), 2, "fractional limits round upward");
assert_eq!(
cap(1_000_000, 0.20),
200_000,
"large corpora must not collapse to the retired fixed 20k ceiling"
);
}
#[tokio::test]
async fn capped_snapshot_selects_newest_threshold_sized_suffix() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 5).await;
register_consumer(&rt, "local", WARM_TEST_MODEL)
.await
.expect("register");
let mut reader = rt.sql().reader().await.expect("reader");
let newest_rows = reader
.query_all(SqlStatement {
sql: "SELECT subject_id FROM ann_write_log \
WHERE namespace = 'local' AND embedding_model = ?1 \
AND field = 'knowledge.atom' \
ORDER BY seq DESC LIMIT 2"
.into(),
params: vec![SqlValue::Text(WARM_TEST_MODEL.into())],
label: None,
})
.await
.expect("latest log rows");
let newest: Vec<Uuid> = newest_rows
.iter()
.map(|row| match row.get("subject_id") {
Some(SqlValue::Text(subject)) => Uuid::parse_str(subject).expect("UUID"),
other => panic!("unexpected subject: {other:?}"),
})
.collect();
let one = fetch_fresh_tail_snapshot(&rt, "local", WARM_TEST_MODEL, 0, Some(0.20))
.await
.expect("20% snapshot");
assert_eq!(one.live_count, Some(5));
assert_eq!(one.ops.len(), 1, "ceil(5 × .20) = 1");
assert_eq!(
one.ops[0].0, newest[0],
"the suffix must start newest-first"
);
let two = fetch_fresh_tail_snapshot(&rt, "local", WARM_TEST_MODEL, 0, Some(0.21))
.await
.expect("21% snapshot");
assert_eq!(two.live_count, Some(5));
assert_eq!(two.ops.len(), 2, "ceil(5 × .21) = 2");
assert_eq!(
two.ops
.iter()
.map(|(subject, _)| *subject)
.collect::<Vec<_>>(),
vec![newest[1], newest[0]],
"selected newest rows are replayed in chronological order"
);
let mut writer = rt.sql().writer().await.expect("writer");
writer
.execute(SqlStatement {
sql: "INSERT INTO ann_write_log \
(namespace, embedding_model, kind, field, subject_id, op) \
VALUES ('local', ?1, 'concept', 'knowledge.atom', ?2, 'upsert')"
.into(),
params: vec![
SqlValue::Text(WARM_TEST_MODEL.into()),
SqlValue::Text(newest[0].to_string()),
],
label: None,
})
.await
.expect("repeat newest write");
drop(writer);
let repeated = fetch_fresh_tail_snapshot(&rt, "local", WARM_TEST_MODEL, 0, Some(0.40))
.await
.expect("40% repeated snapshot");
assert_eq!(
repeated.ops.len(),
1,
"two newest writes coalesce to one subject"
);
assert_eq!(repeated.ops[0].0, newest[0]);
}
#[tokio::test]
async fn ann_segment_dir_encode_decode_round_trip() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let seg_dir = ann_segment_dir(&rt, "local", WARM_TEST_MODEL)
.expect("file backend must return Some(seg_dir)");
let dir_name = seg_dir
.file_name()
.expect("seg_dir must have a basename")
.to_string_lossy()
.into_owned();
let (decoded_ns, decoded_model) =
decode_ann_dir_name(&dir_name).expect("decode must succeed for a valid encode");
assert_eq!(decoded_ns, "local");
assert_eq!(decoded_model, WARM_TEST_MODEL);
let parent = seg_dir.parent().expect("seg_dir must have a parent");
assert_eq!(
parent.file_name().unwrap().to_string_lossy(),
"test.db.ann",
"seg_dir parent must be the database-scoped ANN root"
);
}
#[tokio::test]
async fn ensure_ann_no_data_dir_does_not_panic() {
let rt = KhiveRuntime::memory().expect("in-memory runtime");
let ann = new_shared();
let token = rt.authorize(Namespace::local()).expect("authorize");
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
let key = AnnKey::new("local", WARM_TEST_MODEL);
assert!(
!ann.indexes.read().await.contains_key(&key),
"no index should be loaded when corpus is empty and model is unknown"
);
}
#[tokio::test]
async fn ensure_ann_round_trip_hot() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
let ann = new_shared();
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
let key = AnnKey::new("local", WARM_TEST_MODEL);
assert!(
ann.indexes.read().await.contains_key(&key),
"first call must build the ANN index"
);
let seg_dir = ann_segment_dir(&rt, "local", WARM_TEST_MODEL)
.expect("file backend must have a seg_dir");
assert!(
seg_dir.join("metadata.bin").exists(),
"first call must persist v2 segments (metadata.bin missing)"
);
let info = read_commit_info(&seg_dir)
.expect("read_commit_info must not err")
.expect("metadata.bin must carry a v2 commit record");
let s = info
.last_applied_seq
.expect("checkpoint must persist an extended record with a watermark");
let (live, tail) = scope_counts(&rt, "local", WARM_TEST_MODEL, s)
.await
.expect("scope_counts must succeed");
assert!(live > 0, "seeded corpus must be live");
assert_eq!(
tail, 0,
"watermark must cover every seeded write (empty tail)"
);
use std::os::unix::fs::MetadataExt;
let meta_path = seg_dir.join("metadata.bin");
let ino_before = std::fs::metadata(&meta_path)
.expect("metadata.bin must exist after first build")
.ino();
let ann2 = new_shared();
ensure_ann_for_model(&rt, &token, &ann2, WARM_TEST_MODEL).await;
assert!(
ann2.indexes.read().await.contains_key(&key),
"second call must restore the ANN index from v2 segments"
);
let ino_after = std::fs::metadata(&meta_path)
.expect("metadata.bin must still exist")
.ino();
assert_eq!(
ino_before, ino_after,
"second call must NOT rewrite metadata.bin — proves the v2 Hot load path, not a rebuild"
);
}
#[tokio::test]
async fn knowledge_search_and_suggest_merge_write_above_loaded_bridge_watermark() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
let ann = new_shared();
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
let key = AnnKey::new("local", WARM_TEST_MODEL);
let query = vec![1.0; WARM_DIMS];
let (_, bridge_watermark) = search_loaded_with_seq(&ann, &key, &query, 20)
.await
.expect("serving bridge");
let fresh_id = append_warm_vector(&rt, &token, [1.0; WARM_DIMS]).await;
let now = chrono::Utc::now().timestamp_micros();
let mut writer = rt.sql().writer().await.expect("writer");
writer
.execute(SqlStatement {
sql: "INSERT INTO knowledge_atoms \
(id, namespace, slug, name, content, tags, properties, status, \
finalized, created_at, updated_at) \
VALUES (?1, 'local', 'opaque-fresh-tail', 'Opaque Fresh Tail', \
'content with no lexical overlap', '[\"type:domain\"]', '{}', 'reviewed', \
1, ?2, ?2)"
.into(),
params: vec![SqlValue::Text(fresh_id.to_string()), SqlValue::Integer(now)],
label: None,
})
.await
.expect("insert hydratable knowledge atom");
writer
.execute(SqlStatement {
sql: "INSERT INTO knowledge_domains \
(id, namespace, slug, name, description, members, created_at, updated_at) \
VALUES (?1, 'local', 'opaque-fresh-tail', 'Opaque Fresh Tail', \
'content with no lexical overlap', '[]', ?2, ?2)"
.into(),
params: vec![SqlValue::Text(fresh_id.to_string()), SqlValue::Integer(now)],
label: None,
})
.await
.expect("insert the canonical domain row the mirror atom stands for");
drop(writer);
let (_, still_loaded_watermark) = search_loaded_with_seq(&ann, &key, &query, 20)
.await
.expect("stale serving bridge remains loaded");
assert_eq!(
still_loaded_watermark, bridge_watermark,
"the simulated external write must not mutate the in-process bridge"
);
let result = super::super::KnowledgeHandlers::search(
&rt,
&token,
json!({
"query": "quasar zephyr",
"min_score": 0.1,
"limit": 10,
"rerank": false
}),
&ann,
)
.await
.expect("knowledge.search");
let ids: Vec<&str> = result["results"]
.as_array()
.expect("results")
.iter()
.filter_map(|hit| hit["id"].as_str())
.collect();
let fresh_id = fresh_id.to_string();
assert!(
ids.contains(&fresh_id.as_str()),
"fresh-tail atom must be visible without rebuilding the loaded bridge: {result}"
);
let suggest = super::super::KnowledgeHandlers::suggest(
&rt,
&token,
json!({
"query": "quasar zephyr nebula pulsar aurora",
"limit": 10
}),
&ann,
)
.await
.expect("knowledge.suggest");
let suggest_ids: Vec<&str> = suggest["results"]
.as_array()
.expect("suggest results")
.iter()
.filter_map(|hit| hit["id"].as_str())
.collect();
assert!(
suggest_ids.contains(&fresh_id.as_str()),
"fresh-tail domain atom must be visible to knowledge.suggest: {suggest}"
);
}
#[tokio::test]
async fn fresh_tail_mismatch_replaces_stale_candidates_from_published_segment() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
let ann = new_shared();
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
let key = AnnKey::new("local", WARM_TEST_MODEL);
let query = vec![1.0; WARM_DIMS];
let (old_candidates, old_watermark) = search_loaded_with_seq(&ann, &key, &query, 20)
.await
.expect("old serving bridge");
let fresh_id = append_warm_vector(&rt, &token, [1.0; WARM_DIMS]).await;
assert!(
!old_candidates
.iter()
.any(|(subject, _)| *subject == fresh_id),
"the old bridge must not already contain the peer's fresh write"
);
let replacement = load_and_build_from_vector_store(&rt, &token, WARM_TEST_MODEL)
.await
.expect("scan replacement corpus")
.expect("replacement corpus is non-empty");
let replacement_watermark = replacement
.index
.last_applied_seq()
.expect("replacement carries a watermark");
assert!(replacement_watermark > old_watermark);
persist_ann_v2(&rt, "local", WARM_TEST_MODEL, &replacement)
.expect("publish replacement segment");
raise_watermark(
&rt,
"local",
WARM_TEST_MODEL,
replacement_watermark,
CheckpointAuthority::Incremental,
)
.await
.expect("raise peer watermark");
compact_log(&rt, "local", WARM_TEST_MODEL)
.await
.expect("compact through peer watermark");
assert!(
!tail_exists(&rt, "local", WARM_TEST_MODEL, old_watermark)
.await
.expect("probe compacted old tail"),
"the old bridge's tail must be gone so only segment re-resolution can recover it"
);
let generation_before = current_generation(&ann, "local");
let outcome = fresh_tail_leg(&rt, &ann, &key, &query, 20, Some(old_watermark)).await;
let replacement_candidates = match outcome {
FreshTailOutcome::Replace { candidates, .. } => candidates,
FreshTailOutcome::Ops(_) => {
panic!("a compaction mismatch must replace, not extend, stale candidates")
}
FreshTailOutcome::Skipped => panic!("mismatch re-resolution must not disappear"),
};
assert!(
replacement_candidates
.iter()
.any(|(subject, _)| *subject == fresh_id),
"current-query re-resolution must surface the write covered by the published segment"
);
assert!(
current_generation(&ann, "local") > generation_before,
"mismatch handling must retire the stale cache generation"
);
assert!(
search_loaded_with_seq(&ann, &key, &query, 20)
.await
.is_none(),
"the stale in-process bridge must be evicted so normal warming re-adopts the segment"
);
}
#[tokio::test]
async fn registry_loss_evicts_stale_bridge_and_forces_authoritative_rebuild() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
let ann = new_shared();
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
let key = AnnKey::new("local", WARM_TEST_MODEL);
let query = vec![1.0; WARM_DIMS];
let (_, stale_watermark) = search_loaded_with_seq(&ann, &key, &query, 20)
.await
.expect("serving bridge");
let fresh_id = append_warm_vector(&rt, &token, [1.0; WARM_DIMS]).await;
let mut writer = rt.sql().writer().await.expect("writer");
writer
.execute(SqlStatement {
sql: "DELETE FROM ann_consumer_watermark \
WHERE consumer = ?1 AND namespace = 'local' \
AND embedding_model = ?2"
.into(),
params: vec![
SqlValue::Text(ANN_CONSUMER.into()),
SqlValue::Text(WARM_TEST_MODEL.into()),
],
label: None,
})
.await
.expect("delete own registry row");
writer
.execute(SqlStatement {
sql: "INSERT INTO ann_consumer_watermark \
(consumer, namespace, embedding_model, watermark) \
VALUES ('peer:test', 'local', ?1, 999)"
.into(),
params: vec![SqlValue::Text(WARM_TEST_MODEL.into())],
label: None,
})
.await
.expect("insert peer watermark");
drop(writer);
compact_log(&rt, "local", WARM_TEST_MODEL)
.await
.expect("compact missing interval");
assert!(
!tail_exists(&rt, "local", WARM_TEST_MODEL, stale_watermark)
.await
.expect("tail probe"),
"setup must remove the stale bridge's recovery tail"
);
let outcome = fresh_tail_leg(&rt, &ann, &key, &query, 20, Some(stale_watermark)).await;
assert!(
matches!(
outcome,
FreshTailOutcome::Replace { ref candidates, source_exhausted: true }
if candidates.is_empty()
),
"the query that detects registry loss must discard stale candidates"
);
assert!(search_loaded_with_seq(&ann, &key, &query, 20)
.await
.is_none());
assert_eq!(
read_own_watermark(&rt, "local", WARM_TEST_MODEL)
.await
.expect("registry read"),
Some(-1),
"the cross-process sentinel must stay durable through the Cold transition"
);
assert_eq!(
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await,
AnnWarmOutcome::Ready
);
let rebuilt = search_loaded(&ann, &key, &query, 20)
.await
.expect("authoritative rebuilt bridge");
assert!(
rebuilt.iter().any(|(subject, _)| *subject == fresh_id),
"the next warm must rebuild from the full corpus, not re-adopt the stale segment"
);
assert!(
read_own_watermark(&rt, "local", WARM_TEST_MODEL)
.await
.expect("registry read")
.is_some_and(|watermark| watermark >= 0),
"successful authoritative publication must clear the sentinel"
);
assert!(!force_rebuild_required(&ann, &key));
}
#[tokio::test]
async fn ensure_ann_stale_rebuild() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
let ann = new_shared();
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
let key = AnnKey::new("local", WARM_TEST_MODEL);
assert!(ann.indexes.read().await.contains_key(&key), "initial build");
seed_warm_corpus(&rt, &token, 1).await;
let seg_dir = ann_segment_dir(&rt, "local", WARM_TEST_MODEL)
.expect("file backend must have a seg_dir");
let info_before = read_commit_info(&seg_dir)
.expect("read_commit_info must not err")
.expect("v2 commit record must be present after initial build");
let s_before = info_before
.last_applied_seq
.expect("initial checkpoint must carry a watermark");
let (_, tail) = scope_counts(&rt, "local", WARM_TEST_MODEL, s_before)
.await
.expect("scope_counts must succeed");
assert!(
tail > 0,
"mutation must appear as a tail row above the watermark"
);
let ann2 = new_shared();
ensure_ann_for_model(&rt, &token, &ann2, WARM_TEST_MODEL).await;
assert!(
ann2.indexes.read().await.contains_key(&key),
"must serve an index after corpus mutation (tail replayed)"
);
let info_after = read_commit_info(&seg_dir)
.expect("read_commit_info after replay must not err")
.expect("v2 commit record must be present after replay checkpoint");
assert_eq!(
info_after.vector_count, 5,
"checkpoint must reflect the 5-row corpus (4 initial + 1 replayed)"
);
let s_after = info_after
.last_applied_seq
.expect("replay checkpoint must carry a watermark");
assert!(s_after > s_before, "checkpoint must advance the watermark");
let (_, tail_after) = scope_counts(&rt, "local", WARM_TEST_MODEL, s_after)
.await
.expect("scope_counts must succeed");
assert_eq!(
tail_after, 0,
"replayed tail must be covered by the new watermark"
);
}
#[tokio::test]
async fn ensure_ann_zero_watermark_then_first_write_is_stale_tail() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus_opts(&rt, &token, 4, false).await;
let ann = new_shared();
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
let seg_dir = ann_segment_dir(&rt, "local", WARM_TEST_MODEL).expect("seg_dir");
let info = read_commit_info(&seg_dir)
.expect("read_commit_info")
.expect("v2 commit record");
assert_eq!(
info.last_applied_seq,
Some(0),
"empty-log checkpoint must persist the numeric zero baseline, not a missing watermark"
);
seed_warm_corpus_opts(&rt, &token, 1, true).await;
let ann2 = new_shared();
ensure_ann_for_model(&rt, &token, &ann2, WARM_TEST_MODEL).await;
let key = AnnKey::new("local", WARM_TEST_MODEL);
let n = ann2
.indexes
.read()
.await
.get(&key)
.map(AnnBridge::num_vectors)
.expect("index must be served after the first logged write");
assert_eq!(
n, 5,
"Stale-tail must replay the logged write (Hot would serve 4)"
);
let info2 = read_commit_info(&seg_dir)
.expect("read_commit_info")
.expect("v2 commit record after replay");
assert!(
info2.last_applied_seq.unwrap_or(0) > 0,
"replay checkpoint must advance past the zero baseline"
);
}
#[tokio::test]
async fn ensure_ann_delete_all_is_empty() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 3).await;
let ann = new_shared();
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
let key = AnnKey::new("local", WARM_TEST_MODEL);
assert!(ann.indexes.read().await.contains_key(&key), "initial build");
let table = format!("vec_{}", sanitize_model_key(WARM_TEST_MODEL));
let sql = rt.sql();
let mut w = sql.writer().await.expect("writer");
w.execute(SqlStatement {
sql: format!(
"INSERT INTO ann_write_log \
(namespace, embedding_model, kind, field, subject_id, op) \
SELECT namespace, embedding_model, kind, field, subject_id, 'delete' \
FROM {table} WHERE namespace = ?1 AND embedding_model = ?2"
),
params: vec![
SqlValue::Text("local".into()),
SqlValue::Text(WARM_TEST_MODEL.into()),
],
label: None,
})
.await
.expect("log deletes");
w.execute(SqlStatement {
sql: format!("DELETE FROM {table} WHERE namespace = ?1 AND embedding_model = ?2"),
params: vec![
SqlValue::Text("local".into()),
SqlValue::Text(WARM_TEST_MODEL.into()),
],
label: None,
})
.await
.expect("delete corpus");
drop(w);
let ann2 = new_shared();
ensure_ann_for_model(&rt, &token, &ann2, WARM_TEST_MODEL).await;
assert!(
!ann2.indexes.read().await.contains_key(&key),
"zero live corpus must classify Empty — no ANN served (rule 5 precedes Hot)"
);
assert!(
is_terminally_unavailable(&ann2, &key),
"Empty must set the terminal unavailable marker for wait_ready"
);
}
#[tokio::test]
async fn compact_log_bounded_by_pair_minimum() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
register_consumer(&rt, "local", WARM_TEST_MODEL)
.await
.expect("register pending");
{
let sql = rt.sql();
let mut w = sql.writer().await.expect("writer");
w.execute(SqlStatement {
sql: "INSERT INTO ann_consumer_watermark \
(consumer, namespace, embedding_model, watermark) VALUES (?1, ?2, ?3, 99)"
.into(),
params: vec![
SqlValue::Text("other:test".into()),
SqlValue::Text("local".into()),
SqlValue::Text(WARM_TEST_MODEL.into()),
],
label: None,
})
.await
.expect("insert B row");
}
compact_log(&rt, "local", WARM_TEST_MODEL)
.await
.expect("compact");
let (_, tail_while_pending) = scope_counts(&rt, "local", WARM_TEST_MODEL, 0)
.await
.expect("scope_counts");
assert_eq!(
tail_while_pending, 4,
"a fresh pending row must block pair compaction"
);
raise_watermark(
&rt,
"local",
WARM_TEST_MODEL,
2,
CheckpointAuthority::FullRegistered,
)
.await
.expect("raise");
compact_log(&rt, "local", WARM_TEST_MODEL)
.await
.expect("compact");
let (_, tail_after) = scope_counts(&rt, "local", WARM_TEST_MODEL, 0)
.await
.expect("scope_counts");
assert_eq!(
tail_after, 2,
"compaction must advance exactly to the pair MIN"
);
}
#[tokio::test]
async fn compact_log_retires_expired_never_activated_consumer() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
register_consumer(&rt, "local", WARM_TEST_MODEL)
.await
.expect("register pending");
let sql = rt.sql();
let mut writer = sql.writer().await.expect("writer");
writer
.execute(SqlStatement {
sql: "UPDATE ann_consumer_pending SET registered_at_us = 1 \
WHERE consumer = ?1 AND namespace = 'local' \
AND embedding_model = ?2"
.into(),
params: vec![
SqlValue::Text(ANN_CONSUMER.into()),
SqlValue::Text(WARM_TEST_MODEL.into()),
],
label: Some("test_age_pending_ann_consumer".into()),
})
.await
.expect("age pending registration");
writer
.execute(SqlStatement {
sql: "INSERT INTO ann_consumer_watermark \
(consumer, namespace, embedding_model, watermark) \
VALUES ('other:test', 'local', ?1, 99)"
.into(),
params: vec![SqlValue::Text(WARM_TEST_MODEL.into())],
label: None,
})
.await
.expect("insert active peer");
drop(writer);
compact_log(&rt, "local", WARM_TEST_MODEL)
.await
.expect("retire and compact");
assert_eq!(
read_own_watermark(&rt, "local", WARM_TEST_MODEL)
.await
.expect("read retired registration"),
None,
"expired pending consumer must be removed so its return is Cold"
);
let (_, retained) = scope_counts(&rt, "local", WARM_TEST_MODEL, 0)
.await
.expect("scope counts");
assert_eq!(
retained, 0,
"the retired pending row must no longer pin the active peer's minimum"
);
}
#[tokio::test]
async fn ordinary_checkpoint_cannot_clear_force_rebuild_sentinel() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let ann = new_shared();
let key = AnnKey::new("local", WARM_TEST_MODEL);
prepare_authoritative_rebuild(&rt, &ann, &key)
.await
.expect("publish sentinel");
assert!(
raise_watermark(
&rt,
"local",
WARM_TEST_MODEL,
7,
CheckpointAuthority::Incremental,
)
.await
.is_err(),
"an ordinary checkpoint must lose the conditional publication fence"
);
assert_eq!(
read_own_watermark(&rt, "local", WARM_TEST_MODEL)
.await
.expect("read sentinel"),
Some(-1)
);
raise_watermark(
&rt,
"local",
WARM_TEST_MODEL,
7,
CheckpointAuthority::FullSentinel,
)
.await
.expect("authoritative checkpoint clears sentinel");
assert_eq!(
read_own_watermark(&rt, "local", WARM_TEST_MODEL)
.await
.expect("read raised watermark"),
Some(7)
);
}
#[tokio::test]
async fn full_scan_checkpoint_clears_local_force_rebuild_marker() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
let ann = new_shared();
let key = AnnKey::new("local", WARM_TEST_MODEL);
register_consumer(&rt, "local", WARM_TEST_MODEL)
.await
.expect("register pending consumer");
assert_eq!(
read_own_watermark(&rt, "local", WARM_TEST_MODEL)
.await
.expect("read pending watermark"),
Some(ann_registry::PENDING_WATERMARK)
);
let authority = prepare_full_corpus_scan(&rt, &ann, &key)
.await
.expect("establish full-scan authority");
assert_eq!(authority, CheckpointAuthority::FullSentinel);
assert_eq!(
read_own_watermark(&rt, "local", WARM_TEST_MODEL)
.await
.expect("read recovery fence"),
Some(ann_registry::RECOVERING_WATERMARK),
"a direct full scan must promote pending to the authoritative fence before scanning"
);
assert!(force_rebuild_required(&ann, &key));
let bridge = load_and_build_from_vector_store(&rt, &token, WARM_TEST_MODEL)
.await
.expect("scan corpus")
.expect("non-empty corpus");
assert!(
checkpoint_raise_compact_readopt(
&rt,
&ann,
&key,
bridge,
current_generation(&ann, "local"),
authority,
)
.await,
"authoritative full scan must publish"
);
assert!(
!force_rebuild_required(&ann, &key),
"the shared checkpoint seam must finish local recovery for direct rebuild callers"
);
let query = vec![1.0; WARM_DIMS];
let (_, watermark) = search_loaded_with_seq(&ann, &key, &query, 20)
.await
.expect("published bridge");
assert!(
matches!(
fresh_tail_leg(&rt, &ann, &key, &query, 20, Some(watermark)).await,
FreshTailOutcome::Ops(_)
),
"the next query must retain the freshly published bridge"
);
}
#[tokio::test]
async fn concurrent_full_sentinel_loser_does_not_restore_sentinel() {
let rt = memory_rt_with_embedder();
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
let ann = new_shared();
let key = AnnKey::new("local", WARM_TEST_MODEL);
let authority_a = prepare_full_corpus_scan(&rt, &ann, &key)
.await
.expect("first full-scan authority");
let authority_b = prepare_full_corpus_scan(&rt, &ann, &key)
.await
.expect("second full-scan authority");
assert_eq!(authority_a, CheckpointAuthority::FullSentinel);
assert_eq!(authority_b, CheckpointAuthority::FullSentinel);
let bridge_a = load_and_build_from_vector_store(&rt, &token, WARM_TEST_MODEL)
.await
.expect("scan A")
.expect("non-empty A");
let bridge_b = load_and_build_from_vector_store(&rt, &token, WARM_TEST_MODEL)
.await
.expect("scan B")
.expect("non-empty B");
let generation = current_generation(&ann, "local");
let (published_a, published_b) = tokio::join!(
checkpoint_raise_compact_readopt(&rt, &ann, &key, bridge_a, generation, authority_a,),
checkpoint_raise_compact_readopt(&rt, &ann, &key, bridge_b, generation, authority_b,)
);
assert!(published_a || published_b, "one full scan must win");
assert!(
read_own_watermark(&rt, "local", WARM_TEST_MODEL)
.await
.expect("read winner watermark")
.is_some_and(|watermark| watermark >= 0),
"the losing checkpoint must not demote the winner back to -1"
);
assert!(!force_rebuild_required(&ann, &key));
assert!(
has_current_index(&ann, &key).await,
"the winner must remain available after the losing fence"
);
}
#[tokio::test]
async fn concurrent_pathless_normal_checkpoints_publish_monotonically() {
let rt = memory_rt_with_embedder();
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
let ann = new_shared();
assert_eq!(
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await,
AnnWarmOutcome::Ready
);
let key = AnnKey::new("local", WARM_TEST_MODEL);
let old_authority = prepare_full_corpus_scan(&rt, &ann, &key)
.await
.expect("old authority");
let old_bridge = load_and_build_from_vector_store(&rt, &token, WARM_TEST_MODEL)
.await
.expect("old scan")
.expect("old corpus");
let old_watermark = old_bridge.index.last_applied_seq().unwrap_or(0);
let fresh_id = append_warm_vector(&rt, &token, [1.0; WARM_DIMS]).await;
let new_authority = prepare_full_corpus_scan(&rt, &ann, &key)
.await
.expect("new authority");
let new_bridge = load_and_build_from_vector_store(&rt, &token, WARM_TEST_MODEL)
.await
.expect("new scan")
.expect("new corpus");
let new_watermark = new_bridge.index.last_applied_seq().unwrap_or(0);
assert!(new_watermark > old_watermark);
let generation = current_generation(&ann, "local");
let (old_result, new_result) = tokio::join!(
checkpoint_raise_compact_readopt(
&rt,
&ann,
&key,
old_bridge,
generation,
old_authority,
),
checkpoint_raise_compact_readopt(
&rt,
&ann,
&key,
new_bridge,
generation,
new_authority,
)
);
assert!(old_result && new_result);
assert_eq!(
read_own_watermark(&rt, "local", WARM_TEST_MODEL)
.await
.expect("registry read"),
Some(i64::try_from(new_watermark).expect("watermark range"))
);
let query = vec![1.0; WARM_DIMS];
let (hits, loaded_watermark) = search_loaded_with_seq(&ann, &key, &query, 20)
.await
.expect("monotone winner");
assert_eq!(loaded_watermark, new_watermark);
assert!(hits.iter().any(|(subject, _)| *subject == fresh_id));
}
#[tokio::test]
async fn pathless_lifecycle_helpers_do_not_open_nested_transactions() {
let rt = memory_rt_with_embedder();
let key = AnnKey::new("local", WARM_TEST_MODEL);
let sql = rt.sql();
let mut outer = sql.writer().await.expect("outer writer");
outer
.execute(SqlStatement {
sql: "BEGIN IMMEDIATE".into(),
params: vec![],
label: Some("test_knowledge_pathless_outer_begin".into()),
})
.await
.expect("begin outer transaction");
register_consumer(&rt, "local", WARM_TEST_MODEL)
.await
.expect("register pending inside outer transaction");
write_force_rebuild_sentinel_row(&rt, &key)
.await
.expect("publish recovery sentinel inside outer transaction");
raise_watermark(
&rt,
"local",
WARM_TEST_MODEL,
0,
CheckpointAuthority::FullSentinel,
)
.await
.expect("activate sentinel inside outer transaction");
compact_log(&rt, "local", WARM_TEST_MODEL)
.await
.expect("compact inside outer transaction");
assert_eq!(
read_own_watermark(&rt, "local", WARM_TEST_MODEL)
.await
.expect("read pathless lifecycle state"),
Some(0)
);
outer
.execute(SqlStatement {
sql: "ROLLBACK".into(),
params: vec![],
label: Some("test_knowledge_pathless_outer_rollback".into()),
})
.await
.expect("rollback outer transaction");
}
#[tokio::test]
async fn stale_normal_checkpoint_adopts_newer_publisher_without_overwrite() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
let initial_ann = new_shared();
assert_eq!(
ensure_ann_for_model(&rt, &token, &initial_ann, WARM_TEST_MODEL).await,
AnnWarmOutcome::Ready
);
let key = AnnKey::new("local", WARM_TEST_MODEL);
let stale_authority = prepare_full_corpus_scan(&rt, &initial_ann, &key)
.await
.expect("stale authority");
let stale_bridge = load_and_build_from_vector_store(&rt, &token, WARM_TEST_MODEL)
.await
.expect("stale scan")
.expect("stale corpus");
let stale_watermark = stale_bridge.index.last_applied_seq().unwrap_or(0);
let fresh_id = append_warm_vector(&rt, &token, [1.0; WARM_DIMS]).await;
let winner_ann = new_shared();
let winner_authority = prepare_full_corpus_scan(&rt, &winner_ann, &key)
.await
.expect("winner authority");
let winner_bridge = load_and_build_from_vector_store(&rt, &token, WARM_TEST_MODEL)
.await
.expect("winner scan")
.expect("winner corpus");
let winner_watermark = winner_bridge.index.last_applied_seq().unwrap_or(0);
assert!(winner_watermark > stale_watermark);
assert!(
checkpoint_raise_compact_readopt(
&rt,
&winner_ann,
&key,
winner_bridge,
current_generation(&winner_ann, "local"),
winner_authority,
)
.await,
"newer publisher must commit"
);
let stale_ann = new_shared();
assert!(
checkpoint_raise_compact_readopt(
&rt,
&stale_ann,
&key,
stale_bridge,
current_generation(&stale_ann, "local"),
stale_authority,
)
.await,
"stale publisher must adopt the winner instead of overwriting it"
);
assert_eq!(
read_own_watermark(&rt, "local", WARM_TEST_MODEL)
.await
.expect("registry read"),
Some(i64::try_from(winner_watermark).expect("watermark range"))
);
let info = read_commit_info(
&ann_segment_dir(&rt, "local", WARM_TEST_MODEL).expect("segment directory"),
)
.expect("commit read")
.expect("commit info");
assert_eq!(info.last_applied_seq, Some(winner_watermark));
assert_eq!(info.vector_count, 5);
let query = vec![1.0; WARM_DIMS];
assert!(
search_loaded(&stale_ann, &key, &query, 20)
.await
.expect("adopted winner")
.iter()
.any(|(subject, _)| *subject == fresh_id),
"the stale publisher must serve the winner's fresh row"
);
}
#[tokio::test]
async fn first_use_empty_corpus_retains_cross_process_sentinel() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
let ann = new_shared();
let key = AnnKey::new("local", WARM_TEST_MODEL);
assert_eq!(
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await,
AnnWarmOutcome::Empty
);
assert_eq!(
read_own_watermark(&rt, "local", WARM_TEST_MODEL)
.await
.expect("registry read"),
Some(-1),
"even a fresh process must preserve the only cross-process loss signal"
);
assert!(force_rebuild_required(&ann, &key));
}
#[tokio::test]
async fn delayed_sentinel_request_cannot_demote_completed_recovery() {
let rt = memory_rt_with_embedder();
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
let ann = new_shared();
let key = AnnKey::new("local", WARM_TEST_MODEL);
assert_eq!(
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await,
AnnWarmOutcome::Ready
);
let winner = read_own_watermark(&rt, "local", WARM_TEST_MODEL)
.await
.expect("winner watermark")
.expect("registered winner");
assert!(winner >= 0);
prepare_authoritative_rebuild(&rt, &ann, &key)
.await
.expect("delayed sentinel request");
assert_eq!(
read_own_watermark(&rt, "local", WARM_TEST_MODEL)
.await
.expect("registry read"),
Some(winner),
"revalidation under the publication lock must preserve the winner"
);
assert!(
force_rebuild_required(&ann, &key),
"the delayed detector must remain Cold until it scans or adopts authoritatively"
);
}
#[tokio::test]
async fn ensure_ann_pre_amendment_record_is_cold() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
let ann = new_shared();
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
let seg_dir = ann_segment_dir(&rt, "local", WARM_TEST_MODEL).expect("seg_dir");
let meta_path = seg_dir.join("metadata.bin");
let bytes = std::fs::read(&meta_path).expect("read metadata.bin");
std::fs::write(&meta_path, &bytes[..bytes.len() - 57]).expect("truncate trailer");
let info = read_commit_info(&seg_dir)
.expect("read_commit_info")
.expect("base-length record must still parse");
assert_eq!(
info.last_applied_seq, None,
"trailer removed → no watermark"
);
use std::os::unix::fs::MetadataExt;
let ino_before = std::fs::metadata(&meta_path).expect("meta").ino();
let ann2 = new_shared();
ensure_ann_for_model(&rt, &token, &ann2, WARM_TEST_MODEL).await;
let key = AnnKey::new("local", WARM_TEST_MODEL);
assert!(
ann2.indexes.read().await.contains_key(&key),
"Cold rebuild must still produce a served index"
);
let ino_after = std::fs::metadata(&meta_path).expect("meta").ino();
assert_ne!(
ino_before, ino_after,
"pre-amendment record must force a rebuild (metadata.bin rewritten), not a Hot load"
);
let info2 = read_commit_info(&seg_dir)
.expect("read_commit_info")
.expect("rebuilt record");
assert!(
info2.last_applied_seq.is_some(),
"rebuild must restore the extended watermark record"
);
}
#[tokio::test]
async fn ensure_ann_fast_path_ignores_generation_stale_cached_entry() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
let ann = new_shared();
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
let key = AnnKey::new("local", WARM_TEST_MODEL);
assert!(
ann.indexes.read().await.contains_key(&key),
"setup: first call must build and install the index at generation 0"
);
bump_generation(&ann, "local");
assert_eq!(current_generation(&ann, "local"), 1);
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
assert_eq!(
ann.indexes
.read()
.await
.get(&key)
.expect("entry present")
.generation,
1,
"a present-but-generation-stale entry must not short-circuit via the fast \
path; the reloaded/rebuilt entry must be re-stamped with the namespace's \
new current generation"
);
}
#[tokio::test]
async fn warm_known_snapshots_v2_only_no_legacy_table() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 4).await;
let ann = new_shared();
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
let key = AnnKey::new("local", WARM_TEST_MODEL);
assert!(
ann.indexes.read().await.contains_key(&key),
"setup: first ensure must persist v2 segments"
);
{
let sql = rt.sql();
let mut w = sql.writer().await.expect("writer");
w.execute(SqlStatement {
sql: "DROP TABLE IF EXISTS retrieval_snapshots".into(),
params: vec![],
label: None,
})
.await
.expect("drop retrieval_snapshots");
}
let ann_fresh = new_shared();
warm_known_snapshots(&rt, &ann_fresh).await;
assert!(
ann_fresh.indexes.read().await.contains_key(&key),
"warm_known_snapshots must warm v2 segments when retrieval_snapshots is absent \
(regression: a v1 query error must not abort the v2 filesystem pass)"
);
}
#[tokio::test]
async fn ensure_ann_for_model_empty_corpus_marks_unavailable_and_wait_short_circuits() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
let ann = new_shared();
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
let key = AnnKey::new("local", WARM_TEST_MODEL);
assert!(
!ann.indexes.read().await.contains_key(&key),
"empty corpus must not install an index"
);
let start = std::time::Instant::now();
let ready = wait_ready(&ann, &key, ANN_WARM_WAIT_TIMEOUT_MS, ANN_WARM_WAIT_POLL_MS).await;
let elapsed = start.elapsed();
assert!(!ready, "empty corpus must never become ready");
assert_terminal_wait_latency(elapsed);
}
#[tokio::test]
async fn ensure_ann_for_model_rebuild_error_does_not_mark_unavailable() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 3).await;
let model_key = sanitize_model_key(WARM_TEST_MODEL);
let table = format!("vec_{model_key}");
let sql = rt.sql();
let mut w = sql.writer().await.expect("writer");
w.execute(SqlStatement {
sql: format!("DROP TABLE {table}"),
params: vec![],
label: None,
})
.await
.expect("drop corpus table");
w.execute(SqlStatement {
sql: format!("CREATE VIEW {table} AS SELECT * FROM missing_corpus_table"),
params: vec![],
label: None,
})
.await
.expect("create broken view");
drop(w);
let ann = new_shared();
ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
let key = AnnKey::new("local", WARM_TEST_MODEL);
assert!(
!ann.indexes.read().await.contains_key(&key),
"a failed rebuild must not install an index"
);
assert!(
!unavailable_guard(&ann.unavailable).contains_key(&key),
"a rebuild ERROR must not mark the key unavailable — only a completed \
empty-corpus scan may; a marker here would short-circuit wait_ready \
while the same-generation retry is in flight"
);
let ann2 = ann.clone();
let key2 = key.clone();
tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(40)).await;
let bridge = AnnBridge::build(vec![1.0f32, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build")
.with_generation(0);
install_if_fresher(&ann2, &key2, bridge).await;
});
let ready = wait_ready(&ann, &key, 500, 10).await;
assert!(
ready,
"after a rebuild error the wait must keep polling and observe the \
retry's install, not short-circuit false"
);
}
#[tokio::test]
async fn a_non_warm_host_declines_the_corpus_build_and_writes_nothing() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
seed_warm_corpus(&rt, &token, 3).await;
let ann = new_shared_for_role(false);
let outcome = ensure_ann_for_model(&rt, &token, &ann, WARM_TEST_MODEL).await;
assert_eq!(outcome, AnnWarmOutcome::Declined);
let key = AnnKey::new("local", WARM_TEST_MODEL);
assert!(
!ann.indexes.read().await.contains_key(&key),
"a declined warm must not install an index"
);
if let Some(seg_dir) = ann_segment_dir(&rt, "local", WARM_TEST_MODEL) {
assert!(
!seg_dir.join("metadata.bin").exists(),
"a declined warm must not publish a segment"
);
}
assert!(
matches!(
read_own_watermark(&rt, "local", WARM_TEST_MODEL).await,
Ok(None)
),
"a declined warm must leave the registry untouched: no row, and in \
particular no rebuild sentinel this process will never satisfy"
);
}
#[tokio::test]
async fn ensure_ann_for_model_store_open_failure_does_not_mark_unavailable() {
let dir = TempDir::new().expect("tempdir");
let rt = file_rt_with_embedder(dir.path().join("test.db"));
let token = rt.authorize(Namespace::local()).expect("authorize");
let model = "model-with-no-registered-embedder";
assert!(
rt.vectors_for_model(&token, model).is_err(),
"precondition: opening the vector store for an unregistered model must fail"
);
let ann = new_shared();
ensure_ann_for_model(&rt, &token, &ann, model).await;
let key = AnnKey::new("local", model);
assert!(
!unavailable_guard(&ann.unavailable).contains_key(&key),
"a store-opening failure must not mark the key unavailable"
);
let ann2 = ann.clone();
let key2 = key.clone();
tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(40)).await;
let bridge = AnnBridge::build(vec![1.0f32, 0.0, 0.0, 0.0], 4, vec![Uuid::new_v4()])
.expect("build")
.with_generation(0);
install_if_fresher(&ann2, &key2, bridge).await;
});
let ready = wait_ready(&ann, &key, 500, 10).await;
assert!(
ready,
"after a store-opening failure the wait must keep polling and observe \
the retry's install, not short-circuit false"
);
}
}