use std::collections::{HashMap, HashSet};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use khive_retrieval::ann::corpus::{CorpusScope, WatermarkCapture};
use khive_retrieval::ann::registry::{self as ann_registry, WatermarkAuthority};
use khive_runtime::config::ann_rebuild_threshold_from_env as ann_rebuild_threshold;
use khive_runtime::{KhiveRuntime, Namespace, NamespaceToken, RuntimeError};
use khive_storage::types::{SqlStatement, SqlValue};
use super::snapshot_invalidation::invalidate_legacy_vamana_snapshots;
use khive_vamana::bridge::AnnBridgeCore;
use khive_vamana::distance::l2_normalize;
use khive_vamana::{
read_commit_info, segment_commit_digest, CorpusFingerprint, VamanaIndex, VamanaSnapshot,
};
use tokio::sync::RwLock;
use uuid::Uuid;
#[cfg(test)]
use khive_vamana::{write_external_ids_sidecar, VamanaConfig};
pub(crate) struct AnnBridge {
core: AnnBridgeCore,
generation: u64,
#[cfg(test)]
drop_probe: Option<Arc<()>>,
#[cfg(test)]
search_pause: Option<Arc<TestSearchPause>>,
}
khive_vamana::impl_ann_bridge_helpers!(
AnnBridge,
);
#[cfg(test)]
#[derive(Default)]
struct TestSearchPause {
started: AtomicBool,
released: std::sync::Mutex<bool>,
wake: std::sync::Condvar,
}
#[cfg(test)]
impl TestSearchPause {
fn wait(&self) {
let mut released = self.released.lock().expect("search pause");
self.started.store(true, Ordering::SeqCst);
while !*released {
released = self.wake.wait(released).expect("search pause");
}
}
fn release(&self) {
*self.released.lock().expect("search pause") = true;
self.wake.notify_all();
}
}
#[derive(Clone, Debug, Eq, PartialEq, Hash)]
pub(crate) struct AnnKey {
namespace: String,
model: String,
}
impl AnnKey {
pub(crate) fn new(namespace: &str, model: &str) -> Self {
Self {
namespace: namespace.to_owned(),
model: model.to_owned(),
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum AnnWarmFailure {
EmptyCorpus,
Operational,
Interrupted,
NotWarmHost,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum AnnWarmOutcome {
Ready,
Empty,
Failed,
Declined,
}
#[derive(Debug)]
enum AnnWarmState {
Warming {
attempt_id: u64,
generation: u64,
started_at: std::time::Instant,
},
Ready {
generation: u64,
},
Failed {
generation: u64,
error: AnnWarmFailure,
},
}
struct AnnWarmPermit {
ann: SharedAnn,
key: AnnKey,
attempt_id: u64,
generation: u64,
finished: bool,
}
pub(crate) struct AnnState {
indexes: RwLock<HashMap<AnnKey, AnnBridge>>,
warm_states: std::sync::Mutex<HashMap<AnnKey, AnnWarmState>>,
next_warm_attempt_id: AtomicU64,
generations: std::sync::Mutex<HashMap<String, u64>>,
unavailable: std::sync::Mutex<HashMap<AnnKey, u64>>,
force_rebuild: std::sync::Mutex<HashSet<AnnKey>>,
checkpoint_locks: std::sync::Mutex<HashMap<AnnKey, std::sync::Weak<tokio::sync::Mutex<()>>>>,
rotation_watch_started: AtomicBool,
builds_corpus_indexes: bool,
#[cfg(test)]
test_finish_warm_ready_hook: std::sync::Mutex<Option<Arc<FinishWarmReadyTestHook>>>,
}
#[cfg(test)]
pub(crate) struct FinishWarmReadyTestHook {
pub(crate) incumbent_digest: [u8; 32],
pub(crate) generation: u64,
pub(crate) handle_tx: tokio::sync::mpsc::UnboundedSender<tokio::task::JoinHandle<()>>,
}
pub(crate) type SharedAnn = Arc<AnnState>;
pub(crate) fn new_shared() -> SharedAnn {
new_shared_for_role(true)
}
pub(crate) fn new_shared_for_role(builds_corpus_indexes: bool) -> SharedAnn {
Arc::new(AnnState {
builds_corpus_indexes,
indexes: RwLock::new(HashMap::new()),
warm_states: std::sync::Mutex::new(HashMap::new()),
next_warm_attempt_id: AtomicU64::new(1),
generations: std::sync::Mutex::new(HashMap::new()),
unavailable: std::sync::Mutex::new(HashMap::new()),
force_rebuild: std::sync::Mutex::new(HashSet::new()),
checkpoint_locks: std::sync::Mutex::new(HashMap::new()),
rotation_watch_started: AtomicBool::new(false),
#[cfg(test)]
test_finish_warm_ready_hook: std::sync::Mutex::new(None),
})
}
fn force_rebuild_guard(
m: &std::sync::Mutex<HashSet<AnnKey>>,
) -> std::sync::MutexGuard<'_, HashSet<AnnKey>> {
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn mark_force_rebuild(ann: &SharedAnn, key: &AnnKey) {
force_rebuild_guard(&ann.force_rebuild).insert(key.clone());
}
fn force_rebuild_required(ann: &SharedAnn, key: &AnnKey) -> bool {
force_rebuild_guard(&ann.force_rebuild).contains(key)
}
fn clear_force_rebuild_if_current(ann: &SharedAnn, key: &AnnKey, generation: u64) {
if current_generation(ann, &key.namespace) == generation {
clear_force_rebuild(ann, key);
}
}
fn clear_force_rebuild(ann: &SharedAnn, key: &AnnKey) {
force_rebuild_guard(&ann.force_rebuild).remove(key);
}
fn checkpoint_lock(ann: &SharedAnn, key: &AnnKey) -> Arc<tokio::sync::Mutex<()>> {
let mut locks = ann
.checkpoint_locks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
locks.retain(|_, lock| lock.strong_count() > 0);
if let Some(lock) = locks.get(key).and_then(std::sync::Weak::upgrade) {
return lock;
}
let lock = Arc::new(tokio::sync::Mutex::new(()));
locks.insert(key.clone(), Arc::downgrade(&lock));
lock
}
fn generations_guard(
m: &std::sync::Mutex<HashMap<String, u64>>,
) -> std::sync::MutexGuard<'_, HashMap<String, u64>> {
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn bump_generation(ann: &SharedAnn, namespace: &str) -> u64 {
let mut gens = generations_guard(&ann.generations);
let slot = gens.entry(namespace.to_owned()).or_insert(0);
*slot += 1;
*slot
}
pub(crate) fn current_generation(ann: &SharedAnn, namespace: &str) -> u64 {
generations_guard(&ann.generations)
.get(namespace)
.copied()
.unwrap_or(0)
}
pub(crate) async fn install_if_fresher(ann: &SharedAnn, key: &AnnKey, candidate: AnnBridge) {
let mut idxs = ann.indexes.write().await;
let ns_generation = current_generation(ann, &key.namespace);
if candidate.generation < ns_generation {
tracing::debug!(
key = ?key,
candidate_generation = candidate.generation,
namespace_generation = ns_generation,
"knowledge ANN install skipped: candidate predates namespace's current generation"
);
return;
}
match idxs.get(key) {
Some(existing) if existing.generation >= candidate.generation => {
tracing::debug!(
key = ?key,
existing_generation = existing.generation,
candidate_generation = candidate.generation,
"knowledge ANN install skipped: cached entry already >= this build's generation"
);
}
_ => {
idxs.insert(key.clone(), candidate);
unavailable_guard(&ann.unavailable).remove(key);
}
}
}
pub(crate) async fn install_replacing(ann: &SharedAnn, key: &AnnKey, candidate: AnnBridge) -> bool {
let mut idxs = ann.indexes.write().await;
let ns_generation = current_generation(ann, &key.namespace);
if candidate.generation < ns_generation {
tracing::debug!(
key = ?key,
candidate_generation = candidate.generation,
namespace_generation = ns_generation,
"knowledge ANN replace skipped: candidate predates namespace's current generation"
);
return false;
}
idxs.insert(key.clone(), candidate);
unavailable_guard(&ann.unavailable).remove(key);
true
}
async fn has_current_index(ann: &SharedAnn, key: &AnnKey) -> bool {
let idxs = ann.indexes.read().await;
let current = current_generation(ann, &key.namespace);
idxs.get(key)
.is_some_and(|bridge| bridge.generation >= current)
}
async fn has_current_index_at_watermark(ann: &SharedAnn, key: &AnnKey, watermark: u64) -> bool {
let idxs = ann.indexes.read().await;
let current = current_generation(ann, &key.namespace);
idxs.get(key).is_some_and(|bridge| {
bridge.generation >= current && bridge.index.last_applied_seq().unwrap_or(0) >= watermark
})
}
fn warm_states_guard(
m: &std::sync::Mutex<HashMap<AnnKey, AnnWarmState>>,
) -> std::sync::MutexGuard<'_, HashMap<AnnKey, AnnWarmState>> {
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn begin_warm(ann: &SharedAnn, key: AnnKey) -> Option<AnnWarmPermit> {
let mut states = warm_states_guard(&ann.warm_states);
let generation = current_generation(ann, &key.namespace);
match states.get(&key) {
Some(
AnnWarmState::Warming {
generation: state_generation,
..
}
| AnnWarmState::Ready {
generation: state_generation,
},
) if *state_generation >= generation => return None,
Some(AnnWarmState::Failed {
generation: failed_generation,
error,
}) => {
tracing::debug!(
key = ?key,
failed_generation,
error = ?error,
"retrying failed knowledge ANN warm"
);
}
_ => {}
}
let attempt_id = ann.next_warm_attempt_id.fetch_add(1, Ordering::Relaxed);
states.insert(
key.clone(),
AnnWarmState::Warming {
attempt_id,
generation,
started_at: std::time::Instant::now(),
},
);
Some(AnnWarmPermit {
ann: ann.clone(),
key,
attempt_id,
generation,
finished: false,
})
}
fn finish_warm_state(permit: &mut AnnWarmPermit, next: AnnWarmState) {
let mut states = warm_states_guard(&permit.ann.warm_states);
let started_at = match states.get(&permit.key) {
Some(AnnWarmState::Warming {
attempt_id,
generation,
started_at,
}) if *attempt_id == permit.attempt_id && *generation == permit.generation => *started_at,
_ => {
permit.finished = true;
return;
}
};
tracing::debug!(
key = ?permit.key,
attempt_id = permit.attempt_id,
elapsed_ms = started_at.elapsed().as_millis(),
state = ?next,
"knowledge ANN warm finished"
);
states.insert(permit.key.clone(), next);
permit.finished = true;
}
async fn finish_warm(mut permit: AnnWarmPermit, outcome: AnnWarmOutcome) {
match outcome {
AnnWarmOutcome::Ready => {
let ann = permit.ann.clone();
let idxs = ann.indexes.read().await;
let next = match idxs
.get(&permit.key)
.map(|bridge| bridge.generation)
.filter(|generation| *generation >= permit.generation)
{
Some(generation) => AnnWarmState::Ready { generation },
None => AnnWarmState::Failed {
generation: permit.generation,
error: AnnWarmFailure::Operational,
},
};
#[cfg(test)]
run_finish_warm_ready_test_hook(&ann, &permit.key).await;
finish_warm_state(&mut permit, next);
drop(idxs);
}
AnnWarmOutcome::Empty => {
let next = AnnWarmState::Failed {
generation: permit.generation,
error: AnnWarmFailure::EmptyCorpus,
};
finish_warm_state(&mut permit, next);
}
AnnWarmOutcome::Failed => {
let next = AnnWarmState::Failed {
generation: permit.generation,
error: AnnWarmFailure::Operational,
};
finish_warm_state(&mut permit, next);
}
AnnWarmOutcome::Declined => {
let next = AnnWarmState::Failed {
generation: permit.generation,
error: AnnWarmFailure::NotWarmHost,
};
finish_warm_state(&mut permit, next);
}
}
}
#[cfg(test)]
async fn run_finish_warm_ready_test_hook(ann: &SharedAnn, key: &AnnKey) {
let hook = ann
.test_finish_warm_ready_hook
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let Some(hook) = hook else {
return;
};
let ann = ann.clone();
let key = key.clone();
let incumbent_digest = hook.incumbent_digest;
let generation = hook.generation;
let handle = tokio::spawn(async move {
evict_bridge_and_ready_state(&ann, &key, incumbent_digest, generation).await;
});
let _ = hook.handle_tx.send(handle);
tokio::task::yield_now().await;
}
impl Drop for AnnWarmPermit {
fn drop(&mut self) {
if !self.finished {
let next = AnnWarmState::Failed {
generation: self.generation,
error: AnnWarmFailure::Interrupted,
};
finish_warm_state(self, next);
}
}
}
#[cfg(test)]
impl AnnWarmPermit {
fn leave_in_flight_for_test(mut self) {
self.finished = true;
}
}
fn unavailable_guard(
m: &std::sync::Mutex<HashMap<AnnKey, u64>>,
) -> std::sync::MutexGuard<'_, HashMap<AnnKey, u64>> {
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn mark_unavailable(ann: &SharedAnn, key: &AnnKey, generation: u64) {
unavailable_guard(&ann.unavailable).insert(key.clone(), generation);
}
fn is_terminally_unavailable(ann: &SharedAnn, key: &AnnKey) -> bool {
let current = current_generation(ann, &key.namespace);
let mut guard = unavailable_guard(&ann.unavailable);
match guard.get(key) {
Some(&marked_generation) if marked_generation >= current => true,
Some(_) => {
guard.remove(key);
false
}
None => false,
}
}
#[cfg(test)]
pub(crate) async fn insert_ann_if_absent(ann: &SharedAnn, key: AnnKey, bridge: AnnBridge) -> bool {
use std::collections::hash_map::Entry;
let mut guard = ann.indexes.write().await;
match guard.entry(key) {
Entry::Occupied(_) => false,
Entry::Vacant(e) => {
e.insert(bridge);
true
}
}
}
pub(crate) async fn clear_namespace(ann: &SharedAnn, namespace: &str) {
let mut idxs = ann.indexes.write().await;
let mut states = warm_states_guard(&ann.warm_states);
idxs.retain(|k, _| k.namespace != namespace);
states.retain(|k, _| k.namespace != namespace);
bump_generation(ann, namespace);
}
#[cfg(test)]
pub(crate) async fn search_loaded(
ann: &SharedAnn,
key: &AnnKey,
query: &[f32],
k: usize,
) -> Option<Vec<(Uuid, f32)>> {
let guard = ann.indexes.read().await;
guard.get(key).map(|bridge| bridge.search(query, k))
}
pub(crate) async fn search_loaded_with_seq(
ann: &SharedAnn,
key: &AnnKey,
query: &[f32],
k: usize,
) -> Option<(Vec<(Uuid, f32)>, u64)> {
if !ann.indexes.read().await.contains_key(key) {
return None;
}
let ann = Arc::clone(ann);
let key = key.clone();
let query = query.to_vec();
tokio::task::spawn_blocking(move || {
let guard = ann.indexes.blocking_read();
guard.get(&key).map(|bridge| {
(
bridge.search(&query, k),
bridge.index.last_applied_seq().unwrap_or(0),
)
})
})
.await
.expect("loaded ANN traversal task panicked")
}
pub(crate) fn is_warming_not_loaded(ann: &SharedAnn, key: &AnnKey) -> bool {
let in_warming = {
let states = warm_states_guard(&ann.warm_states);
let generation = current_generation(ann, &key.namespace);
matches!(
states.get(key),
Some(AnnWarmState::Warming {
generation: state_generation,
..
}) if *state_generation >= generation
)
};
if !in_warming {
return false;
}
match ann.indexes.try_read() {
Ok(guard) => !guard.contains_key(key),
Err(_) => true,
}
}
pub(crate) async fn wait_ready(
ann: &SharedAnn,
key: &AnnKey,
timeout_ms: u64,
poll_ms: u64,
) -> bool {
let deadline = std::time::Instant::now() + std::time::Duration::from_millis(timeout_ms);
loop {
if ann.indexes.read().await.contains_key(key) {
return true;
}
if is_terminally_unavailable(ann, key) {
return false;
}
if std::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(std::time::Duration::from_millis(poll_ms)).await;
}
}
pub(crate) const ANN_WARM_WAIT_TIMEOUT_MS: u64 = 5_000;
pub(crate) const ANN_WARM_WAIT_POLL_MS: u64 = 50;
const ROTATION_WATCH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5);
static ANN_WARM_WAIT_TIMEOUT_OVERRIDE_MS: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
pub(crate) fn warm_wait_timeout_ms() -> u64 {
let o = ANN_WARM_WAIT_TIMEOUT_OVERRIDE_MS.load(std::sync::atomic::Ordering::Relaxed);
if o > 0 {
o
} else {
ANN_WARM_WAIT_TIMEOUT_MS
}
}
#[cfg(test)]
pub(crate) fn set_warm_wait_timeout_override_ms(ms: u64) {
ANN_WARM_WAIT_TIMEOUT_OVERRIDE_MS.store(ms, std::sync::atomic::Ordering::Relaxed);
}
impl AnnBridge {
fn from_core(core: AnnBridgeCore) -> Self {
Self {
core,
generation: 0,
#[cfg(test)]
drop_probe: None,
#[cfg(test)]
search_pause: None,
}
}
pub fn build(vectors: Vec<f32>, dim: usize, id_map: Vec<Uuid>) -> Result<Self, String> {
let core = AnnBridgeCore::build(vectors, dim, id_map)?;
Ok(Self::from_core(core))
}
pub(crate) fn set_applied_seq(&mut self, seq: u64) {
self.index.set_last_applied_seq(Some(seq));
}
pub(crate) fn reverse_map_for(
&self,
subjects: impl IntoIterator<Item = Uuid>,
) -> HashMap<Uuid, u32> {
let wanted: Vec<Uuid> = subjects.into_iter().collect();
let wanted_set = (wanted.len() > 4).then(|| wanted.iter().copied().collect::<HashSet<_>>());
let mut reverse: HashMap<Uuid, u32> =
HashMap::with_capacity(wanted.len().min(self.index.live_count()));
for (ordinal, uuid) in self.id_map.iter().enumerate() {
let in_tail = match &wanted_set {
Some(set) => set.contains(uuid),
None => wanted.contains(uuid),
};
if !in_tail || self.index.is_tombstoned(ordinal as u32) {
continue;
}
reverse.insert(*uuid, ordinal as u32);
}
reverse
}
pub(crate) fn apply_final_op(
&mut self,
reverse: &mut HashMap<Uuid, u32>,
uuid: Uuid,
op: Option<Vec<f32>>,
) -> Result<(), String> {
match op {
None => {
if let Some(&ordinal) = reverse.get(&uuid) {
if self.id_map.get(ordinal as usize) != Some(&uuid) {
tracing::warn!(
subject = %uuid,
ordinal,
"replay delete: ordinal reassigned within batch, skipping tombstone"
);
reverse.remove(&uuid);
return Ok(());
}
self.index
.tombstone(ordinal)
.map_err(|e| format!("replay tombstone({ordinal}): {e}"))?;
reverse.remove(&uuid);
}
}
Some(mut embedding) => {
l2_normalize(&mut embedding);
if let Some(&old) = reverse.get(&uuid) {
self.index
.tombstone(old)
.map_err(|e| format!("replay tombstone({old}): {e}"))?;
}
let ordinal = self
.index
.insert(&embedding)
.map_err(|e| format!("replay insert: {e}"))?;
let slot = ordinal as usize;
match slot.cmp(&self.id_map.len()) {
std::cmp::Ordering::Less => self.id_map[slot] = uuid,
std::cmp::Ordering::Equal => self.id_map.push(uuid),
std::cmp::Ordering::Greater => {
return Err(format!(
"replay insert returned ordinal {ordinal} beyond id_map len {}",
self.id_map.len()
));
}
}
reverse.insert(uuid, ordinal);
}
}
Ok(())
}
pub fn search(&self, query: &[f32], k: usize) -> Vec<(Uuid, f32)> {
#[cfg(test)]
if let Some(pause) = &self.search_pause {
pause.wait();
}
match self.core.search_hits(query, k) {
Ok(hits) => hits
.into_iter()
.map(|(uuid, dist)| {
let cosine = 1.0 - dist / 2.0;
(uuid, cosine.max(0.0))
})
.collect(),
Err(e) => {
tracing::warn!(error = %e, "vamana ANN search failed");
Vec::new()
}
}
}
pub fn num_vectors(&self) -> usize {
self.index.num_vectors()
}
pub fn from_vamana_snapshot(snapshot: VamanaSnapshot) -> Result<Self, String> {
let id_map: Vec<Uuid> = snapshot
.external_ids
.iter()
.map(|s| Uuid::parse_str(s).map_err(|e| format!("bad UUID {s}: {e}")))
.collect::<Result<_, _>>()?;
let index =
VamanaIndex::from_snapshot(&snapshot).map_err(|e| format!("snapshot restore: {e}"))?;
let core = AnnBridgeCore {
index,
id_map,
commit_digest: None,
};
Ok(Self::from_core(core))
}
#[allow(dead_code)]
pub fn save_atomic(&self, dir: &std::path::Path) -> Result<(), String> {
let _publication_lock =
khive_retrieval::ann::acquire_checkpoint_lock(dir, BRIDGE_LOCK_MESSAGE_PREFIX)?;
self.save_atomic_locked(dir)
}
fn save_atomic_locked(&self, dir: &std::path::Path) -> Result<(), String> {
self.core.save_atomic(dir)?;
Ok(())
}
#[allow(dead_code)]
pub fn load(dir: &std::path::Path) -> Result<Self, String> {
let (core, _commit_digest) = AnnBridgeCore::load(dir)?;
Ok(Self::from_core(core))
}
}
const BRIDGE_LOCK_MESSAGE_PREFIX: &str = "ANN bridge";
mod segment_key;
pub(crate) use segment_key::snapshot_key;
use segment_key::{ann_segment_dir, ann_segment_dir_from_root, decode_ann_dir_name};
pub(crate) use khive_runtime::config::sanitize_key as sanitize_model_key;
#[cfg(test)]
pub(crate) fn persist_ann_v2(
rt: &KhiveRuntime,
ns: &str,
model: &str,
bridge: &AnnBridge,
) -> Result<(), String> {
match ann_segment_dir(rt, ns, model) {
Some(dir) => bridge.save_atomic(&dir),
None => Ok(()), }
}
fn persist_ann_v2_locked(
rt: &KhiveRuntime,
ns: &str,
model: &str,
bridge: &AnnBridge,
) -> Result<(), String> {
match ann_segment_dir(rt, ns, model) {
Some(dir) => bridge.save_atomic_locked(&dir),
None => Ok(()),
}
}
const ANN_CONSUMER: &str = "knowledge:knowledge.atom";
#[cfg(test)]
async fn register_consumer(rt: &KhiveRuntime, ns: &str, model: &str) -> Result<(), String> {
let sql = rt.sql();
ann_registry::register_pending_dispatch(
sql.as_ref(),
"knowledge_",
ANN_CONSUMER,
ns,
model,
ann_segment_dir(rt, ns, model).is_none(),
)
.await
.map_err(|e| e.to_string())
}
async fn write_force_rebuild_sentinel_row(rt: &KhiveRuntime, key: &AnnKey) -> Result<(), String> {
let sql = rt.sql();
if ann_segment_dir(rt, &key.namespace, &key.model).is_none() {
let mut writer = sql.writer().await.map_err(|error| error.to_string())?;
writer
.execute(ann_registry::pathless_mark_recovering(
"knowledge_",
ANN_CONSUMER,
&key.namespace,
&key.model,
))
.await
.map_err(|error| error.to_string())?;
return Ok(());
}
ann_registry::mark_recovering(sql.as_ref(), ANN_CONSUMER, &key.namespace, &key.model)
.await
.map_err(|error| error.to_string())
}
async fn write_force_rebuild_sentinel(rt: &KhiveRuntime, key: &AnnKey) -> Result<(), String> {
let _publication_lock = match ann_segment_dir(rt, &key.namespace, &key.model) {
Some(dir) => Some(
khive_retrieval::ann::acquire_checkpoint_lock_async(dir, BRIDGE_LOCK_MESSAGE_PREFIX)
.await?,
),
None => None,
};
if matches!(
read_own_watermark(rt, &key.namespace, &key.model).await?,
Some(watermark) if watermark >= 0
) {
return Ok(());
}
write_force_rebuild_sentinel_row(rt, key).await
}
async fn prepare_authoritative_rebuild(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
) -> Result<(), String> {
mark_force_rebuild(ann, key);
let local_publication_lock = checkpoint_lock(ann, key);
let _local_publication_guard = local_publication_lock.lock().await;
write_force_rebuild_sentinel(rt, key).await
}
async fn read_own_watermark(
rt: &KhiveRuntime,
ns: &str,
model: &str,
) -> Result<Option<i64>, String> {
let sql = rt.sql();
ann_registry::read_watermark(sql.as_ref(), "", ANN_CONSUMER, ns, model)
.await
.map_err(|e| e.to_string())
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum CheckpointAuthority {
Incremental,
FullRegistered,
FullSentinel,
}
pub(crate) async fn prepare_full_corpus_scan(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
) -> Result<CheckpointAuthority, String> {
match read_own_watermark(rt, &key.namespace, &key.model).await? {
Some(watermark) if watermark >= 0 => Ok(CheckpointAuthority::FullRegistered),
Some(watermark) if watermark == ann_registry::RECOVERING_WATERMARK => {
mark_force_rebuild(ann, key);
Ok(CheckpointAuthority::FullSentinel)
}
Some(_) | None => {
prepare_authoritative_rebuild(rt, ann, key).await?;
Ok(CheckpointAuthority::FullSentinel)
}
}
}
async fn raise_watermark(
rt: &KhiveRuntime,
ns: &str,
model: &str,
s: u64,
authority: CheckpointAuthority,
) -> Result<(), String> {
let shared_authority = match authority {
CheckpointAuthority::FullSentinel => WatermarkAuthority::Recovering,
CheckpointAuthority::Incremental => WatermarkAuthority::Active,
CheckpointAuthority::FullRegistered => WatermarkAuthority::PendingOrActive,
};
let sql = rt.sql();
let pathless = ann_segment_dir(rt, ns, model).is_none();
if pathless {
i64::try_from(s)
.map_err(|_| format!("knowledge ANN watermark {s} exceeds SQLite INTEGER range"))?;
}
let raised = ann_registry::raise_watermark_dispatch(
sql.as_ref(),
"knowledge_",
ANN_CONSUMER,
ns,
model,
s,
shared_authority,
pathless,
)
.await
.map_err(|e| e.to_string())?;
if !raised {
return Err(format!(
"ANN watermark publication fence rejected {authority:?}: affected 0 rows"
));
}
Ok(())
}
fn knowledge_corpus(ns: &str) -> CorpusScope<'_> {
CorpusScope {
namespace: Some(ns),
record_kind: None,
field: "knowledge.atom",
live_join: None,
watermark_capture: WatermarkCapture::LogHighWater,
}
}
async fn compact_log(rt: &KhiveRuntime, ns: &str, model: &str) -> Result<(), String> {
let sql = rt.sql();
ann_registry::compact_dispatch(
sql.as_ref(),
"knowledge_",
knowledge_corpus(ns).compaction_scope(),
model,
ann_segment_dir(rt, ns, model).is_none(),
)
.await
.map_err(|e| e.to_string())
}
async fn tail_exists(rt: &KhiveRuntime, ns: &str, model: &str, s: u64) -> Result<bool, String> {
let sql = rt.sql();
let mut reader = sql.reader().await.map_err(|e| e.to_string())?;
let rows = reader
.query_all(knowledge_corpus(ns).tail_exists(model, s, "ann_tail_probe"))
.await
.map_err(|e| e.to_string())?;
match rows.first().and_then(|r| r.get("has_tail")) {
Some(SqlValue::Integer(n)) => Ok(*n != 0),
other => Err(format!("tail probe: unexpected value {other:?}")),
}
}
async fn scope_counts(
rt: &KhiveRuntime,
ns: &str,
model: &str,
s: u64,
) -> Result<(u64, u64), String> {
let table_name = format!("vec_{}", sanitize_model_key(model));
let sql = rt.sql();
let mut reader = sql.reader().await.map_err(|e| e.to_string())?;
let rows = reader
.query_all(knowledge_corpus(ns).scope_counts(&table_name, model, s, "ann_scope_counts"))
.await
.map_err(|e| e.to_string())?;
let row = rows
.into_iter()
.next()
.ok_or("scope_counts returned no row")?;
let get = |col: &str| match row.get(col) {
Some(SqlValue::Integer(n)) => u64::try_from(*n).map_err(|_| format!("negative {col}")),
other => Err(format!("scope_counts {col}: unexpected value {other:?}")),
};
Ok((get("live")?, get("tail")?))
}
struct ClassificationScopeCounts {
live_lower_bound: u64,
tail: u64,
live_count_exact: bool,
}
async fn classification_scope_counts(
rt: &KhiveRuntime,
ns: &str,
model: &str,
s: u64,
threshold: f64,
) -> Result<ClassificationScopeCounts, String> {
let multiplier = (2.0 / threshold).ceil();
let multiplier = if multiplier.is_finite() && multiplier < i64::MAX as f64 {
SqlValue::Integer(multiplier as i64)
} else {
SqlValue::Null
};
let table_name = format!("vec_{}", sanitize_model_key(model));
let sql = rt.sql();
let mut reader = sql.reader().await.map_err(|e| e.to_string())?;
let rows = reader
.query_all(knowledge_corpus(ns).classification_scope_counts(
&table_name,
model,
s,
multiplier,
"ann_classification_scope_counts",
))
.await
.map_err(|e| e.to_string())?;
let row = rows
.into_iter()
.next()
.ok_or("classification_scope_counts returned no row")?;
let get = |col: &str| match row.get(col) {
Some(SqlValue::Integer(n)) => u64::try_from(*n).map_err(|_| format!("negative {col}")),
other => Err(format!(
"classification_scope_counts {col}: unexpected value {other:?}"
)),
};
let live_lower_bound = get("live")?;
let tail = get("tail")?;
let cap = match row.get("cap") {
Some(SqlValue::Integer(n)) => *n,
other => return Err(format!("classification_scope_counts cap: {other:?}")),
};
Ok(ClassificationScopeCounts {
live_lower_bound,
tail,
live_count_exact: cap < 0 || live_lower_bound < cap as u64,
})
}
async fn fetch_final_states(
rt: &KhiveRuntime,
ns: &str,
model: &str,
s: u64,
) -> Result<(Vec<(Uuid, bool)>, u64), String> {
let sql = rt.sql();
let mut reader = sql.reader().await.map_err(|e| e.to_string())?;
let rows = reader
.query_all(SqlStatement {
sql: khive_runtime::sql!("knowledge_ann_final_states").into(),
params: vec![
SqlValue::Text(ns.to_owned()),
SqlValue::Text(model.to_owned()),
SqlValue::Integer(s as i64),
],
label: Some("ann_fetch_final_states".into()),
})
.await
.map_err(|e| e.to_string())?;
let mut new_s = s;
let mut finals: Vec<(Uuid, bool)> = Vec::with_capacity(rows.len());
for row in &rows {
let seq = match row.get("seq") {
Some(SqlValue::Integer(n)) => *n,
_ => return Err("ann_write_log.seq: unexpected value".into()),
};
new_s = new_s.max(u64::try_from(seq).map_err(|_| "negative seq")?);
let uuid = match row.get("subject_id") {
Some(SqlValue::Text(t)) => {
Uuid::parse_str(t).map_err(|e| format!("tail subject_id {t}: {e}"))?
}
_ => return Err("ann_write_log.subject_id: unexpected value".into()),
};
let is_delete = match row.get("op") {
Some(SqlValue::Text(t)) => t == "delete",
_ => return Err("ann_write_log.op: unexpected value".into()),
};
finals.push((uuid, is_delete));
}
Ok((finals, new_s))
}
const REPLAY_BATCH: usize = 500;
async fn replay_final_states(
rt: &KhiveRuntime,
token: &NamespaceToken,
bridge: &mut AnnBridge,
ns: &str,
model: &str,
finals: &[(Uuid, bool)],
) -> Result<(), String> {
let mut store: Option<Arc<dyn khive_storage::VectorStore>> = None;
let mut reverse = bridge.reverse_map_for(finals.iter().map(|(uuid, _)| *uuid));
for batch in finals.chunks(REPLAY_BATCH) {
let ids: Vec<Uuid> = batch
.iter()
.filter_map(|(id, delete)| (!*delete).then_some(*id))
.collect();
if store.is_none() && !ids.is_empty() {
if rt
.resolve_embedding_model(Some(model))
.is_ok_and(|resolved| resolved.to_string() != model)
{
return Err(format!(
"replay model {model} is not canonical (contradiction -> Cold)"
));
}
store = Some(
rt.vectors_for_model(token, model)
.map_err(|e| e.to_string())?,
);
}
let mut embeddings = match &store {
Some(store) if !ids.is_empty() => store
.get_vectors(&ids, ns, "knowledge.atom")
.await
.map_err(|e| e.to_string())?,
_ => HashMap::new(),
};
for uuid in &ids {
if !embeddings.contains_key(uuid) {
return Err(format!(
"final upsert for {uuid} has no source row (contradiction → Cold)"
));
}
}
for (uuid, is_delete) in batch {
let op =
if *is_delete {
None
} else {
Some(embeddings.remove(uuid).ok_or_else(|| {
format!("final upsert for {uuid}: embedding lost in batch")
})?)
};
bridge.apply_final_op(&mut reverse, *uuid, op)?;
}
}
Ok(())
}
struct FreshTailSnapshot {
own_watermark: Option<i64>,
registry_min: Option<i64>,
live_count: Option<u64>,
ops: Vec<(Uuid, Option<Vec<f32>>)>,
}
fn fresh_tail_snapshot_statement(
ns: &str,
model: &str,
watermark: i64,
live_threshold: Option<f64>,
) -> SqlStatement {
let table_name = format!("vec_{}", sanitize_model_key(model));
knowledge_corpus(ns).final_tail(
&table_name,
model,
watermark,
live_threshold,
khive_retrieval::ann::corpus::TailFloor::RegistryMinimum {
registry_namespace: ns,
consumer: ANN_CONSUMER,
},
"knowledge_ann_fresh_tail_snapshot",
)
}
async fn fetch_fresh_tail_snapshot(
rt: &KhiveRuntime,
ns: &str,
model: &str,
watermark: u64,
live_threshold: Option<f64>,
) -> Result<FreshTailSnapshot, String> {
let watermark = i64::try_from(watermark)
.map_err(|_| "fresh-tail watermark exceeds SQLite INTEGER range".to_string())?;
let statement = fresh_tail_snapshot_statement(ns, model, watermark, live_threshold);
let sql = rt.sql();
let mut reader = sql.reader().await.map_err(|error| error.to_string())?;
let rows = reader
.query_all(statement)
.await
.map_err(|error| error.to_string())?;
let first = rows
.first()
.ok_or_else(|| "fresh-tail snapshot returned no registry row".to_string())?;
let read_optional_i64 = |column: &str| -> Result<Option<i64>, String> {
match first.get(column) {
Some(SqlValue::Integer(value)) => Ok(Some(*value)),
Some(SqlValue::Null) | None => Ok(None),
other => Err(format!("fresh-tail {column}: unexpected value {other:?}")),
}
};
let own_watermark = read_optional_i64("own_watermark")?;
let registry_min = read_optional_i64("registry_min")?;
let live_count = match read_optional_i64("live_count")? {
Some(value) => {
Some(u64::try_from(value).map_err(|_| "negative fresh-tail live_count".to_string())?)
}
None => None,
};
type RawVector = (
Option<String>,
Option<String>,
Option<String>,
Option<Vec<u8>>,
);
let mut finals: Vec<(Uuid, bool, RawVector)> = Vec::new();
let mut index_by_id: HashMap<Uuid, usize> = HashMap::new();
for row in &rows {
let Some(SqlValue::Integer(_)) = row.get("seq") else {
continue;
};
let subject = match row.get("subject_id") {
Some(SqlValue::Text(value)) => Uuid::parse_str(value)
.map_err(|error| format!("fresh-tail subject_id {value}: {error}"))?,
other => return Err(format!("fresh-tail subject_id: unexpected value {other:?}")),
};
let is_delete = match row.get("op") {
Some(SqlValue::Text(value)) if value == "delete" => true,
Some(SqlValue::Text(value)) if value == "upsert" => false,
other => return Err(format!("fresh-tail op: unexpected value {other:?}")),
};
let raw_vector = (
match row.get("vector_namespace") {
Some(SqlValue::Text(value)) => Some(value.clone()),
_ => None,
},
match row.get("vector_model") {
Some(SqlValue::Text(value)) => Some(value.clone()),
_ => None,
},
match row.get("vector_field") {
Some(SqlValue::Text(value)) => Some(value.clone()),
_ => None,
},
match row.get("embedding") {
Some(SqlValue::Blob(value)) => Some(value.clone()),
_ => None,
},
);
match index_by_id.get(&subject) {
Some(&index) => finals[index] = (subject, is_delete, raw_vector),
None => {
index_by_id.insert(subject, finals.len());
finals.push((subject, is_delete, raw_vector));
}
}
}
let mut ops = Vec::with_capacity(finals.len());
for (subject, is_delete, (vector_ns, vector_model, vector_field, embedding)) in finals {
if is_delete {
ops.push((subject, None));
continue;
}
let in_scope = vector_ns.as_deref() == Some(ns)
&& vector_model.as_deref() == Some(model)
&& vector_field.as_deref() == Some("knowledge.atom");
if !in_scope {
return Err(format!(
"fresh-tail upsert {subject}: vector row outside consumer scope"
));
}
let Some(bytes) = embedding else {
return Err(format!(
"fresh-tail upsert {subject}: embedding is not a blob"
));
};
let embedding = khive_storage::decode_f32_native(&bytes).map_err(|_| {
format!(
"fresh-tail upsert {subject}: malformed embedding byte length {}",
bytes.len()
)
})?;
ops.push((subject, Some(embedding)));
}
Ok(FreshTailSnapshot {
own_watermark,
registry_min,
live_count,
ops,
})
}
fn exact_cosine(query: &[f32], embedding: &[f32]) -> f32 {
if query.len() != embedding.len() || query.is_empty() {
return 0.0;
}
let mut query = query.to_vec();
let mut embedding = embedding.to_vec();
l2_normalize(&mut query);
l2_normalize(&mut embedding);
query
.iter()
.zip(embedding.iter())
.map(|(left, right)| left * right)
.sum::<f32>()
.max(0.0)
}
pub(crate) fn merge_fresh_tail(
candidates: Vec<(Uuid, f32)>,
query: &[f32],
ops: Vec<(Uuid, Option<Vec<f32>>)>,
) -> Vec<(Uuid, f32)> {
khive_retrieval::ann::merge_fresh_tail(candidates, ops, |embedding| {
Ok::<_, std::convert::Infallible>(exact_cosine(query, embedding))
})
.unwrap_or_else(|never| match never {})
}
pub(crate) async fn merge_fresh_tail_off_thread(
candidates: Vec<(Uuid, f32)>,
query: &[f32],
ops: Vec<(Uuid, Option<Vec<f32>>)>,
) -> Vec<(Uuid, f32)> {
if ops.is_empty() {
return candidates;
}
let query = query.to_vec();
tokio::task::spawn_blocking(move || merge_fresh_tail(candidates, &query, ops))
.await
.expect("ANN fresh-tail merge task panicked")
}
pub(crate) enum FreshTailOutcome {
Ops(Vec<(Uuid, Option<Vec<f32>>)>),
Replace {
candidates: Vec<(Uuid, f32)>,
source_exhausted: bool,
},
Skipped,
}
pub(crate) async fn fresh_tail_leg(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
query: &[f32],
k: usize,
watermark: Option<u64>,
) -> FreshTailOutcome {
let ns = key.namespace.as_str();
let model = key.model.as_str();
if force_rebuild_required(ann, key) {
return FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
};
}
if !rt.ann_fresh_tail_enabled() {
return match read_own_watermark(rt, ns, model).await {
Ok(Some(watermark)) if watermark >= 0 => FreshTailOutcome::Skipped,
Ok(_) => force_cold_after_registry_loss(rt, ann, key).await,
Err(error) => {
tracing::warn!(
error = %error,
namespace = ns,
model,
"knowledge ANN registry read failed while fresh-tail was disabled"
);
FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
}
}
};
}
match watermark {
Some(watermark) => fresh_tail_serving(rt, ann, key, query, k, watermark).await,
None => fresh_tail_capped(rt, ann, key).await,
}
}
async fn force_cold_after_registry_loss(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
) -> FreshTailOutcome {
if rt.is_read_only() {
clear_namespace(ann, &key.namespace).await;
return FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
};
}
if let Err(error) = prepare_authoritative_rebuild(rt, ann, key).await {
tracing::warn!(
error = %error,
namespace = key.namespace,
model = key.model,
"knowledge ANN failed to publish registry-loss rebuild sentinel"
);
}
clear_namespace(ann, &key.namespace).await;
FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
}
}
fn ready_snapshot_watermark(snapshot: &FreshTailSnapshot) -> Option<u64> {
snapshot
.own_watermark
.filter(|watermark| *watermark >= 0)
.and_then(|watermark| u64::try_from(watermark).ok())
}
fn nonnegative_registry_min(snapshot: &FreshTailSnapshot) -> u64 {
snapshot
.registry_min
.filter(|watermark| *watermark >= 0)
.and_then(|watermark| u64::try_from(watermark).ok())
.unwrap_or(0)
}
async fn fresh_tail_serving(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
query: &[f32],
k: usize,
watermark: u64,
) -> FreshTailOutcome {
let ns = key.namespace.as_str();
let model = key.model.as_str();
let snapshot = match fetch_fresh_tail_snapshot(rt, ns, model, watermark, None).await {
Ok(snapshot) => snapshot,
Err(error) => {
tracing::warn!(
error = %error,
namespace = ns,
model,
"knowledge fresh-tail snapshot failed; dropping stale vector leg"
);
return FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
};
}
};
let Some(_own_watermark) = ready_snapshot_watermark(&snapshot) else {
return force_cold_after_registry_loss(rt, ann, key).await;
};
let registry_min = nonnegative_registry_min(&snapshot);
if registry_min > watermark {
let published_watermark = ann_segment_dir(rt, ns, model)
.and_then(|dir| read_commit_info(&dir).ok().flatten())
.and_then(|info| info.last_applied_seq)
.filter(|published| *published >= registry_min);
match published_watermark {
Some(published_watermark) => {
clear_namespace(ann, ns).await;
return fresh_tail_reresolve(rt, ann, key, query, k, published_watermark).await;
}
None => {
clear_namespace(ann, ns).await;
return FreshTailOutcome::Replace {
candidates: merge_fresh_tail_off_thread(Vec::new(), query, snapshot.ops).await,
source_exhausted: true,
};
}
}
}
FreshTailOutcome::Ops(snapshot.ops)
}
const FRESH_TAIL_RERESOLVE_MAX_ROUNDS: u32 = 3;
async fn fresh_tail_reresolve(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
query: &[f32],
k: usize,
published_watermark: u64,
) -> FreshTailOutcome {
let ns = key.namespace.as_str();
let model = key.model.as_str();
let mut expected_watermark = published_watermark;
for round in 1..=FRESH_TAIL_RERESOLVE_MAX_ROUNDS {
let Some(dir) = ann_segment_dir(rt, ns, model) else {
tracing::warn!(
key = ?key,
namespace = ns,
model,
"knowledge fresh-tail published segment disappeared during re-resolution"
);
return FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
};
};
let query_for_search = query.to_vec();
let loaded = tokio::task::spawn_blocking(move || {
let bridge = AnnBridge::load(&dir)?;
let loaded_watermark = bridge.index.last_applied_seq();
let candidates = bridge.search(&query_for_search, k);
Ok::<_, String>((candidates, loaded_watermark))
})
.await;
let (candidates, loaded_watermark) = match loaded {
Ok(Ok(loaded)) => loaded,
Ok(Err(error)) => {
tracing::warn!(
error = %error,
key = ?key,
namespace = ns,
model,
"knowledge fresh-tail published segment load failed"
);
return FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
};
}
Err(error) => {
tracing::warn!(
error = %error,
key = ?key,
namespace = ns,
model,
"knowledge fresh-tail segment worker failed during re-resolution"
);
return FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
};
}
};
let loaded_watermark = loaded_watermark.unwrap_or(expected_watermark);
let source_exhausted = candidates.len() < k;
let snapshot = match fetch_fresh_tail_snapshot(rt, ns, model, loaded_watermark, None).await
{
Ok(snapshot) => snapshot,
Err(error) => {
tracing::warn!(
error = %error,
key = ?key,
namespace = ns,
model,
"knowledge fresh-tail re-resolution snapshot failed"
);
return FreshTailOutcome::Replace {
candidates: Vec::new(),
source_exhausted: true,
};
}
};
let Some(_own_watermark) = ready_snapshot_watermark(&snapshot) else {
return force_cold_after_registry_loss(rt, ann, key).await;
};
let registry_min = nonnegative_registry_min(&snapshot);
if registry_min <= loaded_watermark {
return FreshTailOutcome::Replace {
candidates: merge_fresh_tail_off_thread(candidates, query, snapshot.ops).await,
source_exhausted,
};
}
if round == FRESH_TAIL_RERESOLVE_MAX_ROUNDS {
tracing::warn!(
key = ?key,
namespace = ns,
model,
rounds = round,
floor = registry_min,
"knowledge fresh-tail re-resolution did not converge; using exact-only floored suffix"
);
return FreshTailOutcome::Replace {
candidates: merge_fresh_tail_off_thread(Vec::new(), query, snapshot.ops).await,
source_exhausted: true,
};
}
expected_watermark = registry_min;
}
unreachable!("fresh-tail re-resolution loop returns within its bounded rounds")
}
async fn fresh_tail_capped(rt: &KhiveRuntime, ann: &SharedAnn, key: &AnnKey) -> FreshTailOutcome {
let snapshot = match fetch_fresh_tail_snapshot(
rt,
&key.namespace,
&key.model,
0,
Some(ann_rebuild_threshold()),
)
.await
{
Ok(snapshot) => snapshot,
Err(error) => {
tracing::warn!(
error = %error,
namespace = key.namespace,
model = key.model,
"knowledge capped fresh-tail fetch failed"
);
return FreshTailOutcome::Skipped;
}
};
if ready_snapshot_watermark(&snapshot).is_none() {
return force_cold_after_registry_loss(rt, ann, key).await;
}
debug_assert!(snapshot.live_count.is_some());
FreshTailOutcome::Ops(snapshot.ops)
}
async fn adopt_checkpoint_winner(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
generation: u64,
observed_watermark: u64,
) -> bool {
let installed = match ann_segment_dir(rt, &key.namespace, &key.model) {
Some(dir) => match AnnBridge::load(&dir) {
Ok(bridge) if bridge.index.last_applied_seq().unwrap_or(0) >= observed_watermark => {
ann.indexes.write().await.remove(key);
install_replacing(ann, key, bridge.with_generation(generation)).await
}
Ok(bridge) => {
tracing::warn!(
key = ?key,
bridge_watermark = bridge.index.last_applied_seq().unwrap_or(0),
observed_watermark,
"checkpoint winner segment trails its durable watermark"
);
ann.indexes.write().await.remove(key);
false
}
Err(error) => {
tracing::warn!(
error = %error,
key = ?key,
"failed to adopt checkpoint race winner"
);
ann.indexes.write().await.remove(key);
false
}
},
None => {
let current = has_current_index_at_watermark(ann, key, observed_watermark).await;
if !current {
ann.indexes.write().await.remove(key);
}
current
}
};
clear_force_rebuild(ann, key);
installed
}
pub(crate) async fn checkpoint_raise_compact_readopt(
rt: &KhiveRuntime,
ann: &SharedAnn,
key: &AnnKey,
bridge: AnnBridge,
generation: u64,
authority: CheckpointAuthority,
) -> bool {
let ns = key.namespace.as_str();
let model = key.model.as_str();
let local_publication_lock = checkpoint_lock(ann, key);
let _local_publication_guard = local_publication_lock.lock().await;
let publication_lock = match ann_segment_dir(rt, ns, model) {
Some(dir) => match khive_retrieval::ann::acquire_checkpoint_lock_async(
dir,
BRIDGE_LOCK_MESSAGE_PREFIX,
)
.await
{
Ok(lock) => Some(lock),
Err(error) => {
tracing::warn!(error = %error, "failed to acquire ANN checkpoint lock");
return false;
}
},
None => None,
};
let applied = bridge.index.last_applied_seq().unwrap_or(0);
let own_watermark = match read_own_watermark(rt, ns, model).await {
Ok(value) => value,
Err(error) => {
tracing::warn!(error = %error, "ann checkpoint registry read failed");
return false;
}
};
if authority == CheckpointAuthority::FullSentinel {
if let Some(observed) = own_watermark.filter(|watermark| *watermark >= 0) {
tracing::info!(
key = ?key,
observed_watermark = observed,
"authoritative ANN rebuild lost publication race; adopting winner"
);
let observed = u64::try_from(observed).unwrap_or(0);
return adopt_checkpoint_winner(rt, ann, key, generation, observed).await;
}
} else if let Some(observed) = own_watermark.filter(|watermark| *watermark >= 0) {
let observed = u64::try_from(observed).unwrap_or(0);
if observed > applied {
tracing::info!(
key = ?key,
candidate_watermark = applied,
observed_watermark = observed,
"stale ANN checkpoint lost publication race; adopting winner"
);
return adopt_checkpoint_winner(rt, ann, key, generation, observed).await;
}
}
let authorized = match authority {
CheckpointAuthority::FullSentinel => own_watermark == Some(-1),
CheckpointAuthority::Incremental | CheckpointAuthority::FullRegistered => {
own_watermark.is_some_and(|watermark| watermark >= 0)
}
};
if !authorized {
mark_force_rebuild(ann, key);
if let Err(error) = write_force_rebuild_sentinel_row(rt, key).await {
tracing::warn!(error = %error, "failed to fence unauthorized ANN checkpoint");
}
return false;
}
if let Err(e) = persist_ann_v2_locked(rt, ns, model, &bridge) {
tracing::error!(error = %e, "failed to persist v2 Vamana segment");
install_replacing(ann, key, bridge.with_generation(generation)).await;
return false;
}
let published = match ann_segment_dir(rt, ns, model) {
Some(dir) => match AnnBridge::load(&dir) {
Ok(mmap_bridge) => mmap_bridge,
Err(e) => {
tracing::warn!(error = %e, "mmap re-adoption failed; serving Owned build");
bridge
}
},
None => bridge,
};
let file_backed = publication_lock.is_some();
let mut pending_in_memory = Some(published.with_generation(generation));
let mut installed = false;
if file_backed {
installed = install_replacing(
ann,
key,
pending_in_memory.take().expect("published bridge"),
)
.await;
}
if let Err(e) = raise_watermark(rt, ns, model, applied, authority).await {
tracing::warn!(error = %e, "ann watermark raise failed (under-compacts; safe)");
match read_own_watermark(rt, ns, model).await {
Ok(Some(observed)) if observed >= 0 => {
let observed = u64::try_from(observed).unwrap_or(0);
if observed > applied {
return adopt_checkpoint_winner(rt, ann, key, generation, observed).await;
}
if let Some(published) = pending_in_memory.take() {
installed = install_replacing(ann, key, published).await;
}
clear_force_rebuild(ann, key);
return installed || has_current_index_at_watermark(ann, key, observed).await;
}
Ok(_) => {}
Err(read_error) => {
tracing::warn!(
error = %read_error,
"failed to reconcile ANN checkpoint race"
);
}
}
mark_force_rebuild(ann, key);
if let Err(error) = write_force_rebuild_sentinel_row(rt, key).await {
tracing::warn!(error = %error, "failed to restore ANN checkpoint fence");
}
return false;
}
if let Some(published) = pending_in_memory {
installed = install_replacing(ann, key, published).await;
}
if authority != CheckpointAuthority::Incremental {
clear_force_rebuild_if_current(ann, key, generation);
}
drop(publication_lock);
if let Err(e) = compact_log(rt, ns, model).await {
tracing::warn!(error = %e, "ann log compaction failed (retries next checkpoint)");
}
installed
}
async fn try_load_snapshot(
rt: &KhiveRuntime,
namespace: &str,
model: &str,
) -> Option<VamanaSnapshot> {
let key = snapshot_key(namespace, model);
let sql = rt.sql();
let mut reader = sql.reader().await.ok()?;
let rows = reader
.query_all(SqlStatement {
sql: khive_runtime::sql!("knowledge_legacy_snapshot_load").into(),
params: vec![SqlValue::Text(key), SqlValue::Text("vamana".into())],
label: None,
})
.await
.ok()?;
let row = rows.into_iter().next()?;
let blob = match row.get("snapshot")? {
SqlValue::Blob(b) => b.clone(),
_ => return None,
};
serde_json::from_slice::<VamanaSnapshot>(&blob).ok()
}
pub(crate) async fn compute_fingerprint(
rt: &KhiveRuntime,
token: &NamespaceToken,
model: &str,
) -> Option<CorpusFingerprint> {
let store = rt.vectors_for_model(token, model).ok()?;
let info = store.info().await.ok()?;
Some(CorpusFingerprint {
vector_count: info.entry_count,
dimensions: info.dimensions as u32,
})
}
async fn scan_corpus_raw(
rt: &KhiveRuntime,
token: &NamespaceToken,
model: &str,
) -> Result<Option<(Vec<f32>, Vec<Uuid>, u64)>, RuntimeError> {
let store = rt
.vectors_for_model(token, model)
.map_err(|e| RuntimeError::Internal(e.to_string()))?;
let info = store
.info()
.await
.map_err(|e| RuntimeError::Internal(e.to_string()))?;
let count = info.entry_count;
let dims = info.dimensions;
if count == 0 || dims == 0 {
return Ok(None);
}
let ns = token.namespace().as_str().to_owned();
let model_key = sanitize_model_key(model);
let table_name = format!("vec_{model_key}");
let model_str = model.to_owned();
let sql = rt.sql();
let mut reader = sql
.reader()
.await
.map_err(|e| RuntimeError::Internal(e.to_string()))?;
let rows = reader
.query_all(knowledge_corpus(&ns).corpus_scan(&table_name, &model_str, "vamana_corpus_scan"))
.await
.map_err(|e| RuntimeError::Internal(e.to_string()))?;
if rows.is_empty() {
return Ok(None);
}
let mut id_map: Vec<Uuid> = Vec::with_capacity(rows.len());
let mut flat: Vec<f32> = Vec::with_capacity(rows.len() * dims);
let scan_watermark = rows
.first()
.and_then(|row| match row.get("log_s") {
Some(SqlValue::Integer(n)) => u64::try_from(*n).ok(),
_ => None,
})
.unwrap_or(0);
for row in &rows {
let id_str = match row.get("subject_id") {
Some(SqlValue::Text(s)) => s.as_str(),
_ => continue,
};
let uuid = match Uuid::parse_str(id_str) {
Ok(id) => id,
Err(_) => continue,
};
let bytes = match row.get("embedding") {
Some(SqlValue::Blob(b)) => b.as_slice(),
_ => continue,
};
if bytes.len() != dims * 4 {
continue;
}
let Ok(vec) = khive_storage::decode_f32_native(bytes) else {
continue;
};
id_map.push(uuid);
flat.extend_from_slice(&vec);
}
if id_map.is_empty() {
return Ok(None);
}
Ok(Some((flat, id_map, scan_watermark)))
}
pub(crate) async fn load_and_build_from_vector_store(
rt: &KhiveRuntime,
token: &NamespaceToken,
model: &str,
) -> Result<Option<AnnBridge>, RuntimeError> {
let Some((flat, id_map, scan_watermark)) = scan_corpus_raw(rt, token, model).await? else {
return Ok(None);
};
let dims = flat.len() / id_map.len();
AnnBridge::build(flat, dims, id_map)
.map(|mut bridge| {
bridge.set_applied_seq(scan_watermark);
Some(bridge)
})
.map_err(RuntimeError::Internal)
}
pub(crate) async fn invalidate_snapshot(rt: &KhiveRuntime, namespace: &str) {
let sql = rt.sql();
let mut w = match sql.writer().await {
Ok(w) => w,
Err(e) => {
tracing::warn!(error = %e, "failed to open writer for Vamana snapshot invalidation");
return;
}
};
match invalidate_legacy_vamana_snapshots(w.as_mut(), namespace, "invalidate_vamana_snapshot")
.await
{
Ok(_) => {}
Err(e) if e.to_string().contains("no such table") => {}
Err(e) => {
tracing::warn!(error = %e, "failed to invalidate Vamana snapshot");
}
}
}
async fn run_warm_attempt(
rt: &KhiveRuntime,
token: &NamespaceToken,
ann: &SharedAnn,
model: &str,
permit: AnnWarmPermit,
) {
let outcome = ensure_ann_for_model(rt, token, ann, model).await;
finish_warm(permit, outcome).await;
}
async fn warm_ann_for_model_once(
rt: &KhiveRuntime,
token: &NamespaceToken,
ann: &SharedAnn,
model: &str,
) {
if model.is_empty() {
return;
}
let key = AnnKey::new(token.namespace().as_str(), model);
let Some(permit) = begin_warm(ann, key) else {
return;
};
run_warm_attempt(rt, token, ann, model, permit).await;
}
pub(crate) async fn warm_known_snapshots(rt: &KhiveRuntime, ann: &SharedAnn) {
let rows = {
let sql = rt.sql();
match sql.reader().await {
Ok(mut reader) => reader
.query_all(SqlStatement {
sql: khive_runtime::sql!("knowledge_legacy_snapshot_namespaces").into(),
params: vec![SqlValue::Text("%::vamana::%".into())],
label: None,
})
.await
.unwrap_or_default(),
Err(_) => Vec::new(),
}
};
for row in &rows {
let ns_key = match row.get("namespace") {
Some(SqlValue::Text(s)) => s.as_str(),
_ => continue,
};
let Some((ns_str, model)) = ns_key.split_once("::vamana::") else {
continue;
};
if ns_str.is_empty() || model.is_empty() {
continue;
}
let ns = match Namespace::parse(ns_str) {
Ok(n) => n,
Err(_) => continue,
};
let token = match rt.authorize(ns) {
Ok(t) => t,
Err(_) => continue,
};
warm_ann_for_model_once(rt, &token, ann, model).await;
}
let ann_root = match rt.backend_ann_root() {
Some(d) => d,
None => return,
};
let read_dir = match std::fs::read_dir(&ann_root) {
Ok(rd) => rd,
Err(_) => return, };
for entry in read_dir.flatten() {
let name = entry.file_name();
let hex = name.to_string_lossy();
let Some((ns_str, model)) = decode_ann_dir_name(hex.as_ref()) else {
continue;
};
let ns = match Namespace::parse(&ns_str) {
Ok(n) => n,
Err(_) => continue,
};
let token = match rt.authorize(ns) {
Ok(t) => t,
Err(_) => continue,
};
warm_ann_for_model_once(rt, &token, ann, &model).await;
}
}
#[cfg(test)]
pub(crate) fn rotation_watch_started_for_test(ann: &SharedAnn) -> bool {
ann.rotation_watch_started.load(Ordering::Acquire)
}
pub(crate) fn start_rotation_watcher(rt: &KhiveRuntime, ann: &SharedAnn) {
drop(start_rotation_watcher_with_shutdown(
rt,
ann,
khive_runtime::daemon_shutdown_token(),
));
}
fn start_rotation_watcher_with_shutdown(
rt: &KhiveRuntime,
ann: &SharedAnn,
shutdown: tokio_util::sync::CancellationToken,
) -> Option<tokio::task::JoinHandle<()>> {
let ann_root = rt.backend_ann_root()?;
let watch = khive_retrieval::ann::rotation_watch_future(
ann,
&ann.rotation_watch_started,
ann_root,
ROTATION_WATCH_INTERVAL,
shutdown,
|ann, ann_root| async move {
refresh_rotated_segments_in_root(&ann_root, &ann).await;
},
)?;
Some(khive_runtime::spawn_named_tracked_task(
"knowledge_ann_rotation_watch",
watch,
))
}
#[cfg(test)]
async fn refresh_rotated_segments_once(rt: &KhiveRuntime, ann: &SharedAnn) {
let Some(ann_root) = rt.backend_ann_root() else {
return;
};
refresh_rotated_segments_in_root(&ann_root, ann).await;
}
async fn refresh_rotated_segments_in_root(ann_root: &std::path::Path, ann: &SharedAnn) {
let installed: Vec<(AnnKey, [u8; 32])> = ann
.indexes
.read()
.await
.iter()
.filter_map(|(key, bridge)| bridge.commit_digest.map(|digest| (key.clone(), digest)))
.collect();
for (key, expected) in installed {
let dir = ann_segment_dir_from_root(ann_root, &key.namespace, &key.model);
match segment_commit_digest(&dir) {
Ok(Some(observed)) if observed != expected => {
refresh_rotated_segment(ann, &key, expected, dir).await;
}
Ok(_) => {}
Err(error) => {
tracing::warn!(
error = %error,
namespace = %key.namespace,
model = %key.model,
"knowledge ANN rotation check failed; retaining the installed generation"
);
}
}
}
}
async fn evict_bridge_and_ready_state(
ann: &SharedAnn,
key: &AnnKey,
incumbent_digest: [u8; 32],
generation: u64,
) -> bool {
let mut indexes = ann.indexes.write().await;
if indexes
.get(key)
.is_none_or(|bridge| bridge.commit_digest != Some(incumbent_digest))
{
return false;
}
indexes.remove(key);
let mut states = warm_states_guard(&ann.warm_states);
if matches!(
states.get(key),
Some(AnnWarmState::Ready { generation: state_generation }) if *state_generation == generation
) {
states.remove(key);
}
true
}
async fn refresh_rotated_segment(
ann: &SharedAnn,
key: &AnnKey,
expected: [u8; 32],
dir: std::path::PathBuf,
) {
let local_lock = checkpoint_lock(ann, key);
let _local_guard = local_lock.lock().await;
let incumbent = ann.indexes.read().await.get(key).map(|bridge| {
(
bridge.commit_digest,
bridge.generation,
bridge.index.last_applied_seq().unwrap_or(0),
)
});
let Some((Some(incumbent_digest), generation, incumbent_seq)) = incumbent else {
return;
};
if incumbent_digest != expected {
return;
}
let _publication_guard = match khive_retrieval::ann::acquire_checkpoint_lock_async(
dir.clone(),
BRIDGE_LOCK_MESSAGE_PREFIX,
)
.await
{
Ok(lock) => lock,
Err(error) => {
tracing::warn!(
error = %error,
namespace = %key.namespace,
model = %key.model,
"knowledge ANN rotation reload could not acquire the publication lock"
);
return;
}
};
let observed = match segment_commit_digest(&dir) {
Ok(Some(digest)) if digest != incumbent_digest => digest,
Ok(_) => return,
Err(error) => {
tracing::warn!(
error = %error,
namespace = %key.namespace,
model = %key.model,
"knowledge ANN rotated commit identity could not be read"
);
return;
}
};
let replacement = AnnBridge::load(&dir).and_then(|bridge| {
if bridge.commit_digest != Some(observed) {
return Err("commit identity changed during locked rotation reload".to_string());
}
let replacement_seq = bridge.index.last_applied_seq().unwrap_or(0);
if replacement_seq < incumbent_seq {
return Err(format!(
"rotated segment watermark {replacement_seq} regressed below installed {incumbent_seq}"
));
}
Ok(bridge.with_generation(generation))
});
match replacement {
Ok(bridge) => {
if install_replacing(ann, key, bridge).await {
tracing::debug!(
namespace = %key.namespace,
model = %key.model,
"knowledge ANN adopted rotated mmap generation and released its predecessor"
);
}
}
Err(error) => {
evict_bridge_and_ready_state(ann, key, incumbent_digest, generation).await;
tracing::warn!(
error = %error,
namespace = %key.namespace,
model = %key.model,
"knowledge ANN rotated generation failed validation; released predecessor and will rebuild on demand"
);
}
}
}
pub(crate) fn ensure_ann_background(rt: &KhiveRuntime, token: &NamespaceToken, ann: &SharedAnn) {
if rt.is_read_only() {
return;
}
let model = rt.default_embedder_name().to_string();
if model.is_empty() {
return;
}
let ns = token.namespace().as_str().to_owned();
let key = AnnKey::new(&ns, &model);
let Some(permit) = begin_warm(ann, key) else {
return;
};
let rt = rt.clone();
let ann = ann.clone();
let token = token.clone();
tokio::spawn(async move {
run_warm_attempt(&rt, &token, &ann, &model, permit).await;
});
}
enum SegmentOutcome {
Installed,
Empty,
Cold,
ForceRebuild,
}
#[allow(clippy::too_many_arguments)]
async fn classify_and_adopt_segment(
rt: &KhiveRuntime,
token: &NamespaceToken,
ann: &SharedAnn,
key: &AnnKey,
ns: &str,
model: &str,
seg_dir: &std::path::Path,
target_generation: u64,
) -> SegmentOutcome {
if force_rebuild_required(ann, key) {
return SegmentOutcome::ForceRebuild;
}
let info = match read_commit_info(seg_dir) {
Ok(Some(info)) => info,
Ok(None) => return SegmentOutcome::Cold,
Err(e) => {
tracing::warn!(error = %e, dir = %seg_dir.display(),
"error reading v2 commit record; Cold");
return SegmentOutcome::Cold;
}
};
let Some(s) = info.last_applied_seq else {
tracing::info!(namespace = %ns, model = %model,
"pre-amendment v2 segment (no watermark); Cold rebuild");
return SegmentOutcome::Cold;
};
match rt.embedder_dimensions(model) {
Some(dims) if dims as u64 == info.dimensions => {}
Some(dims) => {
tracing::info!(namespace = %ns, model = %model,
segment_dims = info.dimensions, live_dims = dims,
"v2 segment dimension mismatch; Cold rebuild");
return SegmentOutcome::Cold;
}
None => return SegmentOutcome::Cold,
}
match read_own_watermark(rt, ns, model).await {
Ok(Some(watermark)) if watermark >= 0 => {}
Ok(Some(_)) => {
mark_force_rebuild(ann, key);
return SegmentOutcome::ForceRebuild;
}
Ok(None) => {
tracing::info!(namespace = %ns, model = %model,
"ann consumer registry row absent; fencing for authoritative rebuild");
if let Err(error) = prepare_authoritative_rebuild(rt, ann, key).await {
tracing::warn!(error = %error, "ann consumer re-registration failed");
}
return SegmentOutcome::ForceRebuild;
}
Err(e) => {
tracing::warn!(error = %e, "ann registry read failed; Cold");
return SegmentOutcome::Cold;
}
}
match tail_exists(rt, ns, model, s).await {
Ok(false) => {
return match AnnBridge::load(seg_dir) {
Ok(bridge) => {
install_if_fresher(ann, key, bridge.with_generation(target_generation)).await;
SegmentOutcome::Installed
}
Err(e) => {
tracing::warn!(error = %e, dir = %seg_dir.display(),
"Hot segment load failed; Cold rebuild");
SegmentOutcome::Cold
}
};
}
Ok(true) => {}
Err(e) => {
tracing::warn!(error = %e, "ann tail probe failed; Cold");
return SegmentOutcome::Cold;
}
}
let rebuild_fraction = ann_rebuild_threshold();
let counts = match classification_scope_counts(rt, ns, model, s, rebuild_fraction).await {
Ok(counts) => counts,
Err(error) => {
tracing::warn!(error = %error, "bounded ANN scope count failed; retrying exact count");
match scope_counts(rt, ns, model, s).await {
Ok((live, tail)) => ClassificationScopeCounts {
live_lower_bound: live,
tail,
live_count_exact: true,
},
Err(e) => {
tracing::warn!(error = %e, "ann scope-count read failed; Cold");
return SegmentOutcome::Cold;
}
}
}
};
let mut live = counts.live_lower_bound;
let mut tail = counts.tail;
let mut threshold = (rebuild_fraction * live as f64).ceil() as u64;
if !counts.live_count_exact && tail > threshold {
match scope_counts(rt, ns, model, s).await {
Ok((exact_live, exact_tail)) => {
live = exact_live;
tail = exact_tail;
threshold = (rebuild_fraction * live as f64).ceil() as u64;
}
Err(e) => {
tracing::warn!(error = %e, "ANN exact scope-count fallback failed; Cold");
return SegmentOutcome::Cold;
}
}
}
if live == 0 {
tracing::info!(namespace = %ns, model = %model,
"zero live corpus for scope; Empty (FTS/degraded path)");
return SegmentOutcome::Empty;
}
if tail <= threshold {
let mut bridge = match AnnBridge::load(seg_dir) {
Ok(b) => b,
Err(e) => {
tracing::warn!(error = %e, dir = %seg_dir.display(),
"Stale-tail segment load failed; Cold rebuild");
return SegmentOutcome::Cold;
}
};
let (finals, new_s) = match fetch_final_states(rt, ns, model, s).await {
Ok(t) => t,
Err(e) => {
tracing::warn!(error = %e, "tail replay contradiction; Cold rebuild");
return SegmentOutcome::Cold;
}
};
if let Err(e) = replay_final_states(rt, token, &mut bridge, ns, model, &finals).await {
tracing::warn!(error = %e, "tail replay failed; Cold rebuild");
return SegmentOutcome::Cold;
}
bridge.set_applied_seq(new_s);
if !ann.builds_corpus_indexes {
install_if_fresher(ann, key, bridge.with_generation(target_generation)).await;
return SegmentOutcome::Installed;
}
let checkpointed = checkpoint_raise_compact_readopt(
rt,
ann,
key,
bridge,
target_generation,
CheckpointAuthority::Incremental,
)
.await;
if !checkpointed && force_rebuild_required(ann, key) {
return SegmentOutcome::ForceRebuild;
}
return SegmentOutcome::Installed;
}
match AnnBridge::load(seg_dir) {
Ok(bridge) => {
tracing::info!(namespace = %ns, model = %model, tail, live,
"tail above rebuild threshold; serving stale segment during rebuild");
install_if_fresher(ann, key, bridge.with_generation(target_generation)).await;
if !ann.builds_corpus_indexes {
return SegmentOutcome::Installed;
}
}
Err(e) => {
tracing::warn!(error = %e, dir = %seg_dir.display(),
"Stale-rebuild segment load failed; rebuilding without serve-stale");
}
}
SegmentOutcome::Cold
}
pub(crate) async fn ensure_ann_for_model(
rt: &KhiveRuntime,
token: &NamespaceToken,
ann: &SharedAnn,
model: &str,
) -> AnnWarmOutcome {
if model.is_empty() {
return AnnWarmOutcome::Empty;
}
let ns = token.namespace().as_str().to_owned();
let key = AnnKey::new(&ns, model);
let warm_host = ann.builds_corpus_indexes;
let mut force_rebuild = force_rebuild_required(ann, &key);
match read_own_watermark(rt, &ns, model).await {
Ok(Some(watermark)) if watermark < 0 => {
if !warm_host {
tracing::debug!(namespace = %ns, model = %model,
"ANN rebuild is pending and this process is not the warm index host; \
declining");
return AnnWarmOutcome::Declined;
}
mark_force_rebuild(ann, &key);
force_rebuild = true;
}
Ok(Some(_)) => {}
Ok(None) => {
if !warm_host {
tracing::debug!(namespace = %ns, model = %model,
"ANN registry row absent and this process is not the warm index host; \
declining rather than claiming the rebuild");
return AnnWarmOutcome::Declined;
}
if let Err(error) = prepare_authoritative_rebuild(rt, ann, &key).await {
tracing::warn!(error = %error, "failed to fence ANN registry loss");
return AnnWarmOutcome::Failed;
}
force_rebuild = true;
}
Err(error) => {
tracing::warn!(error = %error, "failed to read ANN registration before scan");
return AnnWarmOutcome::Failed;
}
}
if force_rebuild && !matches!(read_own_watermark(rt, &ns, model).await, Ok(Some(-1))) {
if let Err(error) = prepare_authoritative_rebuild(rt, ann, &key).await {
tracing::warn!(error = %error, "failed to establish ANN rebuild sentinel");
return AnnWarmOutcome::Failed;
}
}
let target_generation = current_generation(ann, &ns);
if !force_rebuild {
if let Some(loaded_generation) = ann
.indexes
.read()
.await
.get(&key)
.map(|bridge| bridge.generation)
{
if loaded_generation >= target_generation {
return AnnWarmOutcome::Ready;
}
tracing::debug!(
namespace = %ns,
model = %model,
loaded_generation,
target_generation,
"knowledge ANN fast path skipped: cached entry generation stale; rebuilding"
);
}
}
if !force_rebuild {
if let Some(seg_dir) = ann_segment_dir(rt, &ns, model) {
match classify_and_adopt_segment(
rt,
token,
ann,
&key,
&ns,
model,
&seg_dir,
target_generation,
)
.await
{
SegmentOutcome::Installed => return AnnWarmOutcome::Ready,
SegmentOutcome::Empty => {
mark_unavailable(ann, &key, target_generation);
return AnnWarmOutcome::Empty;
}
SegmentOutcome::Cold => {} SegmentOutcome::ForceRebuild => force_rebuild = true,
}
}
}
if !force_rebuild {
if let Some(snapshot) = try_load_snapshot(rt, &ns, model).await {
let current_fp = compute_fingerprint(rt, token, model).await;
if let Some(fp) = current_fp {
if snapshot.fingerprint == fp {
match AnnBridge::from_vamana_snapshot(snapshot) {
Ok(bridge) => {
install_if_fresher(
ann,
&key,
bridge.with_generation(target_generation),
)
.await;
return AnnWarmOutcome::Ready;
}
Err(e) => {
tracing::warn!(error = %e, "corrupt Vamana v1 snapshot; rebuilding");
}
}
} else {
tracing::info!(
namespace = %ns,
model = %model,
"stale Vamana v1 snapshot (fingerprint mismatch); rebuilding"
);
}
}
}
}
if !warm_host {
tracing::info!(namespace = %ns, model = %model,
"no adoptable ANN segment and this process is not the warm index host; \
serving degraded and leaving the corpus build to the daemon");
return AnnWarmOutcome::Declined;
}
let scan_authority = match prepare_full_corpus_scan(rt, ann, &key).await {
Ok(authority) => authority,
Err(error) => {
tracing::warn!(error = %error, "failed to establish ANN full-scan authority");
return AnnWarmOutcome::Failed;
}
};
match load_and_build_from_vector_store(rt, token, model).await {
Ok(Some(bridge)) => {
let checkpointed = checkpoint_raise_compact_readopt(
rt,
ann,
&key,
bridge,
target_generation,
scan_authority,
)
.await;
if checkpointed {
AnnWarmOutcome::Ready
} else if force_rebuild_required(ann, &key) {
AnnWarmOutcome::Failed
} else if has_current_index(ann, &key).await {
AnnWarmOutcome::Ready
} else {
AnnWarmOutcome::Failed
}
}
Ok(None) => {
mark_unavailable(ann, &key, target_generation);
AnnWarmOutcome::Empty
}
Err(e) => {
tracing::warn!(error = %e, "failed to rebuild Vamana ANN index");
AnnWarmOutcome::Failed
}
}
}
#[cfg(test)]
pub(crate) fn simulate_warming_in_flight(ann: &SharedAnn, key: AnnKey) {
begin_warm(ann, key)
.expect("fresh test ANN state must accept a warm")
.leave_in_flight_for_test();
}
#[cfg(test)]
#[path = "vamana_owned_build_tests.rs"]
mod owned_build_tests;
#[cfg(test)]
#[path = "vamana_tests.rs"]
mod tests;
#[cfg(test)]
#[path = "vamana_corpus_statement_tests.rs"]
mod corpus_statement_tests;
#[cfg(test)]
#[path = "vamana_corpus_capture_tests.rs"]
mod corpus_capture_tests;