use super::*;
pub(crate) struct AnnBridge {
pub(super) core: AnnBridgeCore,
pub(super) generation: u64,
#[cfg(test)]
pub(super) drop_probe: Option<Arc<()>>,
#[cfg(test)]
pub(super) search_pause: Option<Arc<TestSearchPause>>,
}
khive_vamana::impl_ann_bridge_helpers!(
AnnBridge,
);
#[cfg(test)]
#[derive(Default)]
pub(super) struct TestSearchPause {
pub(super) started: AtomicBool,
pub(super) released: std::sync::Mutex<bool>,
pub(super) wake: std::sync::Condvar,
}
#[cfg(test)]
impl TestSearchPause {
pub(super) fn wait(&self) {
let mut released = self.released.lock().expect("search pause");
self.started.store(true, Ordering::SeqCst);
while !*released {
released = self.wake.wait(released).expect("search pause");
}
}
pub(super) fn release(&self) {
*self.released.lock().expect("search pause") = true;
self.wake.notify_all();
}
}
#[derive(Clone, Debug, Eq, PartialEq, Hash)]
pub(crate) struct AnnKey {
pub(super) namespace: String,
pub(super) model: String,
}
impl AnnKey {
pub(crate) fn new(namespace: &str, model: &str) -> Self {
Self {
namespace: namespace.to_owned(),
model: model.to_owned(),
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum AnnWarmFailure {
EmptyCorpus,
Operational,
Interrupted,
NotWarmHost,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum AnnWarmOutcome {
Ready,
Empty,
Failed,
Declined,
}
#[derive(Debug)]
pub(super) enum AnnWarmState {
Warming {
attempt_id: u64,
generation: u64,
started_at: std::time::Instant,
},
Ready {
generation: u64,
},
Failed {
generation: u64,
error: AnnWarmFailure,
},
}
pub(super) struct AnnWarmPermit {
pub(super) ann: SharedAnn,
pub(super) key: AnnKey,
pub(super) attempt_id: u64,
pub(super) generation: u64,
pub(super) finished: bool,
}
pub(crate) struct AnnState {
pub(super) indexes: RwLock<HashMap<AnnKey, AnnBridge>>,
pub(super) warm_states: std::sync::Mutex<HashMap<AnnKey, AnnWarmState>>,
pub(super) next_warm_attempt_id: AtomicU64,
pub(super) generations: std::sync::Mutex<HashMap<String, u64>>,
pub(super) unavailable: std::sync::Mutex<HashMap<AnnKey, u64>>,
pub(super) force_rebuild: std::sync::Mutex<HashSet<AnnKey>>,
pub(super) checkpoint_locks:
std::sync::Mutex<HashMap<AnnKey, std::sync::Weak<tokio::sync::Mutex<()>>>>,
pub(super) rotation_watch_started: AtomicBool,
pub(super) builds_corpus_indexes: bool,
#[cfg(test)]
pub(super) test_finish_warm_ready_hook: std::sync::Mutex<Option<Arc<FinishWarmReadyTestHook>>>,
}
#[cfg(test)]
pub(crate) struct FinishWarmReadyTestHook {
pub(crate) incumbent_digest: [u8; 32],
pub(crate) generation: u64,
pub(crate) handle_tx: tokio::sync::mpsc::UnboundedSender<tokio::task::JoinHandle<()>>,
}
pub(crate) type SharedAnn = Arc<AnnState>;
pub(crate) fn new_shared() -> SharedAnn {
new_shared_for_role(true)
}
pub(crate) fn new_shared_for_role(builds_corpus_indexes: bool) -> SharedAnn {
Arc::new(AnnState {
builds_corpus_indexes,
indexes: RwLock::new(HashMap::new()),
warm_states: std::sync::Mutex::new(HashMap::new()),
next_warm_attempt_id: AtomicU64::new(1),
generations: std::sync::Mutex::new(HashMap::new()),
unavailable: std::sync::Mutex::new(HashMap::new()),
force_rebuild: std::sync::Mutex::new(HashSet::new()),
checkpoint_locks: std::sync::Mutex::new(HashMap::new()),
rotation_watch_started: AtomicBool::new(false),
#[cfg(test)]
test_finish_warm_ready_hook: std::sync::Mutex::new(None),
})
}
pub(super) fn force_rebuild_guard(
m: &std::sync::Mutex<HashSet<AnnKey>>,
) -> std::sync::MutexGuard<'_, HashSet<AnnKey>> {
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
pub(super) fn mark_force_rebuild(ann: &SharedAnn, key: &AnnKey) {
force_rebuild_guard(&ann.force_rebuild).insert(key.clone());
}
pub(super) fn force_rebuild_required(ann: &SharedAnn, key: &AnnKey) -> bool {
force_rebuild_guard(&ann.force_rebuild).contains(key)
}
pub(super) fn clear_force_rebuild_if_current(ann: &SharedAnn, key: &AnnKey, generation: u64) {
if current_generation(ann, &key.namespace) == generation {
clear_force_rebuild(ann, key);
}
}
pub(super) fn clear_force_rebuild(ann: &SharedAnn, key: &AnnKey) {
force_rebuild_guard(&ann.force_rebuild).remove(key);
}
pub(super) fn checkpoint_lock(ann: &SharedAnn, key: &AnnKey) -> Arc<tokio::sync::Mutex<()>> {
let mut locks = ann
.checkpoint_locks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
locks.retain(|_, lock| lock.strong_count() > 0);
if let Some(lock) = locks.get(key).and_then(std::sync::Weak::upgrade) {
return lock;
}
let lock = Arc::new(tokio::sync::Mutex::new(()));
locks.insert(key.clone(), Arc::downgrade(&lock));
lock
}
pub(super) fn generations_guard(
m: &std::sync::Mutex<HashMap<String, u64>>,
) -> std::sync::MutexGuard<'_, HashMap<String, u64>> {
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
pub(super) fn bump_generation(ann: &SharedAnn, namespace: &str) -> u64 {
let mut gens = generations_guard(&ann.generations);
let slot = gens.entry(namespace.to_owned()).or_insert(0);
*slot += 1;
*slot
}
pub(crate) fn current_generation(ann: &SharedAnn, namespace: &str) -> u64 {
generations_guard(&ann.generations)
.get(namespace)
.copied()
.unwrap_or(0)
}
pub(crate) async fn install_if_fresher(ann: &SharedAnn, key: &AnnKey, candidate: AnnBridge) {
let mut idxs = ann.indexes.write().await;
let ns_generation = current_generation(ann, &key.namespace);
if candidate.generation < ns_generation {
tracing::debug!(
key = ?key,
candidate_generation = candidate.generation,
namespace_generation = ns_generation,
"knowledge ANN install skipped: candidate predates namespace's current generation"
);
return;
}
match idxs.get(key) {
Some(existing) if existing.generation >= candidate.generation => {
tracing::debug!(
key = ?key,
existing_generation = existing.generation,
candidate_generation = candidate.generation,
"knowledge ANN install skipped: cached entry already >= this build's generation"
);
}
_ => {
idxs.insert(key.clone(), candidate);
unavailable_guard(&ann.unavailable).remove(key);
}
}
}
pub(crate) async fn install_replacing(ann: &SharedAnn, key: &AnnKey, candidate: AnnBridge) -> bool {
let mut idxs = ann.indexes.write().await;
let ns_generation = current_generation(ann, &key.namespace);
if candidate.generation < ns_generation {
tracing::debug!(
key = ?key,
candidate_generation = candidate.generation,
namespace_generation = ns_generation,
"knowledge ANN replace skipped: candidate predates namespace's current generation"
);
return false;
}
idxs.insert(key.clone(), candidate);
unavailable_guard(&ann.unavailable).remove(key);
true
}
pub(super) async fn has_current_index(ann: &SharedAnn, key: &AnnKey) -> bool {
let idxs = ann.indexes.read().await;
let current = current_generation(ann, &key.namespace);
idxs.get(key)
.is_some_and(|bridge| bridge.generation >= current)
}
pub(super) async fn has_current_index_at_watermark(
ann: &SharedAnn,
key: &AnnKey,
watermark: u64,
) -> bool {
let idxs = ann.indexes.read().await;
let current = current_generation(ann, &key.namespace);
idxs.get(key).is_some_and(|bridge| {
bridge.generation >= current && bridge.index.last_applied_seq().unwrap_or(0) >= watermark
})
}
pub(super) fn warm_states_guard(
m: &std::sync::Mutex<HashMap<AnnKey, AnnWarmState>>,
) -> std::sync::MutexGuard<'_, HashMap<AnnKey, AnnWarmState>> {
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
pub(super) fn begin_warm(ann: &SharedAnn, key: AnnKey) -> Option<AnnWarmPermit> {
let mut states = warm_states_guard(&ann.warm_states);
let generation = current_generation(ann, &key.namespace);
match states.get(&key) {
Some(
AnnWarmState::Warming {
generation: state_generation,
..
}
| AnnWarmState::Ready {
generation: state_generation,
},
) if *state_generation >= generation => return None,
Some(AnnWarmState::Failed {
generation: failed_generation,
error,
}) => {
tracing::debug!(
key = ?key,
failed_generation,
error = ?error,
"retrying failed knowledge ANN warm"
);
}
_ => {}
}
let attempt_id = ann.next_warm_attempt_id.fetch_add(1, Ordering::Relaxed);
states.insert(
key.clone(),
AnnWarmState::Warming {
attempt_id,
generation,
started_at: std::time::Instant::now(),
},
);
Some(AnnWarmPermit {
ann: ann.clone(),
key,
attempt_id,
generation,
finished: false,
})
}
pub(super) fn finish_warm_state(permit: &mut AnnWarmPermit, next: AnnWarmState) {
let mut states = warm_states_guard(&permit.ann.warm_states);
let started_at = match states.get(&permit.key) {
Some(AnnWarmState::Warming {
attempt_id,
generation,
started_at,
}) if *attempt_id == permit.attempt_id && *generation == permit.generation => *started_at,
_ => {
permit.finished = true;
return;
}
};
tracing::debug!(
key = ?permit.key,
attempt_id = permit.attempt_id,
elapsed_ms = started_at.elapsed().as_millis(),
state = ?next,
"knowledge ANN warm finished"
);
states.insert(permit.key.clone(), next);
permit.finished = true;
}
pub(super) async fn finish_warm(mut permit: AnnWarmPermit, outcome: AnnWarmOutcome) {
match outcome {
AnnWarmOutcome::Ready => {
let ann = permit.ann.clone();
let idxs = ann.indexes.read().await;
let next = match idxs
.get(&permit.key)
.map(|bridge| bridge.generation)
.filter(|generation| *generation >= permit.generation)
{
Some(generation) => AnnWarmState::Ready { generation },
None => AnnWarmState::Failed {
generation: permit.generation,
error: AnnWarmFailure::Operational,
},
};
#[cfg(test)]
run_finish_warm_ready_test_hook(&ann, &permit.key).await;
finish_warm_state(&mut permit, next);
drop(idxs);
}
AnnWarmOutcome::Empty => {
let next = AnnWarmState::Failed {
generation: permit.generation,
error: AnnWarmFailure::EmptyCorpus,
};
finish_warm_state(&mut permit, next);
}
AnnWarmOutcome::Failed => {
let next = AnnWarmState::Failed {
generation: permit.generation,
error: AnnWarmFailure::Operational,
};
finish_warm_state(&mut permit, next);
}
AnnWarmOutcome::Declined => {
let next = AnnWarmState::Failed {
generation: permit.generation,
error: AnnWarmFailure::NotWarmHost,
};
finish_warm_state(&mut permit, next);
}
}
}
#[cfg(test)]
pub(super) async fn run_finish_warm_ready_test_hook(ann: &SharedAnn, key: &AnnKey) {
let hook = ann
.test_finish_warm_ready_hook
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
let Some(hook) = hook else {
return;
};
let ann = ann.clone();
let key = key.clone();
let incumbent_digest = hook.incumbent_digest;
let generation = hook.generation;
let handle = tokio::spawn(async move {
evict_bridge_and_ready_state(&ann, &key, incumbent_digest, generation).await;
});
let _ = hook.handle_tx.send(handle);
tokio::task::yield_now().await;
}
impl Drop for AnnWarmPermit {
fn drop(&mut self) {
if !self.finished {
let next = AnnWarmState::Failed {
generation: self.generation,
error: AnnWarmFailure::Interrupted,
};
finish_warm_state(self, next);
}
}
}
#[cfg(test)]
impl AnnWarmPermit {
pub(super) fn leave_in_flight_for_test(mut self) {
self.finished = true;
}
}
pub(super) fn unavailable_guard(
m: &std::sync::Mutex<HashMap<AnnKey, u64>>,
) -> std::sync::MutexGuard<'_, HashMap<AnnKey, u64>> {
m.lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
pub(super) fn mark_unavailable(ann: &SharedAnn, key: &AnnKey, generation: u64) {
unavailable_guard(&ann.unavailable).insert(key.clone(), generation);
}
pub(super) fn is_terminally_unavailable(ann: &SharedAnn, key: &AnnKey) -> bool {
let current = current_generation(ann, &key.namespace);
let mut guard = unavailable_guard(&ann.unavailable);
match guard.get(key) {
Some(&marked_generation) if marked_generation >= current => true,
Some(_) => {
guard.remove(key);
false
}
None => false,
}
}
#[cfg(test)]
pub(crate) async fn insert_ann_if_absent(ann: &SharedAnn, key: AnnKey, bridge: AnnBridge) -> bool {
use std::collections::hash_map::Entry;
let mut guard = ann.indexes.write().await;
match guard.entry(key) {
Entry::Occupied(_) => false,
Entry::Vacant(e) => {
e.insert(bridge);
true
}
}
}
pub(crate) async fn clear_namespace(ann: &SharedAnn, namespace: &str) {
let mut idxs = ann.indexes.write().await;
let mut states = warm_states_guard(&ann.warm_states);
idxs.retain(|k, _| k.namespace != namespace);
states.retain(|k, _| k.namespace != namespace);
bump_generation(ann, namespace);
}
#[cfg(test)]
pub(crate) async fn search_loaded(
ann: &SharedAnn,
key: &AnnKey,
query: &[f32],
k: usize,
) -> Option<Vec<(Uuid, f32)>> {
let guard = ann.indexes.read().await;
guard.get(key).map(|bridge| bridge.search(query, k))
}
pub(crate) async fn search_loaded_with_seq(
ann: &SharedAnn,
key: &AnnKey,
query: &[f32],
k: usize,
) -> Option<(Vec<(Uuid, f32)>, u64)> {
if !ann.indexes.read().await.contains_key(key) {
return None;
}
let ann = Arc::clone(ann);
let key = key.clone();
let query = query.to_vec();
tokio::task::spawn_blocking(move || {
let guard = ann.indexes.blocking_read();
guard.get(&key).map(|bridge| {
(
bridge.search(&query, k),
bridge.index.last_applied_seq().unwrap_or(0),
)
})
})
.await
.expect("loaded ANN traversal task panicked")
}
pub(crate) fn is_warming_not_loaded(ann: &SharedAnn, key: &AnnKey) -> bool {
let in_warming = {
let states = warm_states_guard(&ann.warm_states);
let generation = current_generation(ann, &key.namespace);
matches!(
states.get(key),
Some(AnnWarmState::Warming {
generation: state_generation,
..
}) if *state_generation >= generation
)
};
if !in_warming {
return false;
}
match ann.indexes.try_read() {
Ok(guard) => !guard.contains_key(key),
Err(_) => true,
}
}
pub(crate) async fn wait_ready(
ann: &SharedAnn,
key: &AnnKey,
timeout_ms: u64,
poll_ms: u64,
) -> bool {
let deadline = std::time::Instant::now() + std::time::Duration::from_millis(timeout_ms);
loop {
if ann.indexes.read().await.contains_key(key) {
return true;
}
if is_terminally_unavailable(ann, key) {
return false;
}
if std::time::Instant::now() >= deadline {
return false;
}
tokio::time::sleep(std::time::Duration::from_millis(poll_ms)).await;
}
}
pub(crate) const ANN_WARM_WAIT_TIMEOUT_MS: u64 = 5_000;
pub(crate) const ANN_WARM_WAIT_POLL_MS: u64 = 50;
pub(super) const ROTATION_WATCH_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5);
pub(super) static ANN_WARM_WAIT_TIMEOUT_OVERRIDE_MS: std::sync::atomic::AtomicU64 =
std::sync::atomic::AtomicU64::new(0);
pub(crate) fn warm_wait_timeout_ms() -> u64 {
let o = ANN_WARM_WAIT_TIMEOUT_OVERRIDE_MS.load(std::sync::atomic::Ordering::Relaxed);
if o > 0 {
o
} else {
ANN_WARM_WAIT_TIMEOUT_MS
}
}
#[cfg(test)]
pub(crate) fn set_warm_wait_timeout_override_ms(ms: u64) {
ANN_WARM_WAIT_TIMEOUT_OVERRIDE_MS.store(ms, std::sync::atomic::Ordering::Relaxed);
}