use super::*;
pub(super) async fn run_warm_attempt(
rt: &KhiveRuntime,
token: &NamespaceToken,
ann: &SharedAnn,
model: &str,
permit: AnnWarmPermit,
) {
let outcome = ensure_ann_for_model(rt, token, ann, model).await;
finish_warm(permit, outcome).await;
}
pub(super) async fn warm_ann_for_model_once(
rt: &KhiveRuntime,
token: &NamespaceToken,
ann: &SharedAnn,
model: &str,
) {
if model.is_empty() {
return;
}
let key = AnnKey::new(token.namespace().as_str(), model);
let Some(permit) = begin_warm(ann, key) else {
return;
};
run_warm_attempt(rt, token, ann, model, permit).await;
}
pub(crate) async fn warm_known_snapshots(rt: &KhiveRuntime, ann: &SharedAnn) {
let rows = {
let sql = rt.sql();
match sql.reader().await {
Ok(mut reader) => reader
.query_all(SqlStatement {
sql: ::khive_runtime::sql!("knowledge_legacy_snapshot_namespaces").into(),
params: vec![SqlValue::Text("%::vamana::%".into())],
label: None,
})
.await
.unwrap_or_default(),
Err(_) => Vec::new(),
}
};
for row in &rows {
let ns_key = match row.get("namespace") {
Some(SqlValue::Text(s)) => s.as_str(),
_ => continue,
};
let Some((ns_str, model)) = ns_key.split_once("::vamana::") else {
continue;
};
if ns_str.is_empty() || model.is_empty() {
continue;
}
let ns = match Namespace::parse(ns_str) {
Ok(n) => n,
Err(_) => continue,
};
let token = match rt.authorize(ns) {
Ok(t) => t,
Err(_) => continue,
};
warm_ann_for_model_once(rt, &token, ann, model).await;
}
let ann_root = match rt.backend_ann_root() {
Some(d) => d,
None => return,
};
let read_dir = match std::fs::read_dir(&ann_root) {
Ok(rd) => rd,
Err(_) => return, };
for entry in read_dir.flatten() {
let name = entry.file_name();
let hex = name.to_string_lossy();
let Some((ns_str, model)) = decode_ann_dir_name(hex.as_ref()) else {
continue;
};
let ns = match Namespace::parse(&ns_str) {
Ok(n) => n,
Err(_) => continue,
};
let token = match rt.authorize(ns) {
Ok(t) => t,
Err(_) => continue,
};
warm_ann_for_model_once(rt, &token, ann, &model).await;
}
}
#[cfg(test)]
pub(crate) fn rotation_watch_started_for_test(ann: &SharedAnn) -> bool {
ann.rotation_watch_started.load(Ordering::Acquire)
}
pub(crate) fn start_rotation_watcher(rt: &KhiveRuntime, ann: &SharedAnn) {
drop(start_rotation_watcher_with_shutdown(
rt,
ann,
khive_runtime::daemon_shutdown_token(),
));
}
pub(super) fn start_rotation_watcher_with_shutdown(
rt: &KhiveRuntime,
ann: &SharedAnn,
shutdown: tokio_util::sync::CancellationToken,
) -> Option<tokio::task::JoinHandle<()>> {
let ann_root = rt.backend_ann_root()?;
let watch = khive_retrieval::ann::rotation_watch_future(
ann,
&ann.rotation_watch_started,
ann_root,
ROTATION_WATCH_INTERVAL,
shutdown,
|ann, ann_root| async move {
refresh_rotated_segments_in_root(&ann_root, &ann).await;
},
)?;
Some(khive_runtime::spawn_named_tracked_task(
"knowledge_ann_rotation_watch",
watch,
))
}
#[cfg(test)]
pub(super) async fn refresh_rotated_segments_once(rt: &KhiveRuntime, ann: &SharedAnn) {
let Some(ann_root) = rt.backend_ann_root() else {
return;
};
refresh_rotated_segments_in_root(&ann_root, ann).await;
}
pub(super) async fn refresh_rotated_segments_in_root(ann_root: &std::path::Path, ann: &SharedAnn) {
let installed: Vec<(AnnKey, [u8; 32])> = ann
.indexes
.read()
.await
.iter()
.filter_map(|(key, bridge)| bridge.commit_digest.map(|digest| (key.clone(), digest)))
.collect();
for (key, expected) in installed {
let dir = ann_segment_dir_from_root(ann_root, &key.namespace, &key.model);
match segment_commit_digest(&dir) {
Ok(Some(observed)) if observed != expected => {
refresh_rotated_segment(ann, &key, expected, dir).await;
}
Ok(_) => {}
Err(error) => {
tracing::warn!(
error = %error,
namespace = %key.namespace,
model = %key.model,
"knowledge ANN rotation check failed; retaining the installed generation"
);
}
}
}
}
pub(super) async fn evict_bridge_and_ready_state(
ann: &SharedAnn,
key: &AnnKey,
incumbent_digest: [u8; 32],
generation: u64,
) -> bool {
let mut indexes = ann.indexes.write().await;
if indexes
.get(key)
.is_none_or(|bridge| bridge.commit_digest != Some(incumbent_digest))
{
return false;
}
indexes.remove(key);
let mut states = warm_states_guard(&ann.warm_states);
if matches!(
states.get(key),
Some(AnnWarmState::Ready { generation: state_generation }) if *state_generation == generation
) {
states.remove(key);
}
true
}
pub(super) async fn refresh_rotated_segment(
ann: &SharedAnn,
key: &AnnKey,
expected: [u8; 32],
dir: std::path::PathBuf,
) {
let local_lock = checkpoint_lock(ann, key);
let _local_guard = local_lock.lock().await;
let incumbent = ann.indexes.read().await.get(key).map(|bridge| {
(
bridge.commit_digest,
bridge.generation,
bridge.index.last_applied_seq().unwrap_or(0),
)
});
let Some((Some(incumbent_digest), generation, incumbent_seq)) = incumbent else {
return;
};
if incumbent_digest != expected {
return;
}
let _publication_guard = match khive_retrieval::ann::acquire_checkpoint_lock_async(
dir.clone(),
BRIDGE_LOCK_MESSAGE_PREFIX,
)
.await
{
Ok(lock) => lock,
Err(error) => {
tracing::warn!(
error = %error,
namespace = %key.namespace,
model = %key.model,
"knowledge ANN rotation reload could not acquire the publication lock"
);
return;
}
};
let observed = match segment_commit_digest(&dir) {
Ok(Some(digest)) if digest != incumbent_digest => digest,
Ok(_) => return,
Err(error) => {
tracing::warn!(
error = %error,
namespace = %key.namespace,
model = %key.model,
"knowledge ANN rotated commit identity could not be read"
);
return;
}
};
let replacement = AnnBridge::load(&dir).and_then(|bridge| {
if bridge.commit_digest != Some(observed) {
return Err("commit identity changed during locked rotation reload".to_string());
}
let replacement_seq = bridge.index.last_applied_seq().unwrap_or(0);
if replacement_seq < incumbent_seq {
return Err(format!(
"rotated segment watermark {replacement_seq} regressed below installed {incumbent_seq}"
));
}
Ok(bridge.with_generation(generation))
});
match replacement {
Ok(bridge) => {
if install_replacing(ann, key, bridge).await {
tracing::debug!(
namespace = %key.namespace,
model = %key.model,
"knowledge ANN adopted rotated mmap generation and released its predecessor"
);
}
}
Err(error) => {
evict_bridge_and_ready_state(ann, key, incumbent_digest, generation).await;
tracing::warn!(
error = %error,
namespace = %key.namespace,
model = %key.model,
"knowledge ANN rotated generation failed validation; released predecessor and will rebuild on demand"
);
}
}
}
pub(crate) fn ensure_ann_background(rt: &KhiveRuntime, token: &NamespaceToken, ann: &SharedAnn) {
if rt.is_read_only() {
return;
}
let model = rt.default_embedder_name().to_string();
if model.is_empty() {
return;
}
let ns = token.namespace().as_str().to_owned();
let key = AnnKey::new(&ns, &model);
let Some(permit) = begin_warm(ann, key) else {
return;
};
let rt = rt.clone();
let ann = ann.clone();
let token = token.clone();
tokio::spawn(async move {
run_warm_attempt(&rt, &token, &ann, &model, permit).await;
});
}
pub(super) enum SegmentOutcome {
Installed,
Empty,
Cold,
ForceRebuild,
}
#[allow(clippy::too_many_arguments)]
pub(super) async fn classify_and_adopt_segment(
rt: &KhiveRuntime,
token: &NamespaceToken,
ann: &SharedAnn,
key: &AnnKey,
ns: &str,
model: &str,
seg_dir: &std::path::Path,
target_generation: u64,
) -> SegmentOutcome {
if force_rebuild_required(ann, key) {
return SegmentOutcome::ForceRebuild;
}
let info = match read_commit_info(seg_dir) {
Ok(Some(info)) => info,
Ok(None) => return SegmentOutcome::Cold,
Err(e) => {
tracing::warn!(error = %e, dir = %seg_dir.display(),
"error reading v2 commit record; Cold");
return SegmentOutcome::Cold;
}
};
let Some(s) = info.last_applied_seq else {
tracing::info!(namespace = %ns, model = %model,
"pre-amendment v2 segment (no watermark); Cold rebuild");
return SegmentOutcome::Cold;
};
match rt.embedder_dimensions(model) {
Some(dims) if dims as u64 == info.dimensions => {}
Some(dims) => {
tracing::info!(namespace = %ns, model = %model,
segment_dims = info.dimensions, live_dims = dims,
"v2 segment dimension mismatch; Cold rebuild");
return SegmentOutcome::Cold;
}
None => return SegmentOutcome::Cold,
}
match read_own_watermark(rt, ns, model).await {
Ok(Some(watermark)) if watermark >= 0 => {}
Ok(Some(_)) => {
mark_force_rebuild(ann, key);
return SegmentOutcome::ForceRebuild;
}
Ok(None) => {
tracing::info!(namespace = %ns, model = %model,
"ann consumer registry row absent; fencing for authoritative rebuild");
if let Err(error) = prepare_authoritative_rebuild(rt, ann, key).await {
tracing::warn!(error = %error, "ann consumer re-registration failed");
}
return SegmentOutcome::ForceRebuild;
}
Err(e) => {
tracing::warn!(error = %e, "ann registry read failed; Cold");
return SegmentOutcome::Cold;
}
}
match tail_exists(rt, ns, model, s).await {
Ok(false) => {
return match AnnBridge::load(seg_dir) {
Ok(bridge) => {
install_if_fresher(ann, key, bridge.with_generation(target_generation)).await;
SegmentOutcome::Installed
}
Err(e) => {
tracing::warn!(error = %e, dir = %seg_dir.display(),
"Hot segment load failed; Cold rebuild");
SegmentOutcome::Cold
}
};
}
Ok(true) => {}
Err(e) => {
tracing::warn!(error = %e, "ann tail probe failed; Cold");
return SegmentOutcome::Cold;
}
}
let rebuild_fraction = ann_rebuild_threshold();
let counts = match classification_scope_counts(rt, ns, model, s, rebuild_fraction).await {
Ok(counts) => counts,
Err(error) => {
tracing::warn!(error = %error, "bounded ANN scope count failed; retrying exact count");
match scope_counts(rt, ns, model, s).await {
Ok((live, tail)) => ClassificationScopeCounts {
live_lower_bound: live,
tail,
live_count_exact: true,
},
Err(e) => {
tracing::warn!(error = %e, "ann scope-count read failed; Cold");
return SegmentOutcome::Cold;
}
}
}
};
let mut live = counts.live_lower_bound;
let mut tail = counts.tail;
let mut threshold = (rebuild_fraction * live as f64).ceil() as u64;
if !counts.live_count_exact && tail > threshold {
match scope_counts(rt, ns, model, s).await {
Ok((exact_live, exact_tail)) => {
live = exact_live;
tail = exact_tail;
threshold = (rebuild_fraction * live as f64).ceil() as u64;
}
Err(e) => {
tracing::warn!(error = %e, "ANN exact scope-count fallback failed; Cold");
return SegmentOutcome::Cold;
}
}
}
if live == 0 {
tracing::info!(namespace = %ns, model = %model,
"zero live corpus for scope; Empty (FTS/degraded path)");
return SegmentOutcome::Empty;
}
if tail <= threshold {
let mut bridge = match AnnBridge::load(seg_dir) {
Ok(b) => b,
Err(e) => {
tracing::warn!(error = %e, dir = %seg_dir.display(),
"Stale-tail segment load failed; Cold rebuild");
return SegmentOutcome::Cold;
}
};
let (finals, new_s) = match fetch_final_states(rt, ns, model, s).await {
Ok(t) => t,
Err(e) => {
tracing::warn!(error = %e, "tail replay contradiction; Cold rebuild");
return SegmentOutcome::Cold;
}
};
if let Err(e) = replay_final_states(rt, token, &mut bridge, ns, model, &finals).await {
tracing::warn!(error = %e, "tail replay failed; Cold rebuild");
return SegmentOutcome::Cold;
}
bridge.set_applied_seq(new_s);
if !ann.builds_corpus_indexes {
install_if_fresher(ann, key, bridge.with_generation(target_generation)).await;
return SegmentOutcome::Installed;
}
let checkpointed = checkpoint_raise_compact_readopt(
rt,
ann,
key,
bridge,
target_generation,
CheckpointAuthority::Incremental,
)
.await;
if !checkpointed && force_rebuild_required(ann, key) {
return SegmentOutcome::ForceRebuild;
}
return SegmentOutcome::Installed;
}
match AnnBridge::load(seg_dir) {
Ok(bridge) => {
tracing::info!(namespace = %ns, model = %model, tail, live,
"tail above rebuild threshold; serving stale segment during rebuild");
install_if_fresher(ann, key, bridge.with_generation(target_generation)).await;
if !ann.builds_corpus_indexes {
return SegmentOutcome::Installed;
}
}
Err(e) => {
tracing::warn!(error = %e, dir = %seg_dir.display(),
"Stale-rebuild segment load failed; rebuilding without serve-stale");
}
}
SegmentOutcome::Cold
}
pub(crate) async fn ensure_ann_for_model(
rt: &KhiveRuntime,
token: &NamespaceToken,
ann: &SharedAnn,
model: &str,
) -> AnnWarmOutcome {
if model.is_empty() {
return AnnWarmOutcome::Empty;
}
let ns = token.namespace().as_str().to_owned();
let key = AnnKey::new(&ns, model);
let warm_host = ann.builds_corpus_indexes;
let mut force_rebuild = force_rebuild_required(ann, &key);
match read_own_watermark(rt, &ns, model).await {
Ok(Some(watermark)) if watermark < 0 => {
if !warm_host {
tracing::debug!(namespace = %ns, model = %model,
"ANN rebuild is pending and this process is not the warm index host; \
declining");
return AnnWarmOutcome::Declined;
}
mark_force_rebuild(ann, &key);
force_rebuild = true;
}
Ok(Some(_)) => {}
Ok(None) => {
if !warm_host {
tracing::debug!(namespace = %ns, model = %model,
"ANN registry row absent and this process is not the warm index host; \
declining rather than claiming the rebuild");
return AnnWarmOutcome::Declined;
}
if let Err(error) = prepare_authoritative_rebuild(rt, ann, &key).await {
tracing::warn!(error = %error, "failed to fence ANN registry loss");
return AnnWarmOutcome::Failed;
}
force_rebuild = true;
}
Err(error) => {
tracing::warn!(error = %error, "failed to read ANN registration before scan");
return AnnWarmOutcome::Failed;
}
}
if force_rebuild && !matches!(read_own_watermark(rt, &ns, model).await, Ok(Some(-1))) {
if let Err(error) = prepare_authoritative_rebuild(rt, ann, &key).await {
tracing::warn!(error = %error, "failed to establish ANN rebuild sentinel");
return AnnWarmOutcome::Failed;
}
}
let target_generation = current_generation(ann, &ns);
if !force_rebuild {
if let Some(loaded_generation) = ann
.indexes
.read()
.await
.get(&key)
.map(|bridge| bridge.generation)
{
if loaded_generation >= target_generation {
return AnnWarmOutcome::Ready;
}
tracing::debug!(
namespace = %ns,
model = %model,
loaded_generation,
target_generation,
"knowledge ANN fast path skipped: cached entry generation stale; rebuilding"
);
}
}
if !force_rebuild {
if let Some(seg_dir) = ann_segment_dir(rt, &ns, model) {
match classify_and_adopt_segment(
rt,
token,
ann,
&key,
&ns,
model,
&seg_dir,
target_generation,
)
.await
{
SegmentOutcome::Installed => return AnnWarmOutcome::Ready,
SegmentOutcome::Empty => {
mark_unavailable(ann, &key, target_generation);
return AnnWarmOutcome::Empty;
}
SegmentOutcome::Cold => {} SegmentOutcome::ForceRebuild => force_rebuild = true,
}
}
}
if !force_rebuild {
if let Some(snapshot) = try_load_snapshot(rt, &ns, model).await {
let current_fp = compute_fingerprint(rt, token, model).await;
if let Some(fp) = current_fp {
if snapshot.fingerprint == fp {
match AnnBridge::from_vamana_snapshot(snapshot) {
Ok(bridge) => {
install_if_fresher(
ann,
&key,
bridge.with_generation(target_generation),
)
.await;
return AnnWarmOutcome::Ready;
}
Err(e) => {
tracing::warn!(error = %e, "corrupt Vamana v1 snapshot; rebuilding");
}
}
} else {
tracing::info!(
namespace = %ns,
model = %model,
"stale Vamana v1 snapshot (fingerprint mismatch); rebuilding"
);
}
}
}
}
if !warm_host {
tracing::info!(namespace = %ns, model = %model,
"no adoptable ANN segment and this process is not the warm index host; \
serving degraded and leaving the corpus build to the daemon");
return AnnWarmOutcome::Declined;
}
let scan_authority = match prepare_full_corpus_scan(rt, ann, &key).await {
Ok(authority) => authority,
Err(error) => {
tracing::warn!(error = %error, "failed to establish ANN full-scan authority");
return AnnWarmOutcome::Failed;
}
};
match load_and_build_from_vector_store(rt, token, model).await {
Ok(Some(bridge)) => {
let checkpointed = checkpoint_raise_compact_readopt(
rt,
ann,
&key,
bridge,
target_generation,
scan_authority,
)
.await;
if checkpointed {
AnnWarmOutcome::Ready
} else if force_rebuild_required(ann, &key) {
AnnWarmOutcome::Failed
} else if has_current_index(ann, &key).await {
AnnWarmOutcome::Ready
} else {
AnnWarmOutcome::Failed
}
}
Ok(None) => {
mark_unavailable(ann, &key, target_generation);
AnnWarmOutcome::Empty
}
Err(e) => {
tracing::warn!(error = %e, "failed to rebuild Vamana ANN index");
AnnWarmOutcome::Failed
}
}
}
#[cfg(test)]
pub(crate) fn simulate_warming_in_flight(ann: &SharedAnn, key: AnnKey) {
begin_warm(ann, key)
.expect("fresh test ANN state must accept a warm")
.leave_in_flight_for_test();
}