use std::collections::HashSet;
use super::{
DEFAULT_SKETCH_CANDIDATE_BUDGET, ValiseFile, VectorFidelity, VectorHit, VectorSearchQuery,
query_profile,
};
use crate::codec::VectorCodec as _;
use crate::codec::upq::{UpqCodec, dot_i8};
use crate::error::{Error, Result};
use crate::format::catalog::EmbeddingSpaceDesc;
use crate::format::{CollectionId, VectorId};
struct UpqCand {
vid: u32,
ptr: usize,
}
struct Scored {
score: f32,
vid: u32,
ptr: usize,
}
impl ValiseFile {
pub(super) fn upq_sketch_then_rerank(
&self,
space: &EmbeddingSpaceDesc,
query: &VectorSearchQuery,
) -> Result<Vec<VectorHit>> {
let qprof_on = query_profile::enabled();
let mut qprof = query_profile::QueryProfile::default();
let mut qprof_mark = qprof_on.then(std::time::Instant::now);
let pending_in_space = self
.pending_vector_batches
.keys()
.any(|(esp, _)| *esp == query.embedding_space_id);
if pending_in_space {
return Err(Error::Format(format!(
"vector_search: embedding_space {} has uncommitted vectors; call commit() first",
query.embedding_space_id.0
)));
}
let primary_id = space.primary_codec_id.ok_or_else(|| {
Error::Integrity(format!(
"vector_search: space {} has no primary_codec_id",
space.embedding_space_id.0
))
})?;
let primary_box = self.codec_cache.get(&primary_id).ok_or_else(|| {
Error::Integrity(format!(
"upq_sketch_then_rerank: primary codec {} missing from cache",
primary_id.0
))
})?;
let upq = primary_box
.as_ref()
.as_any()
.downcast_ref::<UpqCodec>()
.ok_or_else(|| {
Error::Integrity(format!(
"upq_sketch_then_rerank: space {} primary codec is not UPQ",
space.embedding_space_id.0
))
})?;
let active_n = self.vector_by_id_cache.len();
let candidates_budget = query
.channel_k
.unwrap_or_else(|| {
query
.k
.saturating_mul(4)
.max(DEFAULT_SKETCH_CANDIDATE_BUDGET)
})
.min(active_n.max(1));
let _fanout_permit = crate::file::query_admission::global().try_acquire();
let parallel = _fanout_permit.is_some();
let (q_rot, q_norm) = upq.rotate_query(&query.query)?;
let q_sketch = crate::retrieval::sketch::pack_query_sketch(&q_rot);
let dim = upq.dim;
let vb = self.vector_base.read();
qprof.prep = query_profile::lap(&mut qprof_mark);
let candidates: Vec<(VectorId, i32)> = {
let (words, sketches) = vb
.sketches_by_space
.get(&space.embedding_space_id)
.ok_or_else(|| {
Error::Integrity(format!(
"vector_search: no sketch index for space {}",
space.embedding_space_id.0
))
})?;
let rows: &[(u32, usize)] = vb
.candidates_by_space
.get(&space.embedding_space_id)
.map(|v| v.as_slice())
.unwrap_or(&[]);
qprof.candidates_scanned = rows.len();
let mut hbuf: Vec<u16> = Vec::new();
crate::retrieval::sketch::scan_candidates(
sketches,
*words,
rows,
&q_sketch,
candidates_budget,
dim,
parallel,
&mut hbuf,
)
};
qprof.stage1_sketch = query_profile::lap(&mut qprof_mark);
let i8_cache = vb
.upq_i8_by_space
.get(&space.embedding_space_id)
.ok_or_else(|| {
Error::Integrity(format!(
"vector_search: no UPQ i8 cache for space {}",
space.embedding_space_id.0
))
})?;
let dequant = vb
.upq_dequant_by_space
.get(&space.embedding_space_id)
.ok_or_else(|| {
Error::Integrity(format!(
"vector_search: no UPQ dequant cache for space {}",
space.embedding_space_id.0
))
})?;
let mut q_i8 = vec![0i8; dim];
UpqCodec::prepare_query_i8(&q_rot, &mut q_i8);
let filter: Option<&HashSet<CollectionId>> = query.collection_filter.as_ref();
let ptrs: &[usize] = &vb.ptrs;
let mut cands: Vec<UpqCand> = Vec::with_capacity(candidates.len());
for (vid, _ham) in &candidates {
let v = vid.0 as usize;
if let Some(set) = filter {
let Some(desc) = self.vector_by_id_cache.get(vid) else {
continue;
};
if !set.contains(&desc.collection_id) {
continue;
}
}
let ptr = ptrs.get(v).copied().unwrap_or(0);
if ptr == 0 {
continue; }
cands.push(UpqCand {
vid: vid.0 as u32,
ptr,
});
}
if parallel {
use rayon::slice::ParallelSliceMut;
cands.par_sort_unstable_by_key(|c| c.vid);
} else {
cands.sort_unstable_by_key(|c| c.vid);
}
let score_one = |c: &UpqCand| -> Scored {
let v = c.vid as usize;
let row = &i8_cache[v * dim..(v + 1) * dim];
let score = -(dot_i8(&q_i8, row) as f32) * dequant[v];
Scored {
score,
vid: c.vid,
ptr: c.ptr,
}
};
let mut scored: Vec<Scored> = if parallel {
use rayon::prelude::*;
cands.par_iter().map(score_one).collect()
} else {
cands.iter().map(score_one).collect()
};
let pool_size = if query.fidelity == VectorFidelity::Full {
(3 * query.k).min(scored.len())
} else {
query.k.min(scored.len())
};
if scored.len() > pool_size && pool_size > 0 {
scored.select_nth_unstable_by(pool_size - 1, |a, b| {
a.score.total_cmp(&b.score).then_with(|| a.vid.cmp(&b.vid))
});
scored.truncate(pool_size);
}
scored.sort_by(|a, b| a.score.total_cmp(&b.score).then_with(|| a.vid.cmp(&b.vid)));
qprof.stage2_rerank = query_profile::lap(&mut qprof_mark);
qprof.candidates_reranked = cands.len();
qprof.survivors = scored.len();
if query.fidelity == VectorFidelity::Full {
let base_bytes = upq.base_bytes_per_vector();
let rescore_one = |item: &mut Scored| -> Result<()> {
let base: &[u8] =
unsafe { std::slice::from_raw_parts(item.ptr as *const u8, base_bytes) };
item.score = upq.decode_dot_from_base(base, &q_rot, q_norm)?;
Ok(())
};
if parallel {
use rayon::prelude::*;
scored.par_iter_mut().try_for_each(rescore_one)?;
} else {
scored.iter_mut().try_for_each(rescore_one)?;
}
scored.sort_by(|a, b| a.score.total_cmp(&b.score).then_with(|| a.vid.cmp(&b.vid)));
scored.truncate(query.k);
qprof.stage3_exact = query_profile::lap(&mut qprof_mark);
}
if qprof_on {
query_profile::record_query(qprof);
}
Ok(scored
.iter()
.take(query.k)
.filter_map(|si| {
self.vector_by_id_cache
.get(&VectorId(si.vid as u64))
.map(|desc| VectorHit {
vector_id: desc.vector_id,
frame_id: desc.owner_frame_id,
collection_id: desc.collection_id,
score: si.score,
})
})
.collect())
}
}