use crate::codec::bitvectors::flat_bit_vectors_scorer::FlatBitVectorsScorer;
use crate::core::codecs::hnsw::flat_vectors_format::FlatVectorsFormat;
use crate::core::codecs::knn_field_vectors_writer::VectorInput;
use crate::core::codecs::knn_vectors_format::KnnVectorsFormat;
use crate::core::codecs::knn_vectors_writer::KnnVectorsWriter;
use crate::core::codecs::lucene99::lucene99_flat_vectors_format::Lucene99FlatVectorsFormat;
use crate::core::codecs::lucene99::lucene99_flat_vectors_reader::Lucene99FlatVectorsReader;
use crate::core::codecs::lucene99::lucene99_flat_vectors_writer::Lucene99FlatVectorsWriter;
use crate::core::codecs::lucene99::lucene99_hnsw_vectors_format::{
DEFAULT_BEAM_WIDTH, DEFAULT_MAX_CONN, DEFAULT_NUM_MERGE_WORKER, MAXIMUM_BEAM_WIDTH,
MAXIMUM_MAX_CONN,
};
use crate::core::codecs::lucene99::lucene99_hnsw_vectors_reader::Lucene99HnswVectorsReader;
use crate::core::codecs::lucene99::lucene99_hnsw_vectors_writer::Lucene99HnswVectorsWriter;
use crate::core::index::codec_reader::CodecReader;
use crate::core::index::field_info::FieldInfo;
use crate::core::index::index_reader::Identity;
use crate::core::index::merge_state::MergeState;
use crate::core::index::segment_info::SegmentInfo;
use crate::core::index::segment_read_state::SegmentReadState;
use crate::core::index::segment_write_state::SegmentWriteState;
use crate::core::index::sorter::DocMap;
use crate::core::index::vector_encoding::VectorEncoding;
use crate::core::search::task_executor::TaskExecutor;
use crate::core::store::directory::Directory;
use crate::core::store::{IndexInput, IndexOutput};
use crate::core::util::HasIdentity;
use crate::core::util::accountable::Accountable;
use crate::core::util::close::Closeable;
use crate::core::util::error::lucene_error::{LuceneError, Result};
use rayon::ThreadPool;
use std::fmt::{Display, Formatter};
use std::sync::{Arc, OnceLock};
pub struct HnswBitVectorsFormat {
max_conn: usize,
beam_width: usize,
flat_vectors_format: Lucene99FlatVectorsFormat<FlatBitVectorsScorer>,
num_merge_workers: usize,
merge_exec: Option<Arc<TaskExecutor>>,
identity: Identity,
}
pub const NAME: &str = "HnswBitVectorsFormat";
impl HnswBitVectorsFormat {
pub fn new() -> Result<Self> {
Self::with_graph_para_with_threads(
DEFAULT_MAX_CONN,
DEFAULT_BEAM_WIDTH,
DEFAULT_NUM_MERGE_WORKER,
None,
)
}
pub fn with_graph_para(max_conn: usize, beam_width: usize) -> Result<Self> {
Self::with_graph_para_with_threads(max_conn, beam_width, DEFAULT_NUM_MERGE_WORKER, None)
}
pub fn with_graph_para_with_threads(
max_conn: usize,
beam_width: usize,
num_merge_workers: usize,
merge_exec: Option<Arc<ThreadPool>>,
) -> Result<Self> {
if max_conn == 0 || max_conn > MAXIMUM_MAX_CONN {
return Err(LuceneError::illegal_argument(format!(
"maxConn must be positive and less than or equal to {}; maxConn={}",
MAXIMUM_MAX_CONN, max_conn
)));
}
if beam_width == 0 || beam_width > MAXIMUM_BEAM_WIDTH {
return Err(LuceneError::illegal_argument(format!(
"beamWidth must be positive and less than or equal to {}; beamWidth={}",
MAXIMUM_BEAM_WIDTH, beam_width
)));
}
if num_merge_workers == 1 && merge_exec.is_some() {
return Err(LuceneError::illegal_argument(
"No executor service is needed as we'll use single thread to merge",
));
}
Ok(Self {
max_conn,
beam_width,
num_merge_workers,
merge_exec: merge_exec.map(|executor| Arc::new(TaskExecutor::new(executor))),
flat_vectors_format: Lucene99FlatVectorsFormat::new(FlatBitVectorsScorer),
identity: Identity::new(),
})
}
}
impl HasIdentity for HnswBitVectorsFormat {
fn identity(&self) -> &Identity {
&self.identity
}
}
impl KnnVectorsFormat for HnswBitVectorsFormat {
fn get_name(&self) -> &str {
NAME
}
type KnnVectorsWriter<T: IndexOutput> = FlatBitVectorsWriter<
Lucene99HnswVectorsWriter<Lucene99FlatVectorsWriter<T, FlatBitVectorsScorer>>,
>;
fn fields_writer<D1, D2>(
&self,
state: &SegmentWriteState<D1>,
segment_info: &SegmentInfo<D2>,
) -> Result<Self::KnnVectorsWriter<D1::IndexOutput>>
where
D1: Directory,
{
Ok(FlatBitVectorsWriter::new(Lucene99HnswVectorsWriter::new(
state,
self.max_conn,
self.beam_width,
FlatVectorsFormat::fields_writer(&self.flat_vectors_format, state, segment_info)?,
self.num_merge_workers,
self.merge_exec.clone(),
segment_info,
)?))
}
type KnnVectorsReader<T: IndexInput> =
Lucene99HnswVectorsReader<Lucene99FlatVectorsReader<T, FlatBitVectorsScorer>, T>;
fn fields_reader<D1, D2>(
&self,
state: &SegmentReadState<D1>,
segment_info: &SegmentInfo<D2>,
) -> Result<Self::KnnVectorsReader<D1::IndexInput>>
where
D1: Directory,
{
Lucene99HnswVectorsReader::new(
state,
FlatVectorsFormat::fields_reader(&self.flat_vectors_format, state, segment_info)?,
segment_info,
)
}
fn get_max_dimensions(&self, _field_name: &str) -> Result<usize> {
Ok(1024)
}
fn for_name(name: &str) -> Result<Arc<Self>> {
static FORMAT: OnceLock<Arc<HnswBitVectorsFormat>> = OnceLock::new();
match name {
NAME => {
if let Some(format) = FORMAT.get() {
return Ok(Arc::clone(format));
}
let format = Arc::new(Self::new()?);
if FORMAT.set(Arc::clone(&format)).is_ok() {
Ok(format)
} else {
FORMAT.get().map(Arc::clone).ok_or_else(|| {
LuceneError::illegal_state(format!(
"failed to initialize vectors format named \"{NAME}\""
))
})
}
},
_ => Err(LuceneError::illegal_argument(format!(
"Could not load vectors format named \"{name}\""
))),
}
}
}
impl Display for HnswBitVectorsFormat {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}(name={}, maxConn={}, beamWidth={}, flatVectorFormat={})",
NAME, NAME, self.max_conn, self.beam_width, self.flat_vectors_format
)
}
}
pub struct FlatBitVectorsWriter<W> {
delegate: W,
}
impl<W> FlatBitVectorsWriter<W> {
pub fn new(delegate: W) -> Self {
Self { delegate }
}
}
impl<O, W> KnnVectorsWriter<O> for FlatBitVectorsWriter<W>
where
O: IndexOutput,
W: KnnVectorsWriter<O>,
{
fn merge_one_field<D1, D2, CR>(
&mut self,
field_info: &Arc<FieldInfo>,
merge_state: &MergeState<'_, D1, CR>,
segment_write_state: &SegmentWriteState<&D2>,
) -> Result<()>
where
D2: Directory<IndexOutput = O>,
CR: CodecReader,
{
self
.delegate
.merge_one_field(field_info, merge_state, segment_write_state)
}
fn finish(&mut self) -> Result<()> {
self.delegate.finish()
}
fn add_field<D1, D2>(
&mut self,
write_state: &SegmentWriteState<D1>,
segment_info: &SegmentInfo<D2>,
field_info: Arc<FieldInfo>,
) -> Result<usize>
where
D1: Directory<IndexOutput = O>,
{
if !matches!(field_info.get_vector_encoding(), VectorEncoding::BYTE(_)) {
return Err(LuceneError::illegal_argument(
"HnswBitVectorsFormat only supports BYTE encoding",
));
}
self
.delegate
.add_field(write_state, segment_info, field_info)
}
fn flush<DM>(&mut self, max_doc: i32, sort_map: Option<&DM>) -> Result<()>
where
DM: DocMap,
{
self.delegate.flush(max_doc, sort_map)
}
fn add_value<V: VectorInput + ?Sized>(
&mut self,
doc_id: i32,
vector_value: &V,
field_vectors_writers_idx: usize,
) -> Result<()> {
self
.delegate
.add_value(doc_id, vector_value, field_vectors_writers_idx)
}
}
impl<W> Closeable for FlatBitVectorsWriter<W>
where
W: Closeable,
{
fn close(&mut self) -> Result<()> {
self.delegate.close()
}
}
impl<W> Accountable for FlatBitVectorsWriter<W>
where
W: Accountable,
{
fn ram_bytes_used(&self) -> Result<i64> {
self.delegate.ram_bytes_used()
}
}