Struct lance_index::vector::pq::ProductQuantizerImpl

pub struct ProductQuantizerImpl<T: ArrowFloatType + Cosine + Dot + L2> {
    pub num_bits: u32,
    pub num_sub_vectors: usize,
    pub dimension: usize,
    pub metric_type: MetricType,
    pub codebook: Arc<T::ArrayType>,
}

Product Quantization, optimized for [Apache Arrow] buffer memory layout.

Fields

num_bits: u32

Number of bits for the centroids.

Only support 8, as one of u8 byte now.

num_sub_vectors: usize

Number of sub-vectors.

dimension: usize

Vector dimension.

metric_type: MetricType

Distance type.

codebook: Arc<T::ArrayType>

PQ codebook

((2 ^ nbits) * num_subvector * sub_vector_length) of f32

Use a layout that is cache / SIMD friendly to compute centroid. But not sure how to make distance lookup via PQ code lookup be cache friendly tho.

Layout:

• row: all centroids for the same sub-vector.
• column: the centroid value of the n-th sub-vector.

// Centroids for a sub-vector.
Codebook[sub_vector_id][pq_code]

Implementations

impl<T: ArrowFloatType + Cosine + Dot + L2> ProductQuantizerImpl<T>

pub fn new( m: usize, nbits: u32, dimension: usize, codebook: Arc<T::ArrayType>, metric_type: MetricType ) -> Self

Create a ProductQuantizer with pre-trained codebook.

pub fn num_centroids(num_bits: u32) -> usize

pub fn codebook_length(num_bits: u32, num_sub_vectors: usize) -> usize

Calculate codebook length.

pub fn centroids(&self, sub_vector_idx: usize) -> &[T::Native]

Get the centroids for one sub-vector.

Returns a flatten num_centroids * sub_vector_width f32 array.

Trait Implementations

impl<T: Debug + ArrowFloatType + Cosine + Dot + L2> Debug for ProductQuantizerImpl<T>

where T::ArrayType: Debug,

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<T: ArrowFloatType + Cosine + Dot + L2 + 'static> ProductQuantizer for ProductQuantizerImpl<T>

fn as_any(&self) -> &dyn Any

fn transform<'life0, 'life1, 'async_trait>( &'life0 self, data: &'life1 dyn Array ) -> Pin<Box<dyn Future<Output = Result<ArrayRef>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Transform a vector column to PQ code column.

fn build_distance_table( &self, query: &dyn Array, code: &UInt8Array ) -> Result<ArrayRef>

Build the distance lookup in f32.

fn num_bits(&self) -> u32

Get the centroids for one sub-vector.

fn num_sub_vectors(&self) -> usize

Number of sub-vectors

fn dimension(&self) -> usize

fn codebook_as_fsl(&self) -> FixedSizeListArray

fn use_residual(&self) -> bool

Whether to use residual as input or not.