Struct ProductQuantizer 

pub struct ProductQuantizer<S: EmbeddingSpace, const M: usize, const NBITS: usize>
where
    [(); { _ }]:,
{ /* private fields */ }

Product Quantizer for vector compression and fast distance computation.

Product Quantization splits a D-dimensional vector into M subvectors, learns a codebook for each subspace via k-means, and encodes each subvector as the index of its nearest centroid.

Const generics:

• M: number of subquantizers
• NBITS: bits per centroid index (determines storage and centroid count)

The embedding dimension must be divisible by M.

After training, ProductQuantizer implements EmbeddingSpace with EmbeddingData = PQCode<M, NBITS>, allowing direct use with FlatIndex and other structures.

This implementation supports:

• Training via k-means++ on each subspace
• Encoding: vector -> PQCode<M, NBITS>
• Decoding: PQCode<M, NBITS> -> reconstructed vector
• ADC: Asymmetric Distance Computation via precomputed distance tables
• SDC: Symmetric Distance Computation via precomputed centroid-to-centroid distances

Note: PQ internally uses L2 distance for subspace quantization.

Implementations

impl<S: EmbeddingSpace, const M: usize, const NBITS: usize> ProductQuantizer<S, M, NBITS>

where [(); { _ }]:, <S::EmbeddingData as Embedding>::Scalar: Into<f32> + From<f32>,

pub const KSUB: usize

Number of centroids per subspace (2^NBITS)

pub fn new(space: S) -> Self

Create a new Product Quantizer.

Arguments

• space - The embedding space

Panics

Panics if dimension is not divisible by M.

pub fn space(&self) -> &S

Get a reference to the underlying embedding space.

pub fn m(&self) -> usize

Number of subquantizers.

pub fn ksub(&self) -> usize

Number of centroids per subspace.

pub fn dsub(&self) -> usize

pub fn encode_embedding( &mut self, embedding: &S::EmbeddingData, ) -> PQCode<M, NBITS>

Encode a single embedding to a PQ code.

pub fn decode_code(&self, code: &PQCode<M, NBITS>) -> S::EmbeddingData

Decode a PQ code to a reconstructed embedding.

pub fn train_on(&mut self, data: &[S::EmbeddingData])

Train the quantizer on a dataset using k-means++.

pub fn build_distance_table( &mut self, query: &S::EmbeddingData, ) -> PQDistanceTable<S, M, NBITS>

Build a distance table for ADC (Asymmetric Distance Computation).

pub fn sdc_table(&self) -> Option<&SDCTable<M, NBITS>>

Get the SDC table for symmetric distance computation.

Returns None if the quantizer is not trained.

impl<S: EmbeddingSpace, const M: usize, const NBITS: usize> ProductQuantizer<S, M, NBITS>

where [(); { _ }]:, <S::EmbeddingData as Embedding>::Scalar: Into<f32> + From<f32>,

pub fn save_codebook(&self, path: &Path) -> Result<(), PQError>

Save the trained codebook to a file.

pub fn load_codebook(path: &Path, space: S) -> Result<Self, PQError>

Load a trained codebook from a file and reconstruct the ProductQuantizer.

Trait Implementations

impl<S: Clone + EmbeddingSpace, const M: usize, const NBITS: usize> Clone for ProductQuantizer<S, M, NBITS>

where [(); { _ }]:,

fn clone(&self) -> ProductQuantizer<S, M, NBITS>

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<S: EmbeddingSpace, const M: usize, const NBITS: usize> Codec<S> for ProductQuantizer<S, M, NBITS>

where [(); { _ }]:, <S::EmbeddingData as Embedding>::Scalar: Into<f32> + From<f32>,

type Encoded = PQCode<M, NBITS>

The compressed representation produced by encode.

type EncodeRef<'b> = EagerOpRef<PQCode<M, NBITS>, PQError> where Self: 'b

Handle to an in-flight encode operation.

type DecodeRef<'b> = EagerOpRef<<S as EmbeddingSpace>::EmbeddingData, PQError> where Self: 'b

Handle to an in-flight decode operation.

type TrainRef<'b> = EagerOpRef<(), PQError> where Self: 'b

Handle to an in-flight train operation.

type ObserveRef<'b> = EagerOpRef<(), PQError> where Self: 'b

Handle to an in-flight observe operation.

fn encode(&mut self, embedding: &S::EmbeddingData) -> Self::EncodeRef<'_>

Compress a single embedding.

fn encode_batch( &mut self, embeddings: &[S::EmbeddingData], ) -> Vec<Self::EncodeRef<'_>>

Compress a batch of embeddings.

fn decode(&self, encoded: &Self::Encoded) -> Self::DecodeRef<'_>

Reconstruct an embedding from its compressed form.

fn decode_batch(&self, encoded: &[Self::Encoded]) -> Vec<Self::DecodeRef<'_>>

Reconstruct a batch of embeddings from their compressed forms.

fn code_size(&self) -> Option<usize>

Returns the fixed byte-size of an encoded vector, if the encoding is fixed-size. Variable-length encodings return None.

fn train(&mut self, embeddings: &[S::EmbeddingData]) -> Self::TrainRef<'_>

Batch training from a set of embeddings.

fn observe(&mut self, _embedding: &S::EmbeddingData) -> Self::ObserveRef<'_>

Online/incremental update from a single observation.

fn observe_batch( &mut self, embeddings: &[S::EmbeddingData], ) -> Vec<Self::ObserveRef<'_>>

Online/incremental update from a batch of observations.

fn is_trained(&self) -> bool

Whether the codec has been trained and is ready to encode/decode.

impl<S: EmbeddingSpace, const M: usize, const NBITS: usize> Debug for ProductQuantizer<S, M, NBITS>

where [(); { _ }]:,

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<S: EmbeddingSpace + Default, const M: usize, const NBITS: usize> Default for ProductQuantizer<S, M, NBITS>

where [(); { _ }]:, <S::EmbeddingData as Embedding>::Scalar: Into<f32> + From<f32>,

fn default() -> Self

Returns the “default value” for a type.

impl<S: EmbeddingSpace, const M: usize, const NBITS: usize> EmbeddingSpace for ProductQuantizer<S, M, NBITS>

where [(); { _ }]:, <S::EmbeddingData as Embedding>::Scalar: Into<f32> + From<f32>,

type EmbeddingData = PQCode<M, NBITS>

type DistanceValue = <S as EmbeddingSpace>::DistanceValue

type Prepared = PQCode<M, NBITS>

Precomputed state for efficient distance queries.

fn space_id(&self) -> &'static str

fn distance( &self, lhs: &Self::EmbeddingData, rhs: &Self::EmbeddingData, ) -> Self::DistanceValue

Compute distance between two embeddings.

fn prepare(&self, embedding: &Self::EmbeddingData) -> Self::Prepared

Prepare a query embedding for efficient repeated distance computations.

fn distance_prepared( &self, prepared: &Self::Prepared, target: &Self::EmbeddingData, ) -> Self::DistanceValue

Compute distance using prepared query state.

fn length() -> usize

fn slice_distance(a: &[f32], b: &[f32]) -> f32

Compute the space’s distance metric on raw scalar slices of arbitrary length.

fn infinite_mapping(native_distance: &Self::DistanceValue) -> f32

Maps a finite distance range to an infinite range (e.g., tan(pi * x / 4))

fn create_embedding( data: Vec<<Self::EmbeddingData as Embedding>::Scalar>, ) -> Self::EmbeddingData

fn create_distance(dist: f32) -> Self::DistanceValue

fn zero_vector() -> Self::EmbeddingData

fn zero_distance() -> Self::DistanceValue

impl<S: EmbeddingSpace, const M: usize, const NBITS: usize> PartialEq for ProductQuantizer<S, M, NBITS>

where [(); { _ }]:,

fn eq(&self, other: &Self) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl<S: EmbeddingSpace, const M: usize, const NBITS: usize> Eq for ProductQuantizer<S, M, NBITS>

where [(); { _ }]:,