Struct LinearQuantizer 

pub struct LinearQuantizer { /* private fields */ }

Linear uniform quantizer

Implementations

impl LinearQuantizer

pub fn new(min: f32, max: f32, bits: u8) -> TokenizerResult<Self>

Create a new linear quantizer

pub fn normalized(bits: u8) -> TokenizerResult<Self>

Create quantizer for normalized [-1, 1] range

pub fn range(&self) -> (f32, f32)

Get the range

pub fn bits(&self) -> u8

Get number of bits

pub fn step_size(&self) -> f32

Get the quantization step size

Trait Implementations

impl Clone for LinearQuantizer

fn clone(&self) -> LinearQuantizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for LinearQuantizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Quantizer for LinearQuantizer

fn quantize(&self, value: f32) -> i32

Quantize a continuous value to discrete level

fn dequantize(&self, level: i32) -> f32

Dequantize a discrete level back to continuous value

fn num_levels(&self) -> usize

Get number of quantization levels

impl SignalTokenizer for LinearQuantizer

fn encode(&self, signal: &Array1<f32>) -> TokenizerResult<Array1<f32>>

Encode a continuous signal into tokens/embeddings

fn decode(&self, tokens: &Array1<f32>) -> TokenizerResult<Array1<f32>>

Decode tokens/embeddings back to continuous signal

fn embed_dim(&self) -> usize

Get the embedding dimension

fn vocab_size(&self) -> usize

Get the vocabulary size (for discrete tokenizers, 0 for continuous)

impl TokenizerIO for LinearQuantizer

fn save_json<P: AsRef<Path>>(&self, path: P) -> TokenizerResult<()>

Save to JSON file

fn load_json<P: AsRef<Path>>(path: P) -> TokenizerResult<Self>

Load from JSON file

fn save_binary<P: AsRef<Path>>(&self, path: P) -> TokenizerResult<()>

Save to binary file (using bincode)

fn load_binary<P: AsRef<Path>>(path: P) -> TokenizerResult<Self>

Load from binary file (using bincode)