Struct PyramidTokenizer 

pub struct PyramidTokenizer { /* private fields */ }

Pyramid tokenizer with residual connections

Each level encodes the residual from the previous level’s reconstruction, similar to residual VQ (RVQ) used in audio codecs.

Implementations

impl PyramidTokenizer

pub fn new( input_dim: usize, embed_dim_per_level: usize, num_levels: usize, ) -> Self

Create a new pyramid tokenizer

pub fn without_residual(self) -> Self

Disable residual encoding (use independent levels)

pub fn encode_pyramid( &self, signal: &Array1<f32>, ) -> TokenizerResult<Vec<Array1<f32>>>

Encode with residual pyramid

pub fn decode_pyramid( &self, embeddings: &[Array1<f32>], ) -> TokenizerResult<Array1<f32>>

Decode from pyramid embeddings

pub fn num_levels(&self) -> usize

Get number of levels

Trait Implementations

impl Clone for PyramidTokenizer

fn clone(&self) -> PyramidTokenizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for PyramidTokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl SignalTokenizer for PyramidTokenizer

fn encode(&self, signal: &Array1<f32>) -> TokenizerResult<Array1<f32>>

Encode a continuous signal into tokens/embeddings

fn decode(&self, tokens: &Array1<f32>) -> TokenizerResult<Array1<f32>>

Decode tokens/embeddings back to continuous signal

fn embed_dim(&self) -> usize

Get the embedding dimension

fn vocab_size(&self) -> usize

Get the vocabulary size (for discrete tokenizers, 0 for continuous)