Struct ContinuousTokenizer 

pub struct ContinuousTokenizer { /* private fields */ }

Continuous tokenizer that projects signals to embedding space

Implementations

impl ContinuousTokenizer

pub fn new(input_dim: usize, embed_dim: usize) -> Self

Create a new continuous tokenizer

pub fn input_dim(&self) -> usize

Get input dimension

pub fn set_encoder(&mut self, weights: Array2<f32>) -> TokenizerResult<()>

Set encoder weights

pub fn set_decoder(&mut self, weights: Array2<f32>) -> TokenizerResult<()>

Set decoder weights

pub fn encoder(&self) -> &Array2<f32>

Get encoder weights

pub fn decoder(&self) -> &Array2<f32>

Get decoder weights

Trait Implementations

impl Clone for ContinuousTokenizer

fn clone(&self) -> ContinuousTokenizer

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ContinuousTokenizer

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl SignalTokenizer for ContinuousTokenizer

fn encode(&self, signal: &Array1<f32>) -> TokenizerResult<Array1<f32>>

Encode a continuous signal into tokens/embeddings

fn decode(&self, tokens: &Array1<f32>) -> TokenizerResult<Array1<f32>>

Decode tokens/embeddings back to continuous signal

fn embed_dim(&self) -> usize

Get the embedding dimension

fn vocab_size(&self) -> usize

Get the vocabulary size (for discrete tokenizers, 0 for continuous)

impl TokenizerIO for ContinuousTokenizer

fn save_json<P: AsRef<Path>>(&self, path: P) -> TokenizerResult<()>

Save to JSON file

fn load_json<P: AsRef<Path>>(path: P) -> TokenizerResult<Self>

Load from JSON file

fn save_binary<P: AsRef<Path>>(&self, path: P) -> TokenizerResult<()>

Save to binary file (using bincode)

fn load_binary<P: AsRef<Path>>(path: P) -> TokenizerResult<Self>

Load from binary file (using bincode)