Struct TrainableContinuousTokenizer 

pub struct TrainableContinuousTokenizer { /* private fields */ }

Trainable continuous tokenizer with gradient descent

Implementations

impl TrainableContinuousTokenizer

pub fn new(input_dim: usize, embed_dim: usize) -> CandleResult<Self>

Create a new trainable continuous tokenizer

pub fn train_batch( &self, signals: &[Array1<f32>], optimizer: &mut AdamW, ) -> TokenizerResult<f32>

Train on a batch of signals

pub fn train( &self, training_data: &[Array1<f32>], config: &TrainingConfig, ) -> TokenizerResult<Vec<f32>>

Train on a dataset

pub fn encode(&self, signal: &Array1<f32>) -> TokenizerResult<Array1<f32>>

Encode a signal (inference)

pub fn decode(&self, embeddings: &Array1<f32>) -> TokenizerResult<Array1<f32>>

Decode embeddings (inference)

pub fn get_encoder_weights(&self) -> TokenizerResult<Array2<f32>>

Get encoder weights as Array2

pub fn get_decoder_weights(&self) -> TokenizerResult<Array2<f32>>

Get decoder weights as Array2

pub fn evaluate( &self, test_data: &[Array1<f32>], ) -> TokenizerResult<ReconstructionMetrics>

Evaluate reconstruction quality on test data

pub fn embed_dim(&self) -> usize

Get embedding dimension

pub fn input_dim(&self) -> usize

Get input dimension

pub fn save_checkpoint<P: AsRef<Path>>( &self, path: P, version: &str, training_config: Option<TrainingConfig>, metrics: Option<ReconstructionMetrics>, ) -> TokenizerResult<()>

Save model to checkpoint

pub fn load_checkpoint<P: AsRef<Path>>(path: P) -> TokenizerResult<Self>

Load model from checkpoint

pub fn peek_checkpoint<P: AsRef<Path>>( path: P, ) -> TokenizerResult<ModelMetadata>

Get metadata from a checkpoint file without loading the full model