Struct MessagePackSerializer 

pub struct MessagePackSerializer { /* private fields */ }

MessagePack serializer for tokenizers and tokenized inputs

Implementations

impl MessagePackSerializer

pub fn new(config: MessagePackConfig) -> Self

Create a new MessagePack serializer with the given configuration

pub fn default() -> Self

Create a new MessagePack serializer with default configuration

pub fn serialize_tokenizer<T: Tokenizer>( &self, tokenizer: &T, metadata: Option<HashMap<String, String>>, ) -> Result<Vec<u8>>

Serialize a tokenizer to MessagePack format

pub fn serialize_tokenized_input( &self, input: &TokenizedInput, ) -> Result<Vec<u8>>

Serialize a tokenized input to MessagePack format

pub fn serialize_tokenized_batch( &self, batch: &[TokenizedInput], ) -> Result<Vec<u8>>

Serialize a TokenizedInput batch to MessagePack format

pub fn deserialize_tokenizer_config( &self, data: &[u8], ) -> Result<MessagePackTokenizerConfig>

Deserialize a tokenizer configuration from MessagePack format

pub fn deserialize_tokenized_input(&self, data: &[u8]) -> Result<TokenizedInput>

Deserialize a tokenized input from MessagePack format

pub fn deserialize_tokenized_batch( &self, data: &[u8], ) -> Result<Vec<TokenizedInput>>

Deserialize a batch of tokenized inputs from MessagePack format

pub fn save_tokenizer_to_file<T: Tokenizer, P: AsRef<Path>>( &self, tokenizer: &T, path: P, metadata: Option<HashMap<String, String>>, ) -> Result<()>

Save a tokenizer to a MessagePack file

pub fn save_tokenized_input_to_file<P: AsRef<Path>>( &self, input: &TokenizedInput, path: P, ) -> Result<()>

Save a tokenized input to a MessagePack file

pub fn load_tokenizer_config_from_file<P: AsRef<Path>>( &self, path: P, ) -> Result<MessagePackTokenizerConfig>

Load a tokenizer configuration from a MessagePack file

pub fn load_tokenized_input_from_file<P: AsRef<Path>>( &self, path: P, ) -> Result<TokenizedInput>

Load a tokenized input from a MessagePack file

pub fn validate_messagepack_data(&self, data: &[u8]) -> Result<bool>

Validate MessagePack data structure

pub fn get_messagepack_info( &self, data: &[u8], ) -> Result<HashMap<String, String>>

Get information about MessagePack data

pub fn compare_messagepack_files<P1: AsRef<Path>, P2: AsRef<Path>>( &self, path1: P1, path2: P2, ) -> Result<HashMap<String, String>>

Compare two MessagePack files