pub struct BinarySerializer { /* private fields */ }Expand description
Binary serializer/deserializer for tokenizers
Implementations§
Source§impl BinarySerializer
impl BinarySerializer
Sourcepub fn new(config: BinaryConfig) -> Self
pub fn new(config: BinaryConfig) -> Self
Create a new binary serializer with the given configuration
Sourcepub fn serialize<P: AsRef<Path>>(
&self,
tokenizer: &BinaryTokenizer,
tokenizer_type: &str,
path: P,
) -> Result<BinaryHeader>
pub fn serialize<P: AsRef<Path>>( &self, tokenizer: &BinaryTokenizer, tokenizer_type: &str, path: P, ) -> Result<BinaryHeader>
Serialize a tokenizer to binary format
Sourcepub fn deserialize<P: AsRef<Path>>(
&self,
path: P,
) -> Result<(BinaryTokenizer, BinaryHeader)>
pub fn deserialize<P: AsRef<Path>>( &self, path: P, ) -> Result<(BinaryTokenizer, BinaryHeader)>
Deserialize a tokenizer from binary format
Sourcepub fn get_file_info<P: AsRef<Path>>(&self, path: P) -> Result<BinaryHeader>
pub fn get_file_info<P: AsRef<Path>>(&self, path: P) -> Result<BinaryHeader>
Get file info without fully loading the tokenizer
Auto Trait Implementations§
impl Freeze for BinarySerializer
impl RefUnwindSafe for BinarySerializer
impl Send for BinarySerializer
impl Sync for BinarySerializer
impl Unpin for BinarySerializer
impl UnsafeUnpin for BinarySerializer
impl UnwindSafe for BinarySerializer
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
Source§impl<T> Instrument for T
impl<T> Instrument for T
Source§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
Source§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more