pub struct Tokens { /* private fields */ }
Expand description
A state metadata holding a list of tokens
Implementations
sourceimpl Tokens
impl Tokens
The metadata used for token mutators
sourcepub fn add_tokens<IT, V>(&mut self, tokens: IT) -> &mut Self where
IT: IntoIterator<Item = V>,
V: AsRef<Vec<u8>>,
pub fn add_tokens<IT, V>(&mut self, tokens: IT) -> &mut Self where
IT: IntoIterator<Item = V>,
V: AsRef<Vec<u8>>,
Add tokens from a slice of Vecs of bytes
sourcepub fn add_from_files<IT, P>(self, files: IT) -> Result<Self, Error> where
IT: IntoIterator<Item = P>,
P: AsRef<Path>,
pub fn add_from_files<IT, P>(self, files: IT) -> Result<Self, Error> where
IT: IntoIterator<Item = P>,
P: AsRef<Path>,
Build tokens from files
sourcepub fn parse_autodict(&mut self, slice: &[u8], size: usize)
pub fn parse_autodict(&mut self, slice: &[u8], size: usize)
Parse autodict section
sourcepub unsafe fn from_ptrs(
token_start: *const u8,
token_stop: *const u8
) -> Result<Self, Error>
pub unsafe fn from_ptrs(
token_start: *const u8,
token_stop: *const u8
) -> Result<Self, Error>
Create a token section from a start and an end pointer Reads from an autotokens section, returning the count of new entries read
Safety
The caller must ensure that the region between token_start
and token_stop
is a valid region, containing autotokens in the exepcted format.
sourcepub fn from_file<P>(file: P) -> Result<Self, Error> where
P: AsRef<Path>,
pub fn from_file<P>(file: P) -> Result<Self, Error> where
P: AsRef<Path>,
Creates a new instance from a file
sourcepub fn add_token(&mut self, token: &Vec<u8>) -> bool
pub fn add_token(&mut self, token: &Vec<u8>) -> bool
Adds a token to a dictionary, checking it is not a duplicate
Returns false
if the token was already present and did not get added.
Trait Implementations
sourceimpl AddAssign<&[Vec<u8, Global>]> for Tokens
impl AddAssign<&[Vec<u8, Global>]> for Tokens
sourcefn add_assign(&mut self, other: &[Vec<u8>])
fn add_assign(&mut self, other: &[Vec<u8>])
Performs the +=
operation. Read more
sourceimpl AddAssign<Tokens> for Tokens
impl AddAssign<Tokens> for Tokens
sourcefn add_assign(&mut self, other: Self)
fn add_assign(&mut self, other: Self)
Performs the +=
operation. Read more
sourceimpl<'de> Deserialize<'de> for Tokens
impl<'de> Deserialize<'de> for Tokens
sourcefn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error> where
__D: Deserializer<'de>,
fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error> where
__D: Deserializer<'de>,
Deserialize this value from the given Serde deserializer. Read more
sourceimpl<'it> IntoIterator for &'it Tokens
impl<'it> IntoIterator for &'it Tokens
Auto Trait Implementations
impl RefUnwindSafe for Tokens
impl Send for Tokens
impl Sync for Tokens
impl Unpin for Tokens
impl UnwindSafe for Tokens
Blanket Implementations
sourceimpl<T> BorrowMut<T> for T where
T: ?Sized,
impl<T> BorrowMut<T> for T where
T: ?Sized,
const: unstable · sourcefn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
sourceimpl<Tail, T> Prepend<T> for Tail
impl<Tail, T> Prepend<T> for Tail
type PreprendResult = Tail
type PreprendResult = Tail
The Resulting TupleList
, of an Prepend::prepend()
call,
including the prepended entry. Read more
sourcefn prepend(self, value: T) -> (T, <Tail as Prepend<T>>::PreprendResult)
fn prepend(self, value: T) -> (T, <Tail as Prepend<T>>::PreprendResult)
Prepend a value to this tuple, returning a new tuple with prepended value.