[−][src]Struct rust_tokenizers::preprocessing::tokenizer::base_tokenizer::TokenRef
A token that references the original text
Fields
text: &'a str
offset: Offset
reference_offsets: &'a [OffsetSize]
mask: Mask
Implementations
impl<'a> TokenRef<'a>
[src]
pub fn new(text: &'a str, offsets: &'a [OffsetSize]) -> TokenRef<'a>
[src]
pub fn to_owned(self) -> Token
[src]
Trait Implementations
impl<'a> Clone for TokenRef<'a>
[src]
impl<'a> ConsolidatableTokens<TokenRef<'a>> for Vec<TokenRef<'a>>
[src]
fn iter_consolidate_tokens(&self) -> ConsolidatedTokenIterator<TokenRef<'a>>
[src]
impl<'a> Copy for TokenRef<'a>
[src]
impl<'a> Debug for TokenRef<'a>
[src]
impl<'a> From<&'a Token> for TokenRef<'a>
[src]
impl<'a> From<TokenRef<'a>> for Token
[src]
impl<'a> PartialEq<TokenRef<'a>> for TokenRef<'a>
[src]
impl<'a> StructuralPartialEq for TokenRef<'a>
[src]
impl<'a> TokenTrait for TokenRef<'a>
[src]
Auto Trait Implementations
impl<'a> RefUnwindSafe for TokenRef<'a>
impl<'a> Send for TokenRef<'a>
impl<'a> Sync for TokenRef<'a>
impl<'a> Unpin for TokenRef<'a>
impl<'a> UnwindSafe for TokenRef<'a>
Blanket Implementations
impl<T> Any for T where
T: 'static + ?Sized,
[src]
T: 'static + ?Sized,
impl<T> Borrow<T> for T where
T: ?Sized,
[src]
T: ?Sized,
impl<T> BorrowMut<T> for T where
T: ?Sized,
[src]
T: ?Sized,
fn borrow_mut(&mut self) -> &mut T
[src]
impl<T> From<T> for T
[src]
impl<T, U> Into<U> for T where
U: From<T>,
[src]
U: From<T>,
impl<T> ToOwned for T where
T: Clone,
[src]
T: Clone,
type Owned = T
The resulting type after obtaining ownership.
fn to_owned(&self) -> T
[src]
fn clone_into(&self, target: &mut T)
[src]
impl<T, U> TryFrom<U> for T where
U: Into<T>,
[src]
U: Into<T>,
type Error = Infallible
The type returned in the event of a conversion error.
fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>
[src]
impl<T, U> TryInto<U> for T where
U: TryFrom<T>,
[src]
U: TryFrom<T>,