Struct rust_tokenizers::Token [−][src]
Owned token that references the original text but stores its own string representation.
Fields
text: String
String representation
offset: Offset
Start and end positions of the token with respect to the original text
reference_offsets: Vec<OffsetSize>
Sequence of positions with respect to the original text contained in the token.
For example, if the token offset is start: 4, end: 10
, corresponding reference_offsets are [4, 5, 6, 7, 8, 9]
mask: Mask
Mask indicating the type of the token
Implementations
impl Token
[src]
pub fn new(text: String) -> Token
[src]
Creates a new owned token from a String
.
Parameters
- text (
String
): text reference
Example
use rust_tokenizers::Token; let text = "world".to_string(); let token = Token::new(text);
pub fn as_ref(&self) -> TokenRef<'_>
[src]
Converts an owned token to a reference form
Example
use rust_tokenizers::Token; let text = "world".to_string(); let token = Token::new(text); let token_ref = token.as_ref();
Trait Implementations
impl Clone for Token
[src]
impl ConsolidatableTokens<Token> for Vec<Token>
[src]
fn iter_consolidate_tokens(&self) -> ConsolidatedTokenIterator<'_, Token>ⓘNotable traits for ConsolidatedTokenIterator<'a, T>
impl<'a, T> Iterator for ConsolidatedTokenIterator<'a, T> where
T: TokenTrait, type Item = &'a [T];
[src]
Notable traits for ConsolidatedTokenIterator<'a, T>
impl<'a, T> Iterator for ConsolidatedTokenIterator<'a, T> where
T: TokenTrait, type Item = &'a [T];
impl Debug for Token
[src]
impl From<&'_ str> for Token
[src]
impl<'a> From<&'a Token> for TokenRef<'a>
[src]
impl<'a> From<TokenRef<'a>> for Token
[src]
impl PartialEq<Token> for Token
[src]
impl StructuralPartialEq for Token
[src]
impl TokenTrait for Token
[src]
Auto Trait Implementations
impl RefUnwindSafe for Token
impl Send for Token
impl Sync for Token
impl Unpin for Token
impl UnwindSafe for Token
Blanket Implementations
impl<T> Any for T where
T: 'static + ?Sized,
[src]
T: 'static + ?Sized,
impl<T> Borrow<T> for T where
T: ?Sized,
[src]
T: ?Sized,
impl<T> BorrowMut<T> for T where
T: ?Sized,
[src]
T: ?Sized,
pub fn borrow_mut(&mut self) -> &mut T
[src]
impl<T> From<T> for T
[src]
impl<T, U> Into<U> for T where
U: From<T>,
[src]
U: From<T>,
impl<T> Pointable for T
pub const ALIGN: usize
type Init = T
The type for initializers.
pub unsafe fn init(init: <T as Pointable>::Init) -> usize
pub unsafe fn deref<'a>(ptr: usize) -> &'a T
pub unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T
pub unsafe fn drop(ptr: usize)
impl<T> ToOwned for T where
T: Clone,
[src]
T: Clone,
type Owned = T
The resulting type after obtaining ownership.
pub fn to_owned(&self) -> T
[src]
pub fn clone_into(&self, target: &mut T)
[src]
impl<T, U> TryFrom<U> for T where
U: Into<T>,
[src]
U: Into<T>,
type Error = Infallible
The type returned in the event of a conversion error.
pub fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>
[src]
impl<T, U> TryInto<U> for T where
U: TryFrom<T>,
[src]
U: TryFrom<T>,