Struct TextSequence 

pub struct TextSequence {
    pub text: String,
    pub tokens: Option<Vec<String>>,
    pub token_ids: Option<Vec<usize>>,
}

Text sequence data container

Fields

text: String

tokens: Option<Vec<String>>

token_ids: Option<Vec<usize>>

Implementations

impl TextSequence

pub fn new(text: String) -> Self

pub fn with_tokens(self, tokens: Vec<String>) -> Self

pub fn with_token_ids(self, token_ids: Vec<usize>) -> Self

pub fn len(&self) -> usize

pub fn is_empty(&self) -> bool

Trait Implementations

impl Clone for TextSequence

fn clone(&self) -> TextSequence

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TextSequence

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Transform<TextSequence> for AddSpecialTokens

type Output = TextSequence

fn transform(&self, input: TextSequence) -> Result<Self::Output>

Apply the transformation to the input

fn is_deterministic(&self) -> bool

Check if the transform is deterministic

impl Transform<TextSequence> for CharTokenizer

type Output = TextSequence

fn transform(&self, input: TextSequence) -> Result<Self::Output>

Apply the transformation to the input

fn is_deterministic(&self) -> bool

Check if the transform is deterministic

impl Transform<TextSequence> for FixedLength

type Output = TextSequence

fn transform(&self, input: TextSequence) -> Result<Self::Output>

Apply the transformation to the input

fn is_deterministic(&self) -> bool

Check if the transform is deterministic

impl Transform<TextSequence> for NGrams

type Output = TextSequence

fn transform(&self, input: TextSequence) -> Result<Self::Output>

Apply the transformation to the input

fn is_deterministic(&self) -> bool

Check if the transform is deterministic

impl Transform<TextSequence> for RemovePunctuation

type Output = TextSequence

fn transform(&self, input: TextSequence) -> Result<Self::Output>

Apply the transformation to the input

fn is_deterministic(&self) -> bool

Check if the transform is deterministic

impl Transform<TextSequence> for SimpleBPE

type Output = TextSequence

fn transform(&self, input: TextSequence) -> Result<Self::Output>

Apply the transformation to the input

fn is_deterministic(&self) -> bool

Check if the transform is deterministic

impl Transform<TextSequence> for ToLowercase

type Output = TextSequence

fn transform(&self, input: TextSequence) -> Result<Self::Output>

Apply the transformation to the input

fn is_deterministic(&self) -> bool

Check if the transform is deterministic

impl Transform<TextSequence> for TokenIdsToTensor

type Output = Tensor

fn transform(&self, input: TextSequence) -> Result<Self::Output>

Apply the transformation to the input

fn is_deterministic(&self) -> bool

Check if the transform is deterministic