Trait TokenSource

pub trait TokenSource {
    type Token: Hash + Eq;
    type Tokenizer: Iterator<Item = Self::Token>;

    // Required methods
    fn tokenize(&self) -> Self::Tokenizer;
    fn estimate_tokens(&self) -> u32;
}

Required Associated Types

type Token: Hash + Eq

type Tokenizer: Iterator<Item = Self::Token>

Required Methods

fn tokenize(&self) -> Self::Tokenizer

fn estimate_tokens(&self) -> u32

Implementations on Foreign Types

impl<'a> TokenSource for &'a str

By default, a line diff is produced for a string

type Token = &'a str

type Tokenizer = Lines<'a>

fn tokenize(&self) -> Self::Tokenizer

fn estimate_tokens(&self) -> u32

impl<'a> TokenSource for &'a [u8]

By default, a line diff is produced for a bytes

type Token = &'a [u8]

type Tokenizer = ByteLines<'a>

fn tokenize(&self) -> Self::Tokenizer

fn estimate_tokens(&self) -> u32

Implementors

impl<'a> TokenSource for ByteLines<'a>

By default a line diff is produced for a string

type Token = &'a [u8]

type Tokenizer = ByteLines<'a>

impl<'a> TokenSource for Lines<'a>

By default a line diff is produced for a string

type Token = &'a str

type Tokenizer = Lines<'a>