1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

use std::fmt;

use {TextFrame, Error};

/// A general tokenizer interface.
pub trait Tokenize<'a> {
    /// Token type.
    type Token: fmt::Debug;

    /// Constructs a new `Tokenizer` from string.
    fn from_str(text: &'a str) -> Self;
    /// Constructs a new `Tokenizer` from [`TextFrame`].
    /// [`TextFrame`]: struct.TextFrame.html
    fn from_frame(text: TextFrame<'a>) -> Self;
    /// Parses a next token.
    fn parse_next(&mut self) -> Result<Self::Token, Error>;
}