Struct InputTokenizer

pub struct InputTokenizer<C: Character, S: TextLineSource<C>> {
    pub state: MouthState,
    pub source: S,
    /* private fields */
}

Takes a TextLineSource and lazily turns it into Tokens, given a CategoryCodeScheme and an optional end-of-line Character. The primary use case is to process an input .tex file.

Example:

type T = StandardToken<u8,Ptr<str>>;
let mut cs_handler = ();
let cc = &DEFAULT_SCHEME_U8;

let string = "\\foo   \n  \n   {a}{!}";
let input: StringLineSource<u8> = string.into();
let mut tokenizer = InputTokenizer::new(input);
let eol = Some(b'\r');
let next = tokenizer.get_next(&mut cs_handler,cc,None); // \foo
assert!(matches!(next,Ok(Some(T::ControlSequence(s))) if &*s == "foo"));
let next = tokenizer.get_next(&mut cs_handler,cc,eol); // \par
assert!(matches!(next,Ok(Some(T::ControlSequence(s))) if &*s == "par"));
let next : T = tokenizer.get_next(&mut cs_handler,cc,eol).unwrap().unwrap(); // {
assert_eq!(next.command_code(), CommandCode::BeginGroup);
let next : T = tokenizer.get_next(&mut cs_handler,cc,eol).unwrap().unwrap(); // a
assert_eq!(next.command_code(), CommandCode::Letter);
let next : T = tokenizer.get_next(&mut cs_handler,cc,eol).unwrap().unwrap(); // }
assert_eq!(next.command_code(), CommandCode::EndGroup);
let next : T = tokenizer.get_next(&mut cs_handler,cc,eol).unwrap().unwrap(); // {
assert_eq!(next.command_code(), CommandCode::BeginGroup);
let next : T = tokenizer.get_next(&mut cs_handler,cc,eol).unwrap().unwrap(); // !
assert_eq!(next.command_code(), CommandCode::Other);
let next : T = tokenizer.get_next(&mut cs_handler,cc,eol).unwrap().unwrap(); // }
assert_eq!(next.command_code(), CommandCode::EndGroup);
let next : T = tokenizer.get_next(&mut cs_handler,cc,eol).unwrap().unwrap(); // end of line => space
assert_eq!(next.command_code(), CommandCode::Space);
assert!(tokenizer.get_next::<T>(&mut cs_handler,cc,eol).unwrap().is_none()); // EOF

Fields

state: MouthState

source: S

Implementations

impl<C: Character, S: TextLineSource<C>> InputTokenizer<C, S>

pub fn new(source: S) -> Self

Create a new InputTokenizer from a TextLineSource

pub fn line(&self) -> usize

The current line

pub fn column(&self) -> usize

The current column

pub fn eof(&self) -> bool

whether the file end has been reached

pub fn readline<T: Token<Char = C>, F: FnMut(T)>(&mut self, f: F)

\readline - read a line of input as Characters of CategoryCode::Other (except for , which has Space) and passing each token to the given function.

pub fn read<T: Token<Char = C>, F: FnMut(T)>( &mut self, handler: &mut <<T as Token>::CS as CSName<<T as Token>::Char>>::Handler, cc: &CategoryCodeScheme<C>, endline: Option<C>, f: F, ) -> Result<(), InvalidCharacter<C>>

\read - read a line of input as Characters in the currenct CategoryCodeScheme, respecting braces (CategoryCode::BeginGroup and EndGroup) and passing each token to the given function.

pub fn get_next<T: Token<Char = C>>( &mut self, handler: &mut <<T as Token>::CS as CSName<<T as Token>::Char>>::Handler, cc: &CategoryCodeScheme<C>, endline: Option<C>, ) -> Result<Option<T>, InvalidCharacter<C>>

Get the next Token from the InputTokenizer (if not empty). Throws InvalidCharacter on encountering a character of code CategoryCode::Invalid.

pub fn preview<W: Write>(&self, len: &mut usize, f: W) -> Result

Only useful for debugging purposes: Print the next len Characters to the given Writer.

Trait Implementations

impl<C: Clone + Character, S: Clone + TextLineSource<C>> Clone for InputTokenizer<C, S>

fn clone(&self) -> InputTokenizer<C, S>

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<C: Debug + Character, S: Debug + TextLineSource<C>> Debug for InputTokenizer<C, S>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.