Function combine::tokens

pub fn tokens<C, T, I>(cmp: C,
                       expected: Info<I::Item, I::Range>,
                       tokens: T)
                       -> Tokens<C, T, I> where C: FnMut(T::Item, I::Item) -> bool, T: Clone + IntoIterator, I: Stream

Parses tokens.

use std::ascii::AsciiExt;
let result = tokens(|l, r| l.eq_ignore_ascii_case(&r), "abc".into(), "abc".chars())
    .parse("AbC")
    .map(|x| x.0.as_str());
assert_eq!(result, Ok("abc"));
let result = tokens(|&l, r| (if l < r { r - l } else { l - r }) <= 2, Info::Range(&b"025"[..]), &b"025"[..])
    .parse(&b"123"[..])
    .map(|x| x.0);
assert_eq!(result, Ok(&b"025"[..]));