Trait yap::Tokens

pub trait Tokens: Sized {
    type Item;
    type Location: TokenLocation + PartialEq + Debug + Clone;

    // Required methods
    fn next(&mut self) -> Option<Self::Item>;
    fn location(&self) -> Self::Location;
    fn set_location(&mut self, location: Self::Location);
    fn is_at_location(&self, location: &Self::Location) -> bool;

    // Provided methods
    fn as_iter(&mut self) -> TokensAsIter<'_, Self> { ... }
    fn into_iter(self) -> TokensIntoIter<Self> { ... }
    fn parse<Out, Buf>(&mut self) -> Result<Out, <Out as FromStr>::Err>
       where Out: FromStr,
             Buf: FromIterator<Self::Item> + Deref<Target = str> { ... }
    fn with_context<C>(self, context: C) -> WithContext<Self, C> { ... }
    fn with_context_mut<C>(
        &mut self,
        context: C
    ) -> WithContextMut<&mut Self, C> { ... }
    fn slice(
        &mut self,
        from: Self::Location,
        to: Self::Location
    ) -> Slice<'_, Self> { ... }
    fn offset(&self) -> usize { ... }
    fn peek(&mut self) -> Option<Self::Item> { ... }
    fn token<I>(&mut self, t: I) -> bool
       where Self::Item: PartialEq,
             I: Borrow<Self::Item> { ... }
    fn tokens<It>(&mut self, ts: It) -> bool
       where Self::Item: PartialEq,
             It: IntoIterator,
             It::Item: Borrow<Self::Item> { ... }
    fn one_of_tokens<It>(&mut self, ts: It) -> Option<Self::Item>
       where Self::Item: PartialEq,
             It: IntoIterator,
             It::Item: Borrow<Self::Item> { ... }
    fn take(&mut self, n: usize) -> Take<'_, Self> { ... }
    fn take_while<F>(&mut self, f: F) -> TakeWhile<'_, Self, F>
       where F: FnMut(&Self::Item) -> bool { ... }
    fn skip_while<F>(&mut self, f: F) -> usize
       where F: FnMut(&Self::Item) -> bool { ... }
    fn many<F, Output>(&mut self, parser: F) -> Many<'_, Self, F>
       where F: FnMut(&mut Self) -> Option<Output> { ... }
    fn many_err<F, Output, E>(&mut self, parser: F) -> ManyErr<'_, Self, F>
       where F: FnMut(&mut Self) -> Result<Output, E> { ... }
    fn skip_many<F>(&mut self, parser: F) -> usize
       where F: FnMut(&mut Self) -> bool { ... }
    fn skip_many1<F, E, Ignored>(&mut self, parser: F) -> Result<usize, E>
       where F: FnMut(&mut Self) -> Result<Ignored, E> { ... }
    fn sep_by<F, S, Output>(
        &mut self,
        parser: F,
        separator: S
    ) -> SepBy<'_, Self, F, S>
       where F: FnMut(&mut Self) -> Option<Output>,
             S: FnMut(&mut Self) -> bool { ... }
    fn sep_by_err<F, S, E, Output>(
        &mut self,
        parser: F,
        separator: S
    ) -> SepByErr<'_, Self, F, S>
       where F: FnMut(&mut Self) -> Result<Output, E>,
             S: FnMut(&mut Self) -> bool { ... }
    fn sep_by_all<F, S, Output>(
        &mut self,
        parser: F,
        separator: S
    ) -> SepByAll<'_, Self, F, S, Output>
       where F: FnMut(&mut Self) -> Option<Output>,
             S: FnMut(&mut Self) -> Option<Output> { ... }
    fn sep_by_all_err<F, S, Output, E>(
        &mut self,
        parser: F,
        separator: S
    ) -> SepByAllErr<'_, Self, F, S, Output>
       where F: FnMut(&mut Self) -> Result<Output, E>,
             S: FnMut(&mut Self) -> Option<Output> { ... }
    fn surrounded_by<F, S, Output>(
        &mut self,
        parser: F,
        surrounding: S
    ) -> Output
       where F: FnOnce(&mut Self) -> Output,
             S: FnMut(&mut Self) { ... }
    fn optional<F, Output>(&mut self, f: F) -> Output
       where F: FnOnce(&mut Self) -> Output,
             Output: IsMatch { ... }
    fn optional_err<F, Output, Error>(&mut self, f: F) -> Result<Output, Error>
       where F: FnOnce(&mut Self) -> Result<Output, Error> { ... }
    fn eof(&mut self) -> bool { ... }
    fn consume(&mut self) { ... }
    fn collect<B: FromIterator<Self::Item>>(&mut self) -> B { ... }
}

The tokens trait is an extension of the Iterator trait, and adds a bunch of useful methods for parsing tokens from the underlying iterable type.

Implementations don’t need to directly implement Iterator; instead there exists Tokens::as_iter() and Tokens::into_iter() methods to return an iterator that is based on the methods implemented here and keeps iterator methods in a separate namespace.

Required Associated Types

type Item

The item returned from Tokens::next().

type Location: TokenLocation + PartialEq + Debug + Clone

An object which can be used to reset the token stream to some position.

Required Methods

fn next(&mut self) -> Option<Self::Item>

Return the next token. This is also the basis of the Iterator implementation that’s returned when you call Tokens::as_iter(). By implementing it here, we can keep all of the methods provided by Iterator in a separate “namespace” to avoid confusion and potential name collisions.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abc".into_tokens();

assert_eq!(s.next(), Some('a'));
assert_eq!(s.next(), Some('b'));
assert_eq!(s.next(), Some('c'));
assert_eq!(s.next(), None);

fn location(&self) -> Self::Location

Return a “location” pointer. This can be passed to Tokens::set_location to set the tokens location back to the state at the time it was handed out. If the crate::TokenLocation trait is in scope, you can also call the crate::TokenLocation::offset() method on it to obtain the current offset.

Example

use yap::{ Tokens, IntoTokens, TokenLocation };

let mut s = "abcde".into_tokens();

let location = s.location();
assert_eq!(s.next().unwrap(), 'a');
assert_eq!(s.location().offset(), 1);
assert_eq!(s.next().unwrap(), 'b');
assert_eq!(s.location().offset(), 2);

s.set_location(location);

assert_eq!(s.next().unwrap(), 'a');
assert_eq!(s.location().offset(), 1);
assert_eq!(s.next().unwrap(), 'b');
assert_eq!(s.location().offset(), 2);

fn set_location(&mut self, location: Self::Location)

Set the tokens to the location provided. See Tokens::location.

fn is_at_location(&self, location: &Self::Location) -> bool

Return true if the current cursor location matches the location given, or false otherwise.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abc".into_tokens();
let location = s.location();
assert_eq!(s.is_at_location(&location), true);
s.next();
assert_eq!(s.is_at_location(&location), false);
s.set_location(location);
assert_eq!(s.is_at_location(&location), true);

Provided Methods

fn as_iter(&mut self) -> TokensAsIter<'_, Self>

Return an iterator over our tokens. The Tokens trait already mirrors the Iterator interface by providing Tokens::Item and Tokens::next(), but we keep the Iterator separate to avoid collisions, and because some iterator methods don’t consume tokens as you might expect, and so must be used with care when parsing input.

fn into_iter(self) -> TokensIntoIter<Self>

Like Tokens::as_iter(), except it consumes self, which can be useful in some situations.

fn parse<Out, Buf>(&mut self) -> Result<Out, <Out as FromStr>::Err>where Out: FromStr, Buf: FromIterator<Self::Item> + Deref<Target = str>,

Attempt to parse the remaining tokens into the first Out generic using str::parse(). The second generic type may be used to buffer tokens, and can be any type that implements FromIterator<Self::Item> + Deref<Target = str>.

If the parsing fails, then no tokens are consumed.

As an optimisation, implementations may choose not to use the provided buffer type if they have a suitable internal buffer of their own already. This is the case for crate::types::StrTokens.

This is mostly expected to be used in conjunction with Tokens::take and Tokens::take_while, which themselves return the matching Tokens.

Example

use yap::{ Tokens, IntoTokens };

let mut tokens = "123abc456".into_tokens();

let n = tokens.take(3).parse::<u8, String>().unwrap();
assert_eq!(n, 123);

let s = tokens.take_while(|t| t.is_alphabetic()).parse::<String, String>().unwrap();
assert_eq!(s, "abc".to_string());

// This will fail to parse; the number is out of bounds. Failure will consume
// no tokens.
assert!(tokens.parse::<u8, String>().is_err());

// This will work; the number can fit into a u16:
let n2 = tokens.parse::<u16, String>().unwrap();
assert_eq!(n2, 456);

fn with_context<C>(self, context: C) -> WithContext<Self, C>

Attach some context to your tokens. The returned struct, WithContext, also implements Tokens, and so has can be used in much the same way. Since this consumes your tokens, it’s better suited to permanent context that you’d like throughout the parsing.

See Tokens::with_context_mut for a version that’s easier to attach temporary context with.

Example

use yap::{ Tokens, IntoTokens, types::WithContext };

fn skip_digits(toks: &mut WithContext<impl Tokens<Item=char>, usize>) {
    let n_skipped = toks.skip_while(|c| c.is_digit(10));
    *toks.context_mut() += n_skipped;
}

let mut tokens = "123abc456".into_tokens().with_context(0usize);

skip_digits(&mut tokens);
tokens.skip_while(|c| c.is_alphabetic());
skip_digits(&mut tokens);

assert_eq!(*tokens.context(), 6);

fn with_context_mut<C>(&mut self, context: C) -> WithContextMut<&mut Self, C>

Unlike Tokens::with_context, which consumes the tokens, this borrows them mutably, allowing it to be used when you only have a mutable reference to tokens (which is a common function signature to use), and making it better suited to attaching temporary contexts.

Be aware that if you attach context in a function called recursively, the type checker may shout at you for constructing a type like WithContextMut<WithContextMut<WithContextMut<..>>>. In these cases, you can “break the cycle” by removing the original WithContextMut by using crate::types::WithContextMut::into_parts() before wrapping the tokens in a new context for the recursive call.

Example

use yap::{ Tokens, IntoTokens };

fn count_digit_comma_calls(toks: &mut impl Tokens<Item=char>) -> (u8, u8) {
    let mut counts = (0u8, 0u8);
    toks.with_context_mut(&mut counts).sep_by(
        |t| {
            t.context_mut().0 += 1;
            let n_skipped = t.skip_while(|c| c.is_digit(10));
            if n_skipped == 0 { None } else { Some(()) }
        },
        |t| {
            t.context_mut().1 += 1;
            t.token(',')
        }
    ).consume();
    counts
}

let n: usize = 0;
let mut tokens = "123,4,56,1,34,1".into_tokens();

let (digits, seps) = count_digit_comma_calls(&mut tokens);

assert_eq!(tokens.remaining().len(), 0);
// digits parsed 6 times:
assert_eq!(digits, 6);
// Attempted to parse seps 6 times; failure on last ends it:
assert_eq!(seps, 6);

fn slice(&mut self, from: Self::Location, to: Self::Location) -> Slice<'_, Self>

Return a slice of tokens starting at the to location provided and ending just prior to the from location provided (ie equivalent to the range to..from).

The slice returned from implements Tokens, so you can use the full range of parsing functions on it.

Note: the slice returned from this prevents the original tokens from being used until it’s dropped, and resets the original tokens to their current location on Drop. if you core::mem::forget it, the original token location will equal whatever the slice location was when it was forgotten.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abcdefghijklmnop".into_tokens();

(0..5).for_each(|_| { s.next(); });
let from = s.location();
(0..5).for_each(|_| { s.next(); });
let to = s.location();

assert_eq!(s.next(), Some('k'));
assert_eq!(s.next(), Some('l'));

// Iterating the from..to range given:
let vals: String = s.slice(from.clone(), to.clone()).collect();
assert_eq!(&*vals, "fghij");

// After the above is dropped, we can continue
// from where we left off:
assert_eq!(s.next(), Some('m'));
assert_eq!(s.next(), Some('n'));

// We can iterate this range again as we please:
let vals: String = s.slice(from, to).collect();
assert_eq!(&*vals, "fghij");

// And the original remains unaffected..
assert_eq!(s.next(), Some('o'));
assert_eq!(s.next(), Some('p'));

fn offset(&self) -> usize

Return the current offset into the tokens that we’ve parsed up to so far. The exact meaning of this can vary by implementation; when parsing slices, it is index of the slice item we’ve consumed up to, and when parsing &str’s it is the number of bytes (not characters) consumed so far.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abc".into_tokens();
assert_eq!(s.offset(), 0);
s.next();
assert_eq!(s.offset(), 1);
s.next();
assert_eq!(s.offset(), 2);

fn peek(&mut self) -> Option<Self::Item>

Return the next item in the input without consuming it.

Prefer this to using the peekable iterator method, which consumes the tokens, and internally keeps hold of the peeked state itself.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abc".into_tokens();
assert_eq!(s.peek(), Some('a'));
assert_eq!(s.peek(), Some('a'));

fn token<I>(&mut self, t: I) -> boolwhere Self::Item: PartialEq, I: Borrow<Self::Item>,

Expect a specific token to be next. If the token is not found, the iterator is not advanced.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abc".into_tokens();
assert_eq!(s.token(&'a'), true);
assert_eq!(s.token(&'b'), true);
assert_eq!(s.token('z'), false);
assert_eq!(s.token('y'), false);
assert_eq!(s.token('c'), true);

fn tokens<It>(&mut self, ts: It) -> boolwhere Self::Item: PartialEq, It: IntoIterator, It::Item: Borrow<Self::Item>,

Expect a specific set of tokens to be next. If the tokens are not found, the iterator is not advanced. Anything that implements IntoIterator with an Item type that can be borrowed to produce &Item can be provided as an input to this.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abcdef".into_tokens();

assert_eq!(s.tokens("abc".chars()), true);
assert_eq!(s.remaining(), "def");

assert_eq!(s.tokens("de".chars()), true);
assert_eq!(s.remaining(), "f");

fn one_of_tokens<It>(&mut self, ts: It) -> Option<Self::Item>where Self::Item: PartialEq, It: IntoIterator, It::Item: Borrow<Self::Item>,

Return the first token that matches the tokens provided, or None if none of them match.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abcdef".into_tokens();

assert_eq!(s.one_of_tokens("abc".chars()), Some('a'));
assert_eq!(s.one_of_tokens("abc".chars()), Some('b'));
assert_eq!(s.one_of_tokens("abc".chars()), Some('c'));
assert_eq!(s.one_of_tokens("abc".chars()), None);
assert_eq!(s.remaining(), "def");

fn take(&mut self, n: usize) -> Take<'_, Self>

Return a Tokens impl that will take the next n tokens from the input (ending early if the input runs early).

Example

use yap::{ Tokens, IntoTokens };

let mut s = "12345abc".into_tokens();
let digits: String = s.take(3).collect();
assert_eq!(&*digits, "123");
assert_eq!(s.remaining(), "45abc");

fn take_while<F>(&mut self, f: F) -> TakeWhile<'_, Self, F>where F: FnMut(&Self::Item) -> bool,

Return a Tokens impl that will consume tokens until the provided function returns false.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "12345abc".into_tokens();
let digits: String = s.take_while(|c| c.is_numeric()).collect();
assert_eq!(&*digits, "12345");
assert_eq!(s.remaining(), "abc");

This exists primarily because Iterator::take_while() will consume the first token that does not match the predicate, which is often not what we’d want. The above example using Iterator::take_while() would look like:

use yap::{ Tokens, IntoTokens };

let mut s = "12345abc".into_tokens();
let digits: String = s.as_iter().take_while(|c| c.is_numeric()).collect();
assert_eq!(&*digits, "12345");

// Note that `Iterator::take_while` consumed the "a" in order to test it,
// whereas `Tokens::take_while` did not:
assert_eq!(s.remaining(), "bc");

fn skip_while<F>(&mut self, f: F) -> usizewhere F: FnMut(&Self::Item) -> bool,

Iterate over the tokens until the provided function returns false on one. Only consume the tokens that the function returned true for, returning the number of tokens that were consumed/skipped. Equivalent to toks.take_while(f).count().

Example

use yap::{ Tokens, IntoTokens };

let mut s = "12345abc".into_tokens();
let n_skipped = s.skip_while(|c| c.is_numeric());

assert_eq!(n_skipped, 5);
assert_eq!(s.remaining(), "abc");

fn many<F, Output>(&mut self, parser: F) -> Many<'_, Self, F>where F: FnMut(&mut Self) -> Option<Output>,

Returns a Tokens impl that runs the provided parser again and again, returning an output from it each time until it returns None.

Example

use yap::{ Tokens, IntoTokens };

fn parse_digit_pair(tokens: &mut impl Tokens<Item=char>) -> Option<u32> {
    let d1 = tokens.next()?;
    let d2 = tokens.next()?;
    // Return the result of adding the 2 digits we saw:
    Some(d1.to_digit(10)? + d2.to_digit(10)?)
}

let mut s = "12345abcde".into_tokens();
let digits: Vec<u32> = s.many(|t| parse_digit_pair(t)).collect();

assert_eq!(digits, vec![3, 7]);
assert_eq!(s.remaining(), "5abcde");

fn many_err<F, Output, E>(&mut self, parser: F) -> ManyErr<'_, Self, F>where F: FnMut(&mut Self) -> Result<Output, E>,

Returns a Tokens impl that runs the provided parser again and again, returning an output from it each time until it returns None. If the parser returns an error, no tokens will be consumed and the error will be returned as the final iteration.

Example

use yap::{ Tokens, IntoTokens };

#[derive(Debug, PartialEq)]
enum Err { NotEnoughTokens, NotADigit(char) }
fn parse_digit_pair(tokens: &mut impl Tokens<Item=char>) -> Result<u32, Err> {
    let n1 = tokens.next()
        .ok_or(Err::NotEnoughTokens)
        .and_then(|c| c.to_digit(10).ok_or(Err::NotADigit(c)))?;
    let n2 = tokens.next()
        .ok_or(Err::NotEnoughTokens)
        .and_then(|c| c.to_digit(10).ok_or(Err::NotADigit(c)))?;
    Ok(n1 + n2)
}

let mut s = "12345abcde".into_tokens();
let mut digits_iter = s.many_err(|t| parse_digit_pair(t));

assert_eq!(digits_iter.next(), Some(Ok(3)));
assert_eq!(digits_iter.next(), Some(Ok(7)));
assert_eq!(digits_iter.next(), Some(Err(Err::NotADigit('a'))));
assert_eq!(digits_iter.next(), None);
assert_eq!(s.remaining(), "5abcde");

fn skip_many<F>(&mut self, parser: F) -> usizewhere F: FnMut(&mut Self) -> bool,

Ignore 0 or more instances of some parser.

Example

use yap::{ Tokens, IntoTokens };

struct ABC;
fn parse_abc(tokens: &mut impl Tokens<Item=char>) -> Option<ABC> {
    let a = tokens.next()?;
    let b = tokens.next()?;
    let c = tokens.next()?;
    if a == 'a' && b == 'b' && c == 'c' {
        Some(ABC)
    } else {
        None
    }
}

let mut s = "abcabcababab".into_tokens();
s.skip_many(|t| parse_abc(t).is_some());

assert_eq!(s.remaining(), "ababab");

fn skip_many1<F, E, Ignored>(&mut self, parser: F) -> Result<usize, E>where F: FnMut(&mut Self) -> Result<Ignored, E>,

Ignore 1 or more instances of some parser. If the provided parser fails immediately, return the error that it produced.

Example

use yap::{ Tokens, IntoTokens };

struct ABC;
fn parse_abc(tokens: &mut impl Tokens<Item=char>) -> Option<ABC> {
    let a = tokens.next()?;
    let b = tokens.next()?;
    let c = tokens.next()?;
    if a == 'a' && b == 'b' && c == 'c' {
        Some(ABC)
    } else {
        None
    }
}

let mut s = "abcabcabcxyz".into_tokens();
let skipped = s.skip_many1(|t| parse_abc(t).ok_or("aaah"));

assert_eq!(skipped, Ok(3));
assert_eq!(s.remaining(), "xyz");

let mut s = "ababababcabc".into_tokens();
let skipped = s.skip_many1(|t| parse_abc(t).ok_or("aaah"));

assert_eq!(skipped, Err("aaah"));
assert_eq!(s.remaining(), "ababababcabc");

fn sep_by<F, S, Output>( &mut self, parser: F, separator: S ) -> SepBy<'_, Self, F, S>where F: FnMut(&mut Self) -> Option<Output>, S: FnMut(&mut Self) -> bool,

Return a Tokens impl that parses anything matching the first parser function, and expects to parse something matching the second separator function between each of these.

Example

use yap::{ Tokens, IntoTokens };

fn parse_digit(tokens: &mut impl Tokens<Item=char>) -> Option<u32> {
    let c = tokens.next()?;
    c.to_digit(10)
}

let mut s = "1,2,3,4,abc".into_tokens();
let digits: Vec<u32> = s.sep_by(|t| parse_digit(t), |t| t.token(',')).collect();
assert_eq!(digits, vec![1,2,3,4]);
assert_eq!(s.remaining(), ",abc");

fn sep_by_err<F, S, E, Output>( &mut self, parser: F, separator: S ) -> SepByErr<'_, Self, F, S>where F: FnMut(&mut Self) -> Result<Output, E>, S: FnMut(&mut Self) -> bool,

Return a Tokens impl that parses anything matching the parser function, and expects to parse something matching the separator function between each one. Unlike Tokens::sep_by, this accepts parsers that return Results, and returns the result on each iteration. Once an error is hit, None is returned thereafter.

Example

use yap::{ Tokens, IntoTokens };

#[derive(Debug, PartialEq)]
enum Err { NoMoreTokens, NotADigit(char) }

fn parse_digit(tokens: &mut impl Tokens<Item=char>) -> Result<u32, Err> {
    let c = tokens.next().ok_or(Err::NoMoreTokens)?;
    c.to_digit(10).ok_or(Err::NotADigit(c))
}

let mut s = "1,2,a,1,2,3".into_tokens();
let mut digits_iter = s.sep_by_err(|t| parse_digit(t), |t| t.token(','));
assert_eq!(digits_iter.next(), Some(Ok(1)));
assert_eq!(digits_iter.next(), Some(Ok(2)));
assert_eq!(digits_iter.next(), Some(Err(Err::NotADigit('a'))));
assert_eq!(digits_iter.next(), None);
assert_eq!(s.remaining(), ",a,1,2,3");

fn sep_by_all<F, S, Output>( &mut self, parser: F, separator: S ) -> SepByAll<'_, Self, F, S, Output>where F: FnMut(&mut Self) -> Option<Output>, S: FnMut(&mut Self) -> Option<Output>,

Returns a Tokens impl that parses anything matching the parser function, and expects to parse something matching the separator function between each one. The Tokens impl hands back the output from both the parser and separator function, which means that they are both expected to return the same type.

Example

use yap::{ Tokens, IntoTokens };

#[derive(PartialEq,Debug)]
enum Op { Plus, Minus, Divide }
#[derive(PartialEq,Debug)]
enum OpOrDigit { Op(Op), Digit(u32) }

fn parse_op(tokens: &mut impl Tokens<Item=char>) -> Option<Op> {
    match tokens.next()? {
        '-' => Some(Op::Minus),
        '+' => Some(Op::Plus),
        '/' => Some(Op::Divide),
        _ => None
    }
}

fn parse_digit(tokens: &mut impl Tokens<Item=char>) -> Option<u32> {
    let c = tokens.next()?;
    c.to_digit(10)
}

let mut s = "1+2/3-4+abc".into_tokens();
let output: Vec<_> = s.sep_by_all(
    |t| parse_digit(t).map(OpOrDigit::Digit),
    |t| parse_op(t).map(OpOrDigit::Op)
).collect();

assert_eq!(output, vec![
    OpOrDigit::Digit(1),
    OpOrDigit::Op(Op::Plus),
    OpOrDigit::Digit(2),
    OpOrDigit::Op(Op::Divide),
    OpOrDigit::Digit(3),
    OpOrDigit::Op(Op::Minus),
    OpOrDigit::Digit(4),
]);
assert_eq!(s.remaining(), "+abc");

fn sep_by_all_err<F, S, Output, E>( &mut self, parser: F, separator: S ) -> SepByAllErr<'_, Self, F, S, Output>where F: FnMut(&mut Self) -> Result<Output, E>, S: FnMut(&mut Self) -> Option<Output>,

Similar to Tokens::sep_by_all, except that the Tokens impl returned also hands back the first error encountered when attempting to run our parser.

Example

use yap::{ Tokens, IntoTokens };

#[derive(PartialEq,Debug)]
enum Op { Plus, Minus, Divide }
#[derive(PartialEq,Debug)]
enum OpOrDigit { Op(Op), Digit(u32) }
#[derive(Debug, PartialEq)]
enum Err { NoMoreTokens, NotADigit(char) }

fn parse_op(tokens: &mut impl Tokens<Item=char>) -> Option<Op> {
    match tokens.next()? {
        '-' => Some(Op::Minus),
        '+' => Some(Op::Plus),
        '/' => Some(Op::Divide),
        _ => None
    }
}

fn parse_digit(tokens: &mut impl Tokens<Item=char>) -> Result<u32, Err> {
    let c = tokens.next().ok_or(Err::NoMoreTokens)?;
    c.to_digit(10).ok_or(Err::NotADigit(c))
}

let mut s = "1+2/3-4+abc".into_tokens();
let output: Vec<_> = s.sep_by_all_err(
    |t| parse_digit(t).map(OpOrDigit::Digit),
    |t| parse_op(t).map(OpOrDigit::Op)
).collect();

assert_eq!(output, vec![
    Ok(OpOrDigit::Digit(1)),
    Ok(OpOrDigit::Op(Op::Plus)),
    Ok(OpOrDigit::Digit(2)),
    Ok(OpOrDigit::Op(Op::Divide)),
    Ok(OpOrDigit::Digit(3)),
    Ok(OpOrDigit::Op(Op::Minus)),
    Ok(OpOrDigit::Digit(4)),
    Err(Err::NotADigit('a'))
]);
assert_eq!(s.remaining(), "+abc");

fn surrounded_by<F, S, Output>(&mut self, parser: F, surrounding: S) -> Outputwhere F: FnOnce(&mut Self) -> Output, S: FnMut(&mut Self),

Parse some tokens that are optionally surrounded by the result of a surrounding parser.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "   hello    ".into_tokens();

let hello: String = s.surrounded_by(
    |t| t.take_while(|c| c.is_ascii_alphabetic()).collect(),
    |t| { t.skip_while(|c| c.is_ascii_whitespace()); }
);

assert_eq!(&*hello, "hello");
assert_eq!(s.remaining(), "");

fn optional<F, Output>(&mut self, f: F) -> Outputwhere F: FnOnce(&mut Self) -> Output, Output: IsMatch,

Attempt to parse some output from the tokens. Returning None or false signifies that no match was found, and no tokens will be consumed. Otherwise, we’ll return the match and consume the tokens.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "foobar".into_tokens();

let res = s.optional(|s| {
    let a = s.next();
    let b = s.next();
    if a == b {
        Some("yay")
    } else {
        None
    }
});

// nothing consumed since None returned from fn
assert_eq!(s.remaining(), "foobar");
assert_eq!(res, None);

let res = s.optional(|s| {
    let a = s.next()?;
    let b = s.next()?;
    Some((a, b))
});

// 2 chars consumed since Some returned from fn
assert_eq!(s.remaining(), "obar");
assert_eq!(res, Some(('f', 'o')));

fn optional_err<F, Output, Error>(&mut self, f: F) -> Result<Output, Error>where F: FnOnce(&mut Self) -> Result<Output, Error>,

Attempt to parse some output from the tokens, returning a Result. If the Result returned is Err, no tokens will be consumed.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "foobar".into_tokens();

let res = s.optional_err(|s| {
    let a = s.next();
    let b = s.next();
    if a == b {
        Ok("yay")
    } else {
        Err("a and b don't match!")
    }
});

// nothing consumed since Err returned from fn
assert_eq!(s.remaining(), "foobar");
assert!(res.is_err());

let res = s.optional_err(|s| {
    let a = s.next();
    let b = s.next();
    if a != b {
        Ok((a, b))
    } else {
        Err("a and b match!")
    }
});

// 2 chars consumed since Ok returned from fn
assert_eq!(s.remaining(), "obar");
assert_eq!(res, Ok((Some('f'), Some('o'))));

fn eof(&mut self) -> bool

Checks next input is None and, if true, consumes the None.

Example

use yap::Tokens;
use yap::types::IterTokens;
use core::iter;

// This will always return None; eof will consume it.
let mut toks = IterTokens::new(iter::empty::<char>());

assert_eq!(toks.eof(), true);
assert_eq!(toks.offset(), 1);

assert_eq!(toks.eof(), true);
assert_eq!(toks.offset(), 2);

// This will return a char; eof won't consume anything.
let mut toks = IterTokens::new(iter::once('a'));

assert_eq!(toks.eof(), false);
assert_eq!(toks.offset(), 0);

assert_eq!(toks.eof(), false);
assert_eq!(toks.offset(), 0);

fn consume(&mut self)

Consume all remaining tokens. This is expected to be used in conjunction with combinators likeTokens::take and Tokens::take_while. This is just a shorthand for toks.as_iter().for_each(drop).

Example

use yap::{Tokens, IntoTokens};

let mut toks = "abc123def".into_tokens();

// Take won't do anything unless it's consumed:
toks.take(3);
assert_eq!(toks.remaining(), "abc123def");

// This is the same as `toks.take(3).as_iter().for_each(drop);`
toks.take(3).consume();
assert_eq!(toks.remaining(), "123def");

toks.take_while(|t| t.is_numeric()).consume();
assert_eq!(toks.remaining(), "def");

fn collect<B: FromIterator<Self::Item>>(&mut self) -> B

Collect up all of the tokens into something that implements FromIterator. If you’d like to call str::parse on the subsequent collection, then prefer Tokens::parse, which can be more optimal in some cases.

This is just a shorthand for calling toks.as_iter().collect().

Object Safety

This trait is not object safe.

Implementors

impl<'a> Tokens for StrTokens<'a>

type Item = char

type Location = StrTokensLocation

impl<'a, Item> Tokens for SliceTokens<'a, Item>

type Item = &'a Item

type Location = SliceTokensLocation

impl<I> Tokens for IterTokens<I>where I: Iterator + Clone,

type Item = <I as Iterator>::Item

type Location = IterTokensLocation<I>

impl<T, C> Tokens for WithContext<T, C>where T: Tokens,

type Item = <T as Tokens>::Item

type Location = <T as Tokens>::Location

impl<T, C> Tokens for WithContextMut<&mut T, C>where T: Tokens,

type Item = <T as Tokens>::Item

type Location = <T as Tokens>::Location