pub trait Tokens: Sized {
type Item;
type Location: TokenLocation + PartialEq + Debug + Clone;
Show 32 methods
// Required methods
fn next(&mut self) -> Option<Self::Item>;
fn location(&self) -> Self::Location;
fn set_location(&mut self, location: Self::Location);
fn is_at_location(&self, location: &Self::Location) -> bool;
// Provided methods
fn as_iter(&mut self) -> TokensAsIter<'_, Self> { ... }
fn into_iter(self) -> TokensIntoIter<Self> { ... }
fn parse<Out, Buf>(&mut self) -> Result<Out, <Out as FromStr>::Err>
where Out: FromStr,
Buf: FromIterator<Self::Item> + Deref<Target = str> { ... }
fn with_context<C>(self, context: C) -> WithContext<Self, C> { ... }
fn with_context_mut<C>(
&mut self,
context: C
) -> WithContextMut<&mut Self, C> { ... }
fn slice(
&mut self,
from: Self::Location,
to: Self::Location
) -> Slice<'_, Self> { ... }
fn offset(&self) -> usize { ... }
fn peek(&mut self) -> Option<Self::Item> { ... }
fn token<I>(&mut self, t: I) -> bool
where Self::Item: PartialEq,
I: Borrow<Self::Item> { ... }
fn tokens<It>(&mut self, ts: It) -> bool
where Self::Item: PartialEq,
It: IntoIterator,
It::Item: Borrow<Self::Item> { ... }
fn one_of_tokens<It>(&mut self, ts: It) -> Option<Self::Item>
where Self::Item: PartialEq,
It: IntoIterator,
It::Item: Borrow<Self::Item> { ... }
fn take(&mut self, n: usize) -> Take<'_, Self> { ... }
fn take_while<F>(&mut self, f: F) -> TakeWhile<'_, Self, F>
where F: FnMut(&Self::Item) -> bool { ... }
fn skip_while<F>(&mut self, f: F) -> usize
where F: FnMut(&Self::Item) -> bool { ... }
fn many<F, Output>(&mut self, parser: F) -> Many<'_, Self, F>
where F: FnMut(&mut Self) -> Option<Output> { ... }
fn many_err<F, Output, E>(&mut self, parser: F) -> ManyErr<'_, Self, F>
where F: FnMut(&mut Self) -> Result<Output, E> { ... }
fn skip_many<F>(&mut self, parser: F) -> usize
where F: FnMut(&mut Self) -> bool { ... }
fn skip_many1<F, E, Ignored>(&mut self, parser: F) -> Result<usize, E>
where F: FnMut(&mut Self) -> Result<Ignored, E> { ... }
fn sep_by<F, S, Output>(
&mut self,
parser: F,
separator: S
) -> SepBy<'_, Self, F, S>
where F: FnMut(&mut Self) -> Option<Output>,
S: FnMut(&mut Self) -> bool { ... }
fn sep_by_err<F, S, E, Output>(
&mut self,
parser: F,
separator: S
) -> SepByErr<'_, Self, F, S>
where F: FnMut(&mut Self) -> Result<Output, E>,
S: FnMut(&mut Self) -> bool { ... }
fn sep_by_all<F, S, Output>(
&mut self,
parser: F,
separator: S
) -> SepByAll<'_, Self, F, S, Output>
where F: FnMut(&mut Self) -> Option<Output>,
S: FnMut(&mut Self) -> Option<Output> { ... }
fn sep_by_all_err<F, S, Output, E>(
&mut self,
parser: F,
separator: S
) -> SepByAllErr<'_, Self, F, S, Output>
where F: FnMut(&mut Self) -> Result<Output, E>,
S: FnMut(&mut Self) -> Option<Output> { ... }
fn surrounded_by<F, S, Output>(
&mut self,
parser: F,
surrounding: S
) -> Output
where F: FnOnce(&mut Self) -> Output,
S: FnMut(&mut Self) { ... }
fn optional<F, Output>(&mut self, f: F) -> Output
where F: FnOnce(&mut Self) -> Output,
Output: IsMatch { ... }
fn optional_err<F, Output, Error>(&mut self, f: F) -> Result<Output, Error>
where F: FnOnce(&mut Self) -> Result<Output, Error> { ... }
fn eof(&mut self) -> bool { ... }
fn consume(&mut self) { ... }
fn collect<B: FromIterator<Self::Item>>(&mut self) -> B { ... }
}
Expand description
The tokens trait is an extension of the Iterator
trait, and adds a bunch of useful methods
for parsing tokens from the underlying iterable type.
Implementations don’t need to directly implement Iterator
; instead there exists
Tokens::as_iter()
and Tokens::into_iter()
methods to return an iterator that is based
on the methods implemented here and keeps iterator methods in a separate namespace.
Required Associated Types§
sourcetype Item
type Item
The item returned from Tokens::next()
.
Required Methods§
sourcefn next(&mut self) -> Option<Self::Item>
fn next(&mut self) -> Option<Self::Item>
Return the next token. This is also the basis of the Iterator
implementation
that’s returned when you call Tokens::as_iter()
. By implementing it here, we can keep
all of the methods provided by Iterator
in a separate “namespace” to avoid confusion
and potential name collisions.
Example
use yap::{ Tokens, IntoTokens };
let mut s = "abc".into_tokens();
assert_eq!(s.next(), Some('a'));
assert_eq!(s.next(), Some('b'));
assert_eq!(s.next(), Some('c'));
assert_eq!(s.next(), None);
sourcefn location(&self) -> Self::Location
fn location(&self) -> Self::Location
Return a “location” pointer. This can be passed to Tokens::set_location
to set the tokens location back to the state at the time it was handed out.
If the crate::TokenLocation
trait is in scope, you can also call the
crate::TokenLocation::offset()
method on it to obtain the current offset.
Example
use yap::{ Tokens, IntoTokens, TokenLocation };
let mut s = "abcde".into_tokens();
let location = s.location();
assert_eq!(s.next().unwrap(), 'a');
assert_eq!(s.location().offset(), 1);
assert_eq!(s.next().unwrap(), 'b');
assert_eq!(s.location().offset(), 2);
s.set_location(location);
assert_eq!(s.next().unwrap(), 'a');
assert_eq!(s.location().offset(), 1);
assert_eq!(s.next().unwrap(), 'b');
assert_eq!(s.location().offset(), 2);
sourcefn set_location(&mut self, location: Self::Location)
fn set_location(&mut self, location: Self::Location)
Set the tokens to the location provided. See Tokens::location
.
sourcefn is_at_location(&self, location: &Self::Location) -> bool
fn is_at_location(&self, location: &Self::Location) -> bool
Return true if the current cursor location matches the location given, or false otherwise.
Example
use yap::{ Tokens, IntoTokens };
let mut s = "abc".into_tokens();
let location = s.location();
assert_eq!(s.is_at_location(&location), true);
s.next();
assert_eq!(s.is_at_location(&location), false);
s.set_location(location);
assert_eq!(s.is_at_location(&location), true);
Provided Methods§
sourcefn as_iter(&mut self) -> TokensAsIter<'_, Self>
fn as_iter(&mut self) -> TokensAsIter<'_, Self>
Return an iterator over our tokens. The Tokens
trait already mirrors the Iterator
interface by providing Tokens::Item
and Tokens::next()
, but we keep the Iterator
separate to avoid collisions, and because some iterator methods don’t consume tokens as you
might expect, and so must be used with care when parsing input.
sourcefn into_iter(self) -> TokensIntoIter<Self>
fn into_iter(self) -> TokensIntoIter<Self>
Like Tokens::as_iter()
, except it consumes self
, which can be useful in some situations.
sourcefn parse<Out, Buf>(&mut self) -> Result<Out, <Out as FromStr>::Err>where
Out: FromStr,
Buf: FromIterator<Self::Item> + Deref<Target = str>,
fn parse<Out, Buf>(&mut self) -> Result<Out, <Out as FromStr>::Err>where Out: FromStr, Buf: FromIterator<Self::Item> + Deref<Target = str>,
Attempt to parse the remaining tokens into the first Out
generic using str::parse()
.
The second generic type may be used to buffer tokens, and can be any type that implements
FromIterator<Self::Item> + Deref<Target = str>
.
If the parsing fails, then no tokens are consumed.
As an optimisation, implementations may choose not to use the provided buffer type if they have a
suitable internal buffer of their own already. This is the case for crate::types::StrTokens
.
This is mostly expected to be used in conjunction with Tokens::take
and Tokens::take_while
,
which themselves return the matching Tokens
.
Example
use yap::{ Tokens, IntoTokens };
let mut tokens = "123abc456".into_tokens();
let n = tokens.take(3).parse::<u8, String>().unwrap();
assert_eq!(n, 123);
let s = tokens.take_while(|t| t.is_alphabetic()).parse::<String, String>().unwrap();
assert_eq!(s, "abc".to_string());
// This will fail to parse; the number is out of bounds. Failure will consume
// no tokens.
assert!(tokens.parse::<u8, String>().is_err());
// This will work; the number can fit into a u16:
let n2 = tokens.parse::<u16, String>().unwrap();
assert_eq!(n2, 456);
sourcefn with_context<C>(self, context: C) -> WithContext<Self, C>
fn with_context<C>(self, context: C) -> WithContext<Self, C>
Attach some context to your tokens. The returned struct, WithContext
, also implements
Tokens
, and so has can be used in much the same way. Since this consumes your tokens, it’s
better suited to permanent context that you’d like throughout the parsing.
See Tokens::with_context_mut
for a version that’s easier to attach temporary context with.
Example
use yap::{ Tokens, IntoTokens, types::WithContext };
fn skip_digits(toks: &mut WithContext<impl Tokens<Item=char>, usize>) {
let n_skipped = toks.skip_while(|c| c.is_digit(10));
*toks.context_mut() += n_skipped;
}
let mut tokens = "123abc456".into_tokens().with_context(0usize);
skip_digits(&mut tokens);
tokens.skip_while(|c| c.is_alphabetic());
skip_digits(&mut tokens);
assert_eq!(*tokens.context(), 6);
sourcefn with_context_mut<C>(&mut self, context: C) -> WithContextMut<&mut Self, C>
fn with_context_mut<C>(&mut self, context: C) -> WithContextMut<&mut Self, C>
Unlike Tokens::with_context
, which consumes the tokens, this borrows them mutably, allowing it to
be used when you only have a mutable reference to tokens (which is a common function signature to use),
and making it better suited to attaching temporary contexts.
Be aware that if you attach context in a function called recursively, the type checker may shout at you
for constructing a type like WithContextMut<WithContextMut<WithContextMut<..>>>
. In these cases, you
can “break the cycle” by removing the original WithContextMut
by using
crate::types::WithContextMut::into_parts()
before wrapping the tokens in a new context for the recursive
call.
Example
use yap::{ Tokens, IntoTokens };
fn count_digit_comma_calls(toks: &mut impl Tokens<Item=char>) -> (u8, u8) {
let mut counts = (0u8, 0u8);
toks.with_context_mut(&mut counts).sep_by(
|t| {
t.context_mut().0 += 1;
let n_skipped = t.skip_while(|c| c.is_digit(10));
if n_skipped == 0 { None } else { Some(()) }
},
|t| {
t.context_mut().1 += 1;
t.token(',')
}
).consume();
counts
}
let n: usize = 0;
let mut tokens = "123,4,56,1,34,1".into_tokens();
let (digits, seps) = count_digit_comma_calls(&mut tokens);
assert_eq!(tokens.remaining().len(), 0);
// digits parsed 6 times:
assert_eq!(digits, 6);
// Attempted to parse seps 6 times; failure on last ends it:
assert_eq!(seps, 6);
sourcefn slice(&mut self, from: Self::Location, to: Self::Location) -> Slice<'_, Self>
fn slice(&mut self, from: Self::Location, to: Self::Location) -> Slice<'_, Self>
Return a slice of tokens starting at the to
location provided and ending just prior to
the from
location provided (ie equivalent to the range to..from
).
The slice returned from implements Tokens
, so you can use the full range
of parsing functions on it.
Note: the slice returned from this prevents the original tokens from being used until
it’s dropped, and resets the original tokens to their current location on Drop
. if you
core::mem::forget
it, the original token location will equal whatever the slice location
was when it was forgotten.
Example
use yap::{ Tokens, IntoTokens };
let mut s = "abcdefghijklmnop".into_tokens();
(0..5).for_each(|_| { s.next(); });
let from = s.location();
(0..5).for_each(|_| { s.next(); });
let to = s.location();
assert_eq!(s.next(), Some('k'));
assert_eq!(s.next(), Some('l'));
// Iterating the from..to range given:
let vals: String = s.slice(from.clone(), to.clone()).collect();
assert_eq!(&*vals, "fghij");
// After the above is dropped, we can continue
// from where we left off:
assert_eq!(s.next(), Some('m'));
assert_eq!(s.next(), Some('n'));
// We can iterate this range again as we please:
let vals: String = s.slice(from, to).collect();
assert_eq!(&*vals, "fghij");
// And the original remains unaffected..
assert_eq!(s.next(), Some('o'));
assert_eq!(s.next(), Some('p'));
sourcefn offset(&self) -> usize
fn offset(&self) -> usize
Return the current offset into the tokens that we’ve parsed up to so far.
The exact meaning of this can vary by implementation; when parsing slices, it
is index of the slice item we’ve consumed up to, and when
parsing &str
’s it is the number of bytes (not characters) consumed so far.
Example
use yap::{ Tokens, IntoTokens };
let mut s = "abc".into_tokens();
assert_eq!(s.offset(), 0);
s.next();
assert_eq!(s.offset(), 1);
s.next();
assert_eq!(s.offset(), 2);
sourcefn peek(&mut self) -> Option<Self::Item>
fn peek(&mut self) -> Option<Self::Item>
Return the next item in the input without consuming it.
Prefer this to using the peekable
iterator method, which consumes
the tokens, and internally keeps hold of the peeked state itself.
Example
use yap::{ Tokens, IntoTokens };
let mut s = "abc".into_tokens();
assert_eq!(s.peek(), Some('a'));
assert_eq!(s.peek(), Some('a'));
sourcefn token<I>(&mut self, t: I) -> boolwhere
Self::Item: PartialEq,
I: Borrow<Self::Item>,
fn token<I>(&mut self, t: I) -> boolwhere Self::Item: PartialEq, I: Borrow<Self::Item>,
Expect a specific token to be next. If the token is not found, the iterator is not advanced.
Example
use yap::{ Tokens, IntoTokens };
let mut s = "abc".into_tokens();
assert_eq!(s.token(&'a'), true);
assert_eq!(s.token(&'b'), true);
assert_eq!(s.token('z'), false);
assert_eq!(s.token('y'), false);
assert_eq!(s.token('c'), true);
sourcefn tokens<It>(&mut self, ts: It) -> boolwhere
Self::Item: PartialEq,
It: IntoIterator,
It::Item: Borrow<Self::Item>,
fn tokens<It>(&mut self, ts: It) -> boolwhere Self::Item: PartialEq, It: IntoIterator, It::Item: Borrow<Self::Item>,
Expect a specific set of tokens to be next. If the tokens are not found, the iterator is not
advanced. Anything that implements IntoIterator
with an Item
type that can be borrowed to
produce &Item
can be provided as an input to this.
Example
use yap::{ Tokens, IntoTokens };
let mut s = "abcdef".into_tokens();
assert_eq!(s.tokens("abc".chars()), true);
assert_eq!(s.remaining(), "def");
assert_eq!(s.tokens("de".chars()), true);
assert_eq!(s.remaining(), "f");
sourcefn one_of_tokens<It>(&mut self, ts: It) -> Option<Self::Item>where
Self::Item: PartialEq,
It: IntoIterator,
It::Item: Borrow<Self::Item>,
fn one_of_tokens<It>(&mut self, ts: It) -> Option<Self::Item>where Self::Item: PartialEq, It: IntoIterator, It::Item: Borrow<Self::Item>,
Return the first token that matches the tokens provided, or None if none of them match.
Example
use yap::{ Tokens, IntoTokens };
let mut s = "abcdef".into_tokens();
assert_eq!(s.one_of_tokens("abc".chars()), Some('a'));
assert_eq!(s.one_of_tokens("abc".chars()), Some('b'));
assert_eq!(s.one_of_tokens("abc".chars()), Some('c'));
assert_eq!(s.one_of_tokens("abc".chars()), None);
assert_eq!(s.remaining(), "def");
sourcefn take_while<F>(&mut self, f: F) -> TakeWhile<'_, Self, F>where
F: FnMut(&Self::Item) -> bool,
fn take_while<F>(&mut self, f: F) -> TakeWhile<'_, Self, F>where F: FnMut(&Self::Item) -> bool,
Return a Tokens
impl that will consume tokens until the provided function returns false.
Example
use yap::{ Tokens, IntoTokens };
let mut s = "12345abc".into_tokens();
let digits: String = s.take_while(|c| c.is_numeric()).collect();
assert_eq!(&*digits, "12345");
assert_eq!(s.remaining(), "abc");
This exists primarily because Iterator::take_while()
will consume the first token that
does not match the predicate, which is often not what we’d want. The above example using
Iterator::take_while()
would look like:
use yap::{ Tokens, IntoTokens };
let mut s = "12345abc".into_tokens();
let digits: String = s.as_iter().take_while(|c| c.is_numeric()).collect();
assert_eq!(&*digits, "12345");
// Note that `Iterator::take_while` consumed the "a" in order to test it,
// whereas `Tokens::take_while` did not:
assert_eq!(s.remaining(), "bc");
sourcefn skip_while<F>(&mut self, f: F) -> usizewhere
F: FnMut(&Self::Item) -> bool,
fn skip_while<F>(&mut self, f: F) -> usizewhere F: FnMut(&Self::Item) -> bool,
Iterate over the tokens until the provided function returns false on one. Only consume the tokens
that the function returned true for, returning the number of tokens that were consumed/skipped.
Equivalent to toks.take_while(f).count()
.
Example
use yap::{ Tokens, IntoTokens };
let mut s = "12345abc".into_tokens();
let n_skipped = s.skip_while(|c| c.is_numeric());
assert_eq!(n_skipped, 5);
assert_eq!(s.remaining(), "abc");
sourcefn many<F, Output>(&mut self, parser: F) -> Many<'_, Self, F>where
F: FnMut(&mut Self) -> Option<Output>,
fn many<F, Output>(&mut self, parser: F) -> Many<'_, Self, F>where F: FnMut(&mut Self) -> Option<Output>,
Returns a Tokens
impl that runs the provided parser again and again, returning
an output from it each time until it returns None
.
Example
use yap::{ Tokens, IntoTokens };
fn parse_digit_pair(tokens: &mut impl Tokens<Item=char>) -> Option<u32> {
let d1 = tokens.next()?;
let d2 = tokens.next()?;
// Return the result of adding the 2 digits we saw:
Some(d1.to_digit(10)? + d2.to_digit(10)?)
}
let mut s = "12345abcde".into_tokens();
let digits: Vec<u32> = s.many(|t| parse_digit_pair(t)).collect();
assert_eq!(digits, vec![3, 7]);
assert_eq!(s.remaining(), "5abcde");
sourcefn many_err<F, Output, E>(&mut self, parser: F) -> ManyErr<'_, Self, F>where
F: FnMut(&mut Self) -> Result<Output, E>,
fn many_err<F, Output, E>(&mut self, parser: F) -> ManyErr<'_, Self, F>where F: FnMut(&mut Self) -> Result<Output, E>,
Returns a Tokens
impl that runs the provided parser again and again, returning
an output from it each time until it returns None
. If the parser returns an error,
no tokens will be consumed and the error will be returned as the final iteration.
Example
use yap::{ Tokens, IntoTokens };
#[derive(Debug, PartialEq)]
enum Err { NotEnoughTokens, NotADigit(char) }
fn parse_digit_pair(tokens: &mut impl Tokens<Item=char>) -> Result<u32, Err> {
let n1 = tokens.next()
.ok_or(Err::NotEnoughTokens)
.and_then(|c| c.to_digit(10).ok_or(Err::NotADigit(c)))?;
let n2 = tokens.next()
.ok_or(Err::NotEnoughTokens)
.and_then(|c| c.to_digit(10).ok_or(Err::NotADigit(c)))?;
Ok(n1 + n2)
}
let mut s = "12345abcde".into_tokens();
let mut digits_iter = s.many_err(|t| parse_digit_pair(t));
assert_eq!(digits_iter.next(), Some(Ok(3)));
assert_eq!(digits_iter.next(), Some(Ok(7)));
assert_eq!(digits_iter.next(), Some(Err(Err::NotADigit('a'))));
assert_eq!(digits_iter.next(), None);
assert_eq!(s.remaining(), "5abcde");
sourcefn skip_many<F>(&mut self, parser: F) -> usizewhere
F: FnMut(&mut Self) -> bool,
fn skip_many<F>(&mut self, parser: F) -> usizewhere F: FnMut(&mut Self) -> bool,
Ignore 0 or more instances of some parser.
Example
use yap::{ Tokens, IntoTokens };
struct ABC;
fn parse_abc(tokens: &mut impl Tokens<Item=char>) -> Option<ABC> {
let a = tokens.next()?;
let b = tokens.next()?;
let c = tokens.next()?;
if a == 'a' && b == 'b' && c == 'c' {
Some(ABC)
} else {
None
}
}
let mut s = "abcabcababab".into_tokens();
s.skip_many(|t| parse_abc(t).is_some());
assert_eq!(s.remaining(), "ababab");
sourcefn skip_many1<F, E, Ignored>(&mut self, parser: F) -> Result<usize, E>where
F: FnMut(&mut Self) -> Result<Ignored, E>,
fn skip_many1<F, E, Ignored>(&mut self, parser: F) -> Result<usize, E>where F: FnMut(&mut Self) -> Result<Ignored, E>,
Ignore 1 or more instances of some parser. If the provided parser fails immediately, return the error that it produced.
Example
use yap::{ Tokens, IntoTokens };
struct ABC;
fn parse_abc(tokens: &mut impl Tokens<Item=char>) -> Option<ABC> {
let a = tokens.next()?;
let b = tokens.next()?;
let c = tokens.next()?;
if a == 'a' && b == 'b' && c == 'c' {
Some(ABC)
} else {
None
}
}
let mut s = "abcabcabcxyz".into_tokens();
let skipped = s.skip_many1(|t| parse_abc(t).ok_or("aaah"));
assert_eq!(skipped, Ok(3));
assert_eq!(s.remaining(), "xyz");
let mut s = "ababababcabc".into_tokens();
let skipped = s.skip_many1(|t| parse_abc(t).ok_or("aaah"));
assert_eq!(skipped, Err("aaah"));
assert_eq!(s.remaining(), "ababababcabc");
sourcefn sep_by<F, S, Output>(
&mut self,
parser: F,
separator: S
) -> SepBy<'_, Self, F, S>where
F: FnMut(&mut Self) -> Option<Output>,
S: FnMut(&mut Self) -> bool,
fn sep_by<F, S, Output>( &mut self, parser: F, separator: S ) -> SepBy<'_, Self, F, S>where F: FnMut(&mut Self) -> Option<Output>, S: FnMut(&mut Self) -> bool,
Return a Tokens
impl that parses anything matching the first parser
function,
and expects to parse something matching the second separator
function between each
of these.
Example
use yap::{ Tokens, IntoTokens };
fn parse_digit(tokens: &mut impl Tokens<Item=char>) -> Option<u32> {
let c = tokens.next()?;
c.to_digit(10)
}
let mut s = "1,2,3,4,abc".into_tokens();
let digits: Vec<u32> = s.sep_by(|t| parse_digit(t), |t| t.token(',')).collect();
assert_eq!(digits, vec![1,2,3,4]);
assert_eq!(s.remaining(), ",abc");
sourcefn sep_by_err<F, S, E, Output>(
&mut self,
parser: F,
separator: S
) -> SepByErr<'_, Self, F, S>where
F: FnMut(&mut Self) -> Result<Output, E>,
S: FnMut(&mut Self) -> bool,
fn sep_by_err<F, S, E, Output>( &mut self, parser: F, separator: S ) -> SepByErr<'_, Self, F, S>where F: FnMut(&mut Self) -> Result<Output, E>, S: FnMut(&mut Self) -> bool,
Return a Tokens
impl that parses anything matching the parser
function, and expects
to parse something matching the separator
function between each one. Unlike Tokens::sep_by
,
this accepts parsers that return Result
s, and returns the result on each iteration. Once
an error is hit, None
is returned thereafter.
Example
use yap::{ Tokens, IntoTokens };
#[derive(Debug, PartialEq)]
enum Err { NoMoreTokens, NotADigit(char) }
fn parse_digit(tokens: &mut impl Tokens<Item=char>) -> Result<u32, Err> {
let c = tokens.next().ok_or(Err::NoMoreTokens)?;
c.to_digit(10).ok_or(Err::NotADigit(c))
}
let mut s = "1,2,a,1,2,3".into_tokens();
let mut digits_iter = s.sep_by_err(|t| parse_digit(t), |t| t.token(','));
assert_eq!(digits_iter.next(), Some(Ok(1)));
assert_eq!(digits_iter.next(), Some(Ok(2)));
assert_eq!(digits_iter.next(), Some(Err(Err::NotADigit('a'))));
assert_eq!(digits_iter.next(), None);
assert_eq!(s.remaining(), ",a,1,2,3");
sourcefn sep_by_all<F, S, Output>(
&mut self,
parser: F,
separator: S
) -> SepByAll<'_, Self, F, S, Output>where
F: FnMut(&mut Self) -> Option<Output>,
S: FnMut(&mut Self) -> Option<Output>,
fn sep_by_all<F, S, Output>( &mut self, parser: F, separator: S ) -> SepByAll<'_, Self, F, S, Output>where F: FnMut(&mut Self) -> Option<Output>, S: FnMut(&mut Self) -> Option<Output>,
Returns a Tokens
impl that parses anything matching the parser
function,
and expects to parse something matching the separator
function between each one.
The Tokens
impl hands back the output from both the parser
and separator
function, which means that they are both expected to return the same type.
Example
use yap::{ Tokens, IntoTokens };
#[derive(PartialEq,Debug)]
enum Op { Plus, Minus, Divide }
#[derive(PartialEq,Debug)]
enum OpOrDigit { Op(Op), Digit(u32) }
fn parse_op(tokens: &mut impl Tokens<Item=char>) -> Option<Op> {
match tokens.next()? {
'-' => Some(Op::Minus),
'+' => Some(Op::Plus),
'/' => Some(Op::Divide),
_ => None
}
}
fn parse_digit(tokens: &mut impl Tokens<Item=char>) -> Option<u32> {
let c = tokens.next()?;
c.to_digit(10)
}
let mut s = "1+2/3-4+abc".into_tokens();
let output: Vec<_> = s.sep_by_all(
|t| parse_digit(t).map(OpOrDigit::Digit),
|t| parse_op(t).map(OpOrDigit::Op)
).collect();
assert_eq!(output, vec![
OpOrDigit::Digit(1),
OpOrDigit::Op(Op::Plus),
OpOrDigit::Digit(2),
OpOrDigit::Op(Op::Divide),
OpOrDigit::Digit(3),
OpOrDigit::Op(Op::Minus),
OpOrDigit::Digit(4),
]);
assert_eq!(s.remaining(), "+abc");
sourcefn sep_by_all_err<F, S, Output, E>(
&mut self,
parser: F,
separator: S
) -> SepByAllErr<'_, Self, F, S, Output>where
F: FnMut(&mut Self) -> Result<Output, E>,
S: FnMut(&mut Self) -> Option<Output>,
fn sep_by_all_err<F, S, Output, E>( &mut self, parser: F, separator: S ) -> SepByAllErr<'_, Self, F, S, Output>where F: FnMut(&mut Self) -> Result<Output, E>, S: FnMut(&mut Self) -> Option<Output>,
Similar to Tokens::sep_by_all
, except that the Tokens
impl returned also
hands back the first error encountered when attempting to run our parser
.
Example
use yap::{ Tokens, IntoTokens };
#[derive(PartialEq,Debug)]
enum Op { Plus, Minus, Divide }
#[derive(PartialEq,Debug)]
enum OpOrDigit { Op(Op), Digit(u32) }
#[derive(Debug, PartialEq)]
enum Err { NoMoreTokens, NotADigit(char) }
fn parse_op(tokens: &mut impl Tokens<Item=char>) -> Option<Op> {
match tokens.next()? {
'-' => Some(Op::Minus),
'+' => Some(Op::Plus),
'/' => Some(Op::Divide),
_ => None
}
}
fn parse_digit(tokens: &mut impl Tokens<Item=char>) -> Result<u32, Err> {
let c = tokens.next().ok_or(Err::NoMoreTokens)?;
c.to_digit(10).ok_or(Err::NotADigit(c))
}
let mut s = "1+2/3-4+abc".into_tokens();
let output: Vec<_> = s.sep_by_all_err(
|t| parse_digit(t).map(OpOrDigit::Digit),
|t| parse_op(t).map(OpOrDigit::Op)
).collect();
assert_eq!(output, vec![
Ok(OpOrDigit::Digit(1)),
Ok(OpOrDigit::Op(Op::Plus)),
Ok(OpOrDigit::Digit(2)),
Ok(OpOrDigit::Op(Op::Divide)),
Ok(OpOrDigit::Digit(3)),
Ok(OpOrDigit::Op(Op::Minus)),
Ok(OpOrDigit::Digit(4)),
Err(Err::NotADigit('a'))
]);
assert_eq!(s.remaining(), "+abc");
sourcefn surrounded_by<F, S, Output>(&mut self, parser: F, surrounding: S) -> Outputwhere
F: FnOnce(&mut Self) -> Output,
S: FnMut(&mut Self),
fn surrounded_by<F, S, Output>(&mut self, parser: F, surrounding: S) -> Outputwhere F: FnOnce(&mut Self) -> Output, S: FnMut(&mut Self),
Parse some tokens that are optionally surrounded by the result of a surrounding
parser.
Example
use yap::{ Tokens, IntoTokens };
let mut s = " hello ".into_tokens();
let hello: String = s.surrounded_by(
|t| t.take_while(|c| c.is_ascii_alphabetic()).collect(),
|t| { t.skip_while(|c| c.is_ascii_whitespace()); }
);
assert_eq!(&*hello, "hello");
assert_eq!(s.remaining(), "");
sourcefn optional<F, Output>(&mut self, f: F) -> Outputwhere
F: FnOnce(&mut Self) -> Output,
Output: IsMatch,
fn optional<F, Output>(&mut self, f: F) -> Outputwhere F: FnOnce(&mut Self) -> Output, Output: IsMatch,
Attempt to parse some output from the tokens. Returning None
or false
signifies that no match was found, and no tokens will
be consumed. Otherwise, we’ll return the match and consume the tokens.
Example
use yap::{ Tokens, IntoTokens };
let mut s = "foobar".into_tokens();
let res = s.optional(|s| {
let a = s.next();
let b = s.next();
if a == b {
Some("yay")
} else {
None
}
});
// nothing consumed since None returned from fn
assert_eq!(s.remaining(), "foobar");
assert_eq!(res, None);
let res = s.optional(|s| {
let a = s.next()?;
let b = s.next()?;
Some((a, b))
});
// 2 chars consumed since Some returned from fn
assert_eq!(s.remaining(), "obar");
assert_eq!(res, Some(('f', 'o')));
sourcefn optional_err<F, Output, Error>(&mut self, f: F) -> Result<Output, Error>where
F: FnOnce(&mut Self) -> Result<Output, Error>,
fn optional_err<F, Output, Error>(&mut self, f: F) -> Result<Output, Error>where F: FnOnce(&mut Self) -> Result<Output, Error>,
Attempt to parse some output from the tokens, returning a Result
.
If the Result
returned is Err
, no tokens will be consumed.
Example
use yap::{ Tokens, IntoTokens };
let mut s = "foobar".into_tokens();
let res = s.optional_err(|s| {
let a = s.next();
let b = s.next();
if a == b {
Ok("yay")
} else {
Err("a and b don't match!")
}
});
// nothing consumed since Err returned from fn
assert_eq!(s.remaining(), "foobar");
assert!(res.is_err());
let res = s.optional_err(|s| {
let a = s.next();
let b = s.next();
if a != b {
Ok((a, b))
} else {
Err("a and b match!")
}
});
// 2 chars consumed since Ok returned from fn
assert_eq!(s.remaining(), "obar");
assert_eq!(res, Ok((Some('f'), Some('o'))));
sourcefn eof(&mut self) -> bool
fn eof(&mut self) -> bool
Checks next input is None
and, if true, consumes the None
.
Example
use yap::Tokens;
use yap::types::IterTokens;
use core::iter;
// This will always return None; eof will consume it.
let mut toks = IterTokens::new(iter::empty::<char>());
assert_eq!(toks.eof(), true);
assert_eq!(toks.offset(), 1);
assert_eq!(toks.eof(), true);
assert_eq!(toks.offset(), 2);
// This will return a char; eof won't consume anything.
let mut toks = IterTokens::new(iter::once('a'));
assert_eq!(toks.eof(), false);
assert_eq!(toks.offset(), 0);
assert_eq!(toks.eof(), false);
assert_eq!(toks.offset(), 0);
sourcefn consume(&mut self)
fn consume(&mut self)
Consume all remaining tokens. This is expected to be used in conjunction
with combinators likeTokens::take
and Tokens::take_while
. This is
just a shorthand for toks.as_iter().for_each(drop)
.
Example
use yap::{Tokens, IntoTokens};
let mut toks = "abc123def".into_tokens();
// Take won't do anything unless it's consumed:
toks.take(3);
assert_eq!(toks.remaining(), "abc123def");
// This is the same as `toks.take(3).as_iter().for_each(drop);`
toks.take(3).consume();
assert_eq!(toks.remaining(), "123def");
toks.take_while(|t| t.is_numeric()).consume();
assert_eq!(toks.remaining(), "def");
sourcefn collect<B: FromIterator<Self::Item>>(&mut self) -> B
fn collect<B: FromIterator<Self::Item>>(&mut self) -> B
Collect up all of the tokens into something that implements
FromIterator
. If you’d like to call str::parse
on the
subsequent collection, then prefer Tokens::parse
, which can
be more optimal in some cases.
This is just a shorthand for calling toks.as_iter().collect()
.