token/
token.rs

1use parseme::{parser::Group, stream::Stream, Parser, Source};
2
3#[derive(Debug, PartialEq)]
4pub enum Token {
5    Whitespace,
6    Ident(String),
7}
8
9fn parse_whitespace(input: &mut Source) -> Result<Token, ()> {
10    parseme::iter::next_if(input, char::is_whitespace).ok_or(())?;
11
12    Ok(Token::Whitespace)
13}
14
15fn parse_iden<'a>(input: &mut Source<'a>) -> Result<Token, ()> {
16    let start_pos = input.pos();
17
18    parseme::iter::next_if(input, parseme::xid::is_start).ok_or(())?;
19    parseme::iter::advance_while(input, parseme::xid::is_continue);
20
21    Ok(Token::Ident(
22        input.src()[start_pos..input.pos()].to_string(),
23    ))
24}
25
26fn main() {
27    let mut source = Source::new("hello world");
28    let mut group = Group::new().add(parse_whitespace).add(parse_iden);
29    let mut parser = group.stream(&mut source);
30
31    assert_eq!(parser.next(), Ok(Token::Ident("hello".to_string())));
32    assert_eq!(parser.next(), Ok(Token::Whitespace));
33    assert_eq!(parser.next(), Ok(Token::Ident("world".to_string())));
34}
token/token.rs

token/
token.rs
