Overview
Hinku is a small library for writing parsers based on token streams.
It provides an integration for lexers created with the logos crate
but provedes flexible trait based interface for integration with other token streams.
Example
Below you can find a rudimentary parser implemented using this library and the Logos crate.
use logos::Logos;
use hinku::{
logos::BufferedLexer,
Either,
ParseError,
ParseResult,
TokenStream,
TokenStreamExt,
};
#[derive(Logos, Debug, Clone, PartialEq)]
enum Token {
#[token("foo")]
Foo,
#[token("bar")]
Bar,
#[error]
#[regex(r"[ \n\r\f\t]+", logos::skip)]
Error,
}
fn foo(stream: &mut dyn TokenStream<Token>) -> ParseResult<Token, String> {
match stream.advance() {
None => Err(ParseError::EndOfStream),
Some((Token::Foo, _)) => Ok(Token::Foo),
Some((other, span)) => Err(ParseError::custom(span, "expected a foo".into())),
}
}
fn bar(stream: &mut dyn TokenStream<Token>) -> ParseResult<Token, String> {
match stream.advance() {
None => Err(ParseError::EndOfStream),
Some((Token::Bar, _)) => Ok(Token::Bar),
Some((other, span)) => Err(ParseError::custom(span, "expected a bar".into())),
}
}
fn foo_or_bar(mut stream: &mut dyn TokenStream<Token>) -> ParseResult<Token, String> {
stream.either(foo, bar)
.map(Either::merge)
.expected("expected either a foo or a bar")
}
fn foobar(mut stream: &mut dyn TokenStream<Token>) -> ParseResult<(Token, Token), String> {
let f = stream.take(foo)?;
let b = stream.take(bar)?;
Ok((f, b))
}
let lex = Token::lexer("foo bar bar foo bar");
let mut stream = BufferedLexer::new(lex);
assert_eq!(stream.take(foo), Ok(Token::Foo));
assert_eq!(stream.take(bar), Ok(Token::Bar));
assert_eq!(stream.take(foo_or_bar), Ok(Token::Bar));
assert_eq!(stream.take(foobar), Ok((Token::Foo, Token::Bar)));