Crate hinku

A lightweight library for writing parsers that work on token streams.

Example

use logos::Logos;
use hinku::{
    logos::BufferedLexer,
    Either,
    ParseError,
    ParseResult,
    TokenStream,
    TokenStreamExt,
};

#[derive(Logos, Debug, Clone, PartialEq)]
enum Token {
    #[token("foo")]
    Foo,

    #[token("bar")]
    Bar,

    #[error]
    #[regex(r"[ \n\r\f\t]+", logos::skip)]
    Error,
}

/// A function that either consumes a Foo token or returns an error.
fn foo(stream: &mut dyn TokenStream<Token>) -> ParseResult<Token, String> {
    match stream.advance() {
        None => Err(ParseError::EndOfStream),
        Some((Token::Foo, _)) => Ok(Token::Foo),
        Some((other, span)) => Err(ParseError::custom(span, "expected a foo".into())),
    }
}

/// A function that either consumes a Bar token or returns an error.
fn bar(stream: &mut dyn TokenStream<Token>) -> ParseResult<Token, String> {
    match stream.advance() {
        None => Err(ParseError::EndOfStream),
        Some((Token::Bar, _)) => Ok(Token::Bar),
        Some((other, span)) => Err(ParseError::custom(span, "expected a bar".into())),
    }
}

/// A function that consumes either one of the tokens.
fn foo_or_bar(mut stream: &mut dyn TokenStream<Token>) -> ParseResult<Token, String> {
    stream.either(foo, bar)
        .map(Either::merge)
        .expected("expected either a foo or a bar")
}

/// A function that expects a Foo token, followed by a Bar token.
fn foobar(mut stream: &mut dyn TokenStream<Token>) -> ParseResult<(Token, Token), String> {
    let f = stream.take(foo)?;
    let b = stream.take(bar)?;

    Ok((f, b))
}

let lex = Token::lexer("foo bar bar foo bar");
let mut stream = BufferedLexer::new(lex);

assert_eq!(stream.take(foo), Ok(Token::Foo));
assert_eq!(stream.take(bar), Ok(Token::Bar));
assert_eq!(stream.take(foo_or_bar), Ok(Token::Bar));
assert_eq!(stream.take(foobar), Ok((Token::Foo, Token::Bar)));

Modules

logos

Structs

SpanningStream

A TokenStream that keeps track of the span advanced while parsing.

TokenStreamFork

Fork of a token stream.

Enums

Either

ParseError

Represents a parsing error.

Traits

ParseResultExt

Implements convinience methods related to error handling for ParseResult.

TokenStream

Token stream with backtracking support.

TokenStreamExt

Type Aliases

ParseResult

Span

Byte range in the source.