tokit 0.0.0

Blazing fast parser combinators: parse-while-lexing (zero-copy), deterministic LALR-style parsing, no backtracking. Flexible emitters for fail-fast runtime or greedy compiler diagnostics
Documentation
use crate::{
  error::UnexpectedEot,
  lexer::Span,
  utils::{Located, marker::PhantomLocated},
};

use super::*;

/// A parser that accepts any token.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
pub struct Any<L, Ctx, Lang: ?Sized = ()> {
  _lxr: PhantomData<L>,
  _ctx: PhantomData<Ctx>,
  _lang: PhantomData<Lang>,
}

impl<L, Ctx> Any<L, Ctx> {
  /// Creates a parser that accepts any token.
  #[cfg_attr(not(tarpaulin), inline(always))]
  pub const fn new() -> Self {
    Self::of()
  }

  /// Creates a parser that yields any token with its span
  #[cfg_attr(not(tarpaulin), inline(always))]
  pub const fn spanned() -> With<Self, PhantomSpan> {
    Self::spanned_of()
  }

  /// Creates a parser that yields any token with its source
  #[cfg_attr(not(tarpaulin), inline(always))]
  pub const fn sliced() -> With<Self, PhantomSliced> {
    Self::sliced_of()
  }

  /// Creates a parser that yields any token without its source and span.
  #[cfg_attr(not(tarpaulin), inline(always))]
  pub const fn located() -> With<Self, PhantomLocated> {
    Self::located_of()
  }
}

impl<L, Ctx, Lang> Any<L, Ctx, Lang> {
  /// Creates a parser that accepts any token.
  #[cfg_attr(not(tarpaulin), inline(always))]
  pub const fn of() -> Self {
    Any {
      _lxr: PhantomData,
      _ctx: PhantomData,
      _lang: PhantomData,
    }
  }

  /// Creates a parser that yields any token with its span.
  #[cfg_attr(not(tarpaulin), inline(always))]
  pub const fn spanned_of() -> With<Self, PhantomSpan> {
    With::new(Self::of(), PhantomSpan::PHANTOM)
  }

  /// Creates a parser that yields any token with its source.
  #[cfg_attr(not(tarpaulin), inline(always))]
  pub const fn sliced_of() -> With<Self, PhantomSliced> {
    With::new(Self::of(), PhantomSliced::PHANTOM)
  }

  /// Creates a parser that yields any token without its source and span.
  #[cfg_attr(not(tarpaulin), inline(always))]
  pub const fn located_of() -> With<Self, PhantomLocated> {
    With::new(Self::of(), PhantomLocated::PHANTOM)
  }
}

impl<'inp, L, Ctx, Lang: ?Sized> ParseInput<'inp, L, L::Token, Ctx, Lang> for Any<L, Ctx, Lang>
where
  L: Lexer<'inp>,
  Ctx: ParseContext<'inp, L, Lang>,
  <Ctx::Emitter as Emitter<'inp, L, Lang>>::Error:
    From<UnexpectedEot<L::Offset, Lang>> + From<<L::Token as Token<'inp>>::Error>,
{
  #[cfg_attr(not(tarpaulin), inline(always))]
  fn parse_input(
    &mut self,
    inp: &mut InputRef<'inp, '_, L, Ctx, Lang>,
  ) -> Result<L::Token, <Ctx::Emitter as Emitter<'inp, L, Lang>>::Error>
  where
    Ctx: ParseContext<'inp, L, Lang>,
  {
    match inp.next() {
      Some(Spanned { data: tok, .. }) => match tok {
        Lexed::Token(tok) => Ok(tok),
        Lexed::Error(err) => Err(err.into()),
      },
      None => Err(UnexpectedEot::eot_of(inp.span().end()).into()),
    }
  }
}

impl<'inp, L, Ctx, Lang: ?Sized> ParseInput<'inp, L, Spanned<L::Token, L::Span>, Ctx, Lang>
  for With<Any<L, Ctx, Lang>, PhantomSpan>
where
  L: Lexer<'inp>,
  Ctx: ParseContext<'inp, L, Lang>,
  <Ctx::Emitter as Emitter<'inp, L, Lang>>::Error:
    From<UnexpectedEot<L::Offset, Lang>> + From<<L::Token as Token<'inp>>::Error>,
{
  #[cfg_attr(not(tarpaulin), inline(always))]
  fn parse_input(
    &mut self,
    inp: &mut InputRef<'inp, '_, L, Ctx, Lang>,
  ) -> Result<Spanned<L::Token, L::Span>, <Ctx::Emitter as Emitter<'inp, L, Lang>>::Error>
  where
    Ctx: ParseContext<'inp, L, Lang>,
  {
    match inp.next() {
      Some(Spanned { data: tok, span }) => match tok {
        Lexed::Token(tok) => Ok(Spanned::new(span, tok)),
        Lexed::Error(err) => Err(err.into()),
      },
      None => Err(UnexpectedEot::eot_of(inp.span().end()).into()),
    }
  }
}

impl<'inp, L, Ctx, Lang: ?Sized>
  ParseInput<'inp, L, Sliced<L::Token, <L::Source as Source<L::Offset>>::Slice<'inp>>, Ctx, Lang>
  for With<Any<L, Ctx, Lang>, PhantomSliced>
where
  L: Lexer<'inp>,
  Ctx: ParseContext<'inp, L, Lang>,
  <Ctx::Emitter as Emitter<'inp, L, Lang>>::Error:
    From<UnexpectedEot<L::Offset, Lang>> + From<<L::Token as Token<'inp>>::Error>,
{
  #[cfg_attr(not(tarpaulin), inline(always))]
  fn parse_input(
    &mut self,
    inp: &mut InputRef<'inp, '_, L, Ctx, Lang>,
  ) -> Result<
    Sliced<L::Token, <L::Source as Source<L::Offset>>::Slice<'inp>>,
    <Ctx::Emitter as Emitter<'inp, L, Lang>>::Error,
  >
  where
    Ctx: ParseContext<'inp, L, Lang>,
  {
    match inp.next() {
      Some(Spanned { data: tok, .. }) => match tok {
        Lexed::Token(tok) => Ok(Sliced::new(
          inp
            .slice()
            .expect("lexer gurantees there must be a valid slice to yield a token"),
          tok,
        )),
        Lexed::Error(err) => Err(err.into()),
      },
      None => Err(UnexpectedEot::eot_of(inp.span().end()).into()),
    }
  }
}

impl<'inp, L, Ctx, Lang: ?Sized>
  ParseInput<
    'inp,
    L,
    Located<L::Token, L::Span, <L::Source as Source<L::Offset>>::Slice<'inp>>,
    Ctx,
    Lang,
  > for With<Any<L, Ctx, Lang>, PhantomLocated>
where
  L: Lexer<'inp>,
  Ctx: ParseContext<'inp, L, Lang>,
  <Ctx::Emitter as Emitter<'inp, L, Lang>>::Error:
    From<UnexpectedEot<L::Offset, Lang>> + From<<L::Token as Token<'inp>>::Error>,
{
  #[cfg_attr(not(tarpaulin), inline(always))]
  fn parse_input(
    &mut self,
    inp: &mut InputRef<'inp, '_, L, Ctx, Lang>,
  ) -> Result<
    Located<L::Token, L::Span, <L::Source as Source<L::Offset>>::Slice<'inp>>,
    <Ctx::Emitter as Emitter<'inp, L, Lang>>::Error,
  >
  where
    Ctx: ParseContext<'inp, L, Lang>,
  {
    match inp.next() {
      Some(Spanned { data: tok, span }) => match tok {
        Lexed::Token(tok) => Ok(Located::new(
          inp
            .slice()
            .expect("lexer gurantees there must be a valid slice to yield a token"),
          span,
          tok,
        )),
        Lexed::Error(err) => Err(err.into()),
      },
      None => Err(UnexpectedEot::eot_of(inp.span().end()).into()),
    }
  }
}

#[cfg(test)]
mod tests {
  use crate::lexer::{DummyLexer, DummyToken};

  use super::*;

  fn assert_any_parse_impl<'inp>() -> impl Parse<'inp, DummyLexer, DummyToken, ()> {
    Parser::new().apply(Any::spanned().map(Spanned::into_data))
  }

  fn assert_any_parse_with_context_impl<'inp>() -> impl Parse<'inp, DummyLexer, DummyToken, ()> {
    Parser::with_context(()).apply(Any::new().spanned().map(Spanned::into_data))
  }

  #[test]
  fn assert_parse_impl() {
    let _ = assert_any_parse_impl();
    let _ = assert_any_parse_with_context_impl();
  }
}