use std::cell::{Cell, RefCell};
use crate::{
ast::{Comment, CommentKind},
error::{Error, Result},
ident::Ident,
lexer,
span::Span,
token::{Token, TokenKind},
};
pub struct ParserState {
cursor: usize,
pending_gts: u8,
}
pub struct ParseStream<'a> {
tokens: &'a [Token],
cursor: Cell<usize>,
errors: RefCell<Vec<Error>>,
pending_gts: Cell<u8>,
synthetic_gt: Token,
pending_comments: RefCell<Vec<Comment>>,
}
impl<'a> ParseStream<'a> {
pub(crate) fn new(tokens: &'a [Token]) -> Self {
ParseStream {
tokens,
cursor: Cell::new(0),
errors: RefCell::new(Vec::new()),
pending_gts: Cell::new(0),
synthetic_gt: Token {
kind: TokenKind::Gt,
span: Span::new(0, 0),
},
pending_comments: RefCell::new(Vec::new()),
}
}
pub fn is_empty(&self) -> bool {
self.skip_comments();
let cursor = self.cursor.get();
cursor >= self.tokens.len() - 1
}
pub fn is_empty_raw(&self) -> bool {
let cursor = self.cursor.get();
cursor >= self.tokens.len() - 1
}
pub fn peek(&self) -> &Token {
if self.pending_gts.get() > 0 {
return &self.synthetic_gt;
}
self.skip_comments();
let cursor = self.cursor.get();
&self.tokens[cursor.min(self.tokens.len() - 1)]
}
pub fn peek_raw(&self) -> &Token {
let cursor = self.cursor.get();
&self.tokens[cursor.min(self.tokens.len() - 1)]
}
pub fn cursor(&self) -> usize {
self.cursor.get()
}
pub fn set_cursor(&self, pos: usize) {
self.cursor.set(pos);
}
pub fn save_state(&self) -> ParserState {
ParserState {
cursor: self.cursor.get(),
pending_gts: self.pending_gts.get(),
}
}
pub fn restore_state(&self, state: ParserState) {
self.cursor.set(state.cursor);
self.pending_gts.set(state.pending_gts);
}
fn advance(&self) -> &Token {
if self.pending_gts.get() > 0 {
self.pending_gts.set(self.pending_gts.get() - 1);
return &self.synthetic_gt;
}
let cursor = self.cursor.get();
let tok = &self.tokens[cursor];
if cursor < self.tokens.len() - 1 {
self.cursor.set(cursor + 1);
}
self.skip_comments();
tok
}
pub fn error<T: std::fmt::Display>(&self, span: Span, msg: T) -> Error {
let err = Error::new(span, msg);
self.errors.borrow_mut().push(err.clone());
err
}
pub fn split_gt(&self) {
let cursor = self.cursor.get();
let kind = &self.tokens[cursor.min(self.tokens.len() - 1)].kind;
match kind {
TokenKind::GtGt => {
self.advance();
self.pending_gts.set(self.pending_gts.get() + 1);
}
TokenKind::GtGtGt => {
self.advance();
self.pending_gts.set(self.pending_gts.get() + 2);
}
_ => {}
}
}
pub fn next(&self) -> &Token {
self.advance()
}
pub fn is(&self, kind: &TokenKind) -> bool {
&self.peek().kind == kind
}
pub fn is_ident(&self, name: &str) -> bool {
match &self.peek().kind {
TokenKind::Ident(s) => s == name,
_ => false,
}
}
pub fn is_any_ident(&self) -> bool {
matches!(
&self.peek().kind,
TokenKind::Ident(_)
| TokenKind::Record
| TokenKind::Sealed
| TokenKind::Var
| TokenKind::Yield
| TokenKind::Open
| TokenKind::Provides
| TokenKind::Requires
| TokenKind::Uses
| TokenKind::With
| TokenKind::When
| TokenKind::To
| TokenKind::Exports
| TokenKind::Opens
| TokenKind::Transitive
| TokenKind::Permits
| TokenKind::NonSealed
| TokenKind::Module
| TokenKind::Byte
| TokenKind::Short
| TokenKind::Int
| TokenKind::Long
| TokenKind::Char
| TokenKind::Float
| TokenKind::Double
| TokenKind::Boolean
| TokenKind::Void
)
}
pub fn is_type_ident(&self) -> bool {
self.is_any_ident() || self.is(&TokenKind::At)
}
pub fn is_keyword(&self, kind: TokenKind) -> bool {
self.peek().kind == kind
}
pub fn eat(&self, expected: &TokenKind) -> bool {
if self.is(expected) {
self.advance();
true
} else {
false
}
}
pub fn expect(&self, kind: TokenKind) -> Result<()> {
if self.is(&kind) {
self.advance();
Ok(())
} else {
Err(Error::expected_token(self.peek().span, &kind.to_string()))
}
}
pub fn look_ahead(&self, n: usize) -> &Token {
let mut pos = self.cursor.get();
let mut remaining = n;
while pos < self.tokens.len() - 1 {
if !is_comment_token(&self.tokens[pos].kind) {
if remaining == 0 {
break;
}
remaining -= 1;
}
pos += 1;
}
&self.tokens[pos.min(self.tokens.len() - 1)]
}
pub fn parse_terminated<T, F>(&self, mut parse_item: F) -> Result<Vec<T>>
where
F: FnMut(&ParseStream) -> Result<T>,
{
let mut items = Vec::new();
if self.is_empty() || !can_start_item(&self.peek().kind) {
return Ok(items);
}
loop {
items.push(parse_item(self)?);
if !self.eat(&TokenKind::Comma) {
break;
}
}
Ok(items)
}
pub fn parse_separated<T, F>(
&self,
can_start_fn: fn(&TokenKind) -> bool,
mut parse_item: F,
) -> Result<Vec<T>>
where
F: FnMut(&ParseStream) -> Result<T>,
{
let mut items = Vec::new();
if self.is_empty() || !can_start_fn(&self.peek().kind) {
return Ok(items);
}
loop {
items.push(parse_item(self)?);
if !self.eat(&TokenKind::Comma) {
break;
}
if self.is_empty() {
break;
}
}
Ok(items)
}
pub fn try_parse<T, F>(&self, f: F) -> Option<T>
where
F: FnOnce(&ParseStream) -> Result<T>,
{
let saved = self.cursor.get();
match f(self) {
Ok(t) => Some(t),
Err(_) => {
self.cursor.set(saved);
None
}
}
}
pub fn parse_parenthesized<T, F>(&self, mut f: F) -> Result<T>
where
F: FnMut(&ParseStream) -> Result<T>,
{
self.expect(TokenKind::LParen)?;
let result = f(self)?;
self.expect(TokenKind::RParen)?;
Ok(result)
}
pub fn parse<T: Parse>(&self) -> Result<T> {
T::parse(self)
}
pub fn parse_braced<T, F>(&self, mut f: F) -> Result<T>
where
F: FnMut(&ParseStream) -> Result<T>,
{
self.expect(TokenKind::LBrace)?;
let result = f(self)?;
self.expect(TokenKind::RBrace)?;
Ok(result)
}
pub fn parse_bracketed<T, F>(&self, mut f: F) -> Result<T>
where
F: FnMut(&ParseStream) -> Result<T>,
{
self.expect(TokenKind::LBracket)?;
let result = f(self)?;
self.expect(TokenKind::RBracket)?;
Ok(result)
}
pub fn expect_then_raw_span(&self, kind: TokenKind) -> Result<Span> {
self.expect(kind)?;
Ok(self.peek_raw().span)
}
pub fn parse_ident(&self) -> Result<Ident> {
match &self.peek().kind {
TokenKind::Ident(s) => {
let span = self.peek().span;
self.advance();
Ok(Ident::new(s.clone(), span))
}
TokenKind::Record
| TokenKind::Sealed
| TokenKind::Var
| TokenKind::Yield
| TokenKind::Open
| TokenKind::Provides
| TokenKind::Requires
| TokenKind::Uses
| TokenKind::With
| TokenKind::When
| TokenKind::To
| TokenKind::Exports
| TokenKind::Opens
| TokenKind::Transitive
| TokenKind::Permits
| TokenKind::NonSealed
| TokenKind::Module
| TokenKind::Byte
| TokenKind::Short
| TokenKind::Int
| TokenKind::Long
| TokenKind::Char
| TokenKind::Float
| TokenKind::Double
| TokenKind::Boolean
| TokenKind::Void => {
let name = format!("{}", self.peek().kind);
let span = self.peek().span;
self.advance();
Ok(Ident::new(name, span))
}
_other => Err(Error::expected_token(self.peek().span, "identifier")),
}
}
pub fn take_errors(&self) -> Vec<Error> {
self.errors.borrow_mut().drain(..).collect()
}
pub fn span_since(&self, start: Span) -> Span {
let end = if self.cursor.get() > 0 {
self.tokens[self.cursor.get() - 1].span
} else {
start
};
start.join(end)
}
pub fn skip_comments_to_peek(&self) {
self.skip_comments();
}
fn skip_comments(&self) {
while self.cursor.get() < self.tokens.len()
&& is_comment_token(&self.tokens[self.cursor.get()].kind)
{
let tok = &self.tokens[self.cursor.get()];
self.pending_comments
.borrow_mut()
.push(token_to_comment(tok));
self.cursor.set(self.cursor.get() + 1);
}
}
pub fn collect_pending_doc_comments(&self) -> Vec<Comment> {
let all = self
.pending_comments
.borrow_mut()
.drain(..)
.collect::<Vec<_>>();
all.into_iter()
.filter(|c| c.kind == CommentKind::DocLine || c.kind == CommentKind::DocBlock)
.collect()
}
pub fn collect_pending_comments(&self) -> Vec<Comment> {
self.pending_comments.borrow_mut().drain(..).collect()
}
pub fn collect_leading_doc_comments(&self) -> Vec<Comment> {
let mut comments = Vec::new();
while self.cursor.get() < self.tokens.len() {
match &self.tokens[self.cursor.get()].kind {
TokenKind::DocLineComment(_) | TokenKind::DocBlockComment(_) => {
let tok = &self.tokens[self.cursor.get()];
comments.push(token_to_comment(tok));
self.cursor.set(self.cursor.get() + 1);
}
TokenKind::LineComment(_) | TokenKind::BlockComment(_) => {
self.cursor.set(self.cursor.get() + 1);
}
_ => break,
}
}
comments
}
pub fn collect_leading_comments(&self) -> Vec<Comment> {
let mut comments = Vec::new();
while self.cursor.get() < self.tokens.len() {
match &self.tokens[self.cursor.get()].kind {
TokenKind::LineComment(_)
| TokenKind::BlockComment(_)
| TokenKind::DocLineComment(_)
| TokenKind::DocBlockComment(_) => {
let tok = &self.tokens[self.cursor.get()];
comments.push(token_to_comment(tok));
self.cursor.set(self.cursor.get() + 1);
}
_ => break,
}
}
comments
}
}
fn is_comment_token(kind: &TokenKind) -> bool {
matches!(
kind,
TokenKind::LineComment(_)
| TokenKind::BlockComment(_)
| TokenKind::DocLineComment(_)
| TokenKind::DocBlockComment(_)
)
}
fn token_to_comment(tok: &Token) -> Comment {
let kind = match &tok.kind {
TokenKind::DocLineComment(_) => CommentKind::DocLine,
TokenKind::DocBlockComment(_) => CommentKind::DocBlock,
TokenKind::LineComment(_) => CommentKind::Line,
TokenKind::BlockComment(_) => CommentKind::Block,
_ => unreachable!(),
};
Comment {
kind,
span: tok.span,
}
}
fn can_start_item(kind: &TokenKind) -> bool {
!matches!(
kind,
TokenKind::Eof
| TokenKind::RParen
| TokenKind::RBrace
| TokenKind::RBracket
| TokenKind::Semicolon
| TokenKind::Comma
)
}
pub trait Parse: Sized {
fn parse(input: &ParseStream) -> Result<Self>;
}
impl Parse for Ident {
fn parse(input: &ParseStream) -> Result<Self> {
input.parse_ident()
}
}
pub fn parse_str<T: Parse>(s: &str) -> Result<T> {
let tokens = lexer::tokenize(s);
let stream = ParseStream::new(&tokens);
let result = T::parse(&stream)?;
if !stream.is_empty() {
return Err(Error::new(stream.peek().span, "unexpected trailing tokens"));
}
Ok(result)
}
pub fn parse<T: Parse>(s: &str) -> Result<T> {
let tokens = lexer::tokenize(s);
let stream = ParseStream::new(&tokens);
T::parse(&stream)
}
pub fn parse_file<T: Parse>(path: &std::path::Path) -> Result<T> {
let content = std::fs::read_to_string(path)
.map_err(|e| Error::new(Span::call_site(), format!("failed to read file: {}", e)))?;
parse_str(&content)
}
#[macro_export]
macro_rules! peek {
($stream:expr, $kind:ident) => {
$stream.is(&$crate::token::TokenKind::$kind)
};
}
#[macro_export]
macro_rules! peek_ident {
($stream:expr) => {
$stream.is_any_ident()
};
}
#[macro_export]
macro_rules! opt {
($stream:expr, $method:ident $(, $arg:expr)*) => {
if $stream.is_empty() {
None
} else {
match $stream.$method($($arg),*) {
Ok(val) => Some(val),
Err(_) => None,
}
}
};
}