use crate::grammar::Grammar;
use crate::lexer::{lex, LexerToken};
use crate::syntax::{SyntaxKind, SyntaxNode};
use crate::token_set::TokenSet;
use drop_bomb::DropBomb;
use rowan::GreenNodeBuilder;
use smol_str::SmolStr;
use std::cell::Cell;
const PARSER_STEP_LIMIT: u32 = 10_000_000;
pub fn parse_owned_ast(grammar: &Grammar, text: &str) -> crate::ast::owned::Root {
let parsed = parse(grammar, text);
crate::ast::concrete::Root::try_from(parsed.syntax())
.unwrap()
.to_owned_ast()
}
pub fn parse(grammar: &Grammar, text: &str) -> Parsed {
let tokens = lex(text);
let mut parser = Parser::new(grammar, TokenSliceSource::new(&tokens));
root(&mut parser);
Parsed::new(parser.end(), tokens.into_iter())
}
trait TokenSource: Clone {
fn peek(&self) -> Option<LexerToken>;
fn lookahead(&self, n: usize) -> Option<LexerToken>;
fn bump(&mut self);
}
#[derive(Clone)]
struct TokenSliceSource<'a>(core::slice::Iter<'a, LexerToken>);
impl<'a> TokenSliceSource<'a> {
pub(crate) fn new(tokens: &'a [LexerToken]) -> Self {
Self(tokens.iter())
}
}
impl<'a> TokenSource for TokenSliceSource<'a> {
fn peek(&self) -> Option<LexerToken> {
self.0.as_slice().get(0).cloned()
}
fn lookahead(&self, n: usize) -> Option<LexerToken> {
self.0.as_slice().get(n).cloned()
}
fn bump(&mut self) {
assert!(self.0.next().is_some());
}
}
pub struct Parsed {
green_node: rowan::GreenNode,
}
impl Parsed {
fn new<Tok: Iterator<Item = LexerToken>>(events: Vec<ParseEvent>, mut tokens: Tok) -> Self {
let mut builder = rowan::GreenNodeBuilder::new();
let mut implicit_text_state: Option<usize> = None;
fn buffer_implicit_text(builder: &mut GreenNodeBuilder, state: &mut Option<usize>, token_count: usize) {
if state.is_none() {
builder.start_node(SyntaxKind::NodeText.into());
}
*state = Some(state.unwrap_or_default() + token_count);
}
fn flush_implicit_text<Tok: Iterator<Item = LexerToken>>(
builder: &mut GreenNodeBuilder,
state: &mut Option<usize>,
tokens: &mut Tok,
) {
match *state {
Some(token_count) if token_count > 0 => {
let mut text: String = String::new();
for _ in 0..token_count {
let token = tokens.next().unwrap();
text += token.text.as_str();
}
builder.token(SyntaxKind::TokenText.into(), text.as_str());
*state = Some(0);
}
_ => {}
}
}
fn close_implicit_text<Tok: Iterator<Item = LexerToken>>(
builder: &mut GreenNodeBuilder,
state: &mut Option<usize>,
tokens: &mut Tok,
) {
if state.is_some() {
flush_implicit_text(builder, state, tokens);
builder.finish_node();
*state = None;
}
}
for ev in events {
if !matches!(
ev,
ParseEvent::Token {
kind: SyntaxKind::TokenBackslash,
..
} | ParseEvent::Token {
kind: SyntaxKind::TokenText,
..
}
) {
close_implicit_text(&mut builder, &mut implicit_text_state, &mut tokens);
}
match ev {
ParseEvent::Start { kind } => {
if kind != SyntaxKind::Tombstone {
builder.start_node(kind.into())
}
}
ParseEvent::Token { kind, raw_token_count } => match kind {
SyntaxKind::TokenBackslash => {
buffer_implicit_text(&mut builder, &mut implicit_text_state, 0);
flush_implicit_text(&mut builder, &mut implicit_text_state, &mut tokens);
let mut text: String = String::new();
for _ in 0..raw_token_count {
let token = tokens.next().unwrap();
text += token.text.as_str();
}
builder.token(kind.into(), text.as_str());
}
SyntaxKind::TokenText => {
buffer_implicit_text(&mut builder, &mut implicit_text_state, raw_token_count);
}
_ => {
let mut text: String = String::new();
for _ in 0..raw_token_count {
let token = tokens.next().unwrap();
text += token.text.as_str();
}
builder.token(kind.into(), text.as_str());
}
},
ParseEvent::End => builder.finish_node(),
}
}
let green_node: rowan::GreenNode = builder.finish();
Parsed { green_node }
}
pub fn syntax(&self) -> SyntaxNode {
SyntaxNode::new_root(self.green_node.clone())
}
}
enum ParseEvent {
Start { kind: SyntaxKind },
End,
Token { kind: SyntaxKind, raw_token_count: usize },
}
pub(crate) struct Marker {
pos: usize,
bomb: DropBomb,
}
impl Marker {
fn new(pos: usize) -> Marker {
Marker {
pos,
bomb: DropBomb::new("Marker must be either completed or abandoned"),
}
}
fn complete<Ts: TokenSource>(mut self, p: &mut Parser<Ts>, kind: SyntaxKind) -> CompletedMarker {
self.bomb.defuse();
match p.events[self.pos] {
ParseEvent::Start { kind: ref mut k, .. } => {
*k = kind;
}
_ => unreachable!(),
}
let finish_pos = p.events.len();
p.push_event(ParseEvent::End);
CompletedMarker::new(self.pos, finish_pos, kind)
}
fn abandon<Ts: TokenSource>(mut self, p: &mut Parser<Ts>) {
self.bomb.defuse();
if self.pos + 1 == p.events.len() {
match p.events.pop() {
Some(ParseEvent::Start {
kind: SyntaxKind::Tombstone,
}) => (),
_ => unreachable!(),
}
}
}
}
struct CompletedMarker {
#[allow(unused)]
start_pos: usize,
#[allow(unused)]
finish_pos: usize,
#[allow(unused)]
kind: SyntaxKind,
}
impl CompletedMarker {
fn new(start_pos: usize, finish_pos: usize, kind: SyntaxKind) -> Self {
CompletedMarker {
start_pos,
finish_pos,
kind,
}
}
#[allow(unused)]
pub(crate) fn kind(&self) -> SyntaxKind {
self.kind
}
}
#[derive(Copy, Clone)]
struct ParserState {
end_kinds: TokenSet,
depth: u8,
}
impl ParserState {
pub fn new() -> Self {
Self {
end_kinds: TokenSet::EMPTY,
depth: 0,
}
}
pub fn insert(self, kind: Option<SyntaxKind>) -> Self {
Self {
end_kinds: self.end_kinds.insert(kind),
depth: self.depth + 1,
}
}
pub fn end_kinds(self) -> TokenSet {
self.end_kinds
}
pub fn depth(self) -> u8 {
self.depth
}
}
struct Parser<'g, Ts: TokenSource> {
pub grammar: &'g Grammar,
tokens: Ts,
events: Vec<ParseEvent>,
steps: Cell<u32>,
}
impl<'g, Ts: TokenSource> Parser<'g, Ts> {
pub fn new(grammar: &'g Grammar, tokens: Ts) -> Self {
Self {
grammar,
tokens,
events: Vec::new(),
steps: Cell::new(0),
}
}
pub fn end(self) -> Vec<ParseEvent> {
self.events
}
pub fn start(&mut self) -> Marker {
let pos = self.events.len();
self.push_event(ParseEvent::Start {
kind: SyntaxKind::Tombstone,
});
Marker::new(pos)
}
pub fn peek_kw(&self) -> Option<SyntaxKind> {
self.tokens.peek().and_then(|t| {
if t.kind != SyntaxKind::TokenText {
return None;
}
match t.text.as_str() {
"admin" => Some(SyntaxKind::TokenAdmin),
"mod" => Some(SyntaxKind::TokenMod),
_ => None,
}
})
}
fn inc_steps(&self) {
let steps = self.steps.get();
assert!(
steps <= PARSER_STEP_LIMIT,
"reached parser step limit: {}",
PARSER_STEP_LIMIT
);
self.steps.set(steps + 1);
}
pub fn peek_kind(&self) -> Option<SyntaxKind> {
self.inc_steps();
self.tokens.peek().map(|t| t.kind)
}
pub fn lookahead_kind(&self, offset: usize) -> Option<SyntaxKind> {
self.inc_steps();
self.tokens.lookahead(offset).map(|t| t.kind)
}
pub fn peek_text(&self) -> Option<SmolStr> {
self.inc_steps();
self.tokens.peek().and_then(|t| {
if t.kind == SyntaxKind::TokenText {
Some(t.text)
} else {
None
}
})
}
pub fn at(&self, kind: SyntaxKind) -> bool {
self.lookahead_at(0, kind)
}
pub fn at_ts(&self, ts: TokenSet) -> bool {
ts.iter().any(|k| match k {
Some(k) => self.at(k),
None => self.peek_kind().is_none(),
})
}
pub fn lookahead_at(&self, offset: usize, kind: SyntaxKind) -> bool {
match kind {
SyntaxKind::TokenStarStar => self.lookahead_at_composite2(offset, [SyntaxKind::TokenStar, SyntaxKind::TokenStar]),
SyntaxKind::TokenTildeTilde => {
self.lookahead_at_composite2(offset, [SyntaxKind::TokenTilde, SyntaxKind::TokenTilde])
}
SyntaxKind::TokenAdmin => match self.tokens.lookahead(offset) {
None => false,
Some(raw) => raw.kind == SyntaxKind::TokenText && raw.text == "admin",
},
SyntaxKind::TokenMod => match self.tokens.lookahead(offset) {
None => false,
Some(raw) => raw.kind == SyntaxKind::TokenText && raw.text == "mod",
},
_ => self.lookahead_kind(offset) == Some(kind),
}
}
fn lookahead_at_composite2(&self, offset: usize, kinds: [SyntaxKind; 2]) -> bool {
kinds
.iter()
.enumerate()
.all(|(i, k)| self.lookahead_kind(offset + i) == Some(*k))
}
pub fn bump(&mut self, kind: SyntaxKind) {
assert!(self.bump_opt(kind), "kind = {:?}", kind)
}
pub fn bump_remap(&mut self, old_kind: SyntaxKind, new_kind: SyntaxKind) {
assert!(self.at(old_kind));
self.inner_bump(new_kind, Parser::<Ts>::get_raw_token_count(old_kind));
}
pub fn bump_opt(&mut self, kind: SyntaxKind) -> bool {
if !self.at(kind) {
false
} else {
self.inner_bump(kind, Parser::<Ts>::get_raw_token_count(kind));
true
}
}
pub fn r#try<F>(&mut self, func: F) -> bool
where
F: FnOnce(&mut Self) -> bool,
{
let tokens = self.tokens.clone();
let event_count: usize = self.events.len();
let is_success = func(self);
if !is_success {
self.tokens = tokens;
self.events.truncate(event_count);
}
is_success
}
fn inner_bump(&mut self, kind: SyntaxKind, raw_token_count: usize) {
for _ in 0..raw_token_count {
self.tokens.bump();
}
self.push_event(ParseEvent::Token { kind, raw_token_count })
}
fn push_event(&mut self, ev: ParseEvent) {
self.inc_steps();
self.events.push(ev);
}
fn get_raw_token_count(kind: SyntaxKind) -> usize {
match kind {
SyntaxKind::TokenStarStar => 2,
SyntaxKind::TokenTildeTilde => 2,
_ => 1,
}
}
}
fn root<Ts: TokenSource>(p: &mut Parser<Ts>) {
let m = p.start();
let state = ParserState::new();
block_content_items(p, state);
m.complete(p, SyntaxKind::NodeContent);
}
fn flat_items<Ts: TokenSource>(p: &mut Parser<Ts>, state: ParserState) {
while p.peek_kind().is_some() {
if p.at_ts(state.end_kinds()) || is_at_closing_tag(p, state) {
break;
}
flat_item(p);
}
}
fn flat_item<Ts: TokenSource>(p: &mut Parser<Ts>) {
if let Some(kind) = p.peek_kind() {
match kind {
SyntaxKind::TokenBackslash => match p.lookahead_kind(1) {
None => p.bump_remap(kind, SyntaxKind::TokenText),
Some(escaped) => {
p.bump(kind);
p.bump_remap(escaped, SyntaxKind::TokenText)
}
},
SyntaxKind::TokenNewline => newline(p),
k => p.bump_remap(k, SyntaxKind::TokenText),
}
}
}
fn block_content_items<Ts: TokenSource>(p: &mut Parser<Ts>, state: ParserState) {
if p.grammar.depth.map_or(false, |max_depth| state.depth() >= max_depth) {
return flat_items(p, state);
}
while p.peek_kind().is_some() {
if p.at_ts(state.end_kinds()) || is_at_closing_tag(p, state) {
break;
}
block_content_item(p, state);
}
}
fn block_content_item<Ts: TokenSource>(p: &mut Parser<Ts>, state: ParserState) {
if let Some(kind) = p.peek_kind() {
match kind {
SyntaxKind::TokenBackslash => match p.lookahead_kind(1) {
None => p.bump_remap(kind, SyntaxKind::TokenText),
Some(escaped) => {
p.bump(kind);
p.bump_remap(escaped, SyntaxKind::TokenText)
}
},
SyntaxKind::TokenColon => {
if !p.r#try(|p| icon(p)) {
p.bump_remap(kind, SyntaxKind::TokenText);
}
}
SyntaxKind::TokenOpenBracket => {
if !p.r#try(|p| block_or_link(p, state).is_some()) {
p.bump_remap(kind, SyntaxKind::TokenText);
}
}
SyntaxKind::TokenStar if p.at(SyntaxKind::TokenStarStar) => {
if !p.r#try(|p| strong(p, state)) {
p.bump_remap(kind, SyntaxKind::TokenText);
}
}
SyntaxKind::TokenTilde if p.at(SyntaxKind::TokenTildeTilde) => {
if !p.r#try(|p| strikethrough(p, state)) {
p.bump_remap(kind, SyntaxKind::TokenText);
}
}
SyntaxKind::TokenUnderscore => {
if !p.r#try(|p| emphasis(p, state)) {
p.bump_remap(kind, SyntaxKind::TokenText);
}
}
SyntaxKind::TokenNewline => newline(p),
SyntaxKind::TokenCloseBracket
| SyntaxKind::TokenCloseParen
| SyntaxKind::TokenOpenParen
| SyntaxKind::TokenSlash
| SyntaxKind::TokenSpace
| SyntaxKind::TokenStar
| SyntaxKind::TokenText
| SyntaxKind::TokenTilde => p.bump_remap(kind, SyntaxKind::TokenText),
_ => p.bump(kind),
}
}
}
fn inline_content_items<Ts: TokenSource>(p: &mut Parser<Ts>, state: ParserState) {
let m = p.start();
if p.grammar.depth.map_or(false, |max_depth| state.depth() >= max_depth) {
flat_items(p, state);
} else {
while p.peek_kind().is_some() {
if p.at(SyntaxKind::TokenNewline) || p.at_ts(state.end_kinds()) || is_at_closing_tag(p, state) {
break;
}
inline_content_item(p, state);
}
}
m.complete(p, SyntaxKind::NodeInlineContent);
}
fn inline_content_item<Ts: TokenSource>(p: &mut Parser<Ts>, state: ParserState) {
if let Some(kind) = p.peek_kind() {
match kind {
SyntaxKind::TokenBackslash => match p.lookahead_kind(1) {
None => p.bump_remap(kind, SyntaxKind::TokenText),
Some(escaped) => {
p.bump(kind);
p.bump_remap(escaped, SyntaxKind::TokenText)
}
},
SyntaxKind::TokenColon => {
if !p.r#try(|p| icon(p)) {
p.bump_remap(kind, SyntaxKind::TokenText);
}
}
SyntaxKind::TokenStar if p.at(SyntaxKind::TokenStarStar) => {
if !p.r#try(|p| strong(p, state)) {
p.bump_remap(kind, SyntaxKind::TokenText);
}
}
SyntaxKind::TokenTilde if p.at(SyntaxKind::TokenTildeTilde) => {
if !p.r#try(|p| strikethrough(p, state)) {
p.bump_remap(kind, SyntaxKind::TokenText);
}
}
SyntaxKind::TokenUnderscore => {
if !p.r#try(|p| emphasis(p, state)) {
p.bump_remap(kind, SyntaxKind::TokenText);
}
}
SyntaxKind::TokenNewline => newline(p),
SyntaxKind::TokenCloseBracket
| SyntaxKind::TokenCloseParen
| SyntaxKind::TokenOpenParen
| SyntaxKind::TokenSlash
| SyntaxKind::TokenSpace
| SyntaxKind::TokenStar
| SyntaxKind::TokenText
| SyntaxKind::TokenTilde => p.bump_remap(kind, SyntaxKind::TokenText),
_ => p.bump(kind),
}
}
}
fn block_or_link<Ts: TokenSource>(p: &mut Parser<Ts>, state: ParserState) -> Option<SyntaxKind> {
if !p.grammar.links.is_empty() && p.r#try(|p| link(p, state)) {
return Some(SyntaxKind::NodeLink);
}
let m = p.start();
let next = opening_tag(p);
let next = match next {
None => {
m.abandon(p);
return None;
}
Some(k) => k,
};
let mut kind: Option<SyntaxKind> = None;
match next {
SyntaxKind::NodeAdminOpen => {
let state = state.insert(Some(SyntaxKind::NodeAdminClose));
block_content_items(p, state);
if p.at(SyntaxKind::TokenOpenBracket) && closing_tag(p, state) == Some(SyntaxKind::NodeAdminClose) {
kind = Some(SyntaxKind::NodeAdmin);
}
}
SyntaxKind::NodeModOpen => {
let state = state.insert(Some(SyntaxKind::NodeModClose));
block_content_items(p, state);
if p.at(SyntaxKind::TokenOpenBracket) && closing_tag(p, state) == Some(SyntaxKind::NodeModClose) {
kind = Some(SyntaxKind::NodeMod);
}
}
_ => unreachable!("Unexpected block end: {:?}", next),
}
match kind {
Some(k) => {
m.complete(p, k);
}
None => m.abandon(p),
};
kind
}
fn link<Ts: TokenSource>(p: &mut Parser<Ts>, state: ParserState) -> bool {
let mut success: bool = false;
let m = p.start();
p.bump(SyntaxKind::TokenOpenBracket);
inline_content_items(p, state.insert(Some(SyntaxKind::TokenCloseBracket)));
if p.bump_opt(SyntaxKind::TokenCloseBracket) && p.bump_opt(SyntaxKind::TokenOpenParen) {
p.bump_opt(SyntaxKind::TokenSpace);
let m = p.start();
if link_uri(p) {
m.complete(p, SyntaxKind::NodeLinkUri);
p.bump_opt(SyntaxKind::TokenSpace);
if p.bump_opt(SyntaxKind::TokenCloseParen) {
success = true;
}
} else {
m.abandon(p);
}
}
fn link_uri<Ts: TokenSource>(p: &mut Parser<Ts>) -> bool {
let protocol = match p.peek_text() {
Some(protocol) => protocol,
None => return false,
};
if !p.grammar.links.contains(protocol.as_str()) {
return false;
}
p.bump(SyntaxKind::TokenText);
if !p.at(SyntaxKind::TokenColon) {
return false;
} else {
p.bump_remap(SyntaxKind::TokenColon, SyntaxKind::TokenText);
}
while let Some(k) = p.peek_kind() {
let at_end = k == SyntaxKind::TokenCloseParen || k == SyntaxKind::TokenNewline;
let almost_at_end = k == SyntaxKind::TokenSpace && p.lookahead_at(1, SyntaxKind::TokenCloseParen);
if at_end || almost_at_end {
break;
}
p.bump_remap(k, SyntaxKind::TokenText);
}
true
}
if success {
m.complete(p, SyntaxKind::NodeLink);
} else {
m.abandon(p);
}
success
}
fn opening_tag<Ts: TokenSource>(p: &mut Parser<Ts>) -> Option<SyntaxKind> {
let m = p.start();
let kind = inner_opening_tag(p);
fn inner_opening_tag<Ts: TokenSource>(p: &mut Parser<Ts>) -> Option<SyntaxKind> {
p.bump(SyntaxKind::TokenOpenBracket);
p.bump_opt(SyntaxKind::TokenSpace);
let (token, node) = match p.peek_kw() {
Some(k @ SyntaxKind::TokenAdmin) if p.grammar.admin => (k, SyntaxKind::NodeAdminOpen),
Some(k @ SyntaxKind::TokenMod) if p.grammar.r#mod => (k, SyntaxKind::NodeModOpen),
_ => return None,
};
p.bump(token);
p.bump_opt(SyntaxKind::TokenSpace);
if !p.bump_opt(SyntaxKind::TokenCloseBracket) {
return None;
}
Some(node)
}
match kind {
Some(k) => {
m.complete(p, k);
}
None => m.abandon(p),
};
kind
}
fn closing_tag<Ts: TokenSource>(p: &mut Parser<Ts>, state: ParserState) -> Option<SyntaxKind> {
let m = p.start();
let kind = inner_closing_tag(p, state);
fn inner_closing_tag<Ts: TokenSource>(p: &mut Parser<Ts>, state: ParserState) -> Option<SyntaxKind> {
p.bump(SyntaxKind::TokenOpenBracket);
p.bump_opt(SyntaxKind::TokenSpace);
if !p.bump_opt(SyntaxKind::TokenSlash) {
return None;
}
p.bump_opt(SyntaxKind::TokenSpace);
let (token, node) = match p.peek_kw() {
Some(k @ SyntaxKind::TokenAdmin) if state.end_kinds().contains(Some(SyntaxKind::NodeAdminClose)) => {
(k, SyntaxKind::NodeAdminClose)
}
Some(k @ SyntaxKind::TokenMod) if state.end_kinds().contains(Some(SyntaxKind::NodeModClose)) => {
(k, SyntaxKind::NodeModClose)
}
_ => return None,
};
p.bump(token);
p.bump_opt(SyntaxKind::TokenSpace);
if !p.bump_opt(SyntaxKind::TokenCloseBracket) {
return None;
}
Some(node)
}
match kind {
Some(k) => {
m.complete(p, k);
}
None => m.abandon(p),
};
kind
}
const CLOSING_TAGS: TokenSet = token_set!(SyntaxKind::NodeAdminClose, SyntaxKind::NodeModClose);
fn is_at_closing_tag<Ts: TokenSource>(p: &mut Parser<Ts>, state: ParserState) -> bool {
if state.end_kinds().intersection(CLOSING_TAGS).is_empty() || !p.at(SyntaxKind::TokenOpenBracket) {
return false;
}
let mut at_end = false;
p.r#try(|p| {
at_end = inner_is_at_closing_tag(p, state);
false
});
fn inner_is_at_closing_tag<Ts: TokenSource>(p: &mut Parser<Ts>, state: ParserState) -> bool {
p.bump(SyntaxKind::TokenOpenBracket);
p.bump_opt(SyntaxKind::TokenSpace);
if !p.bump_opt(SyntaxKind::TokenSlash) {
return false;
}
p.bump_opt(SyntaxKind::TokenSpace);
let k: SyntaxKind = match p.peek_kw() {
Some(k @ SyntaxKind::TokenAdmin) if state.end_kinds().contains(Some(SyntaxKind::NodeAdminClose)) => k,
Some(k @ SyntaxKind::TokenMod) if state.end_kinds().contains(Some(SyntaxKind::NodeModClose)) => k,
_ => return false,
};
p.bump(k);
p.bump_opt(SyntaxKind::TokenSpace);
if !p.bump_opt(SyntaxKind::TokenCloseBracket) {
return false;
}
true
}
at_end
}
fn emphasis<Ts: TokenSource>(p: &mut Parser<Ts>, state: ParserState) -> bool {
let m = p.start();
p.bump(SyntaxKind::TokenUnderscore);
inline_content_items(p, state.insert(Some(SyntaxKind::TokenUnderscore)));
if p.bump_opt(SyntaxKind::TokenUnderscore) {
m.complete(p, SyntaxKind::NodeEmphasis);
true
} else {
m.abandon(p);
false
}
}
fn icon<Ts: TokenSource>(p: &mut Parser<Ts>) -> bool {
let mut success: bool = false;
let m = p.start();
p.bump(SyntaxKind::TokenColon);
if let Some(icon_text) = p.peek_text() {
if p.grammar.icons.contains(icon_text.as_str()) {
p.bump(SyntaxKind::TokenText);
if p.bump_opt(SyntaxKind::TokenColon) {
success = true;
}
}
}
if success {
m.complete(p, SyntaxKind::NodeIcon);
} else {
m.abandon(p);
}
success
}
fn strikethrough<Ts: TokenSource>(p: &mut Parser<Ts>, state: ParserState) -> bool {
let m = p.start();
p.bump(SyntaxKind::TokenTildeTilde);
inline_content_items(p, state.insert(Some(SyntaxKind::TokenTildeTilde)));
if p.bump_opt(SyntaxKind::TokenTildeTilde) {
m.complete(p, SyntaxKind::NodeStrikethrough);
true
} else {
m.abandon(p);
false
}
}
fn strong<Ts: TokenSource>(p: &mut Parser<Ts>, state: ParserState) -> bool {
let m = p.start();
p.bump(SyntaxKind::TokenStarStar);
inline_content_items(p, state.insert(Some(SyntaxKind::TokenStarStar)));
if p.bump_opt(SyntaxKind::TokenStarStar) {
m.complete(p, SyntaxKind::NodeStrong);
true
} else {
m.abandon(p);
false
}
}
fn newline<Ts: TokenSource>(p: &mut Parser<Ts>) {
let m = p.start();
p.bump(SyntaxKind::TokenNewline);
m.complete(p, SyntaxKind::NodeNewline);
}
#[cfg(test)]
mod parser_tests {
use std::convert::TryFrom;
use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use ::test_generator::test_resources;
use rowan::WalkEvent;
use crate::grammar::Grammar;
use crate::parser::parse;
use crate::syntax::SyntaxNode;
#[test_resources("./test-resources/[!.]*/")]
fn test_parse_mkt(path: &str) {
let path: PathBuf = Path::join(Path::new(".."), path);
let _name = path
.components()
.last()
.unwrap()
.as_os_str()
.to_str()
.expect("Failed to retrieve sample name");
let mkt_path = path.join("main.mkt");
let mkt_text: String = ::std::fs::read_to_string(mkt_path).expect("Failed to read input");
let grammar_json = fs::read_to_string(path.join("grammar.json")).expect("Failed to read grammar");
let grammar: Grammar = serde_json::from_str(&grammar_json).expect("Invalid grammar");
let parsed = parse(&grammar, &mkt_text);
let actual_cst = SyntaxNode::new_root(parsed.green_node);
let actual_cst_text = {
let mut actual_cst_text: Vec<u8> = Vec::new();
dump_node(&mut actual_cst_text, &actual_cst).unwrap();
String::from_utf8(actual_cst_text).unwrap()
};
fs::write(path.join("local-main.cst.txt"), &actual_cst_text).unwrap();
let cst_text: String = fs::read_to_string(path.join("main.cst.txt")).expect("Failed to read CST");
assert_eq!(&actual_cst_text, &cst_text);
let actual_ast = crate::ast::concrete::Root::try_from(actual_cst).unwrap().to_owned_ast();
let actual_ast_json = serde_json::to_string_pretty(&actual_ast).unwrap();
let actual_ast_json = format!("{}\n", actual_ast_json);
fs::write(path.join("local-main.ast.json"), &actual_ast_json).unwrap();
let expected_ast_json: String = fs::read_to_string(path.join("main.ast.json")).expect("Failed to read AST");
assert_eq!(actual_ast_json, expected_ast_json);
}
fn dump_node<W: io::Write>(writer: &mut W, node: &SyntaxNode) -> Result<(), io::Error> {
let mut indent = 0;
for event in node.preorder_with_tokens() {
match &event {
WalkEvent::Enter(symbol) => {
write!(
writer,
"{:i$}{:?}@{:?}",
"",
symbol.kind(),
symbol.text_range(),
i = indent
)?;
match symbol {
rowan::NodeOrToken::Node(_) => writeln!(writer, " {{")?,
rowan::NodeOrToken::Token(token) => writeln!(writer, " \"{}\"", token.text().escape_default())?,
};
indent += 2
}
WalkEvent::Leave(symbol) => {
indent -= 2;
if matches!(symbol, rowan::NodeOrToken::Node(_)) {
writeln!(writer, "{:i$}}}", "", i = indent)?
}
}
};
}
Ok(())
}
}