use logos::{Lexer as Lex, Logos, Span};
#[derive(Clone, Default)]
pub struct Extras;
#[derive(Logos, Clone, Debug, Eq, PartialEq)]
#[logos(extras = Extras)]
#[logos(subpattern identifier = r#"[^\s"!#%&'()*+,./;<=>@\[/\]^`{|}~]"#)]
pub enum Block {
#[regex(r"\{\{\{\{~?[\t ]*")]
StartRawBlock,
#[regex(r"\{\{!--")]
StartRawComment,
#[regex(r"\\\{\{\{?")]
StartRawStatement,
#[regex(r"\{\{!")]
StartComment,
#[regex(r"\{\{\{?~?[\t ]*")]
StartStatement,
#[regex(r"\{\{\~?[\t ]*#[\t ]*")]
StartBlockScope,
#[regex(r"\{\{\~?[\t ]*/")]
EndBlockScope,
#[regex(r"\{\{\{\{~?[\t ]*/")]
EndRawBlock,
#[regex(r".")]
Text,
#[token("\n")]
Newline,
#[error]
Error,
}
#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Logos)]
#[logos(extras = Extras)]
pub enum RawComment {
#[regex(r".")]
Text,
#[regex(r"--\}\}")]
End,
#[token("\n")]
Newline,
#[error]
Error,
}
#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Logos)]
#[logos(extras = Extras)]
pub enum RawStatement {
#[regex(r".")]
Text,
#[regex(r"\}?\}\}")]
End,
#[token("\n")]
Newline,
#[error]
Error,
}
#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Logos)]
#[logos(extras = Extras)]
pub enum Comment {
#[regex(r".")]
Text,
#[regex(r"\}\}")]
End,
#[token("\n")]
Newline,
#[error]
Error,
}
#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Logos)]
#[logos(extras = Extras)]
#[logos(subpattern identifier = r#"[^\s"!#%&'()*+,./;<=>@\[/\]^`{|}~]"#)]
pub enum Parameters {
#[token(r">")]
Partial,
#[token(r"else")]
ElseKeyword,
#[token(r"this")]
ExplicitThisKeyword,
#[token("./")]
ExplicitThisDotSlash,
#[token("../")]
ParentRef,
#[regex(r"(?&identifier)+", priority = 2)]
Identifier,
#[regex(r"@(?&identifier)+")]
LocalIdentifier,
#[regex(r"[./]")]
PathDelimiter,
#[token("[")]
StartArray,
#[token("(", priority = 3)]
StartSubExpression,
#[token(")")]
EndSubExpression,
#[regex(r"(?&identifier)+=")]
HashKey,
#[token("\"")]
DoubleQuoteString,
#[token("'")]
SingleQuoteString,
#[regex(r"-?([0-9]+\.)?[0-9]+((e|E)[+-]?[0-9]+)?", priority = 3)]
Number,
#[token("true")]
True,
#[token("false")]
False,
#[token("null")]
Null,
#[regex(r" +")]
WhiteSpace,
#[regex(r"~?\}?\}?\}\}")]
End,
#[token("\n")]
Newline,
#[error]
Error,
}
#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Logos)]
#[logos(extras = Extras)]
pub enum Statement {
#[regex(r"\}?\}\}")]
End,
#[token("\n")]
Newline,
#[error]
Error,
}
#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Logos)]
#[logos(extras = Extras)]
pub enum DoubleQuoteString {
#[regex(r#"[^\\"\n]+"#)]
Text,
#[token("\\n")]
EscapedNewline,
#[token(r#"\""#)]
Escaped,
#[token("\"")]
End,
#[token("\n")]
Newline,
#[error]
Error,
}
#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Logos)]
#[logos(extras = Extras)]
pub enum SingleQuoteString {
#[regex(r#"[^\\'\n]+"#)]
Text,
#[token("\\n")]
EscapedNewline,
#[token(r#"\'"#)]
Escaped,
#[token("'")]
End,
#[token("\n")]
Newline,
#[error]
Error,
}
#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Logos)]
#[logos(extras = Extras)]
pub enum Array {
#[regex(r#"[^\]]+"#)]
Text,
#[token(r#"\]"#)]
Escaped,
#[token("]")]
End,
#[token("\n")]
Newline,
#[error]
Error,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum Token {
Block(Block, Span),
RawComment(RawComment, Span),
RawStatement(RawStatement, Span),
Comment(Comment, Span),
Parameters(Parameters, Span),
DoubleQuoteString(DoubleQuoteString, Span),
SingleQuoteString(SingleQuoteString, Span),
Array(Array, Span),
}
impl Token {
pub fn span(&self) -> &Span {
match self {
Token::Block(_, ref span) => span,
Token::RawComment(_, ref span) => span,
Token::RawStatement(_, ref span) => span,
Token::Comment(_, ref span) => span,
Token::Parameters(_, ref span) => span,
Token::Array(_, ref span) => span,
Token::DoubleQuoteString(_, ref span) => span,
Token::SingleQuoteString(_, ref span) => span,
}
}
pub fn is_text(&self) -> bool {
match self {
Token::Block(ref t, _) => t == &Block::Text || t == &Block::Newline,
Token::RawComment(ref t, _) => {
t == &RawComment::Text || t == &RawComment::Newline
}
Token::RawStatement(ref t, _) => {
t == &RawStatement::Text || t == &RawStatement::Newline
}
Token::Comment(ref t, _) => {
t == &Comment::Text || t == &Comment::Newline
}
Token::Parameters(_, _) => false,
Token::Array(_, _) => false,
Token::DoubleQuoteString(_, _) => false,
Token::SingleQuoteString(_, _) => false,
}
}
pub fn is_newline(&self) -> bool {
match *self {
Token::RawComment(ref lex, _) => lex == &RawComment::Newline,
Token::RawStatement(ref lex, _) => lex == &RawStatement::Newline,
Token::Comment(ref lex, _) => lex == &Comment::Newline,
Token::Block(ref lex, _) => lex == &Block::Newline,
Token::Parameters(ref lex, _) => lex == &Parameters::Newline,
Token::Array(ref lex, _) => lex == &Array::Newline,
Token::DoubleQuoteString(_, _) => false,
Token::SingleQuoteString(_, _) => false,
}
}
}
enum Modes<'source> {
Block(Lex<'source, Block>),
RawComment(Lex<'source, RawComment>),
RawStatement(Lex<'source, RawStatement>),
Comment(Lex<'source, Comment>),
Parameters(Lex<'source, Parameters>),
DoubleQuoteString(Lex<'source, DoubleQuoteString>),
SingleQuoteString(Lex<'source, SingleQuoteString>),
Array(Lex<'source, Array>),
}
impl<'source> Modes<'source> {
fn new(s: &'source str) -> Self {
Self::Block(Block::lexer(s))
}
}
pub struct Lexer<'source> {
mode: Modes<'source>,
}
impl<'source> Lexer<'source> {
pub(crate) fn set_parameters_mode(&mut self) {
match &mut self.mode {
Modes::Block(lexer) => {
self.mode = Modes::Parameters(lexer.to_owned().morph())
}
_ => {}
}
}
pub(crate) fn until_mode(&mut self) -> Option<Token> {
while let Some(token) = self.next() {
match token {
Token::Block(_, _) => return Some(token),
_ => {}
}
}
None
}
}
impl<'source> Iterator for Lexer<'source> {
type Item = Token;
fn next(&mut self) -> Option<Self::Item> {
match &mut self.mode {
Modes::Block(lexer) => {
let result = lexer.next();
let span = lexer.span();
if let Some(token) = result {
if Block::StartRawBlock == token {
self.mode = Modes::Parameters(lexer.to_owned().morph());
} else if Block::EndRawBlock == token {
self.mode = Modes::Parameters(lexer.to_owned().morph());
} else if Block::StartRawComment == token {
self.mode = Modes::RawComment(lexer.to_owned().morph());
} else if Block::StartRawStatement == token {
self.mode =
Modes::RawStatement(lexer.to_owned().morph());
} else if Block::StartComment == token {
self.mode = Modes::Comment(lexer.to_owned().morph());
} else if Block::StartStatement == token {
self.mode = Modes::Parameters(lexer.to_owned().morph());
} else if Block::StartBlockScope == token {
self.mode = Modes::Parameters(lexer.to_owned().morph());
} else if Block::EndBlockScope == token {
self.mode = Modes::Parameters(lexer.to_owned().morph());
}
Some(Token::Block(token, span))
} else {
None
}
}
Modes::RawComment(lexer) => {
let result = lexer.next();
let span = lexer.span();
if let Some(token) = result {
if RawComment::End == token {
self.mode = Modes::Block(lexer.to_owned().morph());
}
Some(Token::RawComment(token, span))
} else {
None
}
}
Modes::RawStatement(lexer) => {
let result = lexer.next();
let span = lexer.span();
if let Some(token) = result {
if RawStatement::End == token {
self.mode = Modes::Block(lexer.to_owned().morph());
}
Some(Token::RawStatement(token, span))
} else {
None
}
}
Modes::Comment(lexer) => {
let result = lexer.next();
let span = lexer.span();
if let Some(token) = result {
if Comment::End == token {
self.mode = Modes::Block(lexer.to_owned().morph());
}
Some(Token::Comment(token, span))
} else {
None
}
}
Modes::Parameters(lexer) => {
let result = lexer.next();
let span = lexer.span();
if let Some(token) = result {
if Parameters::DoubleQuoteString == token {
self.mode =
Modes::DoubleQuoteString(lexer.to_owned().morph());
} else if Parameters::SingleQuoteString == token {
self.mode =
Modes::SingleQuoteString(lexer.to_owned().morph());
} else if Parameters::StartArray == token {
self.mode = Modes::Array(lexer.to_owned().morph());
} else if Parameters::End == token {
self.mode = Modes::Block(lexer.to_owned().morph());
}
Some(Token::Parameters(token, span))
} else {
None
}
}
Modes::DoubleQuoteString(lexer) => {
let result = lexer.next();
let span = lexer.span();
if let Some(token) = result {
if DoubleQuoteString::End == token {
self.mode = Modes::Parameters(lexer.to_owned().morph());
}
Some(Token::DoubleQuoteString(token, span))
} else {
None
}
}
Modes::SingleQuoteString(lexer) => {
let result = lexer.next();
let span = lexer.span();
if let Some(token) = result {
if SingleQuoteString::End == token {
self.mode = Modes::Parameters(lexer.to_owned().morph());
}
Some(Token::SingleQuoteString(token, span))
} else {
None
}
}
Modes::Array(lexer) => {
let result = lexer.next();
let span = lexer.span();
if let Some(token) = result {
if Array::End == token {
self.mode = Modes::Parameters(lexer.to_owned().morph());
}
Some(Token::Array(token, span))
} else {
None
}
}
}
}
}
fn normalize(tokens: Vec<Token>) -> Vec<Token> {
let mut normalized: Vec<Token> = Vec::new();
let mut span: Option<Span> = None;
for t in tokens.into_iter() {
if t.is_text() {
if let Some(ref mut span) = span {
span.end = t.span().end;
} else {
span = Some(t.span().clone());
}
} else {
if let Some(span) = span.take() {
normalized.push(Token::Block(Block::Text, span));
normalized.push(t);
} else {
normalized.push(t);
}
}
}
if let Some(span) = span.take() {
normalized.push(Token::Block(Block::Text, span));
}
normalized
}
pub fn lex(s: &str) -> Lexer {
Lexer {
mode: Modes::new(s),
}
}
pub fn collect(s: &str, normalized: bool) -> Vec<Token> {
let tokens = lex(s).collect();
if normalized {
normalize(tokens)
} else {
tokens
}
}