use logos::internal::LexerInternal;
use logos::Logos;
use crate::span::{Index, LineNumber, Range};
use std::fmt::{Display, Formatter};
#[derive(Clone, Debug, PartialEq, Copy, Eq)]
pub struct FollowedByBracket(pub bool);
#[derive(Clone, Logos, Debug, PartialEq, Copy, Eq)]
pub enum Token {
#[regex(r"\\\r?\n")]
MacroDefNewLine,
#[token("\n")]
Newline,
#[regex(r"//[^\n]*\n", single_line_comment)]
#[token("/*", ignore_multiline_comment)]
Comment(LineNumber),
CommentEnd,
EOF,
#[regex(r"[ \t\f\r]+", logos::skip)]
#[error]
Unexpected,
UnexpectedEOF,
#[regex(r"`[a-zA-Z_][a-zA-Z_0-9\$]*")]
MacroReference,
#[token("`include")]
Include,
#[token("`ifdef")]
MacroIf,
#[token("`ifndef")]
MacroIfn,
#[token("`elsif")]
MacroElsif,
#[token("`else")]
MacroElse,
#[token("`endif")]
MacroEndIf,
#[token("`define")]
MacroDef,
#[regex(r"[a-zA-Z_][[:word:]\$]*", handle_simple_ident)]
SimpleIdentifier(FollowedByBracket),
#[regex(r"\\[[:print:]&&\S]+\s")]
EscapedIdentifier,
#[regex(r"\$[a-zA-Z0-9_\$][a-zA-Z0-9_\$]*")]
SystemCall,
#[token("$temperature")]
Temperature,
#[token("$vt")]
Vt,
#[token("$simparam")]
SimParam,
#[token("$simparam$str")]
SimParamStr,
#[token("$port_connected")]
PortConnected,
#[token("$param_given")]
ParamGiven,
#[token("$display")]
Display,
#[token("$strobe")]
Strobe,
#[token("$write")]
Write,
#[token("$debug")]
Debug,
#[regex(r#""([^\n"\\]|\\[\\tn")])*""#)]
LiteralString,
#[regex(r"[0-9][0-9_]*")]
LiteralUnsignedNumber,
#[regex(r"[0-9][0-9_]*\.[0-9][0-9_]*[TGMKkmupfa]")]
LiteralRealNumberDotScaleChar,
#[regex(r"[0-9][0-9_]*\.[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*")]
LiteralRealNumberDotExp,
#[regex(r"[0-9][0-9_]*[TGMKkmupfa]")]
LiteralRealNumberScaleChar,
#[regex(r"[0-9][0-9_]*[eE][+-]?[0-9][0-9_]*")]
LiteralRealNumberExp,
#[regex(r"[0-9][0-9_]*\.[0-9][0-9_]*")]
LiteralRealNumberDot,
#[token(".")]
Accessor,
#[token(";")]
Semicolon,
#[token(":")]
Colon,
#[token(",")]
Comma,
#[token("(")]
ParenOpen,
#[token(")")]
ParenClose,
#[token("(*")]
AttributeStart,
#[token("*)")]
AttributeEnd,
#[token("[")]
SquareBracketOpen,
#[token("]")]
SquareBracketClose,
#[token("<+")]
Contribute,
#[token("=")]
Assign,
#[token("#")]
Hash,
#[token("*")]
OpMul,
#[token("/")]
OpDiv,
#[token("%")]
OpModulus,
#[token("+")]
Plus,
#[token("-")]
Minus,
#[token("**")]
OpExp,
#[token("!")]
OpLogicNot,
#[token("~")]
OpBitNot,
#[token("<<")]
OpArithmeticShiftLeft,
#[token(">>")]
OpArithmeticShiftRight,
#[token("<")]
OpLess,
#[token("<=")]
OpLessEqual,
#[token(">")]
OpGreater,
#[token(">=")]
OpGreaterEqual,
#[token("==")]
OpEqual,
#[token("!=")]
OpNotEqual,
#[token("&&")]
OpLogicAnd,
#[token("||")]
OpLogicalOr,
#[token("&")]
OpBitAnd,
#[token("^")]
OpBitXor,
#[token("~^")]
#[token("^~")]
OpBitNXor,
#[token("|")]
OpBitOr,
#[token("?")]
OpCondition,
#[token("if")]
If,
#[token("else")]
Else,
#[token("while")]
While,
#[token("begin")]
Begin,
#[token("end")]
End,
#[token("module")]
Module,
#[token("endmodule")]
EndModule,
#[token("discipline")]
Discipline,
#[token("enddiscipline")]
EndDiscipline,
#[token("nature")]
Nature,
#[token("endnature")]
EndNature,
#[token("branch")]
Branch,
#[token("parameter")]
Parameter,
#[token("localparam")]
DefineParameter,
#[token("defparam")]
LocalParameter,
#[token("analog")]
Analog,
#[token("function")]
Function,
#[token("endfunction")]
EndFunction,
#[token("initial")]
AnalogInitial,
#[token("input")]
Input,
#[token("inout")]
Inout,
#[token("output")]
Output,
#[token("signed")]
Signed,
#[token("vectored")]
Vectored,
#[token("scalared")]
Scalared,
#[token("string")]
String,
#[token("time")]
Time,
#[token("realtime")]
Realtime,
#[token("integer")]
Integer,
#[token("real")]
Real,
#[token("reg")]
Reg,
#[token("wreal")]
Wreal,
#[token("supply0")]
Supply0,
#[token("supply1")]
Supply1,
#[token("tri")]
Tri,
#[token("triand")]
TriAnd,
#[token("trior")]
TriOr,
#[token("tri0")]
Tri0,
#[token("tri1")]
Tri1,
#[token("wire")]
Wire,
#[token("uwire")]
Uwire,
#[token("wand")]
Wand,
#[token("wor")]
Wor,
#[token("ground")]
Ground,
#[token("potential")]
Potential,
#[token("flow")]
Flow,
#[token("domain")]
Domain,
#[token("discrete")]
Discrete,
#[token("continuous")]
Continuous,
#[token("ddT")]
TemperatureDerivative,
#[token("ddt")]
TimeDerivative,
#[token("ddx")]
PartialDerivative,
#[token("idt")]
TimeIntegral,
#[token("idtmod")]
TimeIntegralMod,
#[token("limexp")]
LimExp,
#[token("white_noise")]
WhiteNoise,
#[token("flicker_noise")]
FlickerNoise,
#[token("$pow")]
#[token("pow")]
Pow,
#[token("$sqrt")]
#[token("sqrt")]
Sqrt,
#[token("$hypot")]
#[token("hypot")]
Hypot,
#[token("$exp")]
#[token("exp")]
Exp,
#[token("$ln")]
#[token("ln")]
Ln,
#[token("$log10")]
#[token("log")]
Log,
#[token("$min")]
#[token("min")]
Min,
#[token("$max")]
#[token("max")]
Max,
#[token("$abs")]
#[token("abs")]
Abs,
#[token("$floor")]
#[token("floor")]
Floor,
#[token("$ceil")]
#[token("ceil")]
Ceil,
#[token("$sin")]
#[token("sin")]
Sin,
#[token("$cos")]
#[token("cos")]
Cos,
#[token("tan")]
#[token("$tan")]
Tan,
#[token("$asin")]
#[token("asin")]
ArcSin,
#[token("$acos")]
#[token("acos")]
ArcCos,
#[token("atan")]
#[token("$atan")]
ArcTan,
#[token("atan2")]
#[token("$atan2")]
ArcTan2,
#[token("sinh")]
#[token("$sinh")]
SinH,
#[token("cosh")]
#[token("$cosh")]
CosH,
#[token("tanh")]
#[token("$tanh")]
TanH,
#[token("asinh")]
#[token("$asinh")]
ArcSinH,
#[token("acosh")]
#[token("$acosh")]
ArcCosH,
#[token("atanh")]
#[token("$atanh")]
ArcTanH,
#[token("from")]
From,
#[token("exclude")]
Exclude,
#[token("inf")]
Infinity,
#[token("-inf")]
MinusInfinity,
#[token("abstol")]
Abstol,
#[token("access")]
Access,
#[token("ddt_nature")]
TimeDerivativeNature,
#[token("idt_nature")]
TimeIntegralNature,
#[token("units")]
Units,
}
#[inline(always)]
fn single_line_comment<'source>(_: &mut logos::Lexer<'source, Token>) -> LineNumber {
1
}
#[inline]
fn ignore_multiline_comment<'source>(lex: &mut logos::Lexer<'source, Token>) -> Option<LineNumber> {
let mut lines: LineNumber = 0;
loop {
match lex.read()? {
b'*' => {
lex.bump(1);
if lex.read() == Some(b'/') {
lex.bump(1);
break;
}
}
b'\n' => {
lines += 1;
lex.bump(1)
}
_ => lex.bump(1),
}
}
Some(lines)
}
#[inline]
fn handle_simple_ident<'source>(lex: &mut logos::Lexer<'source, Token>) -> FollowedByBracket {
FollowedByBracket(lex.read() == Some(b'('))
}
pub struct Lexer<'lt> {
internal: logos::Lexer<'lt, Token>,
}
impl<'lt> Lexer<'lt> {
pub fn new(source: &'lt str) -> Self {
Self {
internal: Token::lexer(source),
}
}
#[cfg(test)]
pub fn new_test(source: &'lt str) -> Self {
let mut res = Self {
internal: Token::lexer(source),
};
res
}
pub fn peek(&self) -> (Range, Option<Token>) {
let mut lexer = self.internal.clone();
let token = lexer.next();
let range = lexer.span();
let range = Range {
start: range.start as Index,
end: range.end as Index,
};
(range, token)
}
#[cfg(test)]
pub fn test_next(&mut self) -> Option<Token> {
loop {
match self.internal.next() {
Some(Token::Newline) | Some(Token::Comment(_)) => (),
res => return res,
}
}
}
pub fn range(&self) -> Range {
let internal_range = self.internal.span();
Range {
start: internal_range.start as Index,
end: internal_range.end as Index,
}
}
pub fn token_len(&self) -> Index {
self.range().end - self.range().start
}
pub fn slice(&self) -> &str {
self.internal.slice()
}
}
impl<'source> Iterator for Lexer<'source> {
type Item = Token;
fn next(&mut self) -> Option<Self::Item> {
self.internal.next()
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
pub fn macro_if() {
let mut lexer = Lexer::new("`ifdef");
assert_eq!(lexer.next(), Some(Token::MacroIf));
}
#[test]
pub fn macro_ifn() {
let mut lexer = Lexer::new("`ifndef");
assert_eq!(lexer.next(), Some(Token::MacroIfn));
}
#[test]
pub fn macro_else() {
let mut lexer = Lexer::new("`else");
assert_eq!(lexer.next(), Some(Token::MacroElse));
}
#[test]
pub fn macro_elsif() {
let mut lexer = Lexer::new("`elsif");
assert_eq!(lexer.next(), Some(Token::MacroElsif));
}
#[test]
pub fn macro_definition() {
let mut lexer = Lexer::new("`define x(y) \\\n test");
assert_eq!(lexer.test_next(), Some(Token::MacroDef));
assert_eq!(
lexer.test_next(),
Some(Token::SimpleIdentifier(FollowedByBracket(true)))
);
assert_eq!(lexer.test_next(), Some(Token::ParenOpen));
assert_eq!(
lexer.test_next(),
Some(Token::SimpleIdentifier(FollowedByBracket(false)))
);
assert_eq!(lexer.test_next(), Some(Token::ParenClose));
assert_eq!(lexer.test_next(), Some(Token::MacroDefNewLine));
assert_eq!(
lexer.test_next(),
Some(Token::SimpleIdentifier(FollowedByBracket(false)))
);
}
#[test]
pub fn include() {
assert_eq!(Lexer::new("`include").next(), Some(Token::Include));
}
#[test]
pub fn simple_ident() {
let mut lexer = Lexer::new_test("test _test egta test$\ntest2_$ iftest");
assert_eq!(
lexer.test_next(),
Some(Token::SimpleIdentifier(FollowedByBracket(false)))
);
assert_eq!(lexer.slice(), "test");
assert_eq!(
lexer.test_next(),
Some(Token::SimpleIdentifier(FollowedByBracket(false)))
);
assert_eq!(lexer.slice(), "_test");
assert_eq!(
lexer.test_next(),
Some(Token::SimpleIdentifier(FollowedByBracket(false)))
);
assert_eq!(lexer.slice(), "egta");
assert_eq!(
lexer.test_next(),
Some(Token::SimpleIdentifier(FollowedByBracket(false)))
);
assert_eq!(lexer.slice(), "test$");
assert_eq!(
lexer.test_next(),
Some(Token::SimpleIdentifier(FollowedByBracket(false)))
);
assert_eq!(lexer.slice(), "test2_$");
assert_eq!(
lexer.test_next(),
Some(Token::SimpleIdentifier(FollowedByBracket(false)))
);
assert_eq!(lexer.slice(), "iftest");
}
#[test]
pub fn escaped_ident() {
let mut lexer = Lexer::new("\\lel\\\\lel \\if ");
assert_eq!(lexer.test_next(), Some(Token::EscapedIdentifier));
assert_eq!(&lexer.slice()[1..9], "lel\\\\lel");
assert_eq!(lexer.test_next(), Some(Token::EscapedIdentifier));
assert_eq!(&lexer.slice()[1..3], "if");
}
#[test]
pub fn comment() {
let mut lexer = Lexer::new_test("//jdfjdfjw4$%\r%&/**#.,|\ntest");
assert_eq!(
lexer.test_next(),
Some(Token::SimpleIdentifier(FollowedByBracket(false)))
);
assert_eq!(lexer.slice(), "test")
}
#[test]
pub fn block_comment() {
let mut lexer = Lexer::new_test("/*A\nB\n*C*/`test");
assert_eq!(lexer.test_next(), Some(Token::MacroReference));
assert_eq!(lexer.slice(), "`test")
}
#[test]
pub fn string() {
let mut lexer = Lexer::new(r#""lel\"dsd%§.,-032391\t ""#);
assert_eq!(lexer.test_next(), Some(Token::LiteralString));
}
#[test]
pub fn unsigned_number() {
let mut lexer = Lexer::new("1_2345_5678_9");
assert_eq!(lexer.test_next(), Some(Token::LiteralUnsignedNumber));
}
#[test]
pub fn macro_ref() {
let test = "`egta";
let mut lexer = Lexer::new_test(test);
assert_eq!(lexer.test_next(), Some(Token::MacroReference))
}
#[test]
pub fn real_number() {
let mut lexer = Lexer::new_test(
"1.2
0.1
2394.26331
1.2E12 // the exponent symbol can be e or E
1.30e-2
0.1e-0
236.123_763_e-12 // underscores are ignored
1.3u
23E10
29E-2
7k",
);
assert_eq!(lexer.test_next(), Some(Token::LiteralRealNumberDot));
assert_eq!(lexer.test_next(), Some(Token::LiteralRealNumberDot));
assert_eq!(lexer.test_next(), Some(Token::LiteralRealNumberDot));
assert_eq!(lexer.test_next(), Some(Token::LiteralRealNumberDotExp));
assert_eq!(lexer.test_next(), Some(Token::LiteralRealNumberDotExp));
assert_eq!(lexer.test_next(), Some(Token::LiteralRealNumberDotExp));
assert_eq!(lexer.test_next(), Some(Token::LiteralRealNumberDotExp));
assert_eq!(
lexer.test_next(),
Some(Token::LiteralRealNumberDotScaleChar)
);
assert_eq!(lexer.test_next(), Some(Token::LiteralRealNumberExp));
assert_eq!(lexer.test_next(), Some(Token::LiteralRealNumberExp));
assert_eq!(lexer.test_next(), Some(Token::LiteralRealNumberScaleChar));
}
}
impl Display for Token {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
match self {
Self::MacroDefNewLine => f.write_str("\\ (newline)"),
Self::Newline => f.write_str("(newline)"),
Self::Comment(_) => f.write_str("comment"),
Self::CommentEnd => f.write_str("*/"),
Self::EOF => f.write_str("Source file end"),
Self::Unexpected => f.write_str("unexpected Sequence"),
Self::UnexpectedEOF => f.write_str("unexpected EOF"),
Self::MacroReference => f.write_str("macro reference"),
Self::Include => f.write_str("`include"),
Self::MacroIf => f.write_str("`if"),
Self::MacroIfn => f.write_str("`ifn"),
Self::MacroElsif => f.write_str("`elsif"),
Self::MacroElse => f.write_str("`else"),
Self::MacroEndIf => f.write_str("`end"),
Self::MacroDef => f.write_str("`define"),
Self::SimpleIdentifier(_) => f.write_str("simple identifier"),
Self::EscapedIdentifier => f.write_str("escaped identifier"),
Self::SystemCall => f.write_str("system call"),
Self::LiteralString => f.write_str("string literal"),
Self::LiteralUnsignedNumber => f.write_str("unsigned number"),
Self::LiteralRealNumberDotScaleChar => f.write_str("real number"),
Self::LiteralRealNumberDotExp => f.write_str("real number"),
Self::LiteralRealNumberScaleChar => f.write_str("real number"),
Self::LiteralRealNumberExp => f.write_str("real number"),
Self::LiteralRealNumberDot => f.write_str("real number"),
Self::Accessor => f.write_str("."),
Self::Semicolon => f.write_str(";"),
Self::Colon => f.write_str(":"),
Self::Comma => f.write_str(","),
Self::ParenOpen => f.write_str("("),
Self::ParenClose => f.write_str(")"),
Self::AttributeStart => f.write_str("(*"),
Self::AttributeEnd => f.write_str("*)"),
Self::SquareBracketOpen => f.write_str("["),
Self::SquareBracketClose => f.write_str("]"),
Self::Contribute => f.write_str("<+"),
Self::Assign => f.write_str("="),
Self::Hash => f.write_str("#"),
Self::OpMul => f.write_str("*"),
Self::OpDiv => f.write_str(""),
Self::OpModulus => f.write_str(""),
Self::Plus => f.write_str("%"),
Self::Minus => f.write_str("-"),
Self::OpExp => f.write_str("exp"),
Self::OpLogicNot => f.write_str("!"),
Self::OpBitNot => f.write_str("~"),
Self::OpArithmeticShiftLeft => f.write_str("<<"),
Self::OpArithmeticShiftRight => f.write_str(">>"),
Self::OpLess => f.write_str("<"),
Self::OpLessEqual => f.write_str("<="),
Self::OpGreater => f.write_str(">"),
Self::OpGreaterEqual => f.write_str(">="),
Self::OpEqual => f.write_str("=="),
Self::OpNotEqual => f.write_str("!="),
Self::OpLogicAnd => f.write_str("&&"),
Self::OpLogicalOr => f.write_str("||"),
Self::OpBitAnd => f.write_str("&"),
Self::OpBitXor => f.write_str("^"),
Self::OpBitNXor => f.write_str("^~/~^"),
Self::OpBitOr => f.write_str("|"),
Self::OpCondition => f.write_str("?"),
Self::If => f.write_str("if"),
Self::Else => f.write_str("else"),
Self::While => f.write_str("while"),
Self::Begin => f.write_str("begin"),
Self::End => f.write_str("end"),
Self::Module => f.write_str("module"),
Self::EndModule => f.write_str("endmodule"),
Self::Discipline => f.write_str("discipline"),
Self::EndDiscipline => f.write_str("enddiscipline"),
Self::Nature => f.write_str("nature"),
Self::EndNature => f.write_str("endnature"),
Self::Branch => f.write_str("branch"),
Self::Parameter => f.write_str("parameter"),
Self::DefineParameter => f.write_str("defparam"),
Self::LocalParameter => f.write_str("localparam"),
Self::Analog => f.write_str("analog"),
Self::AnalogInitial => f.write_str("inital"),
Self::Input => f.write_str("input"),
Self::Inout => f.write_str("inout"),
Self::Output => f.write_str("output"),
Self::Signed => f.write_str("signed"),
Self::Vectored => f.write_str("vectored"),
Self::Scalared => f.write_str("scalared"),
Self::String => f.write_str("string"),
Self::Time => f.write_str("time"),
Self::Realtime => f.write_str("realtime"),
Self::Integer => f.write_str("integer"),
Self::Real => f.write_str("real"),
Self::Reg => f.write_str("reg"),
Self::Wreal => f.write_str("wreal"),
Self::Supply0 => f.write_str("supply0"),
Self::Supply1 => f.write_str("supply1"),
Self::Tri => f.write_str("tri"),
Self::TriAnd => f.write_str("triand"),
Self::TriOr => f.write_str("trior"),
Self::Tri0 => f.write_str("tri0"),
Self::Tri1 => f.write_str("tri1"),
Self::Wire => f.write_str("wire"),
Self::Uwire => f.write_str("uwire"),
Self::Wand => f.write_str("wand"),
Self::Wor => f.write_str("wor"),
Self::Ground => f.write_str("ground"),
Self::Potential => f.write_str("potential"),
Self::Flow => f.write_str("flow"),
Self::Domain => f.write_str("domain"),
Self::Discrete => f.write_str("discrete"),
Self::Continuous => f.write_str("continuous"),
Self::TimeDerivative => f.write_str("ddt"),
Self::TemperatureDerivative => f.write_str("ddT"),
Self::PartialDerivative => f.write_str("ddx"),
Self::TimeIntegral => f.write_str("idt"),
Self::TimeIntegralMod => f.write_str("idtmod"),
Self::LimExp => f.write_str("limexp"),
Self::WhiteNoise => f.write_str("whitenoise"),
Self::FlickerNoise => f.write_str("flickrnoise"),
Self::Pow => f.write_str("pow"),
Self::Sqrt => f.write_str("sqrt"),
Self::Hypot => f.write_str("hypot"),
Self::Exp => f.write_str("exp"),
Self::Ln => f.write_str("ln"),
Self::Log => f.write_str("log"),
Self::Min => f.write_str("min"),
Self::Max => f.write_str("max"),
Self::Abs => f.write_str("abs"),
Self::Floor => f.write_str("floor"),
Self::Ceil => f.write_str("ceil"),
Self::Sin => f.write_str("sin"),
Self::Cos => f.write_str("cos"),
Self::Tan => f.write_str("tan"),
Self::ArcSin => f.write_str("asin"),
Self::ArcCos => f.write_str("acos"),
Self::ArcTan => f.write_str("atan"),
Self::ArcTan2 => f.write_str("atan2"),
Self::SinH => f.write_str("sinh"),
Self::CosH => f.write_str("cosh"),
Self::TanH => f.write_str("tanh"),
Self::ArcSinH => f.write_str("asinh"),
Self::ArcCosH => f.write_str("acosh"),
Self::ArcTanH => f.write_str("atanh"),
Self::From => f.write_str("from"),
Self::Exclude => f.write_str("exclude"),
Self::Infinity => f.write_str("inf"),
Self::MinusInfinity => f.write_str("-inf"),
Self::Abstol => f.write_str("abstol"),
Self::Access => f.write_str("access"),
Self::TimeDerivativeNature => f.write_str("ddt_nature"),
Self::TimeIntegralNature => f.write_str("idt_nature"),
Self::Units => f.write_str("units"),
Self::Temperature => f.write_str("$temperature"),
Self::Vt => f.write_str("$vt"),
Self::SimParam => f.write_str("$simparam"),
Self::SimParamStr => f.write_str("$simparam$str"),
Self::PortConnected => f.write_str("$port_connected"),
Self::ParamGiven => f.write_str("$param_given"),
Self::Display => f.write_str("$display"),
Self::Strobe => f.write_str("$strobe"),
Self::Write => f.write_str("$write"),
Self::Debug => f.write_str("$debug"),
Self::Function => f.write_str("function"),
Self::EndFunction => f.write_str("endfunction"),
}
}
}