use crate::latin_1::{Latin1Str, Latin1String};
use crate::tokens::tokenizer::LexErr;
use crate::tokens::{Token, TokenKind, Tokenizer};
use std::collections::VecDeque;
fn is_base_specifier(text: &Latin1Str) -> bool {
matches!(
text.to_lowercase().as_bytes(),
b"b" | b"o" | b"x" | b"d" | b"sb" | b"ub" | b"so" | b"uo" | b"sx" | b"ux"
)
}
fn merge_bit_string_literals(
mut tokens: VecDeque<(Token, Option<LexErr>)>,
) -> VecDeque<(Token, Option<LexErr>)> {
let mut result = VecDeque::with_capacity(tokens.len());
while let Some((tok, diag)) = tokens.pop_front() {
match tok.kind() {
TokenKind::Identifier if is_base_specifier(tok.text()) => {
if let Some((next, _)) = tokens.front() {
if next.kind() == TokenKind::StringLiteral && next.leading_trivia().is_empty() {
let (string_tok, string_diag) = tokens.pop_front().unwrap();
let mut merged_text = Latin1String::from(tok.text());
merged_text.extend(string_tok.text());
let merged = Token::new(
TokenKind::BitStringLiteral,
merged_text,
tok.leading_trivia.clone(),
);
result.push_back((merged, diag.or(string_diag)));
continue;
}
}
result.push_back((tok, diag));
}
TokenKind::AbstractLiteral => {
if let (Some((ident, _)), Some((string, _))) = (tokens.front(), tokens.get(1)) {
if ident.kind() == TokenKind::Identifier
&& ident.leading_trivia().is_empty()
&& is_base_specifier(ident.text())
&& string.kind() == TokenKind::StringLiteral
&& string.leading_trivia().is_empty()
{
let (ident_tok, ident_diag) = tokens.pop_front().unwrap();
let (string_tok, string_diag) = tokens.pop_front().unwrap();
let mut merged_text = Latin1String::from(tok.text());
merged_text.extend(ident_tok.text());
merged_text.extend(string_tok.text());
let merged = Token::new(
TokenKind::BitStringLiteral,
merged_text,
tok.leading_trivia.clone(),
);
result.push_back((merged, diag.or(ident_diag).or(string_diag)));
continue;
}
}
result.push_back((tok, diag));
}
_ => result.push_back((tok, diag)),
}
}
result
}
pub struct TokenStream {
inner: VecDeque<(Token, Option<LexErr>)>,
}
impl TokenStream {
fn new(tokens: VecDeque<(Token, Option<LexErr>)>) -> TokenStream {
TokenStream {
inner: merge_bit_string_literals(tokens),
}
}
pub fn from_tokens(tokens: impl IntoIterator<Item = Token>) -> TokenStream {
TokenStream::new(VecDeque::from_iter(
tokens.into_iter().map(|tok| (tok, None)),
))
}
pub fn peek(&self, n: usize) -> Option<&Token> {
self.inner.get(n).map(|(tok, _)| tok)
}
pub fn peek_next(&self) -> Option<&Token> {
self.peek(0)
}
pub fn has_next(&self) -> bool {
self.peek_next().is_some()
}
pub fn next_if(
&mut self,
cond: impl FnOnce(&Token) -> bool,
) -> Option<(Token, Option<LexErr>)> {
if cond(self.peek_next()?) {
self.next()
} else {
None
}
}
}
impl Iterator for TokenStream {
type Item = (Token, Option<LexErr>);
fn next(&mut self) -> Option<Self::Item> {
self.inner.pop_front()
}
}
impl FromIterator<(Token, Option<LexErr>)> for TokenStream {
fn from_iter<T: IntoIterator<Item = (Token, Option<LexErr>)>>(iter: T) -> Self {
TokenStream::new(iter.into_iter().collect())
}
}
impl From<Vec<u8>> for TokenStream {
fn from(value: Vec<u8>) -> Self {
Tokenizer::new(value.into_iter()).collect()
}
}
impl From<&[u8]> for TokenStream {
fn from(value: &[u8]) -> Self {
Tokenizer::new(value.iter().copied()).collect()
}
}
impl<const N: usize> From<&[u8; N]> for TokenStream {
fn from(value: &[u8; N]) -> Self {
Tokenizer::new(value.iter().copied()).collect()
}
}
impl<const N: usize> From<[u8; N]> for TokenStream {
fn from(value: [u8; N]) -> Self {
Tokenizer::new(value.into_iter()).collect()
}
}
impl From<&str> for TokenStream {
fn from(value: &str) -> Self {
Tokenizer::new(value.bytes()).collect()
}
}
impl From<String> for TokenStream {
fn from(value: String) -> Self {
TokenStream::from(value.as_str())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn stream_kinds(input: &str) -> Vec<TokenKind> {
let stream = TokenStream::from(input);
stream.inner.iter().map(|(tok, _)| tok.kind()).collect()
}
fn stream_first_text(input: &str) -> String {
let stream = TokenStream::from(input);
stream.inner[0].0.text().to_string()
}
#[test]
fn merge_bare_base_specifier_string() {
assert_eq!(
stream_kinds("b\"0101\""),
vec![TokenKind::BitStringLiteral, TokenKind::Eof]
);
assert_eq!(stream_first_text("b\"0101\""), "b\"0101\"");
}
#[test]
fn merge_signed_base_specifier_string() {
assert_eq!(
stream_kinds("sx\"FF\""),
vec![TokenKind::BitStringLiteral, TokenKind::Eof]
);
assert_eq!(stream_first_text("sx\"FF\""), "sx\"FF\"");
}
#[test]
fn merge_length_prefixed_bit_string() {
assert_eq!(
stream_kinds("10ub\"0101\""),
vec![TokenKind::BitStringLiteral, TokenKind::Eof]
);
assert_eq!(stream_first_text("10ub\"0101\""), "10ub\"0101\"");
}
#[test]
fn no_merge_with_trivia_between() {
assert_eq!(
stream_kinds("b \"0101\""),
vec![
TokenKind::Identifier,
TokenKind::StringLiteral,
TokenKind::Eof
]
);
assert_eq!(
stream_kinds("10 b\"0101\""),
vec![
TokenKind::AbstractLiteral,
TokenKind::BitStringLiteral,
TokenKind::Eof
]
);
}
#[test]
fn no_merge_non_base_specifier() {
assert_eq!(
stream_kinds("foo\"bar\""),
vec![
TokenKind::Identifier,
TokenKind::StringLiteral,
TokenKind::Eof
]
);
}
#[test]
fn no_merge_number_followed_by_identifier() {
assert_eq!(
stream_kinds("10s"),
vec![
TokenKind::AbstractLiteral,
TokenKind::Identifier,
TokenKind::Eof
]
);
}
#[test]
fn merge_case_insensitive_base_specifier() {
assert_eq!(
stream_kinds("X\"FF\""),
vec![TokenKind::BitStringLiteral, TokenKind::Eof]
);
assert_eq!(
stream_kinds("UB\"01\""),
vec![TokenKind::BitStringLiteral, TokenKind::Eof]
);
}
#[test]
fn merge_all_base_specifiers() {
for base in ["b", "o", "x", "d", "sb", "so", "sx", "ub", "uo", "ux"] {
let input = format!("{base}\"01\"");
assert_eq!(
stream_kinds(&input),
vec![TokenKind::BitStringLiteral, TokenKind::Eof],
"failed for base specifier: {base}"
);
let upper = input.to_uppercase();
assert_eq!(
stream_kinds(&upper),
vec![TokenKind::BitStringLiteral, TokenKind::Eof],
"failed for uppercase base specifier: {base}"
);
}
}
#[test]
fn merge_all_base_specifiers_with_length() {
for base in ["b", "o", "x", "d", "sb", "so", "sx", "ub", "uo", "ux"] {
let input = format!("8{base}\"01\"");
assert_eq!(
stream_kinds(&input),
vec![TokenKind::BitStringLiteral, TokenKind::Eof],
"failed for length-prefixed base specifier: {base}"
);
}
}
}