use crate::types::ParseError;
use muskitty_css_tokenizer::{CssTokenizer, Token, Tokenizer};
use std::cell::{Cell, RefCell};
pub const MAX_NESTING_DEPTH: u32 = 1024;
#[derive(Debug, Clone)]
pub struct TokenStream {
pub tokens: Vec<Token>,
pub index: usize,
marked_indexes: Vec<usize>,
token_spans: Vec<std::ops::Range<usize>>,
source: Option<String>,
depth: Cell<u32>,
nesting_error: RefCell<Option<ParseError>>,
}
impl TokenStream {
pub fn new(mut tokens: Vec<Token>) -> Self {
if tokens.last().is_none_or(|t| !matches!(t, Token::Eof)) {
tokens.push(Token::Eof);
}
Self {
tokens,
index: 0,
marked_indexes: Vec::new(),
token_spans: Vec::new(),
source: None,
depth: Cell::new(0),
nesting_error: RefCell::new(None),
}
}
pub fn with_source(source: &str) -> Self {
let mut tz = CssTokenizer::new(source);
let char_to_byte: Vec<usize> = source.char_indices().map(|(i, _)| i).collect();
let source_len = source.len();
let mut tokens = Vec::new();
let mut spans = Vec::new();
while let Some((token, char_range)) = tz.next_token_with_span() {
let start_byte = char_to_byte
.get(char_range.start)
.copied()
.unwrap_or(source_len);
let end_byte = char_to_byte
.get(char_range.end)
.copied()
.unwrap_or(source_len);
let is_eof = matches!(token, Token::Eof);
tokens.push(token);
spans.push(start_byte..end_byte);
if is_eof {
break;
}
}
if tokens.last().is_none_or(|t| !matches!(t, Token::Eof)) {
tokens.push(Token::Eof);
spans.push(source_len..source_len);
}
Self {
tokens,
index: 0,
marked_indexes: Vec::new(),
token_spans: spans,
source: Some(source.to_string()),
depth: Cell::new(0),
nesting_error: RefCell::new(None),
}
}
pub fn record_nesting_error(&self, depth: u32) {
if self.nesting_error.borrow().is_none() {
self.nesting_error.replace(Some(ParseError::NestingTooDeep {
depth,
limit: MAX_NESTING_DEPTH,
}));
}
}
pub fn nesting_error(&self) -> Option<ParseError> {
self.nesting_error.borrow().clone()
}
pub fn enter_nesting(&self) -> Option<u32> {
let depth = self.depth.get() + 1;
self.depth.set(depth);
if depth > MAX_NESTING_DEPTH {
self.record_nesting_error(depth);
Some(depth)
} else {
None
}
}
pub fn leave_nesting(&self) {
self.depth.set(self.depth.get().saturating_sub(1));
}
pub fn source_slice(&self, start_index: usize, end_index: usize) -> Option<&str> {
let source = self.source.as_ref()?;
if end_index == 0 || start_index >= end_index {
return Some("");
}
let start = self.token_spans.get(start_index)?.start;
let end = self.token_spans.get(end_index - 1)?.end;
source.get(start..end)
}
pub fn next_token(&self) -> Token {
self.tokens.get(self.index).cloned().unwrap_or(Token::Eof)
}
pub fn is_empty(&self) -> bool {
matches!(self.next_token(), Token::Eof)
}
pub fn consume_token(&mut self) -> Token {
let token = self.next_token();
if !matches!(token, Token::Eof) {
self.index += 1;
}
token
}
pub fn discard_token(&mut self) {
if !self.is_empty() {
self.index += 1;
}
}
pub fn mark(&mut self) {
self.marked_indexes.push(self.index);
}
pub fn restore_mark(&mut self) {
if let Some(idx) = self.marked_indexes.pop() {
self.index = idx;
}
}
pub fn discard_mark(&mut self) {
let _ = self.marked_indexes.pop();
}
pub fn discard_whitespace(&mut self) {
while matches!(self.next_token(), Token::Whitespace) {
self.discard_token();
}
}
}