use crate::trait_def::Tokenizer;
use crate::types::{HashType, Numeric, State, Token};
pub struct CssTokenizer {
input: Vec<char>,
pos: usize,
state: State,
eof_emitted: bool,
unicode_ranges_allowed: bool,
}
impl CssTokenizer {
pub fn new(input: &str) -> Self {
let input = preprocess_input(input);
Self {
input,
pos: 0,
state: State::Data,
eof_emitted: false,
unicode_ranges_allowed: false,
}
}
fn peek(&self, offset: usize) -> Option<char> {
self.input.get(self.pos + offset).copied()
}
fn consume(&mut self) -> Option<char> {
let c = self.input.get(self.pos).copied();
if c.is_some() {
self.pos += 1;
}
c
}
fn reconsume(&mut self) {
if self.pos > 0 {
self.pos -= 1;
}
}
fn consume_a_token(&mut self) -> Token {
loop {
let Some(c) = self.consume() else {
self.eof_emitted = true;
self.state = State::Eof;
return Token::Eof;
};
if c == '/' && self.peek(0) == Some('*') {
self.consume(); self.consume_comments_body();
continue;
}
return match c {
' ' | '\t' | '\n' | '\r' | '\u{000C}' => {
while let Some(next) = self.peek(0) {
if matches!(next, ' ' | '\t' | '\n' | '\r' | '\u{000C}') {
self.consume();
} else {
break;
}
}
Token::Whitespace
}
'"' | '\'' => self.consume_a_string_token(c),
'#' => self.consume_a_hash_token(),
'(' => Token::OpenParen,
')' => Token::CloseParen,
'-' => {
if self.starts_with_number(c) {
self.reconsume();
self.consume_a_numeric_token()
} else if self.peek(0) == Some('-') && self.peek(1) == Some('>') {
self.consume(); self.consume(); Token::Cdc
} else if self.would_start_ident_sequence_at(0) {
self.reconsume();
self.consume_an_ident_like_token()
} else {
Token::Delim(c)
}
}
'+' | '.' => {
if self.starts_with_number(c) {
self.reconsume();
self.consume_a_numeric_token()
} else {
Token::Delim(c)
}
}
'<' => {
if self.peek(0) == Some('!')
&& self.peek(1) == Some('-')
&& self.peek(2) == Some('-')
{
self.consume();
self.consume();
self.consume();
Token::Cdo
} else {
Token::Delim(c)
}
}
'@' => self.consume_an_at_keyword_token(),
'[' => Token::OpenBracket,
'\\' => self.consume_ident_or_delim(),
']' => Token::CloseBracket,
'{' => Token::OpenBrace,
'}' => Token::CloseBrace,
'0'..='9' => {
self.reconsume();
self.consume_a_numeric_token()
}
'u' | 'U' => self.consume_u_or_unicode_range(),
_ if is_ident_start_code_point(c) => {
self.reconsume();
self.consume_an_ident_like_token()
}
':' => Token::Colon,
';' => Token::Semicolon,
',' => Token::Comma,
_ => Token::Delim(c),
};
}
}
fn consume_comments_body(&mut self) {
loop {
match self.consume() {
None => return, Some('*') => {
if self.peek(0) == Some('/') {
self.consume(); return;
}
}
Some(_) => {}
}
}
}
fn consume_a_string_token(&mut self, quote: char) -> Token {
let mut value = String::new();
loop {
match self.consume() {
None => {
return Token::String(value);
}
Some(c) if c == quote => {
return Token::String(value);
}
Some('\n') => {
self.reconsume();
return Token::BadString;
}
Some('\\') => {
match self.peek(0) {
None => {
}
Some('\n') => {
self.consume();
}
Some(_) => {
let escaped = self.consume_an_escaped_code_point();
value.push(escaped);
}
}
}
Some(c) => {
value.push(c);
}
}
}
}
fn consume_a_hash_token(&mut self) -> Token {
let next_is_ident = self.peek(0).is_some_and(is_ident_code_point);
let next_is_valid_escape = self.is_valid_escape_at(0);
if !next_is_ident && !next_is_valid_escape {
return Token::Delim('#');
}
let hash_type = if self.would_start_ident_sequence_at(0) {
HashType::Id
} else {
HashType::Unrestricted
};
let name = self.consume_an_ident_sequence();
Token::Hash(name, hash_type)
}
fn consume_a_numeric_token(&mut self) -> Token {
let (value, is_integer) = self.consume_a_number();
let numeric = Numeric { value, is_integer };
if self.would_start_ident_sequence_at(0) {
let unit = self.consume_an_ident_sequence();
return Token::Dimension(numeric, unit);
}
if self.peek(0) == Some('%') {
self.consume();
return Token::Percentage(numeric);
}
Token::Number(numeric)
}
fn consume_a_number(&mut self) -> (f64, bool) {
let mut type_is_number = false;
let mut number_part = String::new();
let mut exponent_part = String::new();
if self.peek(0) == Some('+') || self.peek(0) == Some('-') {
number_part.push(self.consume().unwrap());
}
while let Some(d) = self.peek(0) {
if is_digit(d) {
number_part.push(d);
self.consume();
} else {
break;
}
}
if self.peek(0) == Some('.') && matches!(self.peek(1), Some(d) if is_digit(d)) {
number_part.push(self.consume().unwrap()); type_is_number = true;
while let Some(d) = self.peek(0) {
if is_digit(d) {
number_part.push(d);
self.consume();
} else {
break;
}
}
}
if matches!(self.peek(0), Some('e') | Some('E')) {
let p1 = self.peek(1);
let p2 = self.peek(2);
let valid_exp = match p1 {
Some('+') | Some('-') => matches!(p2, Some(d) if is_digit(d)),
Some(d) if is_digit(d) => true,
_ => false,
};
if valid_exp {
self.consume(); if self.peek(0) == Some('+') || self.peek(0) == Some('-') {
exponent_part.push(self.consume().unwrap());
}
while let Some(d) = self.peek(0) {
if is_digit(d) {
exponent_part.push(d);
self.consume();
} else {
break;
}
}
type_is_number = true;
}
}
let mut value: f64 = number_part.parse().unwrap_or(0.0);
if !exponent_part.is_empty() {
let exp: i32 = exponent_part.parse().unwrap_or(0);
value *= 10.0f64.powi(exp);
}
(value, !type_is_number)
}
fn consume_an_ident_like_token(&mut self) -> Token {
let name = self.consume_an_ident_sequence();
if name.eq_ignore_ascii_case("url") && self.peek(0) == Some('(') {
self.consume(); while self.peek(0).is_some_and(is_whitespace) && self.peek(1).is_some_and(is_whitespace)
{
self.consume();
}
let p0 = self.peek(0);
let p1 = self.peek(1);
let is_quote_case = matches!(p0, Some('"') | Some('\''))
|| (p0.is_some_and(is_whitespace) && matches!(p1, Some('"') | Some('\'')));
if is_quote_case {
return Token::Function(name);
}
return self.consume_a_url_token();
}
if self.peek(0) == Some('(') {
self.consume();
return Token::Function(name);
}
Token::Ident(name)
}
fn consume_an_at_keyword_token(&mut self) -> Token {
if self.would_start_ident_sequence_at(0) {
let name = self.consume_an_ident_sequence();
Token::AtKeyword(name)
} else {
Token::Delim('@')
}
}
fn consume_ident_or_delim(&mut self) -> Token {
if self.is_valid_escape_next() {
self.reconsume();
self.consume_an_ident_like_token()
} else {
Token::Delim('\\')
}
}
fn consume_a_url_token(&mut self) -> Token {
let mut value = String::new();
while self.peek(0).is_some_and(is_whitespace) {
self.consume();
}
loop {
match self.consume() {
Some(')') => return Token::Url(value),
None => return Token::Url(value),
Some(c) if is_whitespace(c) => {
while self.peek(0).is_some_and(is_whitespace) {
self.consume();
}
match self.peek(0) {
Some(')') => {
self.consume();
return Token::Url(value);
}
None => return Token::Url(value), _ => {
self.consume_the_remnants_of_a_bad_url();
return Token::BadUrl;
}
}
}
Some(c) if c == '"' || c == '\'' || c == '(' || is_non_printable(c) => {
self.consume_the_remnants_of_a_bad_url();
return Token::BadUrl;
}
Some('\\') => {
if self.is_valid_escape_next() {
let escaped = self.consume_an_escaped_code_point();
value.push(escaped);
} else {
self.consume_the_remnants_of_a_bad_url();
return Token::BadUrl;
}
}
Some(c) => value.push(c),
}
}
}
fn consume_the_remnants_of_a_bad_url(&mut self) {
loop {
match self.consume() {
Some(')') | None => return,
Some('\\') if self.is_valid_escape_next() => {
let _ = self.consume_an_escaped_code_point();
}
Some(_) => {}
}
}
}
fn consume_u_or_unicode_range(&mut self) -> Token {
self.reconsume(); if self.unicode_ranges_allowed && self.would_start_unicode_range_at(0) {
return self.consume_a_unicode_range_token();
}
self.consume_an_ident_like_token()
}
fn would_start_unicode_range_at(&self, offset: usize) -> bool {
let first = match self.peek(offset) {
Some(c) => c,
None => return false,
};
if !matches!(first, 'U' | 'u') {
return false;
}
if self.peek(offset + 1) != Some('+') {
return false;
}
match self.peek(offset + 2) {
Some('?') => true,
Some(c) if is_hex_digit(c) => true,
_ => false,
}
}
fn consume_a_unicode_range_token(&mut self) -> Token {
self.consume(); self.consume();
let mut first_segment = String::new();
while first_segment.len() < 6 {
match self.peek(0) {
Some(c) if is_hex_digit(c) => {
first_segment.push(c);
self.consume();
}
_ => break,
}
}
while first_segment.len() < 6 {
match self.peek(0) {
Some('?') => {
first_segment.push('?');
self.consume();
}
_ => break,
}
}
if first_segment.contains('?') {
let start_str: String = first_segment
.chars()
.map(|c| if c == '?' { '0' } else { c })
.collect();
let start = u32::from_str_radix(&start_str, 16).unwrap_or(0);
let end_str: String = first_segment
.chars()
.map(|c| if c == '?' { 'F' } else { c })
.collect();
let end = u32::from_str_radix(&end_str, 16).unwrap_or(0);
return Token::UnicodeRange(Some(start), Some(end));
}
let start = u32::from_str_radix(&first_segment, 16).unwrap_or(0);
if self.peek(0) == Some('-') && self.peek(1).is_some_and(is_hex_digit) {
self.consume(); let mut end_segment = String::new();
while end_segment.len() < 6 {
match self.peek(0) {
Some(c) if is_hex_digit(c) => {
end_segment.push(c);
self.consume();
}
_ => break,
}
}
let end = u32::from_str_radix(&end_segment, 16).unwrap_or(0);
return Token::UnicodeRange(Some(start), Some(end));
}
Token::UnicodeRange(Some(start), Some(start))
}
fn consume_an_ident_sequence(&mut self) -> String {
let mut result = String::new();
loop {
let Some(c) = self.consume() else {
return result;
};
if is_ident_code_point(c) {
result.push(c);
} else if c == '\\' && self.is_valid_escape_next() {
let escaped = self.consume_an_escaped_code_point();
result.push(escaped);
} else {
self.reconsume();
return result;
}
}
}
fn consume_an_escaped_code_point(&mut self) -> char {
let Some(c) = self.consume() else {
return '\u{FFFD}';
};
if is_hex_digit(c) {
let mut hex = String::new();
hex.push(c);
while hex.len() < 6 {
match self.peek(0) {
Some(next) if is_hex_digit(next) => {
hex.push(next);
self.consume();
}
_ => break,
}
}
if self.peek(0) == Some(' ') || self.peek(0) == Some('\t') || self.peek(0) == Some('\n')
{
self.consume();
}
let value = u32::from_str_radix(&hex, 16).unwrap_or(0);
if value == 0 || value > 0x10FFFF || (0xD800..=0xDFFF).contains(&value) {
'\u{FFFD}'
} else {
char::from_u32(value).unwrap_or('\u{FFFD}')
}
} else if c == '\n' {
self.reconsume();
'\\'
} else {
c
}
}
fn is_valid_escape_next(&self) -> bool {
match self.peek(0) {
None => false,
Some('\n') => false,
Some(_) => true,
}
}
fn would_start_ident_sequence_at(&self, offset: usize) -> bool {
let first = match self.peek(offset) {
Some(c) => c,
None => return false,
};
match first {
'-' => {
match self.peek(offset + 1) {
Some('-') => true,
Some(next) if is_ident_start_code_point(next) => true,
Some('\\') => {
self.is_valid_escape_at(offset + 1)
}
_ => false,
}
}
'\\' => self.is_valid_escape_at(offset),
_ if is_ident_start_code_point(first) => true,
_ => false,
}
}
fn is_valid_escape_at(&self, offset: usize) -> bool {
if self.peek(offset) != Some('\\') {
return false;
}
match self.peek(offset + 1) {
None => false,
Some('\n') => false,
Some(_) => true,
}
}
fn starts_with_number(&self, first: char) -> bool {
match first {
'+' | '-' => match self.peek(0) {
Some(d) if is_digit(d) => true,
Some('.') => matches!(self.peek(1), Some(d) if is_digit(d)),
_ => false,
},
'.' => matches!(self.peek(0), Some(d) if is_digit(d)),
d if is_digit(d) => true,
_ => false,
}
}
}
fn is_ident_start_code_point(c: char) -> bool {
matches!(c, 'A'..='Z' | 'a'..='z' | '_' | '\u{0080}'..=char::MAX)
}
fn is_ident_code_point(c: char) -> bool {
matches!(c, 'A'..='Z' | 'a'..='z' | '0'..='9' | '_' | '-' | '\u{0080}'..=char::MAX)
}
fn is_digit(c: char) -> bool {
c.is_ascii_digit()
}
fn is_hex_digit(c: char) -> bool {
c.is_ascii_hexdigit()
}
fn is_whitespace(c: char) -> bool {
matches!(c, '\t' | '\n' | '\u{000C}' | '\r' | ' ')
}
fn is_non_printable(c: char) -> bool {
matches!(
c,
'\u{0000}'..='\u{0008}' | '\u{000B}' | '\u{000E}'..='\u{001F}' | '\u{007F}'..='\u{009F}'
)
}
fn preprocess_input(s: &str) -> Vec<char> {
let mut out = Vec::with_capacity(s.len());
let mut chars = s.chars().peekable();
while let Some(c) = chars.next() {
match c {
'\r' => {
if chars.peek() == Some(&'\n') {
chars.next();
}
out.push('\n');
}
'\u{000C}' => out.push('\n'),
_ => out.push(c),
}
}
out
}
impl Tokenizer for CssTokenizer {
fn next_token(&mut self) -> Option<Token> {
if self.eof_emitted {
return None;
}
Some(self.consume_a_token())
}
fn state(&self) -> State {
self.state
}
fn set_state(&mut self, state: State) {
self.state = state;
if state == State::Data {
self.eof_emitted = false;
}
}
fn reset(&mut self) {
self.pos = 0;
self.state = State::Data;
self.eof_emitted = false;
}
fn set_unicode_ranges_allowed(&mut self, allowed: bool) {
self.unicode_ranges_allowed = allowed;
}
fn position(&self) -> usize {
self.pos
}
}
#[cfg(test)]
impl CssTokenizer {
fn collect(input: &str) -> Vec<Token> {
let mut tz = Self::new(input);
let mut out = Vec::new();
while let Some(t) = tz.next_token() {
if matches!(t, Token::Eof) {
break;
}
out.push(t);
}
out
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn empty_input_emits_eof_only() {
let mut tz = CssTokenizer::new("");
assert!(matches!(tz.next_token(), Some(Token::Eof)));
assert!(tz.next_token().is_none());
}
#[test]
fn whitespace_collapses_runs() {
let tokens = CssTokenizer::collect(" \t\n ");
assert_eq!(tokens.len(), 1, "whitespace run → single token");
assert!(matches!(tokens[0], Token::Whitespace));
}
#[test]
fn simple_punctuation_tokens() {
let tokens = CssTokenizer::collect(":;,.{}[]()");
assert_eq!(tokens.len(), 10);
assert!(matches!(tokens[0], Token::Colon));
assert!(matches!(tokens[1], Token::Semicolon));
assert!(matches!(tokens[2], Token::Comma));
assert!(matches!(tokens[3], Token::Delim('.')));
assert!(matches!(tokens[4], Token::OpenBrace));
assert!(matches!(tokens[5], Token::CloseBrace));
assert!(matches!(tokens[6], Token::OpenBracket));
assert!(matches!(tokens[7], Token::CloseBracket));
assert!(matches!(tokens[8], Token::OpenParen));
assert!(matches!(tokens[9], Token::CloseParen));
}
#[test]
fn close_paren_is_emitted() {
let tokens = CssTokenizer::collect(")");
assert_eq!(tokens.len(), 1);
assert!(matches!(tokens[0], Token::CloseParen));
}
#[test]
fn unknown_delim_chars() {
let tokens = CssTokenizer::collect("^~`>");
assert_eq!(tokens.len(), 4);
assert!(matches!(tokens[0], Token::Delim('^')));
assert!(matches!(tokens[1], Token::Delim('~')));
assert!(matches!(tokens[2], Token::Delim('`')));
assert!(matches!(tokens[3], Token::Delim('>')));
}
#[test]
fn preprocessing_normalizes_cr_lf_ff() {
let tz = CssTokenizer::new("a\r\nb");
assert_eq!(tz.input, vec!['a', '\n', 'b']);
let tz = CssTokenizer::new("a\rb");
assert_eq!(tz.input, vec!['a', '\n', 'b']);
let tz = CssTokenizer::new("a\u{000C}b");
assert_eq!(tz.input, vec!['a', '\n', 'b']);
}
#[test]
fn eof_emitted_once() {
let mut tz = CssTokenizer::new(",");
let t1 = tz.next_token();
let t2 = tz.next_token();
let t3 = tz.next_token();
assert!(matches!(t1, Some(Token::Comma)));
assert!(matches!(t2, Some(Token::Eof)));
assert!(t3.is_none());
}
#[test]
fn block_comment_is_skipped() {
let tokens = CssTokenizer::collect("/* hello */:/*x*/;");
assert_eq!(tokens.len(), 2);
assert!(matches!(tokens[0], Token::Colon));
assert!(matches!(tokens[1], Token::Semicolon));
}
#[test]
fn unterminated_comment_consumes_to_eof() {
let tokens = CssTokenizer::collect("/* unfinished");
assert_eq!(tokens.len(), 0, "unterminated comment → no tokens");
}
#[test]
fn comment_between_tokens() {
let tokens = CssTokenizer::collect(":/*c*/;");
assert_eq!(tokens.len(), 2);
assert!(matches!(tokens[0], Token::Colon));
assert!(matches!(tokens[1], Token::Semicolon));
}
#[test]
fn cdo_token_emitted() {
let tokens = CssTokenizer::collect("<!--");
assert_eq!(tokens.len(), 1);
assert!(matches!(tokens[0], Token::Cdo));
}
#[test]
fn cdc_token_emitted() {
let tokens = CssTokenizer::collect("-->");
assert_eq!(tokens.len(), 1);
assert!(matches!(tokens[0], Token::Cdc));
}
#[test]
fn cdo_cdc_wrapping_stylesheet() {
let tokens = CssTokenizer::collect("<!-- : ; -->");
assert_eq!(tokens.len(), 7);
assert!(matches!(tokens[0], Token::Cdo));
assert!(matches!(tokens[1], Token::Whitespace));
assert!(matches!(tokens[2], Token::Colon));
assert!(matches!(tokens[3], Token::Whitespace));
assert!(matches!(tokens[4], Token::Semicolon));
assert!(matches!(tokens[5], Token::Whitespace));
assert!(matches!(tokens[6], Token::Cdc));
}
#[test]
fn less_than_alone_is_delim() {
let tokens = CssTokenizer::collect("< ");
assert_eq!(tokens.len(), 2);
assert!(matches!(tokens[0], Token::Delim('<')));
assert!(matches!(tokens[1], Token::Whitespace));
}
#[test]
fn hyphen_alone_is_delim() {
let tokens = CssTokenizer::collect("- ");
assert_eq!(tokens.len(), 2);
assert!(matches!(tokens[0], Token::Delim('-')));
assert!(matches!(tokens[1], Token::Whitespace));
}
#[test]
fn simple_ident() {
let tokens = CssTokenizer::collect("color");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::Ident(s) if s == "color"));
}
#[test]
fn ident_with_hyphen_prefix() {
let tokens = CssTokenizer::collect("-webkit-flex");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::Ident(s) if s == "-webkit-flex"));
}
#[test]
fn ident_with_double_hyphen() {
let tokens = CssTokenizer::collect("--my-var");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::Ident(s) if s == "--my-var"));
}
#[test]
fn ident_with_underscore() {
let tokens = CssTokenizer::collect("_private");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::Ident(s) if s == "_private"));
}
#[test]
fn ident_with_escape() {
let tokens = CssTokenizer::collect("color\\26 B");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::Ident(s) if s == "color&B"));
}
#[test]
fn function_token() {
let tokens = CssTokenizer::collect("translate(");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::Function(s) if s == "translate"));
}
#[test]
fn at_keyword_token() {
let tokens = CssTokenizer::collect("@media");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::AtKeyword(s) if s == "media"));
}
#[test]
fn at_keyword_with_hyphen() {
let tokens = CssTokenizer::collect("@-webkit-keyframes");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::AtKeyword(s) if s == "-webkit-keyframes"));
}
#[test]
fn at_sign_alone_is_delim() {
let tokens = CssTokenizer::collect("@ ");
assert_eq!(tokens.len(), 2);
assert!(matches!(tokens[0], Token::Delim('@')));
assert!(matches!(tokens[1], Token::Whitespace));
}
#[test]
fn hash_id_type() {
let tokens = CssTokenizer::collect("#main");
assert_eq!(tokens.len(), 1);
match &tokens[0] {
Token::Hash(s, t) => {
assert_eq!(s, "main");
assert_eq!(*t, crate::types::HashType::Id);
}
_ => panic!("expected Hash, got {:?}", tokens[0]),
}
}
#[test]
fn hash_unrestricted_type() {
let tokens = CssTokenizer::collect("#123");
assert_eq!(tokens.len(), 1);
match &tokens[0] {
Token::Hash(s, t) => {
assert_eq!(s, "123");
assert_eq!(*t, crate::types::HashType::Unrestricted);
}
_ => panic!("expected Hash, got {:?}", tokens[0]),
}
}
#[test]
fn hash_hex_color_is_unrestricted() {
let tokens = CssTokenizer::collect("#ff00aa");
assert_eq!(tokens.len(), 1);
match &tokens[0] {
Token::Hash(s, t) => {
assert_eq!(s, "ff00aa");
assert_eq!(*t, crate::types::HashType::Id);
}
_ => panic!("expected Hash, got {:?}", tokens[0]),
}
}
#[test]
fn backslash_escape_in_ident() {
let tokens = CssTokenizer::collect("a\\z");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::Ident(s) if s == "az"));
}
#[test]
fn backslash_alone_is_delim() {
let tokens = CssTokenizer::collect("\\");
assert_eq!(tokens.len(), 1);
assert!(matches!(tokens[0], Token::Delim('\\')));
}
#[test]
fn backslash_newline_is_delim() {
let tokens = CssTokenizer::collect("\\\n");
assert_eq!(tokens.len(), 2);
assert!(matches!(tokens[0], Token::Delim('\\')));
assert!(matches!(tokens[1], Token::Whitespace));
}
#[test]
fn declaration_with_ident_and_value() {
let tokens = CssTokenizer::collect("color: red");
assert_eq!(tokens.len(), 4);
assert!(matches!(&tokens[0], Token::Ident(s) if s == "color"));
assert!(matches!(tokens[1], Token::Colon));
assert!(matches!(tokens[2], Token::Whitespace));
assert!(matches!(&tokens[3], Token::Ident(s) if s == "red"));
}
#[test]
fn unicode_escape_in_ident() {
let tokens = CssTokenizer::collect("\\000026 B");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::Ident(s) if s == "&B"));
}
#[test]
fn null_escape_becomes_replacement_char() {
let tokens = CssTokenizer::collect("\\0");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::Ident(s) if s == "\u{FFFD}"));
}
#[test]
fn surrogate_escape_becomes_replacement_char() {
let tokens = CssTokenizer::collect("\\D800");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::Ident(s) if s == "\u{FFFD}"));
}
#[test]
fn hash_alone_is_delim() {
let tokens = CssTokenizer::collect("#");
assert_eq!(tokens.len(), 1);
assert!(matches!(tokens[0], Token::Delim('#')));
}
#[test]
fn hash_followed_by_space_is_delim() {
let tokens = CssTokenizer::collect("# ");
assert_eq!(tokens.len(), 2);
assert!(matches!(tokens[0], Token::Delim('#')));
assert!(matches!(tokens[1], Token::Whitespace));
}
#[test]
fn hash_followed_by_at_is_delim() {
let tokens = CssTokenizer::collect("#@");
assert_eq!(tokens.len(), 2);
assert!(matches!(tokens[0], Token::Delim('#')));
assert!(matches!(tokens[1], Token::Delim('@')));
}
#[test]
fn string_double_quoted() {
let tokens = CssTokenizer::collect("\"hello\"");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
}
#[test]
fn string_single_quoted() {
let tokens = CssTokenizer::collect("'hello'");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
}
#[test]
fn string_with_escape() {
let tokens = CssTokenizer::collect("\"a\\z\"");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::String(s) if s == "az"));
}
#[test]
fn string_unterminated_eof() {
let tokens = CssTokenizer::collect("\"hello");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::String(s) if s == "hello"));
}
#[test]
fn string_unescaped_newline_is_bad_string() {
let tokens = CssTokenizer::collect("\"\n");
assert_eq!(tokens.len(), 2);
assert!(matches!(tokens[0], Token::BadString));
assert!(matches!(tokens[1], Token::Whitespace));
}
#[test]
fn string_line_continuation() {
let tokens = CssTokenizer::collect("\"a\\\nb\"");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::String(s) if s == "ab"));
}
#[test]
fn string_hex_escape() {
let tokens = CssTokenizer::collect("\"\\26\"");
assert_eq!(tokens.len(), 1);
assert!(matches!(&tokens[0], Token::String(s) if s == "&"));
}
#[test]
fn number_integer() {
let tokens = CssTokenizer::collect("42");
assert_eq!(tokens.len(), 1);
assert_eq!(
tokens[0],
Token::Number(Numeric {
value: 42.0,
is_integer: true
})
);
}
#[test]
fn number_decimal() {
let tokens = CssTokenizer::collect("3.5");
assert_eq!(tokens.len(), 1);
assert_eq!(
tokens[0],
Token::Number(Numeric {
value: 3.5,
is_integer: false
})
);
}
#[test]
fn number_signed() {
let tokens = CssTokenizer::collect("-5");
assert_eq!(tokens.len(), 1);
assert_eq!(
tokens[0],
Token::Number(Numeric {
value: -5.0,
is_integer: true
})
);
}
#[test]
fn number_exponent() {
let tokens = CssTokenizer::collect("1e3");
assert_eq!(tokens.len(), 1);
assert_eq!(
tokens[0],
Token::Number(Numeric {
value: 1000.0,
is_integer: false
})
);
}
#[test]
fn number_decimal_exponent() {
let tokens = CssTokenizer::collect("1.5e2");
assert_eq!(tokens.len(), 1);
assert_eq!(
tokens[0],
Token::Number(Numeric {
value: 150.0,
is_integer: false
})
);
}
#[test]
fn percentage_token() {
let tokens = CssTokenizer::collect("50%");
assert_eq!(tokens.len(), 1);
assert_eq!(
tokens[0],
Token::Percentage(Numeric {
value: 50.0,
is_integer: true
})
);
}
#[test]
fn dimension_px() {
let tokens = CssTokenizer::collect("10px");
assert_eq!(tokens.len(), 1);
assert_eq!(
tokens[0],
Token::Dimension(
Numeric {
value: 10.0,
is_integer: true
},
"px".to_string(),
)
);
}
#[test]
fn dimension_em() {
let tokens = CssTokenizer::collect("1.5em");
assert_eq!(tokens.len(), 1);
assert_eq!(
tokens[0],
Token::Dimension(
Numeric {
value: 1.5,
is_integer: false
},
"em".to_string(),
)
);
}
#[test]
fn dimension_signed() {
let tokens = CssTokenizer::collect("-30deg");
assert_eq!(tokens.len(), 1);
assert_eq!(
tokens[0],
Token::Dimension(
Numeric {
value: -30.0,
is_integer: true
},
"deg".to_string(),
)
);
}
#[test]
fn plus_sign_number() {
let tokens = CssTokenizer::collect("+5");
assert_eq!(tokens.len(), 1);
assert_eq!(
tokens[0],
Token::Number(Numeric {
value: 5.0,
is_integer: true
})
);
}
#[test]
fn dot_starts_number() {
let tokens = CssTokenizer::collect(".5");
assert_eq!(tokens.len(), 1);
assert_eq!(
tokens[0],
Token::Number(Numeric {
value: 0.5,
is_integer: false
})
);
}
#[test]
fn url_unquoted_simple() {
let tokens = CssTokenizer::collect("url(foo)");
assert_eq!(tokens.len(), 1);
assert_eq!(tokens[0], Token::Url("foo".to_string()));
}
#[test]
fn url_unquoted_with_spaces() {
let tokens = CssTokenizer::collect("url( foo )");
assert_eq!(tokens.len(), 1);
assert_eq!(tokens[0], Token::Url("foo".to_string()));
}
#[test]
fn url_empty() {
let tokens = CssTokenizer::collect("url()");
assert_eq!(tokens.len(), 1);
assert_eq!(tokens[0], Token::Url(String::new()));
}
#[test]
fn url_eof_unterminated() {
let tokens = CssTokenizer::collect("url(foo");
assert_eq!(tokens.len(), 1);
assert_eq!(tokens[0], Token::Url("foo".to_string()));
}
#[test]
fn url_quoted_is_function() {
let tokens = CssTokenizer::collect("url(\"foo\")");
assert_eq!(tokens.len(), 3);
assert_eq!(tokens[0], Token::Function("url".to_string()));
assert_eq!(tokens[1], Token::String("foo".to_string()));
assert_eq!(tokens[2], Token::CloseParen);
}
#[test]
fn url_single_quoted_is_function() {
let tokens = CssTokenizer::collect("url('foo')");
assert_eq!(tokens.len(), 3);
assert_eq!(tokens[0], Token::Function("url".to_string()));
assert_eq!(tokens[1], Token::String("foo".to_string()));
assert_eq!(tokens[2], Token::CloseParen);
}
#[test]
fn url_ws_then_quote_is_function() {
let tokens = CssTokenizer::collect("url( \"foo\")");
assert_eq!(tokens.len(), 4);
assert_eq!(tokens[0], Token::Function("url".to_string()));
assert_eq!(tokens[1], Token::Whitespace);
assert_eq!(tokens[2], Token::String("foo".to_string()));
assert_eq!(tokens[3], Token::CloseParen);
}
#[test]
fn url_with_escape() {
let tokens = CssTokenizer::collect("url(foo\\29 bar)");
assert_eq!(tokens.len(), 1);
assert_eq!(tokens[0], Token::Url("foo)bar".to_string()));
}
#[test]
fn url_bad_paren_in_unquoted() {
let tokens = CssTokenizer::collect("url(foo(bar)");
assert_eq!(tokens.len(), 1);
assert_eq!(tokens[0], Token::BadUrl);
}
#[test]
fn url_bad_quote_in_unquoted() {
let tokens = CssTokenizer::collect("url(foo\"bar)");
assert_eq!(tokens.len(), 1);
assert_eq!(tokens[0], Token::BadUrl);
}
#[test]
fn unicode_range_disabled_by_default() {
let tokens = CssTokenizer::collect("U+1234");
assert_eq!(tokens.len(), 2);
assert_eq!(tokens[0], Token::Ident("U".to_string()));
assert_eq!(
tokens[1],
Token::Number(Numeric {
value: 1234.0,
is_integer: true
})
);
}
#[test]
fn unicode_range_simple() {
let mut tz = CssTokenizer::new("U+1234");
tz.set_unicode_ranges_allowed(true);
let t = tz.next_token().unwrap();
assert_eq!(t, Token::UnicodeRange(Some(0x1234), Some(0x1234)));
}
#[test]
fn unicode_range_question_marks() {
let mut tz = CssTokenizer::new("U+12??");
tz.set_unicode_ranges_allowed(true);
let t = tz.next_token().unwrap();
assert_eq!(t, Token::UnicodeRange(Some(0x1200), Some(0x12FF)));
}
#[test]
fn unicode_range_range() {
let mut tz = CssTokenizer::new("U+12-34FF");
tz.set_unicode_ranges_allowed(true);
let t = tz.next_token().unwrap();
assert_eq!(t, Token::UnicodeRange(Some(0x12), Some(0x34FF)));
}
#[test]
fn unicode_range_max_hex() {
let mut tz = CssTokenizer::new("U+10FFFF");
tz.set_unicode_ranges_allowed(true);
let t = tz.next_token().unwrap();
assert_eq!(t, Token::UnicodeRange(Some(0x10FFFF), Some(0x10FFFF)));
}
#[test]
fn unicode_range_lowercase_u() {
let mut tz = CssTokenizer::new("u+abc");
tz.set_unicode_ranges_allowed(true);
let t = tz.next_token().unwrap();
assert_eq!(t, Token::UnicodeRange(Some(0xABC), Some(0xABC)));
}
}