use super::catcodes::{CatCode, CatCodeTable};
#[derive(Debug, Clone, PartialEq)]
pub enum Token {
Char { ch: char, cat: CatCode },
ControlSequence(String),
EndOfFile,
}
#[derive(Debug, Clone)]
pub struct TexLexer<'a> {
input: &'a str,
position: usize,
catcodes: CatCodeTable,
}
impl<'a> TexLexer<'a> {
pub fn new(input: &'a str) -> Self {
Self {
input,
position: 0,
catcodes: CatCodeTable::new(),
}
}
pub fn with_catcodes(input: &'a str, catcodes: CatCodeTable) -> Self {
Self {
input,
position: 0,
catcodes,
}
}
fn peek(&self) -> Option<char> {
self.input[self.position..].chars().next()
}
fn advance(&mut self) -> Option<char> {
let ch = self.input[self.position..].chars().next()?;
self.position += ch.len_utf8();
Some(ch)
}
fn skip_comment(&mut self) {
while let Some(ch) = self.peek() {
if ch == '\n' || ch == '\r' {
self.advance(); break;
}
self.advance();
}
}
fn read_cs_name(&mut self) -> String {
match self.peek() {
Some(ch) if self.catcodes.get(ch) != CatCode::Letter => {
self.advance();
ch.to_string()
}
Some(_) => {
let mut name = String::new();
while let Some(ch) = self.peek() {
if self.catcodes.get(ch) == CatCode::Letter {
name.push(ch);
self.advance();
} else {
break;
}
}
name
}
None => String::new(),
}
}
pub fn next_token(&mut self) -> Option<Token> {
loop {
match self.peek() {
None => return None,
Some(ch) => {
let cat = self.catcodes.get(ch);
match cat {
CatCode::Comment => {
self.advance(); self.skip_comment();
}
CatCode::Space => {
self.advance();
while let Some(c) = self.peek() {
if self.catcodes.get(c) == CatCode::Space {
self.advance();
} else {
break;
}
}
return Some(Token::Char { ch: ' ', cat: CatCode::Space });
}
CatCode::EndOfLine => {
self.advance();
while let Some(c) = self.peek() {
if self.catcodes.get(c) == CatCode::EndOfLine {
self.advance();
} else {
break;
}
}
return Some(Token::Char { ch: ' ', cat: CatCode::Space });
}
_ => break,
}
}
}
}
let ch = self.advance()?;
let cat = self.catcodes.get(ch);
if cat == CatCode::Escape {
let name = self.read_cs_name();
Some(Token::ControlSequence(name))
} else {
Some(Token::Char { ch, cat })
}
}
pub fn tokenize(&mut self) -> Vec<Token> {
let mut tokens = Vec::new();
while let Some(tok) = self.next_token() {
tokens.push(tok);
}
tokens
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tokenize_simple_text() {
let mut lexer = TexLexer::new("Hello world");
let toks = lexer.tokenize();
assert_eq!(toks.len(), 11);
assert_eq!(toks[0], Token::Char { ch: 'H', cat: CatCode::Letter });
assert_eq!(toks[5], Token::Char { ch: ' ', cat: CatCode::Space });
assert_eq!(toks[6], Token::Char { ch: 'w', cat: CatCode::Letter });
}
#[test]
fn tokenize_control_sequence() {
let mut lexer = TexLexer::new("\\section{Intro}");
let toks = lexer.tokenize();
assert_eq!(toks[0], Token::ControlSequence("section".to_string()));
assert_eq!(toks[1], Token::Char { ch: '{', cat: CatCode::BeginGroup });
}
#[test]
fn tokenize_single_char_cs() {
let mut lexer = TexLexer::new("\\$");
let toks = lexer.tokenize();
assert_eq!(toks[0], Token::ControlSequence("$".to_string()));
}
#[test]
fn tokenize_comments_stripped() {
let mut lexer = TexLexer::new("Hello % this is a comment\nworld");
let toks = lexer.tokenize();
assert_eq!(toks.len(), 11);
assert_eq!(toks[6], Token::Char { ch: 'w', cat: CatCode::Letter });
}
#[test]
fn tokenize_collapses_spaces() {
let mut lexer = TexLexer::new("a b");
let toks = lexer.tokenize();
assert_eq!(toks.len(), 3);
assert_eq!(toks[1], Token::Char { ch: ' ', cat: CatCode::Space });
}
#[test]
fn tokenize_empty_input() {
let mut lexer = TexLexer::new("");
assert!(lexer.tokenize().is_empty());
}
}