use crate::catcode::{Cat, CatTable};
use crate::token::Token;
#[derive(Clone, Copy, PartialEq)]
enum State {
NewLine,
MidLine,
SkipBlanks,
}
pub struct Lexer {
chars: Vec<char>,
pos: usize,
state: State,
pub pending: Vec<Token>,
}
impl Lexer {
pub fn new(src: &str) -> Self {
Self {
chars: src.chars().collect(),
pos: 0,
state: State::NewLine,
pending: Vec::new(),
}
}
pub fn push_back(&mut self, toks: &[Token]) {
for t in toks.iter().rev() {
self.pending.push(t.clone());
}
}
fn peek(&self) -> Option<char> {
self.chars.get(self.pos).copied()
}
fn double_superscript(&mut self, cats: &CatTable, c: char) -> Option<char> {
if cats.get(c) != Cat::Superscript {
return None;
}
if self.chars.get(self.pos + 1).copied() != Some(c) {
return None;
}
let third = self.chars.get(self.pos + 2).copied()?;
let code = u32::from(third);
if code >= 128 {
return None;
}
self.pos += 3;
let shifted = match code < 64 {
true => code + 64,
false => code - 64,
};
char::from_u32(shifted)
}
pub fn next_token(&mut self, cats: &CatTable) -> Option<Token> {
if let Some(t) = self.pending.pop() {
return Some(t);
}
loop {
let c = self.peek()?;
if let Some(decoded) = self.double_superscript(cats, c) {
self.chars.splice(self.pos..self.pos, [decoded]);
continue;
}
let cat = cats.get(c);
self.pos += 1;
match cat {
Cat::Escape => return Some(self.control_sequence(cats)),
Cat::Comment => {
while let Some(ch) = self.peek() {
self.pos += 1;
if cats.get(ch) == Cat::EndLine {
break;
}
}
self.state = State::NewLine;
}
Cat::EndLine => {
let out = match self.state {
State::MidLine => Some(Token::Char(' ', Cat::Space)),
State::NewLine => Some(Token::Cs("par".to_string())),
State::SkipBlanks => None,
};
self.state = State::NewLine;
if out.is_some() {
return out;
}
}
Cat::Space => {
if self.state == State::MidLine {
self.state = State::SkipBlanks;
return Some(Token::Char(' ', Cat::Space));
}
}
Cat::Ignored => {}
_ => {
self.state = State::MidLine;
return Some(Token::Char(c, cat));
}
}
}
}
fn control_sequence(&mut self, cats: &CatTable) -> Token {
let Some(first) = self.peek() else {
return Token::Cs(String::new());
};
self.pos += 1;
if cats.get(first) != Cat::Letter {
self.state = match cats.get(first) == Cat::Space {
true => State::SkipBlanks,
false => State::MidLine,
};
return Token::Cs(first.to_string());
}
let mut name = String::from(first);
while let Some(c) = self.peek() {
if cats.get(c) != Cat::Letter {
break;
}
name.push(c);
self.pos += 1;
}
self.state = State::SkipBlanks;
Token::Cs(name)
}
}