use std::{fs::File, io::{BufRead, BufReader}, path::Path};
use oaken::{Registry, State, StateDelta, lex, util::string_state::StringState};
#[derive(Debug, Clone)]
pub enum Token {
Identifier(String),
Value(String),
Assign,
BodyOpen,
BodyClose,
LineEnd,
EOF,
}
impl Token {
pub fn get_infix_binding(&self) -> i32 {
match self {
Self::LineEnd => 1,
Self::EOF | Self::BodyClose => -1,
Self::Assign | Self::BodyOpen => 2,
_ => 0
}
}
}
fn identifier_fallback(str: &String) -> Token {
Token::Identifier(str.clone())
}
pub fn create_registry() -> Registry<Token, impl Fn(&String) -> Token> {
Registry::new(identifier_fallback)
.new_keyword("=", Token::Assign, true).unwrap()
.new_keyword("{", Token::BodyOpen, true).unwrap()
.new_keyword("}", Token::BodyClose, true).unwrap()
.new_keyword(";", Token::LineEnd, true).unwrap()
.new_special_state('"', StringState::new('"'))
.new_special_state('#', CommentState)
}
impl From<String> for Token {
fn from(value: String) -> Self {
Self::Value(value)
}
}
pub fn lex_file<P>(
path: P,
reg: &Registry<Token, impl Fn(&String) -> Token>
) -> std::io::Result<Vec<Token>>
where
P: AsRef<Path>
{
let mut tokens = Vec::new();
let file = File::open(path)?;
let reader = BufReader::new(file);
for line in reader.lines() {
let line = line?;
let mut line = lex(reg, line);
if !line.is_empty() {
tokens.append(&mut line);
}
}
tokens.push(Token::EOF);
Ok(tokens)
}
struct CommentState;
impl State<Token> for CommentState {
fn delta<'a>(&'a self, buffer: &mut oaken::TokenBuffer, _: char) -> StateDelta<'a, Token> {
if !buffer.read().is_empty() {
buffer.flush();
}
StateDelta::Stayed
}
fn construct(&self, _: &oaken::TokenBuffer) -> Option<Token> {
None
}
}