use std::fmt;
#[derive(Debug, Clone, PartialEq)]
pub struct Token {
pub kind: Tok,
pub line: u32,
pub space: bool,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Tok {
Int(i64),
Float(f64),
Str(String, bool),
Regex(String, String),
Symbol(String),
Ident(String),
Const(String), IVar(String), GVar(String), Keyword(String),
Op(String),
Newline,
Semicolon,
Eof,
}
impl fmt::Display for Tok {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Tok::Int(n) => write!(f, "{n}"),
Tok::Float(x) => write!(f, "{x}"),
Tok::Str(s, _) => write!(f, "\"{s}\""),
Tok::Regex(s, fl) => write!(f, "/{s}/{fl}"),
Tok::Symbol(s) => write!(f, ":{s}"),
Tok::Ident(s) | Tok::Const(s) | Tok::Keyword(s) => write!(f, "{s}"),
Tok::IVar(s) => write!(f, "@{s}"),
Tok::GVar(s) => write!(f, "${s}"),
Tok::Op(s) => write!(f, "{s}"),
Tok::Newline => write!(f, "\\n"),
Tok::Semicolon => write!(f, ";"),
Tok::Eof => write!(f, "<eof>"),
}
}
}
const KEYWORDS: &[&str] = &[
"def", "end", "if", "elsif", "else", "unless", "while", "until", "for", "in", "do", "return",
"break", "next", "yield", "then", "case", "when", "nil", "true", "false", "and", "or", "not",
"class", "module", "begin", "rescue", "ensure", "self", "super", "retry",
];
pub fn lex(src: &str) -> Result<Vec<Token>, String> {
let b = src.as_bytes();
let mut i = 0usize;
let mut line = 1u32;
let mut out: Vec<Token> = Vec::new();
let mut sp = true;
let continues = |out: &[Token]| -> bool {
match out.iter().rev().find(|t| t.kind != Tok::Newline) {
None => true,
Some(t) => {
matches!(&t.kind,
Tok::Op(o) if o != ")" && o != "]" && o != "}" )
|| matches!(&t.kind, Tok::Keyword(k)
if matches!(k.as_str(), "and"|"or"|"not"|"if"|"unless"|"while"|"until"|"do"|"then"|"else"|"in"))
}
}
};
while i < b.len() {
let c = b[i];
match c {
b' ' | b'\t' | b'\r' => {
sp = true;
i += 1;
}
b'\\' if i + 1 < b.len() && b[i + 1] == b'\n' => {
sp = true;
line += 1;
i += 2;
}
b'\n' => {
if !continues(&out) {
out.push(Token {
kind: Tok::Newline,
line,
space: core::mem::take(&mut sp),
});
}
sp = true;
line += 1;
i += 1;
}
b'#' => {
while i < b.len() && b[i] != b'\n' {
i += 1;
}
sp = true;
}
b';' => {
out.push(Token {
kind: Tok::Semicolon,
line,
space: core::mem::take(&mut sp),
});
i += 1;
}
b'0'..=b'9' => {
let start = i;
while i < b.len() && (b[i].is_ascii_digit() || b[i] == b'_') {
i += 1;
}
let mut is_float = false;
if i + 1 < b.len() && b[i] == b'.' && b[i + 1].is_ascii_digit() {
is_float = true;
i += 1;
while i < b.len() && (b[i].is_ascii_digit() || b[i] == b'_') {
i += 1;
}
}
if i < b.len() && (b[i] == b'e' || b[i] == b'E') {
is_float = true;
i += 1;
if i < b.len() && (b[i] == b'+' || b[i] == b'-') {
i += 1;
}
while i < b.len() && b[i].is_ascii_digit() {
i += 1;
}
}
let raw: String = src[start..i].chars().filter(|c| *c != '_').collect();
if is_float {
let x: f64 = raw.parse().map_err(|_| format!("bad float: {raw}"))?;
out.push(Token {
kind: Tok::Float(x),
line,
space: core::mem::take(&mut sp),
});
} else {
let n: i64 = raw.parse().map_err(|_| format!("bad integer: {raw}"))?;
out.push(Token {
kind: Tok::Int(n),
line,
space: core::mem::take(&mut sp),
});
}
}
b'"' | b'\'' => {
let quote = c;
let dq = quote == b'"';
i += 1;
let mut s = String::new();
loop {
if i >= b.len() {
return Err(format!("unterminated string at line {line}"));
}
let ch = b[i];
if ch == quote {
i += 1;
break;
}
if ch == b'\\' && i + 1 < b.len() {
let n = b[i + 1];
if dq {
s.push('\\');
s.push(n as char);
} else {
match n {
b'\'' => s.push('\''),
b'\\' => s.push('\\'),
other => {
s.push('\\');
s.push(other as char);
}
}
}
i += 2;
continue;
}
if dq && ch == b'#' && i + 1 < b.len() && b[i + 1] == b'{' {
s.push('#');
s.push('{');
i += 2;
let mut depth = 1u32;
while i < b.len() && depth > 0 {
let d = b[i];
match d {
b'{' => depth += 1,
b'}' => depth -= 1,
b'"' | b'\'' => {
let q = d;
s.push(q as char);
i += 1;
while i < b.len() && b[i] != q {
if b[i] == b'\\' && i + 1 < b.len() {
s.push('\\');
s.push(b[i + 1] as char);
i += 2;
continue;
}
if b[i] == b'\n' {
line += 1;
}
let cl = utf8_len(b[i]);
s.push_str(&src[i..i + cl]);
i += cl;
}
if i < b.len() {
s.push(q as char);
i += 1;
}
continue;
}
b'\n' => line += 1,
_ => {}
}
if depth == 0 {
s.push('}');
i += 1;
break;
}
let cl = utf8_len(d);
s.push_str(&src[i..i + cl]);
i += cl;
}
continue;
}
if ch == b'\n' {
line += 1;
}
let clen = utf8_len(ch);
s.push_str(&src[i..i + clen]);
i += clen;
}
out.push(Token {
kind: Tok::Str(s, dq),
line,
space: core::mem::take(&mut sp),
});
}
b'@' => {
i += 1;
let start = i;
while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_') {
i += 1;
}
out.push(Token {
kind: Tok::IVar(src[start..i].to_string()),
line,
space: core::mem::take(&mut sp),
});
}
b'$' => {
i += 1;
let start = i;
if i < b.len()
&& matches!(
b[i],
b'~' | b'&' | b'`' | b'\'' | b'+' | b',' | b'\\' | b';' | b'/' | b'!'
)
{
i += 1;
} else {
while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_') {
i += 1;
}
}
out.push(Token {
kind: Tok::GVar(src[start..i].to_string()),
line,
space: core::mem::take(&mut sp),
});
}
b'%' if i + 2 < b.len()
&& matches!(b[i + 1], b'w' | b'i' | b'W' | b'I')
&& matches!(b[i + 2], b'[' | b'(' | b'{' | b'<' | b'|') =>
{
let symbols = matches!(b[i + 1], b'i' | b'I');
let open = b[i + 2];
let close = match open {
b'[' => b']',
b'(' => b')',
b'{' => b'}',
b'<' => b'>',
other => other,
};
i += 3;
let start = i;
let mut depth = 1u32;
while i < b.len() {
if b[i] == b'\n' {
line += 1;
}
if b[i] == open && open != close {
depth += 1;
} else if b[i] == close {
depth -= 1;
if depth == 0 {
break;
}
}
i += 1;
}
let body = &src[start..i.min(b.len())];
i += 1; let words: Vec<&str> = body.split_whitespace().collect();
out.push(Token {
kind: Tok::Op("[".into()),
line,
space: core::mem::take(&mut sp),
});
for (idx, w) in words.iter().enumerate() {
if idx > 0 {
out.push(Token {
kind: Tok::Op(",".into()),
line,
space: false,
});
}
let kind = if symbols {
Tok::Symbol((*w).to_string())
} else {
Tok::Str((*w).to_string(), false)
};
out.push(Token {
kind,
line,
space: false,
});
}
out.push(Token {
kind: Tok::Op("]".into()),
line,
space: false,
});
}
b'?' if sp
&& i + 1 < b.len()
&& (b[i + 1] == b'\\'
|| (b[i + 1].is_ascii_alphanumeric()
&& (i + 2 >= b.len()
|| !(b[i + 2].is_ascii_alphanumeric() || b[i + 2] == b'_')))) =>
{
i += 1;
let ch = if b[i] == b'\\' && i + 1 < b.len() {
i += 1;
let e = b[i];
i += 1;
match e {
b'n' => '\n',
b't' => '\t',
b'r' => '\r',
b's' => ' ',
b'0' => '\0',
b'e' => '\x1b',
other => other as char,
}
} else {
let c = b[i] as char;
i += 1;
c
};
out.push(Token {
kind: Tok::Str(ch.to_string(), false),
line,
space: core::mem::take(&mut sp),
});
}
b':' if i + 1 < b.len()
&& op_symbol_at(&src[i + 1..]).is_some()
&& !prev_is_value(&out) =>
{
let op = op_symbol_at(&src[i + 1..]).unwrap();
i += 1 + op.len();
out.push(Token {
kind: Tok::Symbol(op.to_string()),
line,
space: core::mem::take(&mut sp),
});
}
b':' if i + 1 < b.len()
&& (b[i + 1].is_ascii_alphabetic() || b[i + 1] == b'_' || b[i + 1] == b'@') =>
{
i += 1;
let start = i;
while i < b.len() && b[i] == b'@' {
i += 1;
}
while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_') {
i += 1;
}
if i < b.len() && (b[i] == b'?' || b[i] == b'!') {
i += 1;
}
out.push(Token {
kind: Tok::Symbol(src[start..i].to_string()),
line,
space: core::mem::take(&mut sp),
});
}
_ if c.is_ascii_alphabetic() || c == b'_' => {
let start = i;
while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_') {
i += 1;
}
if i < b.len() && (b[i] == b'?' || b[i] == b'!') {
i += 1;
}
let word = &src[start..i];
let kind = if KEYWORDS.contains(&word) {
Tok::Keyword(word.to_string())
} else if word.as_bytes()[0].is_ascii_uppercase() {
Tok::Const(word.to_string())
} else {
Tok::Ident(word.to_string())
};
out.push(Token {
kind,
line,
space: core::mem::take(&mut sp),
});
}
b'/' if regex_start(&out, sp, b.get(i + 1).copied()) => {
i += 1;
let mut pat = String::new();
let mut in_class = false;
while i < b.len() {
let ch = b[i];
if ch == b'\\' && i + 1 < b.len() {
pat.push('\\');
pat.push(b[i + 1] as char);
i += 2;
continue;
}
match ch {
b'[' => in_class = true,
b']' => in_class = false,
b'/' if !in_class => break,
b'\n' => line += 1,
_ => {}
}
let cl = utf8_len(ch);
pat.push_str(&src[i..i + cl]);
i += cl;
}
i += 1; let fstart = i;
while i < b.len() && matches!(b[i], b'i' | b'm' | b'x' | b'o' | b'u' | b'n') {
i += 1;
}
let flags = src[fstart..i].to_string();
out.push(Token {
kind: Tok::Regex(pat, flags),
line,
space: core::mem::take(&mut sp),
});
}
_ => {
let three = if i + 3 <= b.len() { &src[i..i + 3] } else { "" };
let two = if i + 2 <= b.len() { &src[i..i + 2] } else { "" };
let op: &str = if matches!(
three,
"**=" | "<=>" | "===" | "..." | "&&=" | "||=" | "<<=" | ">>="
) {
three
} else if matches!(
two,
"**" | "=="
| "!="
| "<="
| ">="
| "&&"
| "||"
| "+="
| "-="
| "*="
| "/="
| "%="
| "<<"
| ">>"
| ".."
| "::"
| "=>"
| "|="
| "&="
| "^="
| "->"
| "=~"
| "!~"
) {
two
} else {
match c {
b'+' | b'-' | b'*' | b'/' | b'%' | b'=' | b'<' | b'>' | b'!' | b'&'
| b'|' | b'^' | b'~' | b'(' | b')' | b'[' | b']' | b'{' | b'}' | b','
| b'.' | b':' | b'?' => &src[i..i + 1],
other => {
return Err(format!(
"unexpected character '{}' at line {line}",
other as char
))
}
}
};
out.push(Token {
kind: Tok::Op(op.to_string()),
line,
space: core::mem::take(&mut sp),
});
i += op.len();
}
}
}
out.push(Token {
kind: Tok::Eof,
line,
space: core::mem::take(&mut sp),
});
Ok(out)
}
fn regex_start(out: &[Token], sp: bool, next: Option<u8>) -> bool {
if next == Some(b'=') {
return false;
}
let prev = out.iter().rev().find(|t| t.kind != Tok::Newline);
let prev_is_value = match prev.map(|t| &t.kind) {
None => false,
Some(Tok::Int(_))
| Some(Tok::Float(_))
| Some(Tok::Str(_, _))
| Some(Tok::Regex(_, _))
| Some(Tok::Ident(_))
| Some(Tok::Const(_))
| Some(Tok::IVar(_))
| Some(Tok::GVar(_))
| Some(Tok::Symbol(_)) => true,
Some(Tok::Op(o)) => o == ")" || o == "]" || o == "}",
Some(Tok::Keyword(k)) => matches!(k.as_str(), "self" | "end" | "true" | "false" | "nil"),
_ => false,
};
if !prev_is_value {
return true;
}
sp && !matches!(next, Some(b' ') | Some(b'\t') | None)
}
const OP_SYMBOLS: &[&str] = &[
"<=>", "===", "[]=", "**", "==", "!=", "<=", ">=", "<<", ">>", "=~", "[]", "+", "-", "*", "/",
"%", "<", ">", "&", "|", "^", "~", "!",
];
fn op_symbol_at(s: &str) -> Option<&'static str> {
OP_SYMBOLS.iter().find(|op| s.starts_with(**op)).copied()
}
fn prev_is_value(out: &[Token]) -> bool {
let prev = out.iter().rev().find(|t| t.kind != Tok::Newline);
match prev.map(|t| &t.kind) {
Some(Tok::Int(_))
| Some(Tok::Float(_))
| Some(Tok::Str(_, _))
| Some(Tok::Regex(_, _))
| Some(Tok::Ident(_))
| Some(Tok::Const(_))
| Some(Tok::IVar(_))
| Some(Tok::GVar(_))
| Some(Tok::Symbol(_)) => true,
Some(Tok::Op(o)) => o == ")" || o == "]" || o == "}",
Some(Tok::Keyword(k)) => matches!(k.as_str(), "self" | "end" | "true" | "false" | "nil"),
_ => false,
}
}
fn utf8_len(b: u8) -> usize {
match b {
0x00..=0x7F => 1,
0xC0..=0xDF => 2,
0xE0..=0xEF => 3,
_ => 4,
}
}