use std::fmt;
#[derive(Debug, Clone, PartialEq)]
pub struct Token {
pub kind: Tok,
pub line: u32,
pub space: bool,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Tok {
Int(i64),
Float(f64),
Str(String, bool),
Regex(String, String),
Symbol(String),
DSymbol(String),
Ident(String),
Const(String), IVar(String), CVar(String), GVar(String), Keyword(String),
Op(String),
Newline,
Semicolon,
Eof,
}
impl fmt::Display for Tok {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Tok::Int(n) => write!(f, "{n}"),
Tok::Float(x) => write!(f, "{x}"),
Tok::Str(s, _) => write!(f, "\"{s}\""),
Tok::Regex(s, fl) => write!(f, "/{s}/{fl}"),
Tok::Symbol(s) | Tok::DSymbol(s) => write!(f, ":{s}"),
Tok::Ident(s) | Tok::Const(s) | Tok::Keyword(s) => write!(f, "{s}"),
Tok::IVar(s) => write!(f, "@{s}"),
Tok::CVar(s) => write!(f, "@@{s}"),
Tok::GVar(s) => write!(f, "${s}"),
Tok::Op(s) => write!(f, "{s}"),
Tok::Newline => write!(f, "\\n"),
Tok::Semicolon => write!(f, ";"),
Tok::Eof => write!(f, "<eof>"),
}
}
}
fn ident_start(c: u8) -> bool {
c.is_ascii_alphabetic() || c == b'_' || !c.is_ascii()
}
fn ident_byte(c: u8) -> bool {
ident_start(c) || c.is_ascii_digit()
}
pub(crate) const KEYWORDS: &[&str] = &[
"def", "end", "if", "elsif", "else", "unless", "while", "until", "for", "in", "do", "return",
"break", "next", "yield", "then", "case", "when", "nil", "true", "false", "and", "or", "not",
"class", "module", "begin", "rescue", "ensure", "self", "super", "retry", "redo", "alias",
];
struct Heredoc {
delim: String,
squiggly: bool,
indent_term: bool,
interp: bool,
tok_idx: usize,
}
fn collect_heredoc_body(
b: &[u8],
start: usize,
delim: &str,
indent_term: bool,
) -> (String, usize, u32) {
let mut body = String::new();
let mut pos = start;
let mut lines = 0u32;
while pos < b.len() {
let line_start = pos;
while pos < b.len() && b[pos] != b'\n' {
pos += 1;
}
let line = std::str::from_utf8(&b[line_start..pos]).unwrap_or("");
let trimmed = if indent_term { line.trim_start() } else { line };
let had_nl = pos < b.len();
if had_nl {
pos += 1;
lines += 1;
}
if trimmed == delim {
return (body, pos, lines);
}
body.push_str(line);
body.push('\n');
}
(body, pos, lines)
}
fn strip_squiggly(body: &str) -> String {
let width_of = |l: &str| -> usize {
let mut col = 0;
for c in l.chars() {
match c {
' ' => col += 1,
'\t' => col = (col / 8 + 1) * 8,
_ => break,
}
}
col
};
let indent = body
.lines()
.filter(|l| !l.trim().is_empty())
.map(width_of)
.min()
.unwrap_or(0);
let mut out = String::new();
for line in body.split_inclusive('\n') {
let content = line.strip_suffix('\n').unwrap_or(line);
let mut col = 0;
let mut cut = 0;
for c in content.chars() {
let next = match c {
' ' => col + 1,
'\t' => (col / 8 + 1) * 8,
_ => break,
};
if next > indent {
break;
}
col = next;
cut += c.len_utf8();
}
out.push_str(&content[cut..]);
if line.ends_with('\n') {
out.push('\n');
}
}
out
}
fn next_line_leading_dot(b: &[u8], mut j: usize) -> bool {
loop {
while j < b.len() && matches!(b[j], b' ' | b'\t' | b'\r' | b'\n') {
j += 1;
}
if j < b.len() && b[j] == b'#' {
while j < b.len() && b[j] != b'\n' {
j += 1;
}
continue;
}
break;
}
b.get(j) == Some(&b'.') && b.get(j + 1) != Some(&b'.')
}
fn heredoc_here(out: &[Token], after: &[u8], space_before: bool) -> bool {
let squiggly = matches!(after.first(), Some(b'~') | Some(b'-'));
let after_flag = if squiggly { &after[1..] } else { after };
let quoted = matches!(after_flag.first(), Some(b'"') | Some(b'\'') | Some(b'`'));
let bare_ok = if squiggly {
matches!(after_flag.first(), Some(c) if c.is_ascii_alphabetic() || *c == b'_')
} else {
matches!(after_flag.first(), Some(c) if c.is_ascii_uppercase() || *c == b'_')
};
if !quoted && !bare_ok {
return false;
}
match out.iter().rev().find(|t| t.kind != Tok::Newline) {
None => true,
Some(t) => match &t.kind {
Tok::Op(o) if o != ")" && o != "]" && o != "}" => true,
Tok::Keyword(_) => true,
_ => space_before,
},
}
}
pub fn lex(src: &str) -> Result<Vec<Token>, String> {
let b = src.as_bytes();
let mut i = 0usize;
let mut line = 1u32;
let mut out: Vec<Token> = Vec::new();
let mut sp = true;
let mut pending: Vec<Heredoc> = Vec::new();
let continues = |out: &[Token]| -> bool {
let mut iter = out.iter().rev().filter(|t| t.kind != Tok::Newline);
match iter.next() {
None => true,
Some(t) => {
if matches!(&t.kind, Tok::Keyword(k)
if matches!(k.as_str(), "and"|"or"|"not"|"if"|"unless"|"while"|"until"|"do"|"then"|"else"|"in"))
{
return !matches!(
iter.next().map(|p| &p.kind),
Some(Tok::Keyword(k)) if k == "def"
);
}
matches!(&t.kind,
Tok::Op(o) if o != ")" && o != "]" && o != "}" && o != "!" && o != "~" )
}
}
};
while i < b.len() {
let c = b[i];
match c {
b' ' | b'\t' | b'\r' => {
sp = true;
i += 1;
}
b'\\' if i + 1 < b.len() && b[i + 1] == b'\n' => {
sp = true;
line += 1;
i += 2;
}
b'\n' => {
if !continues(&out) && !next_line_leading_dot(b, i + 1) {
out.push(Token {
kind: Tok::Newline,
line,
space: core::mem::take(&mut sp),
});
}
sp = true;
line += 1;
i += 1;
if !pending.is_empty() {
for hd in pending.drain(..) {
let (raw, next, lines) =
collect_heredoc_body(b, i, &hd.delim, hd.indent_term);
let body = if hd.squiggly {
strip_squiggly(&raw)
} else {
raw
};
out[hd.tok_idx].kind = Tok::Str(body, hd.interp);
i = next;
line += lines;
}
}
}
b'=' if (i == 0 || b[i - 1] == b'\n') && src[i..].starts_with("=begin") => {
loop {
while i < b.len() && b[i] != b'\n' {
i += 1;
}
if i >= b.len() {
break;
}
i += 1; line += 1;
if src[i..].starts_with("=end") {
while i < b.len() && b[i] != b'\n' {
i += 1;
}
break;
}
}
sp = true;
}
b'#' => {
while i < b.len() && b[i] != b'\n' {
i += 1;
}
sp = true;
}
b';' => {
out.push(Token {
kind: Tok::Semicolon,
line,
space: core::mem::take(&mut sp),
});
i += 1;
}
b'0'..=b'9' => {
if b[i] == b'0' && i + 1 < b.len() {
let (radix, skip) = match b[i + 1] {
b'b' | b'B' => (2u32, 2),
b'o' | b'O' => (8, 2),
b'x' | b'X' => (16, 2),
b'd' | b'D' => (10, 2),
c if c.is_ascii_digit() => (8, 1),
_ => (0, 0),
};
if radix != 0 {
let dstart = i + skip;
let mut j = dstart;
while j < b.len() {
let c = b[j] as char;
if c == '_' || c.is_digit(radix) {
j += 1;
} else {
break;
}
}
let digits: String = src[dstart..j].chars().filter(|c| *c != '_').collect();
let this_sp = core::mem::take(&mut sp);
match i64::from_str_radix(&digits, radix) {
Ok(n) => out.push(Token {
kind: Tok::Int(n),
line,
space: this_sp,
}),
Err(_) => {
let toks = [
Tok::Str(digits.clone(), false),
Tok::Op(".".into()),
Tok::Ident("to_i".into()),
Tok::Op("(".into()),
Tok::Int(radix as i64),
Tok::Op(")".into()),
];
for (k, kind) in toks.into_iter().enumerate() {
out.push(Token {
kind,
line,
space: k == 0 && this_sp,
});
}
}
}
i = j;
continue;
}
}
let start = i;
while i < b.len() && (b[i].is_ascii_digit() || b[i] == b'_') {
i += 1;
}
let mut is_float = false;
if i + 1 < b.len() && b[i] == b'.' && b[i + 1].is_ascii_digit() {
is_float = true;
i += 1;
while i < b.len() && (b[i].is_ascii_digit() || b[i] == b'_') {
i += 1;
}
}
if i < b.len() && (b[i] == b'e' || b[i] == b'E') {
is_float = true;
i += 1;
if i < b.len() && (b[i] == b'+' || b[i] == b'-') {
i += 1;
}
while i < b.len() && b[i].is_ascii_digit() {
i += 1;
}
}
let raw: String = src[start..i].chars().filter(|c| *c != '_').collect();
let num_suffix = |at: usize| -> Option<(&'static str, usize)> {
let two = b.get(at) == Some(&b'r') && b.get(at + 1) == Some(&b'i');
let (tag, len): (&'static str, usize) = if two {
("ri", 2)
} else if b.get(at) == Some(&b'r') {
("r", 1)
} else if b.get(at) == Some(&b'i') {
("i", 1)
} else {
return None;
};
if b.get(at + len)
.is_some_and(|c| c.is_ascii_alphanumeric() || *c == b'_')
{
return None;
}
Some((tag, len))
};
if is_float {
let plain = !raw.contains(['e', 'E']);
if let Some((tag, len)) = num_suffix(i).filter(|_| plain) {
{
let (int_part, frac) = raw.split_once('.').unwrap_or((&raw, ""));
let den = 10i64.pow(frac.len() as u32);
let digits = format!("{int_part}{frac}");
let num: i64 = digits
.parse()
.map_err(|_| format!("bad rational literal: {raw}"))?;
i += len;
let this_sp = core::mem::take(&mut sp);
let mut synth = vec![
Tok::Ident("Rational".into()),
Tok::Op("(".into()),
Tok::Int(num),
Tok::Op(",".into()),
Tok::Int(den),
Tok::Op(")".into()),
];
if tag != "r" {
if tag == "i" {
synth = vec![Tok::Float(
raw.parse().map_err(|_| format!("bad float: {raw}"))?,
)];
}
let mut wrapped = vec![
Tok::Ident("Complex".into()),
Tok::Op("(".into()),
Tok::Int(0),
Tok::Op(",".into()),
];
wrapped.extend(synth);
wrapped.push(Tok::Op(")".into()));
synth = wrapped;
}
for (k, kind) in synth.into_iter().enumerate() {
out.push(Token {
kind,
line,
space: if k == 0 { this_sp } else { false },
});
}
continue;
}
}
let x: f64 = raw.parse().map_err(|_| format!("bad float: {raw}"))?;
out.push(Token {
kind: Tok::Float(x),
line,
space: core::mem::take(&mut sp),
});
} else {
let n_res = raw.parse::<i64>();
let suffix = num_suffix(i);
let this_sp = core::mem::take(&mut sp);
let synth: Vec<Tok> = match (suffix.map(|(t, _)| t), n_res) {
(Some("r"), Ok(n)) => vec![
Tok::Ident("Rational".into()),
Tok::Op("(".into()),
Tok::Int(n),
Tok::Op(")".into()),
],
(Some("i"), Ok(n)) => vec![
Tok::Ident("Complex".into()),
Tok::Op("(".into()),
Tok::Int(0),
Tok::Op(",".into()),
Tok::Int(n),
Tok::Op(")".into()),
],
(Some("ri"), Ok(n)) => vec![
Tok::Ident("Complex".into()),
Tok::Op("(".into()),
Tok::Int(0),
Tok::Op(",".into()),
Tok::Ident("Rational".into()),
Tok::Op("(".into()),
Tok::Int(n),
Tok::Op(")".into()),
Tok::Op(")".into()),
],
(_, Ok(n)) => vec![Tok::Int(n)],
(_, Err(_)) => vec![
Tok::Str(raw.clone(), false),
Tok::Op(".".into()),
Tok::Ident("to_i".into()),
Tok::Op("(".into()),
Tok::Int(10),
Tok::Op(")".into()),
],
};
if let Some((_, len)) = suffix {
i += len;
}
for (k, kind) in synth.into_iter().enumerate() {
out.push(Token {
kind,
line,
space: if k == 0 { this_sp } else { false },
});
}
}
}
b'"' | b'\'' | b'`' if c != b'`' || !method_name_pos(&out) => {
let quote = c;
let dq = quote != b'\'';
i += 1;
let mut s = String::new();
loop {
if i >= b.len() {
return Err(format!("unterminated string at line {line}"));
}
let ch = b[i];
if ch == quote {
i += 1;
break;
}
if ch == b'\\' && i + 1 < b.len() {
let n = b[i + 1];
let cl = utf8_len(n);
match n {
b'\'' if !dq => s.push('\''),
b'\\' if !dq => s.push('\\'),
_ => {
s.push('\\');
s.push_str(&src[i + 1..i + 1 + cl]);
}
}
i += 1 + cl;
continue;
}
if dq && ch == b'#' && i + 1 < b.len() && b[i + 1] == b'{' {
s.push('#');
s.push('{');
i += 2;
let mut depth = 1u32;
let mut last: u8 = b'{';
while i < b.len() && depth > 0 {
let d = b[i];
match d {
b'{' => depth += 1,
b'}' => depth -= 1,
b'"' | b'\'' => {
let q = d;
s.push(q as char);
i += 1;
while i < b.len() && b[i] != q {
if b[i] == b'\\' && i + 1 < b.len() {
s.push('\\');
let cl = utf8_len(b[i + 1]);
s.push_str(&src[i + 1..i + 1 + cl]);
i += 1 + cl;
continue;
}
if b[i] == b'\n' {
line += 1;
}
let cl = utf8_len(b[i]);
s.push_str(&src[i..i + cl]);
i += cl;
}
if i < b.len() {
s.push(q as char);
i += 1;
}
last = q;
continue;
}
b'/' if matches!(
last,
b'{' | b'('
| b'['
| b','
| b';'
| b':'
| b'='
| b'<'
| b'>'
| b'!'
| b'&'
| b'|'
| b'^'
| b'~'
| b'+'
| b'-'
| b'*'
| b'/'
| b'%'
| b'?'
) =>
{
s.push('/');
i += 1;
let mut in_class = false;
while i < b.len() {
let c = b[i];
if c == b'\\' && i + 1 < b.len() {
s.push('\\');
let cl = utf8_len(b[i + 1]);
s.push_str(&src[i + 1..i + 1 + cl]);
i += 1 + cl;
continue;
}
if c == b'[' {
in_class = true;
} else if c == b']' {
in_class = false;
} else if c == b'/' && !in_class {
break;
}
if c == b'\n' {
line += 1;
}
let cl = utf8_len(c);
s.push_str(&src[i..i + cl]);
i += cl;
}
if i < b.len() {
s.push('/');
i += 1;
}
while i < b.len() && b[i].is_ascii_alphabetic() {
s.push(b[i] as char);
i += 1;
}
last = b'/';
continue;
}
b'\n' => line += 1,
_ => {}
}
if depth == 0 {
s.push('}');
i += 1;
break;
}
if !d.is_ascii_whitespace() {
last = d;
}
let cl = utf8_len(d);
s.push_str(&src[i..i + cl]);
i += cl;
}
continue;
}
if ch == b'\n' {
line += 1;
}
let clen = utf8_len(ch);
s.push_str(&src[i..i + clen]);
i += clen;
}
if quote == b'`' {
push_xstring(&mut out, Tok::Str(s, true), line, core::mem::take(&mut sp));
continue;
}
out.push(Token {
kind: Tok::Str(s, dq),
line,
space: core::mem::take(&mut sp),
});
}
b'@' => {
i += 1;
let class_var = i < b.len() && b[i] == b'@';
if class_var {
i += 1;
}
let start = i;
while i < b.len() && ident_byte(b[i]) {
i += 1;
}
let name = src[start..i].to_string();
out.push(Token {
kind: if class_var {
Tok::CVar(name)
} else {
Tok::IVar(name)
},
line,
space: core::mem::take(&mut sp),
});
}
b'$' => {
i += 1;
let start = i;
if i + 1 < b.len() && b[i] == b'-' {
i += 2;
} else if i < b.len()
&& matches!(
b[i],
b'~' | b'&' | b'`' | b'\'' | b'+' | b',' | b'\\' | b';' | b'/' | b'!'
| b':' | b'"'
| b'?' | b'$' | b'*' | b'@' | b'<' | b'>'
)
{
i += 1;
} else {
while i < b.len() && ident_byte(b[i]) {
i += 1;
}
}
out.push(Token {
kind: Tok::GVar(src[start..i].to_string()),
line,
space: core::mem::take(&mut sp),
});
}
b'%' if i + 2 < b.len()
&& matches!(b[i + 1], b'q' | b'Q' | b'r' | b's' | b'x')
&& !b[i + 2].is_ascii_alphanumeric()
&& !matches!(b[i + 2], b' ' | b'\t' | b'\n' | b'=')
&& !method_name_pos(&out) =>
{
let letter = b[i + 1];
let open = b[i + 2];
let close = match open {
b'[' => b']',
b'(' => b')',
b'{' => b'}',
b'<' => b'>',
other => other,
};
i += 3;
let start = i;
let mut depth = 1u32;
while i < b.len() {
if b[i] == b'\\' && i + 1 < b.len() {
i += 2; continue;
}
if b[i] == b'\n' {
line += 1;
}
if b[i] == open && open != close {
depth += 1;
} else if b[i] == close {
depth -= 1;
if depth == 0 {
break;
}
}
i += 1;
}
let body = src[start..i.min(b.len())].to_string();
i += 1; let kind = match letter {
b'q' => Tok::Str(body, false),
b'Q' => Tok::Str(body, true),
b's' => Tok::Symbol(body),
b'x' => {
push_xstring(
&mut out,
Tok::Str(body, true),
line,
core::mem::take(&mut sp),
);
continue;
}
_ => {
let fstart = i;
while i < b.len() && matches!(b[i], b'i' | b'm' | b'x' | b'o' | b'u' | b'n')
{
i += 1;
}
Tok::Regex(body, src[fstart..i].to_string())
}
};
out.push(Token {
kind,
line,
space: core::mem::take(&mut sp),
});
}
b'%' if i + 2 < b.len()
&& matches!(b[i + 1], b'w' | b'i' | b'W' | b'I')
&& matches!(b[i + 2], b'[' | b'(' | b'{' | b'<' | b'|')
&& !method_name_pos(&out) =>
{
let symbols = matches!(b[i + 1], b'i' | b'I');
let interp = matches!(b[i + 1], b'W' | b'I');
let open = b[i + 2];
let close = match open {
b'[' => b']',
b'(' => b')',
b'{' => b'}',
b'<' => b'>',
other => other,
};
i += 3;
let start = i;
let mut depth = 1u32;
while i < b.len() {
if b[i] == b'\\' && i + 1 < b.len() {
if b[i + 1] == b'\n' {
line += 1;
}
i += 2;
continue;
}
if b[i] == b'\n' {
line += 1;
}
if b[i] == open && open != close {
depth += 1;
} else if b[i] == close {
depth -= 1;
if depth == 0 {
break;
}
}
i += 1;
}
let body = &src[start..i.min(b.len())];
i += 1; let words = percent_words(body, interp, open, close);
out.push(Token {
kind: Tok::Op("[".into()),
line,
space: core::mem::take(&mut sp),
});
for (idx, w) in words.iter().enumerate() {
if idx > 0 {
out.push(Token {
kind: Tok::Op(",".into()),
line,
space: false,
});
}
let kind = match (symbols, interp) {
(true, false) => Tok::Symbol(w.clone()),
(true, true) => Tok::DSymbol(w.clone()),
(false, i) => Tok::Str(w.clone(), i),
};
out.push(Token {
kind,
line,
space: false,
});
}
out.push(Token {
kind: Tok::Op("]".into()),
line,
space: false,
});
}
b'%' if i + 1 < b.len()
&& matches!(b[i + 1], b'(' | b'{' | b'[' | b'<')
&& percent_string_start(&out, sp)
&& !method_name_pos(&out) =>
{
let open = b[i + 1];
let close = match open {
b'[' => b']',
b'(' => b')',
b'{' => b'}',
_ => b'>',
};
i += 2;
let start = i;
let mut depth = 1u32;
while i < b.len() {
if b[i] == b'\\' && i + 1 < b.len() {
i += 2; continue;
}
if b[i] == b'\n' {
line += 1;
}
if b[i] == open && open != close {
depth += 1;
} else if b[i] == close {
depth -= 1;
if depth == 0 {
break;
}
}
i += 1;
}
let body = src[start..i.min(b.len())].to_string();
i += 1; out.push(Token {
kind: Tok::Str(body, true),
line,
space: core::mem::take(&mut sp),
});
}
b'?' if (sp || !prev_is_value(&out))
&& i + 1 < b.len()
&& (b[i + 1] == b'\\'
|| (b[i + 1].is_ascii_alphanumeric()
&& (i + 2 >= b.len()
|| !(b[i + 2].is_ascii_alphanumeric() || b[i + 2] == b'_')))
|| (b[i + 1].is_ascii_graphic() && !b[i + 1].is_ascii_alphanumeric())) =>
{
i += 1;
let ch = if b[i] == b'\\' && i + 1 < b.len() {
i += 1;
let e = b[i];
i += 1;
match e {
b'n' => '\n',
b't' => '\t',
b'r' => '\r',
b's' => ' ',
b'0' => '\0',
b'e' => '\x1b',
other => other as char,
}
} else {
let c = b[i] as char;
i += 1;
c
};
out.push(Token {
kind: Tok::Str(ch.to_string(), false),
line,
space: core::mem::take(&mut sp),
});
}
b':' if i + 1 < b.len()
&& (b[i + 1] == b'"' || b[i + 1] == b'\'')
&& (!prev_is_value(&out) || sp)
&& !glued_keyword_label(&out, sp) =>
{
let quote = b[i + 1];
i += 2;
let mut bytes: Vec<u8> = Vec::new();
let mut has_interp = false;
while i < b.len() && b[i] != quote {
if quote == b'"' && b[i] == b'#' && i + 1 < b.len() && b[i + 1] == b'{' {
has_interp = true;
bytes.push(b'#');
bytes.push(b'{');
i += 2;
let mut depth = 1u32;
while i < b.len() && depth > 0 {
match b[i] {
b'{' => depth += 1,
b'}' => depth -= 1,
q @ (b'"' | b'\'') => {
bytes.push(q);
i += 1;
while i < b.len() && b[i] != q {
if b[i] == b'\\' && i + 1 < b.len() {
bytes.push(b[i]);
bytes.push(b[i + 1]);
i += 2;
continue;
}
bytes.push(b[i]);
i += 1;
}
if i < b.len() {
bytes.push(q);
i += 1;
}
continue;
}
_ => {}
}
bytes.push(b[i]);
i += 1;
}
continue;
}
if quote == b'"' && b[i] == b'\\' && i + 1 < b.len() {
bytes.push(b'\\');
bytes.push(b[i + 1]);
i += 2;
} else {
bytes.push(b[i]);
i += 1;
}
}
i += 1; let body = String::from_utf8_lossy(&bytes).into_owned();
let has_interp =
has_interp || (quote == b'"' && crate::parser::has_sigil_interp(&body));
let kind = if has_interp {
Tok::DSymbol(body)
} else if quote == b'"' {
match decode_dquote_escapes(&body) {
Some(name) => Tok::Symbol(name),
None => return Err(format!("invalid symbol at line {line}")),
}
} else {
Tok::Symbol(body)
};
out.push(Token {
kind,
line,
space: core::mem::take(&mut sp),
});
}
b':' if i + 1 < b.len()
&& op_symbol_at(&src[i + 1..]).is_some()
&& (!prev_is_value(&out) || sp)
&& !glued_keyword_label(&out, sp) =>
{
let op = op_symbol_at(&src[i + 1..]).unwrap();
i += 1 + op.len();
out.push(Token {
kind: Tok::Symbol(op.to_string()),
line,
space: core::mem::take(&mut sp),
});
}
b':' if i + 1 < b.len()
&& (ident_start(b[i + 1])
|| b[i + 1] == b'@'
|| b[i + 1] == b'$')
&& (!prev_is_value(&out) || sp)
&& !glued_keyword_label(&out, sp) =>
{
i += 1;
let start = i;
while i < b.len() && b[i] == b'@' {
i += 1;
}
if i < b.len() && b[i] == b'$' {
i += 1;
}
while i < b.len() && ident_byte(b[i]) {
i += 1;
}
if i < b.len()
&& (((b[i] == b'?' || b[i] == b'!') && b.get(i + 1) != Some(&b'='))
|| (b[i] == b'='
&& !matches!(b.get(i + 1), Some(b'=') | Some(b'~') | Some(b'>'))))
{
i += 1;
}
out.push(Token {
kind: Tok::Symbol(src[start..i].to_string()),
line,
space: core::mem::take(&mut sp),
});
}
b'_' if (i == 0 || b[i - 1] == b'\n')
&& src[i..].starts_with("__END__")
&& matches!(b.get(i + 7), None | Some(b'\n') | Some(b'\r')) =>
{
break;
}
_ if ident_start(c) => {
let start = i;
while i < b.len() && ident_byte(b[i]) {
i += 1;
}
if i < b.len() && (b[i] == b'?' || b[i] == b'!') && b.get(i + 1) != Some(&b'=') {
i += 1;
}
let word = &src[start..i];
let kind = if KEYWORDS.contains(&word) {
Tok::Keyword(word.to_string())
} else if word.chars().next().is_some_and(char::is_uppercase) {
Tok::Const(word.to_string())
} else {
Tok::Ident(word.to_string())
};
out.push(Token {
kind,
line,
space: core::mem::take(&mut sp),
});
}
b'/' if regex_start(&out, sp, b.get(i + 1).copied()) => {
i += 1;
let mut pat = String::new();
let mut in_class = false;
while i < b.len() {
let ch = b[i];
if ch == b'\\' && i + 1 < b.len() {
pat.push('\\');
let cl = utf8_len(b[i + 1]);
pat.push_str(&src[i + 1..i + 1 + cl]);
i += 1 + cl;
continue;
}
if ch == b'#' && i + 1 < b.len() && b[i + 1] == b'{' {
pat.push_str("#{");
i += 2;
let mut depth = 1;
while i < b.len() && depth > 0 {
match b[i] {
b'{' => depth += 1,
b'}' => depth -= 1,
b'\n' => line += 1,
_ => {}
}
let cl = utf8_len(b[i]);
pat.push_str(&src[i..i + cl]);
i += cl;
}
continue;
}
match ch {
b'[' => in_class = true,
b']' => in_class = false,
b'/' if !in_class => break,
b'\n' => line += 1,
_ => {}
}
let cl = utf8_len(ch);
pat.push_str(&src[i..i + cl]);
i += cl;
}
if i < b.len() {
i += 1;
}
let fstart = i;
while i < b.len() && matches!(b[i], b'i' | b'm' | b'x' | b'o' | b'u' | b'n') {
i += 1;
}
let flags = src[fstart..i].to_string();
out.push(Token {
kind: Tok::Regex(pat, flags),
line,
space: core::mem::take(&mut sp),
});
}
b'<' if i + 2 < b.len() && b[i + 1] == b'<' && heredoc_here(&out, &b[i + 2..], sp) => {
let mut j = i + 2;
let squiggly = b[j] == b'~';
let dash = b[j] == b'-';
if squiggly || dash {
j += 1;
}
let (quote, interp) = match b.get(j) {
Some(b'\'') => (Some(b'\''), false),
Some(b'"') => (Some(b'"'), true),
Some(b'`') => (Some(b'`'), true),
_ => (None, true),
};
if quote.is_some() {
j += 1;
}
let delim_start = j;
while j < b.len() && (b[j].is_ascii_alphanumeric() || b[j] == b'_') {
j += 1;
}
let delim = src[delim_start..j].to_string();
if let Some(q) = quote {
if b.get(j) == Some(&q) {
j += 1;
}
}
if quote == Some(b'`') {
push_xstring(
&mut out,
Tok::Str(String::new(), true),
line,
core::mem::take(&mut sp),
);
let close = out.pop().expect("push_xstring pushed `)`");
pending.push(Heredoc {
delim,
squiggly,
indent_term: squiggly || dash,
interp,
tok_idx: out.len() - 1,
});
out.push(close);
i = j;
continue;
}
out.push(Token {
kind: Tok::Str(String::new(), interp),
line,
space: core::mem::take(&mut sp),
});
pending.push(Heredoc {
delim,
squiggly,
indent_term: squiggly || dash,
interp,
tok_idx: out.len() - 1,
});
i = j;
}
_ => {
let three = if i + 3 <= b.len() && src.is_char_boundary(i + 3) {
&src[i..i + 3]
} else {
""
};
let two = if i + 2 <= b.len() && src.is_char_boundary(i + 2) {
&src[i..i + 2]
} else {
""
};
let op: &str = if matches!(
three,
"**=" | "<=>" | "===" | "..." | "&&=" | "||=" | "<<=" | ">>="
) {
three
} else if matches!(
two,
"**" | "=="
| "!="
| "<="
| ">="
| "&&"
| "||"
| "+="
| "-="
| "*="
| "/="
| "%="
| "<<"
| ">>"
| ".."
| "::"
| "=>"
| "|="
| "&="
| "^="
| "->"
| "=~"
| "!~"
| "&."
) {
two
} else {
match c {
b'+' | b'-' | b'*' | b'/' | b'%' | b'=' | b'<' | b'>' | b'!' | b'&'
| b'|' | b'^' | b'~' | b'(' | b')' | b'[' | b']' | b'{' | b'}' | b','
| b'.' | b':' | b'?' | b'`' => &src[i..i + 1],
other => {
return Err(format!(
"unexpected character '{}' at line {line}",
other as char
))
}
}
};
out.push(Token {
kind: Tok::Op(op.to_string()),
line,
space: core::mem::take(&mut sp),
});
i += op.len();
}
}
}
out.push(Token {
kind: Tok::Eof,
line,
space: core::mem::take(&mut sp),
});
Ok(out)
}
fn push_xstring(out: &mut Vec<Token>, body: Tok, line: u32, space: bool) {
let tok = |kind, space| Token { kind, line, space };
out.push(tok(Tok::Ident("`".to_string()), space));
out.push(tok(Tok::Op("(".to_string()), false));
out.push(tok(body, false));
out.push(tok(Tok::Op(")".to_string()), false));
}
fn method_name_pos(out: &[Token]) -> bool {
match out
.iter()
.rev()
.find(|t| t.kind != Tok::Newline)
.map(|t| &t.kind)
{
Some(Tok::Keyword(k)) if k == "def" => true,
Some(Tok::Op(o)) => o == "." || o == "::" || o == "&.",
_ => false,
}
}
fn regex_start(out: &[Token], sp: bool, next: Option<u8>) -> bool {
if method_name_pos(out) {
return false;
}
let prev = out.iter().rev().find(|t| t.kind != Tok::Newline);
let prev_is_value = match prev.map(|t| &t.kind) {
None => false,
Some(Tok::Int(_))
| Some(Tok::Float(_))
| Some(Tok::Str(_, _))
| Some(Tok::Regex(_, _))
| Some(Tok::Ident(_))
| Some(Tok::Const(_))
| Some(Tok::IVar(_))
| Some(Tok::GVar(_))
| Some(Tok::Symbol(_)) => true,
Some(Tok::Op(o)) => o == ")" || o == "]" || o == "}",
Some(Tok::Keyword(k)) => matches!(k.as_str(), "self" | "end" | "true" | "false" | "nil"),
_ => false,
};
if !prev_is_value {
return true;
}
if next == Some(b'=') {
return false;
}
sp && !matches!(next, Some(b' ') | Some(b'\t') | None)
}
const OP_SYMBOLS: &[&str] = &[
"<=>", "===", "[]=", "**", "==", "!=", "!~", "<=", ">=", "<<", ">>", "=~", "[]", "-@", "+@",
"+", "-", "*", "/", "%", "<", ">", "&", "|", "^", "~", "!", "`",
];
fn op_symbol_at(s: &str) -> Option<&'static str> {
OP_SYMBOLS.iter().find(|op| s.starts_with(**op)).copied()
}
fn decode_dquote_escapes(body: &str) -> Option<String> {
crate::parser::decode_dquote_body(body)
}
fn percent_string_start(out: &[Token], sp: bool) -> bool {
if matches!(
out.last().map(|t| &t.kind),
None | Some(Tok::Newline) | Some(Tok::Semicolon)
) {
return true;
}
let prev = out.iter().rev().find(|t| t.kind != Tok::Newline);
match prev.map(|t| &t.kind) {
Some(Tok::Ident(_)) | Some(Tok::Const(_)) => sp,
_ => !prev_is_value(out),
}
}
fn glued_keyword_label(out: &[Token], sp: bool) -> bool {
!sp && matches!(out.last().map(|t| &t.kind), Some(Tok::Keyword(_)))
}
fn prev_is_value(out: &[Token]) -> bool {
let prev = out.iter().rev().find(|t| t.kind != Tok::Newline);
match prev.map(|t| &t.kind) {
Some(Tok::Int(_))
| Some(Tok::Float(_))
| Some(Tok::Str(_, _))
| Some(Tok::Regex(_, _))
| Some(Tok::Ident(_))
| Some(Tok::Const(_))
| Some(Tok::IVar(_))
| Some(Tok::GVar(_))
| Some(Tok::Symbol(_)) => true,
Some(Tok::Op(o)) => o == ")" || o == "]" || o == "}",
Some(Tok::Keyword(k)) => matches!(k.as_str(), "self" | "end" | "true" | "false" | "nil"),
_ => false,
}
}
fn utf8_len(b: u8) -> usize {
match b {
0x00..=0x7F => 1,
0xC0..=0xDF => 2,
0xE0..=0xEF => 3,
_ => 4,
}
}
fn percent_words(body: &str, interp: bool, open: u8, close: u8) -> Vec<String> {
let mut words = Vec::new();
let mut cur = String::new();
let mut in_word = false;
let mut chars = body.chars();
while let Some(c) = chars.next() {
if c == '\\' {
in_word = true;
match chars.next() {
Some(n) if n.is_whitespace() => cur.push(n),
Some(n) if n == open as char || n == close as char || (!interp && n == '\\') => {
cur.push(n)
}
Some(n) => {
cur.push('\\');
cur.push(n);
}
None => cur.push('\\'),
}
} else if c.is_whitespace() {
if in_word {
words.push(std::mem::take(&mut cur));
in_word = false;
}
} else {
in_word = true;
cur.push(c);
}
}
if in_word {
words.push(cur);
}
words
}