use std::fmt;
#[derive(Debug, Clone, PartialEq)]
pub struct Token {
pub kind: Tok,
pub line: u32,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Tok {
Int(i64),
Long(i64),
Float(f64),
Float32(f64),
Str(String),
Char(char),
Ident(String),
Class,
Public,
Static,
Void,
If,
Else,
While,
For,
Do,
Switch,
Case,
Default,
Return,
Break,
Continue,
True,
False,
New,
LBrace,
RBrace,
LParen,
RParen,
LBracket,
RBracket,
Semi,
Comma,
Dot,
Question,
Colon,
Arrow,
ColonColon,
Assign,
PlusAssign,
MinusAssign,
StarAssign,
SlashAssign,
PercentAssign,
Plus,
Minus,
Star,
Slash,
Percent,
PlusPlus,
MinusMinus,
EqEq,
NotEq,
Lt,
Gt,
Le,
Ge,
AndAnd,
OrOr,
Not,
Amp,
Pipe,
Caret,
Tilde,
Shl,
Shr,
Ushr,
AmpAssign,
PipeAssign,
CaretAssign,
ShlAssign,
ShrAssign,
UshrAssign,
Eof,
}
impl Tok {
pub fn generic_closers(&self) -> i32 {
match self {
Tok::Gt => 1,
Tok::Shr => 2,
Tok::Ushr => 3,
_ => 0,
}
}
}
impl fmt::Display for Tok {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{self:?}")
}
}
pub fn lex(src: &str) -> Result<Vec<Token>, String> {
let bytes = src.as_bytes();
let mut i = 0usize;
let mut line = 1u32;
let mut out = Vec::new();
while i < bytes.len() {
let c = bytes[i] as char;
if c == '\n' {
line += 1;
i += 1;
continue;
}
if c.is_whitespace() {
i += 1;
continue;
}
if c == '/' && i + 1 < bytes.len() {
match bytes[i + 1] as char {
'/' => {
i += 2;
while i < bytes.len() && bytes[i] != b'\n' {
i += 1;
}
continue;
}
'*' => {
i += 2;
while i + 1 < bytes.len() && !(bytes[i] == b'*' && bytes[i + 1] == b'/') {
if bytes[i] == b'\n' {
line += 1;
}
i += 1;
}
i += 2; continue;
}
_ => {}
}
}
if c.is_ascii_alphabetic() || c == '_' || c == '$' {
let start = i;
while i < bytes.len() {
let ch = bytes[i] as char;
if ch.is_ascii_alphanumeric() || ch == '_' || ch == '$' {
i += 1;
} else {
break;
}
}
let word = &src[start..i];
out.push(Token {
kind: keyword_or_ident(word),
line,
});
continue;
}
if c == '0'
&& i + 1 < bytes.len()
&& matches!(bytes[i + 1], b'x' | b'X' | b'b' | b'B')
&& i + 2 < bytes.len()
&& (bytes[i + 2] as char).is_ascii_alphanumeric()
{
let radix = if matches!(bytes[i + 1], b'x' | b'X') {
16
} else {
2
};
i += 2;
let start = i;
while i < bytes.len()
&& ((bytes[i] as char).is_ascii_alphanumeric() || bytes[i] == b'_')
{
i += 1;
}
let mut digits = src[start..i].replace('_', "");
let is_long = digits.ends_with(['L', 'l']);
if is_long {
digits.pop();
}
let bits = u64::from_str_radix(&digits, radix)
.map_err(|_| format!("javars: bad integer literal `{digits}` on line {line}"))?;
let v = if is_long {
bits as i64
} else {
bits as u32 as i32 as i64
};
out.push(Token {
kind: if is_long { Tok::Long(v) } else { Tok::Int(v) },
line,
});
continue;
}
if c.is_ascii_digit() {
let start = i;
let mut is_float = false;
while i < bytes.len() && ((bytes[i] as char).is_ascii_digit() || bytes[i] == b'_') {
i += 1;
}
if i < bytes.len()
&& bytes[i] == b'.'
&& i + 1 < bytes.len()
&& (bytes[i + 1] as char).is_ascii_digit()
{
is_float = true;
i += 1;
while i < bytes.len() && ((bytes[i] as char).is_ascii_digit() || bytes[i] == b'_') {
i += 1;
}
}
if i < bytes.len() && matches!(bytes[i], b'e' | b'E') {
let mut j = i + 1;
if j < bytes.len() && matches!(bytes[j], b'+' | b'-') {
j += 1;
}
if j < bytes.len() && (bytes[j] as char).is_ascii_digit() {
is_float = true;
i = j;
while i < bytes.len() && (bytes[i] as char).is_ascii_digit() {
i += 1;
}
}
}
let num_end = i;
let mut is_long = false;
let mut is_f32 = false;
if i < bytes.len() && matches!(bytes[i], b'L' | b'l' | b'f' | b'F' | b'd' | b'D') {
match bytes[i] {
b'f' | b'F' => {
is_float = true;
is_f32 = true;
}
b'd' | b'D' => is_float = true,
_ => is_long = true,
}
i += 1;
}
let text = src[start..num_end].replace('_', "");
let text = text.as_str();
if is_float {
let v: f64 = text
.parse()
.map_err(|_| format!("javars: bad float literal `{text}` on line {line}"))?;
out.push(Token {
kind: if is_f32 {
Tok::Float32(v)
} else {
Tok::Float(v)
},
line,
});
} else {
let v: i64 = if text.len() > 1 && text.starts_with('0') {
i64::from_str_radix(&text[1..], 8)
.map_err(|_| format!("javars: bad octal literal `{text}` on line {line}"))?
} else {
text.parse().map_err(|_| {
format!("javars: bad integer literal `{text}` on line {line}")
})?
};
out.push(Token {
kind: if is_long { Tok::Long(v) } else { Tok::Int(v) },
line,
});
}
continue;
}
if c == '"' && src[i..].starts_with("\"\"\"") {
let (block, consumed, lines) = text_block(src, i, line)?;
i += consumed;
out.push(Token {
kind: Tok::Str(block),
line,
});
line += lines;
continue;
}
if c == '"' {
i += 1;
let mut s = String::new();
while i < bytes.len() && bytes[i] != b'"' {
if bytes[i] == b'\\' && i + 1 < bytes.len() {
i += 1;
s.push(unescape(bytes[i] as char));
i += 1;
} else {
let ch = src[i..].chars().next().unwrap();
if ch == '\n' {
line += 1;
}
s.push(ch);
i += ch.len_utf8();
}
}
if i >= bytes.len() {
return Err(format!(
"javars: unterminated string literal on line {line}"
));
}
i += 1; out.push(Token {
kind: Tok::Str(s),
line,
});
continue;
}
if c == '\'' {
i += 1;
let ch = if bytes[i] == b'\\' {
i += 1;
let c = unescape(bytes[i] as char);
i += 1;
c
} else {
let c = src[i..].chars().next().unwrap();
i += c.len_utf8();
c
};
if i >= bytes.len() || bytes[i] != b'\'' {
return Err(format!("javars: unterminated char literal on line {line}"));
}
i += 1;
out.push(Token {
kind: Tok::Char(ch),
line,
});
continue;
}
let two = if i + 1 < bytes.len() {
&src[i..i + 2]
} else {
""
};
let three = src.get(i..i + 3).unwrap_or("");
let four = src.get(i..i + 4).unwrap_or("");
let (kind, adv) = match four {
">>>=" => (Tok::UshrAssign, 4),
_ => match three {
">>>" => (Tok::Ushr, 3),
">>=" => (Tok::ShrAssign, 3),
"<<=" => (Tok::ShlAssign, 3),
_ => lex_short(two, c, line)?,
},
};
out.push(Token { kind, line });
i += adv;
}
out.push(Token {
kind: Tok::Eof,
line,
});
Ok(out)
}
fn lex_short(two: &str, c: char, line: u32) -> Result<(Tok, usize), String> {
Ok(match two {
"+=" => (Tok::PlusAssign, 2),
"-=" => (Tok::MinusAssign, 2),
"*=" => (Tok::StarAssign, 2),
"/=" => (Tok::SlashAssign, 2),
"%=" => (Tok::PercentAssign, 2),
"++" => (Tok::PlusPlus, 2),
"--" => (Tok::MinusMinus, 2),
"==" => (Tok::EqEq, 2),
"!=" => (Tok::NotEq, 2),
"<=" => (Tok::Le, 2),
">=" => (Tok::Ge, 2),
"&&" => (Tok::AndAnd, 2),
"||" => (Tok::OrOr, 2),
"&=" => (Tok::AmpAssign, 2),
"|=" => (Tok::PipeAssign, 2),
"^=" => (Tok::CaretAssign, 2),
"<<" => (Tok::Shl, 2),
">>" => (Tok::Shr, 2),
"->" => (Tok::Arrow, 2),
"::" => (Tok::ColonColon, 2),
_ => match c {
'{' => (Tok::LBrace, 1),
'}' => (Tok::RBrace, 1),
'(' => (Tok::LParen, 1),
')' => (Tok::RParen, 1),
'[' => (Tok::LBracket, 1),
']' => (Tok::RBracket, 1),
';' => (Tok::Semi, 1),
',' => (Tok::Comma, 1),
'.' => (Tok::Dot, 1),
'?' => (Tok::Question, 1),
':' => (Tok::Colon, 1),
'=' => (Tok::Assign, 1),
'+' => (Tok::Plus, 1),
'-' => (Tok::Minus, 1),
'*' => (Tok::Star, 1),
'/' => (Tok::Slash, 1),
'%' => (Tok::Percent, 1),
'<' => (Tok::Lt, 1),
'>' => (Tok::Gt, 1),
'!' => (Tok::Not, 1),
'&' => (Tok::Amp, 1),
'|' => (Tok::Pipe, 1),
'^' => (Tok::Caret, 1),
'~' => (Tok::Tilde, 1),
other => {
return Err(format!(
"javars: unexpected character `{other}` on line {line}"
))
}
},
})
}
fn keyword_or_ident(word: &str) -> Tok {
match word {
"class" => Tok::Class,
"public" => Tok::Public,
"static" => Tok::Static,
"void" => Tok::Void,
"if" => Tok::If,
"else" => Tok::Else,
"while" => Tok::While,
"for" => Tok::For,
"do" => Tok::Do,
"switch" => Tok::Switch,
"case" => Tok::Case,
"default" => Tok::Default,
"return" => Tok::Return,
"break" => Tok::Break,
"continue" => Tok::Continue,
"true" => Tok::True,
"false" => Tok::False,
"new" => Tok::New,
_ => Tok::Ident(word.to_string()),
}
}
fn unescape(c: char) -> char {
match c {
'n' => '\n',
't' => '\t',
'r' => '\r',
'0' => '\0',
'\\' => '\\',
'"' => '"',
'\'' => '\'',
other => other,
}
}
fn text_block(src: &str, start: usize, line: u32) -> Result<(String, usize, u32), String> {
let after_open = start + 3;
let rest = &src[after_open..];
let header = rest
.find('\n')
.ok_or_else(|| format!("javars: unterminated text block on line {line}"))?;
if rest[..header].trim().is_empty() {
} else {
return Err(format!(
"javars: a text block's opening `\"\"\"` must be followed by a line terminator (line {line})"
));
}
let body_start = after_open + header + 1;
let body_end = src[body_start..]
.find("\"\"\"")
.map(|k| body_start + k)
.ok_or_else(|| format!("javars: unterminated text block on line {line}"))?;
let body = &src[body_start..body_end];
let newlines = body.matches('\n').count() as u32 + 1;
let mut lines: Vec<&str> = body.split('\n').map(|l| l.trim_end_matches('\r')).collect();
let closer_alone = lines
.last()
.is_some_and(|l| l.chars().all(|c| c == ' ' || c == '\t'));
let indent_of = |l: &str| l.len() - l.trim_start_matches([' ', '\t']).len();
let mut indent = usize::MAX;
for (n, l) in lines.iter().enumerate() {
let last = n + 1 == lines.len();
if l.trim().is_empty() && !(last && closer_alone) {
continue;
}
indent = indent.min(indent_of(l));
}
if indent == usize::MAX {
indent = 0;
}
if closer_alone {
lines.pop();
}
let trailing_newline = closer_alone && !lines.is_empty();
let mut value = String::new();
for (n, l) in lines.iter().enumerate() {
if n > 0 {
value.push('\n');
}
let stripped = if l.len() >= indent { &l[indent..] } else { "" };
value.push_str(stripped.trim_end_matches([' ', '\t']));
}
if trailing_newline {
value.push('\n');
}
Ok((unescape_block(&value), body_end + 3 - start, newlines))
}
fn unescape_block(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut chars = s.chars();
while let Some(c) = chars.next() {
if c != '\\' {
out.push(c);
continue;
}
match chars.next() {
Some('\n') => {}
Some('s') => out.push(' '),
Some(e) => out.push(unescape(e)),
None => out.push('\\'),
}
}
out
}