use std::borrow::Cow;
use std::fmt;
#[derive(Debug, Clone, PartialEq)]
pub struct Token {
pub kind: Tok,
pub line: u32,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Tok {
Int(i64),
Long(i64),
Float(f64),
Float32(f64),
Str(String),
Char(char),
Ident(String),
Class,
Public,
Static,
Void,
If,
Else,
While,
For,
Do,
Switch,
Case,
Default,
Return,
Break,
Continue,
True,
False,
New,
LBrace,
RBrace,
LParen,
RParen,
LBracket,
RBracket,
Semi,
Comma,
Dot,
Question,
Colon,
Arrow,
ColonColon,
At,
Assign,
PlusAssign,
MinusAssign,
StarAssign,
SlashAssign,
PercentAssign,
Plus,
Minus,
Star,
Slash,
Percent,
PlusPlus,
MinusMinus,
EqEq,
NotEq,
Lt,
Gt,
Le,
Ge,
AndAnd,
OrOr,
Not,
Amp,
Pipe,
Caret,
Tilde,
Shl,
Shr,
Ushr,
AmpAssign,
PipeAssign,
CaretAssign,
ShlAssign,
ShrAssign,
UshrAssign,
Eof,
}
impl Tok {
pub fn generic_closers(&self) -> i32 {
match self {
Tok::Gt => 1,
Tok::Shr => 2,
Tok::Ushr => 3,
_ => 0,
}
}
}
impl fmt::Display for Tok {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{self:?}")
}
}
pub fn lex(src: &str) -> Result<Vec<Token>, String> {
if let Some((n, c)) = src.lines().enumerate().find_map(|(n, l)| {
l.chars()
.find(|c| crate::jstr::is_reserved(*c as u32))
.map(|c| (n, c))
}) {
return Err(format!(
"{} (line {})",
crate::jstr::reserved_message(c as u32),
n + 1
));
}
lex_translated(&translate_unicode_escapes(src)?)
}
fn translate_unicode_escapes(src: &str) -> Result<Cow<'_, str>, String> {
if !src.contains("\\u") {
return Ok(Cow::Borrowed(src));
}
let cs: Vec<char> = src.chars().collect();
let mut out = String::with_capacity(src.len());
let mut i = 0usize;
let mut line = 1u32;
while i < cs.len() {
if cs[i] != '\\' {
if cs[i] == '\n' {
line += 1;
}
out.push(cs[i]);
i += 1;
continue;
}
let start = i;
while i < cs.len() && cs[i] == '\\' {
i += 1;
}
let run = i - start;
if run % 2 == 0 || i >= cs.len() || cs[i] != 'u' {
out.extend(cs[start..i].iter());
continue;
}
out.extend(std::iter::repeat_n('\\', run - 1));
while i < cs.len() && cs[i] == 'u' {
i += 1;
}
let hex: String = cs.get(i..i + 4).unwrap_or_default().iter().collect();
let value = (hex.len() == 4)
.then(|| u32::from_str_radix(&hex, 16).ok())
.flatten()
.ok_or_else(|| format!("javars: illegal unicode escape on line {line}"))?;
if value == u32::from('\n') {
line += 1;
}
crate::jstr::push_unit(&mut out, value as u16);
i += 4;
}
Ok(Cow::Owned(out))
}
fn lex_translated(src: &str) -> Result<Vec<Token>, String> {
let bytes = src.as_bytes();
let mut i = 0usize;
let mut line = 1u32;
let mut out = Vec::with_capacity(src.len() / 4 + 16);
while i < bytes.len() {
let c = bytes[i] as char;
if c == '\n' {
line += 1;
i += 1;
continue;
}
if c.is_whitespace() {
i += 1;
continue;
}
if c == '/' && i + 1 < bytes.len() {
match bytes[i + 1] as char {
'/' => {
i += 2;
while i < bytes.len() && bytes[i] != b'\n' {
i += 1;
}
continue;
}
'*' => {
i += 2;
while i + 1 < bytes.len() && !(bytes[i] == b'*' && bytes[i + 1] == b'/') {
if bytes[i] == b'\n' {
line += 1;
}
i += 1;
}
i += 2; continue;
}
_ => {}
}
}
if c.is_ascii_alphabetic() || c == '_' || c == '$' {
let start = i;
while i < bytes.len() {
let ch = bytes[i] as char;
if ch.is_ascii_alphanumeric() || ch == '_' || ch == '$' {
i += 1;
} else {
break;
}
}
let word = &src[start..i];
out.push(Token {
kind: keyword_or_ident(word),
line,
});
continue;
}
if c == '0'
&& i + 1 < bytes.len()
&& matches!(bytes[i + 1], b'x' | b'X' | b'b' | b'B')
&& i + 2 < bytes.len()
&& ((bytes[i + 2] as char).is_ascii_alphanumeric()
|| (matches!(bytes[i + 1], b'x' | b'X') && bytes[i + 2] == b'.'))
{
let radix = if matches!(bytes[i + 1], b'x' | b'X') {
16
} else {
2
};
i += 2;
let start = i;
if radix == 16 {
let mut j = i;
while j < bytes.len() && (bytes[j].is_ascii_hexdigit() || bytes[j] == b'_') {
j += 1;
}
if j < bytes.len() && bytes[j] == b'.' {
j += 1;
while j < bytes.len() && (bytes[j].is_ascii_hexdigit() || bytes[j] == b'_') {
j += 1;
}
}
if j < bytes.len() && matches!(bytes[j], b'p' | b'P') {
j += 1;
if j < bytes.len() && matches!(bytes[j], b'+' | b'-') {
j += 1;
}
while j < bytes.len() && (bytes[j].is_ascii_digit() || bytes[j] == b'_') {
j += 1;
}
let text = src[start..j].replace('_', "");
let single = j < bytes.len() && matches!(bytes[j], b'f' | b'F');
if j < bytes.len() && matches!(bytes[j], b'f' | b'F' | b'd' | b'D') {
j += 1;
}
let v = hex_float(&text, single).ok_or_else(|| {
format!("javars: bad floating-point literal `0x{text}` on line {line}")
})?;
i = j;
out.push(Token {
kind: if single {
Tok::Float32(v)
} else {
Tok::Float(v)
},
line,
});
continue;
}
}
while i < bytes.len()
&& ((bytes[i] as char).is_ascii_alphanumeric() || bytes[i] == b'_')
{
i += 1;
}
let mut digits = src[start..i].replace('_', "");
let is_long = digits.ends_with(['L', 'l']);
if is_long {
digits.pop();
}
let bits = u64::from_str_radix(&digits, radix)
.map_err(|_| format!("javars: bad integer literal `{digits}` on line {line}"))?;
let v = if is_long {
bits as i64
} else {
bits as u32 as i32 as i64
};
out.push(Token {
kind: if is_long { Tok::Long(v) } else { Tok::Int(v) },
line,
});
continue;
}
let dot_number = c == '.' && bytes.get(i + 1).is_some_and(u8::is_ascii_digit);
if c.is_ascii_digit() || dot_number {
let start = i;
let mut is_float = false;
while i < bytes.len() && ((bytes[i] as char).is_ascii_digit() || bytes[i] == b'_') {
i += 1;
}
if i < bytes.len()
&& bytes[i] == b'.'
&& i + 1 < bytes.len()
&& (bytes[i + 1] as char).is_ascii_digit()
{
is_float = true;
i += 1;
while i < bytes.len() && ((bytes[i] as char).is_ascii_digit() || bytes[i] == b'_') {
i += 1;
}
}
if !is_float && i < bytes.len() && bytes[i] == b'.' {
let after = bytes.get(i + 1).copied().unwrap_or(b' ');
let after2 = bytes.get(i + 2).copied().unwrap_or(b' ');
let ident = |b: u8| b.is_ascii_alphanumeric() || b == b'_' || b == b'$';
let exp = matches!(after, b'e' | b'E')
&& (after2.is_ascii_digit()
|| (matches!(after2, b'+' | b'-')
&& bytes.get(i + 3).is_some_and(u8::is_ascii_digit)));
let suffix = matches!(after, b'f' | b'F' | b'd' | b'D') && !ident(after2);
if exp || suffix || (!ident(after) && after != b'.') {
is_float = true;
i += 1;
}
}
if i < bytes.len() && matches!(bytes[i], b'e' | b'E') {
let mut j = i + 1;
if j < bytes.len() && matches!(bytes[j], b'+' | b'-') {
j += 1;
}
if j < bytes.len() && (bytes[j] as char).is_ascii_digit() {
is_float = true;
i = j;
while i < bytes.len() && (bytes[i] as char).is_ascii_digit() {
i += 1;
}
}
}
let num_end = i;
let mut is_long = false;
let mut is_f32 = false;
if i < bytes.len() && matches!(bytes[i], b'L' | b'l' | b'f' | b'F' | b'd' | b'D') {
match bytes[i] {
b'f' | b'F' => {
is_float = true;
is_f32 = true;
}
b'd' | b'D' => is_float = true,
_ => is_long = true,
}
i += 1;
}
let text = src[start..num_end].replace('_', "");
let text = text.as_str();
if is_float {
let parsed = if is_f32 {
text.parse::<f32>().map(f64::from)
} else {
text.parse::<f64>()
};
let v: f64 = parsed
.map_err(|_| format!("javars: bad float literal `{text}` on line {line}"))?;
out.push(Token {
kind: if is_f32 {
Tok::Float32(v)
} else {
Tok::Float(v)
},
line,
});
} else {
let v: i64 = if text.len() > 1 && text.starts_with('0') {
i64::from_str_radix(&text[1..], 8)
.map_err(|_| format!("javars: bad octal literal `{text}` on line {line}"))?
} else {
text.parse().map_err(|_| {
format!("javars: bad integer literal `{text}` on line {line}")
})?
};
out.push(Token {
kind: if is_long { Tok::Long(v) } else { Tok::Int(v) },
line,
});
}
continue;
}
if c == '"' && src[i..].starts_with("\"\"\"") {
let (block, consumed, lines) = text_block(src, i, line)?;
i += consumed;
out.push(Token {
kind: Tok::Str(block),
line,
});
line += lines;
continue;
}
if c == '"' {
i += 1;
let mut s = String::new();
while i < bytes.len() && bytes[i] != b'"' {
if bytes[i] == b'\\' && i + 1 < bytes.len() {
i += 1;
let (ch, used) = escape_at(bytes, i);
s.push(ch);
i += used;
} else {
let ch = src[i..].chars().next().unwrap();
if ch == '\n' {
line += 1;
}
s.push(ch);
i += ch.len_utf8();
}
}
if i >= bytes.len() {
return Err(format!(
"javars: unterminated string literal on line {line}"
));
}
i += 1; out.push(Token {
kind: Tok::Str(s),
line,
});
continue;
}
if c == '\'' {
i += 1;
let ch = if bytes[i] == b'\\' {
i += 1;
let (c, used) = escape_at(bytes, i);
i += used;
c
} else {
let c = src[i..].chars().next().unwrap();
i += c.len_utf8();
c
};
let cp = crate::jstr::char_value(ch);
let ch = if cp > 0xFFFF {
crate::jstr::char_of(0xD800 + ((cp - 0x10000) >> 10)).unwrap_or(ch)
} else {
ch
};
if i >= bytes.len() || bytes[i] != b'\'' {
return Err(format!("javars: unterminated char literal on line {line}"));
}
i += 1;
out.push(Token {
kind: Tok::Char(ch),
line,
});
continue;
}
let (kind, adv) = if c == '<' || c == '>' {
match src.get(i..i + 4).unwrap_or("") {
">>>=" => (Tok::UshrAssign, 4),
_ => match src.get(i..i + 3).unwrap_or("") {
">>>" => (Tok::Ushr, 3),
">>=" => (Tok::ShrAssign, 3),
"<<=" => (Tok::ShlAssign, 3),
_ => lex_short(c, bytes.get(i + 1).copied(), line)?,
},
}
} else {
lex_short(c, bytes.get(i + 1).copied(), line)?
};
out.push(Token { kind, line });
i += adv;
}
out.push(Token {
kind: Tok::Eof,
line,
});
strip_annotations(out)
}
fn strip_annotations(toks: Vec<Token>) -> Result<Vec<Token>, String> {
if !toks.iter().any(|t| t.kind == Tok::At) {
return Ok(toks);
}
let mut out = Vec::with_capacity(toks.len());
let mut i = 0;
while i < toks.len() {
if toks[i].kind != Tok::At {
out.push(toks[i].clone());
i += 1;
continue;
}
let line = toks[i].line;
match toks.get(i + 1).map(|t| &t.kind) {
Some(Tok::Ident(w)) if w == "interface" => {
return Err(format!(
"javars: annotation type declarations (`@interface`) are not modeled (line {line})"
))
}
Some(Tok::Ident(_)) => i += 2,
_ => {
return Err(format!(
"javars: `@` must start an annotation (line {line})"
))
}
}
while matches!(toks.get(i).map(|t| &t.kind), Some(Tok::Dot))
&& matches!(toks.get(i + 1).map(|t| &t.kind), Some(Tok::Ident(_)))
{
i += 2;
}
if matches!(toks.get(i).map(|t| &t.kind), Some(Tok::LParen)) {
let mut depth = 0usize;
while i < toks.len() {
match toks[i].kind {
Tok::LParen => depth += 1,
Tok::RParen => {
depth -= 1;
if depth == 0 {
i += 1;
break;
}
}
Tok::Eof => {
return Err(format!("javars: unterminated annotation (line {line})"))
}
_ => {}
}
i += 1;
}
}
}
Ok(out)
}
fn lex_short(c: char, next: Option<u8>, line: u32) -> Result<(Tok, usize), String> {
Ok(match next.map(|n| (c as u8, n)) {
Some((b'+', b'=')) => (Tok::PlusAssign, 2),
Some((b'-', b'=')) => (Tok::MinusAssign, 2),
Some((b'*', b'=')) => (Tok::StarAssign, 2),
Some((b'/', b'=')) => (Tok::SlashAssign, 2),
Some((b'%', b'=')) => (Tok::PercentAssign, 2),
Some((b'+', b'+')) => (Tok::PlusPlus, 2),
Some((b'-', b'-')) => (Tok::MinusMinus, 2),
Some((b'=', b'=')) => (Tok::EqEq, 2),
Some((b'!', b'=')) => (Tok::NotEq, 2),
Some((b'<', b'=')) => (Tok::Le, 2),
Some((b'>', b'=')) => (Tok::Ge, 2),
Some((b'&', b'&')) => (Tok::AndAnd, 2),
Some((b'|', b'|')) => (Tok::OrOr, 2),
Some((b'&', b'=')) => (Tok::AmpAssign, 2),
Some((b'|', b'=')) => (Tok::PipeAssign, 2),
Some((b'^', b'=')) => (Tok::CaretAssign, 2),
Some((b'<', b'<')) => (Tok::Shl, 2),
Some((b'>', b'>')) => (Tok::Shr, 2),
Some((b'-', b'>')) => (Tok::Arrow, 2),
Some((b':', b':')) => (Tok::ColonColon, 2),
_ => match c {
'{' => (Tok::LBrace, 1),
'}' => (Tok::RBrace, 1),
'(' => (Tok::LParen, 1),
')' => (Tok::RParen, 1),
'[' => (Tok::LBracket, 1),
']' => (Tok::RBracket, 1),
';' => (Tok::Semi, 1),
',' => (Tok::Comma, 1),
'.' => (Tok::Dot, 1),
'?' => (Tok::Question, 1),
':' => (Tok::Colon, 1),
'=' => (Tok::Assign, 1),
'+' => (Tok::Plus, 1),
'-' => (Tok::Minus, 1),
'*' => (Tok::Star, 1),
'/' => (Tok::Slash, 1),
'%' => (Tok::Percent, 1),
'<' => (Tok::Lt, 1),
'>' => (Tok::Gt, 1),
'!' => (Tok::Not, 1),
'&' => (Tok::Amp, 1),
'|' => (Tok::Pipe, 1),
'^' => (Tok::Caret, 1),
'~' => (Tok::Tilde, 1),
'@' => (Tok::At, 1),
other => {
return Err(format!(
"javars: unexpected character `{other}` on line {line}"
))
}
},
})
}
fn keyword_or_ident(word: &str) -> Tok {
let kw = match word.len() {
2 => match word {
"if" => Some(Tok::If),
"do" => Some(Tok::Do),
_ => None,
},
3 => match word {
"for" => Some(Tok::For),
"new" => Some(Tok::New),
_ => None,
},
4 => match word {
"void" => Some(Tok::Void),
"else" => Some(Tok::Else),
"case" => Some(Tok::Case),
"true" => Some(Tok::True),
_ => None,
},
5 => match word {
"class" => Some(Tok::Class),
"while" => Some(Tok::While),
"break" => Some(Tok::Break),
"false" => Some(Tok::False),
_ => None,
},
6 => match word {
"public" => Some(Tok::Public),
"static" => Some(Tok::Static),
"switch" => Some(Tok::Switch),
"return" => Some(Tok::Return),
_ => None,
},
7 => match word {
"default" => Some(Tok::Default),
_ => None,
},
8 => match word {
"continue" => Some(Tok::Continue),
_ => None,
},
_ => None,
};
kw.unwrap_or_else(|| Tok::Ident(word.to_string()))
}
fn escape_at(bytes: &[u8], at: usize) -> (char, usize) {
let first = bytes[at];
if !(b'0'..=b'7').contains(&first) {
return (unescape(first as char), 1);
}
let max = if first <= b'3' { 3 } else { 2 };
let mut value = 0u32;
let mut n = 0;
while n < max && bytes.get(at + n).is_some_and(|b| (b'0'..=b'7').contains(b)) {
value = value * 8 + u32::from(bytes[at + n] - b'0');
n += 1;
}
(char::from_u32(value).unwrap_or('\0'), n)
}
fn unescape(c: char) -> char {
match c {
'n' => '\n',
't' => '\t',
'r' => '\r',
'b' => '\u{8}',
'f' => '\u{c}',
's' => ' ',
'0' => '\0',
'\\' => '\\',
'"' => '"',
'\'' => '\'',
other => other,
}
}
fn text_block(src: &str, start: usize, line: u32) -> Result<(String, usize, u32), String> {
let after_open = start + 3;
let rest = &src[after_open..];
let header = rest
.find('\n')
.ok_or_else(|| format!("javars: unterminated text block on line {line}"))?;
if rest[..header].trim().is_empty() {
} else {
return Err(format!(
"javars: a text block's opening `\"\"\"` must be followed by a line terminator (line {line})"
));
}
let body_start = after_open + header + 1;
let bytes = src.as_bytes();
let mut k = body_start;
let body_end = loop {
match bytes.get(k) {
None => return Err(format!("javars: unterminated text block on line {line}")),
Some(b'\\') => k += 2,
Some(b'"') if src[k..].starts_with("\"\"\"") => break k,
Some(_) => k += 1,
}
};
let body = &src[body_start..body_end];
let newlines = body.matches('\n').count() as u32 + 1;
let mut lines: Vec<&str> = body.split('\n').map(|l| l.trim_end_matches('\r')).collect();
let closer_alone = lines
.last()
.is_some_and(|l| l.chars().all(|c| c == ' ' || c == '\t'));
let indent_of = |l: &str| l.len() - l.trim_start_matches([' ', '\t']).len();
let mut indent = usize::MAX;
for (n, l) in lines.iter().enumerate() {
let last = n + 1 == lines.len();
if l.trim().is_empty() && !(last && closer_alone) {
continue;
}
indent = indent.min(indent_of(l));
}
if indent == usize::MAX {
indent = 0;
}
if closer_alone {
lines.pop();
}
let trailing_newline = closer_alone && !lines.is_empty();
let mut value = String::new();
for (n, l) in lines.iter().enumerate() {
if n > 0 {
value.push('\n');
}
let stripped = if l.len() >= indent { &l[indent..] } else { "" };
value.push_str(stripped.trim_end_matches([' ', '\t']));
}
if trailing_newline {
value.push('\n');
}
Ok((unescape_block(&value), body_end + 3 - start, newlines))
}
fn unescape_block(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let bytes = s.as_bytes();
let mut i = 0;
while i < bytes.len() {
if bytes[i] != b'\\' {
let ch = s[i..].chars().next().unwrap_or('\0');
out.push(ch);
i += ch.len_utf8();
continue;
}
i += 1;
match bytes.get(i) {
Some(b'\n') => i += 1,
Some(_) => {
let (ch, used) = escape_at(bytes, i);
out.push(ch);
i += used;
}
None => out.push('\\'),
}
}
out
}
pub fn hex_float(text: &str, single: bool) -> Option<f64> {
let (sig, exp) = text.split_once(['p', 'P'])?;
let exp_digits = exp.strip_prefix(['+', '-']).unwrap_or(exp);
if exp_digits.is_empty() || !exp_digits.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
let mut exp2: i64 = exp_digits
.parse()
.unwrap_or(i64::from(i32::MAX))
.min(i64::from(i32::MAX));
if exp.starts_with('-') {
exp2 = -exp2;
}
let (int_part, frac_part) = sig.split_once('.').unwrap_or((sig, ""));
if int_part.is_empty() && frac_part.is_empty() {
return None;
}
let (mut m, mut kept, mut sticky) = (0u64, 0u32, false);
for (i, ch) in int_part.chars().chain(frac_part.chars()).enumerate() {
let d = u64::from(ch.to_digit(16)?);
let fractional = i >= int_part.len();
if m == 0 && d == 0 {
if fractional {
exp2 -= 4;
}
continue;
}
if kept < 15 {
m = (m << 4) | d;
kept += 1;
if fractional {
exp2 -= 4;
}
} else {
sticky |= d != 0;
if !fractional {
exp2 += 4;
}
}
}
let zero = 0.0;
if m == 0 {
return Some(zero);
}
let (precision, emin, emax) = if single {
(24i64, -126i64, 127i64)
} else {
(53, -1022, 1023)
};
let width = 64 - i64::from(m.leading_zeros());
let top = exp2 + width - 1;
if top > emax {
return Some(f64::INFINITY);
}
let keep = if top < emin {
precision - (emin - top)
} else {
precision
};
let shift = width - keep;
let (mut r, mut e) = (m, exp2);
if shift > 0 {
if shift > 64 {
return Some(zero);
}
let (dropped, half) = if shift == 64 {
(m, 1u64 << 63)
} else {
(m & ((1u64 << shift) - 1), 1u64 << (shift - 1))
};
r = if shift == 64 { 0 } else { m >> shift };
e += shift;
let round_up = dropped > half || (dropped == half && (sticky || r & 1 == 1));
if round_up {
r += 1;
}
if r == 0 {
return Some(zero);
}
} else if shift < 0 {
r = m << -shift;
e += shift;
}
if e + 64 - i64::from(r.leading_zeros()) - 1 > emax {
return Some(f64::INFINITY);
}
Some(scale2(r as f64, e))
}
fn scale2(mut x: f64, mut k: i64) -> f64 {
while k > 1000 {
x *= 2f64.powi(1000);
k -= 1000;
}
while k < -1000 {
x *= 2f64.powi(-1000);
k += 1000;
}
x * 2f64.powi(k as i32)
}