use std::fmt;
#[derive(Debug, Clone, PartialEq)]
pub struct Token {
pub kind: Tok,
pub line: u32,
}
#[derive(Debug, Clone, PartialEq)]
pub enum Tok {
Int(i64),
Float(f64, Option<(i128, i32)>),
Str(String),
Ident(String),
Package,
Import,
Func,
Var,
Const,
Type,
Struct,
Interface,
If,
Else,
For,
Range,
Return,
Break,
Continue,
True,
False,
Go,
Chan,
Select,
Switch,
Fallthrough,
Defer,
LBrace,
RBrace,
LParen,
RParen,
LBracket,
RBracket,
Semi,
Comma,
Dot,
Colon,
Assign,
Define, PlusAssign,
MinusAssign,
StarAssign,
SlashAssign,
PercentAssign,
AmpAssign,
PipeAssign,
CaretAssign,
ShlAssign,
ShrAssign,
AndNotAssign,
Plus,
Minus,
Star,
Slash,
Percent,
PlusPlus,
MinusMinus,
Arrow, EqEq,
NotEq,
Lt,
Gt,
Le,
Ge,
AndAnd,
OrOr,
Not,
Pipe,
Tilde,
Amp,
Caret,
Shl,
Shr,
AndNot,
Ellipsis,
Eof,
}
impl fmt::Display for Tok {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{self:?}")
}
}
impl Tok {
fn ends_statement(&self) -> bool {
matches!(
self,
Tok::Ident(_)
| Tok::Int(_)
| Tok::Float(..)
| Tok::Str(_)
| Tok::True
| Tok::False
| Tok::Return
| Tok::Break
| Tok::Continue
| Tok::Fallthrough
| Tok::RParen
| Tok::RBracket
| Tok::RBrace
| Tok::PlusPlus
| Tok::MinusMinus
)
}
}
fn maybe_asi(out: &mut Vec<Token>, line: u32) {
if let Some(last) = out.last() {
if last.kind.ends_statement() {
out.push(Token {
kind: Tok::Semi,
line,
});
}
}
}
pub fn lex(src: &str) -> Result<Vec<Token>, String> {
let bytes = src.as_bytes();
let mut i = 0usize;
let mut line = 1u32;
let mut out: Vec<Token> = Vec::new();
while i < bytes.len() {
let c = bytes[i] as char;
if c == '\n' {
maybe_asi(&mut out, line);
line += 1;
i += 1;
continue;
}
if c.is_whitespace() {
i += 1;
continue;
}
if c == '/' && i + 1 < bytes.len() {
match bytes[i + 1] as char {
'/' => {
i += 2;
while i < bytes.len() && bytes[i] != b'\n' {
i += 1;
}
continue;
}
'*' => {
i += 2;
let mut saw_newline = false;
while i + 1 < bytes.len() && !(bytes[i] == b'*' && bytes[i + 1] == b'/') {
if bytes[i] == b'\n' {
saw_newline = true;
line += 1;
}
i += 1;
}
i += 2; if saw_newline {
maybe_asi(&mut out, line);
}
continue;
}
_ => {}
}
}
if c.is_ascii_alphabetic() || c == '_' {
let start = i;
while i < bytes.len() {
let ch = bytes[i] as char;
if ch.is_ascii_alphanumeric() || ch == '_' {
i += 1;
} else {
break;
}
}
out.push(Token {
kind: keyword_or_ident(&src[start..i]),
line,
});
continue;
}
if c.is_ascii_digit() {
let start = i;
if c == '0' && i + 1 < bytes.len() {
let (base, allowed): (Option<u32>, &str) = match bytes[i + 1] {
b'x' | b'X' => (Some(16), "0123456789abcdefABCDEF_"),
b'o' | b'O' => (Some(8), "01234567_"),
b'b' | b'B' => (Some(2), "01_"),
_ => (None, ""),
};
if let Some(radix) = base {
i += 2;
let ds = i;
while i < bytes.len() && allowed.contains(bytes[i] as char) {
i += 1;
}
let digits = src[ds..i].replace('_', "");
let v = i64::from_str_radix(&digits, radix)
.or_else(|_| u64::from_str_radix(&digits, radix).map(|u| u as i64))
.map_err(|_| {
format!(
"go-rs: bad integer literal `{}` on line {line}",
&src[start..i]
)
})?;
out.push(Token {
kind: Tok::Int(v),
line,
});
continue;
}
}
let mut is_float = false;
while i < bytes.len() && ((bytes[i] as char).is_ascii_digit() || bytes[i] == b'_') {
i += 1;
}
if i < bytes.len() && bytes[i] == b'.' && !(i + 1 < bytes.len() && bytes[i + 1] == b'.')
{
is_float = true;
i += 1;
while i < bytes.len() && ((bytes[i] as char).is_ascii_digit() || bytes[i] == b'_') {
i += 1;
}
}
if i < bytes.len() && matches!(bytes[i], b'e' | b'E') {
is_float = true;
i += 1;
if i < bytes.len() && matches!(bytes[i], b'+' | b'-') {
i += 1;
}
while i < bytes.len() && (bytes[i] as char).is_ascii_digit() {
i += 1;
}
}
let text = &src[start..i];
let clean = text.replace('_', "");
if is_float {
let v: f64 = clean
.parse()
.map_err(|_| format!("go-rs: bad float literal `{text}` on line {line}"))?;
out.push(Token {
kind: Tok::Float(v, exact_decimal(&clean)),
line,
});
} else {
let v: i64 = clean
.parse()
.or_else(|_| clean.parse::<u64>().map(|u| u as i64))
.map_err(|_| format!("go-rs: bad integer literal `{text}` on line {line}"))?;
out.push(Token {
kind: Tok::Int(v),
line,
});
}
continue;
}
if c == '"' {
i += 1;
let mut s = String::new();
while i < bytes.len() && bytes[i] != b'"' {
if bytes[i] == b'\\' && i + 1 < bytes.len() {
let (cp, next) = scan_escape(src, bytes, i + 1);
s.push(char::from_u32(cp).unwrap_or('\u{FFFD}'));
i = next;
} else {
let ch = src[i..].chars().next().unwrap();
if ch == '\n' {
line += 1;
}
s.push(ch);
i += ch.len_utf8();
}
}
if i >= bytes.len() {
return Err(format!("go-rs: unterminated string literal on line {line}"));
}
i += 1; out.push(Token {
kind: Tok::Str(s),
line,
});
continue;
}
if c == '`' {
i += 1;
let mut s = String::new();
while i < bytes.len() && bytes[i] != b'`' {
let ch = src[i..].chars().next().unwrap();
if ch == '\n' {
line += 1;
}
s.push(ch);
i += ch.len_utf8();
}
if i >= bytes.len() {
return Err(format!(
"go-rs: unterminated raw string literal on line {line}"
));
}
i += 1;
out.push(Token {
kind: Tok::Str(s),
line,
});
continue;
}
if c == '\'' {
i += 1;
let cp: u32 = if bytes[i] == b'\\' {
let (cp, next) = scan_escape(src, bytes, i + 1);
i = next;
cp
} else {
let ch = src[i..].chars().next().unwrap();
i += ch.len_utf8();
ch as u32
};
if i >= bytes.len() || bytes[i] != b'\'' {
return Err(format!("go-rs: unterminated rune literal on line {line}"));
}
i += 1;
out.push(Token {
kind: Tok::Int(cp as i64),
line,
});
continue;
}
let ahead = |n: usize| {
let end = i + n;
if end <= bytes.len() && src.is_char_boundary(end) {
&src[i..end]
} else {
""
}
};
let three = ahead(3);
let two = ahead(2);
let (kind, adv) = match three {
"<<=" => (Tok::ShlAssign, 3),
">>=" => (Tok::ShrAssign, 3),
"&^=" => (Tok::AndNotAssign, 3),
"..." => (Tok::Ellipsis, 3),
_ => match two {
":=" => (Tok::Define, 2),
"+=" => (Tok::PlusAssign, 2),
"-=" => (Tok::MinusAssign, 2),
"*=" => (Tok::StarAssign, 2),
"/=" => (Tok::SlashAssign, 2),
"%=" => (Tok::PercentAssign, 2),
"++" => (Tok::PlusPlus, 2),
"--" => (Tok::MinusMinus, 2),
"<-" => (Tok::Arrow, 2),
"==" => (Tok::EqEq, 2),
"!=" => (Tok::NotEq, 2),
"<=" => (Tok::Le, 2),
">=" => (Tok::Ge, 2),
"&&" => (Tok::AndAnd, 2),
"||" => (Tok::OrOr, 2),
"<<" => (Tok::Shl, 2),
">>" => (Tok::Shr, 2),
"&^" => (Tok::AndNot, 2),
"&=" => (Tok::AmpAssign, 2),
"|=" => (Tok::PipeAssign, 2),
"^=" => (Tok::CaretAssign, 2),
_ => match c {
'{' => (Tok::LBrace, 1),
'}' => (Tok::RBrace, 1),
'(' => (Tok::LParen, 1),
')' => (Tok::RParen, 1),
'[' => (Tok::LBracket, 1),
']' => (Tok::RBracket, 1),
';' => (Tok::Semi, 1),
',' => (Tok::Comma, 1),
'.' => (Tok::Dot, 1),
':' => (Tok::Colon, 1),
'=' => (Tok::Assign, 1),
'+' => (Tok::Plus, 1),
'-' => (Tok::Minus, 1),
'*' => (Tok::Star, 1),
'/' => (Tok::Slash, 1),
'%' => (Tok::Percent, 1),
'<' => (Tok::Lt, 1),
'>' => (Tok::Gt, 1),
'!' => (Tok::Not, 1),
'|' => (Tok::Pipe, 1),
'~' => (Tok::Tilde, 1),
'&' => (Tok::Amp, 1),
'^' => (Tok::Caret, 1),
other => {
return Err(format!(
"go-rs: unexpected character `{other}` on line {line}"
))
}
},
},
};
out.push(Token { kind, line });
i += adv;
}
maybe_asi(&mut out, line);
out.push(Token {
kind: Tok::Eof,
line,
});
Ok(out)
}
fn exact_decimal(text: &str) -> Option<(i128, i32)> {
if text.contains(['e', 'E']) {
return None;
}
let (sign, body) = match text.strip_prefix('-') {
Some(rest) => (-1i128, rest),
None => (1i128, text),
};
let (int_part, frac_part) = match body.split_once('.') {
Some((a, b)) => (a, b),
None => (body, ""),
};
let digits = format!("{int_part}{frac_part}");
let mantissa: i128 = digits.parse().ok()?;
Some((sign * mantissa, frac_part.len() as i32))
}
fn keyword_or_ident(word: &str) -> Tok {
match word {
"package" => Tok::Package,
"import" => Tok::Import,
"func" => Tok::Func,
"var" => Tok::Var,
"const" => Tok::Const,
"type" => Tok::Type,
"struct" => Tok::Struct,
"interface" => Tok::Interface,
"if" => Tok::If,
"else" => Tok::Else,
"for" => Tok::For,
"range" => Tok::Range,
"return" => Tok::Return,
"break" => Tok::Break,
"continue" => Tok::Continue,
"true" => Tok::True,
"false" => Tok::False,
"go" => Tok::Go,
"chan" => Tok::Chan,
"select" => Tok::Select,
"switch" => Tok::Switch,
"fallthrough" => Tok::Fallthrough,
"defer" => Tok::Defer,
_ => Tok::Ident(word.to_string()),
}
}
fn unescape(c: char) -> char {
match c {
'n' => '\n',
't' => '\t',
'r' => '\r',
'0' => '\0',
'\\' => '\\',
'"' => '"',
'`' => '`',
'\'' => '\'',
other => other,
}
}
fn scan_escape(src: &str, bytes: &[u8], i: usize) -> (u32, usize) {
fn digits(src: &str, at: usize, n: usize, radix: u32) -> u32 {
src.get(at..at + n)
.and_then(|s| u32::from_str_radix(s, radix).ok())
.unwrap_or(0xFFFD)
}
match bytes[i] {
b'x' => (digits(src, i + 1, 2, 16), i + 3),
b'u' => (digits(src, i + 1, 4, 16), i + 5),
b'U' => (digits(src, i + 1, 8, 16), i + 9),
b'0'..=b'7' => (digits(src, i, 3, 8), i + 3),
other => (unescape(other as char) as u32, i + 1),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn kinds(src: &str) -> Vec<Tok> {
lex(src).unwrap().into_iter().map(|t| t.kind).collect()
}
#[test]
fn inserts_semicolon_after_statement_line() {
let k = kinds("x := 1\ny := 2\n");
assert_eq!(k.iter().filter(|t| **t == Tok::Semi).count(), 2);
}
#[test]
fn no_semicolon_after_open_brace_or_operator() {
let k = kinds("func f() {\n 1 +\n 2\n}\n");
assert_eq!(k.iter().filter(|t| **t == Tok::Semi).count(), 2);
}
#[test]
fn line_comment_still_terminates() {
let k = kinds("x := 1 // comment\n");
assert!(k.contains(&Tok::Semi));
}
#[test]
fn a_multibyte_rune_next_to_punctuation_lexes() {
for src in [
"f('世')\n",
"f('é')\n",
"x:='世'\n",
"a['界']=1\n",
"(('世'))\n",
"x:=\"世\"+\"界\"\n",
"// 世界\nx := 1\n",
"/* 世 */ x := 1\n",
] {
let toks = lex(src);
assert!(toks.is_ok(), "failed to lex {src:?}: {:?}", toks.err());
}
assert_eq!(kinds("'世'\n")[0], Tok::Int('世' as i64));
assert_eq!(kinds("'a'\n")[0], Tok::Int('a' as i64));
}
#[test]
fn a_truncated_escape_does_not_panic() {
for src in ["'\\x", "\"\\x\"", "'\\u12", "\"\\u12\"", "'\\U1234", "'\\7"] {
let _ = lex(src);
}
assert_eq!(kinds("'\\x41'\n")[0], Tok::Int(0x41));
assert_eq!(kinds("'\\u4e16'\n")[0], Tok::Int('世' as i64));
}
}