use crate::ast::StrPart;
use crate::errlevel::E_DEPRECATED;
use std::cell::RefCell;
#[derive(Debug, Clone, PartialEq)]
pub struct CompileDiag {
pub severity: &'static str,
pub level: i64,
pub line: u32,
pub msg: String,
}
thread_local! {
static DIAGS: RefCell<Vec<CompileDiag>> = const { RefCell::new(Vec::new()) };
}
pub fn clear_diags() {
DIAGS.with(|d| d.borrow_mut().clear());
}
pub fn take_diags() -> Vec<CompileDiag> {
DIAGS.with(|d| std::mem::take(&mut *d.borrow_mut()))
}
pub(crate) fn diag_count() -> usize {
DIAGS.with(|d| d.borrow().len())
}
pub(crate) fn truncate_diags(mark: usize) {
DIAGS.with(|d| d.borrow_mut().truncate(mark));
}
pub(crate) fn push_diag(severity: &'static str, level: i64, line: u32, msg: impl Into<String>) {
DIAGS.with(|d| {
d.borrow_mut().push(CompileDiag {
severity,
level,
line,
msg: msg.into(),
})
});
}
#[derive(Debug, Clone, PartialEq)]
pub enum Tok {
InlineHtml(String),
OpenEcho,
Var(String),
Ident(String),
Int(i64),
Float(f64),
Str(String),
Interp(Vec<StrPart>),
Punct(&'static str),
}
#[derive(Debug, Clone)]
pub struct Spanned {
pub tok: Tok,
pub line: u32,
pub raw: Option<Box<str>>,
pub space_before: bool,
pub doc: Option<std::rc::Rc<(u32, Box<str>)>>,
}
const OPERATORS: &[&str] = &[
"===", "!==", "<=>", "<<=", ">>=", "**=", "...", "?->", "==", "!=", "<>", "<=", ">=", "&&",
"||", "|>", "++", "--", "->", "=>", "::", "+=", "-=", "*=", "/=", "%=", ".=", "&=", "|=", "^=",
"<<", ">>", "**", "(", ")", "{", "}", "[", "]", ";", ",", "=", "+", "-", "*", "/", "%", ".",
"<", ">", "!", "?", ":", "&", "|", "^", "~", "@", "\\",
];
#[derive(Clone, Copy, PartialEq, Eq)]
enum InterpEnd {
DoubleQuote,
EndOfInput,
}
struct Lexer<'a> {
src: &'a [u8],
pos: usize,
line: u32,
out: Vec<Spanned>,
gap: bool,
doc: Option<std::rc::Rc<(u32, Box<str>)>>,
docs: u32,
}
fn located(msg: String, line: u32) -> String {
crate::host::with_host(|h| format!("{msg} in {} on line {line}", h.script_name()))
}
pub fn lex(src: &str) -> Result<Vec<Spanned>, String> {
let mut lx = Lexer {
src: src.as_bytes(),
pos: 0,
line: 1,
out: Vec::new(),
gap: false,
doc: None,
docs: 0,
};
lx.run()?;
Ok(lx.out)
}
impl<'a> Lexer<'a> {
fn run(&mut self) -> Result<(), String> {
loop {
let html = self.read_inline_html();
if !html.is_empty() {
self.push(Tok::InlineHtml(html));
}
if self.pos >= self.src.len() {
return Ok(());
}
if self.starts_with("<?php") {
self.advance(5);
} else if self.starts_with("<?=") {
self.advance(3);
self.push(Tok::OpenEcho);
} else if self.starts_with("<?") {
self.advance(2);
}
self.lex_php()?;
}
}
fn read_inline_html(&mut self) -> String {
let start = self.pos;
while self.pos < self.src.len() {
if self.starts_with("<?") {
break;
}
if self.src[self.pos] == b'\n' {
self.line += 1;
}
self.pos += 1;
}
String::from_utf8_lossy(&self.src[start..self.pos]).into_owned()
}
fn lex_php(&mut self) -> Result<(), String> {
while self.pos < self.src.len() {
let c = self.src[self.pos];
match c {
b' ' | b'\t' | b'\r' => {
self.gap = true;
self.pos += 1;
}
b'\n' => {
self.gap = true;
self.line += 1;
self.pos += 1;
}
_ if self.starts_with("?>") => {
self.advance(2);
if self.pos < self.src.len() && self.src[self.pos] == b'\n' {
self.line += 1;
self.pos += 1;
}
self.push(Tok::Punct(";"));
return Ok(());
}
b'/' if self.peek(1) == Some(b'/') => {
self.gap = true;
self.skip_line_comment()
}
b'#' if self.peek(1) == Some(b'[') => {
self.push(Tok::Punct("#["));
self.advance(2);
}
b'#' => {
self.gap = true;
self.skip_line_comment()
}
b'/' if self.peek(1) == Some(b'*') => {
self.gap = true;
self.skip_block_comment()?
}
b'$' => self.lex_variable(),
_ if self.starts_with("<<<") && self.heredoc_header_at(self.pos) => {
self.lex_heredoc()?
}
b'\'' => self.lex_single_quote()?,
b'"' => self.lex_double_quote()?,
b'0'..=b'9' => self.lex_number(),
b'.' if matches!(self.peek(1), Some(b'0'..=b'9')) => self.lex_number(),
_ if c == b'_' || c.is_ascii_alphabetic() => self.lex_ident(),
_ => self.lex_operator()?,
}
}
Ok(())
}
fn skip_line_comment(&mut self) {
while self.pos < self.src.len() && self.src[self.pos] != b'\n' {
if self.starts_with("?>") {
return;
}
self.pos += 1;
}
}
fn skip_block_comment(&mut self) -> Result<(), String> {
let opened = self.line;
let start = self.pos;
self.advance(2);
while self.pos < self.src.len() {
if self.starts_with("*/") {
self.advance(2);
let text = &self.src[start..self.pos];
if text.len() >= 5 && text[2] == b'*' && text[3].is_ascii_whitespace() {
self.docs += 1;
let text: Box<str> = crate::bytes::from_bytes(text).into();
self.doc = Some(std::rc::Rc::new((self.docs, text)));
}
return Ok(());
}
if self.src[self.pos] == b'\n' {
self.line += 1;
}
self.pos += 1;
}
Err(located(
format!("Unterminated comment starting line {opened}"),
opened,
))
}
fn lex_variable(&mut self) {
self.pos += 1; let start = self.pos;
while self.pos < self.src.len() && is_ident(self.src[self.pos]) {
self.pos += 1;
}
let name = String::from_utf8_lossy(&self.src[start..self.pos]).into_owned();
if name.is_empty() {
self.push(Tok::Punct("$"));
return;
}
self.push(Tok::Var(name));
}
fn lex_ident(&mut self) {
let start = self.pos;
while self.pos < self.src.len() && is_ident(self.src[self.pos]) {
self.pos += 1;
}
let name = String::from_utf8_lossy(&self.src[start..self.pos]).into_owned();
self.push(Tok::Ident(name));
}
fn lex_number(&mut self) {
let tok_start = self.pos;
if self.src[self.pos] == b'0' {
if let Some(radix) = match self.peek(1) {
Some(b'x') | Some(b'X') => Some(16u32),
Some(b'b') | Some(b'B') => Some(2),
Some(b'o') | Some(b'O') => Some(8),
_ => None,
} {
self.advance(2);
let ds = self.pos;
while self.pos < self.src.len() {
let c = self.src[self.pos];
let ok = c == b'_'
|| match radix {
16 => c.is_ascii_hexdigit(),
8 => (b'0'..=b'7').contains(&c),
_ => c == b'0' || c == b'1',
};
if ok {
self.pos += 1;
} else {
break;
}
}
let digits: String = String::from_utf8_lossy(&self.src[ds..self.pos])
.chars()
.filter(|c| *c != '_')
.collect();
let raw = String::from_utf8_lossy(&self.src[tok_start..self.pos]).into_owned();
self.push_num(parse_radix(&digits, radix), &raw);
return;
}
}
let start = self.pos;
let mut is_float = false;
while self.pos < self.src.len() {
let c = self.src[self.pos];
if c.is_ascii_digit() || c == b'_' {
self.pos += 1;
} else if c == b'.' && !is_float && matches!(self.peek(1), Some(b'0'..=b'9')) {
is_float = true;
self.pos += 1;
} else if (c == b'e' || c == b'E')
&& matches!(self.peek(1), Some(b'0'..=b'9' | b'+' | b'-'))
{
is_float = true;
self.pos += 1;
if matches!(self.src.get(self.pos), Some(b'+' | b'-')) {
self.pos += 1;
}
} else {
break;
}
}
let src_text = String::from_utf8_lossy(&self.src[start..self.pos]).into_owned();
let raw: String = src_text.chars().filter(|c| *c != '_').collect();
if is_float {
self.push_num(Tok::Float(raw.parse().unwrap_or(0.0)), &src_text);
return;
}
if raw.len() > 1 && raw.starts_with('0') && raw.bytes().all(|b| (b'0'..=b'7').contains(&b))
{
self.push_num(parse_radix(&raw[1..], 8), &src_text);
return;
}
match raw.parse::<i64>() {
Ok(n) => self.push_num(Tok::Int(n), &src_text),
Err(_) => self.push_num(Tok::Float(raw.parse().unwrap_or(0.0)), &src_text),
}
}
fn lex_single_quote(&mut self) -> Result<(), String> {
let (opened, body) = (self.line, self.pos + 1);
self.pos += 1; let mut s = String::new();
while self.pos < self.src.len() {
let c = self.src[self.pos];
match c {
b'\'' => {
self.pos += 1;
self.push(Tok::Str(s));
return Ok(());
}
b'\\' if matches!(self.peek(1), Some(b'\'' | b'\\')) => {
self.pos += 1;
s.push(self.src[self.pos] as char);
self.pos += 1;
}
b'\n' => {
self.line += 1;
s.push('\n');
self.pos += 1;
}
_ => {
let ch_len = utf8_len(c);
let end = (self.pos + ch_len).min(self.src.len());
s.push_str(&String::from_utf8_lossy(&self.src[self.pos..end]));
self.pos = end;
}
}
}
let rest = String::from_utf8_lossy(&self.src[body.min(self.src.len())..]);
let rest = rest.strip_suffix('\'').unwrap_or(&rest);
Err(located(
format!(
"syntax error, unexpected string content \"{}\"",
quoted_token_text(rest)
),
opened,
))
}
fn lex_double_quote(&mut self) -> Result<(), String> {
self.pos += 1; let parts = self.scan_interp(InterpEnd::DoubleQuote)?;
self.pos += 1; self.push(Tok::Interp(parts));
Ok(())
}
fn scan_interp(&mut self, end: InterpEnd) -> Result<Vec<StrPart>, String> {
let mut parts: Vec<StrPart> = Vec::new();
let mut lit = String::new();
let mut seg_line = self.line;
while self.pos < self.src.len() {
let c = self.src[self.pos];
match c {
b'"' if end == InterpEnd::DoubleQuote => {
if !lit.is_empty() {
parts.push(StrPart::Lit(std::mem::take(&mut lit)));
}
return Ok(parts);
}
b'\\' => {
self.pos += 1;
let e = self.src.get(self.pos).copied().unwrap_or(b'\\');
match e {
b'x' if self.hex_digits_at(self.pos + 1) > 0 => {
self.read_hex_escape(&mut lit);
continue;
}
b'u' if self.peek(1) == Some(b'{') => {
self.read_unicode_escape(&mut lit);
continue;
}
b'0'..=b'7' => {
self.read_octal_escape(&mut lit);
continue;
}
_ => {}
}
let ch = match e {
b'n' => '\n',
b't' => '\t',
b'r' => '\r',
b'v' => '\x0b',
b'f' => '\x0c',
b'"' if end == InterpEnd::DoubleQuote => '"',
b'\\' => '\\',
b'$' => '$',
b'e' => '\x1b',
other => {
lit.push('\\');
other as char
}
};
lit.push(ch);
self.pos += 1;
}
b'{' if self.peek(1) == Some(b'$') => {
if !lit.is_empty() {
parts.push(StrPart::Lit(std::mem::take(&mut lit)));
}
match self.read_braced_interp() {
Some(src) => parts.push(StrPart::Raw(src)),
None => {
lit.push('{');
self.pos += 1;
}
}
seg_line = self.line;
}
b'$' if self.peek(1) == Some(b'{') => {
if !lit.is_empty() {
parts.push(StrPart::Lit(std::mem::take(&mut lit)));
}
push_diag(
"Deprecated",
E_DEPRECATED,
seg_line,
"Using ${var} in strings is deprecated, use {$var} instead",
);
self.pos += 2; let start = self.pos;
while self.pos < self.src.len() && is_ident(self.src[self.pos]) {
self.pos += 1;
}
let name = String::from_utf8_lossy(&self.src[start..self.pos]).into_owned();
if self.src.get(self.pos) == Some(&b'}') {
self.pos += 1;
}
parts.push(StrPart::Var(name));
seg_line = self.line;
}
b'$' if matches!(self.peek(1), Some(b) if b == b'_' || b.is_ascii_alphabetic()) => {
if !lit.is_empty() {
parts.push(StrPart::Lit(std::mem::take(&mut lit)));
}
self.pos += 1; let start = self.pos;
while self.pos < self.src.len() && is_ident(self.src[self.pos]) {
self.pos += 1;
}
let name = String::from_utf8_lossy(&self.src[start..self.pos]).into_owned();
match self.read_simple_interp_suffix(&name) {
Some(src) => parts.push(StrPart::Raw(src)),
None => parts.push(StrPart::Var(name)),
}
seg_line = self.line;
}
b'\n' => {
self.line += 1;
lit.push('\n');
self.pos += 1;
}
_ => {
let ch_len = utf8_len(c);
let stop = (self.pos + ch_len).min(self.src.len());
lit.push_str(&String::from_utf8_lossy(&self.src[self.pos..stop]));
self.pos = stop;
}
}
}
if end == InterpEnd::EndOfInput {
if !lit.is_empty() {
parts.push(StrPart::Lit(lit));
}
return Ok(parts);
}
if end != InterpEnd::DoubleQuote {
return Err(format!("unterminated string (line {})", self.line));
}
let expecting = if parts.iter().any(|p| !matches!(p, StrPart::Lit(_))) {
""
} else if parts.is_empty() && lit.is_empty() {
", expecting variable or string content or \"${\" or \"{$\""
} else {
", expecting variable or \"${\" or \"{$\""
};
Err(located(
format!("syntax error, unexpected end of file{expecting}"),
self.line,
))
}
fn heredoc_header_at(&self, at: usize) -> bool {
let src = &self.src;
let mut i = at + 3;
while matches!(src.get(i), Some(b' ' | b'\t')) {
i += 1;
}
let quote = match src.get(i) {
Some(&q @ (b'\'' | b'"')) => {
i += 1;
Some(q)
}
_ => None,
};
if !matches!(src.get(i), Some(&b) if is_ident(b) && !b.is_ascii_digit()) {
return false;
}
while matches!(src.get(i), Some(&b) if is_ident(b)) {
i += 1;
}
if let Some(q) = quote {
if src.get(i) != Some(&q) {
return false;
}
i += 1;
}
matches!(src.get(i), Some(b'\n' | b'\r'))
}
fn lex_heredoc(&mut self) -> Result<(), String> {
let opened = self.line;
let start = self.pos;
self.pos += 3; while matches!(self.src.get(self.pos), Some(b' ' | b'\t')) {
self.pos += 1;
}
let quote = match self.src.get(self.pos) {
Some(&q @ (b'\'' | b'"')) => {
self.pos += 1;
Some(q)
}
_ => None,
};
let label_start = self.pos;
while matches!(self.src.get(self.pos), Some(&b) if is_ident(b)) {
self.pos += 1;
}
let label = String::from_utf8_lossy(&self.src[label_start..self.pos]).into_owned();
if label.is_empty() || label.as_bytes()[0].is_ascii_digit() {
return Err(located(
"syntax error, unexpected token \"<<<\"".into(),
opened,
));
}
if let Some(q) = quote {
if self.src.get(self.pos) != Some(&q) {
return Err(located(
"syntax error, unexpected token \"<<<\"".into(),
opened,
));
}
self.pos += 1;
}
if self.src.get(self.pos) == Some(&b'\r') {
self.pos += 1;
}
if self.src.get(self.pos) != Some(&b'\n') {
return Err(located(
"syntax error, unexpected token \"<<<\"".into(),
opened,
));
}
self.pos += 1;
let body_start = self.pos;
let (body_end, indent_len, after) = {
let mut i = body_start;
loop {
let line_start = i;
let mut j = i;
while matches!(self.src.get(j), Some(b' ' | b'\t')) {
j += 1;
}
let is_close = self.src[j..].starts_with(label.as_bytes())
&& !matches!(self.src.get(j + label.len()), Some(&b) if is_ident(b));
if is_close {
break (line_start, j - line_start, j + label.len());
}
match self.src[i..].iter().position(|&b| b == b'\n') {
Some(n) => i += n + 1,
None => {
let rest = &self.src[body_start..];
let eof = opened
+ self.src[start..].iter().filter(|&&b| b == b'\n').count() as u32;
let msg = if scanned_interpolation(rest) {
"syntax error, unexpected end of file".to_string()
} else {
"syntax error, unexpected end of file, expecting variable \
or heredoc end or \"${\" or \"{$\""
.to_string()
};
return Err(located(msg, eof));
}
}
}
};
let mut raw_end = body_end;
if raw_end > body_start && self.src[raw_end - 1] == b'\n' {
raw_end -= 1;
if raw_end > body_start && self.src[raw_end - 1] == b'\r' {
raw_end -= 1;
}
}
let raw = String::from_utf8_lossy(&self.src[body_start..raw_end]).into_owned();
let body_line = opened
+ self.src[start..body_start]
.iter()
.filter(|&&b| b == b'\n')
.count() as u32;
let body = dedent_heredoc(&raw, &self.src[body_end..body_end + indent_len], body_line)?;
let consumed = self.src[start..after]
.iter()
.filter(|&&b| b == b'\n')
.count() as u32;
self.pos = after;
if quote == Some(b'\'') {
self.push(Tok::Str(body));
} else {
let mut sub = Lexer {
src: body.as_bytes(),
pos: 0,
line: body_line,
out: Vec::new(),
gap: false,
doc: None,
docs: 0,
};
let parts = sub.scan_interp(InterpEnd::EndOfInput)?;
self.push(Tok::Interp(parts));
}
self.line = opened + consumed;
Ok(())
}
fn read_braced_interp(&mut self) -> Option<String> {
let start = self.pos + 1; let mut i = start;
let mut depth = 1usize;
let mut quote: Option<u8> = None;
while i < self.src.len() {
let c = self.src[i];
match quote {
Some(q) => {
if c == b'\\' {
i += 1;
} else if c == q {
quote = None;
}
}
None => match c {
b'\'' | b'"' => quote = Some(c),
b'{' => depth += 1,
b'}' => {
depth -= 1;
if depth == 0 {
let src = String::from_utf8_lossy(&self.src[start..i]).into_owned();
self.line +=
self.src[start..i].iter().filter(|&&b| b == b'\n').count() as u32;
self.pos = i + 1;
return Some(src);
}
}
b'\n' => {}
_ => {}
},
}
i += 1;
}
None
}
fn read_simple_interp_suffix(&mut self, name: &str) -> Option<String> {
if self.src.get(self.pos) == Some(&b'-') && self.peek(1) == Some(b'>') {
let after = self.pos + 2;
if !matches!(self.src.get(after), Some(&b) if b == b'_' || b.is_ascii_alphabetic()) {
return None;
}
let mut i = after;
while i < self.src.len() && is_ident(self.src[i]) {
i += 1;
}
let prop = String::from_utf8_lossy(&self.src[after..i]).into_owned();
self.pos = i;
return Some(format!("${name}->{prop}"));
}
if self.src.get(self.pos) != Some(&b'[') {
return None;
}
let key_start = self.pos + 1;
let mut i = key_start;
while i < self.src.len() && self.src[i] != b']' {
i += 1;
}
if i >= self.src.len() {
return None;
}
let key = String::from_utf8_lossy(&self.src[key_start..i]).into_owned();
if key.is_empty() {
return None;
}
let key_expr = if key.starts_with('$')
|| key.parse::<i64>().is_ok()
|| (key.starts_with('\'') && key.ends_with('\''))
|| (key.starts_with('"') && key.ends_with('"'))
{
key
} else {
format!("'{}'", key.replace('\\', "\\\\").replace('\'', "\\'"))
};
self.pos = i + 1;
Some(format!("${name}[{key_expr}]"))
}
fn lex_operator(&mut self) -> Result<(), String> {
for op in OPERATORS {
if self.starts_with(op) {
self.advance(op.len());
self.push(Tok::Punct(op));
return Ok(());
}
}
let c = self.src[self.pos] as char;
Err(format!("unexpected character '{c}' (line {})", self.line))
}
fn push(&mut self, tok: Tok) {
let tok = match tok {
Tok::Str(s) => Tok::Str(crate::bytes::canonical(s)),
Tok::Interp(parts) => Tok::Interp(
parts
.into_iter()
.map(|p| match p {
StrPart::Lit(s) => StrPart::Lit(crate::bytes::canonical(s)),
other => other,
})
.collect(),
),
other => other,
};
self.out.push(Spanned {
tok,
line: self.line,
raw: None,
space_before: std::mem::take(&mut self.gap),
doc: self.doc.clone(),
});
if matches!(self.out.last().map(|s| &s.tok), Some(Tok::Punct("{" | "}"))) {
self.doc = None;
}
}
fn push_num(&mut self, tok: Tok, raw: &str) {
self.out.push(Spanned {
tok,
line: self.line,
raw: Some(raw.into()),
space_before: std::mem::take(&mut self.gap),
doc: self.doc.clone(),
});
}
fn starts_with(&self, s: &str) -> bool {
self.src[self.pos..].starts_with(s.as_bytes())
}
fn peek(&self, n: usize) -> Option<u8> {
self.src.get(self.pos + n).copied()
}
fn hex_digits_at(&self, at: usize) -> usize {
(0..2)
.take_while(|i| self.src.get(at + i).is_some_and(|b| b.is_ascii_hexdigit()))
.count()
}
fn read_hex_escape(&mut self, lit: &mut String) {
self.pos += 1; let n = self.hex_digits_at(self.pos);
let hex: String = self.src[self.pos..self.pos + n]
.iter()
.map(|b| *b as char)
.collect();
self.pos += n;
if let Ok(b) = u8::from_str_radix(&hex, 16) {
lit.push_str(&crate::bytes::from_bytes(&[b]));
}
}
fn read_unicode_escape(&mut self, lit: &mut String) {
let close = self.src[self.pos..]
.iter()
.position(|b| *b == b'}')
.map(|i| self.pos + i);
let Some(close) = close else {
lit.push_str("\\u");
self.pos += 1;
return;
};
let hex: String = self.src[self.pos + 2..close]
.iter()
.map(|b| *b as char)
.collect();
self.pos = close + 1;
if let Some(c) = u32::from_str_radix(&hex, 16).ok().and_then(char::from_u32) {
lit.push_str(&crate::bytes::from_bytes(
c.encode_utf8(&mut [0u8; 4]).as_bytes(),
));
}
}
fn read_octal_escape(&mut self, lit: &mut String) {
let n = (0..3)
.take_while(|i| {
self.src
.get(self.pos + i)
.is_some_and(|b| (b'0'..=b'7').contains(b))
})
.count();
let oct: String = self.src[self.pos..self.pos + n]
.iter()
.map(|b| *b as char)
.collect();
self.pos += n;
if let Ok(v) = u32::from_str_radix(&oct, 8) {
lit.push_str(&crate::bytes::from_bytes(&[(v & 0xff) as u8]));
}
}
fn advance(&mut self, n: usize) {
for _ in 0..n {
if self.pos < self.src.len() && self.src[self.pos] == b'\n' {
self.line += 1;
}
self.pos += 1;
}
}
}
fn scanned_interpolation(body: &[u8]) -> bool {
body.windows(2).any(|w| match w {
[b'$', c] => is_ident(*c) && !c.is_ascii_digit(),
[b'{', b'$'] => true,
_ => false,
})
}
fn dedent_heredoc(raw: &str, indent: &[u8], first_line: u32) -> Result<String, String> {
if indent.is_empty() {
return Ok(raw.to_string());
}
let indent = String::from_utf8_lossy(indent).into_owned();
let mut out = String::with_capacity(raw.len());
for (n, line) in raw.split('\n').enumerate() {
if n > 0 {
out.push('\n');
}
if line.strip_suffix('\r').unwrap_or(line).is_empty() {
out.push_str(line);
continue;
}
match line.strip_prefix(indent.as_str()) {
Some(rest) => out.push_str(rest),
None => {
return Err(located(
format!(
"Invalid body indentation level (expecting an indentation \
level of at least {})",
indent.chars().count()
),
first_line + n as u32,
))
}
}
}
Ok(out)
}
fn parse_radix(digits: &str, radix: u32) -> Tok {
match i64::from_str_radix(digits, radix) {
Ok(n) => Tok::Int(n),
Err(_) => Tok::Float(
u128::from_str_radix(digits, radix)
.map(|u| u as f64)
.unwrap_or(0.0),
),
}
}
fn is_ident(c: u8) -> bool {
c == b'_' || c.is_ascii_alphanumeric() || c >= 0x80
}
fn utf8_len(b: u8) -> usize {
match b {
0x00..=0x7f => 1,
0xc0..=0xdf => 2,
0xe0..=0xef => 3,
_ => 4,
}
}
pub(crate) fn quoted_token_text(s: &str) -> String {
let line = s.split('\n').next().unwrap_or_default();
if line.len() <= 33 {
return line.to_string();
}
let mut cut = 30;
while !line.is_char_boundary(cut) {
cut -= 1;
}
format!("{}...", &line[..cut])
}