use std::fmt;
use std::path::Path;
use super::fault::{Fault, Span};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum Kind {
Word,
Text,
Arrow,
Open,
Close,
Break,
End,
}
impl fmt::Display for Kind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(match self {
Self::Word => "a word",
Self::Text => "quoted text",
Self::Arrow => "`->`",
Self::Open => "`{`",
Self::Close => "`}`",
Self::Break => "end of line",
Self::End => "end of file",
})
}
}
#[derive(Clone, Debug)]
pub(super) struct Token {
pub kind: Kind,
pub text: String,
pub span: Span,
}
impl fmt::Display for Token {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self.kind {
Kind::Word | Kind::Text => write!(f, "`{}`", self.text),
kind => fmt::Display::fmt(&kind, f),
}
}
}
const BREAKS: [char; 6] = ['{', '}', '"', ' ', '\t', '\r'];
struct Scan<'a> {
src: &'a [char],
file: &'a Path,
text: &'a str,
out: Vec<Token>,
i: usize,
line: usize,
col: usize,
}
impl Scan<'_> {
fn at(&self, width: usize) -> Span {
Span { file: self.file.to_path_buf(), line: self.line, col: self.col, width }
}
fn push(&mut self, kind: Kind, text: impl Into<String>, span: Span) {
self.out.push(Token { kind, text: text.into(), span });
}
fn emit(&mut self, kind: Kind, text: impl Into<String>, width: usize) {
let where_it_is = self.at(width);
self.push(kind, text, where_it_is);
}
fn starts(&self, at: usize, needle: &str) -> bool {
needle.chars().enumerate().all(|(k, c)| self.src.get(at + k) == Some(&c))
}
fn folded(&mut self) {
let (start_line, start_col) = (self.line, self.col);
let mut parts: Vec<String> = Vec::new();
loop {
let end =
(self.i..self.src.len()).find(|&k| self.src[k] == '\n').unwrap_or(self.src.len());
parts.push(
self.src[(self.i + 2).min(end)..end].iter().collect::<String>().trim().into(),
);
self.i = end + 1;
self.line += 1;
self.col = 1;
let mut probe = self.i;
while matches!(self.src.get(probe), Some(' ' | '\t')) {
probe += 1;
}
if !self.starts(probe, "\\\\") {
break;
}
self.col = probe - end;
self.i = probe;
}
let folded = parts.iter().filter(|p| !p.is_empty()).cloned().collect::<Vec<_>>().join(" ");
let span =
Span { file: self.file.to_path_buf(), line: start_line, col: start_col, width: 2 };
self.push(Kind::Text, folded, span);
let after = Span { file: self.file.to_path_buf(), line: self.line - 1, col: 1, width: 1 };
self.push(Kind::Break, "\n", after);
}
fn quoted(&mut self) -> Result<(), Fault> {
let start_col = self.col;
let unterminated = |scan: &Self| {
Fault::at(
"unterminated text — a `\"` is missing",
Span { file: scan.file.to_path_buf(), line: scan.line, col: start_col, width: 1 },
scan.text,
)
};
let mut j = self.i + 1;
let mut buf = String::new();
while let Some(&ch) = self.src.get(j) {
match ch {
'\n' => return Err(unterminated(self)),
'\\' if j + 1 < self.src.len() => {
buf.push(self.src[j + 1]);
j += 2;
}
'"' => {
let width = j + 1 - self.i;
let span = Span {
file: self.file.to_path_buf(),
line: self.line,
col: start_col,
width,
};
self.push(Kind::Text, buf, span);
self.i = j + 1;
self.col += width;
return Ok(());
}
_ => {
buf.push(ch);
j += 1;
}
}
}
Err(unterminated(self))
}
}
pub(super) fn tokenize(source: &str, file: &Path) -> Result<Vec<Token>, Fault> {
let chars: Vec<char> = source.chars().collect();
let mut scan = Scan { src: &chars, file, text: source, out: Vec::new(), i: 0, line: 1, col: 1 };
while let Some(&ch) = scan.src.get(scan.i) {
if ch == '\n' {
if scan.out.last().is_some_and(|t| t.kind != Kind::Break) {
scan.emit(Kind::Break, "\n", 1);
}
scan.i += 1;
scan.line += 1;
scan.col = 1;
} else if matches!(ch, ' ' | '\t' | '\r') {
scan.i += 1;
scan.col += 1;
} else if scan.starts(scan.i, "//") {
while scan.src.get(scan.i).is_some_and(|&c| c != '\n') {
scan.i += 1;
}
} else if scan.starts(scan.i, "->") {
scan.emit(Kind::Arrow, "->", 2);
scan.i += 2;
scan.col += 2;
} else if matches!(ch, '{' | '}') {
let kind = if ch == '{' { Kind::Open } else { Kind::Close };
scan.emit(kind, ch, 1);
scan.i += 1;
scan.col += 1;
} else if scan.starts(scan.i, "\\\\") {
scan.folded();
} else if ch == '"' {
scan.quoted()?;
} else {
let start = scan.i;
while scan.src.get(scan.i).is_some_and(|c| !BREAKS.contains(c) && *c != '\n')
&& !scan.starts(scan.i, "//")
{
scan.i += 1;
}
let word: String = scan.src[start..scan.i].iter().collect();
let width = scan.i - start;
scan.emit(Kind::Word, word, width);
scan.col += width;
}
}
if scan.out.last().is_none_or(|t| t.kind != Kind::Break) {
scan.emit(Kind::Break, "\n", 1);
}
scan.emit(Kind::End, "", 1);
Ok(scan.out)
}