use std::borrow::Cow;
use std::ops::Range;
#[rustfmt::skip]
const fn accented(accent: u8, base: u8) -> Option<char> {
Some(match (accent, base) {
(b'"', b'a') => 'ä', (b'"', b'e') => 'ë', (b'"', b'i') => 'ï',
(b'"', b'o') => 'ö', (b'"', b'u') => 'ü', (b'"', b'y') => 'ÿ',
(b'"', b'A') => 'Ä', (b'"', b'E') => 'Ë', (b'"', b'I') => 'Ï',
(b'"', b'O') => 'Ö', (b'"', b'U') => 'Ü', (b'"', b'Y') => 'Ÿ',
(b'\'', b'a') => 'á', (b'\'', b'e') => 'é', (b'\'', b'i') => 'í',
(b'\'', b'o') => 'ó', (b'\'', b'u') => 'ú', (b'\'', b'y') => 'ý',
(b'\'', b'A') => 'Á', (b'\'', b'E') => 'É', (b'\'', b'I') => 'Í',
(b'\'', b'O') => 'Ó', (b'\'', b'U') => 'Ú', (b'\'', b'Y') => 'Ý',
(b'`', b'a') => 'à', (b'`', b'e') => 'è', (b'`', b'i') => 'ì',
(b'`', b'o') => 'ò', (b'`', b'u') => 'ù',
(b'`', b'A') => 'À', (b'`', b'E') => 'È', (b'`', b'I') => 'Ì',
(b'`', b'O') => 'Ò', (b'`', b'U') => 'Ù',
(b'^', b'a') => 'â', (b'^', b'e') => 'ê', (b'^', b'i') => 'î',
(b'^', b'o') => 'ô', (b'^', b'u') => 'û',
(b'^', b'A') => 'Â', (b'^', b'E') => 'Ê', (b'^', b'I') => 'Î',
(b'^', b'O') => 'Ô', (b'^', b'U') => 'Û',
(b'~', b'a') => 'ã', (b'~', b'n') => 'ñ', (b'~', b'o') => 'õ',
(b'~', b'A') => 'Ã', (b'~', b'N') => 'Ñ', (b'~', b'O') => 'Õ',
_ => return None,
})
}
#[derive(Debug, Clone, Copy)]
struct Source<'src> {
text: &'src str,
}
impl<'src> Source<'src> {
const fn new(text: &'src str) -> Self {
Self { text }
}
const fn len(self) -> usize {
self.text.len()
}
fn byte(self, at: usize) -> Option<u8> {
self.text.as_bytes().get(at).copied()
}
fn is(self, at: usize, byte: u8) -> bool {
self.byte(at) == Some(byte)
}
fn at_is(self, at: usize, pred: impl FnOnce(u8) -> bool) -> bool {
self.byte(at).is_some_and(pred)
}
fn slice(self, range: Range<usize>) -> &'src str {
let found = self.text.get(range.clone());
debug_assert!(found.is_some(), "slice {range:?} is not a char boundary of {:?}", self.text);
found.unwrap_or_default()
}
}
fn find_matching_brace(src: Source<'_>, open_pos: usize) -> Option<usize> {
let mut depth: u32 = 0;
for (i, byte) in src.text.bytes().enumerate().skip(open_pos) {
match byte {
b'{' => depth += 1,
b'}' => {
depth = depth.checked_sub(1)?;
if depth == 0 {
return Some(i);
}
}
_ => {}
}
}
None
}
#[must_use]
pub fn strip_line(line: &str) -> Cow<'_, str> {
if is_already_plain(line) {
return Cow::Borrowed(line);
}
let without_comment = remove_comment(line);
let stripped = strip_fragment(Source::new(&without_comment));
Cow::Owned(collapse_whitespace(&stripped))
}
const MARKUP_BYTES: &[u8] = b"\\{}%~&[]$";
fn is_already_plain(line: &str) -> bool {
if line.bytes().any(|b| MARKUP_BYTES.contains(&b)) {
return false;
}
let mut prev_was_space = false;
let mut at_start = true;
for ch in line.chars() {
if ch.is_whitespace() {
if at_start || prev_was_space || ch != ' ' {
return false;
}
prev_was_space = true;
} else {
if prev_was_space && CLINGY_PUNCTUATION.contains(&ch) {
return false;
}
prev_was_space = false;
}
at_start = false;
}
!prev_was_space
}
fn remove_comment(line: &str) -> Cow<'_, str> {
let src = Source::new(line);
let mut i = 0;
while i < src.len() {
if src.is(i, b'\\') {
let escapes_next = src.at_is(i + 1, |b| !b.is_ascii_alphabetic() && b != b'@');
i += if escapes_next { 2 } else { 1 };
continue;
}
if src.is(i, b'%') {
return Cow::Borrowed(src.slice(0..i));
}
i += 1;
}
Cow::Borrowed(line)
}
const DROP_ARG_COMMANDS: &[&str] = &[
"url",
"href",
"includegraphics",
"label",
"ref",
"eqref",
"cite",
"cref",
"Cref",
"autoref",
"pageref",
"nameref",
"bibliographystyle",
"bibliography",
"input",
"include",
"usepackage",
"documentclass",
"begin",
"end",
"newcommand",
"renewcommand",
"setlength",
"addtolength",
"setcounter",
"newcounter",
"fnsymbol",
"arabic",
"thispagestyle",
"pagestyle",
"put",
"parbox",
"DeclareGraphicsExtensions",
"graphicspath",
"author",
"IEEEauthorblockN",
"IEEEauthorblockA",
"maketitle",
"thanks",
"hyphenation",
"index",
"nocite",
"markboth",
"markright",
"addcontentsline",
"pagenumbering",
"hspace",
"vspace",
"rule",
"bibitem",
"DescribeMacro",
"DescribeEnv",
];
const TEXT_COMMANDS: &[(&str, &str)] = &[
("LaTeX", "LaTeX"),
("LaTeXe", "LaTeX2e"),
("TeX", "TeX"),
("BibTeX", "BibTeX"),
("ldots", "…"),
("dots", "…"),
("textellipsis", "…"),
("copyright", "(c)"),
("textcopyright", "(c)"),
("textregistered", "(R)"),
("texttrademark", "(TM)"),
("pounds", "GBP"),
("euro", "EUR"),
("textbackslash", " "),
("textbar", "|"),
("S", "Section"),
("P", "Paragraph"),
("dag", "*"),
("ddag", "*"),
("ss", "ss"),
("ae", "ae"),
("AE", "AE"),
("oe", "oe"),
("OE", "OE"),
("aa", "a"),
("AA", "A"),
("o", "o"),
("O", "O"),
("l", "l"),
("L", "L"),
("i", "i"),
("j", "j"),
];
fn text_command(name: &str) -> Option<&'static str> {
TEXT_COMMANDS.iter().find(|(cmd, _)| *cmd == name).map(|(_, text)| *text)
}
const HEADING_COMMANDS: &[&str] = &[
"section",
"subsection",
"subsubsection",
"paragraph",
"subparagraph",
"chapter",
"caption",
"title",
];
fn strip_fragment(src: Source<'_>) -> String {
let mut out = String::with_capacity(src.len());
let mut i = 0;
while i < src.len() {
let Some(byte) = src.byte(i) else { break };
if byte == b'\\' {
let Some(next) = src.byte(i + 1) else { break };
if next.is_ascii_alphabetic() || next == b'@' {
i = skip_command(src, i, &mut out);
} else {
i += 2;
match next {
b'\\' | b' ' => out.push(' '),
b'"' | b'\'' | b'`' | b'^' | b'~' => {
let base = resolve_accent_base(src, &mut i);
match base.as_bytes().first().and_then(|&b| accented(next, b)) {
Some(ch) => {
out.push(ch);
out.push_str(base.get(1..).unwrap_or_default());
}
None => out.push_str(&base),
}
}
b'%' | b'&' | b'#' | b'$' | b'_' | b'-' | b'.' | b'?' => {
out.push(char::from(next));
}
b',' | b';' | b':' | b'!' | b'>' => {
out.push(' ');
}
b'(' | b'[' => {
let closer = if next == b'(' { b')' } else { b']' };
i = close_paren_math(src, i, closer).map_or_else(|| src.len(), |c| c + 2);
out.push(' ');
}
_ => {}
}
}
continue;
}
if byte == b'%' {
break;
}
if byte == b'$' {
let display = src.is(i + 1, b'$');
let delim = if display { 2 } else { 1 };
i = close_dollar_math(src, i, display).map_or_else(|| src.len(), |c| c + delim);
out.push(' ');
continue;
}
if byte == b'[' {
i = skip_brackets(src, i);
continue;
}
if byte == b'{' {
if let Some(close) = find_matching_brace(src, i) {
out.push_str(&strip_fragment(Source::new(src.slice(i + 1..close))));
i = close + 1;
} else {
i += 1;
}
continue;
}
match byte {
b'}' => i += 1,
b'&' | b'~' => {
out.push(' ');
i += 1;
}
_ if !byte.is_ascii() => {
let start = i;
i += 1;
while src.at_is(i, is_utf8_continuation) {
i += 1;
}
out.push_str(src.slice(start..i));
}
_ => {
out.push(char::from(byte));
i += 1;
}
}
}
out
}
const fn is_utf8_continuation(byte: u8) -> bool {
byte & 0b1100_0000 == 0b1000_0000
}
fn resolve_accent_base<'src>(src: Source<'src>, pos: &mut usize) -> Cow<'src, str> {
let Some(byte) = src.byte(*pos) else {
return Cow::Borrowed("");
};
if byte == b'{' {
if let Some(close) = find_matching_brace(src, *pos) {
let inner = strip_fragment(Source::new(src.slice(*pos + 1..close)));
*pos = close + 1;
return Cow::Owned(inner);
}
}
if byte.is_ascii_alphabetic() {
let base = src.slice(*pos..*pos + 1);
*pos += 1;
return Cow::Borrowed(base);
}
Cow::Borrowed("")
}
fn skip_command(src: Source<'_>, start: usize, out: &mut String) -> usize {
let mut j = start + 2;
while src.at_is(j, |b| b.is_ascii_alphabetic() || b == b'@') {
j += 1;
}
let cmd_name = src.slice(start + 1..j);
if cmd_name == "verb" {
let after_star = if src.is(j, b'*') { j + 1 } else { j };
if let Some(delim) = src.byte(after_star) {
out.push(' ');
return skip_delimited(src, after_star + 1, delim);
}
}
if let Some(text) = text_command(cmd_name) {
out.push_str(text);
if src.is(j, b'{') && src.is(j + 1, b'}') {
return j + 2;
}
return j;
}
let drop = DROP_ARG_COMMANDS.contains(&cmd_name);
let heading = HEADING_COMMANDS.contains(&cmd_name);
if src.at_is(j, |b| matches!(b, b'*' | b'^' | b'_')) {
j += 1;
}
if src.is(j, b'[') {
j = skip_brackets(src, j);
}
if src.is(j, b'{') {
if let Some(close) = find_matching_brace(src, j) {
if !drop {
let content = strip_fragment(Source::new(src.slice(j + 1..close)));
out.push_str(&content);
if heading && !content.ends_with(['.', '?', '!']) {
out.push('.');
}
}
j = close + 1;
if drop {
loop {
while src.is(j, b'[') {
j = skip_brackets(src, j);
}
let Some(close2) =
src.is(j, b'{').then(|| find_matching_brace(src, j)).flatten()
else {
break;
};
j = close2 + 1;
}
} else {
while let Some(close2) =
src.is(j, b'{').then(|| find_matching_brace(src, j)).flatten()
{
j = close2 + 1;
}
}
return j;
}
}
j
}
fn skip_delimited(src: Source<'_>, start: usize, delim: u8) -> usize {
let mut i = start;
while i < src.len() {
if src.is(i, delim) {
return i + 1;
}
i += 1;
}
src.len()
}
fn close_paren_math(src: Source<'_>, content_start: usize, closer: u8) -> Option<usize> {
let mut i = content_start;
while i < src.len() {
if src.is(i, b'\\') && src.is(i + 1, closer) {
return Some(i);
}
i += 1;
}
None
}
fn close_dollar_math(src: Source<'_>, open: usize, display: bool) -> Option<usize> {
let mut i = open + if display { 2 } else { 1 };
while i < src.len() {
if src.is(i, b'\\') {
i += 2;
continue;
}
if src.is(i, b'$') && (!display || src.is(i + 1, b'$')) {
return Some(i);
}
i += 1;
}
None
}
fn skip_brackets(src: Source<'_>, pos: usize) -> usize {
let mut depth: u32 = 1;
let mut j = pos + 1;
while depth > 0 {
let Some(byte) = src.byte(j) else { break };
match byte {
b'[' => depth += 1,
b']' => depth -= 1,
_ => {}
}
j += 1;
}
j
}
const CLINGY_PUNCTUATION: [char; 8] = ['.', ',', ';', ':', '!', '?', ')', ']'];
fn collapse_whitespace(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for word in s.split_whitespace() {
if !out.is_empty() && !word.starts_with(CLINGY_PUNCTUATION) {
out.push(' ');
}
out.push_str(word);
}
out
}
#[cfg(test)]
mod tests {
use super::*;
fn strip_fragment(text: &str) -> String {
super::strip_fragment(Source::new(text))
}
fn find_matching_brace(text: &str, open_pos: usize) -> Option<usize> {
super::find_matching_brace(Source::new(text), open_pos)
}
fn strip_line_the_long_way(line: &str) -> String {
let without_comment = remove_comment(line);
collapse_whitespace(&strip_fragment(&without_comment))
}
#[test]
fn borrowed_and_owned_paths_agree() {
let cases = [
"",
"plain prose here",
" leading space",
"trailing space ",
"double space",
"tab\tseparated",
"non\u{a0}breaking space",
"naïve unicode prose",
"hyphen-joined words",
"trailing backslash \\",
r"\textbf{markup}",
"brace } alone",
"percent % comment",
"tilde~space",
"amp & tab",
"bracket [opt] group",
];
for case in cases {
assert_eq!(
strip_line(case),
strip_line_the_long_way(case),
"fast path diverged from the full pipeline on {case:?}"
);
}
}
#[test]
fn plain_lines_are_borrowed() {
assert!(
matches!(strip_line("plain prose here"), Cow::Borrowed(_)),
"a line with no markup must not allocate"
);
assert!(
matches!(strip_line(r"\textbf{markup}"), Cow::Owned(_)),
"a line with markup must be rewritten"
);
}
#[test]
fn math_is_dropped() {
assert_eq!(
strip_line(r"The symbol $\alpha$ tagged as math."),
"The symbol tagged as math.",
"`$…$` must not leave its delimiters behind"
);
assert_eq!(strip_line("Set $x = y + 1$ here."), "Set here.", "a formula is not prose");
assert_eq!(strip_line("before $$a+b$$ after"), "before after", "`$$…$$` is dropped");
assert_eq!(strip_line(r"before \(a+b\) after"), "before after", r"`\(…\)` is dropped");
assert_eq!(strip_line(r"before \[a+b\] after"), "before after", r"`\[…\]` is dropped");
}
#[test]
fn math_notation_does_not_leak() {
for line in [r"a $-58\,\%$ b", r"a $\{t_1, \ldots, t_N\}$ b", r"a $x^2_i$ b"] {
let got = strip_line(line);
assert_eq!(got, "a b", "notation leaked from {line:?} as {got:?}");
}
}
#[test]
fn escaped_dollar_is_not_math() {
assert_eq!(
strip_line(r"costs \$5 and \$6"),
"costs $5 and $6",
r"`\$` is a literal dollar, not a math delimiter"
);
}
#[test]
fn unclosed_math_does_not_eat_the_next_line() {
assert_eq!(strip_line("text $unclosed"), "text", "unclosed math ends at the line");
}
#[test]
fn verb_argument_is_dropped() {
assert_eq!(
strip_line(r"should be given in \verb|\institute|."),
"should be given in.",
"`\\verb` uses an arbitrary delimiter, and its content is code"
);
assert_eq!(
strip_line(r"use \verb+\foo+ here"),
"use here",
"the delimiter is whatever follows \\verb"
);
}
#[test]
fn text_commands_keep_their_word() {
assert_eq!(
strip_line(r"produced under \LaTeX\ using"),
"produced under LaTeX using",
"a vanished \\LaTeX leaves a hole in the sentence"
);
assert_eq!(
strip_line(r"searched by \LaTeX{}, then"),
"searched by LaTeX, then",
"the empty-group spacing idiom is consumed, not read as an argument"
);
}
#[test]
fn dropped_command_does_not_strand_punctuation() {
assert_eq!(
strip_line(r"See Subsection \ref{subsec1}."),
"See Subsection.",
"the space left by a dropped command must not precede the period"
);
assert_eq!(
strip_line(r"Example citation, See \cite{lamport94}."),
"Example citation, See.",
"same for a dropped citation"
);
}
#[test]
fn hyphenation_hints_are_dropped() {
assert_eq!(
strip_line(r"\hyphenation{op-tical net-works}"),
"",
"hyphenation hints are typesetting, not prose"
);
}
#[test]
fn plain_text_unchanged() {
assert_eq!(strip_line("hello world"), "hello world", "plain text is unchanged");
}
#[test]
fn removes_comment() {
assert_eq!(strip_line("visible % hidden"), "visible", "trailing comment is removed");
}
#[test]
fn collapses_whitespace() {
assert_eq!(
strip_line(" lots of space "),
"lots of space",
"whitespace is collapsed and trimmed"
);
}
#[test]
fn nested_commands() {
assert_eq!(
strip_line(r"\textbf{\emph{bold and italic}} and more"),
"bold and italic and more",
"nested commands are stripped, prose kept"
);
}
#[test]
fn escaped_percent() {
assert_eq!(
strip_line(r"100\% done"),
"100% done",
"escaped percent is kept as literal character"
);
}
#[test]
fn fragment_plain_text() {
assert_eq!(
strip_fragment("hello world"),
"hello world",
"plain text passes through unchanged"
);
}
#[test]
fn fragment_simple_command() {
assert_eq!(strip_fragment(r"\textbf{bold}"), "bold", "command arg is kept");
}
#[test]
fn fragment_nested_commands() {
assert_eq!(strip_fragment(r"\textbf{\emph{deep}}"), "deep", "nested commands recurse");
}
#[test]
fn fragment_command_without_braces() {
assert_eq!(
strip_fragment(r"\par some text"),
" some text",
"command without braces is removed"
);
}
#[test]
fn fragment_tilde() {
assert_eq!(strip_fragment("z.~B."), "z. B.", "tilde becomes a regular space");
}
#[test]
fn fragment_percent() {
assert_eq!(strip_fragment("visible % hidden"), "visible ", "percent stops parsing");
}
#[test]
fn fragment_single_char_escape() {
assert_eq!(strip_fragment(r#"Universit\"at"#), "Universität", "diaeresis accent resolved");
}
#[test]
fn fragment_accent_acute() {
assert_eq!(strip_fragment(r"caf\'e"), "café", "acute accent resolved");
}
#[test]
fn fragment_accent_brace_group() {
assert_eq!(strip_fragment(r#"Universit\"{a}t"#), "Universität", "accent with brace group");
}
#[test]
fn fragment_optional_arg_kept() {
assert_eq!(
strip_fragment(r"\textbf[opt]{kept}"),
"kept",
"optional arg is skipped, mandatory arg kept for normal commands"
);
}
#[test]
fn fragment_drop_arg_command() {
assert_eq!(
strip_fragment(r"\usepackage[utf8]{inputenc}"),
"",
"drop-arg command discards its mandatory argument"
);
}
#[test]
fn fragment_url_dropped() {
assert_eq!(strip_fragment(r"\url{https://example.com}"), "", "\\url content is dropped");
}
#[test]
fn fragment_math_spacing() {
assert_eq!(
strip_line(r"a \, b \; c \: d \! e \> f"),
"a b c d e f",
"math spacing commands collapse to a single space"
);
}
#[test]
fn fragment_newcommand_with_arity_and_body_dropped() {
assert_eq!(
strip_line(r"\newcommand{\DB}[1]{\todo[color=cyan]{David: #1}}"),
"",
r"\newcommand body must be dropped even with intervening [N]"
);
assert_eq!(
strip_line(r"prose \newcommand{\foo}[2][def]{body \emph{x}} after"),
"prose after",
r"\newcommand with optional default value drops every following arg"
);
}
#[test]
fn fragment_escaped_punctuation_preserved() {
assert_eq!(
strip_line(r"50\% off, item \#42, \$10, file\_name"),
"50% off, item #42, $10, file_name",
"escaped sentence punctuation is kept verbatim"
);
}
#[test]
fn comment_basic() {
assert_eq!(remove_comment("text % comment"), "text ");
}
#[test]
fn comment_escaped_percent() {
assert_eq!(
remove_comment(r"100\% of cases"),
r"100\% of cases",
r"escaped \% is preserved, strip_fragment handles removal"
);
}
#[test]
fn whitespace_basic() {
assert_eq!(collapse_whitespace(" a b "), "a b");
}
#[test]
fn brace_simple_pair() {
assert_eq!(find_matching_brace("{hello}", 0), Some(6), "simple brace pair");
}
#[test]
fn brace_nested_pair() {
assert_eq!(find_matching_brace("{a{b}c}", 0), Some(6), "nested brace pair");
}
#[test]
fn brace_unmatched() {
assert_eq!(find_matching_brace("{no close", 0), None, "unmatched opening brace");
}
#[test]
fn brace_mid_string() {
assert_eq!(
find_matching_brace("xx{inner}yy", 2),
Some(8),
"brace group starting mid-string"
);
}
#[test]
fn brace_empty_group() {
assert_eq!(find_matching_brace("{}", 0), Some(1), "empty brace pair");
}
}