pub(super) fn escaped_strings(text: &str) -> Vec<String> {
escaped_strings_inner(text, &mut RecoveryWork::default())
.into_iter()
.map(|view| view.text)
.collect()
}
pub(super) struct DecodedSpan {
pub(super) text: String,
pub(super) source: std::ops::Range<usize>,
}
pub(super) fn escaped_views(text: &str) -> Vec<DecodedSpan> {
escaped_strings_inner(text, &mut RecoveryWork::default())
}
fn escaped_strings_inner(text: &str, work: &mut RecoveryWork) -> Vec<DecodedSpan> {
if !text.contains('\\') {
return Vec::new();
}
let Some(mut start) = text.find('"') else {
return Vec::new();
};
let mut previous_quote = start;
let mut decoded = Vec::new();
let mut backslashes = 0usize;
for (i, &b) in text.as_bytes().iter().enumerate().skip(start + 1) {
if b == b'"' {
if backslashes.is_multiple_of(2) {
let literal = &text[start..=i];
if literal.contains('\\') {
if let Ok(value) = serde_json::from_str::<String>(literal) {
decoded.push(DecodedSpan {
text: value,
source: start + 1..i,
});
} else {
recover_valid_spans(&text[start + 1..i], start + 1, &mut decoded, work);
}
}
start = i;
} else {
let fragment = format!("{}\"", &text[previous_quote..=i]);
if serde_json::from_str::<String>(&fragment).is_err() {
recover_valid_spans(&text[start + 1..=i], start + 1, &mut decoded, work);
start = i;
}
}
previous_quote = i;
}
backslashes = if b == b'\\' { backslashes + 1 } else { 0 };
}
let tail = &text[start..];
if tail.contains('\\') {
let literal = format!("{tail}\"");
if let Ok(value) = serde_json::from_str::<String>(&literal) {
decoded.push(DecodedSpan {
text: value,
source: start + 1..text.len(),
});
} else {
recover_valid_spans(&text[start + 1..], start + 1, &mut decoded, work);
}
}
decoded
}
fn recover_valid_spans(
interior: &str,
base: usize,
decoded: &mut Vec<DecodedSpan>,
work: &mut RecoveryWork,
) {
work.begin_pass();
let mut cursor = 0;
let mut start = 0;
let mut escaped = false;
while cursor < interior.len() {
let (len, valid) = recovery_token(&interior[cursor..], work);
if valid {
escaped |= interior.as_bytes()[cursor] == b'\\';
} else {
emit_span(
&interior[start..cursor],
base + start..base + cursor,
escaped,
decoded,
);
start = cursor + len;
escaped = false;
}
cursor += len;
}
emit_span(
&interior[start..],
base + start..base + interior.len(),
escaped,
decoded,
);
}
fn emit_span(
span: &str,
source: std::ops::Range<usize>,
escaped: bool,
decoded: &mut Vec<DecodedSpan>,
) {
if escaped {
let literal = format!("\"{span}\"");
if let Ok(value) = serde_json::from_str::<String>(&literal) {
decoded.push(DecodedSpan {
text: value,
source,
});
}
}
}
fn recovery_token(text: &str, work: &mut RecoveryWork) -> (usize, bool) {
work.visit_token();
let first = text.chars().next().expect("nonempty recovery suffix");
if first != '\\' {
return (first.len_utf8(), first >= ' ' && first != '"');
}
let Some(escaped) = text[1..].chars().next() else {
return (1, false);
};
match escaped {
'"' | '\\' | '/' | 'b' | 'f' | 'n' | 'r' | 't' => (2, true),
'u' => {
let (len, scalar) = unicode_escape(text);
match scalar {
Some(0xD800..=0xDBFF) => {
let next = &text[len..];
if next.starts_with("\\u") {
let (next_len, low) = unicode_escape(next);
if matches!(low, Some(0xDC00..=0xDFFF)) {
return (len + next_len, true);
}
}
(len, false)
}
Some(0xDC00..=0xDFFF) | None => (len, false),
Some(_) => (len, true),
}
}
_ => (1 + escaped.len_utf8(), false),
}
}
fn unicode_escape(text: &str) -> (usize, Option<u16>) {
let mut len = 2;
let mut value = 0;
for &byte in text.as_bytes()[2..].iter().take(4) {
let digit = match byte {
b'0'..=b'9' => byte - b'0',
b'a'..=b'f' => byte - b'a' + 10,
b'A'..=b'F' => byte - b'A' + 10,
_ => break,
};
value = value * 16 + u16::from(digit);
len += 1;
}
(len, (len == 6).then_some(value))
}
#[derive(Default)]
pub(super) struct RecoveryWork {
#[cfg(test)]
pub(super) passes: usize,
#[cfg(test)]
pub(super) tokens: usize,
#[cfg(test)]
max_tokens: Option<usize>,
}
impl RecoveryWork {
fn begin_pass(&mut self) {
#[cfg(test)]
{
self.passes += 1;
}
}
fn visit_token(&mut self) {
#[cfg(test)]
{
self.tokens += 1;
if let Some(limit) = self.max_tokens {
assert!(
self.tokens <= limit,
"recovery token visits exceeded {limit}"
);
}
}
}
}
#[cfg(test)]
pub(super) fn escaped_strings_with_work(
text: &str,
max_tokens: usize,
) -> (Vec<String>, RecoveryWork) {
let mut work = RecoveryWork {
max_tokens: Some(max_tokens),
..RecoveryWork::default()
};
let decoded = escaped_strings_inner(text, &mut work);
(decoded.into_iter().map(|view| view.text).collect(), work)
}
pub(super) fn origin_token(text: &str) -> (usize, bool) {
recovery_token(text, &mut RecoveryWork::default())
}