pub fn is_space(c: char) -> bool {
c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)
}
pub fn split_edges(text: &str) -> (&str, &str, &str) {
let core_start = text.find(|c| !is_space(c)).unwrap_or(text.len());
let rest = &text[core_start..];
let core_len = rest
.char_indices()
.rev()
.find(|&(_, c)| !is_space(c))
.map_or(0, |(i, c)| i + c.len_utf8());
(&text[..core_start], &rest[..core_len], &rest[core_len..])
}
pub fn hoist(text: &str, decorate: impl FnOnce(&str) -> String) -> String {
let (lead, core, trail) = split_edges(text);
if core.is_empty() {
return text.to_string();
}
format!("{lead}{}{trail}", decorate(core))
}
pub fn is_blank(text: &str) -> bool {
text.chars().all(is_space)
}
pub fn trim_run_edges<T>(
mut runs: Vec<T>,
text: impl Fn(&T) -> &str,
set_text: impl Fn(&mut T, String),
trimmable: impl Fn(&T) -> bool,
) -> Vec<T> {
let mut start = 0;
while start < runs.len() && trimmable(&runs[start]) {
let head = text(&runs[start]).trim_start_matches(is_space).to_string();
if !head.is_empty() {
set_text(&mut runs[start], head);
break;
}
start += 1;
}
let mut end = runs.len();
while end > start && trimmable(&runs[end - 1]) {
let tail = text(&runs[end - 1]).trim_end_matches(is_space).to_string();
if !tail.is_empty() {
set_text(&mut runs[end - 1], tail);
break;
}
end -= 1;
}
runs.truncate(end);
runs.drain(..start);
runs.retain(|r| !text(r).is_empty() || !trimmable(r));
runs
}
pub struct LegacyRun<'a> {
pub text: &'a mut String,
pub renders_text: bool,
pub plain: bool,
pub empty_plain: bool,
}
pub fn separate_legacy_runs(runs: &mut [LegacyRun<'_>]) -> Vec<usize> {
let rendered: Vec<usize> = (0..runs.len()).filter(|&i| runs[i].renders_text).collect();
let mut standalone = Vec::new();
for w in rendered.windows(2) {
let (l, r) = (w[0], w[1]);
let carries = |run: &LegacyRun<'_>| run.plain && !run.text.is_empty();
if carries(&runs[l]) {
runs[l].text.push(' ');
} else if carries(&runs[r]) {
runs[r].text.insert(0, ' ');
} else if let Some(e) = (l + 1..r).find(|&i| runs[i].empty_plain) {
runs[e].text.push(' ');
} else {
standalone.push(r);
}
}
standalone
}
pub fn predates_contract(version: &str) -> bool {
let mut it = version.split('.').map(|p| p.trim().parse::<u64>());
match (it.next(), it.next()) {
(Some(Ok(major)), Some(Ok(minor))) => (major, minor) < (1, 11),
_ => false,
}
}
pub fn migrate_legacy_json(doc: &mut serde_json::Value) -> bool {
use serde_json::{json, Value};
if !doc
.get("version")
.and_then(Value::as_str)
.is_some_and(predates_contract)
{
return false;
}
fn locate(r: &Value) -> Option<(bool, usize)> {
let r = r.get("$ref").or_else(|| r.get("cref"))?.as_str()?;
if let Some(i) = r.strip_prefix("#/texts/") {
return Some((true, i.parse().ok()?));
}
Some((false, r.strip_prefix("#/groups/")?.parse().ok()?))
}
fn get(doc: &Value, at: (bool, usize)) -> Option<&Value> {
doc.get(if at.0 { "texts" } else { "groups" })?.get(at.1)
}
fn renders_text(doc: &Value, at: (bool, usize), seen: &mut Vec<(bool, usize)>) -> bool {
if seen.contains(&at) {
return false;
}
seen.push(at);
let Some(item) = get(doc, at) else {
return false;
};
if item
.get("text")
.and_then(Value::as_str)
.is_some_and(|t| !t.is_empty())
{
return true;
}
item.get("children")
.and_then(Value::as_array)
.is_some_and(|cs| {
cs.iter()
.filter_map(locate)
.collect::<Vec<_>>()
.into_iter()
.any(|c| renders_text(doc, c, seen))
})
}
let n_groups = doc
.get("groups")
.and_then(Value::as_array)
.map_or(0, Vec::len);
if doc.get("texts").and_then(Value::as_array).is_none() {
return false;
}
let mut migrated = false;
for g in 0..n_groups {
let group = &doc["groups"][g];
if group.get("label").and_then(Value::as_str) != Some("inline") {
continue;
}
let Some(children) = group.get("children").and_then(Value::as_array) else {
continue;
};
let locs: Vec<Option<(bool, usize)>> = children.iter().map(locate).collect();
let mut info = Vec::with_capacity(locs.len());
for &loc in &locs {
let item = loc.and_then(|at| get(doc, at));
let text = item
.and_then(|i| i.get("text"))
.and_then(Value::as_str)
.map(str::to_string);
let label = item.and_then(|i| i.get("label")).and_then(Value::as_str);
let no_children = item
.and_then(|i| i.get("children"))
.and_then(Value::as_array)
.is_none_or(Vec::is_empty);
let renders = loc.is_some_and(|at| renders_text(doc, at, &mut Vec::new()));
let plain = text.is_some() && !matches!(label, Some("code" | "formula"));
let empty_plain = text.as_deref() == Some("") && label == Some("text") && no_children;
info.push((renders, plain, empty_plain, text.unwrap_or_default()));
}
let mut texts: Vec<String> = info.iter().map(|i| i.3.clone()).collect();
let standalone = {
let mut runs: Vec<LegacyRun<'_>> = texts
.iter_mut()
.zip(&info)
.map(|(text, i)| LegacyRun {
text,
renders_text: i.0,
plain: i.1,
empty_plain: i.2,
})
.collect();
let before = runs.iter().map(|r| r.text.len()).sum::<usize>();
let s = separate_legacy_runs(&mut runs);
migrated |= !s.is_empty() || runs.iter().map(|r| r.text.len()).sum::<usize>() != before;
s
};
for ((loc, new), i) in locs.iter().zip(texts).zip(&info) {
if let Some(at) = *loc {
if new != i.3 {
doc[if at.0 { "texts" } else { "groups" }][at.1]["text"] = Value::String(new);
}
}
}
let self_ref = doc["groups"][g]
.get("self_ref")
.cloned()
.unwrap_or(Value::Null);
let layer = doc["groups"][g]
.get("content_layer")
.cloned()
.unwrap_or_else(|| json!("body"));
for (shift, at) in standalone.into_iter().enumerate() {
let prov_of = |loc: Option<(bool, usize)>, last: bool| {
loc.and_then(|l| get(doc, l))
.and_then(|i| i.get("prov"))
.and_then(Value::as_array)
.and_then(|p| if last { p.last() } else { p.first() })
.cloned()
};
let (left, right) = (prov_of(locs[at - 1], true), prov_of(locs[at], false));
let prov = match (left, right) {
(Some(Value::Object(mut l)), Some(r))
if r.is_object() && l.get("page_no") == r.get("page_no") =>
{
l.insert("charspan".into(), json!([0, 1]));
vec![Value::Object(l)]
}
_ => Vec::new(),
};
let texts = doc["texts"].as_array_mut().expect("checked above");
let r = format!("#/texts/{}", texts.len());
texts.push(json!({
"self_ref": r,
"parent": { "$ref": self_ref },
"label": "text",
"prov": prov,
"orig": " ",
"text": " ",
"children": [],
"content_layer": layer,
}));
if let Some(cs) = doc["groups"][g]["children"].as_array_mut() {
cs.insert(at + shift, json!({ "$ref": r }));
}
}
}
migrated
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn edges_split_like_python_strip() {
assert_eq!(split_edges(" bold \n"), (" ", "bold", " \n"));
assert_eq!(split_edges("x"), ("", "x", ""));
assert_eq!(split_edges(" "), (" ", "", ""));
assert_eq!(split_edges(""), ("", "", ""));
assert_eq!(split_edges("\u{a0}a b\u{1f}"), ("\u{a0}", "a b", "\u{1f}"));
}
#[test]
fn hoisting_keeps_whitespace_outside_the_decoration() {
let bold = |s: &str| format!("**{s}**");
assert_eq!(hoist("bold ", bold), "**bold** ");
assert_eq!(hoist(" x ", bold), " **x** ");
assert_eq!(hoist(" ", bold), " ");
assert_eq!(hoist("", bold), "");
}
#[test]
fn trim_run_edges_matches_docling() {
let t = |v: &[&str]| {
trim_run_edges(
v.iter().map(|s| s.to_string()).collect(),
|s: &String| s.as_str(),
|s, n| *s = n,
|s| s != "<br>",
)
};
assert_eq!(t(&[" ", " a ", " b", " "]), ["a ", " b"]);
assert_eq!(t(&[" a", "", "b "]), ["a", "b"]);
assert_eq!(t(&["<br>", " a "]), ["<br>", " a"]);
assert_eq!(t(&[" "]), Vec::<String>::new());
}
fn run(text: &mut String, plain: bool) -> LegacyRun<'_> {
let renders_text = !text.is_empty();
LegacyRun {
empty_plain: plain && text.is_empty(),
text,
renders_text,
plain,
}
}
#[test]
fn legacy_separators_land_outside_delimiters() {
let (mut a, mut b, mut c) = (
"Normal".to_string(),
"italic".to_string(),
"bold".to_string(),
);
let ins =
separate_legacy_runs(&mut [run(&mut a, true), run(&mut b, true), run(&mut c, true)]);
assert!(ins.is_empty());
assert_eq!(
(a.as_str(), b.as_str(), c.as_str()),
("Normal ", "italic ", "bold")
);
let (mut a, mut b) = ("x".to_string(), "y".to_string());
separate_legacy_runs(&mut [run(&mut a, false), run(&mut b, true)]);
assert_eq!((a.as_str(), b.as_str()), ("x", " y"));
let (mut a, mut e, mut b) = ("a".to_string(), String::new(), "b".to_string());
let ins =
separate_legacy_runs(&mut [run(&mut a, false), run(&mut e, true), run(&mut b, false)]);
assert!(ins.is_empty());
assert_eq!(e, " ");
let (mut a, mut b) = ("a".to_string(), "b".to_string());
assert_eq!(
separate_legacy_runs(&mut [run(&mut a, false), run(&mut b, false)]),
[1]
);
}
#[test]
fn a_pre_contract_document_gets_its_separators_back() {
let mut doc = serde_json::json!({
"version": "1.10.0",
"texts": [
{"self_ref": "#/texts/0", "label": "text", "text": "Normal", "orig": "Normal", "prov": []},
{"self_ref": "#/texts/1", "label": "code", "text": "a", "orig": "a",
"prov": [{"page_no": 1, "charspan": [0, 1]}]},
{"self_ref": "#/texts/2", "label": "code", "text": "b", "orig": "b",
"prov": [{"page_no": 1, "charspan": [0, 1]}]},
],
"groups": [{"self_ref": "#/groups/0", "label": "inline", "content_layer": "body",
"children": [{"$ref": "#/texts/0"}, {"$ref": "#/texts/1"}, {"$ref": "#/texts/2"}]}],
});
assert!(migrate_legacy_json(&mut doc));
assert_eq!(doc["texts"][0]["text"], "Normal ");
assert_eq!(doc["texts"][0]["orig"], "Normal");
assert_eq!(doc["texts"][3]["text"], " ");
assert_eq!(
doc["texts"][3]["prov"][0]["charspan"],
serde_json::json!([0, 1])
);
assert_eq!(doc["groups"][0]["children"][2]["$ref"], "#/texts/3");
for v in [serde_json::json!("1.11.0"), serde_json::Value::Null] {
let mut d = doc.clone();
d["version"] = v;
assert!(!migrate_legacy_json(&mut d));
}
assert!(predates_contract("1.9.2") && !predates_contract("2.0.0"));
}
}