1pub fn is_space(c: char) -> bool {
30 c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)
31}
32
33pub fn split_edges(text: &str) -> (&str, &str, &str) {
37 let core_start = text.find(|c| !is_space(c)).unwrap_or(text.len());
38 let rest = &text[core_start..];
39 let core_len = rest
40 .char_indices()
41 .rev()
42 .find(|&(_, c)| !is_space(c))
43 .map_or(0, |(i, c)| i + c.len_utf8());
44 (&text[..core_start], &rest[..core_len], &rest[core_len..])
45}
46
47pub fn hoist(text: &str, decorate: impl FnOnce(&str) -> String) -> String {
52 let (lead, core, trail) = split_edges(text);
53 if core.is_empty() {
54 return text.to_string();
55 }
56 format!("{lead}{}{trail}", decorate(core))
57}
58
59pub fn is_blank(text: &str) -> bool {
61 text.chars().all(is_space)
62}
63
64pub fn trim_run_edges<T>(
70 mut runs: Vec<T>,
71 text: impl Fn(&T) -> &str,
72 set_text: impl Fn(&mut T, String),
73 trimmable: impl Fn(&T) -> bool,
74) -> Vec<T> {
75 let mut start = 0;
76 while start < runs.len() && trimmable(&runs[start]) {
77 let head = text(&runs[start]).trim_start_matches(is_space).to_string();
78 if !head.is_empty() {
79 set_text(&mut runs[start], head);
80 break;
81 }
82 start += 1;
83 }
84 let mut end = runs.len();
85 while end > start && trimmable(&runs[end - 1]) {
86 let tail = text(&runs[end - 1]).trim_end_matches(is_space).to_string();
87 if !tail.is_empty() {
88 set_text(&mut runs[end - 1], tail);
89 break;
90 }
91 end -= 1;
92 }
93 runs.truncate(end);
94 runs.drain(..start);
95 runs.retain(|r| !text(r).is_empty() || !trimmable(r));
96 runs
97}
98
99pub struct LegacyRun<'a> {
101 pub text: &'a mut String,
103 pub renders_text: bool,
106 pub plain: bool,
109 pub empty_plain: bool,
112}
113
114pub fn separate_legacy_runs(runs: &mut [LegacyRun<'_>]) -> Vec<usize> {
123 let rendered: Vec<usize> = (0..runs.len()).filter(|&i| runs[i].renders_text).collect();
124 let mut standalone = Vec::new();
125 for w in rendered.windows(2) {
126 let (l, r) = (w[0], w[1]);
127 let carries = |run: &LegacyRun<'_>| run.plain && !run.text.is_empty();
128 if carries(&runs[l]) {
129 runs[l].text.push(' ');
130 } else if carries(&runs[r]) {
131 runs[r].text.insert(0, ' ');
132 } else if let Some(e) = (l + 1..r).find(|&i| runs[i].empty_plain) {
133 runs[e].text.push(' ');
134 } else {
135 standalone.push(r);
136 }
137 }
138 standalone
139}
140
141pub fn predates_contract(version: &str) -> bool {
145 let mut it = version.split('.').map(|p| p.trim().parse::<u64>());
146 match (it.next(), it.next()) {
147 (Some(Ok(major)), Some(Ok(minor))) => (major, minor) < (1, 11),
148 _ => false,
149 }
150}
151
152pub fn migrate_legacy_json(doc: &mut serde_json::Value) -> bool {
160 use serde_json::{json, Value};
161 if !doc
162 .get("version")
163 .and_then(Value::as_str)
164 .is_some_and(predates_contract)
165 {
166 return false;
167 }
168 fn locate(r: &Value) -> Option<(bool, usize)> {
169 let r = r.get("$ref").or_else(|| r.get("cref"))?.as_str()?;
170 if let Some(i) = r.strip_prefix("#/texts/") {
171 return Some((true, i.parse().ok()?));
172 }
173 Some((false, r.strip_prefix("#/groups/")?.parse().ok()?))
174 }
175 fn get(doc: &Value, at: (bool, usize)) -> Option<&Value> {
176 doc.get(if at.0 { "texts" } else { "groups" })?.get(at.1)
177 }
178 fn renders_text(doc: &Value, at: (bool, usize), seen: &mut Vec<(bool, usize)>) -> bool {
179 if seen.contains(&at) {
180 return false;
181 }
182 seen.push(at);
183 let Some(item) = get(doc, at) else {
184 return false;
185 };
186 if item
187 .get("text")
188 .and_then(Value::as_str)
189 .is_some_and(|t| !t.is_empty())
190 {
191 return true;
192 }
193 item.get("children")
194 .and_then(Value::as_array)
195 .is_some_and(|cs| {
196 cs.iter()
197 .filter_map(locate)
198 .collect::<Vec<_>>()
199 .into_iter()
200 .any(|c| renders_text(doc, c, seen))
201 })
202 }
203 let n_groups = doc
204 .get("groups")
205 .and_then(Value::as_array)
206 .map_or(0, Vec::len);
207 if doc.get("texts").and_then(Value::as_array).is_none() {
208 return false;
209 }
210 let mut migrated = false;
211 for g in 0..n_groups {
212 let group = &doc["groups"][g];
213 if group.get("label").and_then(Value::as_str) != Some("inline") {
214 continue;
215 }
216 let Some(children) = group.get("children").and_then(Value::as_array) else {
217 continue;
218 };
219 let locs: Vec<Option<(bool, usize)>> = children.iter().map(locate).collect();
220 let mut info = Vec::with_capacity(locs.len());
222 for &loc in &locs {
223 let item = loc.and_then(|at| get(doc, at));
224 let text = item
225 .and_then(|i| i.get("text"))
226 .and_then(Value::as_str)
227 .map(str::to_string);
228 let label = item.and_then(|i| i.get("label")).and_then(Value::as_str);
229 let no_children = item
230 .and_then(|i| i.get("children"))
231 .and_then(Value::as_array)
232 .is_none_or(Vec::is_empty);
233 let renders = loc.is_some_and(|at| renders_text(doc, at, &mut Vec::new()));
234 let plain = text.is_some() && !matches!(label, Some("code" | "formula"));
235 let empty_plain = text.as_deref() == Some("") && label == Some("text") && no_children;
236 info.push((renders, plain, empty_plain, text.unwrap_or_default()));
237 }
238 let mut texts: Vec<String> = info.iter().map(|i| i.3.clone()).collect();
239 let standalone = {
240 let mut runs: Vec<LegacyRun<'_>> = texts
241 .iter_mut()
242 .zip(&info)
243 .map(|(text, i)| LegacyRun {
244 text,
245 renders_text: i.0,
246 plain: i.1,
247 empty_plain: i.2,
248 })
249 .collect();
250 let before = runs.iter().map(|r| r.text.len()).sum::<usize>();
251 let s = separate_legacy_runs(&mut runs);
252 migrated |= !s.is_empty() || runs.iter().map(|r| r.text.len()).sum::<usize>() != before;
253 s
254 };
255 for ((loc, new), i) in locs.iter().zip(texts).zip(&info) {
256 if let Some(at) = *loc {
257 if new != i.3 {
258 doc[if at.0 { "texts" } else { "groups" }][at.1]["text"] = Value::String(new);
259 }
260 }
261 }
262 let self_ref = doc["groups"][g]
263 .get("self_ref")
264 .cloned()
265 .unwrap_or(Value::Null);
266 let layer = doc["groups"][g]
267 .get("content_layer")
268 .cloned()
269 .unwrap_or_else(|| json!("body"));
270 for (shift, at) in standalone.into_iter().enumerate() {
271 let prov_of = |loc: Option<(bool, usize)>, last: bool| {
272 loc.and_then(|l| get(doc, l))
273 .and_then(|i| i.get("prov"))
274 .and_then(Value::as_array)
275 .and_then(|p| if last { p.last() } else { p.first() })
276 .cloned()
277 };
278 let (left, right) = (prov_of(locs[at - 1], true), prov_of(locs[at], false));
279 let prov = match (left, right) {
280 (Some(Value::Object(mut l)), Some(r))
281 if r.is_object() && l.get("page_no") == r.get("page_no") =>
282 {
283 l.insert("charspan".into(), json!([0, 1]));
284 vec![Value::Object(l)]
285 }
286 _ => Vec::new(),
287 };
288 let texts = doc["texts"].as_array_mut().expect("checked above");
289 let r = format!("#/texts/{}", texts.len());
290 texts.push(json!({
291 "self_ref": r,
292 "parent": { "$ref": self_ref },
293 "label": "text",
294 "prov": prov,
295 "orig": " ",
296 "text": " ",
297 "children": [],
298 "content_layer": layer,
299 }));
300 if let Some(cs) = doc["groups"][g]["children"].as_array_mut() {
301 cs.insert(at + shift, json!({ "$ref": r }));
302 }
303 }
304 }
305 migrated
306}
307
308#[cfg(test)]
309mod tests {
310 use super::*;
311
312 #[test]
313 fn edges_split_like_python_strip() {
314 assert_eq!(split_edges(" bold \n"), (" ", "bold", " \n"));
315 assert_eq!(split_edges("x"), ("", "x", ""));
316 assert_eq!(split_edges(" "), (" ", "", ""));
317 assert_eq!(split_edges(""), ("", "", ""));
318 assert_eq!(split_edges("\u{a0}a b\u{1f}"), ("\u{a0}", "a b", "\u{1f}"));
319 }
320
321 #[test]
322 fn hoisting_keeps_whitespace_outside_the_decoration() {
323 let bold = |s: &str| format!("**{s}**");
324 assert_eq!(hoist("bold ", bold), "**bold** ");
325 assert_eq!(hoist(" x ", bold), " **x** ");
326 assert_eq!(hoist(" ", bold), " ");
327 assert_eq!(hoist("", bold), "");
328 }
329
330 #[test]
331 fn trim_run_edges_matches_docling() {
332 let t = |v: &[&str]| {
333 trim_run_edges(
334 v.iter().map(|s| s.to_string()).collect(),
335 |s: &String| s.as_str(),
336 |s, n| *s = n,
337 |s| s != "<br>",
338 )
339 };
340 assert_eq!(t(&[" ", " a ", " b", " "]), ["a ", " b"]);
341 assert_eq!(t(&[" a", "", "b "]), ["a", "b"]);
342 assert_eq!(t(&["<br>", " a "]), ["<br>", " a"]);
343 assert_eq!(t(&[" "]), Vec::<String>::new());
344 }
345
346 fn run(text: &mut String, plain: bool) -> LegacyRun<'_> {
347 let renders_text = !text.is_empty();
348 LegacyRun {
349 empty_plain: plain && text.is_empty(),
350 text,
351 renders_text,
352 plain,
353 }
354 }
355
356 #[test]
357 fn legacy_separators_land_outside_delimiters() {
358 let (mut a, mut b, mut c) = (
360 "Normal".to_string(),
361 "italic".to_string(),
362 "bold".to_string(),
363 );
364 let ins =
365 separate_legacy_runs(&mut [run(&mut a, true), run(&mut b, true), run(&mut c, true)]);
366 assert!(ins.is_empty());
367 assert_eq!(
368 (a.as_str(), b.as_str(), c.as_str()),
369 ("Normal ", "italic ", "bold")
370 );
371 let (mut a, mut b) = ("x".to_string(), "y".to_string());
373 separate_legacy_runs(&mut [run(&mut a, false), run(&mut b, true)]);
374 assert_eq!((a.as_str(), b.as_str()), ("x", " y"));
375 let (mut a, mut e, mut b) = ("a".to_string(), String::new(), "b".to_string());
377 let ins =
378 separate_legacy_runs(&mut [run(&mut a, false), run(&mut e, true), run(&mut b, false)]);
379 assert!(ins.is_empty());
380 assert_eq!(e, " ");
381 let (mut a, mut b) = ("a".to_string(), "b".to_string());
383 assert_eq!(
384 separate_legacy_runs(&mut [run(&mut a, false), run(&mut b, false)]),
385 [1]
386 );
387 }
388
389 #[test]
390 fn a_pre_contract_document_gets_its_separators_back() {
391 let mut doc = serde_json::json!({
392 "version": "1.10.0",
393 "texts": [
394 {"self_ref": "#/texts/0", "label": "text", "text": "Normal", "orig": "Normal", "prov": []},
395 {"self_ref": "#/texts/1", "label": "code", "text": "a", "orig": "a",
396 "prov": [{"page_no": 1, "charspan": [0, 1]}]},
397 {"self_ref": "#/texts/2", "label": "code", "text": "b", "orig": "b",
398 "prov": [{"page_no": 1, "charspan": [0, 1]}]},
399 ],
400 "groups": [{"self_ref": "#/groups/0", "label": "inline", "content_layer": "body",
401 "children": [{"$ref": "#/texts/0"}, {"$ref": "#/texts/1"}, {"$ref": "#/texts/2"}]}],
402 });
403 assert!(migrate_legacy_json(&mut doc));
404 assert_eq!(doc["texts"][0]["text"], "Normal ");
405 assert_eq!(doc["texts"][0]["orig"], "Normal");
406 assert_eq!(doc["texts"][3]["text"], " ");
407 assert_eq!(
408 doc["texts"][3]["prov"][0]["charspan"],
409 serde_json::json!([0, 1])
410 );
411 assert_eq!(doc["groups"][0]["children"][2]["$ref"], "#/texts/3");
412 for v in [serde_json::json!("1.11.0"), serde_json::Value::Null] {
414 let mut d = doc.clone();
415 d["version"] = v;
416 assert!(!migrate_legacy_json(&mut d));
417 }
418 assert!(predates_contract("1.9.2") && !predates_contract("2.0.0"));
419 }
420}