Skip to main content

docling_core/
inline.rs

1//! docling-core's inline whitespace contract (schema 1.11.0, docling-core#693,
2//! docling#3891).
3//!
4//! The runs of an `inline` group carry their own boundary whitespace, and the
5//! Markdown, HTML and LaTeX serializers concatenate them as they are. Before
6//! the contract every serializer joined runs with a hard `" "`, inventing
7//! spaces the source never had — `2nd` → `2 nd`, `H₂O.` → `H 2 O .`,
8//! `**bold** .` — which no backend could undo while the separator lived in the
9//! serializer. Three helpers carry the contract:
10//!
11//! * [`hoist`] — docling-core's `post_process` with
12//!   `hoist_decoration_whitespace`: a run's edge whitespace goes *outside* its
13//!   whole decoration stack (formatting and hyperlink together), so
14//!   `"bold "` renders `**bold** ` and never `**bold **`, which CommonMark
15//!   prints as literal asterisks; a whitespace-only run is returned verbatim,
16//!   undecorated and unlinked.
17//! * [`trim_run_edges`] — docling's `trim_run_edges` (`docling/utils/_inline_runs.py`):
18//!   a backend that keeps each run's whitespace trims only the block's outer
19//!   edges.
20//! * [`separate_legacy_runs`] — docling-core's
21//!   `_migrate_legacy_inline_separators`: the old `" "` join re-injected into
22//!   the runs themselves, for a document stamped below 1.11.0 and for an item
23//!   tree whose backend still builds trimmed runs ([`crate::tree::ItemTree::inline_whitespace`]
24//!   unset), so their visible output — extra-space bugs included — is what it
25//!   was.
26
27/// Python's `str.isspace` set, which docling-core's `strip()` removes: Unicode
28/// `White_Space` plus the ASCII information separators U+001C–U+001F.
29pub fn is_space(c: char) -> bool {
30    c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)
31}
32
33/// `text` split into its leading whitespace, its core and its trailing
34/// whitespace (Python's `lstrip` / `strip` / `rstrip` boundaries). A
35/// whitespace-only text is all lead.
36pub fn split_edges(text: &str) -> (&str, &str, &str) {
37    let core_start = text.find(|c| !is_space(c)).unwrap_or(text.len());
38    let rest = &text[core_start..];
39    let core_len = rest
40        .char_indices()
41        .rev()
42        .find(|&(_, c)| !is_space(c))
43        .map_or(0, |(i, c)| i + c.len_utf8());
44    (&text[..core_start], &rest[..core_len], &rest[core_len..])
45}
46
47/// docling-core's `post_process` under `hoist_decoration_whitespace`: decorate
48/// the run's core and keep its edge whitespace outside the markers. A
49/// whitespace-only (or empty) run comes back verbatim — no markers, no empty
50/// hyperlink.
51pub fn hoist(text: &str, decorate: impl FnOnce(&str) -> String) -> String {
52    let (lead, core, trail) = split_edges(text);
53    if core.is_empty() {
54        return text.to_string();
55    }
56    format!("{lead}{}{trail}", decorate(core))
57}
58
59/// Whether `text` is empty or whitespace only.
60pub fn is_blank(text: &str) -> bool {
61    text.chars().all(is_space)
62}
63
64/// docling's `trim_run_edges`: drop the whitespace-only runs at either edge
65/// of a block, strip the outer whitespace of the first and last remaining
66/// run, keep every interior run as it is, and drop trimmable runs left
67/// without text. A run `trimmable` refuses (a line break, an image) stops the
68/// trim at its edge and is always kept.
69pub fn trim_run_edges<T>(
70    mut runs: Vec<T>,
71    text: impl Fn(&T) -> &str,
72    set_text: impl Fn(&mut T, String),
73    trimmable: impl Fn(&T) -> bool,
74) -> Vec<T> {
75    let mut start = 0;
76    while start < runs.len() && trimmable(&runs[start]) {
77        let head = text(&runs[start]).trim_start_matches(is_space).to_string();
78        if !head.is_empty() {
79            set_text(&mut runs[start], head);
80            break;
81        }
82        start += 1;
83    }
84    let mut end = runs.len();
85    while end > start && trimmable(&runs[end - 1]) {
86        let tail = text(&runs[end - 1]).trim_end_matches(is_space).to_string();
87        if !tail.is_empty() {
88            set_text(&mut runs[end - 1], tail);
89            break;
90        }
91        end -= 1;
92    }
93    runs.truncate(end);
94    runs.drain(..start);
95    runs.retain(|r| !text(r).is_empty() || !trimmable(r));
96    runs
97}
98
99/// One child of an inline group, as [`separate_legacy_runs`] sees it.
100pub struct LegacyRun<'a> {
101    /// The run's own text (empty for a nested group or an empty host).
102    pub text: &'a mut String,
103    /// Whether the run or its subtree renders any text — docling-core's
104    /// `renders_text`; a run that does not takes no part in the join.
105    pub renders_text: bool,
106    /// A plain text run (not code, not a formula, and with no children of its
107    /// own): only such a run with text takes the separator.
108    pub plain: bool,
109    /// An empty plain `text` run without children — the separator a 1.11 →
110    /// 1.10 projection stripped, refilled instead of inserting another.
111    pub empty_plain: bool,
112}
113
114/// docling-core's `_migrate_legacy_inline_separators`, for one inline group:
115/// one `" "` at every boundary the old serializer joined. Between two
116/// consecutive runs that render text the left run takes a trailing space when
117/// it is a plain run with text, else the right one a leading space, else an
118/// empty plain run between them is refilled, else — both neighbours code or
119/// formula runs, where a space would land inside `` ` ``/`$` — a standalone
120/// `" "` run is needed. Returns the child positions such runs go in front of,
121/// in ascending order (the caller inserts them, shifting as it goes).
122pub fn separate_legacy_runs(runs: &mut [LegacyRun<'_>]) -> Vec<usize> {
123    let rendered: Vec<usize> = (0..runs.len()).filter(|&i| runs[i].renders_text).collect();
124    let mut standalone = Vec::new();
125    for w in rendered.windows(2) {
126        let (l, r) = (w[0], w[1]);
127        let carries = |run: &LegacyRun<'_>| run.plain && !run.text.is_empty();
128        if carries(&runs[l]) {
129            runs[l].text.push(' ');
130        } else if carries(&runs[r]) {
131            runs[r].text.insert(0, ' ');
132        } else if let Some(e) = (l + 1..r).find(|&i| runs[i].empty_plain) {
133            runs[e].text.push(' ');
134        } else {
135            standalone.push(r);
136        }
137    }
138    standalone
139}
140
141/// Whether a docling-JSON `version` predates the inline whitespace contract
142/// (below 1.11.0). An unparsable or missing version is not migrated —
143/// docling-core leaves versionless input alone rather than guess.
144pub fn predates_contract(version: &str) -> bool {
145    let mut it = version.split('.').map(|p| p.trim().parse::<u64>());
146    match (it.next(), it.next()) {
147        (Some(Ok(major)), Some(Ok(minor))) => (major, minor) < (1, 11),
148        _ => false,
149    }
150}
151
152/// docling-core's `_migrate_legacy_inline_separators` on a docling-JSON
153/// document: when its `version` predates 1.11.0, every `inline` group's runs
154/// get the `" "` the old join put between them ([`separate_legacy_runs`]),
155/// a standalone separator run appended to `texts` (with the left run's
156/// location, `charspan` `[0, 1]`, when both runs share a page) where both
157/// neighbours are delimited. Returns whether anything changed. Only `text`
158/// moves; `orig` keeps what the producer wrote.
159pub fn migrate_legacy_json(doc: &mut serde_json::Value) -> bool {
160    use serde_json::{json, Value};
161    if !doc
162        .get("version")
163        .and_then(Value::as_str)
164        .is_some_and(predates_contract)
165    {
166        return false;
167    }
168    fn locate(r: &Value) -> Option<(bool, usize)> {
169        let r = r.get("$ref").or_else(|| r.get("cref"))?.as_str()?;
170        if let Some(i) = r.strip_prefix("#/texts/") {
171            return Some((true, i.parse().ok()?));
172        }
173        Some((false, r.strip_prefix("#/groups/")?.parse().ok()?))
174    }
175    fn get(doc: &Value, at: (bool, usize)) -> Option<&Value> {
176        doc.get(if at.0 { "texts" } else { "groups" })?.get(at.1)
177    }
178    fn renders_text(doc: &Value, at: (bool, usize), seen: &mut Vec<(bool, usize)>) -> bool {
179        if seen.contains(&at) {
180            return false;
181        }
182        seen.push(at);
183        let Some(item) = get(doc, at) else {
184            return false;
185        };
186        if item
187            .get("text")
188            .and_then(Value::as_str)
189            .is_some_and(|t| !t.is_empty())
190        {
191            return true;
192        }
193        item.get("children")
194            .and_then(Value::as_array)
195            .is_some_and(|cs| {
196                cs.iter()
197                    .filter_map(locate)
198                    .collect::<Vec<_>>()
199                    .into_iter()
200                    .any(|c| renders_text(doc, c, seen))
201            })
202    }
203    let n_groups = doc
204        .get("groups")
205        .and_then(Value::as_array)
206        .map_or(0, Vec::len);
207    if doc.get("texts").and_then(Value::as_array).is_none() {
208        return false;
209    }
210    let mut migrated = false;
211    for g in 0..n_groups {
212        let group = &doc["groups"][g];
213        if group.get("label").and_then(Value::as_str) != Some("inline") {
214            continue;
215        }
216        let Some(children) = group.get("children").and_then(Value::as_array) else {
217            continue;
218        };
219        let locs: Vec<Option<(bool, usize)>> = children.iter().map(locate).collect();
220        // (renders_text, plain, empty_plain, text) per child.
221        let mut info = Vec::with_capacity(locs.len());
222        for &loc in &locs {
223            let item = loc.and_then(|at| get(doc, at));
224            let text = item
225                .and_then(|i| i.get("text"))
226                .and_then(Value::as_str)
227                .map(str::to_string);
228            let label = item.and_then(|i| i.get("label")).and_then(Value::as_str);
229            let no_children = item
230                .and_then(|i| i.get("children"))
231                .and_then(Value::as_array)
232                .is_none_or(Vec::is_empty);
233            let renders = loc.is_some_and(|at| renders_text(doc, at, &mut Vec::new()));
234            let plain = text.is_some() && !matches!(label, Some("code" | "formula"));
235            let empty_plain = text.as_deref() == Some("") && label == Some("text") && no_children;
236            info.push((renders, plain, empty_plain, text.unwrap_or_default()));
237        }
238        let mut texts: Vec<String> = info.iter().map(|i| i.3.clone()).collect();
239        let standalone = {
240            let mut runs: Vec<LegacyRun<'_>> = texts
241                .iter_mut()
242                .zip(&info)
243                .map(|(text, i)| LegacyRun {
244                    text,
245                    renders_text: i.0,
246                    plain: i.1,
247                    empty_plain: i.2,
248                })
249                .collect();
250            let before = runs.iter().map(|r| r.text.len()).sum::<usize>();
251            let s = separate_legacy_runs(&mut runs);
252            migrated |= !s.is_empty() || runs.iter().map(|r| r.text.len()).sum::<usize>() != before;
253            s
254        };
255        for ((loc, new), i) in locs.iter().zip(texts).zip(&info) {
256            if let Some(at) = *loc {
257                if new != i.3 {
258                    doc[if at.0 { "texts" } else { "groups" }][at.1]["text"] = Value::String(new);
259                }
260            }
261        }
262        let self_ref = doc["groups"][g]
263            .get("self_ref")
264            .cloned()
265            .unwrap_or(Value::Null);
266        let layer = doc["groups"][g]
267            .get("content_layer")
268            .cloned()
269            .unwrap_or_else(|| json!("body"));
270        for (shift, at) in standalone.into_iter().enumerate() {
271            let prov_of = |loc: Option<(bool, usize)>, last: bool| {
272                loc.and_then(|l| get(doc, l))
273                    .and_then(|i| i.get("prov"))
274                    .and_then(Value::as_array)
275                    .and_then(|p| if last { p.last() } else { p.first() })
276                    .cloned()
277            };
278            let (left, right) = (prov_of(locs[at - 1], true), prov_of(locs[at], false));
279            let prov = match (left, right) {
280                (Some(Value::Object(mut l)), Some(r))
281                    if r.is_object() && l.get("page_no") == r.get("page_no") =>
282                {
283                    l.insert("charspan".into(), json!([0, 1]));
284                    vec![Value::Object(l)]
285                }
286                _ => Vec::new(),
287            };
288            let texts = doc["texts"].as_array_mut().expect("checked above");
289            let r = format!("#/texts/{}", texts.len());
290            texts.push(json!({
291                "self_ref": r,
292                "parent": { "$ref": self_ref },
293                "label": "text",
294                "prov": prov,
295                "orig": " ",
296                "text": " ",
297                "children": [],
298                "content_layer": layer,
299            }));
300            if let Some(cs) = doc["groups"][g]["children"].as_array_mut() {
301                cs.insert(at + shift, json!({ "$ref": r }));
302            }
303        }
304    }
305    migrated
306}
307
308#[cfg(test)]
309mod tests {
310    use super::*;
311
312    #[test]
313    fn edges_split_like_python_strip() {
314        assert_eq!(split_edges("  bold \n"), ("  ", "bold", " \n"));
315        assert_eq!(split_edges("x"), ("", "x", ""));
316        assert_eq!(split_edges("   "), ("   ", "", ""));
317        assert_eq!(split_edges(""), ("", "", ""));
318        assert_eq!(split_edges("\u{a0}a b\u{1f}"), ("\u{a0}", "a b", "\u{1f}"));
319    }
320
321    #[test]
322    fn hoisting_keeps_whitespace_outside_the_decoration() {
323        let bold = |s: &str| format!("**{s}**");
324        assert_eq!(hoist("bold ", bold), "**bold** ");
325        assert_eq!(hoist(" x ", bold), " **x** ");
326        assert_eq!(hoist("  ", bold), "  ");
327        assert_eq!(hoist("", bold), "");
328    }
329
330    #[test]
331    fn trim_run_edges_matches_docling() {
332        let t = |v: &[&str]| {
333            trim_run_edges(
334                v.iter().map(|s| s.to_string()).collect(),
335                |s: &String| s.as_str(),
336                |s, n| *s = n,
337                |s| s != "<br>",
338            )
339        };
340        assert_eq!(t(&["  ", " a ", " b", " "]), ["a ", " b"]);
341        assert_eq!(t(&[" a", "", "b "]), ["a", "b"]);
342        assert_eq!(t(&["<br>", " a "]), ["<br>", " a"]);
343        assert_eq!(t(&["   "]), Vec::<String>::new());
344    }
345
346    fn run(text: &mut String, plain: bool) -> LegacyRun<'_> {
347        let renders_text = !text.is_empty();
348        LegacyRun {
349            empty_plain: plain && text.is_empty(),
350            text,
351            renders_text,
352            plain,
353        }
354    }
355
356    #[test]
357    fn legacy_separators_land_outside_delimiters() {
358        // Normal / italic / bold → `Normal ` `italic ` `bold`.
359        let (mut a, mut b, mut c) = (
360            "Normal".to_string(),
361            "italic".to_string(),
362            "bold".to_string(),
363        );
364        let ins =
365            separate_legacy_runs(&mut [run(&mut a, true), run(&mut b, true), run(&mut c, true)]);
366        assert!(ins.is_empty());
367        assert_eq!(
368            (a.as_str(), b.as_str(), c.as_str()),
369            ("Normal ", "italic ", "bold")
370        );
371        // code, plain → the plain run takes a leading space.
372        let (mut a, mut b) = ("x".to_string(), "y".to_string());
373        separate_legacy_runs(&mut [run(&mut a, false), run(&mut b, true)]);
374        assert_eq!((a.as_str(), b.as_str()), ("x", " y"));
375        // code, empty plain, code → the empty run is refilled.
376        let (mut a, mut e, mut b) = ("a".to_string(), String::new(), "b".to_string());
377        let ins =
378            separate_legacy_runs(&mut [run(&mut a, false), run(&mut e, true), run(&mut b, false)]);
379        assert!(ins.is_empty());
380        assert_eq!(e, " ");
381        // code, code → a standalone run in front of the second.
382        let (mut a, mut b) = ("a".to_string(), "b".to_string());
383        assert_eq!(
384            separate_legacy_runs(&mut [run(&mut a, false), run(&mut b, false)]),
385            [1]
386        );
387    }
388
389    #[test]
390    fn a_pre_contract_document_gets_its_separators_back() {
391        let mut doc = serde_json::json!({
392            "version": "1.10.0",
393            "texts": [
394                {"self_ref": "#/texts/0", "label": "text", "text": "Normal", "orig": "Normal", "prov": []},
395                {"self_ref": "#/texts/1", "label": "code", "text": "a", "orig": "a",
396                 "prov": [{"page_no": 1, "charspan": [0, 1]}]},
397                {"self_ref": "#/texts/2", "label": "code", "text": "b", "orig": "b",
398                 "prov": [{"page_no": 1, "charspan": [0, 1]}]},
399            ],
400            "groups": [{"self_ref": "#/groups/0", "label": "inline", "content_layer": "body",
401                        "children": [{"$ref": "#/texts/0"}, {"$ref": "#/texts/1"}, {"$ref": "#/texts/2"}]}],
402        });
403        assert!(migrate_legacy_json(&mut doc));
404        assert_eq!(doc["texts"][0]["text"], "Normal ");
405        assert_eq!(doc["texts"][0]["orig"], "Normal");
406        assert_eq!(doc["texts"][3]["text"], " ");
407        assert_eq!(
408            doc["texts"][3]["prov"][0]["charspan"],
409            serde_json::json!([0, 1])
410        );
411        assert_eq!(doc["groups"][0]["children"][2]["$ref"], "#/texts/3");
412        // A current or versionless document is left alone.
413        for v in [serde_json::json!("1.11.0"), serde_json::Value::Null] {
414            let mut d = doc.clone();
415            d["version"] = v;
416            assert!(!migrate_legacy_json(&mut d));
417        }
418        assert!(predates_contract("1.9.2") && !predates_contract("2.0.0"));
419    }
420}