docling 1.98.1

DocumentConverter and format backends for docling.rs (a Rust port of docling).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248

//! OOXML markup compatibility (ECMA-376 Part 3): `mc:AlternateContent`.
//!
//! Word writes a feature newer than the base schema twice: an `mc:Choice
//! Requires="wps"` holding the modern markup (a DrawingML text box, a 2010
//! shape group, a `w14` text effect) and an `mc:Fallback` holding what an
//! older reader shows instead (the same text box as VML, a picture of the
//! shape). A reader is to take the first `Choice` whose required namespaces it
//! understands, else the `Fallback`, and never both.
//!
//! The DOCX backends used to see both branches: a body-level block was skipped
//! outright (python-docx's `body.iterchildren()` does the same, so docling
//! loses those paragraphs too), and a text box arrived twice — once from the
//! `wps` drawing, once from the VML fallback — which a text-keyed dedup then
//! folded, dropping a paragraph legitimately repeated inside one box along
//! with the copy (#572). [`resolve_alternate_content`] settles the choice
//! before the XML is parsed into a tree: every `mc:AlternateContent` is
//! replaced by the content of its selected branch, so the two backends, with
//! their many `descendants()` scans, read one document with no alternatives
//! in it and need no dedup. A deliberate divergence from docling — Word, its
//! COM mirror of the same file as `.doc`, and anydoc all keep the content.

use std::borrow::Cow;

use roxmltree::{Document, Node};

/// The markup-compatibility namespace of `mc:AlternateContent` and friends.
pub(super) const MC_NS: &str = "http://schemas.openxmlformats.org/markup-compatibility/2006";

/// The namespaces whose markup the DOCX backends read, so a `Choice`
/// requiring them is taken over the fallback: 2010 wordprocessing shapes
/// (text boxes — `txbxContent` under `wps:txbx`), shape groups and canvases
/// (text boxes inside them), the 2010 drawing and `w14` extensions (the same
/// runs and pictures with extra properties — `w14:checkbox` is read), and
/// DrawingML 2010 picture extensions (the `a:blip` is the same). Anything
/// else — ink (`wpi`), chartex (`cx`), the `w15`/`w16*` extensions — takes
/// the fallback Word wrote for readers without it.
const SUPPORTED: &[&str] = &[
    "http://schemas.microsoft.com/office/word/2010/wordprocessingShape",
    "http://schemas.microsoft.com/office/word/2010/wordprocessingGroup",
    "http://schemas.microsoft.com/office/word/2010/wordprocessingCanvas",
    "http://schemas.microsoft.com/office/word/2010/wordprocessingDrawing",
    "http://schemas.microsoft.com/office/word/2010/wordml",
    "http://schemas.microsoft.com/office/drawing/2010/main",
];

/// Nesting the loop tolerates: a selected branch may itself hold an
/// `mc:AlternateContent` (a text box inside a shape group), each pass settles
/// one level, and real documents nest two or three deep.
const MAX_DEPTH: usize = 8;

/// `xml` with every `mc:AlternateContent` replaced by the content of its
/// selected branch — the first `mc:Choice` whose `Requires` prefixes all
/// resolve to a [`SUPPORTED`] namespace, else the `mc:Fallback`, else nothing.
/// Namespace declarations the dropped wrapper elements carried are
/// re-declared on the kept content's top-level elements, so it parses as it
/// did. XML that does not parse, or has no alternatives, is returned as is.
pub(super) fn resolve_alternate_content(xml: &str) -> Cow<'_, str> {
    if !xml.contains("AlternateContent") {
        return Cow::Borrowed(xml);
    }
    let mut current = Cow::Borrowed(xml);
    for _ in 0..MAX_DEPTH {
        let Ok(doc) = Document::parse(&current) else {
            return current;
        };
        let edits = outermost_edits(&doc);
        if edits.is_empty() {
            return current;
        }
        let mut out = String::with_capacity(current.len());
        let mut pos = 0;
        for (range, replacement) in edits {
            out.push_str(&current[pos..range.start]);
            out.push_str(&replacement);
            pos = range.end;
        }
        out.push_str(&current[pos..]);
        current = Cow::Owned(out);
    }
    current
}

/// `(range of the mc:AlternateContent element, its replacement text)` for
/// every alternative not nested in another, in document order.
fn outermost_edits(doc: &Document) -> Vec<(std::ops::Range<usize>, String)> {
    let is_alt = |n: Node| n.is_element() && n.tag_name() == (MC_NS, "AlternateContent").into();
    doc.descendants()
        .filter(|n| is_alt(*n) && !n.ancestors().skip(1).any(is_alt))
        .map(|alt| (alt.range(), branch_text(doc, alt)))
        .collect()
}

/// The selected branch's inner markup, with the namespace declarations of
/// the wrapper elements hoisted onto its top-level elements.
fn branch_text(doc: &Document, alt: Node) -> String {
    let Some(branch) = select_branch(alt) else {
        return String::new();
    };
    let src = doc.input_text();
    let (Some(first), Some(last)) = (branch.first_child(), branch.last_child()) else {
        return String::new();
    };
    let inner = first.range().start..last.range().end;
    // Declarations in scope inside the branch but not outside the
    // alternative: those lived on the wrapper elements being removed.
    let outside = alt.parent().unwrap_or_else(|| doc.root());
    let hoisted: Vec<String> = branch
        .namespaces()
        .filter(|ns| outside.lookup_namespace_uri(ns.name()) != Some(ns.uri()))
        .map(|ns| match ns.name() {
            Some(p) => format!(" xmlns:{p}=\"{}\"", ns.uri()),
            None => format!(" xmlns=\"{}\"", ns.uri()),
        })
        .collect();
    if hoisted.is_empty() {
        return src[inner].to_string();
    }
    // Insert after each top-level element's qualified name, back to front so
    // the earlier offsets stay valid.
    let mut text = src[inner.clone()].to_string();
    let decls = hoisted.concat();
    let mut tops: Vec<Node> = branch.children().filter(Node::is_element).collect();
    tops.reverse();
    for el in tops {
        let at = qname_end(src, el.range().start) - inner.start;
        text.insert_str(at, &decls);
    }
    text
}

/// The offset just past an element's qualified name, `start` being its `<`:
/// where an attribute may be inserted.
fn qname_end(src: &str, start: usize) -> usize {
    let tag = &src[start + 1..];
    let len = tag
        .find(|c: char| c.is_ascii_whitespace() || c == '/' || c == '>')
        .unwrap_or(tag.len());
    start + 1 + len
}

/// The branch a reader with [`SUPPORTED`] takes: the first `mc:Choice` whose
/// `Requires` prefixes (resolved in the choice's own scope — an unknown
/// prefix matches nothing) are all supported, else the `mc:Fallback`.
fn select_branch<'a, 'i>(alt: Node<'a, 'i>) -> Option<Node<'a, 'i>> {
    let mc = |n: &Node, name: &str| n.is_element() && n.tag_name() == (MC_NS, name).into();
    alt.children()
        .filter(|n| mc(n, "Choice"))
        .find(|choice| {
            choice
                .attribute((MC_NS, "Requires"))
                .or_else(|| choice.attribute("Requires"))
                .is_none_or(|req| {
                    req.split_whitespace().all(|prefix| {
                        choice
                            .lookup_namespace_uri(Some(prefix))
                            .is_some_and(|uri| SUPPORTED.contains(&uri))
                    })
                })
        })
        .or_else(|| alt.children().find(|n| mc(n, "Fallback")))
}

#[cfg(test)]
mod tests {
    use super::*;

    const HEAD: &str = r#"<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:wps="http://schemas.microsoft.com/office/word/2010/wordprocessingShape" xmlns:wpi="http://schemas.microsoft.com/office/word/2010/wordprocessingInk" mc:Ignorable="wps wpi"><w:body>"#;
    const TAIL: &str = "</w:body></w:document>";

    fn paragraphs(xml: &str) -> Vec<String> {
        let doc = Document::parse(xml).unwrap();
        doc.descendants()
            .filter(|n| n.has_tag_name("t"))
            .map(|t| t.text().unwrap_or("").to_string())
            .collect()
    }

    #[test]
    fn supported_choice_wins_and_fallback_is_gone() {
        let xml = format!(
            "{HEAD}<w:p><w:r><w:t>before</w:t></w:r></w:p>\
             <mc:AlternateContent><mc:Choice Requires=\"wps\">\
             <w:p><w:r><w:t>choice</w:t></w:r></w:p><w:p><w:r><w:t>twice</w:t></w:r></w:p>\
             <w:p><w:r><w:t>twice</w:t></w:r></w:p></mc:Choice>\
             <mc:Fallback><w:p><w:r><w:t>fallback</w:t></w:r></w:p></mc:Fallback>\
             </mc:AlternateContent><w:p><w:r><w:t>after</w:t></w:r></w:p>{TAIL}"
        );
        let out = resolve_alternate_content(&xml);
        assert!(!out.contains("AlternateContent") && !out.contains("fallback"));
        assert_eq!(
            paragraphs(&out),
            ["before", "choice", "twice", "twice", "after"]
        );
        // The body's children are now the paragraphs themselves.
        let doc = Document::parse(&out).unwrap();
        let body = doc.descendants().find(|n| n.has_tag_name("body")).unwrap();
        assert_eq!(body.children().filter(Node::is_element).count(), 5);
    }

    #[test]
    fn unsupported_requirement_takes_the_fallback() {
        let xml = format!(
            "{HEAD}<mc:AlternateContent><mc:Choice Requires=\"wpi\">\
             <w:p><w:r><w:t>ink</w:t></w:r></w:p></mc:Choice>\
             <mc:Choice Requires=\"wps unknown\"><w:p><w:r><w:t>partly</w:t></w:r></w:p></mc:Choice>\
             <mc:Fallback><w:p><w:r><w:t>picture</w:t></w:r></w:p></mc:Fallback>\
             </mc:AlternateContent>{TAIL}"
        );
        assert_eq!(paragraphs(&resolve_alternate_content(&xml)), ["picture"]);
        // No fallback either: the alternative contributes nothing.
        let xml = format!(
            "{HEAD}<mc:AlternateContent><mc:Choice Requires=\"wpi\">\
             <w:p><w:r><w:t>ink</w:t></w:r></w:p></mc:Choice></mc:AlternateContent>\
             <w:p><w:r><w:t>kept</w:t></w:r></w:p>{TAIL}"
        );
        assert_eq!(paragraphs(&resolve_alternate_content(&xml)), ["kept"]);
    }

    #[test]
    fn nested_alternatives_and_local_namespaces_survive() {
        // A choice declaring a prefix its content uses: the declaration moves
        // onto the kept element. The inner alternative settles on the next pass.
        let xml = format!(
            "{HEAD}<w:p><w:r><mc:AlternateContent xmlns:x=\"urn:x\">\
             <mc:Choice Requires=\"wps\" xmlns:y=\"urn:y\"><x:a y:k=\"1\"><w:t>outer</w:t>\
             <mc:AlternateContent><mc:Choice Requires=\"wps\"><w:t>inner</w:t></mc:Choice>\
             <mc:Fallback><w:t>no</w:t></mc:Fallback></mc:AlternateContent></x:a></mc:Choice>\
             <mc:Fallback><w:t>no</w:t></mc:Fallback></mc:AlternateContent></w:r></w:p>{TAIL}"
        );
        let out = resolve_alternate_content(&xml);
        assert!(!out.contains("AlternateContent"), "{out}");
        assert_eq!(paragraphs(&out), ["outer", "inner"]);
        let doc = Document::parse(&out).unwrap();
        let a = doc.descendants().find(|n| n.has_tag_name("a")).unwrap();
        assert_eq!(a.tag_name().namespace(), Some("urn:x"));
        assert_eq!(a.attribute(("urn:y", "k")), Some("1"));
    }

    #[test]
    fn untouched_without_alternatives() {
        let xml = format!("{HEAD}<w:p/>{TAIL}");
        assert!(matches!(resolve_alternate_content(&xml), Cow::Borrowed(_)));
        assert!(matches!(
            resolve_alternate_content("<a><b"),
            Cow::Borrowed(_)
        ));
    }
}