extend_docx/
source_graph.rs

1use serde::{Deserialize, Serialize};
2
3use crate::{structured::package_identity, OoxmlPackage};
4
5#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
6#[serde(rename_all = "camelCase")]
7pub struct SourceAttribute {
8    pub namespace: Option<String>,
9    pub name: String,
10    pub value: String,
11}
12
13#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
14#[serde(rename_all = "camelCase")]
15pub struct SourceElement {
16    pub id: String,
17    pub kind: String,
18    #[serde(skip_serializing_if = "Option::is_none")]
19    pub namespace: Option<String>,
20    #[serde(skip_serializing_if = "Option::is_none")]
21    pub name: Option<String>,
22    #[serde(skip_serializing_if = "Option::is_none")]
23    pub text: Option<String>,
24    pub byte_start: usize,
25    pub byte_end: usize,
26    pub attributes: Vec<SourceAttribute>,
27    pub children: Vec<SourceElement>,
28}
29
30#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
31#[serde(rename_all = "camelCase")]
32pub struct SourcePart {
33    pub name: String,
34    pub byte_length: usize,
35    pub root: Option<SourceElement>,
36    pub error: Option<String>,
37}
38
39#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
40#[serde(rename_all = "camelCase")]
41pub struct DocumentSourceGraph {
42    pub schema_version: u32,
43    pub document_id: String,
44    pub parts: Vec<SourcePart>,
45    pub assets: Vec<SourcePart>,
46    pub warnings: Vec<String>,
47    pub preserved_features: Vec<PreservedFeature>,
48}
49
50#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
51#[serde(rename_all = "camelCase")]
52pub struct PreservedFeature {
53    pub part_name: String,
54    pub feature: String,
55    pub occurrences: usize,
56    pub support: String,
57}
58
59fn preserved_feature_kind(ns: &str, local: &str) -> Option<(&'static str, &'static str)> {
60    let word = ns.ends_with("wordprocessingml/2006/main") || ns.ends_with("wordprocessingml/main");
61    if word {
62        let feature = match local {
63            "altChunk" => Some("altChunk"),
64            "subDoc" => Some("subDoc"),
65            "object" => Some("object"),
66            "control" => Some("control"),
67            "movie" => Some("movie"),
68            "sound" => Some("sound"),
69            _ => None,
70        };
71        if let Some(feature) = feature {
72            return Some((feature, "preserved_only"));
73        }
74    }
75    if ns.contains("drawingml") && ns.contains("diagram") && local == "dataModel" {
76        Some(("diagram", "preserved_only"))
77    } else if ns.ends_with("/math") && local == "oMath" {
78        Some(("equation", "approximate_rendering"))
79    } else {
80        None
81    }
82}
83
84pub fn preserved_features(package: &OoxmlPackage) -> Vec<PreservedFeature> {
85    let mut features = std::collections::BTreeMap::<(String, String, String), usize>::new();
86    for part in package.parts.values() {
87        if crate::xml_index::contains_local_names(&part.content, &[
88            "altChunk", "subDoc", "object", "control", "movie", "sound", "dataModel", "oMath",
89        ]) == Some(false) {
90            continue;
91        }
92        if ![
93            "altChunk",
94            "subDoc",
95            "object",
96            "control",
97            "movie",
98            "sound",
99            "dataModel",
100            "oMath",
101        ]
102        .iter()
103        .any(|name| part.content.contains(name))
104        {
105            continue;
106        }
107        if crate::xml_index::feature_names(&part.content, &[
108            "altChunk", "subDoc", "object", "control", "movie", "sound", "dataModel", "oMath",
109        ], |namespace, local| {
110            if let Some((feature, support)) = preserved_feature_kind(namespace, local) {
111                *features.entry((part.name.clone(), feature.into(), support.into())).or_default() += 1;
112            }
113        }).is_some() {
114            continue;
115        }
116        let Ok(doc) = roxmltree::Document::parse(&part.content) else {
117            continue;
118        };
119        for node in doc.descendants().filter(|n| n.is_element()) {
120            let ns = node.tag_name().namespace().unwrap_or("");
121            let local = node.tag_name().name();
122            let feature = preserved_feature_kind(ns, local);
123            if let Some((feature, support)) = feature {
124                *features
125                    .entry((part.name.clone(), feature.into(), support.into()))
126                    .or_default() += 1;
127            }
128        }
129    }
130    features
131        .into_iter()
132        .map(
133            |((part_name, feature, support), occurrences)| PreservedFeature {
134                part_name,
135                feature,
136                support,
137                occurrences,
138            },
139        )
140        .collect()
141}
142
143pub fn document_source_graph(package: &OoxmlPackage) -> DocumentSourceGraph {
144    fn element(node: roxmltree::Node<'_, '_>, id: String) -> SourceElement {
145        let range = node.range();
146        SourceElement {
147            id: id.clone(),
148            kind: if node.is_root() {
149                "document"
150            } else if node.is_element() {
151                "element"
152            } else if node.is_text() {
153                "text"
154            } else if node.is_comment() {
155                "comment"
156            } else {
157                "processing_instruction"
158            }
159            .into(),
160            namespace: node.tag_name().namespace().map(str::to_string),
161            name: if node.is_element() {
162                Some(node.tag_name().name().to_string())
163            } else {
164                node.pi().map(|pi| pi.target.to_string())
165            },
166            text: if node.is_element() || node.is_root() {
167                None
168            } else if let Some(pi) = node.pi() {
169                pi.value.map(str::to_string)
170            } else {
171                node.text().map(str::to_string)
172            },
173            byte_start: range.start,
174            byte_end: range.end,
175            attributes: node
176                .attributes()
177                .map(|attribute| SourceAttribute {
178                    namespace: attribute.namespace().map(str::to_string),
179                    name: attribute.name().to_string(),
180                    value: attribute.value().to_string(),
181                })
182                .collect(),
183            children: node
184                .children()
185                .enumerate()
186                .map(|(index, child)| element(child, format!("{id}/{index}")))
187                .collect(),
188        }
189    }
190    let document_id = package_identity(package);
191    let mut parts = package
192        .parts
193        .values()
194        .map(|part| {
195            let parsed = roxmltree::Document::parse(&part.content);
196            SourcePart {
197                name: part.name.clone(),
198                byte_length: part.content.len(),
199                root: parsed
200                    .as_ref()
201                    .ok()
202                    .map(|doc| element(doc.root(), format!("{document_id}/{}/0", part.name))),
203                error: parsed.err().map(|e| e.to_string()),
204            }
205        })
206        .collect::<Vec<_>>();
207    let mut assets = package
208        .binary_assets
209        .iter()
210        .map(|(name, bytes)| SourcePart {
211            name: name.clone(),
212            byte_length: bytes.len(),
213            root: None,
214            error: None,
215        })
216        .collect::<Vec<_>>();
217    parts.sort_by(|a, b| a.name.cmp(&b.name));
218    assets.sort_by(|a, b| a.name.cmp(&b.name));
219    DocumentSourceGraph {
220        schema_version: 1,
221        document_id,
222        parts,
223        assets,
224        warnings: package.warnings.clone(),
225        preserved_features: preserved_features(package),
226    }
227}
228
229#[cfg(test)]
230mod indexed_feature_tests {
231    use super::*;
232    use std::collections::HashMap;
233
234    #[test]
235    fn indexed_counts_match_dom_for_valid_and_malformed_markup() {
236        let headers = [
237            "<w:root xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\" xmlns:m=\"http://schemas.openxmlformats.org/officeDocument/2006/math\" xmlns:x=\"urn:other\">",
238            "<w:root xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\" xmlns:m=\"http://schemas.openxmlformats.org/officeDocument/2006/math\" xmlns:x=\"\">",
239            "<w:root xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\" xmlns:m=\"http://schemas.openxmlformats.org/officeDocument/2006/math\">",
240            "<?xml version=\"1.0\"?><w:root xmlns:w=\"http://purl.oclc.org/ooxml/wordprocessingml/main\" xmlns:m=\"http://purl.oclc.org/ooxml/officeDocument/math\">",
241        ];
242        let bodies = [
243            "<w:object/><w:p x:xmlns=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\"><object/></w:p>",
244            "<w:object/><w:p x:value=\"a\" value=\"b\"/>",
245            "<w:object/><m:oMath/><w:object/><m:OMATH/>",
246            "<w:object w:value=\"&#9;&#10;&#13;\"/>",
247            "<w:object w:value=\"content &amp; detail\"/>",
248            "<w:object w:1invalid=\"value\"/>",
249            "<w:object/><w:1invalid/>",
250            "<w:object/><w:p>\u{0001}</w:p>",
251            "<w:object/><w:p>\u{fffe}</w:p>",
252            "<w:object/><w:p>&#65;</w:p>",
253            "<w:object/><w:p>&#X41;</w:p>",
254            "<w:object/><w:p>&#+65;</w:p>",
255            "<w:object/><w:p>]]></w:p>",
256            "<w:object/><?xml version=\"1.0\"?>",
257            "<w:object/><w:p xmlns:w=\"urn:other\"><w:object/></w:p>",
258            "<w:object/><w:p w:value=\"a\" w:value=\"b\"/>",
259            "<w:object/><w:p><w:t>content &unknown;</w:t></w:p>",
260            "<w:object/><w:p><w:t>content</w:p>",
261        ];
262        let mut fast = 0;
263        for header in headers {
264            for body in bodies {
265                let xml = format!("{header}{body}</w:root>");
266                let part = crate::package::OoxmlPart { name: "part.xml".into(), content: xml };
267                let package = OoxmlPackage {
268                    parts: HashMap::from([(part.name.clone(), part)]),
269                    binary_assets: HashMap::new(), warnings: Vec::new(),
270                };
271                let expected = preserved_features(&package);
272                let part = package.parts.values().next().unwrap();
273                let mut prepared = HashMap::new();
274                if let Some(index) = crate::xml_index::Index::canonical(&part.content)
275                    .or_else(|| crate::xml_index::Index::canonical_normalized(&part.content)) {
276                    prepared.insert(part.name.clone(), index);
277                }
278                let _indexes = crate::xml_index::scope(
279                    package.parts.values().map(|part| (part.name.as_str(), part.content.as_str())), prepared,
280                );
281                if crate::xml_index::feature_names(&part.content, &["object", "oMath"], |_, _| {}).is_some() {
282                    fast += 1;
283                    assert!(roxmltree::Document::parse(&part.content).is_ok());
284                }
285                assert_eq!(serde_json::to_value(preserved_features(&package)).unwrap(),
286                    serde_json::to_value(expected).unwrap());
287            }
288        }
289        assert!(fast >= 6);
290    }
291}
292
293#[cfg(test)]
294mod hoisted_feature_tests {
295    use super::*;
296    use std::collections::HashMap;
297
298    #[test]
299    fn hoisted_counts_preserve_invalid_text_and_depth_fallbacks() {
300        let mut accelerated = 0;
301        for depth in [0, 1, 16, 62, 63, 64, 65] {
302            for body in [
303                "<w:object/><m:oMath/><m:OMATH/>",
304                "<w:object/><w:t>\u{0001}</w:t>",
305                "<w:object/><w:t>\u{fffe}</w:t>",
306                "<w:object/><w:t>é\u{10000}&amp;</w:t>",
307                "<w:object w:value=\"&#9;&#10;&#13;\"/>",
308                "<w:object w:value=\"\u{0001}\"/>",
309            ] {
310                let xml = format!(
311                    "<w:root xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\" xmlns:m=\"http://schemas.openxmlformats.org/officeDocument/2006/math\">{}{body}{}</w:root>",
312                    "<w:p>".repeat(depth), "</w:p>".repeat(depth),
313                );
314                let Some((content, index)) = crate::xml_index::Index::canonical_hoisted(&xml) else { continue; };
315                let part = crate::package::OoxmlPart { name: "part.xml".into(), content };
316                let package = OoxmlPackage {
317                    parts: HashMap::from([(part.name.clone(), part)]),
318                    binary_assets: HashMap::new(), warnings: Vec::new(),
319                };
320                let expected = preserved_features(&package);
321                let part = package.parts.values().next().unwrap();
322                let _indexes = crate::xml_index::scope(
323                    package.parts.values().map(|part| (part.name.as_str(), part.content.as_str())),
324                    HashMap::from([(part.name.clone(), index)]),
325                );
326                if crate::xml_index::feature_names(&part.content, &["object", "oMath"], |_, _| {}).is_some() {
327                    accelerated += 1;
328                    assert!(roxmltree::Document::parse(&part.content).is_ok());
329                }
330                assert_eq!(serde_json::to_value(preserved_features(&package)).unwrap(),
331                    serde_json::to_value(expected).unwrap());
332            }
333        }
334        assert!(accelerated >= 6);
335    }
336}
extend_docx/source_graph.rs

extend_docx/
source_graph.rs
