1use std::collections::HashSet;
2
3use serde::{Deserialize, Serialize};
4use serde_json::{json, Value};
5use sha2::{Digest, Sha256};
6use unicode_segmentation::UnicodeSegmentation;
7
8use crate::model::*;
9use crate::package::OoxmlPackage;
10
11#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
12#[serde(rename_all = "snake_case")]
13pub enum BlockType {
14 Text,
15 Heading,
16 SectionHeading,
17 Figure,
18 Table,
19 TableHead,
20 TableCell,
21 KeyValue,
22 PageNumber,
23 Barcode,
24 Formula,
25 Header,
26 Footer,
27 Watermark,
28 Legend,
29}
30
31#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
32#[serde(rename_all = "camelCase")]
33pub struct SourceRef {
34 pub document_id: String,
35 pub node_id: String,
36 pub part_name: String,
37 pub story: String,
38 #[serde(skip_serializing_if = "Option::is_none")]
39 pub start: Option<usize>,
40 #[serde(skip_serializing_if = "Option::is_none")]
41 pub end: Option<usize>,
42 pub offset_encoding: String,
43}
44
45#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)]
46#[serde(rename_all = "camelCase")]
47pub struct ExtractDocumentBlocksOptions {
48 #[serde(default)]
49 pub include_words: bool,
50}
51
52#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
53#[serde(rename_all = "camelCase")]
54pub struct DocumentWord {
55 pub object: String,
56 pub id: String,
57 pub block_id: String,
58 pub content: String,
59 pub source_refs: Vec<SourceRef>,
60}
61
62#[derive(Clone, Debug, PartialEq, Serialize)]
63#[serde(tag = "type", rename_all = "snake_case")]
64pub enum ParagraphInline {
65 Text {
66 start: usize,
67 end: usize,
68 text: String,
69 style: Option<SharedTextStyle>,
70 link: Option<String>,
71 #[serde(rename = "noteReference")]
72 note_reference: Option<NoteReference>,
73 },
74 FormField {
75 #[serde(rename = "blockId")]
76 block_id: String,
77 start: usize,
78 end: usize,
79 text: String,
80 },
81 Block {
82 #[serde(rename = "blockId")]
83 block_id: String,
84 offset: usize,
85 },
86}
87
88struct FormFieldSemanticDetails<'a>(&'a FormFieldRunNode);
89
90impl Serialize for FormFieldSemanticDetails<'_> {
91 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
92 where
93 S: serde::Serializer,
94 {
95 use serde::ser::SerializeMap;
96 let field = self.0;
97 let mut map = serializer.serialize_map(None)?;
98 map.serialize_entry("type", &field.r#type)?;
99 map.serialize_entry("fieldType", &field.field_type)?;
100 macro_rules! optional {
101 ($name:literal, $value:expr) => {
102 if let Some(value) = $value.as_ref() {
103 map.serialize_entry($name, value)?;
104 }
105 };
106 }
107 optional!("sourceKind", field.source_kind);
108 optional!("id", field.id);
109 optional!("tag", field.tag);
110 optional!("title", field.title);
111 optional!("placeholder", field.placeholder);
112 optional!("checked", field.checked);
113 optional!("value", field.value);
114 optional!("options", field.options);
115 optional!("widget", field.widget);
116 optional!("checkedSymbol", field.checked_symbol);
117 optional!("uncheckedSymbol", field.unchecked_symbol);
118 optional!("style", field.style);
119 optional!("link", field.link);
120 optional!("sourceXml", field.source_xml);
121 map.end()
122 }
123}
124
125#[derive(Clone, Debug, PartialEq, Serialize)]
126#[serde(rename_all = "camelCase")]
127pub struct ParagraphDetails {
128 pub r#type: String,
129 pub kind: String,
130 pub style: Option<SharedParagraphStyle>,
131 pub style_id: String,
132 pub heading_level: Option<HeadingLevel>,
133 pub numbering: Option<ParagraphNumbering>,
134 pub field_instructions: Vec<String>,
135 pub inlines: Vec<ParagraphInline>,
136}
137
138#[derive(Clone, Debug, PartialEq, Serialize)]
139#[serde(rename_all = "camelCase")]
140pub struct TableCellDetails {
141 pub r#type: String,
142 pub row_index: usize,
143 pub column_index: i64,
144 pub cell_index: usize,
145 pub column_span: i64,
146 pub row_span: i64,
147 pub is_header: bool,
148 pub style: Option<TableCellStyle>,
149}
150
151#[derive(Clone, Debug, Serialize)]
153#[serde(untagged)]
154pub enum BlockDetails {
155 Paragraph(Box<ParagraphDetails>),
156 TableCell(Box<TableCellDetails>),
157 Raw(Value),
158}
159
160impl BlockDetails {
161 pub fn to_json_value(&self) -> Value {
162 match self {
163 Self::Paragraph(details) => serde_json::to_value(details)
164 .expect("paragraph details contain JSON-compatible values"),
165 Self::TableCell(details) => serde_json::to_value(details)
166 .expect("table cell details contain JSON-compatible values"),
167 Self::Raw(value) => value.clone(),
168 }
169 }
170}
171
172impl PartialEq for BlockDetails {
173 fn eq(&self, other: &Self) -> bool {
174 match (self, other) {
175 (Self::Raw(left), Self::Raw(right)) => left == right,
176 _ => self.to_json_value() == other.to_json_value(),
177 }
178 }
179}
180
181impl<'de> Deserialize<'de> for BlockDetails {
182 fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
183 Value::deserialize(deserializer).map(Self::Raw)
184 }
185}
186
187impl From<Value> for BlockDetails {
188 fn from(value: Value) -> Self {
189 Self::Raw(value)
190 }
191}
192
193impl From<ParagraphDetails> for BlockDetails {
194 fn from(details: ParagraphDetails) -> Self {
195 Self::Paragraph(Box::new(details))
196 }
197}
198
199impl From<TableCellDetails> for BlockDetails {
200 fn from(details: TableCellDetails) -> Self {
201 Self::TableCell(Box::new(details))
202 }
203}
204
205#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
206#[serde(rename_all = "camelCase")]
207pub struct DocumentBlock {
208 pub object: String,
209 pub id: String,
210 pub r#type: BlockType,
211 pub content: String,
212 pub details: BlockDetails,
213 pub source_refs: Vec<SourceRef>,
214 #[serde(skip_serializing_if = "Option::is_none")]
215 pub parent_block_id: Option<String>,
216 pub children: Vec<DocumentBlock>,
217 #[serde(default, skip_serializing_if = "Option::is_none")]
218 pub words: Option<Vec<DocumentWord>>,
219}
220
221#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
222#[serde(rename_all = "camelCase")]
223pub struct StructuredDocument {
224 pub schema_version: u32,
225 pub parser_version: String,
226 pub document_id: String,
227 pub blocks: Vec<DocumentBlock>,
228 pub warnings: Vec<String>,
229 pub geometry_status: String,
230}
231
232pub fn package_identity(package: &OoxmlPackage) -> String {
233 let mut hash = Sha256::new();
234 let mut parts: Vec<_> = package.parts.iter().collect();
235 parts.sort_by_key(|(name, _)| *name);
236 for (name, part) in parts {
237 hash.update((name.len() as u64).to_le_bytes());
238 hash.update(name.as_bytes());
239 hash.update((part.content.len() as u64).to_le_bytes());
240 hash.update(part.content.as_bytes());
241 }
242 let mut assets: Vec<_> = package.binary_assets.iter().collect();
243 assets.sort_by_key(|(name, _)| *name);
244 for (name, bytes) in assets {
245 hash.update((name.len() as u64).to_le_bytes());
246 hash.update(name.as_bytes());
247 hash.update((bytes.len() as u64).to_le_bytes());
248 hash.update(bytes);
249 }
250 format!("{:x}", hash.finalize())
251}
252
253fn image_url(image: &ImageRunNode) -> Option<String> {
254 if let Some(src) = image.src.as_ref().filter(|src| !src.trim().is_empty()) {
255 return Some(src.clone());
256 }
257 let data = image.data.as_deref().filter(|data| !data.is_empty())?;
258 let content_type = image
259 .content_type
260 .as_deref()
261 .filter(|content_type| !content_type.trim().is_empty())
262 .unwrap_or("application/octet-stream");
263 let capacity = 13usize
264 .checked_add(content_type.len())?
265 .checked_add(base64::encoded_len(data.len(), true)?)?;
266 let mut url = String::with_capacity(capacity);
267 url.push_str("data:");
268 url.push_str(content_type);
269 url.push_str(";base64,");
270 crate::parse::append_base64(data, &mut url);
271 Some(url)
272}
273
274fn source_part(model: &DocModel, kind: &str) -> String {
275 model
276 .metadata
277 .source_part_names
278 .as_ref()
279 .and_then(|names| names.get(kind))
280 .cloned()
281 .unwrap_or_else(|| format!("word/{kind}.xml"))
282}
283
284fn indexed_path<const N: usize>(prefix: &str, components: [(&str, usize); N]) -> String {
285 let mut buffers: [itoa::Buffer; N] = std::array::from_fn(|_| itoa::Buffer::new());
286 let mut numbers = [""; N];
287 let mut capacity = prefix.len();
288 for ((buffer, (separator, number)), rendered) in buffers.iter_mut().zip(components).zip(numbers.iter_mut()) {
289 *rendered = buffer.format(number);
290 capacity += separator.len() + rendered.len();
291 }
292 let mut result = String::with_capacity(capacity);
293 result.push_str(prefix);
294 for ((separator, _), number) in components.into_iter().zip(numbers) {
295 result.push_str(separator);
296 result.push_str(number);
297 }
298 result
299}
300
301pub fn assign_source_ids(model: &mut DocModel, document_id: &str) {
302 fn paragraph(node: &mut ParagraphNode, id: String) {
303 for (index, child) in node.children.iter_mut().enumerate() {
304 if let ParagraphChildNode::Image(image) = child {
305 let image_id = indexed_path(&id, [("/child/", index)]);
306 if let Some(content) = &mut image.content_nodes {
307 nodes(content, &format!("{image_id}/content"));
308 }
309 image.block_id = Some(image_id);
310 }
311 }
312 node.block_id = Some(id);
313 }
314 fn table(node: &mut TableNode, id: String) {
315 for (ri, row) in node.rows.iter_mut().enumerate() {
316 for (ci, cell) in row.cells.iter_mut().enumerate() {
317 for (ni, child) in cell.nodes.iter_mut().enumerate() {
318 let id = indexed_path(&id, [("/row/", ri), ("/cell/", ci), ("/", ni)]);
319 match child {
320 TableCellContentNode::Paragraph(p) => paragraph(p, id),
321 TableCellContentNode::Table(t) => table(t, id),
322 }
323 }
324 }
325 }
326 node.block_id = Some(id);
327 }
328 fn nodes(content: &mut [DocNode], root: &str) {
329 for (index, node) in content.iter_mut().enumerate() {
330 let id = indexed_path(root, [("/", index)]);
331 match node {
332 DocNode::Paragraph(p) => paragraph(p, id),
333 DocNode::Table(t) => table(t, id),
334 }
335 }
336 }
337 model.metadata.document_id = Some(document_id.to_string());
338 let root = format!("docx:{document_id}");
339 let main_part = source_part(model, "document");
340 let footnote_part = source_part(model, "footnotes");
341 let endnote_part = source_part(model, "endnotes");
342 let comment_part = source_part(model, "comments");
343 nodes(&mut model.nodes, &format!("{root}/{main_part}/body"));
344 for header in &mut model.metadata.header_sections {
345 nodes(&mut header.nodes, &format!("{root}/{}", header.part_name));
346 }
347 for footer in &mut model.metadata.footer_sections {
348 nodes(&mut footer.nodes, &format!("{root}/{}", footer.part_name));
349 }
350 for section in model.metadata.sections.iter_mut().flatten() {
351 for header in &mut section.header_sections {
352 nodes(&mut header.nodes, &format!("{root}/{}", header.part_name));
353 }
354 for footer in &mut section.footer_sections {
355 nodes(&mut footer.nodes, &format!("{root}/{}", footer.part_name));
356 }
357 }
358 for (kind, part, notes) in [
359 ("footnote", footnote_part, &mut model.metadata.footnotes),
360 ("endnote", endnote_part, &mut model.metadata.endnotes),
361 ] {
362 for note in notes.iter_mut().flatten() {
363 if let Some(content) = &mut note.nodes {
364 nodes(content, &format!("{root}/{part}/{kind}/{}", note.id));
365 }
366 }
367 }
368 for comment in model.metadata.comments.iter_mut().flatten() {
369 if let Some(content) = &mut comment.nodes {
370 nodes(
371 content,
372 &format!("{root}/{comment_part}/comment/{}", comment.id),
373 );
374 }
375 }
376}
377
378fn joined_block_content(blocks: &[DocumentBlock], separator: &str) -> String {
379 let capacity = blocks.iter().map(|block| block.content.len()).sum::<usize>()
380 + separator.len() * blocks.len().saturating_sub(1);
381 let mut content = String::with_capacity(capacity);
382 for (index, block) in blocks.iter().enumerate() {
383 if index > 0 {
384 content.push_str(separator);
385 }
386 content.push_str(&block.content);
387 }
388 content
389}
390
391fn paragraph_field_instructions(xml: &str) -> Vec<String> {
392 if crate::xml::find_ignore_ascii_case(xml, 0, "<w:instrText").is_none() {
393 return Vec::new();
394 }
395 crate::xml::extract_balanced_tag_ranges(xml, "w:instrText")
396 .into_iter()
397 .map(|range| {
398 xml[range.start..range.end]
399 .split_once('>')
400 .map(|(_, value)| {
401 crate::xml::decode_xml_entities(value.split("</").next().unwrap_or(""))
402 })
403 .unwrap_or_default()
404 })
405 .collect()
406}
407
408struct Extractor {
409 document_id: String,
410 part_name: String,
411 story: String,
412 include_words: bool,
413}
414
415impl Extractor {
416 fn block(
417 &self,
418 id: String,
419 kind: BlockType,
420 content: String,
421 details: impl Into<BlockDetails>,
422 children: Vec<DocumentBlock>,
423 parent: Option<&str>,
424 ) -> DocumentBlock {
425 DocumentBlock {
426 object: "block".into(),
427 id: id.clone(),
428 r#type: kind,
429 content,
430 details: details.into(),
431 source_refs: vec![SourceRef {
432 document_id: self.document_id.clone(),
433 node_id: id,
434 part_name: self.part_name.clone(),
435 story: self.story.clone(),
436 start: None,
437 end: None,
438 offset_encoding: "utf-16".into(),
439 }],
440 parent_block_id: parent.map(str::to_string),
441 children,
442 words: None,
443 }
444 }
445
446 fn words(&self, block_id: &str, content: &str) -> Vec<DocumentWord> {
447 let mut offset = 0;
448 content
449 .split_word_bounds()
450 .filter_map(|word| {
451 let start = offset;
452 offset += word.encode_utf16().count();
453 if word.chars().all(char::is_whitespace) {
454 return None;
455 }
456 Some(DocumentWord {
457 object: "word".into(),
458 id: indexed_path(block_id, [("/word/", start), ("-", offset)]),
459 block_id: block_id.into(),
460 content: word.into(),
461 source_refs: vec![SourceRef {
462 document_id: self.document_id.clone(),
463 node_id: block_id.into(),
464 part_name: self.part_name.clone(),
465 story: self.story.clone(),
466 start: Some(start),
467 end: Some(offset),
468 offset_encoding: "utf-16".into(),
469 }],
470 })
471 })
472 .collect()
473 }
474
475 fn story_block(&self, nodes: &[DocNode], kind: BlockType) -> DocumentBlock {
476 let id = format!("docx:{}/{}/story", self.document_id, self.part_name);
477 let children = self.nodes(nodes, Some(&id));
478 let content = joined_block_content(&children, "\n");
479 let refs = children
480 .iter()
481 .flat_map(|b| b.source_refs.clone())
482 .collect();
483 let mut block = self.block(id, kind, content, json!({"kind":"story"}), children, None);
484 block.source_refs = refs;
485 block
486 }
487
488 fn nodes(&self, nodes: &[DocNode], parent: Option<&str>) -> Vec<DocumentBlock> {
489 nodes
490 .iter()
491 .enumerate()
492 .map(|(index, node)| {
493 let fallback = indexed_path(parent.unwrap_or("document"), [("/", index)]);
494 match node {
495 DocNode::Paragraph(p) => self.paragraph(p, &fallback, parent),
496 DocNode::Table(t) => self.table(t, &fallback, parent),
497 }
498 })
499 .collect()
500 }
501
502 fn paragraph(&self, p: &ParagraphNode, fallback: &str, parent: Option<&str>) -> DocumentBlock {
503 let id = p.block_id.as_deref().unwrap_or(fallback);
504 let style = p.style.as_ref();
505 let style_id = style.and_then(|s| s.style_id.as_deref()).unwrap_or("");
506 let field_instructions = p
507 .source_xml
508 .as_deref()
509 .map(paragraph_field_instructions)
510 .unwrap_or_default();
511 let joined_instructions = field_instructions.join("").to_uppercase();
512 let instruction_kind = joined_instructions.split_whitespace().next().unwrap_or("");
513 let kind = if matches!(instruction_kind, "PAGE" | "NUMPAGES" | "SECTIONPAGES") {
514 BlockType::PageNumber
515 } else if matches!(instruction_kind, "DISPLAYBARCODE" | "MERGEBARCODE") {
516 BlockType::Barcode
517 } else if style.and_then(|s| s.heading_level).is_some() {
518 BlockType::Heading
519 } else if style_id.eq_ignore_ascii_case("title")
520 || style_id.eq_ignore_ascii_case("subtitle")
521 {
522 BlockType::SectionHeading
523 } else if style_id.eq_ignore_ascii_case("caption") {
524 BlockType::Legend
525 } else {
526 BlockType::Text
527 };
528 let mut content = String::new();
529 let mut inline = Vec::with_capacity(p.children.len());
530 let mut children = Vec::new();
531 let mut text_offset = 0;
532 for (index, child) in p.children.iter().enumerate() {
533 match child {
534 ParagraphChildNode::Text(run) => {
535 let start = text_offset;
536 text_offset += run.text.encode_utf16().count();
537 content.push_str(&run.text);
538 inline.push(ParagraphInline::Text {
539 start,
540 end: text_offset,
541 text: run.text.clone(),
542 style: run.style.clone(),
543 link: run.link.clone(),
544 note_reference: run.note_reference.clone(),
545 });
546 }
547 ParagraphChildNode::FormField(field) => {
548 let child_id = indexed_path(&id, [("/child/", index)]);
549 let value = field.value.clone().unwrap_or_else(|| {
550 field
551 .checked
552 .map(|checked| if checked { "☒" } else { "☐" }.to_string())
553 .unwrap_or_default()
554 });
555 let start = text_offset;
556 text_offset += value.encode_utf16().count();
557 content.push_str(&value);
558 inline.push(ParagraphInline::FormField {
559 block_id: child_id.clone(),
560 start,
561 end: text_offset,
562 text: value.clone(),
563 });
564 let details = if field.source_preview.is_some() {
565 json!({"type":"key_value_details", "kind":"form_field", "field":FormFieldSemanticDetails(field)})
566 } else {
567 json!({"type":"key_value_details", "kind":"form_field", "field":field})
568 };
569 children.push(self.block(
570 child_id,
571 BlockType::KeyValue,
572 value,
573 details,
574 Vec::new(),
575 Some(id),
576 ));
577 }
578 ParagraphChildNode::Image(image) => {
579 let child_id = indexed_path(&id, [("/child/", index)]);
580 let kind = match image.semantic_kind.as_deref() {
581 Some("formula") => BlockType::Formula,
582 Some("watermark") => BlockType::Watermark,
583 Some("barcode") => BlockType::Barcode,
584 _ => BlockType::Figure,
585 };
586 let image_id = image.block_id.as_deref().unwrap_or(&child_id);
587 inline.push(ParagraphInline::Block {
588 block_id: image_id.to_owned(),
589 offset: text_offset,
590 });
591 let nested = image
592 .content_nodes
593 .as_ref()
594 .map(|nodes| self.nodes(nodes, Some(image_id)))
595 .unwrap_or_default();
596 let text = if nested.is_empty() {
597 image
598 .text_box_text
599 .clone()
600 .or_else(|| image.alt.clone())
601 .unwrap_or_default()
602 } else {
603 nested
604 .iter()
605 .map(|b| b.content.as_str())
606 .collect::<Vec<_>>()
607 .join("\n")
608 };
609 let mut details = json!({"kind":image.semantic_kind.as_deref().unwrap_or(if image.synthetic_text_box == Some(true) {"text_box"} else {"image"}), "mathMl":image.math_ml, "asset":image.part_name, "alt":image.alt, "contentType":image.content_type, "width":image.width_px, "height":image.height_px, "floating":image.floating});
610 if let Some(paint_bounds) = image.paint_bounds.as_ref() {
611 details
612 .as_object_mut()
613 .unwrap()
614 .insert("paintBounds".into(), json!(paint_bounds));
615 }
616 if let Some(url) = image_url(image) {
617 details
618 .as_object_mut()
619 .unwrap()
620 .insert("imageUrl".into(), Value::String(url));
621 }
622 children.push(self.block(
623 image_id.to_string(),
624 kind,
625 text,
626 details,
627 nested,
628 Some(id),
629 ));
630 }
631 }
632 }
633 let details = ParagraphDetails {
634 r#type: "text_details".into(),
635 kind: "paragraph".into(),
636 style: p.style.clone(),
637 style_id: style_id.into(),
638 heading_level: style.and_then(|s| s.heading_level),
639 numbering: style.and_then(|s| s.numbering.clone()),
640 field_instructions,
641 inlines: inline,
642 };
643 let mut block = self.block(id.to_string(), kind, content, details, children, parent);
644 if self.include_words {
645 block.words = Some(self.words(id, &block.content));
646 }
647 block
648 }
649
650 fn table(&self, table: &TableNode, fallback: &str, parent: Option<&str>) -> DocumentBlock {
651 let id = table.block_id.as_deref().unwrap_or(fallback);
652 let mut children = Vec::new();
653 let mut row_text = String::new();
654 let mut column_count = 0;
655 for (ri, row) in table.rows.iter().enumerate() {
656 let row_id = indexed_path(id, [("/row/", ri)]);
657 let is_header = row
658 .style
659 .as_ref()
660 .and_then(|s| s.is_header)
661 .unwrap_or(false);
662 let mut cells = Vec::with_capacity(row.cells.len());
663 let mut column_index = row.style.as_ref().and_then(|s| s.grid_before).unwrap_or(0);
664 for (ci, cell) in row.cells.iter().enumerate() {
665 let cell_id = indexed_path(&row_id, [("/cell/", ci)]);
666 let content = cell
667 .nodes
668 .iter()
669 .enumerate()
670 .map(|(ni, node)| match node {
671 TableCellContentNode::Paragraph(p) => {
672 self.paragraph(p, &indexed_path(&cell_id, [("/", ni)]), Some(&cell_id))
673 }
674 TableCellContentNode::Table(t) => {
675 self.table(t, &indexed_path(&cell_id, [("/", ni)]), Some(&cell_id))
676 }
677 })
678 .collect::<Vec<_>>();
679 let text = joined_block_content(&content, "\n");
680 cells.push(self.block(
681 cell_id,
682 BlockType::TableCell,
683 text,
684 TableCellDetails {
685 r#type: "table_cell_details".into(),
686 row_index: ri,
687 column_index,
688 cell_index: ci,
689 column_span: cell.style.as_ref().and_then(|s| s.grid_span).unwrap_or(1),
690 row_span: cell.style.as_ref().and_then(|s| s.row_span).unwrap_or(1),
691 is_header,
692 style: cell.style.clone(),
693 },
694 content,
695 Some(if is_header { &row_id } else { id }),
696 ));
697 column_index += cell
698 .style
699 .as_ref()
700 .and_then(|s| s.grid_span)
701 .unwrap_or(1)
702 .max(1);
703 }
704 column_count = column_count.max(column_index);
705 let text = joined_block_content(&cells, "\t");
706 if ri > 0 {
707 row_text.push('\n');
708 }
709 row_text.push_str(&text);
710 if is_header {
711 children.push(self.block(
712 row_id,
713 BlockType::TableHead,
714 text,
715 json!({"rowIndex":ri}),
716 cells,
717 Some(id),
718 ));
719 } else {
720 children.extend(cells);
721 }
722 }
723 self.block(
724 id.to_string(),
725 BlockType::Table,
726 row_text,
727 json!({"type":"table_details", "rowCount":table.rows.len(), "columnCount":column_count}),
728 children,
729 parent,
730 )
731 }
732}
733
734fn block_has_visible_text(block: &DocumentBlock) -> bool {
735 block.content.chars().any(|c| {
736 !c.is_whitespace() && !matches!(c, '\u{200b}' | '\u{feff}' | '\u{00ad}')
737 })
738}
739
740fn without_empty_blocks(mut blocks: Vec<DocumentBlock>, parent: Option<&str>) -> Vec<DocumentBlock> {
741 let mut needs_flattening = false;
742 blocks.retain_mut(|block| {
743 block.children = without_empty_blocks(std::mem::take(&mut block.children), Some(&block.id));
744 if block.parent_block_id.as_deref() != parent {
745 block.parent_block_id = parent.map(str::to_string);
746 }
747 if block_has_visible_text(block) || matches!(block.r#type,
748 BlockType::Figure | BlockType::Formula | BlockType::Barcode
749 | BlockType::Watermark | BlockType::KeyValue)
750 {
751 return true;
752 }
753 if block.children.is_empty() {
754 return false;
755 }
756 needs_flattening |= matches!(block.r#type, BlockType::Text);
757 true
758 });
759 if !needs_flattening {
760 if blocks.capacity() > blocks.len().saturating_mul(4).max(64) {
761 blocks.shrink_to_fit();
762 }
763 return blocks;
764 }
765 let mut retained = Vec::with_capacity(blocks.len());
766 for mut block in blocks {
767 if matches!(block.r#type, BlockType::Text) && !block_has_visible_text(&block) {
768 for child in &mut block.children {
769 if child.parent_block_id.as_deref() != parent {
770 child.parent_block_id = parent.map(str::to_string);
771 }
772 }
773 retained.extend(block.children);
774 } else {
775 retained.push(block);
776 }
777 }
778 retained
779}
780
781pub fn extract_document_blocks(model: &DocModel) -> StructuredDocument {
782 extract_document_blocks_with_options(model, ExtractDocumentBlocksOptions::default())
783}
784
785pub fn extract_document_blocks_with_options(
786 model: &DocModel,
787 options: ExtractDocumentBlocksOptions,
788) -> StructuredDocument {
789 let document_id = model
790 .metadata
791 .document_id
792 .clone()
793 .unwrap_or_else(|| "unsaved".into());
794 let mut extractor = Extractor {
795 document_id: document_id.clone(),
796 part_name: source_part(model, "document"),
797 story: "body".into(),
798 include_words: options.include_words,
799 };
800 let mut blocks = extractor.nodes(&model.nodes, None);
801 let mut seen = HashSet::new();
802 let headers = model.metadata.header_sections.iter().chain(
803 model
804 .metadata
805 .sections
806 .iter()
807 .flatten()
808 .flat_map(|s| s.header_sections.iter()),
809 );
810 for header in headers {
811 if !seen.insert(header.part_name.clone()) {
812 continue;
813 }
814 extractor.part_name = header.part_name.clone();
815 extractor.story = "header".into();
816 blocks.push(extractor.story_block(&header.nodes, BlockType::Header));
817 }
818 let footers = model.metadata.footer_sections.iter().chain(
819 model
820 .metadata
821 .sections
822 .iter()
823 .flatten()
824 .flat_map(|s| s.footer_sections.iter()),
825 );
826 for footer in footers {
827 if !seen.insert(footer.part_name.clone()) {
828 continue;
829 }
830 extractor.part_name = footer.part_name.clone();
831 extractor.story = "footer".into();
832 blocks.push(extractor.story_block(&footer.nodes, BlockType::Footer));
833 }
834 for (kind, notes) in [
835 ("footnote", &model.metadata.footnotes),
836 ("endnote", &model.metadata.endnotes),
837 ] {
838 extractor.part_name = source_part(model, &format!("{kind}s"));
839 extractor.story = kind.into();
840 for note in notes.iter().flatten() {
841 if let Some(nodes) = ¬e.nodes {
842 blocks.extend(extractor.nodes(nodes, None));
843 }
844 }
845 }
846 extractor.part_name = source_part(model, "comments");
847 extractor.story = "comment".into();
848 for comment in model.metadata.comments.iter().flatten() {
849 if let Some(nodes) = &comment.nodes {
850 blocks.extend(extractor.nodes(nodes, None));
851 }
852 }
853 StructuredDocument {
854 schema_version: 1,
855 parser_version: env!("CARGO_PKG_VERSION").into(),
856 document_id,
857 blocks: without_empty_blocks(blocks, None),
858 warnings: model.metadata.warnings.clone(),
859 geometry_status: "requires_layout".into(),
860 }
861}
862
863#[cfg(test)]
864mod typed_table_tests {
865 #[test]
866 fn table_details_round_trip_through_imported_json() {
867 let mut package = crate::package::OoxmlPackage {
868 parts: Default::default(),
869 binary_assets: Default::default(),
870 warnings: vec![],
871 };
872 let name = "word/document.xml".to_owned();
873 package.parts.insert(name.clone(), crate::package::OoxmlPart { name, content: "<w:document xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\"><w:body><w:tbl><w:tblGrid><w:gridCol w:w=\"1200\"/><w:gridCol w:w=\"1200\"/></w:tblGrid><w:tr><w:trPr><w:tblHeader/></w:trPr><w:tc><w:tcPr><w:gridSpan w:val=\"2\"/><w:shd w:fill=\"FFCC00\"/></w:tcPr><w:p><w:r><w:rPr><w:b/></w:rPr><w:t>cell content</w:t></w:r></w:p></w:tc></w:tr></w:tbl></w:body></w:document>".into() });
874 let model = crate::build_doc_model(&package);
875 for include_words in [false, true] {
876 let blocks = super::extract_document_blocks_with_options(
877 &model,
878 super::ExtractDocumentBlocksOptions { include_words },
879 );
880 let value = serde_json::to_value(&blocks).unwrap();
881 let imported: super::StructuredDocument =
882 serde_json::from_value(value.clone()).unwrap();
883 assert_eq!(blocks, imported);
884 assert_eq!(serde_json::to_value(imported).unwrap(), value);
885 let details = &value["blocks"][0]["children"][0]["children"][0]["details"];
886 assert_eq!(details["columnSpan"], 2);
887 assert_eq!(details["isHeader"], true);
888 assert_eq!(details["style"]["backgroundColor"], "#FFCC00");
889 }
890 }
891}
892
893#[cfg(test)]
894mod field_projection_tests {
895 use super::paragraph_field_instructions;
896
897 #[test]
898 fn borrowed_instruction_projection_matches_balanced_block_behavior() {
899 let cases = [
900 "", "<w:p><w:r><w:t>text</w:t></w:r></w:p>",
901 "<w:instrText>PAGE</w:instrText>",
902 "<W:INSTRText>NUMPAGES</W:INSTRText>",
903 "<w:instrText>HYPER&LINK</w:instrText>",
904 "<w:instrText><w:instrText>PAGE</w:instrText></w:instrText>",
905 "<w:instrText>unfinished", "<w:instrText/>",
906 "<w:instrTextExtra>text</w:instrTextExtra>",
907 "<w:instrText xml:space=\"preserve\"> PAGE </w:instrText>",
908 "<w:instrText>PAGE</w:instrText><w:instrText>NUMPAGES</w:instrText>",
909 ];
910 for xml in cases {
911 let expected: Vec<_> = crate::xml::extract_balanced_tag_blocks(xml, "w:instrText")
912 .iter()
913 .map(|field| {
914 field.split_once('>')
915 .map(|(_, value)| crate::xml::decode_xml_entities(value.split("</").next().unwrap_or("")))
916 .unwrap_or_default()
917 })
918 .collect();
919 assert_eq!(paragraph_field_instructions(xml), expected);
920 }
921 }
922}
923
924#[cfg(test)]
925mod indexed_path_tests {
926 use super::indexed_path;
927
928 #[test]
929 fn indexed_paths_preserve_integer_and_unicode_spelling() {
930 for prefix in ["", "prefix", "文😀"] {
931 for index in [0, 1, 9, 10, 99, 100, usize::MAX] {
932 assert_eq!(indexed_path(prefix, [("/", index)]), format!("{prefix}/{index}"));
933 assert_eq!(indexed_path(prefix, [("/row/", index), ("/cell/", 0), ("/", 10)]), format!("{prefix}/row/{index}/cell/0/10"));
934 assert_eq!(indexed_path(prefix, [("/word/", index), ("-", index)]), format!("{prefix}/word/{index}-{index}"));
935 }
936 assert_eq!(indexed_path(prefix, []), prefix);
937 }
938 }
939}
940
941#[cfg(test)]
942mod inplace_block_cleanup_tests {
943 #[test]
944 fn invisible_containers_flatten_objects_and_repair_only_parent_links() {
945 use super::*;
946 let extractor = Extractor {
947 document_id: "document".into(), part_name: "part".into(),
948 story: "body".into(), include_words: false,
949 };
950 let figure = extractor.block("figure".into(), BlockType::Figure, String::new(),
951 json!({}), Vec::new(), Some("container"));
952 let source_refs = figure.source_refs.clone();
953 let container = extractor.block("container".into(), BlockType::Text,
954 " \u{200b}\u{feff}\u{00ad}".into(), json!({}), vec![figure], Some("stale"));
955 let empty = extractor.block("empty".into(), BlockType::Table, String::new(),
956 json!({}), Vec::new(), None);
957 let text = extractor.block("text".into(), BlockType::Text, "content".into(),
958 json!({}), Vec::new(), Some("stale"));
959 let result = without_empty_blocks(vec![container, empty, text], Some("parent"));
960 assert_eq!(result.len(), 2);
961 assert_eq!(result[0].id, "figure");
962 assert_eq!(result[0].source_refs, source_refs);
963 assert_eq!(result[1].content, "content");
964 assert!(result.iter().all(|block| block.parent_block_id.as_deref() == Some("parent")));
965 let root = without_empty_blocks(result, None);
966 assert!(root.iter().all(|block| block.parent_block_id.is_none()));
967 }
968}