1use std::path::Path;
9
10use crate::convert::{ConvertOptions, PageConsumer, read_limited_file};
11use crate::document::html::{HtmlBlock, render_blocks_to_pages};
12use crate::error::{Error, Result};
13use crate::geospatial::xml_tree::{XmlElement, XmlLimits, parse_xml_tree};
14use crate::table::{TableAlign, TableData};
15
16const MAX_TEI_BYTES: u64 = 64 * 1024 * 1024;
17const MAX_TEI_XML_EVENTS: usize = 1_000_000;
18const MAX_TEI_XML_NODES: usize = 500_000;
19const MAX_TEI_XML_DEPTH: usize = 128;
20const MAX_TEI_TEXT_BYTES: usize = 48 * 1024 * 1024;
21const MAX_TEI_ROWS: usize = 200_000;
22const MAX_TEI_DISPLAY_BYTES: usize = 512;
23
24pub(crate) fn looks_like_prefix(bytes: &[u8]) -> bool {
25 crate::geospatial::xml_tree::looks_like_root(
26 bytes,
27 b"TEI",
28 Some(b"http://www.tei-c.org/ns/1.0"),
29 ) || crate::geospatial::xml_tree::looks_like_root(
30 bytes,
31 b"tei",
32 Some(b"http://www.tei-c.org/ns/1.0"),
33 )
34}
35
36struct TeiPageSink<'a> {
37 inner: &'a mut dyn PageConsumer,
38 warnings: &'a [String],
39}
40
41impl PageConsumer for TeiPageSink<'_> {
42 fn consume(&mut self, mut page: crate::ir::Page) -> Result<()> {
43 page.source_format = "tei".into();
44 if page.title.is_empty() {
45 page.title = "TEI scholarly text".into();
46 }
47 page.description =
48 "TEI text and structural metadata is rendered inertly; external targets, facsimiles, images and scripts are not resolved or executed".into();
49 for warning in self.warnings {
50 page.warn(warning.clone());
51 }
52 self.inner.consume(page)
53 }
54}
55
56#[derive(Default)]
57struct Summary {
58 title: String,
59 authors: Vec<String>,
60 date: String,
61 divisions: usize,
62 headings: usize,
63 paragraphs: usize,
64 notes: usize,
65 lists: usize,
66 figures: usize,
67 tables: usize,
68 rows: Vec<Vec<String>>,
69}
70
71pub(crate) fn convert(
72 path: &Path,
73 options: &ConvertOptions,
74 sink: &mut dyn PageConsumer,
75) -> Result<Vec<String>> {
76 let bytes = read_limited_file(
77 path,
78 options.max_input_bytes.min(MAX_TEI_BYTES),
79 "TEI input",
80 )?;
81 let root = parse_xml_tree(
82 &bytes,
83 &XmlLimits {
84 max_events: options.max_xml_events.min(MAX_TEI_XML_EVENTS),
85 max_nodes: MAX_TEI_XML_NODES,
86 max_depth: MAX_TEI_XML_DEPTH,
87 max_text_bytes: MAX_TEI_TEXT_BYTES,
88 },
89 "TEI",
90 )?;
91 if !root.name.eq_ignore_ascii_case("tei") {
92 return Err(Error::InvalidInput("TEI XML root must be TEI".into()));
93 }
94 let mut summary = Summary::default();
95 if let Some(header) = root.children_named("teiHeader").next() {
96 collect_header(header, &mut summary);
97 }
98 if let Some(text) = root.children_named("text").next() {
99 walk_text(text, 0, &mut summary)?;
100 } else {
101 return Err(Error::InvalidInput(
102 "TEI document requires a text element".into(),
103 ));
104 }
105 let mut warnings = vec![
106 "TEI titles, structural text, notes and counts are shown; targets, URLs, facsimiles, identifiers, arbitrary attributes and extension values are omitted".into(),
107 "TEI XML traversal is bounded; DTD/entities, XInclude, external images, scripts and linked resources are never resolved or executed".into(),
108 ];
109 if summary.paragraphs == 0 && summary.headings == 0 {
110 warnings.push("TEI text contains no renderable head or paragraph elements".into());
111 }
112 let authors = if summary.authors.is_empty() {
113 "—".into()
114 } else {
115 summary.authors.join(", ")
116 };
117 let metadata = format!(
118 "Title: {}\nAuthors: {}\nDate: {}\nDivisions: {}\nHeadings: {}\nParagraphs: {}\nNotes: {}\nLists: {}\nFigures: {}\nTables: {}",
119 display_or_dash(&summary.title),
120 truncate(&authors),
121 display_or_dash(&summary.date),
122 summary.divisions,
123 summary.headings,
124 summary.paragraphs,
125 summary.notes,
126 summary.lists,
127 summary.figures,
128 summary.tables,
129 );
130 let rows = if summary.rows.is_empty() {
131 vec![vec!["—".into(), "—".into(), "—".into(), "0".into()]]
132 } else {
133 summary.rows
134 };
135 let blocks = vec![
136 HtmlBlock::Heading {
137 level: 1,
138 text: "TEI scholarly text".into(),
139 },
140 HtmlBlock::Paragraph { text: metadata },
141 HtmlBlock::Table(TableData {
142 headers: vec!["Kind".into(), "Depth".into(), "Text".into(), "N".into()],
143 rows,
144 alignments: vec![TableAlign::Left; 4],
145 raw_source: String::new(),
146 }),
147 ];
148 let mut page_sink = TeiPageSink {
149 inner: sink,
150 warnings: &warnings,
151 };
152 render_blocks_to_pages(&blocks, &mut page_sink, options)?;
153 Ok(std::mem::take(&mut warnings))
154}
155
156fn collect_header(header: &XmlElement, summary: &mut Summary) {
157 if let Some(file_desc) = header.children_named("fileDesc").next() {
158 if let Some(title_stmt) = file_desc.children_named("titleStmt").next() {
159 summary.title = first_text(title_stmt, "title");
160 for author in title_stmt.children_named("author") {
161 let name = first_text(author, "persName");
162 let name = if name.is_empty() {
163 truncate(author.text.trim())
164 } else {
165 name
166 };
167 if !name.is_empty() && summary.authors.len() < 1_000 {
168 summary.authors.push(name);
169 }
170 }
171 }
172 if let Some(publication) = file_desc.children_named("publicationStmt").next() {
173 summary.date = first_text(publication, "date");
174 }
175 }
176}
177
178fn walk_text(element: &XmlElement, depth: usize, summary: &mut Summary) -> Result<()> {
179 for child in &element.children {
180 let next_depth = depth.saturating_add(1);
181 match child.name.as_str() {
182 "front" | "body" | "back" | "group" | "text" => {
183 walk_text(child, depth, summary)?;
184 }
185 "div" | "div1" | "div2" | "div3" | "div4" | "div5" | "div6" | "div7" => {
186 summary.divisions = summary.divisions.saturating_add(1);
187 push_row(
188 summary,
189 "div",
190 next_depth,
191 display_or_dash(child.attribute("type").unwrap_or("")).to_owned(),
192 )?;
193 walk_text(child, next_depth, summary)?;
194 }
195 "head" => {
196 summary.headings = summary.headings.saturating_add(1);
197 push_row(summary, "head", next_depth, text_content(child))?;
198 }
199 "p" | "ab" | "sp" | "quote" | "cit" => {
200 summary.paragraphs = summary.paragraphs.saturating_add(1);
201 push_row(
202 summary,
203 child.name.as_str(),
204 next_depth,
205 text_content(child),
206 )?;
207 walk_text(child, next_depth, summary)?;
208 }
209 "item" | "entry" => {
210 push_row(
211 summary,
212 child.name.as_str(),
213 next_depth,
214 text_content(child),
215 )?;
216 walk_text(child, next_depth, summary)?;
217 }
218 "note" | "noteGrp" => {
219 summary.notes = summary.notes.saturating_add(1);
220 push_row(summary, "note", next_depth, text_content(child))?;
221 walk_text(child, next_depth, summary)?;
222 }
223 "list" | "listBibl" | "listPerson" | "listPlace" => {
224 summary.lists = summary.lists.saturating_add(1);
225 push_row(
226 summary,
227 "list",
228 next_depth,
229 display_or_dash(child.attribute("type").unwrap_or("")).to_owned(),
230 )?;
231 walk_text(child, next_depth, summary)?;
232 }
233 "figure" | "graphic" | "formula" => {
234 summary.figures = summary.figures.saturating_add(1);
235 push_row(
236 summary,
237 child.name.as_str(),
238 next_depth,
239 "external media omitted".into(),
240 )?;
241 }
242 "table" => {
243 summary.tables = summary.tables.saturating_add(1);
244 let cells = count_descendants(child, &["cell"]);
245 push_row(summary, "table", next_depth, format!("{cells} cells"))?;
246 }
247 _ => {
248 walk_text(child, depth, summary)?;
249 }
250 }
251 }
252 Ok(())
253}
254
255fn count_descendants(element: &XmlElement, names: &[&str]) -> usize {
256 let mut count = usize::from(names.iter().any(|name| *name == element.name));
257 for child in &element.children {
258 count = count.saturating_add(count_descendants(child, names));
259 }
260 count
261}
262
263fn push_row(summary: &mut Summary, kind: &str, depth: usize, text: String) -> Result<()> {
264 if summary.rows.len() >= MAX_TEI_ROWS {
265 return Err(Error::LimitExceeded(format!(
266 "TEI rendered rows exceed {MAX_TEI_ROWS}"
267 )));
268 }
269 summary.rows.push(vec![
270 truncate(kind),
271 depth.to_string(),
272 truncate(&text),
273 "1".into(),
274 ]);
275 Ok(())
276}
277
278fn first_text(parent: &XmlElement, name: &str) -> String {
279 parent
280 .children_named(name)
281 .next()
282 .map(text_content)
283 .unwrap_or_default()
284}
285
286fn text_content(element: &XmlElement) -> String {
287 let mut text = element.text.trim().to_owned();
288 for child in &element.children {
289 let child_text = text_content(child);
290 if child_text.is_empty() {
291 continue;
292 }
293 while text.chars().last().is_some_and(char::is_whitespace) {
294 text.pop();
295 }
296 if let Some(last) = text.pop() {
297 if ".,;:!?".contains(last) {
298 while text.chars().last().is_some_and(char::is_whitespace) {
299 text.pop();
300 }
301 text.push(' ');
302 text.push_str(&child_text);
303 text.push(last);
304 } else {
305 text.push(last);
306 if !text.is_empty() {
307 text.push(' ');
308 }
309 text.push_str(&child_text);
310 }
311 } else {
312 text.push_str(&child_text);
313 }
314 }
315 truncate(&text)
316}
317
318fn truncate(value: &str) -> String {
319 if value.len() <= MAX_TEI_DISPLAY_BYTES {
320 return value.to_owned();
321 }
322 let mut end = MAX_TEI_DISPLAY_BYTES;
323 while !value.is_char_boundary(end) {
324 end -= 1;
325 }
326 format!("{}…", &value[..end])
327}
328
329fn display_or_dash(value: &str) -> &str {
330 if value.is_empty() { "—" } else { value }
331}
332
333#[cfg(test)]
334mod tests {
335 use super::*;
336
337 #[test]
338 fn recognizes_namespaced_tei_root() {
339 assert!(looks_like_prefix(
340 br#"<TEI xmlns="http://www.tei-c.org/ns/1.0"><text><body/></text></TEI>"#
341 ));
342 assert!(!looks_like_prefix(br#"<TEI><text/></TEI>"#));
343 }
344
345 #[test]
346 fn walks_text_structure_and_counts_nodes() {
347 let xml = br#"<TEI><teiHeader><fileDesc><titleStmt><title>Work</title><author><persName>Ada</persName></author></titleStmt></fileDesc></teiHeader><text><body><div type="chapter"><head>Intro</head><p>Hello <hi>world</hi>.</p><note>Note</note></div></body></text></TEI>"#;
348 let root = parse_xml_tree(
349 xml,
350 &XmlLimits {
351 max_events: 1000,
352 max_nodes: 1000,
353 max_depth: 32,
354 max_text_bytes: 10000,
355 },
356 "TEI",
357 )
358 .unwrap();
359 let mut summary = Summary::default();
360 collect_header(
361 root.children_named("teiHeader").next().unwrap(),
362 &mut summary,
363 );
364 walk_text(root.children_named("text").next().unwrap(), 0, &mut summary).unwrap();
365 assert_eq!(summary.title, "Work");
366 assert_eq!(summary.authors, vec!["Ada"]);
367 assert_eq!(summary.divisions, 1);
368 assert_eq!(summary.headings, 1);
369 assert_eq!(summary.paragraphs, 1);
370 assert_eq!(summary.notes, 1);
371 }
372}