vole_document/field/document_format.rs
1//! Byte-based document-format detection for the universal observation API
2//! (Phase 12.7, ADR-0031, plan §DEC-5).
3//!
4//! The format of a field is decided from its **source bytes**, never from a file
5//! name or extension. Detection is deliberately conservative: an ambiguous or
6//! malformed input falls back to [`DocumentFormat::Opaque`] rather than guessing,
7//! and a format whose adapter is not compiled in cannot be detected (the input is
8//! then `Opaque`), so the reported capability set always matches what the build
9//! can actually serve.
10//!
11//! * **PDF** — the byte-authoritative physical scanner admits a validated PDF
12//! ([`crate::adapter::pdf::detect`]).
13//! * **DOCX** — a valid ZIP that also carries the OPC content-types part
14//! (`[Content_Types].xml`) and a package `officeDocument` relationship.
15//! * **EPUB** — a valid ZIP that is an OCF container: the mandatory stored
16//! `mimetype` member equals `application/epub+zip`, or `META-INF/container.xml`
17//! names the OCF namespace and an OPF (`application/oebps-package+xml`) rootfile.
18//! * **ODT** — a valid ZIP that is an OpenDocument (ODF) package: the mandatory
19//! stored `mimetype` member is an OpenDocument *text* media type, or
20//! `META-INF/manifest.xml` declares one.
21//! * **ODS** — a valid ZIP that is an OpenDocument (ODF) package whose mandatory
22//! stored `mimetype` member is an OpenDocument *spreadsheet* media type, or whose
23//! `META-INF/manifest.xml` declares one (Phase 21.3.1). Mutually exclusive with
24//! ODT (a text document declares the text media type, a spreadsheet the
25//! spreadsheet one).
26//! * **ODP** — a valid ZIP that is an OpenDocument (ODF) package whose mandatory
27//! stored `mimetype` member is an OpenDocument *presentation* media type, or whose
28//! `META-INF/manifest.xml` declares one (Phase 21.4.1). Mutually exclusive with
29//! ODT/ODS (a presentation declares the presentation media type).
30//! * **Opaque** — everything else, including a ZIP that matches none of the above
31//! (or more than one — an ambiguous ZIP fails safe).
32//! * **JSON** — the whole source parses as exactly one JSON value within the
33//! JSON caps (Phase 21.5.1). JSON is a Wave-2 structured-tree format with no
34//! package layer, so it is detected directly (never via `detect_zip_family`);
35//! a malformed or oversized input stays `Opaque`.
36//!
37//! The detected format is recorded in the field manifest's provenance (a
38//! machine-readable `format=<name>;` prefix, see [`DocumentFormat::from_provenance`]),
39//! so `observe`/`find`/`explain` can dispatch common selectors without reading the
40//! whole source again.
41
42use crate::limits::Limits;
43
44/// The mandatory OCF `mimetype` payload.
45pub const EPUB_MIMETYPE: &[u8] = b"application/epub+zip";
46/// The mandatory OCF container descriptor member.
47pub const CONTAINER_MEMBER: &[u8] = b"META-INF/container.xml";
48/// The OCF container-descriptor namespace.
49pub const CONTAINER_NS: &[u8] = b"urn:oasis:names:tc:opendocument:xmlns:container";
50/// The default (and only normative) package-document media type.
51pub const OPF_MEDIA_TYPE: &[u8] = b"application/oebps-package+xml";
52/// The OPC content-types part.
53#[cfg(feature = "package")]
54const CONTENT_TYPES_MEMBER: &[u8] = b"[Content_Types].xml";
55/// The OPC package-relationships part.
56#[cfg(feature = "package")]
57const PACKAGE_RELS_MEMBER: &[u8] = b"_rels/.rels";
58/// The `officeDocument` relationship type fragment (transitional and strict).
59#[cfg(feature = "package")]
60const OFFICE_DOCUMENT_FRAGMENT: &[u8] = b"officeDocument";
61/// The WordprocessingML document main content-type fragment (Phase 21.1.2).
62#[cfg(feature = "package")]
63const DOCX_MAIN_FRAGMENT: &[u8] = b"wordprocessingml.document.main+xml";
64/// The canonical WordprocessingML main-part target fragment (Phase 21.1.2).
65#[cfg(feature = "package")]
66const DOCX_MAIN_TARGET: &[u8] = b"word/document.xml";
67/// The ODF package manifest member (Phase 13.3).
68#[cfg(feature = "odt")]
69const ODT_MANIFEST_MEMBER: &[u8] = b"META-INF/manifest.xml";
70/// The OpenDocument *text* media-type fragment (Phase 13.3).
71#[cfg(feature = "odt")]
72const ODT_TEXT_FRAGMENT: &[u8] = b"application/vnd.oasis.opendocument.text";
73/// The ODF package manifest member for the ODS detection rule (Phase 21.3.1).
74#[cfg(feature = "ods")]
75const ODS_MANIFEST_MEMBER: &[u8] = b"META-INF/manifest.xml";
76/// The OpenDocument *spreadsheet* media-type fragment (Phase 21.3.1).
77#[cfg(feature = "ods")]
78const ODS_SPREADSHEET_FRAGMENT: &[u8] = b"application/vnd.oasis.opendocument.spreadsheet";
79/// The ODF package manifest member for the ODP detection rule (Phase 21.4.1).
80#[cfg(feature = "odp")]
81const ODP_MANIFEST_MEMBER: &[u8] = b"META-INF/manifest.xml";
82/// The OpenDocument *presentation* media-type fragment (Phase 21.4.1).
83#[cfg(feature = "odp")]
84const ODP_PRESENTATION_FRAGMENT: &[u8] = b"application/vnd.oasis.opendocument.presentation";
85/// The ODF media-type prefix shared by every OpenDocument package.
86#[cfg(any(feature = "odt", feature = "ods", feature = "odp"))]
87const ODF_MEDIA_PREFIX: &[u8] = b"application/vnd.oasis.opendocument.";
88/// The SpreadsheetML workbook main content-type fragment (Phase 21.1.1).
89#[cfg(feature = "xlsx")]
90const XLSX_MAIN_FRAGMENT: &[u8] = b"spreadsheetml.sheet.main+xml";
91/// The SpreadsheetML content-type namespace fragment (Phase 21.1.1).
92#[cfg(feature = "xlsx")]
93const XLSX_NS_FRAGMENT: &[u8] = b"spreadsheetml";
94/// A SpreadsheetML workbook part-target fragment (Phase 21.1.1).
95#[cfg(feature = "xlsx")]
96const XLSX_WORKBOOK_TARGET: &[u8] = b"xl/workbook.xml";
97/// The PresentationML presentation main content-type fragment (Phase 21.2.1).
98#[cfg(feature = "pptx")]
99const PPTX_MAIN_FRAGMENT: &[u8] = b"presentationml.presentation.main+xml";
100/// The PresentationML content-type namespace fragment (Phase 21.2.1).
101#[cfg(feature = "pptx")]
102const PPTX_NS_FRAGMENT: &[u8] = b"presentationml";
103/// The canonical PresentationML main-part target fragment (Phase 21.2.1).
104#[cfg(feature = "pptx")]
105const PPTX_MAIN_TARGET: &[u8] = b"ppt/presentation.xml";
106
107/// A detected document format (the class of the field's source bytes).
108#[derive(Debug, Clone, Copy, PartialEq, Eq)]
109pub enum DocumentFormat {
110 /// A validated PDF (physical indicators).
111 Pdf,
112 /// An OPC package with a WordprocessingML `officeDocument` part.
113 Docx,
114 /// An OCF container with an EPUB package document.
115 Epub,
116 /// An ODF package with an OpenDocument text content part.
117 Odt,
118 /// An ODF package with an OpenDocument spreadsheet content part.
119 Ods,
120 /// An ODF package with an OpenDocument presentation content part.
121 Odp,
122 /// An OPC package with a SpreadsheetML workbook part.
123 Xlsx,
124 /// An OPC package with a PresentationML presentation part.
125 Pptx,
126 /// A structured JSON document (the whole source parses as exactly one JSON
127 /// value). Not a package: the exact leaf is the whole source (Phase 21.5.1).
128 Json,
129 /// A structured YAML document (the whole source parses as a bounded YAML
130 /// stream whose every document root is a mapping or a sequence). Not a package:
131 /// the exact leaf is the whole source (Phase 21.6.1).
132 Yaml,
133 /// A CSV/TSV table (the whole source parses under a comma or tab delimiter as a
134 /// table with a consistent field count across a sampled majority of at least
135 /// two records). Not a package: the exact leaf is the whole source, and the
136 /// recorded dialect distinguishes comma from tab (Phase 21.7.1).
137 Csv,
138 /// A Markdown prose document (the whole source carries at least one structural
139 /// mark — an ATX heading, a fenced code block, front matter, a table, a
140 /// reference definition, or a footnote definition). Not a package: the exact
141 /// leaf is the whole source (Phase 21.8.1).
142 Markdown,
143 /// A standalone XML document (the whole source begins with `<` and parses as
144 /// well-formed XML with exactly one root element under the XML caps). Not a
145 /// package: the exact leaf is the whole source, and every node/attribute span
146 /// is a `Q_gen` projection (Phase 21.9).
147 Xml,
148 /// A standalone HTML document (the source is not well-formed XML and carries
149 /// clear HTML structure — a `<!doctype html>`, an `<html`/`<head`/`<body` tag,
150 /// or a preponderance of known HTML tags). Parsed by a bounded,
151 /// **error-recovering** scanner. Not a package: the exact leaf is the whole
152 /// source, and every element/attribute/text/comment/raw-text span is a `Q_gen`
153 /// projection (Phase 21.10).
154 Html,
155 /// A TOML document (the source is not any earlier format, parses as TOML 1.0
156 /// under the TOML caps with no errors, and carries at least one key/value
157 /// assignment). Not a package: the exact leaf is the whole source, and every
158 /// table/array/key/value/comment span is a `Q_gen` projection. TOML's
159 /// duplicate-key/redefinition rules are enforced (a violation is a typed
160 /// decline) (Phase 21.11).
161 Toml,
162 /// A JSONL / NDJSON line/event stream (the source carries at least two non-blank
163 /// lines and **every** non-blank line parses as exactly one JSON value under the
164 /// caps, each line parsed by the shared JSON parser). Not a package: the exact
165 /// leaf is the whole source, and every record's exact line span, terminator, and
166 /// per-token spans are `Q_gen` projections. A single JSON value stays
167 /// [`DocumentFormat::Json`] (Phase 21.12).
168 Jsonl,
169 /// An EML / MIME internet message (an RFC 5322 header block terminated by a
170 /// blank line, carrying `From`/`Date`/`Message-ID`, or an explicit
171 /// `MIME-Version`). Not a package: the exact leaf is the whole source, and every
172 /// header/part span and every `Content-Transfer-Encoding`-decoded constituent is
173 /// a `Q_gen` projection (Phase 21.13).
174 Eml,
175 /// An Apache Parquet file (the source begins with `PAR1` and ends with `PAR1`,
176 /// and the 4-byte little-endian footer length before the trailing magic is
177 /// consistent with the file length). Not a package: the exact leaf is the whole
178 /// source, and the parsed footer inventory (schema, row groups, column chunks,
179 /// statistics) and any decoded values are `Q_gen` projections (Phase 21.14).
180 Parquet,
181 /// An Apache Arrow IPC file/stream (the source begins with the `ARROW1` magic
182 /// and is either the **file** format — a trailing `ARROW1` magic preceded by a
183 /// consistent little-endian `int32` footer length — or the **stream** format —
184 /// a valid encapsulated `Schema` message at the 8-byte magic+padding prefix).
185 /// Not a package: the exact leaf is the whole source, and the parsed schema and
186 /// record-batch inventory (with each batch's exact source span) and any decoded
187 /// columnar values are `Q_gen` projections (Phase 21.16).
188 ArrowIpc,
189 /// Anything else; preserved exactly by the opaque floor.
190 Opaque,
191}
192
193impl DocumentFormat {
194 /// Stable lower-case name (used in JSON and in the manifest provenance token).
195 pub const fn name(self) -> &'static str {
196 match self {
197 DocumentFormat::Pdf => "pdf",
198 DocumentFormat::Docx => "docx",
199 DocumentFormat::Epub => "epub",
200 DocumentFormat::Odt => "odt",
201 DocumentFormat::Ods => "ods",
202 DocumentFormat::Odp => "odp",
203 DocumentFormat::Xlsx => "xlsx",
204 DocumentFormat::Pptx => "pptx",
205 DocumentFormat::Json => "json",
206 DocumentFormat::Yaml => "yaml",
207 DocumentFormat::Csv => "csv",
208 DocumentFormat::Markdown => "markdown",
209 DocumentFormat::Xml => "xml",
210 DocumentFormat::Html => "html",
211 DocumentFormat::Toml => "toml",
212 DocumentFormat::Jsonl => "jsonl",
213 DocumentFormat::Eml => "eml",
214 DocumentFormat::Parquet => "parquet",
215 DocumentFormat::ArrowIpc => "arrow",
216 DocumentFormat::Opaque => "opaque",
217 }
218 }
219
220 /// The adapter that serves this format (the observation layer's name for it).
221 pub const fn adapter(self) -> &'static str {
222 match self {
223 DocumentFormat::Pdf => "pdf",
224 DocumentFormat::Docx => "docx",
225 DocumentFormat::Epub => "epub",
226 DocumentFormat::Odt => "odt",
227 DocumentFormat::Ods => "ods",
228 DocumentFormat::Odp => "odp",
229 DocumentFormat::Xlsx => "xlsx",
230 DocumentFormat::Pptx => "pptx",
231 DocumentFormat::Json => "json",
232 DocumentFormat::Yaml => "yaml",
233 DocumentFormat::Csv => "csv",
234 DocumentFormat::Markdown => "markdown",
235 DocumentFormat::Xml => "xml",
236 DocumentFormat::Html => "html",
237 DocumentFormat::Toml => "toml",
238 DocumentFormat::Jsonl => "jsonl",
239 DocumentFormat::Eml => "eml",
240 DocumentFormat::Parquet => "parquet",
241 DocumentFormat::ArrowIpc => "arrow",
242 DocumentFormat::Opaque => "opaque",
243 }
244 }
245
246 /// Whether the adapter for this format is compiled into this build.
247 pub const fn compiled(self) -> bool {
248 match self {
249 DocumentFormat::Pdf | DocumentFormat::Opaque => true,
250 DocumentFormat::Docx => cfg!(feature = "docx"),
251 DocumentFormat::Epub => cfg!(feature = "epub"),
252 DocumentFormat::Odt => cfg!(feature = "odt"),
253 DocumentFormat::Ods => cfg!(feature = "ods"),
254 DocumentFormat::Odp => cfg!(feature = "odp"),
255 DocumentFormat::Xlsx => cfg!(feature = "xlsx"),
256 DocumentFormat::Pptx => cfg!(feature = "pptx"),
257 DocumentFormat::Json => cfg!(feature = "json"),
258 DocumentFormat::Yaml => cfg!(feature = "yaml"),
259 DocumentFormat::Csv => cfg!(feature = "csv"),
260 DocumentFormat::Markdown => cfg!(feature = "markdown"),
261 DocumentFormat::Xml => cfg!(feature = "xml"),
262 DocumentFormat::Html => cfg!(feature = "html"),
263 DocumentFormat::Toml => cfg!(feature = "toml"),
264 DocumentFormat::Jsonl => cfg!(feature = "jsonl"),
265 DocumentFormat::Eml => cfg!(feature = "eml"),
266 DocumentFormat::Parquet => cfg!(feature = "parquet"),
267 DocumentFormat::ArrowIpc => cfg!(feature = "arrow"),
268 }
269 }
270
271 /// The machine-readable `format=<name>;` provenance prefix recorded at ingest.
272 pub fn provenance_prefix(self) -> String {
273 format!("format={};", self.name())
274 }
275
276 /// Recover the recorded format from a field manifest's provenance string.
277 ///
278 /// Returns `None` for a manifest that does not carry the token (e.g. a field
279 /// written before this subphase); such a field still serves every native
280 /// selector, but common observations decline typed rather than guessing.
281 pub fn from_provenance(provenance: &str) -> Option<DocumentFormat> {
282 let rest = provenance.strip_prefix("format=")?;
283 let name = rest.split(';').next()?;
284 match name {
285 "pdf" => Some(DocumentFormat::Pdf),
286 "docx" => Some(DocumentFormat::Docx),
287 "epub" => Some(DocumentFormat::Epub),
288 "odt" => Some(DocumentFormat::Odt),
289 "ods" => Some(DocumentFormat::Ods),
290 "odp" => Some(DocumentFormat::Odp),
291 "xlsx" => Some(DocumentFormat::Xlsx),
292 "pptx" => Some(DocumentFormat::Pptx),
293 "json" => Some(DocumentFormat::Json),
294 "yaml" => Some(DocumentFormat::Yaml),
295 "csv" => Some(DocumentFormat::Csv),
296 "markdown" => Some(DocumentFormat::Markdown),
297 "xml" => Some(DocumentFormat::Xml),
298 "html" => Some(DocumentFormat::Html),
299 "toml" => Some(DocumentFormat::Toml),
300 "jsonl" => Some(DocumentFormat::Jsonl),
301 "eml" => Some(DocumentFormat::Eml),
302 "parquet" => Some(DocumentFormat::Parquet),
303 "arrow" => Some(DocumentFormat::ArrowIpc),
304 "opaque" => Some(DocumentFormat::Opaque),
305 _ => None,
306 }
307 }
308}
309
310/// Detect the document format of `source` from its bytes alone.
311///
312/// Never consults a file name or extension. A malformed or ambiguous input (or a
313/// format whose adapter is not compiled) falls back to [`DocumentFormat::Opaque`].
314pub fn detect_document_format(source: &[u8], limits: Limits) -> DocumentFormat {
315 if crate::adapter::pdf::detect(source, limits) {
316 return DocumentFormat::Pdf;
317 }
318 #[cfg(feature = "package")]
319 {
320 if let Some(format) = detect_zip_family(source, limits) {
321 return format;
322 }
323 }
324 // Parquet is the **analytical** Wave-2 format (Phase 21.14), with no package
325 // layer. Its detection is a strong magic-byte contract — `PAR1` at both ends
326 // plus a consistent little-endian footer length — so it runs immediately after
327 // the package families and **before** the weak, no-magic-byte heuristics
328 // (JSON/YAML/CSV/Markdown/XML/HTML), which a binary columnar file must never be
329 // guessed to be. It is deliberately conservative: a file that merely begins or
330 // ends with `PAR1` but whose footer length overruns the leading magic stays
331 // `Opaque`.
332 #[cfg(feature = "parquet")]
333 if crate::adapter::parquet::detect(source, limits) {
334 return DocumentFormat::Parquet;
335 }
336 // Arrow IPC is the **analytical** Wave-2 format (Phase 21.16), with no package
337 // layer. Its detection is a strong magic-byte contract — the `ARROW1` magic at
338 // the start plus either a consistent trailing footer (file format) or a valid
339 // encapsulated `Schema` message (stream format) — so it runs immediately after
340 // Parquet and **before** the weak, no-magic-byte heuristics, which a binary
341 // columnar file must never be guessed to be. It is deliberately conservative:
342 // `ARROW1`-prefixed junk with neither a consistent footer nor a valid schema
343 // message stays `Opaque`.
344 #[cfg(feature = "arrow")]
345 if crate::adapter::arrow::detect(source, limits) {
346 return DocumentFormat::ArrowIpc;
347 }
348 // JSON is a Wave-2 structured-tree format with **no** package layer, so it is
349 // detected directly from the whole source (never through `detect_zip_family`).
350 // Conservative: the entire source must parse as exactly one JSON value within
351 // the JSON caps, else the input stays Opaque (Phase 21.5.1).
352 #[cfg(feature = "json")]
353 if crate::adapter::json::detect(source, limits) {
354 return DocumentFormat::Json;
355 }
356 // JSONL/NDJSON is the **line/event-stream** Wave-2 format (Phase 21.12), also
357 // with no package layer. It is tried **immediately after JSON** because it is the
358 // most specific signal for a newline-separated JSON stream: each non-blank line
359 // must parse as exactly one JSON value, which is stricter than — and must run
360 // before — the YAML stream reader (which would otherwise be free to reinterpret a
361 // line-delimited JSON stream as a stream of documents). Detection is
362 // conservative: at least two non-blank lines are required and every non-blank
363 // line must parse as exactly one JSON value under the caps, so a single JSON
364 // value, a malformed line, and a bag of values with a non-newline separator all
365 // stay `Opaque`.
366 #[cfg(feature = "jsonl")]
367 if crate::adapter::jsonl::detect(source, limits) {
368 return DocumentFormat::Jsonl;
369 }
370 // EML/MIME is the **messaging** Wave-2 format (Phase 21.13), also with no package
371 // layer. It is tried **immediately after JSONL** and **before YAML/TOML/CSV/
372 // Markdown/XML/HTML**: a raw message begins with an RFC 5322 header block, which
373 // the YAML stream reader would otherwise reinterpret as a mapping (e.g. `From: a`
374 // / `Date: b`). Detection is conservative: a genuine header block terminated by a
375 // blank line carrying `From`/`Date`/`Message-ID`, or an explicit `MIME-Version`,
376 // is required, so prose without a header block and a colon-bearing note stay
377 // `Opaque`. Only the header block is scanned (never the MIME tree).
378 #[cfg(feature = "eml")]
379 if crate::adapter::eml::detect(source, limits) {
380 return DocumentFormat::Eml;
381 }
382 // YAML is the second Wave-2 structured-tree format (Phase 21.6.1), also with
383 // **no** package layer. It is detected directly and conservatively: the whole
384 // source must parse as a bounded YAML stream under the YAML caps, and every
385 // document root must be a mapping or a sequence, else the input stays Opaque.
386 #[cfg(feature = "yaml")]
387 if crate::adapter::yaml::detect(source, limits) {
388 return DocumentFormat::Yaml;
389 }
390 // TOML is the next Wave-2 structured-tree format (Phase 21.11). Precedence
391 // (explicit): TOML is tried **after** the strong, fully-parsed tree formats
392 // (JSON, YAML) but **before** the weak, no-magic-byte heuristics (CSV,
393 // Markdown, XML, HTML). TOML's positive signal is a complete, error-free parse,
394 // which is far stronger than the CSV/Markdown heuristics; crucially, a TOML
395 // comment (`# …` at column 0) is otherwise misread as a Markdown ATX heading and
396 // would steal the document. A source that is valid JSON/YAML still wins (it is
397 // tried first); a source that is valid XML/HTML never parses as TOML (it begins
398 // with `<`), so nothing is lost there. TOML has no magic bytes and a plain prose
399 // paragraph is not TOML, so detection is conservative: the whole source must
400 // parse as TOML 1.0 under the TOML caps with no errors **and** carry at least one
401 // key/value assignment. TOML's duplicate-key/redefinition rules are enforced, so
402 // a doc that violates them is not detected. Plain prose and non-TOML text stay
403 // Opaque.
404 #[cfg(feature = "toml")]
405 if crate::adapter::toml::detect(source, limits) {
406 return DocumentFormat::Toml;
407 }
408 // CSV/TSV is the first **tabular** Wave-2 format (Phase 21.7.1). It has no
409 // magic bytes, so it is detected last and conservatively: only after the
410 // PDF/ZIP/JSON/YAML families are declined does a source qualify, and then only
411 // if it parses under a specific delimiter (`,` or tab) as a table with a
412 // consistent field count across a sampled majority of records and at least two
413 // columns. When in doubt the input stays Opaque.
414 #[cfg(feature = "csv")]
415 if crate::adapter::csv::detect(source, limits) {
416 return DocumentFormat::Csv;
417 }
418 // Markdown is the first **prose** Wave-2 format (Phase 21.8.1). It has no
419 // magic bytes either, and a plain prose paragraph is itself valid Markdown, so
420 // detection is deliberately conservative: only after the PDF/ZIP/JSON/YAML/CSV
421 // families are declined, and only when the source carries a clear structural
422 // mark (an ATX heading, a fenced code block, front matter, a table, a reference
423 // definition, or a footnote definition), is it admitted. Plain prose stays
424 // Opaque rather than being guessed to be Markdown.
425 #[cfg(feature = "markdown")]
426 if crate::adapter::markdown::detect(source, limits) {
427 return DocumentFormat::Markdown;
428 }
429 // HTML document-level marker (Phase 21.10 / 21.15 fix): a `<!doctype html>` or
430 // an `<html>` root element is a more specific signal than the generic XML
431 // fallback, so a **well-formed XHTML page** is classified `Html`, not `Xml`
432 // (XML's well-formedness parser would otherwise claim it). The general XML
433 // branch below is still tried before the general HTML branch, so a bare XML
434 // tree stays XML and HTML only claims the sources XML declines.
435 #[cfg(feature = "html")]
436 if crate::adapter::html::has_document_marker(source)
437 && crate::adapter::html::detect(source, limits)
438 {
439 return DocumentFormat::Html;
440 }
441 // XML is the structured-tree Wave-2 format for a bare XML source (Phase 21.9).
442 // It has no package layer and no magic bytes beyond `<`, so it is detected last
443 // and conservatively: only after the PDF/ZIP/JSON/YAML/CSV/Markdown families are
444 // declined, only when the source begins with `<` (an XML prolog or a root
445 // element start), and only when the whole source parses as well-formed XML with
446 // exactly one root element under the XML caps. `<-prefixed` junk and prose stay
447 // Opaque rather than being guessed to be XML.
448 #[cfg(feature = "xml")]
449 if crate::adapter::xml::detect(source, limits) {
450 return DocumentFormat::Xml;
451 }
452 // HTML is the error-recovering markup Wave-2 format (Phase 21.10). Precedence
453 // note (explicit): a *document-level* HTML marker is handled above (before XML);
454 // for every other `<`-bearing source XML is tried before HTML, so any document
455 // XML accepts (a bare XML tree) is genuinely XML. HTML then claims the
456 // `<`-bearing sources XML declines — real HTML5 is almost never well-formed XML
457 // (void elements, optional end tags, unquoted attributes, undeclared named
458 // entities). Detection is conservative: the source must carry clear HTML
459 // structure (a `<!doctype html>`, an `<html`/`<head`/`<body` tag, or a
460 // preponderance of known HTML tags). Plain prose and non-HTML `<`-junk stay
461 // Opaque rather than being guessed to be HTML.
462 #[cfg(feature = "html")]
463 if crate::adapter::html::detect(source, limits) {
464 return DocumentFormat::Html;
465 }
466 DocumentFormat::Opaque
467}
468
469/// Whether `source` is a structurally valid ZIP archive.
470///
471/// Used by the universal ingest dispatcher so a generic ZIP (which detection
472/// reports as `Opaque`) is still inverted through the byte-authoritative package
473/// layer rather than the opaque floor.
474#[cfg(feature = "package")]
475pub fn is_zip(source: &[u8], limits: Limits) -> bool {
476 crate::adapter::package::scan(source, limits).is_ok()
477}
478
479#[cfg(feature = "package")]
480fn detect_zip_family(source: &[u8], limits: Limits) -> Option<DocumentFormat> {
481 let physical = crate::adapter::package::scan(source, limits).ok()?;
482
483 // EPUB (OCF): the mandatory `mimetype` member, or an OCF container that
484 // resolves to an OPF rootfile.
485 let mimetype = member_decoded(&physical, source, EPUB_MIMETYPE_MEMBER, limits);
486 let mimetype_ok = mimetype.as_deref() == Some(EPUB_MIMETYPE);
487 let container = member_decoded(&physical, source, CONTAINER_MEMBER, limits);
488 let container_ok = container
489 .as_deref()
490 .is_some_and(|c| contains(c, CONTAINER_NS) && contains(c, OPF_MEDIA_TYPE));
491 let is_epub = mimetype_ok || container_ok;
492
493 // DOCX: an OPC package whose content types declare a WordprocessingML main
494 // part, or whose package relationships declare an `officeDocument` part that
495 // targets `word/document.xml`. The positive WordprocessingML signal (rather
496 // than the mere absence of a SpreadsheetML one) keeps DOCX and XLSX mutually
497 // exclusive without misclassifying a Word document that *embeds* an Excel
498 // workbook (whose package declares SpreadsheetML content types for the
499 // embedded part, but no SpreadsheetML workbook main part). Without `xlsx`
500 // and `pptx` the legacy relationship-only rule stands.
501 let content_types = member_decoded(&physical, source, CONTENT_TYPES_MEMBER, limits);
502 let rels = member_decoded(&physical, source, PACKAGE_RELS_MEMBER, limits);
503 #[cfg(any(feature = "xlsx", feature = "pptx"))]
504 let is_docx = content_types
505 .as_deref()
506 .is_some_and(|ct| contains(ct, DOCX_MAIN_FRAGMENT))
507 || rels.as_deref().is_some_and(|r| {
508 contains(r, OFFICE_DOCUMENT_FRAGMENT) && contains(r, DOCX_MAIN_TARGET)
509 });
510 #[cfg(not(any(feature = "xlsx", feature = "pptx")))]
511 let is_docx = content_types.is_some()
512 && rels
513 .as_deref()
514 .is_some_and(|r| contains(r, OFFICE_DOCUMENT_FRAGMENT));
515
516 // XLSX: an OPC package whose content types declare a SpreadsheetML workbook
517 // (or whose `officeDocument` relationship targets a workbook part).
518 #[cfg(feature = "xlsx")]
519 let is_xlsx = content_types.as_deref().is_some_and(|ct| {
520 contains(ct, XLSX_MAIN_FRAGMENT)
521 || (contains(ct, XLSX_NS_FRAGMENT)
522 && rels
523 .as_deref()
524 .is_some_and(|r| contains(r, XLSX_WORKBOOK_TARGET)))
525 });
526 #[cfg(not(feature = "xlsx"))]
527 let is_xlsx = false;
528
529 // PPTX: an OPC package whose content types declare a PresentationML main part
530 // (or whose content types name PresentationML and whose `officeDocument`
531 // relationship targets `ppt/presentation.xml`). The positive PresentationML
532 // signal keeps it mutually exclusive with DOCX and XLSX: a Word/Excel document
533 // that *embeds* a PowerPoint part declares only the PresentationML embed type
534 // (`…presentationml.presentation`, not the `.main+xml` main part) and its
535 // `officeDocument` relationship targets `word/document.xml`/`xl/workbook.xml`,
536 // so it is never misclassified as PPTX.
537 #[cfg(feature = "pptx")]
538 let is_pptx = content_types.as_deref().is_some_and(|ct| {
539 contains(ct, PPTX_MAIN_FRAGMENT)
540 || (contains(ct, PPTX_NS_FRAGMENT)
541 && rels
542 .as_deref()
543 .is_some_and(|r| contains(r, PPTX_MAIN_TARGET)))
544 });
545 #[cfg(not(feature = "pptx"))]
546 let is_pptx = false;
547
548 // The mandatory ODF `mimetype` member is **authoritative** when it names an
549 // OpenDocument media type (ODF 1.2 §3.2: the package's own media type). An ODP
550 // that *embeds a spreadsheet* lists both media types in `META-INF/manifest.xml`
551 // (one per file-entry), so scanning the whole manifest would make it look like an
552 // ODP *and* an ODS and force the ambiguous case below to Opaque. Only when the
553 // `mimetype` member is absent or is not an ODF media type do we fall back to the
554 // manifest scan (for a non-conformant package).
555 #[cfg(any(feature = "odt", feature = "ods", feature = "odp"))]
556 let odf_mimetype = mimetype
557 .as_deref()
558 .filter(|m| contains(m, ODF_MEDIA_PREFIX));
559
560 // ODT: an ODF package whose mandatory `mimetype` (or, absent that,
561 // `META-INF/manifest.xml`) declares an OpenDocument text media type.
562 #[cfg(feature = "odt")]
563 let is_odt = match odf_mimetype {
564 Some(m) => contains(m, ODT_TEXT_FRAGMENT),
565 None => member_decoded(&physical, source, ODT_MANIFEST_MEMBER, limits)
566 .as_deref()
567 .is_some_and(|m| contains(m, ODT_TEXT_FRAGMENT)),
568 };
569 #[cfg(not(feature = "odt"))]
570 let is_odt = false;
571
572 // ODS: an ODF package whose mandatory `mimetype` (or, absent that, the manifest)
573 // declares an OpenDocument *spreadsheet* media type. The positive spreadsheet
574 // fragment keeps it mutually exclusive with ODT (a text document declares the
575 // text media type, never the spreadsheet one).
576 #[cfg(feature = "ods")]
577 let is_ods = match odf_mimetype {
578 Some(m) => contains(m, ODS_SPREADSHEET_FRAGMENT),
579 None => member_decoded(&physical, source, ODS_MANIFEST_MEMBER, limits)
580 .as_deref()
581 .is_some_and(|m| contains(m, ODS_SPREADSHEET_FRAGMENT)),
582 };
583 #[cfg(not(feature = "ods"))]
584 let is_ods = false;
585
586 // ODP: an ODF package whose mandatory `mimetype` (or, absent that, the manifest)
587 // declares an OpenDocument *presentation* media type. The positive presentation
588 // fragment keeps it mutually exclusive with ODT/ODS.
589 #[cfg(feature = "odp")]
590 let is_odp = match odf_mimetype {
591 Some(m) => contains(m, ODP_PRESENTATION_FRAGMENT),
592 None => member_decoded(&physical, source, ODP_MANIFEST_MEMBER, limits)
593 .as_deref()
594 .is_some_and(|m| contains(m, ODP_PRESENTATION_FRAGMENT)),
595 };
596 #[cfg(not(feature = "odp"))]
597 let is_odp = false;
598
599 // A ZIP matching more than one native signature is ambiguous: fail safe.
600 let matches = [is_docx, is_epub, is_odt, is_ods, is_odp, is_xlsx, is_pptx]
601 .iter()
602 .filter(|b| **b)
603 .count();
604 match matches {
605 1 if is_docx => Some(DocumentFormat::Docx),
606 1 if is_epub => Some(DocumentFormat::Epub),
607 1 if is_odt => Some(DocumentFormat::Odt),
608 1 if is_ods => Some(DocumentFormat::Ods),
609 1 if is_odp => Some(DocumentFormat::Odp),
610 1 if is_xlsx => Some(DocumentFormat::Xlsx),
611 1 if is_pptx => Some(DocumentFormat::Pptx),
612 _ => None,
613 }
614}
615
616#[cfg(feature = "package")]
617const EPUB_MIMETYPE_MEMBER: &[u8] = b"mimetype";
618
619/// Decode one member's bytes by exact name, bounded and decline-safe: encrypted,
620/// oversized, unsupported-method, or out-of-range members yield `None`. The name
621/// match is ASCII case-insensitive, mirroring the OPC layer
622/// ([`crate::field::opc`]), which resolves the well-known control parts
623/// (`[Content_Types].xml`, `_rels/.rels`) case-insensitively; a package that
624/// lowercases them is still identified (Phase 21.15 fix).
625#[cfg(feature = "package")]
626fn member_decoded(
627 physical: &crate::adapter::package::ZipPhysical,
628 source: &[u8],
629 name: &[u8],
630 limits: Limits,
631) -> Option<Vec<u8>> {
632 const FLAG_ENCRYPTED: u16 = 0x0001;
633 let member = physical
634 .members
635 .iter()
636 .find(|m| m.name.eq_ignore_ascii_case(name))?;
637 if member.flags & FLAG_ENCRYPTED != 0 || member.uncompressed_size > limits.max_xml_part_bytes {
638 return None;
639 }
640 let off = usize::try_from(member.data.0).ok()?;
641 let len = usize::try_from(member.data.1).ok()?;
642 let raw = source.get(off..off.checked_add(len)?)?;
643 match member.method {
644 0 => Some(raw.to_vec()),
645 8 => crate::field::derive::inflate_raw_deflate(raw, member.uncompressed_size, limits).ok(),
646 _ => None,
647 }
648}
649
650/// Byte-substring search (no allocation, case-sensitive).
651#[cfg(feature = "package")]
652fn contains(haystack: &[u8], needle: &[u8]) -> bool {
653 !needle.is_empty()
654 && needle.len() <= haystack.len()
655 && haystack.windows(needle.len()).any(|w| w == needle)
656}
657
658#[cfg(test)]
659mod tests {
660 use super::*;
661
662 #[test]
663 fn names_and_provenance_roundtrip() {
664 for f in [
665 DocumentFormat::Pdf,
666 DocumentFormat::Docx,
667 DocumentFormat::Epub,
668 DocumentFormat::Odt,
669 DocumentFormat::Ods,
670 DocumentFormat::Odp,
671 DocumentFormat::Xlsx,
672 DocumentFormat::Pptx,
673 DocumentFormat::Json,
674 DocumentFormat::Yaml,
675 DocumentFormat::Csv,
676 DocumentFormat::Markdown,
677 DocumentFormat::Xml,
678 DocumentFormat::Html,
679 DocumentFormat::Toml,
680 DocumentFormat::Jsonl,
681 DocumentFormat::Eml,
682 DocumentFormat::Parquet,
683 DocumentFormat::Opaque,
684 ] {
685 let token = format!("{}field:package;members=1", f.provenance_prefix());
686 assert_eq!(DocumentFormat::from_provenance(&token), Some(f));
687 }
688 assert_eq!(DocumentFormat::from_provenance("field:ingest-b"), None);
689 assert_eq!(DocumentFormat::from_provenance("format=exotic;x"), None);
690 }
691
692 #[test]
693 fn plain_bytes_are_opaque() {
694 assert_eq!(
695 detect_document_format(b"not a document", Limits::DEFAULT),
696 DocumentFormat::Opaque
697 );
698 }
699
700 #[test]
701 fn a_corpus_pdf_is_detected() {
702 let (_, pdf) = crate::adapter::pdf::sample_pdfs()
703 .into_iter()
704 .next()
705 .expect("the PDF corpus is non-empty");
706 assert_eq!(
707 detect_document_format(&pdf, Limits::DEFAULT),
708 DocumentFormat::Pdf
709 );
710 }
711}