vole_document/field/document_format.rs
1//! Byte-based document-format detection for the universal observation API
2//! (Phase 12.7, ADR-0031, plan §DEC-5).
3//!
4//! The format of a field is decided from its **source bytes**, never from a file
5//! name or extension. Detection is deliberately conservative: an ambiguous or
6//! malformed input falls back to [`DocumentFormat::Opaque`] rather than guessing,
7//! and a format whose adapter is not compiled in cannot be detected (the input is
8//! then `Opaque`), so the reported capability set always matches what the build
9//! can actually serve.
10//!
11//! * **PDF** — the byte-authoritative physical scanner admits a validated PDF
12//! ([`crate::adapter::pdf::detect`]).
13//! * **DOCX** — a valid ZIP that also carries the OPC content-types part
14//! (`[Content_Types].xml`) and a package `officeDocument` relationship.
15//! * **EPUB** — a valid ZIP that is an OCF container: the mandatory stored
16//! `mimetype` member equals `application/epub+zip`, or `META-INF/container.xml`
17//! names the OCF namespace and an OPF (`application/oebps-package+xml`) rootfile.
18//! * **ODT** — a valid ZIP that is an OpenDocument (ODF) package: the mandatory
19//! stored `mimetype` member is an OpenDocument *text* media type, or
20//! `META-INF/manifest.xml` declares one.
21//! * **ODS** — a valid ZIP that is an OpenDocument (ODF) package whose mandatory
22//! stored `mimetype` member is an OpenDocument *spreadsheet* media type, or whose
23//! `META-INF/manifest.xml` declares one (Phase 21.3.1). Mutually exclusive with
24//! ODT (a text document declares the text media type, a spreadsheet the
25//! spreadsheet one).
26//! * **ODP** — a valid ZIP that is an OpenDocument (ODF) package whose mandatory
27//! stored `mimetype` member is an OpenDocument *presentation* media type, or whose
28//! `META-INF/manifest.xml` declares one (Phase 21.4.1). Mutually exclusive with
29//! ODT/ODS (a presentation declares the presentation media type).
30//! * **Opaque** — everything else, including a ZIP that matches none of the above
31//! (or more than one — an ambiguous ZIP fails safe).
32//! * **JSON** — the whole source parses as exactly one JSON value within the
33//! JSON caps (Phase 21.5.1). JSON is a Wave-2 structured-tree format with no
34//! package layer, so it is detected directly (never via `detect_zip_family`);
35//! a malformed or oversized input stays `Opaque`.
36//!
37//! The detected format is recorded in the field manifest's provenance (a
38//! machine-readable `format=<name>;` prefix, see [`DocumentFormat::from_provenance`]),
39//! so `observe`/`find`/`explain` can dispatch common selectors without reading the
40//! whole source again.
41
42use crate::limits::Limits;
43
44/// The mandatory OCF `mimetype` payload.
45pub const EPUB_MIMETYPE: &[u8] = b"application/epub+zip";
46/// The mandatory OCF container descriptor member.
47pub const CONTAINER_MEMBER: &[u8] = b"META-INF/container.xml";
48/// The OCF container-descriptor namespace.
49pub const CONTAINER_NS: &[u8] = b"urn:oasis:names:tc:opendocument:xmlns:container";
50/// The default (and only normative) package-document media type.
51pub const OPF_MEDIA_TYPE: &[u8] = b"application/oebps-package+xml";
52/// The OPC content-types part.
53#[cfg(feature = "package")]
54const CONTENT_TYPES_MEMBER: &[u8] = b"[Content_Types].xml";
55/// The OPC package-relationships part.
56#[cfg(feature = "package")]
57const PACKAGE_RELS_MEMBER: &[u8] = b"_rels/.rels";
58/// The `officeDocument` relationship type fragment (transitional and strict).
59#[cfg(feature = "package")]
60const OFFICE_DOCUMENT_FRAGMENT: &[u8] = b"officeDocument";
61/// The WordprocessingML document main content-type fragment (Phase 21.1.2).
62#[cfg(feature = "package")]
63const DOCX_MAIN_FRAGMENT: &[u8] = b"wordprocessingml.document.main+xml";
64/// The canonical WordprocessingML main-part target fragment (Phase 21.1.2).
65#[cfg(feature = "package")]
66const DOCX_MAIN_TARGET: &[u8] = b"word/document.xml";
67/// The ODF package manifest member (Phase 13.3).
68#[cfg(feature = "odt")]
69const ODT_MANIFEST_MEMBER: &[u8] = b"META-INF/manifest.xml";
70/// The OpenDocument *text* media-type fragment (Phase 13.3).
71#[cfg(feature = "odt")]
72const ODT_TEXT_FRAGMENT: &[u8] = b"application/vnd.oasis.opendocument.text";
73/// The ODF package manifest member for the ODS detection rule (Phase 21.3.1).
74#[cfg(feature = "ods")]
75const ODS_MANIFEST_MEMBER: &[u8] = b"META-INF/manifest.xml";
76/// The OpenDocument *spreadsheet* media-type fragment (Phase 21.3.1).
77#[cfg(feature = "ods")]
78const ODS_SPREADSHEET_FRAGMENT: &[u8] = b"application/vnd.oasis.opendocument.spreadsheet";
79/// The ODF package manifest member for the ODP detection rule (Phase 21.4.1).
80#[cfg(feature = "odp")]
81const ODP_MANIFEST_MEMBER: &[u8] = b"META-INF/manifest.xml";
82/// The OpenDocument *presentation* media-type fragment (Phase 21.4.1).
83#[cfg(feature = "odp")]
84const ODP_PRESENTATION_FRAGMENT: &[u8] = b"application/vnd.oasis.opendocument.presentation";
85/// The ODF media-type prefix shared by every OpenDocument package.
86#[cfg(any(feature = "odt", feature = "ods", feature = "odp"))]
87const ODF_MEDIA_PREFIX: &[u8] = b"application/vnd.oasis.opendocument.";
88/// The SpreadsheetML workbook main content-type fragment (Phase 21.1.1).
89#[cfg(feature = "xlsx")]
90const XLSX_MAIN_FRAGMENT: &[u8] = b"spreadsheetml.sheet.main+xml";
91/// The SpreadsheetML content-type namespace fragment (Phase 21.1.1).
92#[cfg(feature = "xlsx")]
93const XLSX_NS_FRAGMENT: &[u8] = b"spreadsheetml";
94/// A SpreadsheetML workbook part-target fragment (Phase 21.1.1).
95#[cfg(feature = "xlsx")]
96const XLSX_WORKBOOK_TARGET: &[u8] = b"xl/workbook.xml";
97/// The PresentationML presentation main content-type fragment (Phase 21.2.1).
98#[cfg(feature = "pptx")]
99const PPTX_MAIN_FRAGMENT: &[u8] = b"presentationml.presentation.main+xml";
100/// The PresentationML content-type namespace fragment (Phase 21.2.1).
101#[cfg(feature = "pptx")]
102const PPTX_NS_FRAGMENT: &[u8] = b"presentationml";
103/// The canonical PresentationML main-part target fragment (Phase 21.2.1).
104#[cfg(feature = "pptx")]
105const PPTX_MAIN_TARGET: &[u8] = b"ppt/presentation.xml";
106
107/// A detected document format (the class of the field's source bytes).
108#[derive(Debug, Clone, Copy, PartialEq, Eq)]
109pub enum DocumentFormat {
110 /// A validated PDF (physical indicators).
111 Pdf,
112 /// An OPC package with a WordprocessingML `officeDocument` part.
113 Docx,
114 /// An OCF container with an EPUB package document.
115 Epub,
116 /// An ODF package with an OpenDocument text content part.
117 Odt,
118 /// An ODF package with an OpenDocument spreadsheet content part.
119 Ods,
120 /// An ODF package with an OpenDocument presentation content part.
121 Odp,
122 /// An OPC package with a SpreadsheetML workbook part.
123 Xlsx,
124 /// An OPC package with a PresentationML presentation part.
125 Pptx,
126 /// A structured JSON document (the whole source parses as exactly one JSON
127 /// value). Not a package: the exact leaf is the whole source (Phase 21.5.1).
128 Json,
129 /// A structured JSON5 / JSONC document (the whole source parses as exactly one
130 /// JSON5 value and uses at least one JSON5/JSONC-only construct — a comment, a
131 /// trailing comma, an unquoted key, a single-quoted string, a hex/leading-dot/
132 /// `Infinity`/`NaN` number, a string continuation, or the extended whitespace
133 /// set). A source that is **strict JSON** stays [`DocumentFormat::Json`]; the
134 /// recorded dialect distinguishes `jsonc` (comments/trailing commas only) from
135 /// `json5`. Not a package: the exact leaf is the whole source (Phase 21.17.1).
136 Json5,
137 /// A structured YAML document (the whole source parses as a bounded YAML
138 /// stream whose every document root is a mapping or a sequence). Not a package:
139 /// the exact leaf is the whole source (Phase 21.6.1).
140 Yaml,
141 /// A CSV/TSV table (the whole source parses under a comma or tab delimiter as a
142 /// table with a consistent field count across a sampled majority of at least
143 /// two records). Not a package: the exact leaf is the whole source, and the
144 /// recorded dialect distinguishes comma from tab (Phase 21.7.1).
145 Csv,
146 /// A Markdown prose document (the whole source carries at least one structural
147 /// mark — an ATX heading, a fenced code block, front matter, a table, a
148 /// reference definition, or a footnote definition). Not a package: the exact
149 /// leaf is the whole source (Phase 21.8.1).
150 Markdown,
151 /// A standalone XML document (the whole source begins with `<` and parses as
152 /// well-formed XML with exactly one root element under the XML caps). Not a
153 /// package: the exact leaf is the whole source, and every node/attribute span
154 /// is a `Q_gen` projection (Phase 21.9).
155 Xml,
156 /// A standalone HTML document (the source is not well-formed XML and carries
157 /// clear HTML structure — a `<!doctype html>`, an `<html`/`<head`/`<body` tag,
158 /// or a preponderance of known HTML tags). Parsed by a bounded,
159 /// **error-recovering** scanner. Not a package: the exact leaf is the whole
160 /// source, and every element/attribute/text/comment/raw-text span is a `Q_gen`
161 /// projection (Phase 21.10).
162 Html,
163 /// A TOML document (the source is not any earlier format, parses as TOML 1.0
164 /// under the TOML caps with no errors, and carries at least one key/value
165 /// assignment). Not a package: the exact leaf is the whole source, and every
166 /// table/array/key/value/comment span is a `Q_gen` projection. TOML's
167 /// duplicate-key/redefinition rules are enforced (a violation is a typed
168 /// decline) (Phase 21.11).
169 Toml,
170 /// A JSONL / NDJSON line/event stream (the source carries at least two non-blank
171 /// lines and **every** non-blank line parses as exactly one JSON value under the
172 /// caps, each line parsed by the shared JSON parser). Not a package: the exact
173 /// leaf is the whole source, and every record's exact line span, terminator, and
174 /// per-token spans are `Q_gen` projections. A single JSON value stays
175 /// [`DocumentFormat::Json`] (Phase 21.12).
176 Jsonl,
177 /// An EML / MIME internet message (an RFC 5322 header block terminated by a
178 /// blank line, carrying `From`/`Date`/`Message-ID`, or an explicit
179 /// `MIME-Version`). Not a package: the exact leaf is the whole source, and every
180 /// header/part span and every `Content-Transfer-Encoding`-decoded constituent is
181 /// a `Q_gen` projection (Phase 21.13).
182 Eml,
183 /// An Apache Parquet file (the source begins with `PAR1` and ends with `PAR1`,
184 /// and the 4-byte little-endian footer length before the trailing magic is
185 /// consistent with the file length). Not a package: the exact leaf is the whole
186 /// source, and the parsed footer inventory (schema, row groups, column chunks,
187 /// statistics) and any decoded values are `Q_gen` projections (Phase 21.14).
188 Parquet,
189 /// An Apache Arrow IPC file/stream (the source begins with the `ARROW1` magic
190 /// and is either the **file** format — a trailing `ARROW1` magic preceded by a
191 /// consistent little-endian `int32` footer length — or the **stream** format —
192 /// a valid encapsulated `Schema` message at the 8-byte magic+padding prefix).
193 /// Not a package: the exact leaf is the whole source, and the parsed schema and
194 /// record-batch inventory (with each batch's exact source span) and any decoded
195 /// columnar values are `Q_gen` projections (Phase 21.16).
196 ArrowIpc,
197 /// A CBOR (RFC 8949) structured-tree document. CBOR has **no magic bytes**, so
198 /// detection is deliberately conservative: either the self-described-CBOR tag
199 /// `55799` (`0xd9 0xd9 0xf7`) at the start of a source that then parses, in full,
200 /// as exactly one well-formed CBOR item; or a full-input well-formed parse whose
201 /// root is a container (array/map) or a tag and which reaches at least three
202 /// nodes (so a single scalar, an empty container, and any structurally trivial
203 /// input stay `Opaque`). A container head byte is always `>= 0x80`, so a
204 /// pure-ASCII text document is never claimed; and the whole-number/short-
205 /// container prefix overlaps MessagePack's fixint/fixarray encodings, so an
206 /// ambiguous or trivial input stays `Opaque` rather than being guessed
207 /// (recorded honestly; a Phase-21.19 MessagePack adapter must share the seam).
208 /// Not a package: the exact leaf is the whole source, and every item's kind,
209 /// exact span, encoding width, tag number, float width, and definite/indefinite
210 /// form are `Q_gen` projections (Phase 21.18).
211 Cbor,
212 /// A MessagePack structured-tree document. MessagePack has **no magic bytes**, so
213 /// detection is deliberately conservative: a full-input well-formed parse of
214 /// exactly one item whose root is a container (array/map) reaching a node/byte
215 /// threshold, or the same with an unambiguous MessagePack-only head byte
216 /// (`0xdc..=0xdf`, which CBOR's grammar rejects). A lone scalar, an empty
217 /// container, and any structurally trivial or ambiguous input stay `Opaque`. A
218 /// container head byte is always `>= 0x80`, so a pure-ASCII text document is never
219 /// claimed; and the whole-number/short-container prefix overlaps CBOR's
220 /// `fixint`/short containers, so CBOR (tried first) owns any input well-formed
221 /// under both. Not a package: the exact leaf is the whole source, and every item's
222 /// kind, exact span, exact format byte (encoding width and signedness), `str` vs
223 /// `bin`, float width, and extension type/length are `Q_gen` projections
224 /// (Phase 21.19).
225 Msgpack,
226 /// A config-family document (INI, `.env`, or Java `.properties`). The whole
227 /// source is a bounded, line-based key/value document; detection is
228 /// deliberately conservative and requires a **dialect-distinguishing signal**,
229 /// so the pure `KEY=VALUE` overlap between `env` and `properties` (and plain
230 /// prose, and a `#!` script) stays `Opaque` rather than being guessed. Not a
231 /// package: the exact leaf is the whole source, the recorded dialect lives in
232 /// the model, and every line's content span, key/separator/value span, quoting,
233 /// inline-comment, `export`, continuation, and `\uXXXX`-spelling fact is a
234 /// `Q_gen` projection (Phase 21.20).
235 Config,
236 /// An RSS 2.0 / Atom 1.0 syndication feed. A feed is XML, so its physical
237 /// bytes are shared with [`DocumentFormat::Xml`]; the recorded dialect
238 /// (`rss`/`atom`) lives in the model, exactly as CSV/config record their
239 /// dialects. Detection is a **bounded semantic test** run before the generic
240 /// XML detector: an RSS root `<rss>` with a `<channel>` child, or an Atom root
241 /// `<feed>` in the Atom namespace (`http://www.w3.org/2005/Atom`) with at least
242 /// one `<entry>` child. Not a package: the exact leaf is the whole source, and
243 /// every element/attribute span, element order, attribute spelling, and the
244 /// namespace declaration is a `Q_gen` projection (Phase 21.21).
245 Feed,
246 /// A GeoJSON (RFC 7946) document. GeoJSON is JSON, so its physical bytes are
247 /// shared with [`DocumentFormat::Json`]; the recorded root class (one of the nine
248 /// RFC 7946 type names) lives in the model, exactly as a feed records its
249 /// dialect. Detection is a **bounded semantic test** run before the generic JSON
250 /// detector: the source parses as exactly one JSON value whose root is an object
251 /// with a `"type"` string equal to one of the nine type names and whose shape is
252 /// consistent (a geometry has an array `coordinates`/`geometries`, a `Feature` has
253 /// `geometry`/`properties`, a `FeatureCollection` has an array `features`). A plain
254 /// JSON document, and a JSON document whose `"type"` is an unrelated string, stay
255 /// [`DocumentFormat::Json`]. Not a package: the exact leaf is the whole source, and
256 /// every JSON token span, `coordinates` nesting, `properties` order/duplicates, and
257 /// foreign member is a `Q_gen` projection (Phase 21.22).
258 Geojson,
259 /// A KML 2.2 / GPX 1.1 geospatial document. KML and GPX are XML, so their
260 /// physical bytes are shared with [`DocumentFormat::Xml`]; the recorded dialect
261 /// (`kml`/`gpx`) lives in the model, exactly as a feed records its dialect.
262 /// Detection is a **bounded semantic test** run before the generic XML detector:
263 /// a `<kml>` root in the KML namespace (`http://www.opengis.net/kml/2.2`) with a
264 /// `Document`/`Folder`/`Placemark` child, or a `<gpx>` root in the GPX namespace
265 /// (`http://www.topografix.com/GPX/1/1`) with a `metadata`/`wpt`/`rte`/`trk`
266 /// child. A plain XML document, and a shaped-but-invalid KML/GPX, stay
267 /// [`DocumentFormat::Xml`]. Not a package: the exact leaf is the whole source,
268 /// and every element/attribute span, element order, attribute spelling (KML
269 /// geometry `coordinates`, GPX `lat`/`lon`), and the namespace declaration is a
270 /// `Q_gen` projection (Phase 21.23).
271 Gis,
272 /// A Jupyter notebook (`.ipynb`, nbformat) document. A notebook's physical bytes
273 /// are JSON, so its bytes are shared with [`DocumentFormat::Json`]; the recorded
274 /// `nbformat`/`nbformat_minor` and the cell anchors live in the model, exactly as a
275 /// feed records its dialect. Detection is a **bounded semantic test** run before the
276 /// generic JSON detector: the source parses as exactly one JSON value whose root is
277 /// an object with a plain non-negative integer-literal `nbformat` (≥ 1) and an array
278 /// `cells`, every cell an object with a string `cell_type`, and every present
279 /// recognized field nbformat-shaped. A plain JSON document, and a JSON document that
280 /// merely has a `cells` key but is not nbformat-shaped, stay
281 /// [`DocumentFormat::Json`]. Not a package: the exact leaf is the whole source, and
282 /// every JSON token span, the exact `cell_type`/`output_type` strings, the exact
283 /// `source` representation, and the cell/output order are `Q_gen` projections
284 /// (Phase 21.24).
285 Notebook,
286 /// A fixed-width (column-position) text table. Unlike CSV/TSV/PSV, its columns
287 /// are defined by **character positions**, not a delimiter, so it is a distinct
288 /// format rather than a CSV dialect. Detection is **maximally conservative**
289 /// (fixed-width is inherently ambiguous): at least three sampled records of
290 /// identical byte width, an inferred column layout whose interior whitespace gaps
291 /// are at least two columns wide, at least two non-empty columns, and an outright
292 /// decline of anything that also parses as a delimited (CSV/TSV/PSV) or Markdown
293 /// table. Not a package: the exact leaf is the whole source, and the recorded
294 /// inferred layout, per-record spans, per-column spans, padding, terminator, BOM,
295 /// and header row are `Q_gen` projections (Phase 21.25).
296 FixedWidth,
297 /// A reStructuredText (Docutils) prose document (the whole source carries a
298 /// reST-specific structural mark — an explicit markup start (`.. ` comment,
299 /// directive, target, footnote/citation, or substitution definition), a grid or
300 /// simple table, a field list, or a section adornment whose char is not a
301 /// Markdown construct — so plain prose and a Markdown document are never stolen).
302 /// Not a package: the exact leaf is the whole source (Phase 21.26.1).
303 Rst,
304 /// An AsciiDoc prose document (the whole source carries an AsciiDoc-specific
305 /// structural mark — a level-0 document title followed by a further block, a
306 /// `==`+ section, a `|===` table, a complete delimited block, or a block attribute
307 /// line followed by a block — so plain prose and a Markdown or reStructuredText
308 /// document are never stolen). Not a package: the exact leaf is the whole source
309 /// (Phase 21.26.2).
310 Asciidoc,
311 /// An MDX document (Markdown + JSX/ESM). The whole source carries an MDX-specific
312 /// signal — a top-level ESM `import`/`export` statement, a JSX element with a
313 /// capitalized/namespaced component name (or a JSX-specific attribute), a JSX
314 /// fragment, or a whole-line MDX block expression `{…}` — so plain Markdown stays
315 /// `Markdown`, plain HTML stays `Html`, and plain prose stays `Opaque`. Not a
316 /// package: the exact leaf is the whole source (Phase 21.26.3).
317 Mdx,
318 /// An MHTML (MIME HTML, RFC 2557) web archive. MHTML is a MIME
319 /// `multipart/related` document, so its physical bytes are shared with
320 /// [`DocumentFormat::Eml`]; the adapter **reuses the EML adapter's bounded MIME
321 /// layer** for the envelope, parts, headers, and spans, and the **HTML adapter's
322 /// bounded, error-recovering scanner** for the root part, recording the ordered
323 /// sub-resources (keyed by `Content-Location`/`Content-ID`) and the root HTML part
324 /// (the `start=` `Content-ID`, else the first `text/html` part, else the first
325 /// part). Detection is a **bounded semantic sub-detection** run before the generic
326 /// EML detector: a `multipart/related` root plus an MHTML-specific signal — a
327 /// top-level `Snapshot-Content-Location`/`Content-Base`, or a `From`+`Subject`
328 /// envelope with a `text/html` part. A plain `multipart/related` email, a plain
329 /// HTML document, and a plain MIME message stay `Eml`/`Html`/`Eml`. Not a package:
330 /// the exact leaf is the whole source, and every envelope/part/resource/HTML span
331 /// is a `Q_gen` projection (Phase 21.27).
332 Mhtml,
333 /// A syslog / log-stream document (the source carries at least two non-blank
334 /// lines and **every** non-blank line matches one of three dialects — RFC 5424
335 /// syslog, RFC 3164 (BSD) syslog, or a generic application log line with a
336 /// leading timestamp and/or level token). The recorded dialect lives per record.
337 /// Detection is byte-based and conservative and must not steal a higher-priority
338 /// format (JSON/JSON5/JSONL/CBOR/Msgpack/YAML/TOML/config/CSV or the prose
339 /// family), all of which the whole-source dispatcher tries first; plain prose and
340 /// code stay `Opaque`. Not a package: the exact leaf is the whole source, and
341 /// every record's exact line span/terminator plus every field's exact span (and
342 /// the preserved message, NILVALUE, and timestamp/level spelling) are `Q_gen`
343 /// projections (Phase 21.28).
344 Logstream,
345 /// A package-metadata manifest (an npm `package.json` / `package-lock.json`, a
346 /// Cargo `Cargo.toml` / `Cargo.lock`, or a Python `pyproject.toml`). A manifest's
347 /// physical bytes are JSON or TOML, so its bytes are shared with
348 /// [`DocumentFormat::Json`]/[`DocumentFormat::Toml`]; the **recorded dialect**
349 /// (`npm_package` / `cargo_manifest` / `pyproject_manifest` / `cargo_lock` /
350 /// `npm_lock`) lives in the model, exactly as a feed or CSV records its dialect.
351 /// Detection is a **bounded semantic test** run before the generic JSON and TOML
352 /// detectors: a strong manifest shape (a JSON object with string `name`+`version`
353 /// and a third package-specific key, a JSON `lockfileVersion`+`packages`, a TOML
354 /// `[package]` table with a string `name`, a TOML `[build-system]`/`[project]`/
355 /// `[tool.poetry]`, or a TOML `[[package]]` lockfile shape). A generic JSON
356 /// `name`+`version`, a generic TOML `name`/`version`, and plain prose stay
357 /// `Json`/`Toml`/`Opaque`. Not a package: the exact leaf is the whole source, and
358 /// every reused-arena token span plus every recorded section/entry name/value span
359 /// is a `Q_gen` projection (Phase 21.29).
360 Pkgmeta,
361 /// An API/specification document (a JSON Schema, an OpenAPI 3.x, a Swagger 2.0, or
362 /// an AsyncAPI document). An API spec's physical bytes are JSON, so its bytes are
363 /// shared with [`DocumentFormat::Json`]; the **recorded dialect** (`json_schema` /
364 /// `openapi` / `swagger` / `asyncapi`) lives in the model, exactly as CSV/config/
365 /// feed/pkgmeta record their dialects. Detection is a **bounded semantic test** run
366 /// before the generic JSON detector on a **strong, root-level string marker**: a
367 /// JSON-Schema `$schema` URI, an `openapi` `3.`-prefixed string, a `swagger` equal
368 /// to `"2.0"`, or a non-empty `asyncapi` string. A plain JSON object that merely
369 /// contains a `properties`/`paths`/`components` key but lacks the marker stays
370 /// [`DocumentFormat::Json`], and a JSON-Schema-shaped object with no `$schema` is
371 /// an honest recorded ambiguity that also stays `Json`. Not a package: the exact
372 /// leaf is the whole source, and every JSON token span plus every recorded
373 /// object/member/`$ref` span is a `Q_gen` projection — `$ref` targets are preserved
374 /// verbatim and never resolved (Phase 21.30).
375 Apispec,
376 /// A bibliography document — **BibTeX** (`.bib`), **RIS**, or **CSL-JSON**. One
377 /// adapter covers the three dialects and records the dialect in the model, exactly
378 /// as CSV/config/feed/pkgmeta/apispec record theirs. BibTeX and RIS are text (no
379 /// magic bytes); CSL-JSON's physical bytes are JSON, so its **bounded semantic
380 /// sub-detection runs before the generic JSON detector**: the root must be an
381 /// **array** of objects (or a single object) each carrying a string `id` **and** a
382 /// string `type`. A plain JSON array, a JSON array of non-CSL objects, a generic
383 /// JSON object, and prose stay [`DocumentFormat::Json`]/[`DocumentFormat::Opaque`].
384 /// Not a package: the exact leaf is the whole source, and every entry/field/item
385 /// token span (and each value's exact spelling and delimiters) is a `Q_gen`
386 /// projection (Phase 21.31).
387 Bib,
388 /// An RTF (Rich Text Format) document. RTF is a text, group-brace grammar with
389 /// control words — closer to a bounded tree (like XML) than to a flat line format —
390 /// but it is *not* a package: there is no OPC/ZIP layer, so the exact leaf is the
391 /// whole source (a `DocumentExact`, RAW-like authority) and every derived
392 /// observation is a bounded, span-preserving (`Q_gen`) projection of it. Detection
393 /// is a **strong magic + brace-balance** test: the source must begin (after an
394 /// optional UTF-8 BOM) with `{\rtf` followed by a digit, and its group braces must
395 /// balance under a bounded scan (respecting `\\`/`\{`/`\}`, `\'hh`, control words,
396 /// and `\binN`), with the outermost group closing exactly at end of input. A plain
397 /// text source, a leading-`{` blob, and every other format are therefore never
398 /// stolen; they stay `Opaque`. The adapter preserves the header, every group with
399 /// its exact brace span and nesting, every control word/symbol, every `\'hh` hex
400 /// escape and `\uN` Unicode escape (with its `\ucN` fallback), every destination
401 /// (`\fonttbl`/`\colortbl`/`\stylesheet`/`\info`/`\pict`/`\object`/`\*`) verbatim,
402 /// and every plain text run byte-exactly (never whitespace-normalized) — Phase
403 /// 21.32.
404 Rtf,
405 /// A generic archive container (Phase 21.33): a standalone **ZIP** archive (a
406 /// valid EOCD and a coherent central directory) or a standalone **TAR** archive
407 /// (a valid `ustar`/old-GNU or v7 header whose checksum matches). The recorded
408 /// dialect (`zip`/`tar`) lives in the model, exactly as CSV/config/feed record
409 /// theirs. This is deliberately a **plain archive**, never an office package: the
410 /// ZIP-family office formats (DOCX/EPUB/ODT/ODS/ODP/XLSX/PPTX) are detected
411 /// **before** this adapter, so a `.docx`/`.xlsx`/`.odt` source still detects as
412 /// its office format and is never reclassified. Not a package: the exact leaf is
413 /// the whole source (a `DocumentExact`, RAW-like authority) and every derived
414 /// observation is a bounded, span-preserving (`Q_gen`) projection of it.
415 Container,
416 /// A scientific container (Phase 21.34): a NetCDF classic (CDF-1/CDF-2) file or an
417 /// HDF5 file (a documented bounded subset). The recorded dialect
418 /// (`netcdf`/`hdf5`) lives in the model, exactly as the archive container records
419 /// `zip`/`tar`. This is deliberately a **scan-only scientific format**: it is not
420 /// an office package. Detection is a **strong magic-byte** test — NetCDF requires
421 /// `CDF\x01`/`CDF\x02` at offset 0; HDF5 requires the 8-byte signature
422 /// `\x89HDF\r\n\x1a\n` at offset 0 or 512 with a superblock version in `0..=3`, an
423 /// offset/length size in `{2,4,8}`, and a superblock that fits inside the source.
424 /// Not a package: the exact leaf is the whole source (a `DocumentExact`, RAW-like
425 /// authority) and every derived observation is a bounded, span-preserving (`Q_gen`)
426 /// projection of it. NetCDF classic is supported in full; HDF5 is a documented
427 /// bounded subset (symbol-table groups + contiguous datasets + v1 object headers),
428 /// and everything outside it declines **typed**.
429 Hdf,
430 /// A CFBF / OLE2 compound file (Phase 21.35): the physical-layer container behind
431 /// legacy Word (`.doc`), Excel (`.xls`), PowerPoint (`.ppt`), and Outlook (`.msg`)
432 /// files. Detection is a **strong magic + header validation** test — the 8-byte
433 /// signature `D0 CF 11 E0 A1 B1 1A E1` at offset 0 **and** an internally
434 /// consistent header (byte order `FFFE`, a major version of 3 or 4, a sector shift
435 /// of 9 or 12 consistent with the version, a mini-sector shift of 6, at least one
436 /// FAT sector, and a first directory sector inside the file). A random
437 /// signature-only blob and prose stay `Opaque`. The recorded dialect
438 /// (`cfb3`/`cfb4`) lives in the model. Not a package: the exact leaf is the whole
439 /// source (a `DocumentExact`, RAW-like authority) and every derived observation —
440 /// the header, the DIFAT/FAT, the directory entries with their verbatim UTF-16LE
441 /// names and reconstructed sibling/child tree, and each stream's exact byte span —
442 /// is a bounded, span-preserving (`Q_gen`) projection of it. This is the shared
443 /// physical layer that unlocks the legacy DOC/XLS/PPT/MSG semantic models (later
444 /// subphases).
445 Cfbf,
446 /// Anything else; preserved exactly by the opaque floor.
447 Opaque,
448}
449
450impl DocumentFormat {
451 /// Stable lower-case name (used in JSON and in the manifest provenance token).
452 pub const fn name(self) -> &'static str {
453 match self {
454 DocumentFormat::Pdf => "pdf",
455 DocumentFormat::Docx => "docx",
456 DocumentFormat::Epub => "epub",
457 DocumentFormat::Odt => "odt",
458 DocumentFormat::Ods => "ods",
459 DocumentFormat::Odp => "odp",
460 DocumentFormat::Xlsx => "xlsx",
461 DocumentFormat::Pptx => "pptx",
462 DocumentFormat::Json => "json",
463 DocumentFormat::Json5 => "json5",
464 DocumentFormat::Yaml => "yaml",
465 DocumentFormat::Csv => "csv",
466 DocumentFormat::Markdown => "markdown",
467 DocumentFormat::Xml => "xml",
468 DocumentFormat::Html => "html",
469 DocumentFormat::Toml => "toml",
470 DocumentFormat::Jsonl => "jsonl",
471 DocumentFormat::Eml => "eml",
472 DocumentFormat::Parquet => "parquet",
473 DocumentFormat::ArrowIpc => "arrow",
474 DocumentFormat::Cbor => "cbor",
475 DocumentFormat::Msgpack => "msgpack",
476 DocumentFormat::Config => "config",
477 DocumentFormat::Feed => "feed",
478 DocumentFormat::Geojson => "geojson",
479 DocumentFormat::Gis => "gis",
480 DocumentFormat::Notebook => "notebook",
481 DocumentFormat::FixedWidth => "fixedwidth",
482 DocumentFormat::Rst => "rst",
483 DocumentFormat::Asciidoc => "asciidoc",
484 DocumentFormat::Mdx => "mdx",
485 DocumentFormat::Mhtml => "mhtml",
486 DocumentFormat::Logstream => "logstream",
487 DocumentFormat::Pkgmeta => "pkgmeta",
488 DocumentFormat::Apispec => "apispec",
489 DocumentFormat::Bib => "bib",
490 DocumentFormat::Rtf => "rtf",
491 DocumentFormat::Container => "container",
492 DocumentFormat::Hdf => "hdf",
493 DocumentFormat::Cfbf => "cfbf",
494 DocumentFormat::Opaque => "opaque",
495 }
496 }
497
498 /// The adapter that serves this format (the observation layer's name for it).
499 pub const fn adapter(self) -> &'static str {
500 match self {
501 DocumentFormat::Pdf => "pdf",
502 DocumentFormat::Docx => "docx",
503 DocumentFormat::Epub => "epub",
504 DocumentFormat::Odt => "odt",
505 DocumentFormat::Ods => "ods",
506 DocumentFormat::Odp => "odp",
507 DocumentFormat::Xlsx => "xlsx",
508 DocumentFormat::Pptx => "pptx",
509 DocumentFormat::Json => "json",
510 DocumentFormat::Json5 => "json5",
511 DocumentFormat::Yaml => "yaml",
512 DocumentFormat::Csv => "csv",
513 DocumentFormat::Markdown => "markdown",
514 DocumentFormat::Xml => "xml",
515 DocumentFormat::Html => "html",
516 DocumentFormat::Toml => "toml",
517 DocumentFormat::Jsonl => "jsonl",
518 DocumentFormat::Eml => "eml",
519 DocumentFormat::Parquet => "parquet",
520 DocumentFormat::ArrowIpc => "arrow",
521 DocumentFormat::Cbor => "cbor",
522 DocumentFormat::Msgpack => "msgpack",
523 DocumentFormat::Config => "config",
524 DocumentFormat::Feed => "feed",
525 DocumentFormat::Geojson => "geojson",
526 DocumentFormat::Gis => "gis",
527 DocumentFormat::Notebook => "notebook",
528 DocumentFormat::FixedWidth => "fixedwidth",
529 DocumentFormat::Rst => "rst",
530 DocumentFormat::Asciidoc => "asciidoc",
531 DocumentFormat::Mdx => "mdx",
532 DocumentFormat::Mhtml => "mhtml",
533 DocumentFormat::Logstream => "logstream",
534 DocumentFormat::Pkgmeta => "pkgmeta",
535 DocumentFormat::Apispec => "apispec",
536 DocumentFormat::Bib => "bib",
537 DocumentFormat::Rtf => "rtf",
538 DocumentFormat::Container => "container",
539 DocumentFormat::Hdf => "hdf",
540 DocumentFormat::Cfbf => "cfbf",
541 DocumentFormat::Opaque => "opaque",
542 }
543 }
544
545 /// Whether the adapter for this format is compiled into this build.
546 pub const fn compiled(self) -> bool {
547 match self {
548 DocumentFormat::Pdf | DocumentFormat::Opaque => true,
549 DocumentFormat::Docx => cfg!(feature = "docx"),
550 DocumentFormat::Epub => cfg!(feature = "epub"),
551 DocumentFormat::Odt => cfg!(feature = "odt"),
552 DocumentFormat::Ods => cfg!(feature = "ods"),
553 DocumentFormat::Odp => cfg!(feature = "odp"),
554 DocumentFormat::Xlsx => cfg!(feature = "xlsx"),
555 DocumentFormat::Pptx => cfg!(feature = "pptx"),
556 DocumentFormat::Json => cfg!(feature = "json"),
557 DocumentFormat::Json5 => cfg!(feature = "json5"),
558 DocumentFormat::Yaml => cfg!(feature = "yaml"),
559 DocumentFormat::Csv => cfg!(feature = "csv"),
560 DocumentFormat::Markdown => cfg!(feature = "markdown"),
561 DocumentFormat::Xml => cfg!(feature = "xml"),
562 DocumentFormat::Html => cfg!(feature = "html"),
563 DocumentFormat::Toml => cfg!(feature = "toml"),
564 DocumentFormat::Jsonl => cfg!(feature = "jsonl"),
565 DocumentFormat::Eml => cfg!(feature = "eml"),
566 DocumentFormat::Parquet => cfg!(feature = "parquet"),
567 DocumentFormat::ArrowIpc => cfg!(feature = "arrow"),
568 DocumentFormat::Cbor => cfg!(feature = "cbor"),
569 DocumentFormat::Msgpack => cfg!(feature = "msgpack"),
570 DocumentFormat::Config => cfg!(feature = "config"),
571 DocumentFormat::Feed => cfg!(feature = "feed"),
572 DocumentFormat::Geojson => cfg!(feature = "geojson"),
573 DocumentFormat::Gis => cfg!(feature = "gis"),
574 DocumentFormat::Notebook => cfg!(feature = "notebook"),
575 DocumentFormat::FixedWidth => cfg!(feature = "fixedwidth"),
576 DocumentFormat::Rst => cfg!(feature = "rst"),
577 DocumentFormat::Asciidoc => cfg!(feature = "asciidoc"),
578 DocumentFormat::Mdx => cfg!(feature = "mdx"),
579 DocumentFormat::Mhtml => cfg!(feature = "mhtml"),
580 DocumentFormat::Logstream => cfg!(feature = "logstream"),
581 DocumentFormat::Pkgmeta => cfg!(feature = "pkgmeta"),
582 DocumentFormat::Apispec => cfg!(feature = "apispec"),
583 DocumentFormat::Bib => cfg!(feature = "bib"),
584 DocumentFormat::Rtf => cfg!(feature = "rtf"),
585 DocumentFormat::Container => cfg!(feature = "container"),
586 DocumentFormat::Hdf => cfg!(feature = "hdf"),
587 DocumentFormat::Cfbf => cfg!(feature = "cfbf"),
588 }
589 }
590
591 /// The machine-readable `format=<name>;` provenance prefix recorded at ingest.
592 pub fn provenance_prefix(self) -> String {
593 format!("format={};", self.name())
594 }
595
596 /// Recover the recorded format from a field manifest's provenance string.
597 ///
598 /// Returns `None` for a manifest that does not carry the token (e.g. a field
599 /// written before this subphase); such a field still serves every native
600 /// selector, but common observations decline typed rather than guessing.
601 pub fn from_provenance(provenance: &str) -> Option<DocumentFormat> {
602 let rest = provenance.strip_prefix("format=")?;
603 let name = rest.split(';').next()?;
604 match name {
605 "pdf" => Some(DocumentFormat::Pdf),
606 "docx" => Some(DocumentFormat::Docx),
607 "epub" => Some(DocumentFormat::Epub),
608 "odt" => Some(DocumentFormat::Odt),
609 "ods" => Some(DocumentFormat::Ods),
610 "odp" => Some(DocumentFormat::Odp),
611 "xlsx" => Some(DocumentFormat::Xlsx),
612 "pptx" => Some(DocumentFormat::Pptx),
613 "json" => Some(DocumentFormat::Json),
614 "json5" => Some(DocumentFormat::Json5),
615 "yaml" => Some(DocumentFormat::Yaml),
616 "csv" => Some(DocumentFormat::Csv),
617 "markdown" => Some(DocumentFormat::Markdown),
618 "xml" => Some(DocumentFormat::Xml),
619 "html" => Some(DocumentFormat::Html),
620 "toml" => Some(DocumentFormat::Toml),
621 "jsonl" => Some(DocumentFormat::Jsonl),
622 "eml" => Some(DocumentFormat::Eml),
623 "parquet" => Some(DocumentFormat::Parquet),
624 "arrow" => Some(DocumentFormat::ArrowIpc),
625 "cbor" => Some(DocumentFormat::Cbor),
626 "msgpack" => Some(DocumentFormat::Msgpack),
627 "config" => Some(DocumentFormat::Config),
628 "feed" => Some(DocumentFormat::Feed),
629 "geojson" => Some(DocumentFormat::Geojson),
630 "gis" => Some(DocumentFormat::Gis),
631 "notebook" => Some(DocumentFormat::Notebook),
632 "fixedwidth" => Some(DocumentFormat::FixedWidth),
633 "rst" => Some(DocumentFormat::Rst),
634 "asciidoc" => Some(DocumentFormat::Asciidoc),
635 "mdx" => Some(DocumentFormat::Mdx),
636 "mhtml" => Some(DocumentFormat::Mhtml),
637 "logstream" => Some(DocumentFormat::Logstream),
638 "pkgmeta" => Some(DocumentFormat::Pkgmeta),
639 "apispec" => Some(DocumentFormat::Apispec),
640 "bib" => Some(DocumentFormat::Bib),
641 "rtf" => Some(DocumentFormat::Rtf),
642 "container" => Some(DocumentFormat::Container),
643 "hdf" => Some(DocumentFormat::Hdf),
644 "cfbf" => Some(DocumentFormat::Cfbf),
645 "opaque" => Some(DocumentFormat::Opaque),
646 _ => None,
647 }
648 }
649}
650
651/// Detect the document format of `source` from its bytes alone.
652///
653/// Never consults a file name or extension. A malformed or ambiguous input (or a
654/// format whose adapter is not compiled) falls back to [`DocumentFormat::Opaque`].
655pub fn detect_document_format(source: &[u8], limits: Limits) -> DocumentFormat {
656 if crate::adapter::pdf::detect(source, limits) {
657 return DocumentFormat::Pdf;
658 }
659 #[cfg(feature = "package")]
660 {
661 if let Some(format) = detect_zip_family(source, limits) {
662 return format;
663 }
664 }
665 // The generic archive container (Phase 21.33) is a **standalone** format, never
666 // an office package: a plain ZIP (a valid EOCD + coherent central directory) or a
667 // plain TAR (a strong 512-byte header whose checksum matches). It is inserted
668 // **after** the entire office ZIP family above — so an office package is never
669 // reclassified as a plain archive — and **before** every weak, no-magic-byte
670 // heuristic (JSON/YAML/CSV/Markdown/XML/HTML), which a binary archive must never
671 // be guessed to be. It is deliberately conservative: a `PK`-prefixed blob without
672 // a valid EOCD, a 512-byte-junk blob whose checksum fails, and prose all stay
673 // `Opaque`.
674 #[cfg(feature = "container")]
675 if crate::adapter::container::detect(source, limits) {
676 return DocumentFormat::Container;
677 }
678 // The scientific container (Phase 21.34) is a **standalone** format with strong
679 // magics: NetCDF classic (`CDF\x01`/`CDF\x02` at offset 0) and HDF5 (the 8-byte
680 // signature at offset 0 or 512 plus a validated superblock). It is inserted with
681 // the other strong-magic binaries (after the office ZIP family and the archive
682 // container, before Parquet) and **before** every weak, no-magic-byte heuristic
683 // (JSON/YAML/CSV/Markdown/XML/HTML), which a binary scientific file must never be
684 // guessed to be. It is deliberately conservative on HDF5: a random
685 // `\x89HDF`-prefixed blob with a bad superblock, and prose, stay `Opaque`.
686 // NetCDF's magic is the 4-byte `CDF\x01`/`CDF\x02`; a `CDF`-prefixed blob is
687 // admitted here and its derived model then declines typed if the header is
688 // malformed (the whole-source exact leaf is always unaffected).
689 #[cfg(feature = "hdf")]
690 if crate::adapter::hdf::detect(source, limits) {
691 return DocumentFormat::Hdf;
692 }
693 // The CFBF / OLE2 compound file (Phase 21.35) is a **standalone** physical-layer
694 // format with a strong 8-byte signature (`D0 CF 11 E0 A1 B1 1A E1`) plus a
695 // validated header. It is inserted with the other strong-magic binaries (after the
696 // office ZIP family, the archive container, and the scientific container, and
697 // **before** Parquet) and **before** every weak, no-magic-byte heuristic
698 // (JSON/YAML/CSV/Markdown/XML/HTML), which a binary compound file must never be
699 // guessed to be. It is deliberately conservative: a signature-only blob (its
700 // header fields are zero) and prose stay `Opaque`; a blob with a plausible header
701 // but a malformed body is admitted and its derived model then declines typed (the
702 // whole-source exact leaf is always unaffected).
703 #[cfg(feature = "cfbf")]
704 if crate::adapter::cfbf::detect(source, limits) {
705 return DocumentFormat::Cfbf;
706 }
707 // Parquet is the **analytical** Wave-2 format (Phase 21.14), with no package
708 // layer. Its detection is a strong magic-byte contract — `PAR1` at both ends
709 // plus a consistent little-endian footer length — so it runs immediately after
710 // the package families and **before** the weak, no-magic-byte heuristics
711 // (JSON/YAML/CSV/Markdown/XML/HTML), which a binary columnar file must never be
712 // guessed to be. It is deliberately conservative: a file that merely begins or
713 // ends with `PAR1` but whose footer length overruns the leading magic stays
714 // `Opaque`.
715 #[cfg(feature = "parquet")]
716 if crate::adapter::parquet::detect(source, limits) {
717 return DocumentFormat::Parquet;
718 }
719 // Arrow IPC is the **analytical** Wave-2 format (Phase 21.16), with no package
720 // layer. Its detection is a strong magic-byte contract — the `ARROW1` magic at
721 // the start plus either a consistent trailing footer (file format) or a valid
722 // encapsulated `Schema` message (stream format) — so it runs immediately after
723 // Parquet and **before** the weak, no-magic-byte heuristics, which a binary
724 // columnar file must never be guessed to be. It is deliberately conservative:
725 // `ARROW1`-prefixed junk with neither a consistent footer nor a valid schema
726 // message stays `Opaque`.
727 #[cfg(feature = "arrow")]
728 if crate::adapter::arrow::detect(source, limits) {
729 return DocumentFormat::ArrowIpc;
730 }
731 // JSON is a Wave-2 structured-tree format with **no** package layer, so it is
732 // detected directly from the whole source (never through `detect_zip_family`).
733 // Conservative: the entire source must parse as exactly one JSON value within
734 // the JSON caps, else the input stays Opaque (Phase 21.5.1).
735 //
736 // GeoJSON is the **spatial** Wave-2 format (Phase 21.22), whose physical bytes
737 // are JSON, so it is a **bounded semantic sub-detection** run **before** the
738 // generic JSON detector below: the source must parse as exactly one JSON value
739 // whose root is an object with a `"type"` string equal to one of the nine RFC 7946
740 // type names **and** whose shape is consistent (a geometry has an array
741 // `coordinates`/`geometries`, a `Feature` has `geometry`/`properties`, a
742 // `FeatureCollection` has an array `features`). A more specific claim than a bare
743 // JSON value, so it is tried first; a plain JSON document, and a JSON document
744 // whose `"type"` is an unrelated string, decline here and are then claimed by the
745 // JSON detector (staying `Json`). Prose and malformed input stay `Opaque`.
746 #[cfg(feature = "geojson")]
747 if crate::adapter::geojson::detect(source, limits) {
748 return DocumentFormat::Geojson;
749 }
750 // A Jupyter notebook (`.ipynb`, nbformat) is the **document-shaped** Wave-2 format
751 // (Phase 21.24), whose physical bytes are JSON, so it is a **bounded semantic
752 // sub-detection** run **before** the generic JSON detector below: the source must
753 // parse as exactly one JSON value that is an nbformat-shaped object — a plain
754 // non-negative integer-literal `nbformat` (≥ 1), an array `cells`, every cell an
755 // object with a string `cell_type`, and every present recognized field
756 // nbformat-shaped (a string-or-array-of-strings `source`, a number-or-null
757 // `execution_count`, object `metadata`/`attachments`, an array `outputs` of objects
758 // with a string `output_type`). A more specific claim than a bare JSON value, so it
759 // is tried first; a plain JSON document, and a JSON document that merely has a
760 // `cells` key but is not nbformat-shaped, decline here and are then claimed by the
761 // JSON detector (staying `Json`). Prose and malformed input stay `Opaque`.
762 #[cfg(feature = "notebook")]
763 if crate::adapter::notebook::detect(source, limits) {
764 return DocumentFormat::Notebook;
765 }
766 // A package-metadata manifest (Phase 21.29) — an npm `package.json` /
767 // `package-lock.json`, a Cargo `Cargo.toml` / `Cargo.lock`, or a Python
768 // `pyproject.toml` — is the **manifest** Wave-2 format, whose physical bytes are
769 // JSON or TOML. It is a **bounded semantic sub-detection** run **before** the
770 // generic JSON and TOML detectors below: the source must carry a strong manifest
771 // shape (a JSON object with string `name`+`version` and a third package-specific
772 // key, a JSON `lockfileVersion`+`packages`/`dependencies`, a TOML `[package]` table
773 // with a string `name`, a TOML `[build-system]`/`[project]`/`[tool.poetry]`, or a
774 // TOML `[[package]]` lockfile shape). A more specific claim than a bare JSON/TOML
775 // value, so it is tried first; a generic JSON `{"name":…,"version":…}`, a generic
776 // TOML `name = …` / `version = …`, and plain prose decline here and are then
777 // claimed by the JSON detector (staying `Json`), the TOML detector (staying
778 // `Toml`), or the opaque floor. Detection is content-only and never consults a
779 // file name.
780 #[cfg(feature = "pkgmeta")]
781 if crate::adapter::pkgmeta::detect(source, limits) {
782 return DocumentFormat::Pkgmeta;
783 }
784 // An API-specification document (Phase 21.30) — a JSON Schema, an OpenAPI 3.x, a
785 // Swagger 2.0, or an AsyncAPI document — is the **API/spec** Wave-2 format, whose
786 // physical bytes are JSON. It is a **bounded semantic sub-detection** run **before**
787 // the generic JSON detector below: the root must be a JSON object carrying a
788 // **strong, root-level string marker** — a JSON-Schema `$schema` URI, an `openapi`
789 // `3.`-prefixed string, a `swagger` equal to `"2.0"`, or a non-empty `asyncapi`
790 // string. A more specific claim than a bare JSON value, so it is tried first; a
791 // plain JSON object that merely has a `properties`/`paths` key, a JSON-Schema-shaped
792 // object with **no** `$schema`, an unrelated `openapi`/`swagger` value, and prose
793 // decline here and are then claimed by the JSON detector (staying `Json`) or the
794 // opaque floor. Detection is content-only and never consults a file name.
795 #[cfg(feature = "apispec")]
796 if crate::adapter::apispec::detect(source, limits) {
797 return DocumentFormat::Apispec;
798 }
799 // A bibliography (Phase 21.31) — BibTeX, RIS, or CSL-JSON — is the
800 // **bibliography** Wave-2 format. BibTeX/RIS are line-anchored text and CSL-JSON's
801 // physical bytes are JSON, so this is a **bounded content-only test** run **before**
802 // the generic JSON detector below (a CSL-JSON array is otherwise claimed by the JSON
803 // detector, which is the weaker claim) and **before** the weak text heuristics
804 // (YAML/TOML/config/CSV/Markdown/RST/AsciiDoc/logstream/fixedwidth), which a BibTeX
805 // `@type{…}` file or an RIS `TY - ` stream must never be guessed to be. It claims
806 // only on a strong signal: a line-anchored `@type{` entry sequence, an RIS `TY - `
807 // entry sequence whose every entry is `ER - `-terminated, or a JSON array (or
808 // single object) whose every element carries a string `id` **and** a string `type`.
809 // A plain JSON array, a JSON array of non-CSL objects, a generic JSON object, and
810 // prose decline here and are then claimed by the JSON detector (staying `Json`) or
811 // the opaque floor. Detection is content-only and never consults a file name.
812 #[cfg(feature = "bib")]
813 if crate::adapter::bib::detect(source, limits) {
814 return DocumentFormat::Bib;
815 }
816 // RTF (Rich Text Format, Phase 21.32) is a text group-brace grammar with a strong
817 // header magic (`{\rtf<digit>`), so it is tried here with the **strong-magic**
818 // detectors (right after the binary magics PDF/ZIP/Parquet/Arrow and before the
819 // JSON family). Its detection is magic + a bounded brace-balance scan, so no other
820 // format's bytes can steal it and it can steal no other format: an RTF source
821 // begins with `{`, which is not a valid JSON document (JSON would need a valid
822 // value), not a config `KEY=VALUE` line, and not a Markdown/RST structural mark;
823 // and the binary detectors above reject it. A plain text or leading-`{` blob stays
824 // `Opaque`.
825 #[cfg(feature = "rtf")]
826 if crate::adapter::rtf::detect(source, limits) {
827 return DocumentFormat::Rtf;
828 }
829 #[cfg(feature = "json")]
830 if crate::adapter::json::detect(source, limits) {
831 return DocumentFormat::Json;
832 }
833 // JSON5/JSONC is the **structured-extra** Wave-2 format (Phase 21.17.1), also
834 // with no package layer. It is tried **immediately after JSON** and before
835 // JSONL/YAML/TOML/CSV/Markdown/XML/HTML: JSON5 is a superset of JSON, so a
836 // strict JSON source is already claimed by the JSON detector above, and this
837 // detector requires at least one JSON5/JSONC-only construct (a comment, a
838 // trailing comma, an unquoted key, a single quote, a hex/leading-dot/
839 // `Infinity`/`NaN` number, a string continuation, or the extended whitespace
840 // set). A strict JSON document is therefore never reclassified, and a plain
841 // non-JSON blob stays Opaque.
842 #[cfg(feature = "json5")]
843 if crate::adapter::json5::detect(source, limits) {
844 return DocumentFormat::Json5;
845 }
846 // JSONL/NDJSON is the **line/event-stream** Wave-2 format (Phase 21.12), also
847 // with no package layer. It is tried **immediately after JSON** because it is the
848 // most specific signal for a newline-separated JSON stream: each non-blank line
849 // must parse as exactly one JSON value, which is stricter than — and must run
850 // before — the YAML stream reader (which would otherwise be free to reinterpret a
851 // line-delimited JSON stream as a stream of documents). Detection is
852 // conservative: at least two non-blank lines are required and every non-blank
853 // line must parse as exactly one JSON value under the caps, so a single JSON
854 // value, a malformed line, and a bag of values with a non-newline separator all
855 // stay `Opaque`.
856 #[cfg(feature = "jsonl")]
857 if crate::adapter::jsonl::detect(source, limits) {
858 return DocumentFormat::Jsonl;
859 }
860 // CBOR is the **binary** structured-tree Wave-2 format (Phase 21.18), also with
861 // no package layer. It is tried **immediately after the JSON family** (JSON,
862 // JSON5, JSONL) and **before** the remaining textual heuristics
863 // (EML/YAML/TOML/CSV/Markdown/XML/HTML): the strong magic-byte binaries
864 // (PDF/ZIP/Parquet/Arrow) run above and are never reconsidered. CBOR has **no
865 // magic bytes**, so its detector is deliberately conservative — the
866 // self-described-CBOR tag `55799` (`0xd9 0xd9 0xf7`), or a full-input,
867 // well-formed parse whose root is a container/tag and which reaches at least
868 // three nodes. A container head byte is always `>= 0x80`, so no pure-ASCII
869 // document (JSON/YAML/TOML/XML/HTML/prose) can be claimed; a lone scalar, an
870 // empty container, a truncated item, and a structurally trivial input all stay
871 // `Opaque` rather than being guessed. The small-integer/short-array encodings
872 // overlap MessagePack's fixint/fixarray (recorded honestly; a Phase-21.19
873 // MessagePack adapter must share this seam).
874 #[cfg(feature = "cbor")]
875 if crate::adapter::cbor::detect(source, limits) {
876 return DocumentFormat::Cbor;
877 }
878 // MessagePack is the **binary** structured-tree Wave-2 sibling of CBOR (Phase
879 // 21.19), also with no package layer. It is tried **immediately after CBOR** and
880 // **before** the remaining textual heuristics (EML/YAML/TOML/CSV/Markdown/XML/
881 // HTML): the strong magic-byte binaries (PDF/ZIP/Parquet/Arrow) and the JSON
882 // family run above and are never reconsidered. **Placement is load-bearing: the
883 // two binary detectors share the whole-number/short-container prefix, so putting
884 // CBOR first guarantees its stronger self-described-tag signal (and every input
885 // well-formed under both grammars) is never stolen from it.** MessagePack has
886 // **no magic bytes**, so its detector is deliberately conservative — a
887 // full-input, well-formed parse whose root is a container (array/map) reaching at
888 // least three items and eight bytes, or the same with an unambiguous
889 // MessagePack-only head byte (`0xdc..=0xdf`, which CBOR's grammar rejects). A
890 // container head byte is always `>= 0x80`, so no pure-ASCII document can be
891 // claimed; a lone scalar, an empty container, the ambiguous
892 // `fixarray(3)`/`fixmap(2)` overlap fixtures, and a structurally trivial input
893 // all stay `Opaque` rather than being guessed.
894 #[cfg(feature = "msgpack")]
895 if crate::adapter::msgpack::detect(source, limits) {
896 return DocumentFormat::Msgpack;
897 }
898 // MHTML (MIME HTML, RFC 2557) is the **web-archive** Wave-2 format (Phase
899 // 21.27), whose physical bytes are a MIME `multipart/related` message. It is a
900 // **bounded semantic sub-detection** run **before** the generic EML detector
901 // below (a `multipart/related` message is otherwise claimed by EML, which is a
902 // weaker claim): the root must be `multipart/related` **and** carry an
903 // MHTML-specific signal — a top-level `Snapshot-Content-Location`/`Content-Base`,
904 // or a `From`+`Subject` envelope with a `text/html` part. A plain
905 // `multipart/related` email without those markers, a plain HTML document (no MIME
906 // envelope), and a plain MIME message (not `multipart/related`) all decline here
907 // and are then claimed by their own detectors (`Eml`/`Html`/`Eml`).
908 #[cfg(feature = "mhtml")]
909 if crate::adapter::mhtml::detect(source, limits) {
910 return DocumentFormat::Mhtml;
911 }
912 // EML/MIME is the **messaging** Wave-2 format (Phase 21.13), also with no package
913 // layer. It is tried **immediately after JSONL** and **before YAML/TOML/CSV/
914 // Markdown/XML/HTML**: a raw message begins with an RFC 5322 header block, which
915 // the YAML stream reader would otherwise reinterpret as a mapping (e.g. `From: a`
916 // / `Date: b`). Detection is conservative: a genuine header block terminated by a
917 // blank line carrying `From`/`Date`/`Message-ID`, or an explicit `MIME-Version`,
918 // is required, so prose without a header block and a colon-bearing note stay
919 // `Opaque`. Only the header block is scanned (never the MIME tree).
920 #[cfg(feature = "eml")]
921 if crate::adapter::eml::detect(source, limits) {
922 return DocumentFormat::Eml;
923 }
924 // YAML is the second Wave-2 structured-tree format (Phase 21.6.1), also with
925 // **no** package layer. It is detected directly and conservatively: the whole
926 // source must parse as a bounded YAML stream under the YAML caps, and every
927 // document root must be a mapping or a sequence, else the input stays Opaque.
928 #[cfg(feature = "yaml")]
929 if crate::adapter::yaml::detect(source, limits) {
930 return DocumentFormat::Yaml;
931 }
932 // TOML is the next Wave-2 structured-tree format (Phase 21.11). Precedence
933 // (explicit): TOML is tried **after** the strong, fully-parsed tree formats
934 // (JSON, YAML) but **before** the weak, no-magic-byte heuristics (CSV,
935 // Markdown, XML, HTML). TOML's positive signal is a complete, error-free parse,
936 // which is far stronger than the CSV/Markdown heuristics; crucially, a TOML
937 // comment (`# …` at column 0) is otherwise misread as a Markdown ATX heading and
938 // would steal the document. A source that is valid JSON/YAML still wins (it is
939 // tried first); a source that is valid XML/HTML never parses as TOML (it begins
940 // with `<`), so nothing is lost there. TOML has no magic bytes and a plain prose
941 // paragraph is not TOML, so detection is conservative: the whole source must
942 // parse as TOML 1.0 under the TOML caps with no errors **and** carry at least one
943 // key/value assignment. TOML's duplicate-key/redefinition rules are enforced, so
944 // a doc that violates them is not detected. Plain prose and non-TOML text stay
945 // Opaque.
946 #[cfg(feature = "toml")]
947 if crate::adapter::toml::detect(source, limits) {
948 return DocumentFormat::Toml;
949 }
950 // The config family (INI / `.env` / Java `.properties`) is the key/value line
951 // Wave-2 format (Phase 21.20). Precedence (explicit): it is tried **after** the
952 // strong, fully-parsed tree formats (JSON/JSON5/JSONL/YAML/TOML) — a TOML
953 // document is very often a syntactically valid INI file, so a source that is
954 // valid TOML must stay `Toml` (never reclassified) — and **before** CSV/Markdown/
955 // XML/HTML. It precedes CSV deliberately: a config file whose values contain
956 // commas (e.g. `A=x,y`) has a consistent field count and would otherwise be
957 // stolen by the CSV detector, whereas the config `KEY=VALUE` shape is the more
958 // specific signal. Config has **no magic bytes**, so detection is deliberately
959 // conservative and requires a **dialect-distinguishing signal**: an INI
960 // `[section]` header; a properties `:`/whitespace separator, `!` comment,
961 // `\uXXXX` escape, continuation, or non-identifier key (plus a strong `=`/`:`
962 // separator somewhere); or an `env` `export ` prefix. The pure `KEY=VALUE`
963 // overlap between `env` and `properties`, plain prose, a two-column `a b`
964 // blob, and a `#!` script all stay `Opaque` rather than being guessed; a plain
965 // `.txt`/Markdown/code source is never claimed.
966 #[cfg(feature = "config")]
967 if crate::adapter::config::detect(source, limits) {
968 return DocumentFormat::Config;
969 }
970 // CSV/TSV is the first **tabular** Wave-2 format (Phase 21.7.1). It has no
971 // magic bytes, so it is detected last and conservatively: only after the
972 // PDF/ZIP/JSON/YAML families are declined does a source qualify, and then only
973 // if it parses under a specific delimiter (`,`, tab, or `|`) as a table with a
974 // consistent field count across a sampled majority of records and at least two
975 // columns. Phase 21.25 adds the pipe delimiter to this dialect set; the pipe
976 // dialect is tried after comma/tab and is declined when the source carries a
977 // GFM/Markdown delimiter row (so a Markdown table is never stolen). When in
978 // doubt the input stays Opaque.
979 #[cfg(feature = "csv")]
980 if crate::adapter::csv::detect(source, limits) {
981 return DocumentFormat::Csv;
982 }
983 // MDX (Phase 21.26.3) is Markdown + JSX/ESM, so it is tried **immediately before
984 // Markdown** (and therefore before HTML/XML and the other prose/tabular
985 // heuristics): an MDX document's bytes are Markdown-plus-JSX, so generic Markdown
986 // must never claim it first. Detection is conservative and **MDX-specific** — the
987 // source must parse under the Markdown model **and** carry a top-level ESM
988 // `import`/`export` statement, a JSX component (a capitalized/namespaced element
989 // name) or fragment, a JSX-specific attribute (a brace value or spread), or a
990 // whole-line MDX block expression `{…}` — and it declines anything already
991 // claimed by JSON/YAML/CSV/HTML/XML/RST/AsciiDoc, so a plain Markdown document
992 // stays `Markdown`, a plain HTML document stays `Html`, and plain prose stays
993 // `Opaque`.
994 #[cfg(feature = "mdx")]
995 if crate::adapter::mdx::detect(source, limits) {
996 return DocumentFormat::Mdx;
997 }
998 // Markdown is the first **prose** Wave-2 format (Phase 21.8.1). It has no
999 // magic bytes either, and a plain prose paragraph is itself valid Markdown, so
1000 // detection is deliberately conservative: only after the PDF/ZIP/JSON/YAML/CSV
1001 // families are declined, and only when the source carries a clear structural
1002 // mark (an ATX heading, a fenced code block, front matter, a table, a reference
1003 // definition, or a footnote definition), is it admitted. Plain prose stays
1004 // Opaque rather than being guessed to be Markdown.
1005 #[cfg(feature = "markdown")]
1006 if crate::adapter::markdown::detect(source, limits) {
1007 return DocumentFormat::Markdown;
1008 }
1009 // reStructuredText (Phase 21.26.1) is the **next prose** Wave-2 format, also with
1010 // no package layer. It is tried **immediately after Markdown** and **before
1011 // fixed-width**: Markdown must be tried first so a Markdown document is never
1012 // stolen (reST detection requires a reST-**specific** signal — an explicit markup
1013 // start, a grid/simple table, a field list, or an adornment whose char is not a
1014 // Markdown construct), and a reST document must be claimed before the maximally
1015 // ambiguous fixed-width heuristic (which could otherwise read an aligned reST body
1016 // as a column-position table). Plain prose stays Opaque.
1017 #[cfg(feature = "rst")]
1018 if crate::adapter::rst::detect(source, limits) {
1019 return DocumentFormat::Rst;
1020 }
1021 // AsciiDoc (Phase 21.26.2) is the **third prose** Wave-2 format, also with no
1022 // package layer. It is tried **immediately after reStructuredText** and **before
1023 // fixed-width**: Markdown and reST must be tried first so neither is stolen (a
1024 // valid Markdown or reST document always keeps its own format), and AsciiDoc must
1025 // be claimed before the maximally ambiguous fixed-width heuristic (which could
1026 // otherwise read an aligned AsciiDoc body as a column-position table). The
1027 // dispatcher already tries PDF/ZIP/JSON/YAML/config/CSV ahead of it, so a
1028 // `:name: value`-shaped config source and a `#`/fence-bearing Markdown source are
1029 // claimed earlier. Detection is conservative and AsciiDoc-specific (a document
1030 // title followed by a block, a `==`+ section, a `|===` table, a complete
1031 // delimited block, or a block attribute line followed by a block); a lone `= text`
1032 // line and plain prose stay Opaque.
1033 #[cfg(feature = "asciidoc")]
1034 if crate::adapter::asciidoc::detect(source, limits) {
1035 return DocumentFormat::Asciidoc;
1036 }
1037 // A syslog / log stream is the **line/event-stream** Wave-2 format (Phase 21.28),
1038 // also with no package layer. Precedence (explicit): it is tried **after** every
1039 // structured and prose format above — the JSON family (JSON/JSON5/JSONL/CBOR/
1040 // Msgpack), MHTML/EML, YAML/TOML, the config family, CSV, and the prose family
1041 // (MDX/Markdown/RST/AsciiDoc) — so a JSON/YAML/CSV/prose document is never stolen;
1042 // and **before** the maximally ambiguous fixed-width heuristic, which could
1043 // otherwise read a column-aligned log stream as a column-position table. It is
1044 // tried before the XML/HTML family too, but a log line never begins with a valid
1045 // XML/HTML root (a syslog `<PRI>` is not well-formed XML). Detection is byte-based
1046 // and conservative: at least two non-blank lines, and **every** non-blank line must
1047 // match a dialect — an RFC 5424 syslog header, an RFC 3164 (BSD) syslog line, or a
1048 // generic application log line with a leading timestamp and/or level token. Plain
1049 // prose and code stay `Opaque` (a prose line that does not begin with a timestamp/
1050 // level declines the whole file; a file whose every line begins with a level word
1051 // IS claimed — recorded honestly in the adapter docs).
1052 #[cfg(feature = "logstream")]
1053 if crate::adapter::logstream::detect(source, limits) {
1054 return DocumentFormat::Logstream;
1055 }
1056 // Fixed-width (column-position) text is the second **tabular** Wave-2 format
1057 // (Phase 21.25), but its columns are defined by character positions, not a
1058 // delimiter, so it is a distinct format (and adapter), never a CSV dialect. It is
1059 // tried **last among the tabular/prose heuristics** — after CSV/TSV/PSV, after
1060 // Markdown, and after reStructuredText — so a delimited table, a Markdown table, or
1061 // a reST document always keeps its own format (the fixed-width detector also
1062 // declines a delimited or Markdown table explicitly, defence in depth).
1063 // Fixed-width is **genuinely ambiguous** (nearly any aligned text can look
1064 // tabular), so detection is maximally conservative: at least three sampled records
1065 // of identical byte width, an inferred column layout whose interior whitespace gaps
1066 // are at least two columns wide, and at least two non-empty columns; a
1067 // variable-length or single-space-separated blob stays Opaque.
1068 #[cfg(feature = "fixedwidth")]
1069 if crate::adapter::fixedwidth::detect(source, limits) {
1070 return DocumentFormat::FixedWidth;
1071 }
1072 // RSS/Atom is the **syndication** Wave-2 format (Phase 21.21). A feed's physical
1073 // bytes are XML, so this is a **bounded semantic sub-detection** run **before**
1074 // the generic standalone-XML detector below: an RSS root `<rss>` with a
1075 // `<channel>` child, or an Atom root `<feed>` in the Atom namespace
1076 // (`http://www.w3.org/2005/Atom`) with at least one `<entry>` child. A feed is a
1077 // more specific claim than a bare XML tree, so it is tried first; a plain XML
1078 // document (a root that is neither `<rss>` nor an Atom `<feed>`, or an
1079 // `<rss>`-shaped-but-invalid / non-Atom / record-less `<feed>`) declines here and
1080 // falls through to the XML detector (staying `Xml`) or to `Opaque`. An HTML
1081 // document (`<!doctype html>` / an `<html>` root) simply declines here and is
1082 // then claimed by the HTML document-level marker immediately below.
1083 #[cfg(feature = "feed")]
1084 if crate::adapter::feed::detect(source, limits) {
1085 return DocumentFormat::Feed;
1086 }
1087 // KML/GPX is the **geospatial** Wave-2 format (Phase 21.23). Its physical bytes
1088 // are XML, so this is a **bounded semantic sub-detection** run **before** the
1089 // generic standalone-XML detector below: a `<kml>` root in the KML namespace
1090 // (`http://www.opengis.net/kml/2.2`) with a `Document`/`Folder`/`Placemark` child,
1091 // or a `<gpx>` root in the GPX namespace (`http://www.topografix.com/GPX/1/1`)
1092 // with a `metadata`/`wpt`/`rte`/`trk` child. A geospatial document is a more
1093 // specific claim than a bare XML tree, so it is tried first; a plain XML document
1094 // (a root that is neither `<kml>` nor `<gpx>`, or a shaped-but-invalid / non-GIS
1095 // namespace / child-less root) declines here and falls through to the XML detector
1096 // (staying `Xml`) or to `Opaque`. An older KML (2.0/2.1) or GPX 1.0 namespace is
1097 // not the claimed URI, so it also stays `Xml`. An HTML document simply declines
1098 // here and is then claimed by the HTML document-level marker immediately below.
1099 #[cfg(feature = "gis")]
1100 if crate::adapter::gis::detect(source, limits) {
1101 return DocumentFormat::Gis;
1102 }
1103 // HTML document-level marker (Phase 21.10 / 21.15 fix): a `<!doctype html>` or
1104 // an `<html>` root element is a more specific signal than the generic XML
1105 // fallback, so a **well-formed XHTML page** is classified `Html`, not `Xml`
1106 // (XML's well-formedness parser would otherwise claim it). The general XML
1107 // branch below is still tried before the general HTML branch, so a bare XML
1108 // tree stays XML and HTML only claims the sources XML declines.
1109 #[cfg(feature = "html")]
1110 if crate::adapter::html::has_document_marker(source)
1111 && crate::adapter::html::detect(source, limits)
1112 {
1113 return DocumentFormat::Html;
1114 }
1115 // XML is the structured-tree Wave-2 format for a bare XML source (Phase 21.9).
1116 // It has no package layer and no magic bytes beyond `<`, so it is detected last
1117 // and conservatively: only after the PDF/ZIP/JSON/YAML/CSV/Markdown families are
1118 // declined, only when the source begins with `<` (an XML prolog or a root
1119 // element start), and only when the whole source parses as well-formed XML with
1120 // exactly one root element under the XML caps. `<-prefixed` junk and prose stay
1121 // Opaque rather than being guessed to be XML.
1122 #[cfg(feature = "xml")]
1123 if crate::adapter::xml::detect(source, limits) {
1124 return DocumentFormat::Xml;
1125 }
1126 // HTML is the error-recovering markup Wave-2 format (Phase 21.10). Precedence
1127 // note (explicit): a *document-level* HTML marker is handled above (before XML);
1128 // for every other `<`-bearing source XML is tried before HTML, so any document
1129 // XML accepts (a bare XML tree) is genuinely XML. HTML then claims the
1130 // `<`-bearing sources XML declines — real HTML5 is almost never well-formed XML
1131 // (void elements, optional end tags, unquoted attributes, undeclared named
1132 // entities). Detection is conservative: the source must carry clear HTML
1133 // structure (a `<!doctype html>`, an `<html`/`<head`/`<body` tag, or a
1134 // preponderance of known HTML tags). Plain prose and non-HTML `<`-junk stay
1135 // Opaque rather than being guessed to be HTML.
1136 #[cfg(feature = "html")]
1137 if crate::adapter::html::detect(source, limits) {
1138 return DocumentFormat::Html;
1139 }
1140 DocumentFormat::Opaque
1141}
1142
1143/// Whether `source` is a structurally valid ZIP archive.
1144///
1145/// Used by the universal ingest dispatcher so a generic ZIP (which detection
1146/// reports as `Opaque`) is still inverted through the byte-authoritative package
1147/// layer rather than the opaque floor.
1148#[cfg(feature = "package")]
1149pub fn is_zip(source: &[u8], limits: Limits) -> bool {
1150 crate::adapter::package::scan(source, limits).is_ok()
1151}
1152
1153#[cfg(feature = "package")]
1154fn detect_zip_family(source: &[u8], limits: Limits) -> Option<DocumentFormat> {
1155 let physical = crate::adapter::package::scan(source, limits).ok()?;
1156
1157 // EPUB (OCF): the mandatory `mimetype` member, or an OCF container that
1158 // resolves to an OPF rootfile.
1159 let mimetype = member_decoded(&physical, source, EPUB_MIMETYPE_MEMBER, limits);
1160 let mimetype_ok = mimetype.as_deref() == Some(EPUB_MIMETYPE);
1161 let container = member_decoded(&physical, source, CONTAINER_MEMBER, limits);
1162 let container_ok = container
1163 .as_deref()
1164 .is_some_and(|c| contains(c, CONTAINER_NS) && contains(c, OPF_MEDIA_TYPE));
1165 let is_epub = mimetype_ok || container_ok;
1166
1167 // DOCX: an OPC package whose content types declare a WordprocessingML main
1168 // part, or whose package relationships declare an `officeDocument` part that
1169 // targets `word/document.xml`. The positive WordprocessingML signal (rather
1170 // than the mere absence of a SpreadsheetML one) keeps DOCX and XLSX mutually
1171 // exclusive without misclassifying a Word document that *embeds* an Excel
1172 // workbook (whose package declares SpreadsheetML content types for the
1173 // embedded part, but no SpreadsheetML workbook main part). Without `xlsx`
1174 // and `pptx` the legacy relationship-only rule stands.
1175 let content_types = member_decoded(&physical, source, CONTENT_TYPES_MEMBER, limits);
1176 let rels = member_decoded(&physical, source, PACKAGE_RELS_MEMBER, limits);
1177 #[cfg(any(feature = "xlsx", feature = "pptx"))]
1178 let is_docx = content_types
1179 .as_deref()
1180 .is_some_and(|ct| contains(ct, DOCX_MAIN_FRAGMENT))
1181 || rels.as_deref().is_some_and(|r| {
1182 contains(r, OFFICE_DOCUMENT_FRAGMENT) && contains(r, DOCX_MAIN_TARGET)
1183 });
1184 #[cfg(not(any(feature = "xlsx", feature = "pptx")))]
1185 let is_docx = content_types.is_some()
1186 && rels
1187 .as_deref()
1188 .is_some_and(|r| contains(r, OFFICE_DOCUMENT_FRAGMENT));
1189
1190 // XLSX: an OPC package whose content types declare a SpreadsheetML workbook
1191 // (or whose `officeDocument` relationship targets a workbook part).
1192 #[cfg(feature = "xlsx")]
1193 let is_xlsx = content_types.as_deref().is_some_and(|ct| {
1194 contains(ct, XLSX_MAIN_FRAGMENT)
1195 || (contains(ct, XLSX_NS_FRAGMENT)
1196 && rels
1197 .as_deref()
1198 .is_some_and(|r| contains(r, XLSX_WORKBOOK_TARGET)))
1199 });
1200 #[cfg(not(feature = "xlsx"))]
1201 let is_xlsx = false;
1202
1203 // PPTX: an OPC package whose content types declare a PresentationML main part
1204 // (or whose content types name PresentationML and whose `officeDocument`
1205 // relationship targets `ppt/presentation.xml`). The positive PresentationML
1206 // signal keeps it mutually exclusive with DOCX and XLSX: a Word/Excel document
1207 // that *embeds* a PowerPoint part declares only the PresentationML embed type
1208 // (`…presentationml.presentation`, not the `.main+xml` main part) and its
1209 // `officeDocument` relationship targets `word/document.xml`/`xl/workbook.xml`,
1210 // so it is never misclassified as PPTX.
1211 #[cfg(feature = "pptx")]
1212 let is_pptx = content_types.as_deref().is_some_and(|ct| {
1213 contains(ct, PPTX_MAIN_FRAGMENT)
1214 || (contains(ct, PPTX_NS_FRAGMENT)
1215 && rels
1216 .as_deref()
1217 .is_some_and(|r| contains(r, PPTX_MAIN_TARGET)))
1218 });
1219 #[cfg(not(feature = "pptx"))]
1220 let is_pptx = false;
1221
1222 // The mandatory ODF `mimetype` member is **authoritative** when it names an
1223 // OpenDocument media type (ODF 1.2 §3.2: the package's own media type). An ODP
1224 // that *embeds a spreadsheet* lists both media types in `META-INF/manifest.xml`
1225 // (one per file-entry), so scanning the whole manifest would make it look like an
1226 // ODP *and* an ODS and force the ambiguous case below to Opaque. Only when the
1227 // `mimetype` member is absent or is not an ODF media type do we fall back to the
1228 // manifest scan (for a non-conformant package).
1229 #[cfg(any(feature = "odt", feature = "ods", feature = "odp"))]
1230 let odf_mimetype = mimetype
1231 .as_deref()
1232 .filter(|m| contains(m, ODF_MEDIA_PREFIX));
1233
1234 // ODT: an ODF package whose mandatory `mimetype` (or, absent that,
1235 // `META-INF/manifest.xml`) declares an OpenDocument text media type.
1236 #[cfg(feature = "odt")]
1237 let is_odt = match odf_mimetype {
1238 Some(m) => contains(m, ODT_TEXT_FRAGMENT),
1239 None => member_decoded(&physical, source, ODT_MANIFEST_MEMBER, limits)
1240 .as_deref()
1241 .is_some_and(|m| contains(m, ODT_TEXT_FRAGMENT)),
1242 };
1243 #[cfg(not(feature = "odt"))]
1244 let is_odt = false;
1245
1246 // ODS: an ODF package whose mandatory `mimetype` (or, absent that, the manifest)
1247 // declares an OpenDocument *spreadsheet* media type. The positive spreadsheet
1248 // fragment keeps it mutually exclusive with ODT (a text document declares the
1249 // text media type, never the spreadsheet one).
1250 #[cfg(feature = "ods")]
1251 let is_ods = match odf_mimetype {
1252 Some(m) => contains(m, ODS_SPREADSHEET_FRAGMENT),
1253 None => member_decoded(&physical, source, ODS_MANIFEST_MEMBER, limits)
1254 .as_deref()
1255 .is_some_and(|m| contains(m, ODS_SPREADSHEET_FRAGMENT)),
1256 };
1257 #[cfg(not(feature = "ods"))]
1258 let is_ods = false;
1259
1260 // ODP: an ODF package whose mandatory `mimetype` (or, absent that, the manifest)
1261 // declares an OpenDocument *presentation* media type. The positive presentation
1262 // fragment keeps it mutually exclusive with ODT/ODS.
1263 #[cfg(feature = "odp")]
1264 let is_odp = match odf_mimetype {
1265 Some(m) => contains(m, ODP_PRESENTATION_FRAGMENT),
1266 None => member_decoded(&physical, source, ODP_MANIFEST_MEMBER, limits)
1267 .as_deref()
1268 .is_some_and(|m| contains(m, ODP_PRESENTATION_FRAGMENT)),
1269 };
1270 #[cfg(not(feature = "odp"))]
1271 let is_odp = false;
1272
1273 // A ZIP matching more than one native signature is ambiguous: fail safe.
1274 let matches = [is_docx, is_epub, is_odt, is_ods, is_odp, is_xlsx, is_pptx]
1275 .iter()
1276 .filter(|b| **b)
1277 .count();
1278 match matches {
1279 1 if is_docx => Some(DocumentFormat::Docx),
1280 1 if is_epub => Some(DocumentFormat::Epub),
1281 1 if is_odt => Some(DocumentFormat::Odt),
1282 1 if is_ods => Some(DocumentFormat::Ods),
1283 1 if is_odp => Some(DocumentFormat::Odp),
1284 1 if is_xlsx => Some(DocumentFormat::Xlsx),
1285 1 if is_pptx => Some(DocumentFormat::Pptx),
1286 _ => None,
1287 }
1288}
1289
1290#[cfg(feature = "package")]
1291const EPUB_MIMETYPE_MEMBER: &[u8] = b"mimetype";
1292
1293/// Decode one member's bytes by exact name, bounded and decline-safe: encrypted,
1294/// oversized, unsupported-method, or out-of-range members yield `None`. The name
1295/// match is ASCII case-insensitive, mirroring the OPC layer
1296/// ([`crate::field::opc`]), which resolves the well-known control parts
1297/// (`[Content_Types].xml`, `_rels/.rels`) case-insensitively; a package that
1298/// lowercases them is still identified (Phase 21.15 fix).
1299#[cfg(feature = "package")]
1300fn member_decoded(
1301 physical: &crate::adapter::package::ZipPhysical,
1302 source: &[u8],
1303 name: &[u8],
1304 limits: Limits,
1305) -> Option<Vec<u8>> {
1306 const FLAG_ENCRYPTED: u16 = 0x0001;
1307 let member = physical
1308 .members
1309 .iter()
1310 .find(|m| m.name.eq_ignore_ascii_case(name))?;
1311 if member.flags & FLAG_ENCRYPTED != 0 || member.uncompressed_size > limits.max_xml_part_bytes {
1312 return None;
1313 }
1314 let off = usize::try_from(member.data.0).ok()?;
1315 let len = usize::try_from(member.data.1).ok()?;
1316 let raw = source.get(off..off.checked_add(len)?)?;
1317 match member.method {
1318 0 => Some(raw.to_vec()),
1319 8 => crate::field::derive::inflate_raw_deflate(raw, member.uncompressed_size, limits).ok(),
1320 _ => None,
1321 }
1322}
1323
1324/// Byte-substring search (no allocation, case-sensitive).
1325#[cfg(feature = "package")]
1326fn contains(haystack: &[u8], needle: &[u8]) -> bool {
1327 !needle.is_empty()
1328 && needle.len() <= haystack.len()
1329 && haystack.windows(needle.len()).any(|w| w == needle)
1330}
1331
1332#[cfg(test)]
1333mod tests {
1334 use super::*;
1335
1336 #[test]
1337 fn names_and_provenance_roundtrip() {
1338 for f in [
1339 DocumentFormat::Pdf,
1340 DocumentFormat::Docx,
1341 DocumentFormat::Epub,
1342 DocumentFormat::Odt,
1343 DocumentFormat::Ods,
1344 DocumentFormat::Odp,
1345 DocumentFormat::Xlsx,
1346 DocumentFormat::Pptx,
1347 DocumentFormat::Json,
1348 DocumentFormat::Json5,
1349 DocumentFormat::Yaml,
1350 DocumentFormat::Csv,
1351 DocumentFormat::Markdown,
1352 DocumentFormat::Xml,
1353 DocumentFormat::Html,
1354 DocumentFormat::Toml,
1355 DocumentFormat::Jsonl,
1356 DocumentFormat::Eml,
1357 DocumentFormat::Parquet,
1358 DocumentFormat::ArrowIpc,
1359 DocumentFormat::Cbor,
1360 DocumentFormat::Msgpack,
1361 DocumentFormat::Config,
1362 DocumentFormat::Feed,
1363 DocumentFormat::Geojson,
1364 DocumentFormat::Gis,
1365 DocumentFormat::Notebook,
1366 DocumentFormat::FixedWidth,
1367 DocumentFormat::Rst,
1368 DocumentFormat::Asciidoc,
1369 DocumentFormat::Mdx,
1370 DocumentFormat::Mhtml,
1371 DocumentFormat::Logstream,
1372 DocumentFormat::Pkgmeta,
1373 DocumentFormat::Apispec,
1374 DocumentFormat::Bib,
1375 DocumentFormat::Rtf,
1376 DocumentFormat::Opaque,
1377 ] {
1378 let token = format!("{}field:package;members=1", f.provenance_prefix());
1379 assert_eq!(DocumentFormat::from_provenance(&token), Some(f));
1380 }
1381 assert_eq!(DocumentFormat::from_provenance("field:ingest-b"), None);
1382 assert_eq!(DocumentFormat::from_provenance("format=exotic;x"), None);
1383 }
1384
1385 #[test]
1386 fn plain_bytes_are_opaque() {
1387 assert_eq!(
1388 detect_document_format(b"not a document", Limits::DEFAULT),
1389 DocumentFormat::Opaque
1390 );
1391 }
1392
1393 #[test]
1394 fn a_corpus_pdf_is_detected() {
1395 let (_, pdf) = crate::adapter::pdf::sample_pdfs()
1396 .into_iter()
1397 .next()
1398 .expect("the PDF corpus is non-empty");
1399 assert_eq!(
1400 detect_document_format(&pdf, Limits::DEFAULT),
1401 DocumentFormat::Pdf
1402 );
1403 }
1404}