Skip to main content

vole_document/field/
document_format.rs

1//! Byte-based document-format detection for the universal observation API
2//! (Phase 12.7, ADR-0031, plan §DEC-5).
3//!
4//! The format of a field is decided from its **source bytes**, never from a file
5//! name or extension. Detection is deliberately conservative: an ambiguous or
6//! malformed input falls back to [`DocumentFormat::Opaque`] rather than guessing,
7//! and a format whose adapter is not compiled in cannot be detected (the input is
8//! then `Opaque`), so the reported capability set always matches what the build
9//! can actually serve.
10//!
11//! * **PDF** — the byte-authoritative physical scanner admits a validated PDF
12//!   ([`crate::adapter::pdf::detect`]).
13//! * **DOCX** — a valid ZIP that also carries the OPC content-types part
14//!   (`[Content_Types].xml`) and a package `officeDocument` relationship.
15//! * **EPUB** — a valid ZIP that is an OCF container: the mandatory stored
16//!   `mimetype` member equals `application/epub+zip`, or `META-INF/container.xml`
17//!   names the OCF namespace and an OPF (`application/oebps-package+xml`) rootfile.
18//! * **ODT** — a valid ZIP that is an OpenDocument (ODF) package: the mandatory
19//!   stored `mimetype` member is an OpenDocument *text* media type, or
20//!   `META-INF/manifest.xml` declares one.
21//! * **ODS** — a valid ZIP that is an OpenDocument (ODF) package whose mandatory
22//!   stored `mimetype` member is an OpenDocument *spreadsheet* media type, or whose
23//!   `META-INF/manifest.xml` declares one (Phase 21.3.1). Mutually exclusive with
24//!   ODT (a text document declares the text media type, a spreadsheet the
25//!   spreadsheet one).
26//! * **ODP** — a valid ZIP that is an OpenDocument (ODF) package whose mandatory
27//!   stored `mimetype` member is an OpenDocument *presentation* media type, or whose
28//!   `META-INF/manifest.xml` declares one (Phase 21.4.1). Mutually exclusive with
29//!   ODT/ODS (a presentation declares the presentation media type).
30//! * **Opaque** — everything else, including a ZIP that matches none of the above
31//!   (or more than one — an ambiguous ZIP fails safe).
32//! * **JSON** — the whole source parses as exactly one JSON value within the
33//!   JSON caps (Phase 21.5.1). JSON is a Wave-2 structured-tree format with no
34//!   package layer, so it is detected directly (never via `detect_zip_family`);
35//!   a malformed or oversized input stays `Opaque`.
36//!
37//! The detected format is recorded in the field manifest's provenance (a
38//! machine-readable `format=<name>;` prefix, see [`DocumentFormat::from_provenance`]),
39//! so `observe`/`find`/`explain` can dispatch common selectors without reading the
40//! whole source again.
41
42use crate::limits::Limits;
43
44/// The mandatory OCF `mimetype` payload.
45pub const EPUB_MIMETYPE: &[u8] = b"application/epub+zip";
46/// The mandatory OCF container descriptor member.
47pub const CONTAINER_MEMBER: &[u8] = b"META-INF/container.xml";
48/// The OCF container-descriptor namespace.
49pub const CONTAINER_NS: &[u8] = b"urn:oasis:names:tc:opendocument:xmlns:container";
50/// The default (and only normative) package-document media type.
51pub const OPF_MEDIA_TYPE: &[u8] = b"application/oebps-package+xml";
52/// The OPC content-types part.
53#[cfg(feature = "package")]
54const CONTENT_TYPES_MEMBER: &[u8] = b"[Content_Types].xml";
55/// The OPC package-relationships part.
56#[cfg(feature = "package")]
57const PACKAGE_RELS_MEMBER: &[u8] = b"_rels/.rels";
58/// The `officeDocument` relationship type fragment (transitional and strict).
59#[cfg(feature = "package")]
60const OFFICE_DOCUMENT_FRAGMENT: &[u8] = b"officeDocument";
61/// The WordprocessingML document main content-type fragment (Phase 21.1.2).
62#[cfg(feature = "package")]
63const DOCX_MAIN_FRAGMENT: &[u8] = b"wordprocessingml.document.main+xml";
64/// The canonical WordprocessingML main-part target fragment (Phase 21.1.2).
65#[cfg(feature = "package")]
66const DOCX_MAIN_TARGET: &[u8] = b"word/document.xml";
67/// The ODF package manifest member (Phase 13.3).
68#[cfg(feature = "odt")]
69const ODT_MANIFEST_MEMBER: &[u8] = b"META-INF/manifest.xml";
70/// The OpenDocument *text* media-type fragment (Phase 13.3).
71#[cfg(feature = "odt")]
72const ODT_TEXT_FRAGMENT: &[u8] = b"application/vnd.oasis.opendocument.text";
73/// The ODF package manifest member for the ODS detection rule (Phase 21.3.1).
74#[cfg(feature = "ods")]
75const ODS_MANIFEST_MEMBER: &[u8] = b"META-INF/manifest.xml";
76/// The OpenDocument *spreadsheet* media-type fragment (Phase 21.3.1).
77#[cfg(feature = "ods")]
78const ODS_SPREADSHEET_FRAGMENT: &[u8] = b"application/vnd.oasis.opendocument.spreadsheet";
79/// The ODF package manifest member for the ODP detection rule (Phase 21.4.1).
80#[cfg(feature = "odp")]
81const ODP_MANIFEST_MEMBER: &[u8] = b"META-INF/manifest.xml";
82/// The OpenDocument *presentation* media-type fragment (Phase 21.4.1).
83#[cfg(feature = "odp")]
84const ODP_PRESENTATION_FRAGMENT: &[u8] = b"application/vnd.oasis.opendocument.presentation";
85/// The ODF media-type prefix shared by every OpenDocument package.
86#[cfg(any(feature = "odt", feature = "ods", feature = "odp"))]
87const ODF_MEDIA_PREFIX: &[u8] = b"application/vnd.oasis.opendocument.";
88/// The SpreadsheetML workbook main content-type fragment (Phase 21.1.1).
89#[cfg(feature = "xlsx")]
90const XLSX_MAIN_FRAGMENT: &[u8] = b"spreadsheetml.sheet.main+xml";
91/// The SpreadsheetML content-type namespace fragment (Phase 21.1.1).
92#[cfg(feature = "xlsx")]
93const XLSX_NS_FRAGMENT: &[u8] = b"spreadsheetml";
94/// A SpreadsheetML workbook part-target fragment (Phase 21.1.1).
95#[cfg(feature = "xlsx")]
96const XLSX_WORKBOOK_TARGET: &[u8] = b"xl/workbook.xml";
97/// The PresentationML presentation main content-type fragment (Phase 21.2.1).
98#[cfg(feature = "pptx")]
99const PPTX_MAIN_FRAGMENT: &[u8] = b"presentationml.presentation.main+xml";
100/// The PresentationML content-type namespace fragment (Phase 21.2.1).
101#[cfg(feature = "pptx")]
102const PPTX_NS_FRAGMENT: &[u8] = b"presentationml";
103/// The canonical PresentationML main-part target fragment (Phase 21.2.1).
104#[cfg(feature = "pptx")]
105const PPTX_MAIN_TARGET: &[u8] = b"ppt/presentation.xml";
106
107/// A detected document format (the class of the field's source bytes).
108#[derive(Debug, Clone, Copy, PartialEq, Eq)]
109pub enum DocumentFormat {
110    /// A validated PDF (physical indicators).
111    Pdf,
112    /// An OPC package with a WordprocessingML `officeDocument` part.
113    Docx,
114    /// An OCF container with an EPUB package document.
115    Epub,
116    /// An ODF package with an OpenDocument text content part.
117    Odt,
118    /// An ODF package with an OpenDocument spreadsheet content part.
119    Ods,
120    /// An ODF package with an OpenDocument presentation content part.
121    Odp,
122    /// An OPC package with a SpreadsheetML workbook part.
123    Xlsx,
124    /// An OPC package with a PresentationML presentation part.
125    Pptx,
126    /// A structured JSON document (the whole source parses as exactly one JSON
127    /// value). Not a package: the exact leaf is the whole source (Phase 21.5.1).
128    Json,
129    /// A structured JSON5 / JSONC document (the whole source parses as exactly one
130    /// JSON5 value and uses at least one JSON5/JSONC-only construct — a comment, a
131    /// trailing comma, an unquoted key, a single-quoted string, a hex/leading-dot/
132    /// `Infinity`/`NaN` number, a string continuation, or the extended whitespace
133    /// set). A source that is **strict JSON** stays [`DocumentFormat::Json`]; the
134    /// recorded dialect distinguishes `jsonc` (comments/trailing commas only) from
135    /// `json5`. Not a package: the exact leaf is the whole source (Phase 21.17.1).
136    Json5,
137    /// A structured YAML document (the whole source parses as a bounded YAML
138    /// stream whose every document root is a mapping or a sequence). Not a package:
139    /// the exact leaf is the whole source (Phase 21.6.1).
140    Yaml,
141    /// A CSV/TSV table (the whole source parses under a comma or tab delimiter as a
142    /// table with a consistent field count across a sampled majority of at least
143    /// two records). Not a package: the exact leaf is the whole source, and the
144    /// recorded dialect distinguishes comma from tab (Phase 21.7.1).
145    Csv,
146    /// A Markdown prose document (the whole source carries at least one structural
147    /// mark — an ATX heading, a fenced code block, front matter, a table, a
148    /// reference definition, or a footnote definition). Not a package: the exact
149    /// leaf is the whole source (Phase 21.8.1).
150    Markdown,
151    /// A standalone XML document (the whole source begins with `<` and parses as
152    /// well-formed XML with exactly one root element under the XML caps). Not a
153    /// package: the exact leaf is the whole source, and every node/attribute span
154    /// is a `Q_gen` projection (Phase 21.9).
155    Xml,
156    /// A standalone HTML document (the source is not well-formed XML and carries
157    /// clear HTML structure — a `<!doctype html>`, an `<html`/`<head`/`<body` tag,
158    /// or a preponderance of known HTML tags). Parsed by a bounded,
159    /// **error-recovering** scanner. Not a package: the exact leaf is the whole
160    /// source, and every element/attribute/text/comment/raw-text span is a `Q_gen`
161    /// projection (Phase 21.10).
162    Html,
163    /// A TOML document (the source is not any earlier format, parses as TOML 1.0
164    /// under the TOML caps with no errors, and carries at least one key/value
165    /// assignment). Not a package: the exact leaf is the whole source, and every
166    /// table/array/key/value/comment span is a `Q_gen` projection. TOML's
167    /// duplicate-key/redefinition rules are enforced (a violation is a typed
168    /// decline) (Phase 21.11).
169    Toml,
170    /// A JSONL / NDJSON line/event stream (the source carries at least two non-blank
171    /// lines and **every** non-blank line parses as exactly one JSON value under the
172    /// caps, each line parsed by the shared JSON parser). Not a package: the exact
173    /// leaf is the whole source, and every record's exact line span, terminator, and
174    /// per-token spans are `Q_gen` projections. A single JSON value stays
175    /// [`DocumentFormat::Json`] (Phase 21.12).
176    Jsonl,
177    /// An EML / MIME internet message (an RFC 5322 header block terminated by a
178    /// blank line, carrying `From`/`Date`/`Message-ID`, or an explicit
179    /// `MIME-Version`). Not a package: the exact leaf is the whole source, and every
180    /// header/part span and every `Content-Transfer-Encoding`-decoded constituent is
181    /// a `Q_gen` projection (Phase 21.13).
182    Eml,
183    /// An Apache Parquet file (the source begins with `PAR1` and ends with `PAR1`,
184    /// and the 4-byte little-endian footer length before the trailing magic is
185    /// consistent with the file length). Not a package: the exact leaf is the whole
186    /// source, and the parsed footer inventory (schema, row groups, column chunks,
187    /// statistics) and any decoded values are `Q_gen` projections (Phase 21.14).
188    Parquet,
189    /// An Apache Arrow IPC file/stream (the source begins with the `ARROW1` magic
190    /// and is either the **file** format — a trailing `ARROW1` magic preceded by a
191    /// consistent little-endian `int32` footer length — or the **stream** format —
192    /// a valid encapsulated `Schema` message at the 8-byte magic+padding prefix).
193    /// Not a package: the exact leaf is the whole source, and the parsed schema and
194    /// record-batch inventory (with each batch's exact source span) and any decoded
195    /// columnar values are `Q_gen` projections (Phase 21.16).
196    ArrowIpc,
197    /// A CBOR (RFC 8949) structured-tree document. CBOR has **no magic bytes**, so
198    /// detection is deliberately conservative: either the self-described-CBOR tag
199    /// `55799` (`0xd9 0xd9 0xf7`) at the start of a source that then parses, in full,
200    /// as exactly one well-formed CBOR item; or a full-input well-formed parse whose
201    /// root is a container (array/map) or a tag and which reaches at least three
202    /// nodes (so a single scalar, an empty container, and any structurally trivial
203    /// input stay `Opaque`). A container head byte is always `>= 0x80`, so a
204    /// pure-ASCII text document is never claimed; and the whole-number/short-
205    /// container prefix overlaps MessagePack's fixint/fixarray encodings, so an
206    /// ambiguous or trivial input stays `Opaque` rather than being guessed
207    /// (recorded honestly; a Phase-21.19 MessagePack adapter must share the seam).
208    /// Not a package: the exact leaf is the whole source, and every item's kind,
209    /// exact span, encoding width, tag number, float width, and definite/indefinite
210    /// form are `Q_gen` projections (Phase 21.18).
211    Cbor,
212    /// A MessagePack structured-tree document. MessagePack has **no magic bytes**, so
213    /// detection is deliberately conservative: a full-input well-formed parse of
214    /// exactly one item whose root is a container (array/map) reaching a node/byte
215    /// threshold, or the same with an unambiguous MessagePack-only head byte
216    /// (`0xdc..=0xdf`, which CBOR's grammar rejects). A lone scalar, an empty
217    /// container, and any structurally trivial or ambiguous input stay `Opaque`. A
218    /// container head byte is always `>= 0x80`, so a pure-ASCII text document is never
219    /// claimed; and the whole-number/short-container prefix overlaps CBOR's
220    /// `fixint`/short containers, so CBOR (tried first) owns any input well-formed
221    /// under both. Not a package: the exact leaf is the whole source, and every item's
222    /// kind, exact span, exact format byte (encoding width and signedness), `str` vs
223    /// `bin`, float width, and extension type/length are `Q_gen` projections
224    /// (Phase 21.19).
225    Msgpack,
226    /// A config-family document (INI, `.env`, or Java `.properties`). The whole
227    /// source is a bounded, line-based key/value document; detection is
228    /// deliberately conservative and requires a **dialect-distinguishing signal**,
229    /// so the pure `KEY=VALUE` overlap between `env` and `properties` (and plain
230    /// prose, and a `#!` script) stays `Opaque` rather than being guessed. Not a
231    /// package: the exact leaf is the whole source, the recorded dialect lives in
232    /// the model, and every line's content span, key/separator/value span, quoting,
233    /// inline-comment, `export`, continuation, and `\uXXXX`-spelling fact is a
234    /// `Q_gen` projection (Phase 21.20).
235    Config,
236    /// An RSS 2.0 / Atom 1.0 syndication feed. A feed is XML, so its physical
237    /// bytes are shared with [`DocumentFormat::Xml`]; the recorded dialect
238    /// (`rss`/`atom`) lives in the model, exactly as CSV/config record their
239    /// dialects. Detection is a **bounded semantic test** run before the generic
240    /// XML detector: an RSS root `<rss>` with a `<channel>` child, or an Atom root
241    /// `<feed>` in the Atom namespace (`http://www.w3.org/2005/Atom`) with at least
242    /// one `<entry>` child. Not a package: the exact leaf is the whole source, and
243    /// every element/attribute span, element order, attribute spelling, and the
244    /// namespace declaration is a `Q_gen` projection (Phase 21.21).
245    Feed,
246    /// A GeoJSON (RFC 7946) document. GeoJSON is JSON, so its physical bytes are
247    /// shared with [`DocumentFormat::Json`]; the recorded root class (one of the nine
248    /// RFC 7946 type names) lives in the model, exactly as a feed records its
249    /// dialect. Detection is a **bounded semantic test** run before the generic JSON
250    /// detector: the source parses as exactly one JSON value whose root is an object
251    /// with a `"type"` string equal to one of the nine type names and whose shape is
252    /// consistent (a geometry has an array `coordinates`/`geometries`, a `Feature` has
253    /// `geometry`/`properties`, a `FeatureCollection` has an array `features`). A plain
254    /// JSON document, and a JSON document whose `"type"` is an unrelated string, stay
255    /// [`DocumentFormat::Json`]. Not a package: the exact leaf is the whole source, and
256    /// every JSON token span, `coordinates` nesting, `properties` order/duplicates, and
257    /// foreign member is a `Q_gen` projection (Phase 21.22).
258    Geojson,
259    /// A KML 2.2 / GPX 1.1 geospatial document. KML and GPX are XML, so their
260    /// physical bytes are shared with [`DocumentFormat::Xml`]; the recorded dialect
261    /// (`kml`/`gpx`) lives in the model, exactly as a feed records its dialect.
262    /// Detection is a **bounded semantic test** run before the generic XML detector:
263    /// a `<kml>` root in the KML namespace (`http://www.opengis.net/kml/2.2`) with a
264    /// `Document`/`Folder`/`Placemark` child, or a `<gpx>` root in the GPX namespace
265    /// (`http://www.topografix.com/GPX/1/1`) with a `metadata`/`wpt`/`rte`/`trk`
266    /// child. A plain XML document, and a shaped-but-invalid KML/GPX, stay
267    /// [`DocumentFormat::Xml`]. Not a package: the exact leaf is the whole source,
268    /// and every element/attribute span, element order, attribute spelling (KML
269    /// geometry `coordinates`, GPX `lat`/`lon`), and the namespace declaration is a
270    /// `Q_gen` projection (Phase 21.23).
271    Gis,
272    /// A Jupyter notebook (`.ipynb`, nbformat) document. A notebook's physical bytes
273    /// are JSON, so its bytes are shared with [`DocumentFormat::Json`]; the recorded
274    /// `nbformat`/`nbformat_minor` and the cell anchors live in the model, exactly as a
275    /// feed records its dialect. Detection is a **bounded semantic test** run before the
276    /// generic JSON detector: the source parses as exactly one JSON value whose root is
277    /// an object with a plain non-negative integer-literal `nbformat` (≥ 1) and an array
278    /// `cells`, every cell an object with a string `cell_type`, and every present
279    /// recognized field nbformat-shaped. A plain JSON document, and a JSON document that
280    /// merely has a `cells` key but is not nbformat-shaped, stay
281    /// [`DocumentFormat::Json`]. Not a package: the exact leaf is the whole source, and
282    /// every JSON token span, the exact `cell_type`/`output_type` strings, the exact
283    /// `source` representation, and the cell/output order are `Q_gen` projections
284    /// (Phase 21.24).
285    Notebook,
286    /// A fixed-width (column-position) text table. Unlike CSV/TSV/PSV, its columns
287    /// are defined by **character positions**, not a delimiter, so it is a distinct
288    /// format rather than a CSV dialect. Detection is **maximally conservative**
289    /// (fixed-width is inherently ambiguous): at least three sampled records of
290    /// identical byte width, an inferred column layout whose interior whitespace gaps
291    /// are at least two columns wide, at least two non-empty columns, and an outright
292    /// decline of anything that also parses as a delimited (CSV/TSV/PSV) or Markdown
293    /// table. Not a package: the exact leaf is the whole source, and the recorded
294    /// inferred layout, per-record spans, per-column spans, padding, terminator, BOM,
295    /// and header row are `Q_gen` projections (Phase 21.25).
296    FixedWidth,
297    /// A reStructuredText (Docutils) prose document (the whole source carries a
298    /// reST-specific structural mark — an explicit markup start (`.. ` comment,
299    /// directive, target, footnote/citation, or substitution definition), a grid or
300    /// simple table, a field list, or a section adornment whose char is not a
301    /// Markdown construct — so plain prose and a Markdown document are never stolen).
302    /// Not a package: the exact leaf is the whole source (Phase 21.26.1).
303    Rst,
304    /// An AsciiDoc prose document (the whole source carries an AsciiDoc-specific
305    /// structural mark — a level-0 document title followed by a further block, a
306    /// `==`+ section, a `|===` table, a complete delimited block, or a block attribute
307    /// line followed by a block — so plain prose and a Markdown or reStructuredText
308    /// document are never stolen). Not a package: the exact leaf is the whole source
309    /// (Phase 21.26.2).
310    Asciidoc,
311    /// An MDX document (Markdown + JSX/ESM). The whole source carries an MDX-specific
312    /// signal — a top-level ESM `import`/`export` statement, a JSX element with a
313    /// capitalized/namespaced component name (or a JSX-specific attribute), a JSX
314    /// fragment, or a whole-line MDX block expression `{…}` — so plain Markdown stays
315    /// `Markdown`, plain HTML stays `Html`, and plain prose stays `Opaque`. Not a
316    /// package: the exact leaf is the whole source (Phase 21.26.3).
317    Mdx,
318    /// An MHTML (MIME HTML, RFC 2557) web archive. MHTML is a MIME
319    /// `multipart/related` document, so its physical bytes are shared with
320    /// [`DocumentFormat::Eml`]; the adapter **reuses the EML adapter's bounded MIME
321    /// layer** for the envelope, parts, headers, and spans, and the **HTML adapter's
322    /// bounded, error-recovering scanner** for the root part, recording the ordered
323    /// sub-resources (keyed by `Content-Location`/`Content-ID`) and the root HTML part
324    /// (the `start=` `Content-ID`, else the first `text/html` part, else the first
325    /// part). Detection is a **bounded semantic sub-detection** run before the generic
326    /// EML detector: a `multipart/related` root plus an MHTML-specific signal — a
327    /// top-level `Snapshot-Content-Location`/`Content-Base`, or a `From`+`Subject`
328    /// envelope with a `text/html` part. A plain `multipart/related` email, a plain
329    /// HTML document, and a plain MIME message stay `Eml`/`Html`/`Eml`. Not a package:
330    /// the exact leaf is the whole source, and every envelope/part/resource/HTML span
331    /// is a `Q_gen` projection (Phase 21.27).
332    Mhtml,
333    /// A syslog / log-stream document (the source carries at least two non-blank
334    /// lines and **every** non-blank line matches one of three dialects — RFC 5424
335    /// syslog, RFC 3164 (BSD) syslog, or a generic application log line with a
336    /// leading timestamp and/or level token). The recorded dialect lives per record.
337    /// Detection is byte-based and conservative and must not steal a higher-priority
338    /// format (JSON/JSON5/JSONL/CBOR/Msgpack/YAML/TOML/config/CSV or the prose
339    /// family), all of which the whole-source dispatcher tries first; plain prose and
340    /// code stay `Opaque`. Not a package: the exact leaf is the whole source, and
341    /// every record's exact line span/terminator plus every field's exact span (and
342    /// the preserved message, NILVALUE, and timestamp/level spelling) are `Q_gen`
343    /// projections (Phase 21.28).
344    Logstream,
345    /// A package-metadata manifest (an npm `package.json` / `package-lock.json`, a
346    /// Cargo `Cargo.toml` / `Cargo.lock`, or a Python `pyproject.toml`). A manifest's
347    /// physical bytes are JSON or TOML, so its bytes are shared with
348    /// [`DocumentFormat::Json`]/[`DocumentFormat::Toml`]; the **recorded dialect**
349    /// (`npm_package` / `cargo_manifest` / `pyproject_manifest` / `cargo_lock` /
350    /// `npm_lock`) lives in the model, exactly as a feed or CSV records its dialect.
351    /// Detection is a **bounded semantic test** run before the generic JSON and TOML
352    /// detectors: a strong manifest shape (a JSON object with string `name`+`version`
353    /// and a third package-specific key, a JSON `lockfileVersion`+`packages`, a TOML
354    /// `[package]` table with a string `name`, a TOML `[build-system]`/`[project]`/
355    /// `[tool.poetry]`, or a TOML `[[package]]` lockfile shape). A generic JSON
356    /// `name`+`version`, a generic TOML `name`/`version`, and plain prose stay
357    /// `Json`/`Toml`/`Opaque`. Not a package: the exact leaf is the whole source, and
358    /// every reused-arena token span plus every recorded section/entry name/value span
359    /// is a `Q_gen` projection (Phase 21.29).
360    Pkgmeta,
361    /// An API/specification document (a JSON Schema, an OpenAPI 3.x, a Swagger 2.0, or
362    /// an AsyncAPI document). An API spec's physical bytes are JSON, so its bytes are
363    /// shared with [`DocumentFormat::Json`]; the **recorded dialect** (`json_schema` /
364    /// `openapi` / `swagger` / `asyncapi`) lives in the model, exactly as CSV/config/
365    /// feed/pkgmeta record their dialects. Detection is a **bounded semantic test** run
366    /// before the generic JSON detector on a **strong, root-level string marker**: a
367    /// JSON-Schema `$schema` URI, an `openapi` `3.`-prefixed string, a `swagger` equal
368    /// to `"2.0"`, or a non-empty `asyncapi` string. A plain JSON object that merely
369    /// contains a `properties`/`paths`/`components` key but lacks the marker stays
370    /// [`DocumentFormat::Json`], and a JSON-Schema-shaped object with no `$schema` is
371    /// an honest recorded ambiguity that also stays `Json`. Not a package: the exact
372    /// leaf is the whole source, and every JSON token span plus every recorded
373    /// object/member/`$ref` span is a `Q_gen` projection — `$ref` targets are preserved
374    /// verbatim and never resolved (Phase 21.30).
375    Apispec,
376    /// A bibliography document — **BibTeX** (`.bib`), **RIS**, or **CSL-JSON**. One
377    /// adapter covers the three dialects and records the dialect in the model, exactly
378    /// as CSV/config/feed/pkgmeta/apispec record theirs. BibTeX and RIS are text (no
379    /// magic bytes); CSL-JSON's physical bytes are JSON, so its **bounded semantic
380    /// sub-detection runs before the generic JSON detector**: the root must be an
381    /// **array** of objects (or a single object) each carrying a string `id` **and** a
382    /// string `type`. A plain JSON array, a JSON array of non-CSL objects, a generic
383    /// JSON object, and prose stay [`DocumentFormat::Json`]/[`DocumentFormat::Opaque`].
384    /// Not a package: the exact leaf is the whole source, and every entry/field/item
385    /// token span (and each value's exact spelling and delimiters) is a `Q_gen`
386    /// projection (Phase 21.31).
387    Bib,
388    /// An RTF (Rich Text Format) document. RTF is a text, group-brace grammar with
389    /// control words — closer to a bounded tree (like XML) than to a flat line format —
390    /// but it is *not* a package: there is no OPC/ZIP layer, so the exact leaf is the
391    /// whole source (a `DocumentExact`, RAW-like authority) and every derived
392    /// observation is a bounded, span-preserving (`Q_gen`) projection of it. Detection
393    /// is a **strong magic + brace-balance** test: the source must begin (after an
394    /// optional UTF-8 BOM) with `{\rtf` followed by a digit, and its group braces must
395    /// balance under a bounded scan (respecting `\\`/`\{`/`\}`, `\'hh`, control words,
396    /// and `\binN`), with the outermost group closing exactly at end of input. A plain
397    /// text source, a leading-`{` blob, and every other format are therefore never
398    /// stolen; they stay `Opaque`. The adapter preserves the header, every group with
399    /// its exact brace span and nesting, every control word/symbol, every `\'hh` hex
400    /// escape and `\uN` Unicode escape (with its `\ucN` fallback), every destination
401    /// (`\fonttbl`/`\colortbl`/`\stylesheet`/`\info`/`\pict`/`\object`/`\*`) verbatim,
402    /// and every plain text run byte-exactly (never whitespace-normalized) — Phase
403    /// 21.32.
404    Rtf,
405    /// A generic archive container (Phase 21.33): a standalone **ZIP** archive (a
406    /// valid EOCD and a coherent central directory) or a standalone **TAR** archive
407    /// (a valid `ustar`/old-GNU or v7 header whose checksum matches). The recorded
408    /// dialect (`zip`/`tar`) lives in the model, exactly as CSV/config/feed record
409    /// theirs. This is deliberately a **plain archive**, never an office package: the
410    /// ZIP-family office formats (DOCX/EPUB/ODT/ODS/ODP/XLSX/PPTX) are detected
411    /// **before** this adapter, so a `.docx`/`.xlsx`/`.odt` source still detects as
412    /// its office format and is never reclassified. Not a package: the exact leaf is
413    /// the whole source (a `DocumentExact`, RAW-like authority) and every derived
414    /// observation is a bounded, span-preserving (`Q_gen`) projection of it.
415    Container,
416    /// A scientific container (Phase 21.34): a NetCDF classic (CDF-1/CDF-2) file or an
417    /// HDF5 file (a documented bounded subset). The recorded dialect
418    /// (`netcdf`/`hdf5`) lives in the model, exactly as the archive container records
419    /// `zip`/`tar`. This is deliberately a **scan-only scientific format**: it is not
420    /// an office package. Detection is a **strong magic-byte** test — NetCDF requires
421    /// `CDF\x01`/`CDF\x02` at offset 0; HDF5 requires the 8-byte signature
422    /// `\x89HDF\r\n\x1a\n` at offset 0 or 512 with a superblock version in `0..=3`, an
423    /// offset/length size in `{2,4,8}`, and a superblock that fits inside the source.
424    /// Not a package: the exact leaf is the whole source (a `DocumentExact`, RAW-like
425    /// authority) and every derived observation is a bounded, span-preserving (`Q_gen`)
426    /// projection of it. NetCDF classic is supported in full; HDF5 is a documented
427    /// bounded subset (symbol-table groups + contiguous datasets + v1 object headers),
428    /// and everything outside it declines **typed**.
429    Hdf,
430    /// A CFBF / OLE2 compound file (Phase 21.35): the physical-layer container behind
431    /// legacy Word (`.doc`), Excel (`.xls`), PowerPoint (`.ppt`), and Outlook (`.msg`)
432    /// files. Detection is a **strong magic + header validation** test — the 8-byte
433    /// signature `D0 CF 11 E0 A1 B1 1A E1` at offset 0 **and** an internally
434    /// consistent header (byte order `FFFE`, a major version of 3 or 4, a sector shift
435    /// of 9 or 12 consistent with the version, a mini-sector shift of 6, at least one
436    /// FAT sector, and a first directory sector inside the file). A random
437    /// signature-only blob and prose stay `Opaque`. The recorded dialect
438    /// (`cfb3`/`cfb4`) lives in the model. Not a package: the exact leaf is the whole
439    /// source (a `DocumentExact`, RAW-like authority) and every derived observation —
440    /// the header, the DIFAT/FAT, the directory entries with their verbatim UTF-16LE
441    /// names and reconstructed sibling/child tree, and each stream's exact byte span —
442    /// is a bounded, span-preserving (`Q_gen`) projection of it. This is the shared
443    /// physical layer that unlocks the legacy DOC/XLS/PPT/MSG semantic models (later
444    /// subphases).
445    Cfbf,
446    /// Anything else; preserved exactly by the opaque floor.
447    Opaque,
448}
449
450impl DocumentFormat {
451    /// Stable lower-case name (used in JSON and in the manifest provenance token).
452    pub const fn name(self) -> &'static str {
453        match self {
454            DocumentFormat::Pdf => "pdf",
455            DocumentFormat::Docx => "docx",
456            DocumentFormat::Epub => "epub",
457            DocumentFormat::Odt => "odt",
458            DocumentFormat::Ods => "ods",
459            DocumentFormat::Odp => "odp",
460            DocumentFormat::Xlsx => "xlsx",
461            DocumentFormat::Pptx => "pptx",
462            DocumentFormat::Json => "json",
463            DocumentFormat::Json5 => "json5",
464            DocumentFormat::Yaml => "yaml",
465            DocumentFormat::Csv => "csv",
466            DocumentFormat::Markdown => "markdown",
467            DocumentFormat::Xml => "xml",
468            DocumentFormat::Html => "html",
469            DocumentFormat::Toml => "toml",
470            DocumentFormat::Jsonl => "jsonl",
471            DocumentFormat::Eml => "eml",
472            DocumentFormat::Parquet => "parquet",
473            DocumentFormat::ArrowIpc => "arrow",
474            DocumentFormat::Cbor => "cbor",
475            DocumentFormat::Msgpack => "msgpack",
476            DocumentFormat::Config => "config",
477            DocumentFormat::Feed => "feed",
478            DocumentFormat::Geojson => "geojson",
479            DocumentFormat::Gis => "gis",
480            DocumentFormat::Notebook => "notebook",
481            DocumentFormat::FixedWidth => "fixedwidth",
482            DocumentFormat::Rst => "rst",
483            DocumentFormat::Asciidoc => "asciidoc",
484            DocumentFormat::Mdx => "mdx",
485            DocumentFormat::Mhtml => "mhtml",
486            DocumentFormat::Logstream => "logstream",
487            DocumentFormat::Pkgmeta => "pkgmeta",
488            DocumentFormat::Apispec => "apispec",
489            DocumentFormat::Bib => "bib",
490            DocumentFormat::Rtf => "rtf",
491            DocumentFormat::Container => "container",
492            DocumentFormat::Hdf => "hdf",
493            DocumentFormat::Cfbf => "cfbf",
494            DocumentFormat::Opaque => "opaque",
495        }
496    }
497
498    /// The adapter that serves this format (the observation layer's name for it).
499    pub const fn adapter(self) -> &'static str {
500        match self {
501            DocumentFormat::Pdf => "pdf",
502            DocumentFormat::Docx => "docx",
503            DocumentFormat::Epub => "epub",
504            DocumentFormat::Odt => "odt",
505            DocumentFormat::Ods => "ods",
506            DocumentFormat::Odp => "odp",
507            DocumentFormat::Xlsx => "xlsx",
508            DocumentFormat::Pptx => "pptx",
509            DocumentFormat::Json => "json",
510            DocumentFormat::Json5 => "json5",
511            DocumentFormat::Yaml => "yaml",
512            DocumentFormat::Csv => "csv",
513            DocumentFormat::Markdown => "markdown",
514            DocumentFormat::Xml => "xml",
515            DocumentFormat::Html => "html",
516            DocumentFormat::Toml => "toml",
517            DocumentFormat::Jsonl => "jsonl",
518            DocumentFormat::Eml => "eml",
519            DocumentFormat::Parquet => "parquet",
520            DocumentFormat::ArrowIpc => "arrow",
521            DocumentFormat::Cbor => "cbor",
522            DocumentFormat::Msgpack => "msgpack",
523            DocumentFormat::Config => "config",
524            DocumentFormat::Feed => "feed",
525            DocumentFormat::Geojson => "geojson",
526            DocumentFormat::Gis => "gis",
527            DocumentFormat::Notebook => "notebook",
528            DocumentFormat::FixedWidth => "fixedwidth",
529            DocumentFormat::Rst => "rst",
530            DocumentFormat::Asciidoc => "asciidoc",
531            DocumentFormat::Mdx => "mdx",
532            DocumentFormat::Mhtml => "mhtml",
533            DocumentFormat::Logstream => "logstream",
534            DocumentFormat::Pkgmeta => "pkgmeta",
535            DocumentFormat::Apispec => "apispec",
536            DocumentFormat::Bib => "bib",
537            DocumentFormat::Rtf => "rtf",
538            DocumentFormat::Container => "container",
539            DocumentFormat::Hdf => "hdf",
540            DocumentFormat::Cfbf => "cfbf",
541            DocumentFormat::Opaque => "opaque",
542        }
543    }
544
545    /// Whether the adapter for this format is compiled into this build.
546    pub const fn compiled(self) -> bool {
547        match self {
548            DocumentFormat::Pdf | DocumentFormat::Opaque => true,
549            DocumentFormat::Docx => cfg!(feature = "docx"),
550            DocumentFormat::Epub => cfg!(feature = "epub"),
551            DocumentFormat::Odt => cfg!(feature = "odt"),
552            DocumentFormat::Ods => cfg!(feature = "ods"),
553            DocumentFormat::Odp => cfg!(feature = "odp"),
554            DocumentFormat::Xlsx => cfg!(feature = "xlsx"),
555            DocumentFormat::Pptx => cfg!(feature = "pptx"),
556            DocumentFormat::Json => cfg!(feature = "json"),
557            DocumentFormat::Json5 => cfg!(feature = "json5"),
558            DocumentFormat::Yaml => cfg!(feature = "yaml"),
559            DocumentFormat::Csv => cfg!(feature = "csv"),
560            DocumentFormat::Markdown => cfg!(feature = "markdown"),
561            DocumentFormat::Xml => cfg!(feature = "xml"),
562            DocumentFormat::Html => cfg!(feature = "html"),
563            DocumentFormat::Toml => cfg!(feature = "toml"),
564            DocumentFormat::Jsonl => cfg!(feature = "jsonl"),
565            DocumentFormat::Eml => cfg!(feature = "eml"),
566            DocumentFormat::Parquet => cfg!(feature = "parquet"),
567            DocumentFormat::ArrowIpc => cfg!(feature = "arrow"),
568            DocumentFormat::Cbor => cfg!(feature = "cbor"),
569            DocumentFormat::Msgpack => cfg!(feature = "msgpack"),
570            DocumentFormat::Config => cfg!(feature = "config"),
571            DocumentFormat::Feed => cfg!(feature = "feed"),
572            DocumentFormat::Geojson => cfg!(feature = "geojson"),
573            DocumentFormat::Gis => cfg!(feature = "gis"),
574            DocumentFormat::Notebook => cfg!(feature = "notebook"),
575            DocumentFormat::FixedWidth => cfg!(feature = "fixedwidth"),
576            DocumentFormat::Rst => cfg!(feature = "rst"),
577            DocumentFormat::Asciidoc => cfg!(feature = "asciidoc"),
578            DocumentFormat::Mdx => cfg!(feature = "mdx"),
579            DocumentFormat::Mhtml => cfg!(feature = "mhtml"),
580            DocumentFormat::Logstream => cfg!(feature = "logstream"),
581            DocumentFormat::Pkgmeta => cfg!(feature = "pkgmeta"),
582            DocumentFormat::Apispec => cfg!(feature = "apispec"),
583            DocumentFormat::Bib => cfg!(feature = "bib"),
584            DocumentFormat::Rtf => cfg!(feature = "rtf"),
585            DocumentFormat::Container => cfg!(feature = "container"),
586            DocumentFormat::Hdf => cfg!(feature = "hdf"),
587            DocumentFormat::Cfbf => cfg!(feature = "cfbf"),
588        }
589    }
590
591    /// The machine-readable `format=<name>;` provenance prefix recorded at ingest.
592    pub fn provenance_prefix(self) -> String {
593        format!("format={};", self.name())
594    }
595
596    /// Recover the recorded format from a field manifest's provenance string.
597    ///
598    /// Returns `None` for a manifest that does not carry the token (e.g. a field
599    /// written before this subphase); such a field still serves every native
600    /// selector, but common observations decline typed rather than guessing.
601    pub fn from_provenance(provenance: &str) -> Option<DocumentFormat> {
602        let rest = provenance.strip_prefix("format=")?;
603        let name = rest.split(';').next()?;
604        match name {
605            "pdf" => Some(DocumentFormat::Pdf),
606            "docx" => Some(DocumentFormat::Docx),
607            "epub" => Some(DocumentFormat::Epub),
608            "odt" => Some(DocumentFormat::Odt),
609            "ods" => Some(DocumentFormat::Ods),
610            "odp" => Some(DocumentFormat::Odp),
611            "xlsx" => Some(DocumentFormat::Xlsx),
612            "pptx" => Some(DocumentFormat::Pptx),
613            "json" => Some(DocumentFormat::Json),
614            "json5" => Some(DocumentFormat::Json5),
615            "yaml" => Some(DocumentFormat::Yaml),
616            "csv" => Some(DocumentFormat::Csv),
617            "markdown" => Some(DocumentFormat::Markdown),
618            "xml" => Some(DocumentFormat::Xml),
619            "html" => Some(DocumentFormat::Html),
620            "toml" => Some(DocumentFormat::Toml),
621            "jsonl" => Some(DocumentFormat::Jsonl),
622            "eml" => Some(DocumentFormat::Eml),
623            "parquet" => Some(DocumentFormat::Parquet),
624            "arrow" => Some(DocumentFormat::ArrowIpc),
625            "cbor" => Some(DocumentFormat::Cbor),
626            "msgpack" => Some(DocumentFormat::Msgpack),
627            "config" => Some(DocumentFormat::Config),
628            "feed" => Some(DocumentFormat::Feed),
629            "geojson" => Some(DocumentFormat::Geojson),
630            "gis" => Some(DocumentFormat::Gis),
631            "notebook" => Some(DocumentFormat::Notebook),
632            "fixedwidth" => Some(DocumentFormat::FixedWidth),
633            "rst" => Some(DocumentFormat::Rst),
634            "asciidoc" => Some(DocumentFormat::Asciidoc),
635            "mdx" => Some(DocumentFormat::Mdx),
636            "mhtml" => Some(DocumentFormat::Mhtml),
637            "logstream" => Some(DocumentFormat::Logstream),
638            "pkgmeta" => Some(DocumentFormat::Pkgmeta),
639            "apispec" => Some(DocumentFormat::Apispec),
640            "bib" => Some(DocumentFormat::Bib),
641            "rtf" => Some(DocumentFormat::Rtf),
642            "container" => Some(DocumentFormat::Container),
643            "hdf" => Some(DocumentFormat::Hdf),
644            "cfbf" => Some(DocumentFormat::Cfbf),
645            "opaque" => Some(DocumentFormat::Opaque),
646            _ => None,
647        }
648    }
649}
650
651/// Detect the document format of `source` from its bytes alone.
652///
653/// Never consults a file name or extension. A malformed or ambiguous input (or a
654/// format whose adapter is not compiled) falls back to [`DocumentFormat::Opaque`].
655pub fn detect_document_format(source: &[u8], limits: Limits) -> DocumentFormat {
656    if crate::adapter::pdf::detect(source, limits) {
657        return DocumentFormat::Pdf;
658    }
659    #[cfg(feature = "package")]
660    {
661        if let Some(format) = detect_zip_family(source, limits) {
662            return format;
663        }
664    }
665    // The generic archive container (Phase 21.33) is a **standalone** format, never
666    // an office package: a plain ZIP (a valid EOCD + coherent central directory) or a
667    // plain TAR (a strong 512-byte header whose checksum matches). It is inserted
668    // **after** the entire office ZIP family above — so an office package is never
669    // reclassified as a plain archive — and **before** every weak, no-magic-byte
670    // heuristic (JSON/YAML/CSV/Markdown/XML/HTML), which a binary archive must never
671    // be guessed to be. It is deliberately conservative: a `PK`-prefixed blob without
672    // a valid EOCD, a 512-byte-junk blob whose checksum fails, and prose all stay
673    // `Opaque`.
674    #[cfg(feature = "container")]
675    if crate::adapter::container::detect(source, limits) {
676        return DocumentFormat::Container;
677    }
678    // The scientific container (Phase 21.34) is a **standalone** format with strong
679    // magics: NetCDF classic (`CDF\x01`/`CDF\x02` at offset 0) and HDF5 (the 8-byte
680    // signature at offset 0 or 512 plus a validated superblock). It is inserted with
681    // the other strong-magic binaries (after the office ZIP family and the archive
682    // container, before Parquet) and **before** every weak, no-magic-byte heuristic
683    // (JSON/YAML/CSV/Markdown/XML/HTML), which a binary scientific file must never be
684    // guessed to be. It is deliberately conservative on HDF5: a random
685    // `\x89HDF`-prefixed blob with a bad superblock, and prose, stay `Opaque`.
686    // NetCDF's magic is the 4-byte `CDF\x01`/`CDF\x02`; a `CDF`-prefixed blob is
687    // admitted here and its derived model then declines typed if the header is
688    // malformed (the whole-source exact leaf is always unaffected).
689    #[cfg(feature = "hdf")]
690    if crate::adapter::hdf::detect(source, limits) {
691        return DocumentFormat::Hdf;
692    }
693    // The CFBF / OLE2 compound file (Phase 21.35) is a **standalone** physical-layer
694    // format with a strong 8-byte signature (`D0 CF 11 E0 A1 B1 1A E1`) plus a
695    // validated header. It is inserted with the other strong-magic binaries (after the
696    // office ZIP family, the archive container, and the scientific container, and
697    // **before** Parquet) and **before** every weak, no-magic-byte heuristic
698    // (JSON/YAML/CSV/Markdown/XML/HTML), which a binary compound file must never be
699    // guessed to be. It is deliberately conservative: a signature-only blob (its
700    // header fields are zero) and prose stay `Opaque`; a blob with a plausible header
701    // but a malformed body is admitted and its derived model then declines typed (the
702    // whole-source exact leaf is always unaffected).
703    #[cfg(feature = "cfbf")]
704    if crate::adapter::cfbf::detect(source, limits) {
705        return DocumentFormat::Cfbf;
706    }
707    // Parquet is the **analytical** Wave-2 format (Phase 21.14), with no package
708    // layer. Its detection is a strong magic-byte contract — `PAR1` at both ends
709    // plus a consistent little-endian footer length — so it runs immediately after
710    // the package families and **before** the weak, no-magic-byte heuristics
711    // (JSON/YAML/CSV/Markdown/XML/HTML), which a binary columnar file must never be
712    // guessed to be. It is deliberately conservative: a file that merely begins or
713    // ends with `PAR1` but whose footer length overruns the leading magic stays
714    // `Opaque`.
715    #[cfg(feature = "parquet")]
716    if crate::adapter::parquet::detect(source, limits) {
717        return DocumentFormat::Parquet;
718    }
719    // Arrow IPC is the **analytical** Wave-2 format (Phase 21.16), with no package
720    // layer. Its detection is a strong magic-byte contract — the `ARROW1` magic at
721    // the start plus either a consistent trailing footer (file format) or a valid
722    // encapsulated `Schema` message (stream format) — so it runs immediately after
723    // Parquet and **before** the weak, no-magic-byte heuristics, which a binary
724    // columnar file must never be guessed to be. It is deliberately conservative:
725    // `ARROW1`-prefixed junk with neither a consistent footer nor a valid schema
726    // message stays `Opaque`.
727    #[cfg(feature = "arrow")]
728    if crate::adapter::arrow::detect(source, limits) {
729        return DocumentFormat::ArrowIpc;
730    }
731    // JSON is a Wave-2 structured-tree format with **no** package layer, so it is
732    // detected directly from the whole source (never through `detect_zip_family`).
733    // Conservative: the entire source must parse as exactly one JSON value within
734    // the JSON caps, else the input stays Opaque (Phase 21.5.1).
735    //
736    // GeoJSON is the **spatial** Wave-2 format (Phase 21.22), whose physical bytes
737    // are JSON, so it is a **bounded semantic sub-detection** run **before** the
738    // generic JSON detector below: the source must parse as exactly one JSON value
739    // whose root is an object with a `"type"` string equal to one of the nine RFC 7946
740    // type names **and** whose shape is consistent (a geometry has an array
741    // `coordinates`/`geometries`, a `Feature` has `geometry`/`properties`, a
742    // `FeatureCollection` has an array `features`). A more specific claim than a bare
743    // JSON value, so it is tried first; a plain JSON document, and a JSON document
744    // whose `"type"` is an unrelated string, decline here and are then claimed by the
745    // JSON detector (staying `Json`). Prose and malformed input stay `Opaque`.
746    #[cfg(feature = "geojson")]
747    if crate::adapter::geojson::detect(source, limits) {
748        return DocumentFormat::Geojson;
749    }
750    // A Jupyter notebook (`.ipynb`, nbformat) is the **document-shaped** Wave-2 format
751    // (Phase 21.24), whose physical bytes are JSON, so it is a **bounded semantic
752    // sub-detection** run **before** the generic JSON detector below: the source must
753    // parse as exactly one JSON value that is an nbformat-shaped object — a plain
754    // non-negative integer-literal `nbformat` (≥ 1), an array `cells`, every cell an
755    // object with a string `cell_type`, and every present recognized field
756    // nbformat-shaped (a string-or-array-of-strings `source`, a number-or-null
757    // `execution_count`, object `metadata`/`attachments`, an array `outputs` of objects
758    // with a string `output_type`). A more specific claim than a bare JSON value, so it
759    // is tried first; a plain JSON document, and a JSON document that merely has a
760    // `cells` key but is not nbformat-shaped, decline here and are then claimed by the
761    // JSON detector (staying `Json`). Prose and malformed input stay `Opaque`.
762    #[cfg(feature = "notebook")]
763    if crate::adapter::notebook::detect(source, limits) {
764        return DocumentFormat::Notebook;
765    }
766    // A package-metadata manifest (Phase 21.29) — an npm `package.json` /
767    // `package-lock.json`, a Cargo `Cargo.toml` / `Cargo.lock`, or a Python
768    // `pyproject.toml` — is the **manifest** Wave-2 format, whose physical bytes are
769    // JSON or TOML. It is a **bounded semantic sub-detection** run **before** the
770    // generic JSON and TOML detectors below: the source must carry a strong manifest
771    // shape (a JSON object with string `name`+`version` and a third package-specific
772    // key, a JSON `lockfileVersion`+`packages`/`dependencies`, a TOML `[package]` table
773    // with a string `name`, a TOML `[build-system]`/`[project]`/`[tool.poetry]`, or a
774    // TOML `[[package]]` lockfile shape). A more specific claim than a bare JSON/TOML
775    // value, so it is tried first; a generic JSON `{"name":…,"version":…}`, a generic
776    // TOML `name = …` / `version = …`, and plain prose decline here and are then
777    // claimed by the JSON detector (staying `Json`), the TOML detector (staying
778    // `Toml`), or the opaque floor. Detection is content-only and never consults a
779    // file name.
780    #[cfg(feature = "pkgmeta")]
781    if crate::adapter::pkgmeta::detect(source, limits) {
782        return DocumentFormat::Pkgmeta;
783    }
784    // An API-specification document (Phase 21.30) — a JSON Schema, an OpenAPI 3.x, a
785    // Swagger 2.0, or an AsyncAPI document — is the **API/spec** Wave-2 format, whose
786    // physical bytes are JSON. It is a **bounded semantic sub-detection** run **before**
787    // the generic JSON detector below: the root must be a JSON object carrying a
788    // **strong, root-level string marker** — a JSON-Schema `$schema` URI, an `openapi`
789    // `3.`-prefixed string, a `swagger` equal to `"2.0"`, or a non-empty `asyncapi`
790    // string. A more specific claim than a bare JSON value, so it is tried first; a
791    // plain JSON object that merely has a `properties`/`paths` key, a JSON-Schema-shaped
792    // object with **no** `$schema`, an unrelated `openapi`/`swagger` value, and prose
793    // decline here and are then claimed by the JSON detector (staying `Json`) or the
794    // opaque floor. Detection is content-only and never consults a file name.
795    #[cfg(feature = "apispec")]
796    if crate::adapter::apispec::detect(source, limits) {
797        return DocumentFormat::Apispec;
798    }
799    // A bibliography (Phase 21.31) — BibTeX, RIS, or CSL-JSON — is the
800    // **bibliography** Wave-2 format. BibTeX/RIS are line-anchored text and CSL-JSON's
801    // physical bytes are JSON, so this is a **bounded content-only test** run **before**
802    // the generic JSON detector below (a CSL-JSON array is otherwise claimed by the JSON
803    // detector, which is the weaker claim) and **before** the weak text heuristics
804    // (YAML/TOML/config/CSV/Markdown/RST/AsciiDoc/logstream/fixedwidth), which a BibTeX
805    // `@type{…}` file or an RIS `TY  - ` stream must never be guessed to be. It claims
806    // only on a strong signal: a line-anchored `@type{` entry sequence, an RIS `TY  - `
807    // entry sequence whose every entry is `ER  - `-terminated, or a JSON array (or
808    // single object) whose every element carries a string `id` **and** a string `type`.
809    // A plain JSON array, a JSON array of non-CSL objects, a generic JSON object, and
810    // prose decline here and are then claimed by the JSON detector (staying `Json`) or
811    // the opaque floor. Detection is content-only and never consults a file name.
812    #[cfg(feature = "bib")]
813    if crate::adapter::bib::detect(source, limits) {
814        return DocumentFormat::Bib;
815    }
816    // RTF (Rich Text Format, Phase 21.32) is a text group-brace grammar with a strong
817    // header magic (`{\rtf<digit>`), so it is tried here with the **strong-magic**
818    // detectors (right after the binary magics PDF/ZIP/Parquet/Arrow and before the
819    // JSON family). Its detection is magic + a bounded brace-balance scan, so no other
820    // format's bytes can steal it and it can steal no other format: an RTF source
821    // begins with `{`, which is not a valid JSON document (JSON would need a valid
822    // value), not a config `KEY=VALUE` line, and not a Markdown/RST structural mark;
823    // and the binary detectors above reject it. A plain text or leading-`{` blob stays
824    // `Opaque`.
825    #[cfg(feature = "rtf")]
826    if crate::adapter::rtf::detect(source, limits) {
827        return DocumentFormat::Rtf;
828    }
829    #[cfg(feature = "json")]
830    if crate::adapter::json::detect(source, limits) {
831        return DocumentFormat::Json;
832    }
833    // JSON5/JSONC is the **structured-extra** Wave-2 format (Phase 21.17.1), also
834    // with no package layer. It is tried **immediately after JSON** and before
835    // JSONL/YAML/TOML/CSV/Markdown/XML/HTML: JSON5 is a superset of JSON, so a
836    // strict JSON source is already claimed by the JSON detector above, and this
837    // detector requires at least one JSON5/JSONC-only construct (a comment, a
838    // trailing comma, an unquoted key, a single quote, a hex/leading-dot/
839    // `Infinity`/`NaN` number, a string continuation, or the extended whitespace
840    // set). A strict JSON document is therefore never reclassified, and a plain
841    // non-JSON blob stays Opaque.
842    #[cfg(feature = "json5")]
843    if crate::adapter::json5::detect(source, limits) {
844        return DocumentFormat::Json5;
845    }
846    // JSONL/NDJSON is the **line/event-stream** Wave-2 format (Phase 21.12), also
847    // with no package layer. It is tried **immediately after JSON** because it is the
848    // most specific signal for a newline-separated JSON stream: each non-blank line
849    // must parse as exactly one JSON value, which is stricter than — and must run
850    // before — the YAML stream reader (which would otherwise be free to reinterpret a
851    // line-delimited JSON stream as a stream of documents). Detection is
852    // conservative: at least two non-blank lines are required and every non-blank
853    // line must parse as exactly one JSON value under the caps, so a single JSON
854    // value, a malformed line, and a bag of values with a non-newline separator all
855    // stay `Opaque`.
856    #[cfg(feature = "jsonl")]
857    if crate::adapter::jsonl::detect(source, limits) {
858        return DocumentFormat::Jsonl;
859    }
860    // CBOR is the **binary** structured-tree Wave-2 format (Phase 21.18), also with
861    // no package layer. It is tried **immediately after the JSON family** (JSON,
862    // JSON5, JSONL) and **before** the remaining textual heuristics
863    // (EML/YAML/TOML/CSV/Markdown/XML/HTML): the strong magic-byte binaries
864    // (PDF/ZIP/Parquet/Arrow) run above and are never reconsidered. CBOR has **no
865    // magic bytes**, so its detector is deliberately conservative — the
866    // self-described-CBOR tag `55799` (`0xd9 0xd9 0xf7`), or a full-input,
867    // well-formed parse whose root is a container/tag and which reaches at least
868    // three nodes. A container head byte is always `>= 0x80`, so no pure-ASCII
869    // document (JSON/YAML/TOML/XML/HTML/prose) can be claimed; a lone scalar, an
870    // empty container, a truncated item, and a structurally trivial input all stay
871    // `Opaque` rather than being guessed. The small-integer/short-array encodings
872    // overlap MessagePack's fixint/fixarray (recorded honestly; a Phase-21.19
873    // MessagePack adapter must share this seam).
874    #[cfg(feature = "cbor")]
875    if crate::adapter::cbor::detect(source, limits) {
876        return DocumentFormat::Cbor;
877    }
878    // MessagePack is the **binary** structured-tree Wave-2 sibling of CBOR (Phase
879    // 21.19), also with no package layer. It is tried **immediately after CBOR** and
880    // **before** the remaining textual heuristics (EML/YAML/TOML/CSV/Markdown/XML/
881    // HTML): the strong magic-byte binaries (PDF/ZIP/Parquet/Arrow) and the JSON
882    // family run above and are never reconsidered. **Placement is load-bearing: the
883    // two binary detectors share the whole-number/short-container prefix, so putting
884    // CBOR first guarantees its stronger self-described-tag signal (and every input
885    // well-formed under both grammars) is never stolen from it.** MessagePack has
886    // **no magic bytes**, so its detector is deliberately conservative — a
887    // full-input, well-formed parse whose root is a container (array/map) reaching at
888    // least three items and eight bytes, or the same with an unambiguous
889    // MessagePack-only head byte (`0xdc..=0xdf`, which CBOR's grammar rejects). A
890    // container head byte is always `>= 0x80`, so no pure-ASCII document can be
891    // claimed; a lone scalar, an empty container, the ambiguous
892    // `fixarray(3)`/`fixmap(2)` overlap fixtures, and a structurally trivial input
893    // all stay `Opaque` rather than being guessed.
894    #[cfg(feature = "msgpack")]
895    if crate::adapter::msgpack::detect(source, limits) {
896        return DocumentFormat::Msgpack;
897    }
898    // MHTML (MIME HTML, RFC 2557) is the **web-archive** Wave-2 format (Phase
899    // 21.27), whose physical bytes are a MIME `multipart/related` message. It is a
900    // **bounded semantic sub-detection** run **before** the generic EML detector
901    // below (a `multipart/related` message is otherwise claimed by EML, which is a
902    // weaker claim): the root must be `multipart/related` **and** carry an
903    // MHTML-specific signal — a top-level `Snapshot-Content-Location`/`Content-Base`,
904    // or a `From`+`Subject` envelope with a `text/html` part. A plain
905    // `multipart/related` email without those markers, a plain HTML document (no MIME
906    // envelope), and a plain MIME message (not `multipart/related`) all decline here
907    // and are then claimed by their own detectors (`Eml`/`Html`/`Eml`).
908    #[cfg(feature = "mhtml")]
909    if crate::adapter::mhtml::detect(source, limits) {
910        return DocumentFormat::Mhtml;
911    }
912    // EML/MIME is the **messaging** Wave-2 format (Phase 21.13), also with no package
913    // layer. It is tried **immediately after JSONL** and **before YAML/TOML/CSV/
914    // Markdown/XML/HTML**: a raw message begins with an RFC 5322 header block, which
915    // the YAML stream reader would otherwise reinterpret as a mapping (e.g. `From: a`
916    // / `Date: b`). Detection is conservative: a genuine header block terminated by a
917    // blank line carrying `From`/`Date`/`Message-ID`, or an explicit `MIME-Version`,
918    // is required, so prose without a header block and a colon-bearing note stay
919    // `Opaque`. Only the header block is scanned (never the MIME tree).
920    #[cfg(feature = "eml")]
921    if crate::adapter::eml::detect(source, limits) {
922        return DocumentFormat::Eml;
923    }
924    // YAML is the second Wave-2 structured-tree format (Phase 21.6.1), also with
925    // **no** package layer. It is detected directly and conservatively: the whole
926    // source must parse as a bounded YAML stream under the YAML caps, and every
927    // document root must be a mapping or a sequence, else the input stays Opaque.
928    #[cfg(feature = "yaml")]
929    if crate::adapter::yaml::detect(source, limits) {
930        return DocumentFormat::Yaml;
931    }
932    // TOML is the next Wave-2 structured-tree format (Phase 21.11). Precedence
933    // (explicit): TOML is tried **after** the strong, fully-parsed tree formats
934    // (JSON, YAML) but **before** the weak, no-magic-byte heuristics (CSV,
935    // Markdown, XML, HTML). TOML's positive signal is a complete, error-free parse,
936    // which is far stronger than the CSV/Markdown heuristics; crucially, a TOML
937    // comment (`# …` at column 0) is otherwise misread as a Markdown ATX heading and
938    // would steal the document. A source that is valid JSON/YAML still wins (it is
939    // tried first); a source that is valid XML/HTML never parses as TOML (it begins
940    // with `<`), so nothing is lost there. TOML has no magic bytes and a plain prose
941    // paragraph is not TOML, so detection is conservative: the whole source must
942    // parse as TOML 1.0 under the TOML caps with no errors **and** carry at least one
943    // key/value assignment. TOML's duplicate-key/redefinition rules are enforced, so
944    // a doc that violates them is not detected. Plain prose and non-TOML text stay
945    // Opaque.
946    #[cfg(feature = "toml")]
947    if crate::adapter::toml::detect(source, limits) {
948        return DocumentFormat::Toml;
949    }
950    // The config family (INI / `.env` / Java `.properties`) is the key/value line
951    // Wave-2 format (Phase 21.20). Precedence (explicit): it is tried **after** the
952    // strong, fully-parsed tree formats (JSON/JSON5/JSONL/YAML/TOML) — a TOML
953    // document is very often a syntactically valid INI file, so a source that is
954    // valid TOML must stay `Toml` (never reclassified) — and **before** CSV/Markdown/
955    // XML/HTML. It precedes CSV deliberately: a config file whose values contain
956    // commas (e.g. `A=x,y`) has a consistent field count and would otherwise be
957    // stolen by the CSV detector, whereas the config `KEY=VALUE` shape is the more
958    // specific signal. Config has **no magic bytes**, so detection is deliberately
959    // conservative and requires a **dialect-distinguishing signal**: an INI
960    // `[section]` header; a properties `:`/whitespace separator, `!` comment,
961    // `\uXXXX` escape, continuation, or non-identifier key (plus a strong `=`/`:`
962    // separator somewhere); or an `env` `export ` prefix. The pure `KEY=VALUE`
963    // overlap between `env` and `properties`, plain prose, a two-column `a b`
964    // blob, and a `#!` script all stay `Opaque` rather than being guessed; a plain
965    // `.txt`/Markdown/code source is never claimed.
966    #[cfg(feature = "config")]
967    if crate::adapter::config::detect(source, limits) {
968        return DocumentFormat::Config;
969    }
970    // CSV/TSV is the first **tabular** Wave-2 format (Phase 21.7.1). It has no
971    // magic bytes, so it is detected last and conservatively: only after the
972    // PDF/ZIP/JSON/YAML families are declined does a source qualify, and then only
973    // if it parses under a specific delimiter (`,`, tab, or `|`) as a table with a
974    // consistent field count across a sampled majority of records and at least two
975    // columns. Phase 21.25 adds the pipe delimiter to this dialect set; the pipe
976    // dialect is tried after comma/tab and is declined when the source carries a
977    // GFM/Markdown delimiter row (so a Markdown table is never stolen). When in
978    // doubt the input stays Opaque.
979    #[cfg(feature = "csv")]
980    if crate::adapter::csv::detect(source, limits) {
981        return DocumentFormat::Csv;
982    }
983    // MDX (Phase 21.26.3) is Markdown + JSX/ESM, so it is tried **immediately before
984    // Markdown** (and therefore before HTML/XML and the other prose/tabular
985    // heuristics): an MDX document's bytes are Markdown-plus-JSX, so generic Markdown
986    // must never claim it first. Detection is conservative and **MDX-specific** — the
987    // source must parse under the Markdown model **and** carry a top-level ESM
988    // `import`/`export` statement, a JSX component (a capitalized/namespaced element
989    // name) or fragment, a JSX-specific attribute (a brace value or spread), or a
990    // whole-line MDX block expression `{…}` — and it declines anything already
991    // claimed by JSON/YAML/CSV/HTML/XML/RST/AsciiDoc, so a plain Markdown document
992    // stays `Markdown`, a plain HTML document stays `Html`, and plain prose stays
993    // `Opaque`.
994    #[cfg(feature = "mdx")]
995    if crate::adapter::mdx::detect(source, limits) {
996        return DocumentFormat::Mdx;
997    }
998    // Markdown is the first **prose** Wave-2 format (Phase 21.8.1). It has no
999    // magic bytes either, and a plain prose paragraph is itself valid Markdown, so
1000    // detection is deliberately conservative: only after the PDF/ZIP/JSON/YAML/CSV
1001    // families are declined, and only when the source carries a clear structural
1002    // mark (an ATX heading, a fenced code block, front matter, a table, a reference
1003    // definition, or a footnote definition), is it admitted. Plain prose stays
1004    // Opaque rather than being guessed to be Markdown.
1005    #[cfg(feature = "markdown")]
1006    if crate::adapter::markdown::detect(source, limits) {
1007        return DocumentFormat::Markdown;
1008    }
1009    // reStructuredText (Phase 21.26.1) is the **next prose** Wave-2 format, also with
1010    // no package layer. It is tried **immediately after Markdown** and **before
1011    // fixed-width**: Markdown must be tried first so a Markdown document is never
1012    // stolen (reST detection requires a reST-**specific** signal — an explicit markup
1013    // start, a grid/simple table, a field list, or an adornment whose char is not a
1014    // Markdown construct), and a reST document must be claimed before the maximally
1015    // ambiguous fixed-width heuristic (which could otherwise read an aligned reST body
1016    // as a column-position table). Plain prose stays Opaque.
1017    #[cfg(feature = "rst")]
1018    if crate::adapter::rst::detect(source, limits) {
1019        return DocumentFormat::Rst;
1020    }
1021    // AsciiDoc (Phase 21.26.2) is the **third prose** Wave-2 format, also with no
1022    // package layer. It is tried **immediately after reStructuredText** and **before
1023    // fixed-width**: Markdown and reST must be tried first so neither is stolen (a
1024    // valid Markdown or reST document always keeps its own format), and AsciiDoc must
1025    // be claimed before the maximally ambiguous fixed-width heuristic (which could
1026    // otherwise read an aligned AsciiDoc body as a column-position table). The
1027    // dispatcher already tries PDF/ZIP/JSON/YAML/config/CSV ahead of it, so a
1028    // `:name: value`-shaped config source and a `#`/fence-bearing Markdown source are
1029    // claimed earlier. Detection is conservative and AsciiDoc-specific (a document
1030    // title followed by a block, a `==`+ section, a `|===` table, a complete
1031    // delimited block, or a block attribute line followed by a block); a lone `= text`
1032    // line and plain prose stay Opaque.
1033    #[cfg(feature = "asciidoc")]
1034    if crate::adapter::asciidoc::detect(source, limits) {
1035        return DocumentFormat::Asciidoc;
1036    }
1037    // A syslog / log stream is the **line/event-stream** Wave-2 format (Phase 21.28),
1038    // also with no package layer. Precedence (explicit): it is tried **after** every
1039    // structured and prose format above — the JSON family (JSON/JSON5/JSONL/CBOR/
1040    // Msgpack), MHTML/EML, YAML/TOML, the config family, CSV, and the prose family
1041    // (MDX/Markdown/RST/AsciiDoc) — so a JSON/YAML/CSV/prose document is never stolen;
1042    // and **before** the maximally ambiguous fixed-width heuristic, which could
1043    // otherwise read a column-aligned log stream as a column-position table. It is
1044    // tried before the XML/HTML family too, but a log line never begins with a valid
1045    // XML/HTML root (a syslog `<PRI>` is not well-formed XML). Detection is byte-based
1046    // and conservative: at least two non-blank lines, and **every** non-blank line must
1047    // match a dialect — an RFC 5424 syslog header, an RFC 3164 (BSD) syslog line, or a
1048    // generic application log line with a leading timestamp and/or level token. Plain
1049    // prose and code stay `Opaque` (a prose line that does not begin with a timestamp/
1050    // level declines the whole file; a file whose every line begins with a level word
1051    // IS claimed — recorded honestly in the adapter docs).
1052    #[cfg(feature = "logstream")]
1053    if crate::adapter::logstream::detect(source, limits) {
1054        return DocumentFormat::Logstream;
1055    }
1056    // Fixed-width (column-position) text is the second **tabular** Wave-2 format
1057    // (Phase 21.25), but its columns are defined by character positions, not a
1058    // delimiter, so it is a distinct format (and adapter), never a CSV dialect. It is
1059    // tried **last among the tabular/prose heuristics** — after CSV/TSV/PSV, after
1060    // Markdown, and after reStructuredText — so a delimited table, a Markdown table, or
1061    // a reST document always keeps its own format (the fixed-width detector also
1062    // declines a delimited or Markdown table explicitly, defence in depth).
1063    // Fixed-width is **genuinely ambiguous** (nearly any aligned text can look
1064    // tabular), so detection is maximally conservative: at least three sampled records
1065    // of identical byte width, an inferred column layout whose interior whitespace gaps
1066    // are at least two columns wide, and at least two non-empty columns; a
1067    // variable-length or single-space-separated blob stays Opaque.
1068    #[cfg(feature = "fixedwidth")]
1069    if crate::adapter::fixedwidth::detect(source, limits) {
1070        return DocumentFormat::FixedWidth;
1071    }
1072    // RSS/Atom is the **syndication** Wave-2 format (Phase 21.21). A feed's physical
1073    // bytes are XML, so this is a **bounded semantic sub-detection** run **before**
1074    // the generic standalone-XML detector below: an RSS root `<rss>` with a
1075    // `<channel>` child, or an Atom root `<feed>` in the Atom namespace
1076    // (`http://www.w3.org/2005/Atom`) with at least one `<entry>` child. A feed is a
1077    // more specific claim than a bare XML tree, so it is tried first; a plain XML
1078    // document (a root that is neither `<rss>` nor an Atom `<feed>`, or an
1079    // `<rss>`-shaped-but-invalid / non-Atom / record-less `<feed>`) declines here and
1080    // falls through to the XML detector (staying `Xml`) or to `Opaque`. An HTML
1081    // document (`<!doctype html>` / an `<html>` root) simply declines here and is
1082    // then claimed by the HTML document-level marker immediately below.
1083    #[cfg(feature = "feed")]
1084    if crate::adapter::feed::detect(source, limits) {
1085        return DocumentFormat::Feed;
1086    }
1087    // KML/GPX is the **geospatial** Wave-2 format (Phase 21.23). Its physical bytes
1088    // are XML, so this is a **bounded semantic sub-detection** run **before** the
1089    // generic standalone-XML detector below: a `<kml>` root in the KML namespace
1090    // (`http://www.opengis.net/kml/2.2`) with a `Document`/`Folder`/`Placemark` child,
1091    // or a `<gpx>` root in the GPX namespace (`http://www.topografix.com/GPX/1/1`)
1092    // with a `metadata`/`wpt`/`rte`/`trk` child. A geospatial document is a more
1093    // specific claim than a bare XML tree, so it is tried first; a plain XML document
1094    // (a root that is neither `<kml>` nor `<gpx>`, or a shaped-but-invalid / non-GIS
1095    // namespace / child-less root) declines here and falls through to the XML detector
1096    // (staying `Xml`) or to `Opaque`. An older KML (2.0/2.1) or GPX 1.0 namespace is
1097    // not the claimed URI, so it also stays `Xml`. An HTML document simply declines
1098    // here and is then claimed by the HTML document-level marker immediately below.
1099    #[cfg(feature = "gis")]
1100    if crate::adapter::gis::detect(source, limits) {
1101        return DocumentFormat::Gis;
1102    }
1103    // HTML document-level marker (Phase 21.10 / 21.15 fix): a `<!doctype html>` or
1104    // an `<html>` root element is a more specific signal than the generic XML
1105    // fallback, so a **well-formed XHTML page** is classified `Html`, not `Xml`
1106    // (XML's well-formedness parser would otherwise claim it). The general XML
1107    // branch below is still tried before the general HTML branch, so a bare XML
1108    // tree stays XML and HTML only claims the sources XML declines.
1109    #[cfg(feature = "html")]
1110    if crate::adapter::html::has_document_marker(source)
1111        && crate::adapter::html::detect(source, limits)
1112    {
1113        return DocumentFormat::Html;
1114    }
1115    // XML is the structured-tree Wave-2 format for a bare XML source (Phase 21.9).
1116    // It has no package layer and no magic bytes beyond `<`, so it is detected last
1117    // and conservatively: only after the PDF/ZIP/JSON/YAML/CSV/Markdown families are
1118    // declined, only when the source begins with `<` (an XML prolog or a root
1119    // element start), and only when the whole source parses as well-formed XML with
1120    // exactly one root element under the XML caps. `<-prefixed` junk and prose stay
1121    // Opaque rather than being guessed to be XML.
1122    #[cfg(feature = "xml")]
1123    if crate::adapter::xml::detect(source, limits) {
1124        return DocumentFormat::Xml;
1125    }
1126    // HTML is the error-recovering markup Wave-2 format (Phase 21.10). Precedence
1127    // note (explicit): a *document-level* HTML marker is handled above (before XML);
1128    // for every other `<`-bearing source XML is tried before HTML, so any document
1129    // XML accepts (a bare XML tree) is genuinely XML. HTML then claims the
1130    // `<`-bearing sources XML declines — real HTML5 is almost never well-formed XML
1131    // (void elements, optional end tags, unquoted attributes, undeclared named
1132    // entities). Detection is conservative: the source must carry clear HTML
1133    // structure (a `<!doctype html>`, an `<html`/`<head`/`<body` tag, or a
1134    // preponderance of known HTML tags). Plain prose and non-HTML `<`-junk stay
1135    // Opaque rather than being guessed to be HTML.
1136    #[cfg(feature = "html")]
1137    if crate::adapter::html::detect(source, limits) {
1138        return DocumentFormat::Html;
1139    }
1140    DocumentFormat::Opaque
1141}
1142
1143/// Whether `source` is a structurally valid ZIP archive.
1144///
1145/// Used by the universal ingest dispatcher so a generic ZIP (which detection
1146/// reports as `Opaque`) is still inverted through the byte-authoritative package
1147/// layer rather than the opaque floor.
1148#[cfg(feature = "package")]
1149pub fn is_zip(source: &[u8], limits: Limits) -> bool {
1150    crate::adapter::package::scan(source, limits).is_ok()
1151}
1152
1153#[cfg(feature = "package")]
1154fn detect_zip_family(source: &[u8], limits: Limits) -> Option<DocumentFormat> {
1155    let physical = crate::adapter::package::scan(source, limits).ok()?;
1156
1157    // EPUB (OCF): the mandatory `mimetype` member, or an OCF container that
1158    // resolves to an OPF rootfile.
1159    let mimetype = member_decoded(&physical, source, EPUB_MIMETYPE_MEMBER, limits);
1160    let mimetype_ok = mimetype.as_deref() == Some(EPUB_MIMETYPE);
1161    let container = member_decoded(&physical, source, CONTAINER_MEMBER, limits);
1162    let container_ok = container
1163        .as_deref()
1164        .is_some_and(|c| contains(c, CONTAINER_NS) && contains(c, OPF_MEDIA_TYPE));
1165    let is_epub = mimetype_ok || container_ok;
1166
1167    // DOCX: an OPC package whose content types declare a WordprocessingML main
1168    // part, or whose package relationships declare an `officeDocument` part that
1169    // targets `word/document.xml`. The positive WordprocessingML signal (rather
1170    // than the mere absence of a SpreadsheetML one) keeps DOCX and XLSX mutually
1171    // exclusive without misclassifying a Word document that *embeds* an Excel
1172    // workbook (whose package declares SpreadsheetML content types for the
1173    // embedded part, but no SpreadsheetML workbook main part). Without `xlsx`
1174    // and `pptx` the legacy relationship-only rule stands.
1175    let content_types = member_decoded(&physical, source, CONTENT_TYPES_MEMBER, limits);
1176    let rels = member_decoded(&physical, source, PACKAGE_RELS_MEMBER, limits);
1177    #[cfg(any(feature = "xlsx", feature = "pptx"))]
1178    let is_docx = content_types
1179        .as_deref()
1180        .is_some_and(|ct| contains(ct, DOCX_MAIN_FRAGMENT))
1181        || rels.as_deref().is_some_and(|r| {
1182            contains(r, OFFICE_DOCUMENT_FRAGMENT) && contains(r, DOCX_MAIN_TARGET)
1183        });
1184    #[cfg(not(any(feature = "xlsx", feature = "pptx")))]
1185    let is_docx = content_types.is_some()
1186        && rels
1187            .as_deref()
1188            .is_some_and(|r| contains(r, OFFICE_DOCUMENT_FRAGMENT));
1189
1190    // XLSX: an OPC package whose content types declare a SpreadsheetML workbook
1191    // (or whose `officeDocument` relationship targets a workbook part).
1192    #[cfg(feature = "xlsx")]
1193    let is_xlsx = content_types.as_deref().is_some_and(|ct| {
1194        contains(ct, XLSX_MAIN_FRAGMENT)
1195            || (contains(ct, XLSX_NS_FRAGMENT)
1196                && rels
1197                    .as_deref()
1198                    .is_some_and(|r| contains(r, XLSX_WORKBOOK_TARGET)))
1199    });
1200    #[cfg(not(feature = "xlsx"))]
1201    let is_xlsx = false;
1202
1203    // PPTX: an OPC package whose content types declare a PresentationML main part
1204    // (or whose content types name PresentationML and whose `officeDocument`
1205    // relationship targets `ppt/presentation.xml`). The positive PresentationML
1206    // signal keeps it mutually exclusive with DOCX and XLSX: a Word/Excel document
1207    // that *embeds* a PowerPoint part declares only the PresentationML embed type
1208    // (`…presentationml.presentation`, not the `.main+xml` main part) and its
1209    // `officeDocument` relationship targets `word/document.xml`/`xl/workbook.xml`,
1210    // so it is never misclassified as PPTX.
1211    #[cfg(feature = "pptx")]
1212    let is_pptx = content_types.as_deref().is_some_and(|ct| {
1213        contains(ct, PPTX_MAIN_FRAGMENT)
1214            || (contains(ct, PPTX_NS_FRAGMENT)
1215                && rels
1216                    .as_deref()
1217                    .is_some_and(|r| contains(r, PPTX_MAIN_TARGET)))
1218    });
1219    #[cfg(not(feature = "pptx"))]
1220    let is_pptx = false;
1221
1222    // The mandatory ODF `mimetype` member is **authoritative** when it names an
1223    // OpenDocument media type (ODF 1.2 §3.2: the package's own media type). An ODP
1224    // that *embeds a spreadsheet* lists both media types in `META-INF/manifest.xml`
1225    // (one per file-entry), so scanning the whole manifest would make it look like an
1226    // ODP *and* an ODS and force the ambiguous case below to Opaque. Only when the
1227    // `mimetype` member is absent or is not an ODF media type do we fall back to the
1228    // manifest scan (for a non-conformant package).
1229    #[cfg(any(feature = "odt", feature = "ods", feature = "odp"))]
1230    let odf_mimetype = mimetype
1231        .as_deref()
1232        .filter(|m| contains(m, ODF_MEDIA_PREFIX));
1233
1234    // ODT: an ODF package whose mandatory `mimetype` (or, absent that,
1235    // `META-INF/manifest.xml`) declares an OpenDocument text media type.
1236    #[cfg(feature = "odt")]
1237    let is_odt = match odf_mimetype {
1238        Some(m) => contains(m, ODT_TEXT_FRAGMENT),
1239        None => member_decoded(&physical, source, ODT_MANIFEST_MEMBER, limits)
1240            .as_deref()
1241            .is_some_and(|m| contains(m, ODT_TEXT_FRAGMENT)),
1242    };
1243    #[cfg(not(feature = "odt"))]
1244    let is_odt = false;
1245
1246    // ODS: an ODF package whose mandatory `mimetype` (or, absent that, the manifest)
1247    // declares an OpenDocument *spreadsheet* media type. The positive spreadsheet
1248    // fragment keeps it mutually exclusive with ODT (a text document declares the
1249    // text media type, never the spreadsheet one).
1250    #[cfg(feature = "ods")]
1251    let is_ods = match odf_mimetype {
1252        Some(m) => contains(m, ODS_SPREADSHEET_FRAGMENT),
1253        None => member_decoded(&physical, source, ODS_MANIFEST_MEMBER, limits)
1254            .as_deref()
1255            .is_some_and(|m| contains(m, ODS_SPREADSHEET_FRAGMENT)),
1256    };
1257    #[cfg(not(feature = "ods"))]
1258    let is_ods = false;
1259
1260    // ODP: an ODF package whose mandatory `mimetype` (or, absent that, the manifest)
1261    // declares an OpenDocument *presentation* media type. The positive presentation
1262    // fragment keeps it mutually exclusive with ODT/ODS.
1263    #[cfg(feature = "odp")]
1264    let is_odp = match odf_mimetype {
1265        Some(m) => contains(m, ODP_PRESENTATION_FRAGMENT),
1266        None => member_decoded(&physical, source, ODP_MANIFEST_MEMBER, limits)
1267            .as_deref()
1268            .is_some_and(|m| contains(m, ODP_PRESENTATION_FRAGMENT)),
1269    };
1270    #[cfg(not(feature = "odp"))]
1271    let is_odp = false;
1272
1273    // A ZIP matching more than one native signature is ambiguous: fail safe.
1274    let matches = [is_docx, is_epub, is_odt, is_ods, is_odp, is_xlsx, is_pptx]
1275        .iter()
1276        .filter(|b| **b)
1277        .count();
1278    match matches {
1279        1 if is_docx => Some(DocumentFormat::Docx),
1280        1 if is_epub => Some(DocumentFormat::Epub),
1281        1 if is_odt => Some(DocumentFormat::Odt),
1282        1 if is_ods => Some(DocumentFormat::Ods),
1283        1 if is_odp => Some(DocumentFormat::Odp),
1284        1 if is_xlsx => Some(DocumentFormat::Xlsx),
1285        1 if is_pptx => Some(DocumentFormat::Pptx),
1286        _ => None,
1287    }
1288}
1289
1290#[cfg(feature = "package")]
1291const EPUB_MIMETYPE_MEMBER: &[u8] = b"mimetype";
1292
1293/// Decode one member's bytes by exact name, bounded and decline-safe: encrypted,
1294/// oversized, unsupported-method, or out-of-range members yield `None`. The name
1295/// match is ASCII case-insensitive, mirroring the OPC layer
1296/// ([`crate::field::opc`]), which resolves the well-known control parts
1297/// (`[Content_Types].xml`, `_rels/.rels`) case-insensitively; a package that
1298/// lowercases them is still identified (Phase 21.15 fix).
1299#[cfg(feature = "package")]
1300fn member_decoded(
1301    physical: &crate::adapter::package::ZipPhysical,
1302    source: &[u8],
1303    name: &[u8],
1304    limits: Limits,
1305) -> Option<Vec<u8>> {
1306    const FLAG_ENCRYPTED: u16 = 0x0001;
1307    let member = physical
1308        .members
1309        .iter()
1310        .find(|m| m.name.eq_ignore_ascii_case(name))?;
1311    if member.flags & FLAG_ENCRYPTED != 0 || member.uncompressed_size > limits.max_xml_part_bytes {
1312        return None;
1313    }
1314    let off = usize::try_from(member.data.0).ok()?;
1315    let len = usize::try_from(member.data.1).ok()?;
1316    let raw = source.get(off..off.checked_add(len)?)?;
1317    match member.method {
1318        0 => Some(raw.to_vec()),
1319        8 => crate::field::derive::inflate_raw_deflate(raw, member.uncompressed_size, limits).ok(),
1320        _ => None,
1321    }
1322}
1323
1324/// Byte-substring search (no allocation, case-sensitive).
1325#[cfg(feature = "package")]
1326fn contains(haystack: &[u8], needle: &[u8]) -> bool {
1327    !needle.is_empty()
1328        && needle.len() <= haystack.len()
1329        && haystack.windows(needle.len()).any(|w| w == needle)
1330}
1331
1332#[cfg(test)]
1333mod tests {
1334    use super::*;
1335
1336    #[test]
1337    fn names_and_provenance_roundtrip() {
1338        for f in [
1339            DocumentFormat::Pdf,
1340            DocumentFormat::Docx,
1341            DocumentFormat::Epub,
1342            DocumentFormat::Odt,
1343            DocumentFormat::Ods,
1344            DocumentFormat::Odp,
1345            DocumentFormat::Xlsx,
1346            DocumentFormat::Pptx,
1347            DocumentFormat::Json,
1348            DocumentFormat::Json5,
1349            DocumentFormat::Yaml,
1350            DocumentFormat::Csv,
1351            DocumentFormat::Markdown,
1352            DocumentFormat::Xml,
1353            DocumentFormat::Html,
1354            DocumentFormat::Toml,
1355            DocumentFormat::Jsonl,
1356            DocumentFormat::Eml,
1357            DocumentFormat::Parquet,
1358            DocumentFormat::ArrowIpc,
1359            DocumentFormat::Cbor,
1360            DocumentFormat::Msgpack,
1361            DocumentFormat::Config,
1362            DocumentFormat::Feed,
1363            DocumentFormat::Geojson,
1364            DocumentFormat::Gis,
1365            DocumentFormat::Notebook,
1366            DocumentFormat::FixedWidth,
1367            DocumentFormat::Rst,
1368            DocumentFormat::Asciidoc,
1369            DocumentFormat::Mdx,
1370            DocumentFormat::Mhtml,
1371            DocumentFormat::Logstream,
1372            DocumentFormat::Pkgmeta,
1373            DocumentFormat::Apispec,
1374            DocumentFormat::Bib,
1375            DocumentFormat::Rtf,
1376            DocumentFormat::Opaque,
1377        ] {
1378            let token = format!("{}field:package;members=1", f.provenance_prefix());
1379            assert_eq!(DocumentFormat::from_provenance(&token), Some(f));
1380        }
1381        assert_eq!(DocumentFormat::from_provenance("field:ingest-b"), None);
1382        assert_eq!(DocumentFormat::from_provenance("format=exotic;x"), None);
1383    }
1384
1385    #[test]
1386    fn plain_bytes_are_opaque() {
1387        assert_eq!(
1388            detect_document_format(b"not a document", Limits::DEFAULT),
1389            DocumentFormat::Opaque
1390        );
1391    }
1392
1393    #[test]
1394    fn a_corpus_pdf_is_detected() {
1395        let (_, pdf) = crate::adapter::pdf::sample_pdfs()
1396            .into_iter()
1397            .next()
1398            .expect("the PDF corpus is non-empty");
1399        assert_eq!(
1400            detect_document_format(&pdf, Limits::DEFAULT),
1401            DocumentFormat::Pdf
1402        );
1403    }
1404}