vole_document/field/
document_format.rs

1//! Byte-based document-format detection for the universal observation API
2//! (Phase 12.7, ADR-0031, plan §DEC-5).
3//!
4//! The format of a field is decided from its **source bytes**, never from a file
5//! name or extension. Detection is deliberately conservative: an ambiguous or
6//! malformed input falls back to [`DocumentFormat::Opaque`] rather than guessing,
7//! and a format whose adapter is not compiled in cannot be detected (the input is
8//! then `Opaque`), so the reported capability set always matches what the build
9//! can actually serve.
10//!
11//! * **PDF** — the byte-authoritative physical scanner admits a validated PDF
12//!   ([`crate::adapter::pdf::detect`]).
13//! * **DOCX** — a valid ZIP that also carries the OPC content-types part
14//!   (`[Content_Types].xml`) and a package `officeDocument` relationship.
15//! * **EPUB** — a valid ZIP that is an OCF container: the mandatory stored
16//!   `mimetype` member equals `application/epub+zip`, or `META-INF/container.xml`
17//!   names the OCF namespace and an OPF (`application/oebps-package+xml`) rootfile.
18//! * **ODT** — a valid ZIP that is an OpenDocument (ODF) package: the mandatory
19//!   stored `mimetype` member is an OpenDocument *text* media type, or
20//!   `META-INF/manifest.xml` declares one.
21//! * **Opaque** — everything else, including a ZIP that matches none of the above
22//!   (or more than one — an ambiguous ZIP fails safe).
23//!
24//! The detected format is recorded in the field manifest's provenance (a
25//! machine-readable `format=<name>;` prefix, see [`DocumentFormat::from_provenance`]),
26//! so `observe`/`find`/`explain` can dispatch common selectors without reading the
27//! whole source again.
28
29use crate::limits::Limits;
30
31/// The mandatory OCF `mimetype` payload.
32pub const EPUB_MIMETYPE: &[u8] = b"application/epub+zip";
33/// The mandatory OCF container descriptor member.
34pub const CONTAINER_MEMBER: &[u8] = b"META-INF/container.xml";
35/// The OCF container-descriptor namespace.
36pub const CONTAINER_NS: &[u8] = b"urn:oasis:names:tc:opendocument:xmlns:container";
37/// The default (and only normative) package-document media type.
38pub const OPF_MEDIA_TYPE: &[u8] = b"application/oebps-package+xml";
39/// The OPC content-types part.
40#[cfg(feature = "package")]
41const CONTENT_TYPES_MEMBER: &[u8] = b"[Content_Types].xml";
42/// The OPC package-relationships part.
43#[cfg(feature = "package")]
44const PACKAGE_RELS_MEMBER: &[u8] = b"_rels/.rels";
45/// The `officeDocument` relationship type fragment (transitional and strict).
46#[cfg(feature = "package")]
47const OFFICE_DOCUMENT_FRAGMENT: &[u8] = b"officeDocument";
48/// The ODF package manifest member (Phase 13.3).
49#[cfg(feature = "odt")]
50const ODT_MANIFEST_MEMBER: &[u8] = b"META-INF/manifest.xml";
51/// The OpenDocument *text* media-type fragment (Phase 13.3).
52#[cfg(feature = "odt")]
53const ODT_TEXT_FRAGMENT: &[u8] = b"application/vnd.oasis.opendocument.text";
54
55/// A detected document format (the class of the field's source bytes).
56#[derive(Debug, Clone, Copy, PartialEq, Eq)]
57pub enum DocumentFormat {
58    /// A validated PDF (physical indicators).
59    Pdf,
60    /// An OPC package with a WordprocessingML `officeDocument` part.
61    Docx,
62    /// An OCF container with an EPUB package document.
63    Epub,
64    /// An ODF package with an OpenDocument text content part.
65    Odt,
66    /// Anything else; preserved exactly by the opaque floor.
67    Opaque,
68}
69
70impl DocumentFormat {
71    /// Stable lower-case name (used in JSON and in the manifest provenance token).
72    pub const fn name(self) -> &'static str {
73        match self {
74            DocumentFormat::Pdf => "pdf",
75            DocumentFormat::Docx => "docx",
76            DocumentFormat::Epub => "epub",
77            DocumentFormat::Odt => "odt",
78            DocumentFormat::Opaque => "opaque",
79        }
80    }
81
82    /// The adapter that serves this format (the observation layer's name for it).
83    pub const fn adapter(self) -> &'static str {
84        match self {
85            DocumentFormat::Pdf => "pdf",
86            DocumentFormat::Docx => "docx",
87            DocumentFormat::Epub => "epub",
88            DocumentFormat::Odt => "odt",
89            DocumentFormat::Opaque => "opaque",
90        }
91    }
92
93    /// Whether the adapter for this format is compiled into this build.
94    pub const fn compiled(self) -> bool {
95        match self {
96            DocumentFormat::Pdf | DocumentFormat::Opaque => true,
97            DocumentFormat::Docx => cfg!(feature = "docx"),
98            DocumentFormat::Epub => cfg!(feature = "epub"),
99            DocumentFormat::Odt => cfg!(feature = "odt"),
100        }
101    }
102
103    /// The machine-readable `format=<name>;` provenance prefix recorded at ingest.
104    pub fn provenance_prefix(self) -> String {
105        format!("format={};", self.name())
106    }
107
108    /// Recover the recorded format from a field manifest's provenance string.
109    ///
110    /// Returns `None` for a manifest that does not carry the token (e.g. a field
111    /// written before this subphase); such a field still serves every native
112    /// selector, but common observations decline typed rather than guessing.
113    pub fn from_provenance(provenance: &str) -> Option<DocumentFormat> {
114        let rest = provenance.strip_prefix("format=")?;
115        let name = rest.split(';').next()?;
116        match name {
117            "pdf" => Some(DocumentFormat::Pdf),
118            "docx" => Some(DocumentFormat::Docx),
119            "epub" => Some(DocumentFormat::Epub),
120            "odt" => Some(DocumentFormat::Odt),
121            "opaque" => Some(DocumentFormat::Opaque),
122            _ => None,
123        }
124    }
125}
126
127/// Detect the document format of `source` from its bytes alone.
128///
129/// Never consults a file name or extension. A malformed or ambiguous input (or a
130/// format whose adapter is not compiled) falls back to [`DocumentFormat::Opaque`].
131pub fn detect_document_format(source: &[u8], limits: Limits) -> DocumentFormat {
132    if crate::adapter::pdf::detect(source, limits) {
133        return DocumentFormat::Pdf;
134    }
135    #[cfg(feature = "package")]
136    {
137        if let Some(format) = detect_zip_family(source, limits) {
138            return format;
139        }
140    }
141    DocumentFormat::Opaque
142}
143
144/// Whether `source` is a structurally valid ZIP archive.
145///
146/// Used by the universal ingest dispatcher so a generic ZIP (which detection
147/// reports as `Opaque`) is still inverted through the byte-authoritative package
148/// layer rather than the opaque floor.
149#[cfg(feature = "package")]
150pub fn is_zip(source: &[u8], limits: Limits) -> bool {
151    crate::adapter::package::scan(source, limits).is_ok()
152}
153
154#[cfg(feature = "package")]
155fn detect_zip_family(source: &[u8], limits: Limits) -> Option<DocumentFormat> {
156    let physical = crate::adapter::package::scan(source, limits).ok()?;
157
158    // EPUB (OCF): the mandatory `mimetype` member, or an OCF container that
159    // resolves to an OPF rootfile.
160    let mimetype = member_decoded(&physical, source, EPUB_MIMETYPE_MEMBER, limits);
161    let mimetype_ok = mimetype.as_deref() == Some(EPUB_MIMETYPE);
162    let container = member_decoded(&physical, source, CONTAINER_MEMBER, limits);
163    let container_ok = container
164        .as_deref()
165        .is_some_and(|c| contains(c, CONTAINER_NS) && contains(c, OPF_MEDIA_TYPE));
166    let is_epub = mimetype_ok || container_ok;
167
168    // DOCX: an OPC package (`[Content_Types].xml`) whose package relationships
169    // declare an `officeDocument` part.
170    let content_types = member_decoded(&physical, source, CONTENT_TYPES_MEMBER, limits);
171    let rels = member_decoded(&physical, source, PACKAGE_RELS_MEMBER, limits);
172    let is_docx = content_types.is_some()
173        && rels
174            .as_deref()
175            .is_some_and(|r| contains(r, OFFICE_DOCUMENT_FRAGMENT));
176
177    // ODT: an ODF package whose mandatory `mimetype` (or `META-INF/manifest.xml`)
178    // declares an OpenDocument text media type.
179    #[cfg(feature = "odt")]
180    let is_odt = mimetype
181        .as_deref()
182        .is_some_and(|m| contains(m, ODT_TEXT_FRAGMENT))
183        || member_decoded(&physical, source, ODT_MANIFEST_MEMBER, limits)
184            .as_deref()
185            .is_some_and(|m| contains(m, ODT_TEXT_FRAGMENT));
186    #[cfg(not(feature = "odt"))]
187    let is_odt = false;
188
189    // A ZIP matching more than one native signature is ambiguous: fail safe.
190    let matches = [is_docx, is_epub, is_odt].iter().filter(|b| **b).count();
191    match matches {
192        1 if is_docx => Some(DocumentFormat::Docx),
193        1 if is_epub => Some(DocumentFormat::Epub),
194        1 if is_odt => Some(DocumentFormat::Odt),
195        _ => None,
196    }
197}
198
199#[cfg(feature = "package")]
200const EPUB_MIMETYPE_MEMBER: &[u8] = b"mimetype";
201
202/// Decode one member's bytes by exact name, bounded and decline-safe: encrypted,
203/// oversized, unsupported-method, or out-of-range members yield `None`.
204#[cfg(feature = "package")]
205fn member_decoded(
206    physical: &crate::adapter::package::ZipPhysical,
207    source: &[u8],
208    name: &[u8],
209    limits: Limits,
210) -> Option<Vec<u8>> {
211    const FLAG_ENCRYPTED: u16 = 0x0001;
212    let member = physical.members.iter().find(|m| m.name == name)?;
213    if member.flags & FLAG_ENCRYPTED != 0 || member.uncompressed_size > limits.max_xml_part_bytes {
214        return None;
215    }
216    let off = usize::try_from(member.data.0).ok()?;
217    let len = usize::try_from(member.data.1).ok()?;
218    let raw = source.get(off..off.checked_add(len)?)?;
219    match member.method {
220        0 => Some(raw.to_vec()),
221        8 => crate::field::derive::inflate_raw_deflate(raw, member.uncompressed_size, limits).ok(),
222        _ => None,
223    }
224}
225
226/// Byte-substring search (no allocation, case-sensitive).
227#[cfg(feature = "package")]
228fn contains(haystack: &[u8], needle: &[u8]) -> bool {
229    !needle.is_empty()
230        && needle.len() <= haystack.len()
231        && haystack.windows(needle.len()).any(|w| w == needle)
232}
233
234#[cfg(test)]
235mod tests {
236    use super::*;
237
238    #[test]
239    fn names_and_provenance_roundtrip() {
240        for f in [
241            DocumentFormat::Pdf,
242            DocumentFormat::Docx,
243            DocumentFormat::Epub,
244            DocumentFormat::Opaque,
245        ] {
246            let token = format!("{}field:package;members=1", f.provenance_prefix());
247            assert_eq!(DocumentFormat::from_provenance(&token), Some(f));
248        }
249        assert_eq!(DocumentFormat::from_provenance("field:ingest-b"), None);
250        assert_eq!(DocumentFormat::from_provenance("format=exotic;x"), None);
251    }
252
253    #[test]
254    fn plain_bytes_are_opaque() {
255        assert_eq!(
256            detect_document_format(b"not a document", Limits::DEFAULT),
257            DocumentFormat::Opaque
258        );
259    }
260
261    #[test]
262    fn a_corpus_pdf_is_detected() {
263        let (_, pdf) = crate::adapter::pdf::sample_pdfs()
264            .into_iter()
265            .next()
266            .expect("the PDF corpus is non-empty");
267        assert_eq!(
268            detect_document_format(&pdf, Limits::DEFAULT),
269            DocumentFormat::Pdf
270        );
271    }
272}
vole_document/field/document_format.rs

vole_document/field/
document_format.rs
