docling_core/
metadata.rs

1//! Document metadata (#679): what a file says about itself — an EPUB's
2//! Dublin Core, an Office package's core and application properties, an
3//! ODF `meta.xml`, a photo's EXIF, an audio file's tags, a PDF's Info
4//! dictionary — read into [`DocumentMetadata`] on the document, and written
5//! out on request as a `metadata` object in the JSON or as YAML front
6//! matter at the top of the Markdown ([`MetadataOutput`]). Nothing is
7//! written by default, so every export stays byte for byte what it was.
8//!
9//! The common fields are the ones every source has a name for (a title,
10//! authors, a subject, keywords, dates, the tool that wrote the file); what
11//! only one kind of file carries keeps its own names in a section of its
12//! own: `exif` (the tag names as exiftool prints them: `DateTimeOriginal`,
13//! `Make`, `Model`, …), `audio` (`album`, `artist`, `genre`, `duration`,
14//! …), `properties` (the rest of an Office package's properties:
15//! `revision`, `category`, `pages`, …) and a decoded `gps` position.
16//!
17//! Under the PII pass (`redact_pii`) the author fields are masked as
18//! persons and the position is dropped; see [`crate::redact`].
19
20use std::collections::BTreeMap;
21use std::fmt::Write as _;
22
23use serde::{Deserialize, Serialize};
24
25/// Where the metadata goes in the exports. `None` (the default) keeps it to
26/// the in-memory document; `Json` adds a `metadata` object to the docling
27/// JSON (after `origin`); `Frontmatter` opens the Markdown with a YAML front
28/// matter block as well.
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
30pub enum MetadataOutput {
31    #[default]
32    None,
33    Json,
34    Frontmatter,
35}
36
37impl MetadataOutput {
38    /// The option's accepted spellings, for error messages.
39    pub const ACCEPTED: &'static str = "none, json, frontmatter";
40
41    /// Parse the wire spelling (case-insensitive; `front_matter` and
42    /// `front-matter` read as `frontmatter`).
43    pub fn parse(s: &str) -> Option<Self> {
44        match s
45            .trim()
46            .to_ascii_lowercase()
47            .replace(['_', '-'], "")
48            .as_str()
49        {
50            "none" => Some(Self::None),
51            "json" => Some(Self::Json),
52            "frontmatter" => Some(Self::Frontmatter),
53            _ => None,
54        }
55    }
56
57    pub fn as_str(self) -> &'static str {
58        match self {
59            Self::None => "none",
60            Self::Json => "json",
61            Self::Frontmatter => "frontmatter",
62        }
63    }
64
65    /// Whether the JSON export carries the `metadata` object.
66    pub fn in_json(self) -> bool {
67        self != Self::None
68    }
69
70    /// Whether the Markdown export opens with the front matter.
71    pub fn front_matter(self) -> bool {
72        self == Self::Frontmatter
73    }
74}
75
76/// A decoded GPS position (EXIF `GPSLatitude` / `GPSLongitude` /
77/// `GPSAltitude` with their reference hemispheres applied): signed decimal
78/// degrees, metres above sea level.
79#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
80pub struct GpsPosition {
81    pub latitude: f64,
82    pub longitude: f64,
83    #[serde(default, skip_serializing_if = "Option::is_none")]
84    pub altitude: Option<f64>,
85}
86
87/// The metadata of a document. Every field is optional; a document whose
88/// file carries nothing has the default, [`is_empty`](Self::is_empty).
89/// Dates are kept as the file wrote them, normalized to ISO 8601 where the
90/// format defines one (Office and ODF `dcterms` dates, a PDF's `D:` dates);
91/// EXIF's own `YYYY:MM:DD HH:MM:SS` stays as it is in `exif` and is turned
92/// into ISO 8601 for `date`.
93#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
94#[serde(default)]
95pub struct DocumentMetadata {
96    #[serde(skip_serializing_if = "Option::is_none")]
97    pub title: Option<String>,
98    #[serde(skip_serializing_if = "Vec::is_empty")]
99    pub authors: Vec<String>,
100    #[serde(skip_serializing_if = "Option::is_none")]
101    pub subject: Option<String>,
102    #[serde(skip_serializing_if = "Option::is_none")]
103    pub description: Option<String>,
104    #[serde(skip_serializing_if = "Vec::is_empty")]
105    pub keywords: Vec<String>,
106    #[serde(skip_serializing_if = "Option::is_none")]
107    pub language: Option<String>,
108    #[serde(skip_serializing_if = "Option::is_none")]
109    pub publisher: Option<String>,
110    #[serde(skip_serializing_if = "Option::is_none")]
111    pub identifier: Option<String>,
112    #[serde(skip_serializing_if = "Option::is_none")]
113    pub rights: Option<String>,
114    /// The document's own date: an EPUB's `dc:date`, a photo's
115    /// `DateTimeOriginal`, an audio file's `date`.
116    #[serde(skip_serializing_if = "Option::is_none")]
117    pub date: Option<String>,
118    #[serde(skip_serializing_if = "Option::is_none")]
119    pub created: Option<String>,
120    #[serde(skip_serializing_if = "Option::is_none")]
121    pub modified: Option<String>,
122    #[serde(skip_serializing_if = "Option::is_none")]
123    pub last_modified_by: Option<String>,
124    /// The application that wrote the file (Office's `Application`, ODF's
125    /// `meta:generator`, EXIF's `Software`, a PDF's `Producer`).
126    #[serde(skip_serializing_if = "Option::is_none")]
127    pub creator_tool: Option<String>,
128    /// The source's other named properties, by a lower-case name
129    /// (`revision`, `category`, `content_status`, `company`, `pages`,
130    /// `words`, `image_size`, …).
131    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
132    pub properties: BTreeMap<String, String>,
133    #[serde(skip_serializing_if = "Option::is_none")]
134    pub gps: Option<GpsPosition>,
135    /// EXIF tags by their exiftool name, each value as it displays
136    /// (`Make`, `Model`, `DateTimeOriginal`, `ExposureTime`, `GPSLatitude`,
137    /// …); binary tags (`MakerNote`, the thumbnail) are left out.
138    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
139    pub exif: BTreeMap<String, String>,
140    /// An audio file's tags and stream facts by a lower-case name
141    /// (`title`, `artist`, `album`, `genre`, `track`, `date`, `duration`,
142    /// `sample_rate`, `channels`, `codec`, …).
143    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
144    pub audio: BTreeMap<String, String>,
145}
146
147impl DocumentMetadata {
148    pub fn is_empty(&self) -> bool {
149        *self == Self::default()
150    }
151
152    /// The JSON object the docling JSON export carries as `metadata`
153    /// (empty fields left out).
154    pub fn to_json(&self) -> serde_json::Value {
155        serde_json::to_value(self).unwrap_or(serde_json::Value::Null)
156    }
157
158    /// Read back what [`to_json`](Self::to_json) wrote (unknown keys are
159    /// ignored, missing ones default).
160    pub fn from_json(value: &serde_json::Value) -> Self {
161        serde_json::from_value(value.clone()).unwrap_or_default()
162    }
163
164    /// The YAML front matter block — `---`, one line per present field in
165    /// a fixed order, `---` — or an empty string for empty metadata.
166    /// Scalars print plain when YAML would read them back as the same
167    /// string (`authors: [Test Author]`) and double-quoted otherwise (a
168    /// date, a number, a colon, a leading symbol), so a front-matter parser
169    /// gets the strings the file held.
170    pub fn to_front_matter(&self) -> String {
171        if self.is_empty() {
172            return String::new();
173        }
174        let mut out = String::from("---\n");
175        fn scalar(out: &mut String, key: &str, value: &Option<String>) {
176            if let Some(v) = value {
177                let _ = writeln!(out, "{}: {}", yaml_key(key), yaml_scalar(v));
178            }
179        }
180        scalar(&mut out, "title", &self.title);
181        yaml_list(&mut out, "authors", &self.authors);
182        scalar(&mut out, "subject", &self.subject);
183        scalar(&mut out, "description", &self.description);
184        yaml_list(&mut out, "keywords", &self.keywords);
185        scalar(&mut out, "language", &self.language);
186        scalar(&mut out, "publisher", &self.publisher);
187        scalar(&mut out, "identifier", &self.identifier);
188        scalar(&mut out, "rights", &self.rights);
189        scalar(&mut out, "date", &self.date);
190        scalar(&mut out, "created", &self.created);
191        scalar(&mut out, "modified", &self.modified);
192        scalar(&mut out, "last_modified_by", &self.last_modified_by);
193        scalar(&mut out, "creator_tool", &self.creator_tool);
194        yaml_map(&mut out, "properties", &self.properties);
195        if let Some(gps) = &self.gps {
196            let _ = write!(
197                out,
198                "gps: {{latitude: {}, longitude: {}",
199                gps.latitude, gps.longitude
200            );
201            if let Some(alt) = gps.altitude {
202                let _ = write!(out, ", altitude: {alt}");
203            }
204            out.push_str("}\n");
205        }
206        yaml_map(&mut out, "exif", &self.exif);
207        yaml_map(&mut out, "audio", &self.audio);
208        out.push_str("---\n");
209        out
210    }
211}
212
213fn yaml_list(out: &mut String, key: &str, items: &[String]) {
214    if items.is_empty() {
215        return;
216    }
217    // A flow list when every item prints plain and holds no `,`; a block
218    // list otherwise (a quoted item may hold anything).
219    if items
220        .iter()
221        .all(|i| yaml_plain(i) && !i.contains([',', '[', ']']))
222    {
223        let _ = writeln!(out, "{}: [{}]", yaml_key(key), items.join(", "));
224    } else {
225        let _ = writeln!(out, "{}:", yaml_key(key));
226        for i in items {
227            let _ = writeln!(out, "  - {}", yaml_scalar(i));
228        }
229    }
230}
231
232fn yaml_map(out: &mut String, key: &str, map: &BTreeMap<String, String>) {
233    if map.is_empty() {
234        return;
235    }
236    let _ = writeln!(out, "{}:", yaml_key(key));
237    for (k, v) in map {
238        let _ = writeln!(out, "  {}: {}", yaml_key(k), yaml_scalar(v));
239    }
240}
241
242/// A key as YAML reads it back: plain for the usual identifier shapes,
243/// quoted for anything else.
244fn yaml_key(key: &str) -> String {
245    if !key.is_empty()
246        && key
247            .chars()
248            .all(|c| c.is_alphanumeric() || c == '_' || c == '-' || c == '.')
249        && key.starts_with(|c: char| c.is_alphabetic() || c == '_')
250    {
251        key.to_string()
252    } else {
253        yaml_quote(key)
254    }
255}
256
257/// Whether `s` can print as a plain scalar and read back unchanged: it
258/// starts with a letter, holds none of YAML's indicators or line breaks,
259/// and is not one of the words YAML 1.1 reads as a boolean or null.
260fn yaml_plain(s: &str) -> bool {
261    let starts_ok = s.starts_with(|c: char| c.is_alphabetic());
262    let clean = !s.contains(['\n', '\r', '\t', '"', '\\', '#', '`', '{', '}'])
263        && !s.contains(": ")
264        && !s.ends_with(':')
265        && !s.ends_with(' ');
266    let word = s.to_ascii_lowercase();
267    let reserved = matches!(
268        word.as_str(),
269        "true" | "false" | "null" | "yes" | "no" | "on" | "off" | "y" | "n"
270    );
271    starts_ok && clean && !reserved
272}
273
274fn yaml_scalar(s: &str) -> String {
275    if yaml_plain(s) {
276        s.to_string()
277    } else {
278        yaml_quote(s)
279    }
280}
281
282/// A double-quoted YAML scalar: `\` and `"` escaped, control characters
283/// as `\n`, `\t` or `\uXXXX`.
284fn yaml_quote(s: &str) -> String {
285    let mut out = String::with_capacity(s.len() + 2);
286    out.push('"');
287    for c in s.chars() {
288        match c {
289            '"' => out.push_str("\\\""),
290            '\\' => out.push_str("\\\\"),
291            '\n' => out.push_str("\\n"),
292            '\r' => out.push_str("\\r"),
293            '\t' => out.push_str("\\t"),
294            c if c.is_control() => {
295                let _ = write!(out, "\\u{:04X}", c as u32);
296            }
297            c => out.push(c),
298        }
299    }
300    out.push('"');
301    out
302}
303
304#[cfg(test)]
305mod tests {
306    use super::*;
307
308    #[test]
309    fn front_matter_prints_plain_where_safe_and_quotes_the_rest() {
310        let mut m = DocumentMetadata {
311            title: Some("Test EPUB Document".into()),
312            authors: vec!["Test Author".into()],
313            language: Some("en".into()),
314            date: Some("2024-12-18".into()),
315            description: Some("A line: with a colon, and \"quotes\"".into()),
316            ..Default::default()
317        };
318        m.exif
319            .insert("DateTimeOriginal".into(), "2024:01:02 03:04:05".into());
320        m.properties.insert("revision".into(), "3".into());
321        m.gps = Some(GpsPosition {
322            latitude: 48.8566,
323            longitude: 2.3522,
324            altitude: None,
325        });
326        assert_eq!(
327            m.to_front_matter(),
328            "---\n\
329             title: Test EPUB Document\n\
330             authors: [Test Author]\n\
331             description: \"A line: with a colon, and \\\"quotes\\\"\"\n\
332             language: en\n\
333             date: \"2024-12-18\"\n\
334             properties:\n  revision: \"3\"\n\
335             gps: {latitude: 48.8566, longitude: 2.3522}\n\
336             exif:\n  DateTimeOriginal: \"2024:01:02 03:04:05\"\n\
337             ---\n"
338        );
339    }
340
341    #[test]
342    fn lists_with_commas_or_quotes_go_block_style() {
343        let m = DocumentMetadata {
344            authors: vec!["Smith, John".into(), "Ada".into()],
345            keywords: vec!["yes".into()],
346            ..Default::default()
347        };
348        assert_eq!(
349            m.to_front_matter(),
350            "---\nauthors:\n  - Smith, John\n  - Ada\nkeywords:\n  - \"yes\"\n---\n"
351        );
352        assert!(DocumentMetadata::default().to_front_matter().is_empty());
353        assert!(DocumentMetadata::default().is_empty());
354    }
355
356    #[test]
357    fn json_round_trips_and_leaves_empty_fields_out() {
358        let m = DocumentMetadata {
359            title: Some("T".into()),
360            authors: vec!["A".into()],
361            ..Default::default()
362        };
363        let j = m.to_json();
364        assert_eq!(j, serde_json::json!({"title": "T", "authors": ["A"]}));
365        assert_eq!(DocumentMetadata::from_json(&j), m);
366        assert_eq!(
367            DocumentMetadata::from_json(&serde_json::json!({"title": "T", "unknown": 1})).title,
368            Some("T".into())
369        );
370    }
371
372    #[test]
373    fn the_output_mode_parses_its_spellings() {
374        assert_eq!(
375            MetadataOutput::parse("Front-Matter"),
376            Some(MetadataOutput::Frontmatter)
377        );
378        assert_eq!(MetadataOutput::parse("json"), Some(MetadataOutput::Json));
379        assert_eq!(MetadataOutput::parse("none"), Some(MetadataOutput::None));
380        assert_eq!(MetadataOutput::parse("yaml"), None);
381        assert!(
382            MetadataOutput::Frontmatter.in_json() && MetadataOutput::Frontmatter.front_matter()
383        );
384        assert!(MetadataOutput::Json.in_json() && !MetadataOutput::Json.front_matter());
385        assert!(!MetadataOutput::None.in_json());
386    }
387}
docling_core/metadata.rs

docling_core/
metadata.rs
