cloudiful_docling_convert/
document.rs

1use std::path::Path;
2use std::str::FromStr;
3
4use bytes::Bytes;
5use serde::{Deserialize, Serialize};
6use serde_json::Value;
7
8use crate::error::{PdfConvertError, Result};
9use crate::models::{Bookmark, ChunkMetadata};
10
11mod input_kind;
12
13pub use input_kind::InputKind;
14
15#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
16pub enum OutputFormat {
17    Json,
18    Md,
19    Text,
20    Html,
21    Doctags,
22}
23
24impl OutputFormat {
25    pub fn as_api_value(self) -> &'static str {
26        match self {
27            Self::Json => "json",
28            Self::Md => "md",
29            Self::Text => "text",
30            Self::Html => "html",
31            Self::Doctags => "doctags",
32        }
33    }
34
35    pub fn extension(self) -> &'static str {
36        self.as_api_value()
37    }
38}
39
40impl std::fmt::Display for OutputFormat {
41    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
42        f.write_str(self.as_api_value())
43    }
44}
45
46impl FromStr for OutputFormat {
47    type Err = PdfConvertError;
48
49    fn from_str(value: &str) -> Result<Self> {
50        match value {
51            "json" => Ok(Self::Json),
52            "md" => Ok(Self::Md),
53            "text" => Ok(Self::Text),
54            "html" => Ok(Self::Html),
55            "doctags" => Ok(Self::Doctags),
56            other => Err(PdfConvertError::validation_error(
57                "format",
58                format!("unsupported output format: {}", other),
59            )),
60        }
61    }
62}
63
64#[derive(Debug, Clone)]
65pub struct InputDocument {
66    pub filename: String,
67    pub media_type: String,
68    pub bytes: Bytes,
69    pub input_kind_override: Option<InputKind>,
70}
71
72impl InputDocument {
73    pub fn new(
74        filename: impl Into<String>,
75        media_type: impl Into<String>,
76        bytes: impl Into<Bytes>,
77    ) -> Self {
78        Self {
79            filename: filename.into(),
80            media_type: media_type.into(),
81            bytes: bytes.into(),
82            input_kind_override: None,
83        }
84    }
85
86    pub fn with_input_kind(mut self, input_kind: InputKind) -> Self {
87        self.input_kind_override = Some(input_kind);
88        self
89    }
90
91    pub fn from_path_and_bytes(path: &Path, bytes: impl Into<Bytes>) -> Result<Self> {
92        let filename = path
93            .file_name()
94            .and_then(|name| name.to_str())
95            .ok_or_else(|| {
96                PdfConvertError::validation_error(
97                    "input_path",
98                    format!("path '{}' does not have a valid file name", path.display()),
99                )
100            })?;
101        let kind = InputKind::from_path(path).ok_or_else(|| {
102            PdfConvertError::validation_error(
103                "input_path",
104                format!("unsupported file type for '{}'", path.display()),
105            )
106        })?;
107
108        Ok(Self::new(
109            filename,
110            kind.canonical_media_type(filename, None),
111            bytes,
112        ))
113    }
114
115    pub fn from_path_and_bytes_with_kind(
116        path: &Path,
117        bytes: impl Into<Bytes>,
118        input_kind: InputKind,
119    ) -> Result<Self> {
120        let filename = path
121            .file_name()
122            .and_then(|name| name.to_str())
123            .ok_or_else(|| {
124                PdfConvertError::validation_error(
125                    "input_path",
126                    format!("path '{}' does not have a valid file name", path.display()),
127                )
128            })?;
129
130        Ok(Self::new(
131            filename,
132            input_kind.canonical_media_type(filename, None),
133            bytes,
134        )
135        .with_input_kind(input_kind))
136    }
137
138    pub fn kind(&self) -> Result<InputKind> {
139        if let Some(input_kind) = self.input_kind_override {
140            return Ok(input_kind);
141        }
142
143        InputKind::from_filename_and_media_type(&self.filename, Some(&self.media_type))
144            .ok_or_else(|| {
145                let reason = if InputKind::requires_explicit_override(
146                    &self.filename,
147                    Some(&self.media_type),
148                ) {
149                    format!(
150                        "ambiguous input type for '{}' ({}); provide an explicit input_format override",
151                        self.filename, self.media_type
152                    )
153                } else {
154                    format!(
155                        "unsupported input type for '{}' ({})",
156                        self.filename, self.media_type
157                    )
158                };
159                PdfConvertError::validation_error(
160                    "input",
161                    reason,
162                )
163            })
164    }
165}
166
167#[derive(Debug, Clone)]
168pub struct ConvertRequest {
169    pub input: InputDocument,
170    pub output_formats: Vec<OutputFormat>,
171    pub options: ConvertOptions,
172}
173
174impl ConvertRequest {
175    pub fn validate(&self) -> Result<InputKind> {
176        let kind = self.input.kind()?;
177        match (&self.options, kind) {
178            (ConvertOptions::Pdf(_), InputKind::Pdf)
179            | (ConvertOptions::Text(_), InputKind::Text) => {}
180            (ConvertOptions::Generic(_), _) if kind.uses_generic_convert_options() => {}
181            (_, InputKind::Pdf) => {
182                return Err(PdfConvertError::validation_error(
183                    "options",
184                    "PDF input requires Pdf convert options",
185                ));
186            }
187            (_, InputKind::Text) => {
188                return Err(PdfConvertError::validation_error(
189                    "options",
190                    "txt input requires TextConvertOptions",
191                ));
192            }
193            _ => {
194                return Err(PdfConvertError::validation_error(
195                    "options",
196                    "non-pdf, non-text input requires GenericFileConvertOptions",
197                ));
198            }
199        }
200
201        if self.output_formats.is_empty() {
202            return Err(PdfConvertError::validation_error(
203                "output_formats",
204                "at least one output format is required",
205            ));
206        }
207
208        if let ConvertOptions::Pdf(options) = &self.options {
209            options.validate()?;
210        }
211
212        Ok(kind)
213    }
214}
215
216#[derive(Debug, Clone)]
217pub enum ConvertOptions {
218    Pdf(PdfConvertOptions),
219    Generic(GenericFileConvertOptions),
220    Text(TextConvertOptions),
221}
222
223#[derive(Debug, Clone)]
224pub struct PdfConvertOptions {
225    pub pages_per_file: u32,
226    pub split_input: bool,
227    pub split_by_bookmark: bool,
228    pub chunking: bool,
229    pub batch_size: usize,
230}
231
232impl Default for PdfConvertOptions {
233    fn default() -> Self {
234        Self {
235            pages_per_file: 5,
236            split_input: true,
237            split_by_bookmark: false,
238            chunking: false,
239            batch_size: 2,
240        }
241    }
242}
243
244impl PdfConvertOptions {
245    pub fn validate(&self) -> Result<()> {
246        if self.pages_per_file == 0 {
247            return Err(PdfConvertError::validation_error(
248                "pages_per_file",
249                "value must be 1 or greater",
250            ));
251        }
252
253        if self.batch_size == 0 {
254            return Err(PdfConvertError::validation_error(
255                "batch_size",
256                "value must be 1 or greater",
257            ));
258        }
259
260        Ok(())
261    }
262}
263
264#[derive(Debug, Clone, Default)]
265pub struct GenericFileConvertOptions {
266    pub chunking: bool,
267}
268
269#[derive(Debug, Clone)]
270pub struct TextConvertOptions {
271    pub normalize_line_endings: bool,
272    pub trim_utf8_bom: bool,
273}
274
275impl Default for TextConvertOptions {
276    fn default() -> Self {
277        Self {
278            normalize_line_endings: true,
279            trim_utf8_bom: true,
280        }
281    }
282}
283
284#[derive(Debug, Clone, Serialize, Deserialize)]
285pub struct ConvertedChunk {
286    pub metadata: Option<ChunkMetadata>,
287    pub markdown: Option<String>,
288    pub text: Option<String>,
289    pub json: Option<Value>,
290    pub html: Option<String>,
291    pub doctags: Option<String>,
292    pub raw_result: Value,
293}
294
295#[derive(Debug, Clone, Serialize, Deserialize)]
296pub struct ConvertedDocumentMetadata {
297    pub input_kind: InputKind,
298    pub media_type: String,
299    pub page_count: Option<u32>,
300    pub outlines: Vec<Bookmark>,
301}
302
303#[derive(Debug, Clone, Serialize, Deserialize)]
304pub struct ConvertedDocument {
305    pub filename: String,
306    pub markdown: Option<String>,
307    pub text: Option<String>,
308    pub json: Option<Value>,
309    pub html: Option<String>,
310    pub doctags: Option<String>,
311    pub chunks: Vec<ConvertedChunk>,
312    pub metadata: ConvertedDocumentMetadata,
313    pub errors: Vec<String>,
314}
315
316#[derive(Debug, Clone)]
317pub struct FileConvertRequest {
318    pub request: ConvertRequest,
319    pub output_dir: std::path::PathBuf,
320    pub selected_output: OutputFormat,
321    pub overwrite: bool,
322}
323
324#[derive(Debug, Clone)]
325pub struct ConvertedFile {
326    pub document: ConvertedDocument,
327    pub output_paths: Vec<std::path::PathBuf>,
328}
329
330pub fn supported_input_kind(path: &Path) -> bool {
331    InputKind::from_path(path).is_some()
332}
333
334#[cfg(test)]
335mod tests {
336    use super::*;
337
338    #[test]
339    fn output_format_supports_html_and_doctags() {
340        assert_eq!("html".parse::<OutputFormat>().unwrap(), OutputFormat::Html);
341        assert_eq!(
342            "doctags".parse::<OutputFormat>().unwrap(),
343            OutputFormat::Doctags
344        );
345        assert_eq!(OutputFormat::Html.extension(), "html");
346        assert_eq!(OutputFormat::Doctags.extension(), "doctags");
347    }
348
349    #[test]
350    fn request_validation_rejects_mismatched_options() {
351        let request = ConvertRequest {
352            input: InputDocument::new("a.txt", "text/plain", Bytes::from_static(b"hello")),
353            output_formats: vec![OutputFormat::Text],
354            options: ConvertOptions::Generic(GenericFileConvertOptions::default()),
355        };
356
357        let err = request.validate().unwrap_err();
358        assert!(err.to_string().contains("TextConvertOptions"));
359    }
360
361    #[test]
362    fn request_validation_rejects_zero_pages_per_file() {
363        let request = ConvertRequest {
364            input: InputDocument::new("a.pdf", "application/pdf", Bytes::from_static(b"%PDF")),
365            output_formats: vec![OutputFormat::Text],
366            options: ConvertOptions::Pdf(PdfConvertOptions {
367                pages_per_file: 0,
368                ..PdfConvertOptions::default()
369            }),
370        };
371
372        let err = request.validate().unwrap_err();
373        assert!(err.to_string().contains("pages_per_file"));
374    }
375
376    #[test]
377    fn request_validation_rejects_zero_batch_size() {
378        let request = ConvertRequest {
379            input: InputDocument::new("a.pdf", "application/pdf", Bytes::from_static(b"%PDF")),
380            output_formats: vec![OutputFormat::Text],
381            options: ConvertOptions::Pdf(PdfConvertOptions {
382                batch_size: 0,
383                ..PdfConvertOptions::default()
384            }),
385        };
386
387        let err = request.validate().unwrap_err();
388        assert!(err.to_string().contains("batch_size"));
389    }
390
391    #[test]
392    fn ambiguous_xml_requires_explicit_override() {
393        let err = InputDocument::new(
394            "paper.xml",
395            "application/xml",
396            Bytes::from_static(b"<article />"),
397        )
398        .kind()
399        .unwrap_err();
400
401        assert!(err.to_string().contains("explicit input_format override"));
402    }
403
404    #[test]
405    fn override_resolves_ambiguous_sources() {
406        let input = InputDocument::new(
407            "paper.xml",
408            "application/xml",
409            Bytes::from_static(b"<article />"),
410        )
411        .with_input_kind(InputKind::XmlJats);
412
413        assert_eq!(input.kind().unwrap(), InputKind::XmlJats);
414    }
415
416    #[test]
417    fn from_path_and_bytes_with_kind_preserves_override_kind() {
418        let input = InputDocument::from_path_and_bytes_with_kind(
419            Path::new("filing.json"),
420            Bytes::from_static(br#"{"schema":"docling"}"#),
421            InputKind::JsonDocling,
422        )
423        .unwrap();
424
425        assert_eq!(input.kind().unwrap(), InputKind::JsonDocling);
426        assert_eq!(input.media_type, "application/json");
427    }
428}
cloudiful_docling_convert/document.rs

cloudiful_docling_convert/
document.rs
