1use std::path::Path;
2use std::str::FromStr;
3
4use bytes::Bytes;
5use serde::{Deserialize, Serialize};
6
7use crate::error::{PdfConvertError, Result};
8use crate::models::{ChunkDocumentResponse, DoclingChunk};
9
10mod input_kind;
11
12pub use input_kind::InputKind;
13
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
15#[serde(rename_all = "snake_case")]
16pub enum OutputFormat {
17 Json,
18 Md,
19 Yaml,
20 Html,
21 HtmlSplitPage,
22 Text,
23 Doctags,
24 Vtt,
25 Doclang,
26 Dclx,
27 Chunks,
28}
29
30impl OutputFormat {
31 pub fn as_api_value(self) -> &'static str {
32 match self {
33 Self::Json => "json",
34 Self::Md => "md",
35 Self::Yaml => "yaml",
36 Self::Html => "html",
37 Self::HtmlSplitPage => "html_split_page",
38 Self::Text => "text",
39 Self::Doctags => "doctags",
40 Self::Vtt => "vtt",
41 Self::Doclang => "doclang",
42 Self::Dclx => "dclx",
43 Self::Chunks => "chunks",
44 }
45 }
46
47 pub fn extension(self) -> &'static str {
48 match self {
49 Self::Chunks => "chunks.json",
50 Self::Yaml | Self::HtmlSplitPage | Self::Vtt | Self::Dclx => "zip",
51 _ => self.as_api_value(),
52 }
53 }
54
55 pub fn is_archive(self) -> bool {
56 matches!(
57 self,
58 Self::Yaml | Self::HtmlSplitPage | Self::Vtt | Self::Dclx
59 )
60 }
61
62 pub fn is_chunk_output(self) -> bool {
63 matches!(self, Self::Chunks)
64 }
65}
66
67impl std::fmt::Display for OutputFormat {
68 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
69 f.write_str(self.as_api_value())
70 }
71}
72
73impl FromStr for OutputFormat {
74 type Err = PdfConvertError;
75
76 fn from_str(value: &str) -> Result<Self> {
77 match value.trim().to_ascii_lowercase().as_str() {
78 "json" => Ok(Self::Json),
79 "md" | "markdown" => Ok(Self::Md),
80 "yaml" | "yml" => Ok(Self::Yaml),
81 "html" => Ok(Self::Html),
82 "html_split_page" | "html-split-page" => Ok(Self::HtmlSplitPage),
83 "text" | "txt" => Ok(Self::Text),
84 "doctags" => Ok(Self::Doctags),
85 "vtt" => Ok(Self::Vtt),
86 "doclang" => Ok(Self::Doclang),
87 "dclx" => Ok(Self::Dclx),
88 "chunks" => Ok(Self::Chunks),
89 other => Err(PdfConvertError::validation_error(
90 "format",
91 format!("unsupported output format: {other}"),
92 )),
93 }
94 }
95}
96
97#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
98#[serde(rename_all = "snake_case")]
99pub enum ChunkerKind {
100 #[default]
101 None,
102 Hybrid,
103 Hierarchical,
104}
105
106impl ChunkerKind {
107 pub fn is_enabled(self) -> bool {
108 !matches!(self, Self::None)
109 }
110}
111
112impl std::fmt::Display for ChunkerKind {
113 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
114 let value = match self {
115 Self::None => "none",
116 Self::Hybrid => "hybrid",
117 Self::Hierarchical => "hierarchical",
118 };
119 f.write_str(value)
120 }
121}
122
123impl FromStr for ChunkerKind {
124 type Err = PdfConvertError;
125
126 fn from_str(value: &str) -> Result<Self> {
127 match value.trim().to_ascii_lowercase().as_str() {
128 "none" | "" => Ok(Self::None),
129 "hybrid" => Ok(Self::Hybrid),
130 "hierarchical" => Ok(Self::Hierarchical),
131 other => Err(PdfConvertError::validation_error(
132 "chunker",
133 format!("unsupported chunker: {other}"),
134 )),
135 }
136 }
137}
138
139#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
140pub struct ChunkingOptions {
141 pub use_markdown_tables: bool,
142 pub use_markdown_images: bool,
143 pub image_placeholder: String,
144 pub include_raw_text: bool,
145 pub max_tokens: Option<u32>,
146 pub tokenizer: Option<String>,
147 pub merge_peers: bool,
148}
149
150impl ChunkingOptions {
151 pub fn hybrid_defaults() -> Self {
152 Self {
153 image_placeholder: "![IMAGE]".to_string(),
154 tokenizer: Some("sentence-transformers/all-MiniLM-L6-v2".to_string()),
155 merge_peers: true,
156 ..Self::default()
157 }
158 }
159
160 pub fn hierarchical_defaults() -> Self {
161 Self {
162 image_placeholder: "![IMAGE]".to_string(),
163 merge_peers: true,
164 ..Self::default()
165 }
166 }
167}
168
169#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
170#[serde(rename_all = "lowercase")]
171pub enum PipelineKind {
172 Legacy,
173 Standard,
174 Vlm,
175 Asr,
176}
177
178impl FromStr for PipelineKind {
179 type Err = PdfConvertError;
180
181 fn from_str(value: &str) -> Result<Self> {
182 match value.trim().to_ascii_lowercase().as_str() {
183 "legacy" => Ok(Self::Legacy),
184 "standard" => Ok(Self::Standard),
185 "vlm" => Ok(Self::Vlm),
186 "asr" => Ok(Self::Asr),
187 other => Err(PdfConvertError::validation_error(
188 "pipeline",
189 format!("unsupported pipeline: {other}"),
190 )),
191 }
192 }
193}
194
195#[derive(Debug, Clone)]
196pub struct InputDocument {
197 pub filename: String,
198 pub media_type: String,
199 pub bytes: Bytes,
200 pub input_kind_override: Option<InputKind>,
201}
202
203impl InputDocument {
204 pub fn new(
205 filename: impl Into<String>,
206 media_type: impl Into<String>,
207 bytes: impl Into<Bytes>,
208 ) -> Self {
209 Self {
210 filename: filename.into(),
211 media_type: media_type.into(),
212 bytes: bytes.into(),
213 input_kind_override: None,
214 }
215 }
216
217 pub fn with_input_kind(mut self, input_kind: InputKind) -> Self {
218 self.input_kind_override = Some(input_kind);
219 self
220 }
221
222 pub fn from_path_and_bytes(path: &Path, bytes: impl Into<Bytes>) -> Result<Self> {
223 let filename = path
224 .file_name()
225 .and_then(|name| name.to_str())
226 .ok_or_else(|| {
227 PdfConvertError::validation_error(
228 "input_path",
229 format!("path '{}' does not have a valid file name", path.display()),
230 )
231 })?;
232 let kind = InputKind::from_path(path).ok_or_else(|| {
233 PdfConvertError::validation_error(
234 "input_path",
235 format!("unsupported file type for '{}'", path.display()),
236 )
237 })?;
238
239 Ok(Self::new(
240 filename,
241 kind.canonical_media_type(filename, None),
242 bytes,
243 ))
244 }
245
246 pub fn from_path_and_bytes_with_kind(
247 path: &Path,
248 bytes: impl Into<Bytes>,
249 input_kind: InputKind,
250 ) -> Result<Self> {
251 let filename = path
252 .file_name()
253 .and_then(|name| name.to_str())
254 .ok_or_else(|| {
255 PdfConvertError::validation_error(
256 "input_path",
257 format!("path '{}' does not have a valid file name", path.display()),
258 )
259 })?;
260
261 Ok(Self::new(
262 filename,
263 input_kind.canonical_media_type(filename, None),
264 bytes,
265 )
266 .with_input_kind(input_kind))
267 }
268
269 pub fn kind(&self) -> Result<InputKind> {
270 if let Some(input_kind) = self.input_kind_override {
271 return Ok(input_kind);
272 }
273
274 InputKind::from_filename_and_media_type(&self.filename, Some(&self.media_type)).ok_or_else(
275 || {
276 let reason = if InputKind::requires_explicit_override(
277 &self.filename,
278 Some(&self.media_type),
279 ) {
280 format!(
281 "ambiguous input type for '{}' ({}); provide an explicit input_format override",
282 self.filename, self.media_type
283 )
284 } else {
285 format!(
286 "unsupported input type for '{}' ({})",
287 self.filename, self.media_type
288 )
289 };
290 PdfConvertError::validation_error("input", reason)
291 },
292 )
293 }
294}
295
296#[derive(Debug, Clone)]
297pub struct ConvertRequest {
298 pub input: InputDocument,
299 pub output_formats: Vec<OutputFormat>,
300 pub options: ConvertOptions,
301}
302
303impl ConvertRequest {
304 pub fn validate(&self) -> Result<InputKind> {
305 let kind = self.input.kind()?;
306 match (&self.options, kind) {
307 (ConvertOptions::Pdf(_), InputKind::Pdf)
308 | (ConvertOptions::Text(_), InputKind::Text) => {}
309 (ConvertOptions::Generic(_), _) if kind.uses_generic_convert_options() => {}
310 (_, InputKind::Pdf) => {
311 return Err(PdfConvertError::validation_error(
312 "options",
313 "PDF input requires PdfConvertOptions",
314 ));
315 }
316 (_, InputKind::Text) => {
317 return Err(PdfConvertError::validation_error(
318 "options",
319 "txt input requires TextConvertOptions",
320 ));
321 }
322 _ => {
323 return Err(PdfConvertError::validation_error(
324 "options",
325 "non-pdf, non-text input requires GenericFileConvertOptions",
326 ));
327 }
328 }
329
330 if self.output_formats.is_empty() {
331 return Err(PdfConvertError::validation_error(
332 "output_formats",
333 "at least one output format is required",
334 ));
335 }
336
337 let chunker = match &self.options {
338 ConvertOptions::Pdf(options) | ConvertOptions::Generic(options) => options.chunker,
339 ConvertOptions::Text(_) => ChunkerKind::None,
340 };
341
342 if chunker.is_enabled() && self.output_formats.iter().any(|format| format.is_archive()) {
343 return Err(PdfConvertError::validation_error(
344 "output_formats",
345 "native chunking cannot be combined with archive outputs",
346 ));
347 }
348
349 if self
350 .output_formats
351 .iter()
352 .any(|format| format.is_chunk_output())
353 && !chunker.is_enabled()
354 {
355 return Err(PdfConvertError::validation_error(
356 "chunker",
357 "chunks output requires hybrid or hierarchical chunking",
358 ));
359 }
360
361 Ok(kind)
362 }
363}
364
365#[derive(Debug, Clone)]
366pub enum ConvertOptions {
367 Pdf(PdfConvertOptions),
368 Generic(GenericFileConvertOptions),
369 Text(TextConvertOptions),
370}
371
372#[derive(Debug, Clone, PartialEq, Eq)]
373pub struct RemoteConvertOptions {
374 pub chunker: ChunkerKind,
375 pub chunking: ChunkingOptions,
376 pub pipeline: Option<PipelineKind>,
377 pub picture_description_preset: Option<String>,
383}
384
385impl Default for RemoteConvertOptions {
386 fn default() -> Self {
387 Self {
388 chunker: ChunkerKind::None,
389 chunking: ChunkingOptions::hybrid_defaults(),
390 pipeline: None,
391 picture_description_preset: None,
392 }
393 }
394}
395
396pub type PdfConvertOptions = RemoteConvertOptions;
397pub type GenericFileConvertOptions = RemoteConvertOptions;
398
399#[derive(Debug, Clone)]
400pub struct TextConvertOptions {
401 pub normalize_line_endings: bool,
402 pub trim_utf8_bom: bool,
403}
404
405impl Default for TextConvertOptions {
406 fn default() -> Self {
407 Self {
408 normalize_line_endings: true,
409 trim_utf8_bom: true,
410 }
411 }
412}
413
414#[derive(Debug, Clone, Serialize, Deserialize)]
415pub struct ConvertedDocumentMetadata {
416 pub input_kind: InputKind,
417 pub media_type: String,
418}
419
420#[derive(Debug, Clone, Serialize, Deserialize)]
421pub struct ConvertedDocument {
422 pub filename: String,
423 pub markdown: Option<String>,
424 pub text: Option<String>,
425 pub json: Option<serde_json::Value>,
426 pub html: Option<String>,
427 pub doctags: Option<String>,
428 pub doclang: Option<String>,
429 pub chunks: Vec<DoclingChunk>,
430 pub chunk_response: Option<ChunkDocumentResponse>,
431 #[serde(skip)]
432 pub archive: Option<Vec<u8>>,
433 pub metadata: ConvertedDocumentMetadata,
434 pub errors: Vec<String>,
435}
436
437#[derive(Debug, Clone)]
438pub struct FileConvertRequest {
439 pub request: ConvertRequest,
440 pub output_dir: std::path::PathBuf,
441 pub selected_output: OutputFormat,
442 pub overwrite: bool,
443}
444
445#[derive(Debug, Clone)]
446pub struct ConvertedFile {
447 pub document: ConvertedDocument,
448 pub output_paths: Vec<std::path::PathBuf>,
449}
450
451pub fn supported_input_kind(path: &Path) -> bool {
452 InputKind::from_path(path).is_some()
453}
454
455#[cfg(test)]
456mod tests {
457 use super::*;
458
459 #[test]
460 fn output_format_supports_native_and_archive_outputs() {
461 for (value, format) in [
462 ("md", OutputFormat::Md),
463 ("markdown", OutputFormat::Md),
464 ("json", OutputFormat::Json),
465 ("yaml", OutputFormat::Yaml),
466 ("yml", OutputFormat::Yaml),
467 ("html", OutputFormat::Html),
468 ("html_split_page", OutputFormat::HtmlSplitPage),
469 ("html-split-page", OutputFormat::HtmlSplitPage),
470 ("text", OutputFormat::Text),
471 ("txt", OutputFormat::Text),
472 ("doctags", OutputFormat::Doctags),
473 ("vtt", OutputFormat::Vtt),
474 ("doclang", OutputFormat::Doclang),
475 ("dclx", OutputFormat::Dclx),
476 ("chunks", OutputFormat::Chunks),
477 ] {
478 assert_eq!(value.parse::<OutputFormat>().unwrap(), format);
479 }
480
481 assert_eq!(OutputFormat::Chunks.extension(), "chunks.json");
482 for format in [
483 OutputFormat::Yaml,
484 OutputFormat::HtmlSplitPage,
485 OutputFormat::Vtt,
486 OutputFormat::Dclx,
487 ] {
488 assert_eq!(format.extension(), "zip");
489 assert!(format.is_archive());
490 }
491 }
492
493 #[test]
494 fn chunking_rejects_archive_outputs() {
495 let request = ConvertRequest {
496 input: InputDocument::new("a.pdf", "application/pdf", Bytes::from_static(b"%PDF")),
497 output_formats: vec![OutputFormat::Yaml],
498 options: ConvertOptions::Pdf(RemoteConvertOptions {
499 chunker: ChunkerKind::Hybrid,
500 ..RemoteConvertOptions::default()
501 }),
502 };
503
504 assert!(
505 request
506 .validate()
507 .unwrap_err()
508 .to_string()
509 .contains("archive")
510 );
511 }
512
513 #[test]
514 fn chunks_require_a_native_chunker() {
515 let request = ConvertRequest {
516 input: InputDocument::new("a.pdf", "application/pdf", Bytes::from_static(b"%PDF")),
517 output_formats: vec![OutputFormat::Chunks],
518 options: ConvertOptions::Pdf(RemoteConvertOptions::default()),
519 };
520
521 assert!(
522 request
523 .validate()
524 .unwrap_err()
525 .to_string()
526 .contains("requires")
527 );
528 }
529
530 #[test]
531 fn ambiguous_xml_requires_explicit_override() {
532 let error = InputDocument::new(
533 "paper.xml",
534 "application/xml",
535 Bytes::from_static(b"<article />"),
536 )
537 .kind()
538 .unwrap_err();
539
540 assert!(error.to_string().contains("explicit input_format override"));
541 }
542
543 #[test]
544 fn remote_convert_options_default_omits_picture_description_preset() {
545 let options = RemoteConvertOptions::default();
546 assert!(
547 options.picture_description_preset.is_none(),
548 "default RemoteConvertOptions must not carry a picture_description_preset so the legacy custom VLM bundle is preserved"
549 );
550 }
551
552 #[test]
553 fn remote_convert_options_round_trip_picture_description_preset() {
554 let options = RemoteConvertOptions {
555 chunker: ChunkerKind::None,
556 chunking: ChunkingOptions::default(),
557 pipeline: None,
558 picture_description_preset: Some("granite_vision".to_string()),
559 };
560 assert_eq!(
561 options.picture_description_preset.as_deref(),
562 Some("granite_vision")
563 );
564 }
565}