1use std::path::Path;
2use std::str::FromStr;
3
4use bytes::Bytes;
5use serde::{Deserialize, Serialize};
6
7use crate::error::{PdfConvertError, Result};
8use crate::models::{ChunkDocumentResponse, DoclingChunk};
9
10mod input_kind;
11
12pub use input_kind::InputKind;
13
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
15#[serde(rename_all = "snake_case")]
16pub enum OutputFormat {
17 Json,
18 Md,
19 Yaml,
20 Html,
21 HtmlSplitPage,
22 Text,
23 Doctags,
24 Vtt,
25 Doclang,
26 Dclx,
27 Chunks,
28}
29
30impl OutputFormat {
31 pub fn as_api_value(self) -> &'static str {
32 match self {
33 Self::Json => "json",
34 Self::Md => "md",
35 Self::Yaml => "yaml",
36 Self::Html => "html",
37 Self::HtmlSplitPage => "html_split_page",
38 Self::Text => "text",
39 Self::Doctags => "doctags",
40 Self::Vtt => "vtt",
41 Self::Doclang => "doclang",
42 Self::Dclx => "dclx",
43 Self::Chunks => "chunks",
44 }
45 }
46
47 pub fn extension(self) -> &'static str {
48 match self {
49 Self::Chunks => "chunks.json",
50 Self::Yaml | Self::HtmlSplitPage | Self::Vtt | Self::Dclx => "zip",
51 _ => self.as_api_value(),
52 }
53 }
54
55 pub fn is_archive(self) -> bool {
56 matches!(
57 self,
58 Self::Yaml | Self::HtmlSplitPage | Self::Vtt | Self::Dclx
59 )
60 }
61
62 pub fn is_chunk_output(self) -> bool {
63 matches!(self, Self::Chunks)
64 }
65}
66
67impl std::fmt::Display for OutputFormat {
68 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
69 f.write_str(self.as_api_value())
70 }
71}
72
73impl FromStr for OutputFormat {
74 type Err = PdfConvertError;
75
76 fn from_str(value: &str) -> Result<Self> {
77 match value.trim().to_ascii_lowercase().as_str() {
78 "json" => Ok(Self::Json),
79 "md" | "markdown" => Ok(Self::Md),
80 "yaml" | "yml" => Ok(Self::Yaml),
81 "html" => Ok(Self::Html),
82 "html_split_page" | "html-split-page" => Ok(Self::HtmlSplitPage),
83 "text" | "txt" => Ok(Self::Text),
84 "doctags" => Ok(Self::Doctags),
85 "vtt" => Ok(Self::Vtt),
86 "doclang" => Ok(Self::Doclang),
87 "dclx" => Ok(Self::Dclx),
88 "chunks" => Ok(Self::Chunks),
89 other => Err(PdfConvertError::validation_error(
90 "format",
91 format!("unsupported output format: {other}"),
92 )),
93 }
94 }
95}
96
97#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
98#[serde(rename_all = "snake_case")]
99pub enum ChunkerKind {
100 #[default]
101 None,
102 Hybrid,
103 Hierarchical,
104}
105
106impl ChunkerKind {
107 pub fn is_enabled(self) -> bool {
108 !matches!(self, Self::None)
109 }
110}
111
112impl std::fmt::Display for ChunkerKind {
113 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
114 let value = match self {
115 Self::None => "none",
116 Self::Hybrid => "hybrid",
117 Self::Hierarchical => "hierarchical",
118 };
119 f.write_str(value)
120 }
121}
122
123impl FromStr for ChunkerKind {
124 type Err = PdfConvertError;
125
126 fn from_str(value: &str) -> Result<Self> {
127 match value.trim().to_ascii_lowercase().as_str() {
128 "none" | "" => Ok(Self::None),
129 "hybrid" => Ok(Self::Hybrid),
130 "hierarchical" => Ok(Self::Hierarchical),
131 other => Err(PdfConvertError::validation_error(
132 "chunker",
133 format!("unsupported chunker: {other}"),
134 )),
135 }
136 }
137}
138
139#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
140pub struct ChunkingOptions {
141 pub use_markdown_tables: bool,
142 pub use_markdown_images: bool,
143 pub image_placeholder: String,
144 pub include_raw_text: bool,
145 pub max_tokens: Option<u32>,
146 pub tokenizer: Option<String>,
147 pub merge_peers: bool,
148}
149
150impl ChunkingOptions {
151 pub fn hybrid_defaults() -> Self {
152 Self {
153 image_placeholder: "![IMAGE]".to_string(),
154 tokenizer: Some("sentence-transformers/all-MiniLM-L6-v2".to_string()),
155 merge_peers: true,
156 ..Self::default()
157 }
158 }
159
160 pub fn hierarchical_defaults() -> Self {
161 Self {
162 image_placeholder: "![IMAGE]".to_string(),
163 merge_peers: true,
164 ..Self::default()
165 }
166 }
167}
168
169#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
170#[serde(rename_all = "lowercase")]
171pub enum PipelineKind {
172 Legacy,
173 Standard,
174 Vlm,
175 Asr,
176}
177
178impl FromStr for PipelineKind {
179 type Err = PdfConvertError;
180
181 fn from_str(value: &str) -> Result<Self> {
182 match value.trim().to_ascii_lowercase().as_str() {
183 "legacy" => Ok(Self::Legacy),
184 "standard" => Ok(Self::Standard),
185 "vlm" => Ok(Self::Vlm),
186 "asr" => Ok(Self::Asr),
187 other => Err(PdfConvertError::validation_error(
188 "pipeline",
189 format!("unsupported pipeline: {other}"),
190 )),
191 }
192 }
193}
194
195#[derive(Debug, Clone)]
196pub struct InputDocument {
197 pub filename: String,
198 pub media_type: String,
199 pub bytes: Bytes,
200 pub input_kind_override: Option<InputKind>,
201}
202
203impl InputDocument {
204 pub fn new(
205 filename: impl Into<String>,
206 media_type: impl Into<String>,
207 bytes: impl Into<Bytes>,
208 ) -> Self {
209 Self {
210 filename: filename.into(),
211 media_type: media_type.into(),
212 bytes: bytes.into(),
213 input_kind_override: None,
214 }
215 }
216
217 pub fn with_input_kind(mut self, input_kind: InputKind) -> Self {
218 self.input_kind_override = Some(input_kind);
219 self
220 }
221
222 pub fn from_path_and_bytes(path: &Path, bytes: impl Into<Bytes>) -> Result<Self> {
223 let filename = path
224 .file_name()
225 .and_then(|name| name.to_str())
226 .ok_or_else(|| {
227 PdfConvertError::validation_error(
228 "input_path",
229 format!("path '{}' does not have a valid file name", path.display()),
230 )
231 })?;
232 let kind = InputKind::from_path(path).ok_or_else(|| {
233 PdfConvertError::validation_error(
234 "input_path",
235 format!("unsupported file type for '{}'", path.display()),
236 )
237 })?;
238
239 Ok(Self::new(
240 filename,
241 kind.canonical_media_type(filename, None),
242 bytes,
243 ))
244 }
245
246 pub fn from_path_and_bytes_with_kind(
247 path: &Path,
248 bytes: impl Into<Bytes>,
249 input_kind: InputKind,
250 ) -> Result<Self> {
251 let filename = path
252 .file_name()
253 .and_then(|name| name.to_str())
254 .ok_or_else(|| {
255 PdfConvertError::validation_error(
256 "input_path",
257 format!("path '{}' does not have a valid file name", path.display()),
258 )
259 })?;
260
261 Ok(Self::new(
262 filename,
263 input_kind.canonical_media_type(filename, None),
264 bytes,
265 )
266 .with_input_kind(input_kind))
267 }
268
269 pub fn kind(&self) -> Result<InputKind> {
270 if let Some(input_kind) = self.input_kind_override {
271 return Ok(input_kind);
272 }
273
274 InputKind::from_filename_and_media_type(&self.filename, Some(&self.media_type)).ok_or_else(
275 || {
276 let reason = if InputKind::requires_explicit_override(
277 &self.filename,
278 Some(&self.media_type),
279 ) {
280 format!(
281 "ambiguous input type for '{}' ({}); provide an explicit input_format override",
282 self.filename, self.media_type
283 )
284 } else {
285 format!(
286 "unsupported input type for '{}' ({})",
287 self.filename, self.media_type
288 )
289 };
290 PdfConvertError::validation_error("input", reason)
291 },
292 )
293 }
294}
295
296#[derive(Debug, Clone)]
297pub struct ConvertRequest {
298 pub input: InputDocument,
299 pub output_formats: Vec<OutputFormat>,
300 pub options: ConvertOptions,
301}
302
303impl ConvertRequest {
304 pub fn validate(&self) -> Result<InputKind> {
305 let kind = self.input.kind()?;
306 match (&self.options, kind) {
307 (ConvertOptions::Pdf(_), InputKind::Pdf)
308 | (ConvertOptions::Text(_), InputKind::Text) => {}
309 (ConvertOptions::Generic(_), _) if kind.uses_generic_convert_options() => {}
310 (_, InputKind::Pdf) => {
311 return Err(PdfConvertError::validation_error(
312 "options",
313 "PDF input requires PdfConvertOptions",
314 ));
315 }
316 (_, InputKind::Text) => {
317 return Err(PdfConvertError::validation_error(
318 "options",
319 "txt input requires TextConvertOptions",
320 ));
321 }
322 _ => {
323 return Err(PdfConvertError::validation_error(
324 "options",
325 "non-pdf, non-text input requires GenericFileConvertOptions",
326 ));
327 }
328 }
329
330 if self.output_formats.is_empty() {
331 return Err(PdfConvertError::validation_error(
332 "output_formats",
333 "at least one output format is required",
334 ));
335 }
336
337 let chunker = match &self.options {
338 ConvertOptions::Pdf(options) | ConvertOptions::Generic(options) => options.chunker,
339 ConvertOptions::Text(_) => ChunkerKind::None,
340 };
341
342 if chunker.is_enabled() && self.output_formats.iter().any(|format| format.is_archive()) {
343 return Err(PdfConvertError::validation_error(
344 "output_formats",
345 "native chunking cannot be combined with archive outputs",
346 ));
347 }
348
349 if self
350 .output_formats
351 .iter()
352 .any(|format| format.is_chunk_output())
353 && !chunker.is_enabled()
354 {
355 return Err(PdfConvertError::validation_error(
356 "chunker",
357 "chunks output requires hybrid or hierarchical chunking",
358 ));
359 }
360
361 Ok(kind)
362 }
363}
364
365#[derive(Debug, Clone)]
366pub enum ConvertOptions {
367 Pdf(PdfConvertOptions),
368 Generic(GenericFileConvertOptions),
369 Text(TextConvertOptions),
370}
371
372#[derive(Debug, Clone, PartialEq, Eq)]
373pub struct RemoteConvertOptions {
374 pub chunker: ChunkerKind,
375 pub chunking: ChunkingOptions,
376 pub pipeline: Option<PipelineKind>,
377}
378
379impl Default for RemoteConvertOptions {
380 fn default() -> Self {
381 Self {
382 chunker: ChunkerKind::None,
383 chunking: ChunkingOptions::hybrid_defaults(),
384 pipeline: None,
385 }
386 }
387}
388
389pub type PdfConvertOptions = RemoteConvertOptions;
390pub type GenericFileConvertOptions = RemoteConvertOptions;
391
392#[derive(Debug, Clone)]
393pub struct TextConvertOptions {
394 pub normalize_line_endings: bool,
395 pub trim_utf8_bom: bool,
396}
397
398impl Default for TextConvertOptions {
399 fn default() -> Self {
400 Self {
401 normalize_line_endings: true,
402 trim_utf8_bom: true,
403 }
404 }
405}
406
407#[derive(Debug, Clone, Serialize, Deserialize)]
408pub struct ConvertedDocumentMetadata {
409 pub input_kind: InputKind,
410 pub media_type: String,
411}
412
413#[derive(Debug, Clone, Serialize, Deserialize)]
414pub struct ConvertedDocument {
415 pub filename: String,
416 pub markdown: Option<String>,
417 pub text: Option<String>,
418 pub json: Option<serde_json::Value>,
419 pub html: Option<String>,
420 pub doctags: Option<String>,
421 pub doclang: Option<String>,
422 pub chunks: Vec<DoclingChunk>,
423 pub chunk_response: Option<ChunkDocumentResponse>,
424 #[serde(skip)]
425 pub archive: Option<Vec<u8>>,
426 pub metadata: ConvertedDocumentMetadata,
427 pub errors: Vec<String>,
428}
429
430#[derive(Debug, Clone)]
431pub struct FileConvertRequest {
432 pub request: ConvertRequest,
433 pub output_dir: std::path::PathBuf,
434 pub selected_output: OutputFormat,
435 pub overwrite: bool,
436}
437
438#[derive(Debug, Clone)]
439pub struct ConvertedFile {
440 pub document: ConvertedDocument,
441 pub output_paths: Vec<std::path::PathBuf>,
442}
443
444pub fn supported_input_kind(path: &Path) -> bool {
445 InputKind::from_path(path).is_some()
446}
447
448#[cfg(test)]
449mod tests {
450 use super::*;
451
452 #[test]
453 fn output_format_supports_native_and_archive_outputs() {
454 for (value, format) in [
455 ("md", OutputFormat::Md),
456 ("markdown", OutputFormat::Md),
457 ("json", OutputFormat::Json),
458 ("yaml", OutputFormat::Yaml),
459 ("yml", OutputFormat::Yaml),
460 ("html", OutputFormat::Html),
461 ("html_split_page", OutputFormat::HtmlSplitPage),
462 ("html-split-page", OutputFormat::HtmlSplitPage),
463 ("text", OutputFormat::Text),
464 ("txt", OutputFormat::Text),
465 ("doctags", OutputFormat::Doctags),
466 ("vtt", OutputFormat::Vtt),
467 ("doclang", OutputFormat::Doclang),
468 ("dclx", OutputFormat::Dclx),
469 ("chunks", OutputFormat::Chunks),
470 ] {
471 assert_eq!(value.parse::<OutputFormat>().unwrap(), format);
472 }
473
474 assert_eq!(OutputFormat::Chunks.extension(), "chunks.json");
475 for format in [
476 OutputFormat::Yaml,
477 OutputFormat::HtmlSplitPage,
478 OutputFormat::Vtt,
479 OutputFormat::Dclx,
480 ] {
481 assert_eq!(format.extension(), "zip");
482 assert!(format.is_archive());
483 }
484 }
485
486 #[test]
487 fn chunking_rejects_archive_outputs() {
488 let request = ConvertRequest {
489 input: InputDocument::new("a.pdf", "application/pdf", Bytes::from_static(b"%PDF")),
490 output_formats: vec![OutputFormat::Yaml],
491 options: ConvertOptions::Pdf(RemoteConvertOptions {
492 chunker: ChunkerKind::Hybrid,
493 ..RemoteConvertOptions::default()
494 }),
495 };
496
497 assert!(
498 request
499 .validate()
500 .unwrap_err()
501 .to_string()
502 .contains("archive")
503 );
504 }
505
506 #[test]
507 fn chunks_require_a_native_chunker() {
508 let request = ConvertRequest {
509 input: InputDocument::new("a.pdf", "application/pdf", Bytes::from_static(b"%PDF")),
510 output_formats: vec![OutputFormat::Chunks],
511 options: ConvertOptions::Pdf(RemoteConvertOptions::default()),
512 };
513
514 assert!(
515 request
516 .validate()
517 .unwrap_err()
518 .to_string()
519 .contains("requires")
520 );
521 }
522
523 #[test]
524 fn ambiguous_xml_requires_explicit_override() {
525 let error = InputDocument::new(
526 "paper.xml",
527 "application/xml",
528 Bytes::from_static(b"<article />"),
529 )
530 .kind()
531 .unwrap_err();
532
533 assert!(error.to_string().contains("explicit input_format override"));
534 }
535}