1use std::fmt;
40use std::str::FromStr;
41
42use serde::de::DeserializeOwned;
43use serde::{Deserialize, Serialize};
44
45use docling_core::{
46 CustomPattern, ImageRedaction, MetadataOutput, PiiKind, RedactionOptions, Replacement,
47 TrackChanges,
48};
49
50use crate::DocumentConverter;
51
52#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
55#[serde(default)]
56pub struct ConvertOptions {
57 pub strict: Option<bool>,
60 pub compact_tables: Option<bool>,
62 pub picture_alt_text: Option<bool>,
65 #[serde(alias = "md_page_break_placeholder")]
69 pub page_break_placeholder: Option<String>,
70
71 pub fetch_images: Option<bool>,
76 pub image_sources: Option<String>,
83 #[serde(default, deserialize_with = "de_string_list")]
87 pub image_hosts: Option<Vec<String>>,
88 pub max_image_bytes: Option<u64>,
91 pub max_images: Option<usize>,
94 pub max_image_total_mb: Option<u64>,
97 pub min_image_bytes: Option<u64>,
100 pub list_attachments: Option<bool>,
102 pub skip_empty_cells: Option<bool>,
104 pub ebcdic_layout: Option<String>,
106 pub encoding: Option<String>,
108 pub ipynb_outputs: Option<String>,
112 pub metadata: Option<String>,
118 pub track_changes: Option<String>,
126 pub extract_diagrams: Option<String>,
134 pub use_outline_level_for_headings: Option<bool>,
142 pub use_web_browser: Option<bool>,
144 pub xbrl_taxonomy: Option<String>,
146
147 pub asr_model: Option<String>,
150 pub asr_lang: Option<String>,
152 #[serde(deserialize_with = "de_video_frames")]
156 pub video_frames: Option<usize>,
157 pub video_scene_threshold: Option<f32>,
160 pub video_frame_max_side: Option<u32>,
164 pub video_frame_dedupe: Option<u32>,
168
169 pub pages: Option<String>,
172 pub document_timeout: Option<f64>,
174 #[serde(deserialize_with = "de_text_layer_only")]
179 pub text_layer_only: Option<TextLayerOnly>,
180 pub no_ocr: Option<bool>,
183 pub skip_ocr: Option<bool>,
188 #[serde(alias = "pdf_password")]
192 pub password: Option<String>,
193 pub force_full_page_ocr: Option<bool>,
195 pub no_table_former: Option<bool>,
197 pub no_text_panels: Option<bool>,
199 pub heading_hierarchy: Option<bool>,
201 pub ocr_engine: Option<String>,
203 pub ocr_lang: Option<String>,
206 pub ocr_mode: Option<String>,
209 pub ocr_scale: Option<f32>,
211 pub ocr_textline_orientation: Option<bool>,
215 pub images_scale: Option<f32>,
217 #[serde(alias = "generate_page_images")]
220 pub page_images: Option<bool>,
221 pub do_picture_classification: Option<bool>,
223 pub do_code_enrichment: Option<bool>,
225 pub do_formula_enrichment: Option<bool>,
227 pub do_picture_ocr: Option<bool>,
230 pub picture_ocr_classes: Option<String>,
233 pub picture_ocr_min_side: Option<u32>,
236 pub picture_ocr_scale: Option<f32>,
240 pub keep_picture_images: Option<bool>,
243 pub do_picture_description: Option<bool>,
247 pub picture_description_endpoint: Option<String>,
250 pub picture_description_model: Option<String>,
252 pub picture_description_api_key: Option<String>,
254 pub picture_description_prompt: Option<String>,
257 pub picture_description_mode: Option<String>,
259 pub picture_description_concurrency: Option<usize>,
261
262 pub redact_pii: Option<bool>,
265 pub redact_mode: Option<String>,
267 pub redact_kinds: Option<String>,
271 pub redact_pattern: Option<String>,
274 pub redact_images: Option<String>,
276 pub redact_allow_terms: Option<String>,
279 pub redact_deny_terms: Option<String>,
282 pub redact_allow_patterns: Option<String>,
285 pub redact_ner_min_score: Option<f32>,
287
288 pub pipeline: Option<String>,
292 pub vlm_endpoint: Option<String>,
294 pub vlm_model: Option<String>,
296 pub vlm_api_key: Option<String>,
298 pub vlm_prompt: Option<String>,
300 pub vlm_max_tokens: Option<usize>,
302}
303
304#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
310pub enum TextLayerOnly {
311 #[default]
314 Off,
315 On,
318 Auto,
327}
328
329impl TextLayerOnly {
330 pub const ACCEPTED: &'static str = "true, false, auto";
332
333 pub fn parse(s: &str) -> Option<Self> {
335 match s.trim().to_ascii_lowercase().as_str() {
336 "auto" => Some(Self::Auto),
337 "1" | "true" | "yes" | "on" => Some(Self::On),
338 "0" | "false" | "no" | "off" => Some(Self::Off),
339 _ => None,
340 }
341 }
342
343 pub fn as_str(self) -> &'static str {
345 match self {
346 Self::Off => "false",
347 Self::On => "true",
348 Self::Auto => "auto",
349 }
350 }
351}
352
353impl From<bool> for TextLayerOnly {
354 fn from(on: bool) -> Self {
355 if on {
356 Self::On
357 } else {
358 Self::Off
359 }
360 }
361}
362
363impl std::fmt::Display for TextLayerOnly {
364 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
365 f.write_str(self.as_str())
366 }
367}
368
369impl std::str::FromStr for TextLayerOnly {
370 type Err = String;
371
372 fn from_str(s: &str) -> Result<Self, Self::Err> {
373 Self::parse(s).ok_or_else(|| {
374 format!(
375 "text_layer_only: expected one of {}, got {s:?}",
376 Self::ACCEPTED
377 )
378 })
379 }
380}
381
382impl Serialize for TextLayerOnly {
386 fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
387 match self {
388 Self::Off => s.serialize_bool(false),
389 Self::On => s.serialize_bool(true),
390 Self::Auto => s.serialize_str("auto"),
391 }
392 }
393}
394
395impl<'de> Deserialize<'de> for TextLayerOnly {
396 fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
397 #[derive(Deserialize)]
398 #[serde(untagged)]
399 enum Raw {
400 Flag(bool),
401 Count(u64),
402 Text(String),
403 }
404 let parsed = match Raw::deserialize(d)? {
405 Raw::Flag(b) => Some(Self::from(b)),
406 Raw::Count(0) => Some(Self::Off),
407 Raw::Count(1) => Some(Self::On),
408 Raw::Count(_) => None,
409 Raw::Text(s) => Self::parse(&s),
410 };
411 parsed.ok_or_else(|| {
412 serde::de::Error::custom(format!(
413 "text_layer_only: expected one of {}",
414 Self::ACCEPTED
415 ))
416 })
417 }
418}
419
420fn de_text_layer_only<'de, D: serde::Deserializer<'de>>(
423 d: D,
424) -> Result<Option<TextLayerOnly>, D::Error> {
425 Option::<TextLayerOnly>::deserialize(d)
426}
427
428#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
430pub enum PictureDescriptionMode {
431 #[default]
433 Caption,
434 Ocr,
438}
439
440impl PictureDescriptionMode {
441 pub fn parse(s: &str) -> Option<Self> {
443 match s.trim().to_ascii_lowercase().as_str() {
444 "caption" => Some(Self::Caption),
445 "ocr" => Some(Self::Ocr),
446 _ => None,
447 }
448 }
449
450 pub fn as_str(self) -> &'static str {
451 match self {
452 Self::Caption => "caption",
453 Self::Ocr => "ocr",
454 }
455 }
456
457 pub fn default_prompt(self) -> &'static str {
459 match self {
460 Self::Caption => CAPTION_PROMPT,
461 Self::Ocr => OCR_PROMPT,
462 }
463 }
464}
465
466pub const CAPTION_PROMPT: &str = "Write a detailed caption for this image.";
468
469pub const OCR_PROMPT: &str = "Extract all text from this image. Return ONLY the extracted text, \
471 maintaining the original layout and order. Do not add any commentary \
472 or description.";
473
474#[derive(Debug, Clone, PartialEq, Eq)]
478pub struct OptionsError {
479 pub field: &'static str,
481 pub message: String,
483}
484
485impl OptionsError {
486 fn new(field: &'static str, message: impl Into<String>) -> Self {
487 Self {
488 field,
489 message: message.into(),
490 }
491 }
492
493 pub fn cli_message(&self) -> String {
496 let flag = cli_flag(self.field);
497 if self.message.contains(self.field) {
498 self.message.replacen(self.field, &flag, 1)
499 } else {
500 format!("{flag}: {}", self.message)
501 }
502 }
503}
504
505impl fmt::Display for OptionsError {
506 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
507 f.write_str(&self.message)
508 }
509}
510
511impl std::error::Error for OptionsError {}
512
513#[derive(Debug, Clone, Copy, PartialEq, Eq)]
515pub enum PipelineKind {
516 Standard,
518 Vlm,
520}
521
522#[derive(Debug, Clone, Copy)]
526pub struct OptionInfo {
527 pub name: &'static str,
529 pub aliases: &'static [&'static str],
531 pub cli: Option<&'static str>,
533 pub python: Option<&'static str>,
537 pub node: Option<&'static str>,
541}
542
543const fn row(name: &'static str) -> OptionInfo {
544 OptionInfo {
545 name,
546 aliases: &[],
547 cli: None,
548 python: None,
549 node: None,
550 }
551}
552
553pub const OPTIONS: &[OptionInfo] = &[
555 OptionInfo {
558 python: Some(""),
559 ..row("strict")
560 },
561 row("compact_tables"),
562 row("picture_alt_text"),
563 OptionInfo {
566 aliases: &["md_page_break_placeholder"],
567 python: Some(""),
568 ..row("page_break_placeholder")
569 },
570 row("fetch_images"),
571 row("image_sources"),
572 row("image_hosts"),
573 row("max_image_bytes"),
574 row("max_images"),
575 row("max_image_total_mb"),
576 row("min_image_bytes"),
577 row("list_attachments"),
578 row("skip_empty_cells"),
579 row("ebcdic_layout"),
580 row("encoding"),
581 row("ipynb_outputs"),
582 row("metadata"),
583 row("track_changes"),
584 row("extract_diagrams"),
585 OptionInfo {
586 cli: Some("--no-outline-level-headings"),
587 ..row("use_outline_level_for_headings")
588 },
589 OptionInfo {
592 node: Some(""),
593 ..row("use_web_browser")
594 },
595 row("xbrl_taxonomy"),
596 row("asr_model"),
597 row("asr_lang"),
598 row("video_frames"),
599 row("video_scene_threshold"),
600 row("video_frame_max_side"),
601 row("video_frame_dedupe"),
602 OptionInfo {
604 python: Some("page_range"),
605 ..row("pages")
606 },
607 row("document_timeout"),
608 row("text_layer_only"),
614 OptionInfo {
615 python: Some("do_ocr"),
616 ..row("no_ocr")
617 },
618 OptionInfo {
619 python: Some(""),
620 ..row("skip_ocr")
621 },
622 OptionInfo {
625 aliases: &["pdf_password"],
626 ..row("password")
627 },
628 row("force_full_page_ocr"),
629 OptionInfo {
630 python: Some("do_table_structure"),
631 ..row("no_table_former")
632 },
633 row("no_text_panels"),
634 row("heading_hierarchy"),
635 row("ocr_engine"),
636 row("ocr_lang"),
637 row("ocr_mode"),
638 row("ocr_scale"),
639 row("ocr_textline_orientation"),
640 row("images_scale"),
641 OptionInfo {
642 aliases: &["generate_page_images"],
643 python: Some("generate_page_images"),
644 ..row("page_images")
645 },
646 OptionInfo {
647 cli: Some("--enrich-picture-classes"),
648 ..row("do_picture_classification")
649 },
650 OptionInfo {
651 cli: Some("--enrich-code"),
652 ..row("do_code_enrichment")
653 },
654 OptionInfo {
655 cli: Some("--enrich-formula"),
656 ..row("do_formula_enrichment")
657 },
658 OptionInfo {
659 cli: Some("--picture-ocr"),
660 ..row("do_picture_ocr")
661 },
662 row("picture_ocr_classes"),
663 row("picture_ocr_min_side"),
664 row("picture_ocr_scale"),
665 OptionInfo {
666 cli: Some("--no-picture-images"),
667 ..row("keep_picture_images")
668 },
669 OptionInfo {
670 cli: Some("--picture-description"),
671 ..row("do_picture_description")
672 },
673 row("picture_description_endpoint"),
674 row("picture_description_model"),
675 row("picture_description_api_key"),
676 row("picture_description_prompt"),
677 row("picture_description_mode"),
678 row("picture_description_concurrency"),
679 row("redact_pii"),
680 row("redact_mode"),
681 row("redact_kinds"),
682 row("redact_pattern"),
683 row("redact_images"),
684 row("redact_allow_terms"),
685 row("redact_deny_terms"),
686 OptionInfo {
687 cli: Some("--redact-allow-pattern"),
688 ..row("redact_allow_patterns")
689 },
690 row("redact_ner_min_score"),
691 row("pipeline"),
692 row("vlm_endpoint"),
693 row("vlm_model"),
694 row("vlm_api_key"),
695 row("vlm_prompt"),
696 row("vlm_max_tokens"),
697];
698
699pub fn cli_flag(field: &str) -> String {
702 OPTIONS
703 .iter()
704 .find(|o| o.name == field)
705 .and_then(|o| o.cli)
706 .map(str::to_string)
707 .unwrap_or_else(|| format!("--{}", field.replace('_', "-")))
708}
709
710pub fn merge_options<T: Serialize + DeserializeOwned + Default>(over: T, base: T) -> T {
716 let mut merged = serde_json::to_value(base).unwrap_or(serde_json::Value::Null);
717 let over = serde_json::to_value(over).unwrap_or(serde_json::Value::Null);
718 if let (Some(into), Some(from)) = (merged.as_object_mut(), over.as_object()) {
719 for (key, value) in from {
720 if !value.is_null() {
721 into.insert(key.clone(), value.clone());
722 }
723 }
724 }
725 serde_json::from_value(merged).unwrap_or_default()
728}
729
730fn de_string_list<'de, D: serde::Deserializer<'de>>(d: D) -> Result<Option<Vec<String>>, D::Error> {
733 #[derive(Deserialize)]
734 #[serde(untagged)]
735 enum Raw {
736 List(Vec<String>),
737 Text(String),
738 }
739 Ok(Option::<Raw>::deserialize(d)?.map(|raw| {
740 let items: Vec<String> = match raw {
741 Raw::List(v) => v,
742 Raw::Text(s) => s.split(',').map(str::to_string).collect(),
743 };
744 items
745 .into_iter()
746 .map(|s| s.trim().to_string())
747 .filter(|s| !s.is_empty())
748 .collect()
749 }))
750}
751
752fn de_video_frames<'de, D: serde::Deserializer<'de>>(d: D) -> Result<Option<usize>, D::Error> {
757 #[derive(Deserialize)]
758 #[serde(untagged)]
759 enum Raw {
760 Count(u64),
761 Text(String),
762 }
763 let raw = Option::<Raw>::deserialize(d)?;
764 let n = match raw {
765 None => return Ok(None),
766 Some(Raw::Count(n)) => n,
767 Some(Raw::Text(s)) => {
768 let s = s.trim();
769 if s.eq_ignore_ascii_case("all") {
770 return Ok(Some(crate::ALL_VIDEO_FRAMES));
771 }
772 s.parse::<u64>().map_err(|_| {
773 serde::de::Error::custom(format!(
774 "video_frames: expected a non-negative integer or \"all\", got {s:?}"
775 ))
776 })?
777 }
778 };
779 Ok(Some(if n >= u32::MAX as u64 {
780 crate::ALL_VIDEO_FRAMES
781 } else {
782 n as usize
783 }))
784}
785
786impl ConvertOptions {
787 pub fn image_sources(&self) -> Result<Option<crate::ImageSources>, OptionsError> {
790 self.image_sources
791 .as_deref()
792 .filter(|s| !s.trim().is_empty())
793 .map(|s| {
794 s.parse::<crate::ImageSources>()
795 .map_err(|e| OptionsError::new("image_sources", e))
796 })
797 .transpose()
798 }
799
800 pub fn ipynb_outputs(&self) -> Result<Option<crate::IpynbOutputs>, OptionsError> {
803 self.ipynb_outputs
804 .as_deref()
805 .filter(|s| !s.trim().is_empty())
806 .map(|s| {
807 s.parse::<crate::IpynbOutputs>()
808 .map_err(|e| OptionsError::new("ipynb_outputs", e))
809 })
810 .transpose()
811 }
812
813 pub fn track_changes(&self) -> Result<Option<TrackChanges>, OptionsError> {
817 self.track_changes
818 .as_deref()
819 .filter(|s| !s.trim().is_empty())
820 .map(|s| {
821 s.parse::<TrackChanges>()
822 .map_err(|e| OptionsError::new("track_changes", e))
823 })
824 .transpose()
825 }
826
827 pub fn extract_diagrams(&self) -> Result<Option<DiagramFormat>, OptionsError> {
831 self.extract_diagrams
832 .as_deref()
833 .filter(|s| !s.trim().is_empty())
834 .map(|s| {
835 s.parse::<DiagramFormat>()
836 .map_err(|e| OptionsError::new("extract_diagrams", e))
837 })
838 .transpose()
839 }
840
841 pub fn metadata_output(&self) -> Result<Option<MetadataOutput>, OptionsError> {
845 self.metadata
846 .as_deref()
847 .filter(|s| !s.trim().is_empty())
848 .map(|s| {
849 MetadataOutput::parse(s).ok_or_else(|| {
850 OptionsError::new(
851 "metadata",
852 format!(
853 "metadata must be one of {}; got {s:?}",
854 MetadataOutput::ACCEPTED
855 ),
856 )
857 })
858 })
859 .transpose()
860 }
861
862 pub fn field_names() -> Vec<String> {
867 match serde_json::to_value(Self::default()) {
868 Ok(serde_json::Value::Object(map)) => map.keys().cloned().collect(),
869 _ => Vec::new(),
870 }
871 }
872
873 pub fn unknown_keys<'a>(
877 keys: impl IntoIterator<Item = &'a str>,
878 also_known: &[&str],
879 ) -> Vec<String> {
880 let fields = Self::field_names();
881 keys.into_iter()
882 .filter(|k| {
883 !fields.iter().any(|f| f == k)
884 && !also_known.contains(k)
885 && !OPTIONS.iter().any(|o| o.aliases.contains(k))
886 })
887 .map(str::to_string)
888 .collect()
889 }
890
891 pub fn merge_over(self, base: ConvertOptions) -> ConvertOptions {
893 merge_options(self, base)
894 }
895
896 pub fn text_candidates(text: &str) -> Vec<serde_json::Value> {
900 let trimmed = text.trim();
901 let mut candidates = vec![serde_json::Value::String(text.to_string())];
902 if let Ok(n) = trimmed.parse::<u64>() {
903 candidates.push(serde_json::Value::from(n));
904 } else if let Ok(f) = trimmed.parse::<f64>() {
905 if let Some(n) = serde_json::Number::from_f64(f) {
906 candidates.push(serde_json::Value::Number(n));
907 }
908 }
909 match trimmed.to_ascii_lowercase().as_str() {
910 "1" | "true" | "yes" | "on" => candidates.push(serde_json::Value::Bool(true)),
911 "0" | "false" | "no" | "off" => candidates.push(serde_json::Value::Bool(false)),
912 _ => {}
913 }
914 candidates
915 }
916
917 pub fn set_text(&mut self, name: &str, text: &str) -> Result<bool, OptionsError> {
926 let Some(field) = OPTIONS
927 .iter()
928 .find(|o| o.name == name || o.aliases.contains(&name))
929 .map(|o| o.name)
930 else {
931 return Ok(false);
932 };
933 for value in Self::text_candidates(text) {
934 let mut patch = serde_json::Map::new();
935 patch.insert(name.to_string(), value);
936 if let Ok(parsed) =
937 serde_json::from_value::<ConvertOptions>(serde_json::Value::Object(patch))
938 {
939 *self = parsed.merge_over(std::mem::take(self));
940 return Ok(true);
941 }
942 }
943 Err(OptionsError::new(
944 field,
945 format!("{name}: cannot read {text:?} as that option's value"),
946 ))
947 }
948
949 pub fn from_loose_json(
958 map: serde_json::Map<String, serde_json::Value>,
959 ) -> Result<Self, OptionsError> {
960 let mut out = Self::default();
961 for (name, value) in map {
962 let known = OPTIONS
963 .iter()
964 .find(|o| o.name == name || o.aliases.contains(&name.as_str()))
965 .map(|o| o.name);
966 let Some(field) = known else {
967 return Err(OptionsError::new(
968 "options",
969 format!("unknown field `{name}` (see docs/OPTIONS.md for the list)"),
970 ));
971 };
972 match value {
973 serde_json::Value::Null => {}
974 serde_json::Value::String(text) => {
975 out.set_text(&name, &text)?;
976 }
977 other => {
978 let text = match &other {
979 serde_json::Value::Number(n) => Some(n.to_string()),
980 serde_json::Value::Bool(b) => Some(b.to_string()),
981 _ => None,
982 };
983 let mut patch = serde_json::Map::new();
984 patch.insert(name.clone(), other);
985 match serde_json::from_value::<ConvertOptions>(serde_json::Value::Object(patch))
986 {
987 Ok(parsed) => out = parsed.merge_over(out),
988 Err(e) => match text {
989 Some(t) if out.set_text(&name, &t).is_ok() => {}
990 _ => return Err(OptionsError::new(field, format!("{name}: {e}"))),
991 },
992 }
993 }
994 }
995 }
996 Ok(out)
997 }
998
999 pub fn validate(&self) -> Result<(), OptionsError> {
1010 self.page_range()?;
1011 self.document_timeout()?;
1012 self.ipynb_outputs()?;
1013 self.track_changes()?;
1014 self.extract_diagrams()?;
1015 self.metadata_output()?;
1016 if let Some(s) = self.ocr_scale {
1017 if !(s.is_finite() && s > 0.0) {
1018 return Err(OptionsError::new(
1019 "ocr_scale",
1020 format!("ocr_scale must be a positive number, got {s}"),
1021 ));
1022 }
1023 }
1024 if let Some(s) = self.picture_ocr_scale {
1025 if !(s.is_finite() && s > 0.0) {
1026 return Err(OptionsError::new(
1027 "picture_ocr_scale",
1028 format!("picture_ocr_scale must be a positive number, got {s}"),
1029 ));
1030 }
1031 }
1032 self.picture_description_mode()?;
1033 if self.picture_description_concurrency == Some(0) {
1034 return Err(OptionsError::new(
1035 "picture_description_concurrency",
1036 "picture_description_concurrency must be a positive integer, got 0",
1037 ));
1038 }
1039 if let Some(s) = self.images_scale {
1040 if !(0.1..=4.0).contains(&s) {
1041 return Err(OptionsError::new(
1042 "images_scale",
1043 format!("images_scale must be a number in 0.1-4.0, got {s}"),
1044 ));
1045 }
1046 }
1047 self.image_sources()?;
1048 if self.max_image_bytes == Some(0) {
1049 return Err(OptionsError::new(
1050 "max_image_bytes",
1051 "max_image_bytes must be a positive number of bytes, got 0",
1052 ));
1053 }
1054 if let Some(t) = self.video_scene_threshold {
1055 if !(t.is_finite() && (0.0..=1.0).contains(&t)) {
1056 return Err(OptionsError::new(
1057 "video_scene_threshold",
1058 format!("video_scene_threshold must be a number in 0.0-1.0, got {t}"),
1059 ));
1060 }
1061 }
1062 if let Some(d) = self.video_frame_dedupe {
1063 if d > 64 {
1064 return Err(OptionsError::new(
1065 "video_frame_dedupe",
1066 format!("video_frame_dedupe must be a Hamming distance in 0-64, got {d}"),
1067 ));
1068 }
1069 }
1070 self.pipeline()?;
1071 if self.vlm_max_tokens == Some(0) {
1072 return Err(OptionsError::new(
1073 "vlm_max_tokens",
1074 "vlm_max_tokens must be a positive integer, got 0",
1075 ));
1076 }
1077 #[cfg(feature = "pdf")]
1078 {
1079 self.ocr_engine()?;
1080 self.ocr_mode()?;
1081 self.ocr_lang()?;
1082 }
1083 self.picture_ocr_classes()?;
1084 self.redaction()?;
1085 Ok(())
1086 }
1087
1088 pub fn picture_ocr_classes(&self) -> Result<Vec<String>, OptionsError> {
1094 let labels: Vec<String> = self
1095 .picture_ocr_classes
1096 .as_deref()
1097 .unwrap_or("")
1098 .split(',')
1099 .map(|s| s.trim().to_ascii_lowercase())
1100 .filter(|s| !s.is_empty())
1101 .collect();
1102 #[cfg(feature = "pdf")]
1103 if let Some(bad) = labels
1104 .iter()
1105 .find(|l| !docling_pdf::enrich::PICTURE_CLASSES.contains(&l.as_str()))
1106 {
1107 return Err(OptionsError::new(
1108 "picture_ocr_classes",
1109 format!(
1110 "picture_ocr_classes: {bad:?} is not a DocumentFigureClassifier label ({})",
1111 docling_pdf::enrich::PICTURE_CLASSES.join(", ")
1112 ),
1113 ));
1114 }
1115 Ok(labels)
1116 }
1117
1118 pub fn picture_description_mode(&self) -> Result<PictureDescriptionMode, OptionsError> {
1121 match self.picture_description_mode.as_deref().map(str::trim) {
1122 None | Some("") => Ok(PictureDescriptionMode::Caption),
1123 Some(m) => PictureDescriptionMode::parse(m).ok_or_else(|| {
1124 OptionsError::new(
1125 "picture_description_mode",
1126 format!("picture_description_mode must be caption or ocr, got {m:?}"),
1127 )
1128 }),
1129 }
1130 }
1131
1132 #[cfg(feature = "vlm")]
1138 pub fn picture_description_options(
1139 &self,
1140 ) -> Result<Option<crate::vlm::PictureDescriptionOptions>, OptionsError> {
1141 if self.do_picture_description != Some(true) {
1142 return Ok(None);
1143 }
1144 let set = |s: &Option<String>| s.clone().filter(|v| !v.trim().is_empty());
1145 let endpoint = set(&self.picture_description_endpoint).or_else(|| set(&self.vlm_endpoint));
1146 let model = set(&self.picture_description_model).or_else(|| set(&self.vlm_model));
1147 let mut o =
1148 crate::vlm::PictureDescriptionOptions::resolve(endpoint, model).map_err(|e| {
1149 let msg = match e {
1150 crate::ConversionError::Parse(m) => m,
1151 other => other.to_string(),
1152 }
1153 .replace(
1154 "pass --picture-description-endpoint",
1155 "set picture_description_endpoint",
1156 )
1157 .replace(
1158 "pass --picture-description-model",
1159 "set picture_description_model",
1160 );
1161 let field = if msg.contains("picture_description_model") {
1162 "picture_description_model"
1163 } else {
1164 "picture_description_endpoint"
1165 };
1166 OptionsError::new(field, msg)
1167 })?;
1168 if let Some(k) = set(&self.picture_description_api_key).or_else(|| set(&self.vlm_api_key)) {
1169 o.api_key = Some(k);
1170 }
1171 o.prompt = set(&self.picture_description_prompt);
1172 o.mode = self.picture_description_mode()?;
1173 if let Some(n) = self.picture_description_concurrency {
1174 o.concurrency = n;
1175 }
1176 Ok(Some(o))
1177 }
1178
1179 pub fn redaction(&self) -> Result<Option<RedactionOptions>, OptionsError> {
1184 let mut opts = RedactionOptions::default();
1185 if let Some(m) = self.redact_mode.as_deref().filter(|m| !m.trim().is_empty()) {
1186 opts.replacement = Replacement::parse(m).ok_or_else(|| {
1187 OptionsError::new(
1188 "redact_mode",
1189 format!(
1190 "redact_mode must be one of {} , got {m:?}",
1191 Replacement::ACCEPTED
1192 ),
1193 )
1194 })?;
1195 }
1196 if let Some(k) = self.redact_kinds.as_deref() {
1197 for name in k.split(',').map(str::trim).filter(|s| !s.is_empty()) {
1198 let kind = PiiKind::parse(name).ok_or_else(|| {
1199 OptionsError::new(
1200 "redact_kinds",
1201 format!(
1202 "redact_kinds: {name:?} is not a PII kind ({})",
1203 PiiKind::ACCEPTED
1204 ),
1205 )
1206 })?;
1207 if !opts.kinds.contains(&kind) {
1208 opts.kinds.push(kind);
1209 }
1210 }
1211 }
1212 if let Some(p) = self.redact_pattern.as_deref() {
1213 opts.custom_patterns = p.lines().filter_map(CustomPattern::parse).collect();
1214 docling_core::redact::PatternDetector::new(&opts)
1216 .map_err(|e| OptionsError::new("redact_pattern", e.0))?;
1217 }
1218 let list = |s: &Option<String>| -> Vec<String> {
1219 s.as_deref()
1220 .unwrap_or("")
1221 .split(',')
1222 .map(str::trim)
1223 .filter(|t| !t.is_empty())
1224 .map(str::to_string)
1225 .collect()
1226 };
1227 opts.allow_terms = list(&self.redact_allow_terms);
1228 opts.deny_terms = list(&self.redact_deny_terms);
1229 if let Some(p) = self.redact_allow_patterns.as_deref() {
1230 opts.allow_patterns = p
1231 .lines()
1232 .map(str::trim)
1233 .filter(|l| !l.is_empty())
1234 .map(str::to_string)
1235 .collect();
1236 docling_core::redact::AllowList::new(&opts).map_err(|e| {
1237 OptionsError::new(
1238 "redact_allow_patterns",
1239 format!("redact_allow_patterns: {}", e.0),
1240 )
1241 })?;
1242 }
1243 if let Some(score) = self.redact_ner_min_score {
1244 if !(score.is_finite() && (0.0..=1.0).contains(&score)) {
1245 return Err(OptionsError::new(
1246 "redact_ner_min_score",
1247 format!("redact_ner_min_score must be a number in 0.0-1.0, got {score}"),
1248 ));
1249 }
1250 opts.ner_min_score = score;
1251 }
1252 if let Some(i) = self
1253 .redact_images
1254 .as_deref()
1255 .filter(|i| !i.trim().is_empty())
1256 {
1257 opts.images = ImageRedaction::parse(i).ok_or_else(|| {
1258 OptionsError::new(
1259 "redact_images",
1260 format!(
1261 "redact_images must be one of {}, got {i:?}",
1262 ImageRedaction::ACCEPTED
1263 ),
1264 )
1265 })?;
1266 }
1267 Ok(self.redact_pii.unwrap_or(false).then_some(opts))
1268 }
1269
1270 pub fn apply(&self, mut c: DocumentConverter) -> Result<DocumentConverter, OptionsError> {
1275 self.validate()?;
1276 if let Some(v) = self.strict {
1277 c = c.strict(v);
1278 }
1279 if let Some(v) = self.compact_tables {
1280 c = c.compact_tables(v);
1281 }
1282 if let Some(v) = self.picture_alt_text {
1283 c = c.picture_alt_text(v);
1284 }
1285 if self.page_break_placeholder.is_some() {
1286 c = c.page_break_placeholder(self.page_break_placeholder.clone());
1287 }
1288 if let Some(v) = self.fetch_images {
1289 c = c.fetch_images(v);
1290 }
1291 if let Some(sources) = self.image_sources()? {
1292 c = c.image_sources(sources);
1293 }
1294 if let Some(hosts) = &self.image_hosts {
1295 c = c.image_hosts(hosts.iter().cloned());
1296 }
1297 if let Some(v) = self.max_image_bytes {
1298 c = c.max_image_bytes(v);
1299 }
1300 if let Some(v) = self.max_images {
1301 c = c.max_images(v);
1302 }
1303 if let Some(v) = self.max_image_total_mb {
1304 c = c.max_image_total_mb(v);
1305 }
1306 if let Some(v) = self.min_image_bytes {
1307 c = c.min_image_bytes(v);
1308 }
1309 if let Some(v) = self.list_attachments {
1310 c = c.list_attachments(v);
1311 }
1312 if let Some(v) = self.skip_empty_cells {
1313 c = c.skip_empty_cells(v);
1314 }
1315 if self.ebcdic_layout.is_some() {
1316 c = c.ebcdic_layout_opt(self.ebcdic_layout.clone());
1317 }
1318 if self.encoding.is_some() {
1319 c = c.encoding(self.encoding.clone());
1320 }
1321 if let Some(outputs) = self.ipynb_outputs()? {
1322 c = c.ipynb_outputs(outputs);
1323 }
1324 if let Some(mode) = self.metadata_output()? {
1325 c = c.metadata_output(mode);
1326 }
1327 if let Some(policy) = self.track_changes()? {
1328 c = c.track_changes(Some(policy));
1329 }
1330 if let Some(format) = self.extract_diagrams()? {
1331 c = c.extract_diagrams(format);
1332 }
1333 if let Some(v) = self.use_outline_level_for_headings {
1334 c = c.use_outline_level_for_headings(v);
1335 }
1336 if let Some(v) = self.use_web_browser {
1337 c = c.use_web_browser(v);
1338 }
1339 if let Some(dir) = &self.xbrl_taxonomy {
1340 c = c.xbrl_taxonomy(dir);
1341 }
1342 if self.asr_model.is_some() {
1343 c = c.asr_model(self.asr_model.clone());
1344 }
1345 if self.asr_lang.is_some() {
1346 c = c.asr_lang(self.asr_lang.clone());
1347 }
1348 if let Some(n) = self.video_frames {
1349 c = c.video_frames(n);
1350 }
1351 if let Some(t) = self.video_scene_threshold {
1352 c = c.video_scene_threshold(t);
1353 }
1354 if let Some(px) = self.video_frame_max_side {
1355 c = c.video_frame_max_side(px);
1356 }
1357 if let Some(d) = self.video_frame_dedupe {
1358 c = c.video_frame_dedupe(Some(d));
1359 }
1360 if let Some((first, last)) = self.page_range()? {
1361 c = c.page_range(first, last);
1362 }
1363 if self.document_timeout.is_some() {
1364 c = c.document_timeout(self.document_timeout()?);
1365 }
1366 if let Some(v) = self.text_layer_only {
1367 c = c.text_layer_only(v);
1368 }
1369 if self.no_ocr.is_some() || self.skip_ocr.is_some() {
1370 c = c.no_ocr(self.ocr_disabled());
1371 }
1372 if self.password.is_some() {
1373 c = c.password(self.password.clone());
1374 }
1375 if let Some(v) = self.force_full_page_ocr {
1376 c = c.force_full_page_ocr(v);
1377 }
1378 if let Some(v) = self.no_table_former {
1379 c = c.no_table_former(v);
1380 }
1381 if let Some(v) = self.no_text_panels {
1382 c = c.no_text_panels(v);
1383 }
1384 if let Some(v) = self.heading_hierarchy {
1385 c = c.heading_hierarchy(v);
1386 }
1387 if let Some(v) = &self.ocr_engine {
1388 c = c.ocr_engine(v.clone());
1389 }
1390 if let Some(v) = &self.ocr_lang {
1391 c = c.ocr_lang(v.clone());
1392 }
1393 if let Some(v) = &self.ocr_mode {
1394 c = c.ocr_mode(v.clone());
1395 }
1396 if let Some(s) = self.ocr_scale {
1397 c = c.ocr_scale(s);
1398 }
1399 if let Some(v) = self.ocr_textline_orientation {
1400 c = c.ocr_textline_orientation(v);
1401 }
1402 if let Some(s) = self.images_scale {
1403 c = c.images_scale(s);
1404 }
1405 if let Some(v) = self.page_images {
1406 c = c.generate_page_images(v);
1407 }
1408 if let Some(v) = self.do_picture_classification {
1409 c = c.do_picture_classification(v);
1410 }
1411 if let Some(v) = self.do_code_enrichment {
1412 c = c.do_code_enrichment(v);
1413 }
1414 if let Some(v) = self.do_formula_enrichment {
1415 c = c.do_formula_enrichment(v);
1416 }
1417 if let Some(v) = self.do_picture_ocr {
1418 c = c.do_picture_ocr(v);
1419 }
1420 if self.picture_ocr_classes.is_some() {
1421 c = c.picture_ocr_classes(self.picture_ocr_classes()?);
1422 }
1423 if let Some(v) = self.picture_ocr_min_side {
1424 c = c.picture_ocr_min_side(v);
1425 }
1426 if let Some(s) = self.picture_ocr_scale {
1427 c = c.picture_ocr_scale(s);
1428 }
1429 if let Some(v) = self.keep_picture_images {
1430 c = c.keep_picture_images(v);
1431 }
1432 #[cfg(feature = "vlm")]
1433 if let Some(o) = self.picture_description_options()? {
1434 c = c.picture_description(o);
1435 }
1436 if self.redact_pii.is_some() {
1437 c = c.redact_pii_opt(self.redaction()?);
1438 }
1439 Ok(c)
1440 }
1441
1442 pub fn page_range(&self) -> Result<Option<(usize, usize)>, OptionsError> {
1447 self.pages
1448 .as_deref()
1449 .map(|p| {
1450 crate::parse_page_range(p)
1451 .map_err(|e| OptionsError::new("pages", format!("pages: {e}")))
1452 })
1453 .transpose()
1454 }
1455
1456 pub fn document_timeout(&self) -> Result<Option<std::time::Duration>, OptionsError> {
1458 match self.document_timeout {
1459 Some(s) if s.is_finite() && s > 0.0 => Ok(Some(std::time::Duration::from_secs_f64(s))),
1460 Some(s) => Err(OptionsError::new(
1461 "document_timeout",
1462 format!("document_timeout must be a positive number of seconds, got {s}"),
1463 )),
1464 None => Ok(None),
1465 }
1466 }
1467
1468 pub fn pipeline(&self) -> Result<PipelineKind, OptionsError> {
1470 match self.pipeline.as_deref().map(str::trim) {
1471 None | Some("standard") => Ok(PipelineKind::Standard),
1472 Some("vlm") => Ok(PipelineKind::Vlm),
1473 Some(other) => Err(OptionsError::new(
1474 "pipeline",
1475 format!("unknown pipeline {other:?} (expected: standard, vlm)"),
1476 )),
1477 }
1478 }
1479
1480 pub fn enrichments(&self) -> crate::EnrichmentOptions {
1482 crate::EnrichmentOptions {
1483 picture_classification: self.do_picture_classification.unwrap_or(false),
1484 code: self.do_code_enrichment.unwrap_or(false),
1485 formula: self.do_formula_enrichment.unwrap_or(false),
1486 }
1487 }
1488
1489 #[cfg(feature = "pdf")]
1491 pub fn ocr_engine(&self) -> Result<Option<crate::OcrEngine>, OptionsError> {
1492 self.ocr_engine
1493 .as_deref()
1494 .map(|v| {
1495 crate::OcrEngine::parse(v).ok_or_else(|| {
1496 OptionsError::new(
1497 "ocr_engine",
1498 format!("ocr_engine {v:?} is not {}", crate::OcrEngine::ACCEPTED),
1499 )
1500 })
1501 })
1502 .transpose()
1503 }
1504
1505 #[cfg(feature = "pdf")]
1508 pub fn effective_ocr_engine(&self) -> Result<crate::OcrEngine, OptionsError> {
1509 Ok(self
1510 .ocr_engine()?
1511 .unwrap_or_else(crate::OcrEngine::from_env))
1512 }
1513
1514 #[cfg(feature = "pdf")]
1518 pub fn ocr_lang(&self) -> Result<Option<crate::OcrLang>, OptionsError> {
1519 let Some(v) = self.ocr_lang.as_deref() else {
1520 return Ok(None);
1521 };
1522 let engine = self.effective_ocr_engine()?;
1523 engine.validate_lang(v).map_err(|e| {
1524 OptionsError::new(
1525 "ocr_lang",
1526 if e.contains("ocr_lang") {
1527 e
1528 } else {
1529 format!("ocr_lang: {e}")
1530 },
1531 )
1532 })?;
1533 Ok(match engine {
1534 crate::OcrEngine::PpOcr => crate::OcrLang::parse(v),
1535 crate::OcrEngine::Tesseract => None,
1536 })
1537 }
1538
1539 #[cfg(feature = "pdf")]
1542 pub fn tesseract_lang(&self) -> Result<Option<String>, OptionsError> {
1543 let Some(v) = self.ocr_lang.as_deref() else {
1544 return Ok(None);
1545 };
1546 match self.effective_ocr_engine()? {
1547 crate::OcrEngine::Tesseract => crate::tesseract_lang_arg(v)
1548 .map(Some)
1549 .map_err(|e| OptionsError::new("ocr_lang", e)),
1550 crate::OcrEngine::PpOcr => Ok(None),
1551 }
1552 }
1553
1554 pub fn ocr_disabled(&self) -> bool {
1557 self.no_ocr.unwrap_or(false) || self.skip_ocr.unwrap_or(false)
1558 }
1559
1560 #[cfg(feature = "pdf")]
1562 pub fn ocr_mode(&self) -> Result<Option<crate::OcrMode>, OptionsError> {
1563 self.ocr_mode
1564 .as_deref()
1565 .map(|v| {
1566 crate::OcrMode::parse(v).ok_or_else(|| {
1567 OptionsError::new(
1568 "ocr_mode",
1569 format!(
1570 "ocr_mode {v:?} is not \
1571 default|full_page|layout_regions|pdf_aware_layout_regions"
1572 ),
1573 )
1574 })
1575 })
1576 .transpose()
1577 }
1578
1579 #[cfg(feature = "pdf")]
1581 pub fn image_output(&self) -> crate::ImageOutput {
1582 crate::ImageOutput {
1583 scale: self.images_scale,
1584 page_images: self.page_images.unwrap_or(false),
1585 }
1586 }
1587
1588 #[cfg(feature = "vlm")]
1596 pub fn vlm_options(&self) -> Result<Option<crate::vlm::VlmOptions>, OptionsError> {
1597 if self.pipeline()? != PipelineKind::Vlm {
1598 return Ok(None);
1599 }
1600 let set = |s: &Option<String>| s.clone().filter(|v| !v.trim().is_empty());
1601 let mut v = crate::vlm::VlmOptions::resolve(set(&self.vlm_endpoint), set(&self.vlm_model))
1602 .map_err(|e| {
1603 let msg = match e {
1606 crate::ConversionError::Parse(m) => m,
1607 other => other.to_string(),
1608 }
1609 .replace("pass --vlm-endpoint", "set vlm_endpoint")
1610 .replace("pass --vlm-model", "set vlm_model");
1611 let field = if msg.contains("vlm_model") {
1612 "vlm_model"
1613 } else {
1614 "vlm_endpoint"
1615 };
1616 OptionsError::new(field, msg)
1617 })?;
1618 if let Some(p) = set(&self.vlm_prompt) {
1619 v.prompt = Some(p);
1620 }
1621 if let Some(k) = set(&self.vlm_api_key) {
1622 v.api_key = Some(k);
1623 }
1624 match self.vlm_max_tokens {
1625 Some(0) => {
1628 return Err(OptionsError::new(
1629 "vlm_max_tokens",
1630 "vlm_max_tokens must be a positive integer, got 0",
1631 ))
1632 }
1633 Some(n) => v.max_tokens = n,
1634 None => {}
1635 }
1636 v.page_range = self.page_range()?;
1637 Ok(Some(v))
1638 }
1639}
1640
1641impl DocumentConverter {
1642 pub fn from_options(options: &ConvertOptions) -> Result<Self, OptionsError> {
1645 options.apply(Self::new())
1646 }
1647}
1648
1649#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
1652#[serde(rename_all = "lowercase")]
1653pub enum DiagramFormat {
1654 #[default]
1656 Mermaid,
1657 None,
1659}
1660
1661impl DiagramFormat {
1662 pub const ACCEPTED: &'static str = "mermaid, none";
1664
1665 pub fn as_str(self) -> &'static str {
1666 match self {
1667 DiagramFormat::Mermaid => "mermaid",
1668 DiagramFormat::None => "none",
1669 }
1670 }
1671}
1672
1673impl FromStr for DiagramFormat {
1674 type Err = String;
1675
1676 fn from_str(s: &str) -> Result<Self, Self::Err> {
1677 match s.trim().to_ascii_lowercase().as_str() {
1678 "mermaid" => Ok(DiagramFormat::Mermaid),
1679 "none" | "off" => Ok(DiagramFormat::None),
1680 other => Err(format!(
1681 "extract_diagrams must be one of {}; got {other:?}",
1682 DiagramFormat::ACCEPTED
1683 )),
1684 }
1685 }
1686}
1687
1688impl fmt::Display for DiagramFormat {
1689 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1690 f.write_str(self.as_str())
1691 }
1692}
1693
1694#[cfg(test)]
1695mod tests {
1696 use super::*;
1697
1698 #[test]
1701 fn options_table_matches_the_struct() {
1702 let fields = ConvertOptions::field_names();
1703 for o in OPTIONS {
1704 assert!(
1705 fields.iter().any(|f| f == o.name),
1706 "no field for row {}",
1707 o.name
1708 );
1709 for a in o.aliases {
1710 assert!(!fields.iter().any(|f| f == a), "alias {a} is also a field");
1711 }
1712 }
1713 for f in &fields {
1714 assert!(
1715 OPTIONS.iter().any(|o| o.name == f),
1716 "no OPTIONS row for field {f}"
1717 );
1718 }
1719 assert_eq!(cli_flag("ocr_scale"), "--ocr-scale");
1720 assert_eq!(cli_flag("do_code_enrichment"), "--enrich-code");
1721 }
1722
1723 #[test]
1726 fn text_values_read_as_the_options_type() {
1727 let mut o = ConvertOptions::default();
1728 assert!(o.set_text("strict", "true").unwrap());
1729 assert!(o.set_text("compact_tables", "YES").unwrap());
1730 assert!(o.set_text("ocr_scale", "1.5").unwrap());
1731 assert!(o.set_text("video_frames", "4").unwrap());
1732 assert!(o.set_text("pages", "5").unwrap());
1733 assert!(o.set_text("pdf_password", "0").unwrap(), "an alias");
1734 assert!(!o.set_text("not_an_option", "x").unwrap());
1735 assert_eq!(o.strict, Some(true));
1736 assert_eq!(o.compact_tables, Some(true));
1737 assert_eq!(o.ocr_scale, Some(1.5));
1738 assert_eq!(o.video_frames, Some(4));
1739 assert_eq!(o.pages.as_deref(), Some("5"));
1740 assert_eq!(o.password.as_deref(), Some("0"));
1741 let e = o.set_text("strict", "maybe").unwrap_err();
1742 assert_eq!(e.field, "strict");
1743 assert_eq!(
1744 e.cli_message(),
1745 "--strict: cannot read \"maybe\" as that option's value"
1746 );
1747
1748 let o = ConvertOptions::from_loose_json(
1749 serde_json::json!({
1750 "force_full_page_ocr": "true",
1751 "ocr_scale": "2",
1752 "pages": 3,
1753 "strict": 1,
1754 "text_layer_only": false,
1755 "ocr_lang": null,
1756 "redact_kinds": "email, phone"
1757 })
1758 .as_object()
1759 .unwrap()
1760 .clone(),
1761 )
1762 .unwrap();
1763 assert_eq!(o.force_full_page_ocr, Some(true));
1764 assert_eq!(o.ocr_scale, Some(2.0));
1765 assert_eq!(o.pages.as_deref(), Some("3"));
1766 assert_eq!(o.strict, Some(true));
1767 assert_eq!(o.text_layer_only, Some(TextLayerOnly::Off));
1768 assert_eq!(o.ocr_lang, None);
1769 assert_eq!(o.redact_kinds.as_deref(), Some("email, phone"));
1770 let bad = |v: serde_json::Value| {
1771 ConvertOptions::from_loose_json(v.as_object().unwrap().clone())
1772 .unwrap_err()
1773 .to_string()
1774 };
1775 assert!(bad(serde_json::json!({"strict": "maybe"})).contains("strict: cannot read"));
1776 assert!(bad(serde_json::json!({"strict": [1]})).contains("strict: invalid type"));
1777 assert!(bad(serde_json::json!({"strictt": true})).contains("unknown field `strictt`"));
1778 }
1779
1780 #[test]
1781 fn aliases_deserialize_and_unknown_keys_are_reported() {
1782 let o: ConvertOptions = serde_json::from_str(
1783 r#"{"md_page_break_placeholder": "<!-- p -->", "generate_page_images": true}"#,
1784 )
1785 .unwrap();
1786 assert_eq!(o.page_break_placeholder.as_deref(), Some("<!-- p -->"));
1787 assert_eq!(o.page_images, Some(true));
1788 assert_eq!(
1789 ConvertOptions::unknown_keys(
1790 ["to", "strict", "md_page_break_placeholder", "strictness"],
1791 &["to"]
1792 ),
1793 vec!["strictness".to_string()]
1794 );
1795 }
1796
1797 #[test]
1801 fn video_frames_spellings_and_video_ranges() {
1802 let parse = |json: &str| serde_json::from_str::<ConvertOptions>(json).unwrap();
1803 assert_eq!(parse(r#"{"video_frames": 3}"#).video_frames, Some(3));
1804 assert_eq!(parse(r#"{"video_frames": "12"}"#).video_frames, Some(12));
1805 assert_eq!(
1806 parse(r#"{"video_frames": "all"}"#).video_frames,
1807 Some(crate::ALL_VIDEO_FRAMES)
1808 );
1809 assert_eq!(
1810 parse(r#"{"video_frames": 4294967295}"#).video_frames,
1811 Some(crate::ALL_VIDEO_FRAMES)
1812 );
1813 assert_eq!(parse("{}").video_frames, None);
1814 assert!(serde_json::from_str::<ConvertOptions>(r#"{"video_frames": "some"}"#).is_err());
1815 let merged = ConvertOptions {
1817 video_frames: Some(crate::ALL_VIDEO_FRAMES),
1818 ..Default::default()
1819 }
1820 .merge_over(ConvertOptions::default());
1821 assert_eq!(merged.video_frames, Some(crate::ALL_VIDEO_FRAMES));
1822
1823 let ok = ConvertOptions {
1824 video_scene_threshold: Some(0.6),
1825 video_frame_max_side: Some(640),
1826 video_frame_dedupe: Some(5),
1827 ..Default::default()
1828 };
1829 ok.validate().unwrap();
1830 let bad = ConvertOptions {
1831 video_scene_threshold: Some(1.5),
1832 ..Default::default()
1833 };
1834 assert_eq!(bad.validate().unwrap_err().field, "video_scene_threshold");
1835 let bad = ConvertOptions {
1836 video_frame_dedupe: Some(65),
1837 ..Default::default()
1838 };
1839 assert_eq!(bad.validate().unwrap_err().field, "video_frame_dedupe");
1840 }
1841
1842 #[test]
1843 fn merge_prefers_the_overlay_and_keeps_the_rest() {
1844 let base = ConvertOptions {
1845 strict: Some(true),
1846 ocr_scale: Some(2.0),
1847 pages: Some("1-3".into()),
1848 ..Default::default()
1849 };
1850 let over = ConvertOptions {
1851 strict: Some(false),
1852 video_frames: Some(0),
1853 ..Default::default()
1854 };
1855 let merged = over.merge_over(base);
1856 assert_eq!(merged.strict, Some(false));
1857 assert_eq!(merged.ocr_scale, Some(2.0));
1858 assert_eq!(merged.pages.as_deref(), Some("1-3"));
1859 assert_eq!(merged.video_frames, Some(0));
1860 }
1861
1862 #[test]
1863 fn validation_rules_and_cli_spelling() {
1864 let bad = |o: ConvertOptions| o.validate().unwrap_err();
1865 let e = bad(ConvertOptions {
1866 ocr_scale: Some(0.0),
1867 ..Default::default()
1868 });
1869 assert_eq!(e.field, "ocr_scale");
1870 assert_eq!(e.to_string(), "ocr_scale must be a positive number, got 0");
1871 assert_eq!(
1872 e.cli_message(),
1873 "--ocr-scale must be a positive number, got 0"
1874 );
1875 let e = bad(ConvertOptions {
1877 picture_ocr_scale: Some(-1.0),
1878 ..Default::default()
1879 });
1880 assert_eq!(
1881 e.cli_message(),
1882 "--picture-ocr-scale must be a positive number, got -1"
1883 );
1884
1885 let e = bad(ConvertOptions {
1886 images_scale: Some(9.0),
1887 ..Default::default()
1888 });
1889 assert_eq!(e.field, "images_scale");
1890 let e = bad(ConvertOptions {
1891 document_timeout: Some(-1.0),
1892 ..Default::default()
1893 });
1894 assert!(e.message.contains("document_timeout must be a positive"));
1895 let e = bad(ConvertOptions {
1896 pages: Some("5-3".into()),
1897 ..Default::default()
1898 });
1899 assert_eq!(e.field, "pages");
1900 assert!(
1901 e.cli_message().starts_with("--pages: "),
1902 "{}",
1903 e.cli_message()
1904 );
1905 let e = bad(ConvertOptions {
1906 pipeline: Some("magic".into()),
1907 ..Default::default()
1908 });
1909 assert!(e.message.contains("unknown pipeline"), "{e}");
1910 let e = bad(ConvertOptions {
1911 vlm_max_tokens: Some(0),
1912 ..Default::default()
1913 });
1914 assert_eq!(
1915 e.cli_message(),
1916 "--vlm-max-tokens must be a positive integer, got 0"
1917 );
1918 assert!(ConvertOptions::default().validate().is_ok());
1919 }
1920
1921 #[cfg(feature = "pdf")]
1922 #[test]
1923 fn ocr_options_are_validated_against_the_engine() {
1924 let e = ConvertOptions {
1925 ocr_engine: Some("easyocr".into()),
1926 ..Default::default()
1927 }
1928 .validate()
1929 .unwrap_err();
1930 assert_eq!(e.field, "ocr_engine");
1931 assert!(
1932 e.cli_message().starts_with("--ocr-engine \"easyocr\""),
1933 "{}",
1934 e.cli_message()
1935 );
1936 let e = ConvertOptions {
1938 ocr_engine: Some("ppocr".into()),
1939 ocr_lang: Some("deu".into()),
1940 ..Default::default()
1941 }
1942 .validate()
1943 .unwrap_err();
1944 assert_eq!(e.field, "ocr_lang");
1945 assert!(e.message.contains("ocr_lang"), "{e}");
1946 let tess = ConvertOptions {
1947 ocr_engine: Some("tesseract".into()),
1948 ocr_lang: Some("deu+fra".into()),
1949 ..Default::default()
1950 };
1951 assert!(tess.validate().is_ok());
1952 assert_eq!(tess.ocr_lang().unwrap(), None);
1953 assert_eq!(tess.tesseract_lang().unwrap().as_deref(), Some("deu+fra"));
1954 let e = ConvertOptions {
1955 ocr_mode: Some("sideways".into()),
1956 ..Default::default()
1957 }
1958 .validate()
1959 .unwrap_err();
1960 assert_eq!(e.field, "ocr_mode");
1961 }
1962
1963 #[test]
1964 fn apply_sets_only_what_is_given() {
1965 let base = DocumentConverter::new().strict(true);
1968 let mut doc = docling_core::DoclingDocument::new("t");
1969 ConvertOptions::default()
1970 .apply(base.clone())
1971 .unwrap()
1972 .finish_document(&mut doc);
1973 assert!(doc.strict_markdown, "unset keeps the base's strict");
1974 let mut doc = docling_core::DoclingDocument::new("t");
1975 ConvertOptions {
1976 strict: Some(false),
1977 page_break_placeholder: Some("<!-- p -->".into()),
1978 ..Default::default()
1979 }
1980 .apply(base)
1981 .unwrap()
1982 .finish_document(&mut doc);
1983 assert!(!doc.strict_markdown);
1984 assert_eq!(doc.page_break_placeholder.as_deref(), Some("<!-- p -->"));
1985 assert!(DocumentConverter::from_options(&ConvertOptions {
1987 pages: Some("0-2".into()),
1988 ..Default::default()
1989 })
1990 .is_err());
1991 }
1992
1993 #[cfg(feature = "vlm")]
1994 #[test]
1995 fn vlm_options_follow_the_pipeline_switch() {
1996 let stray = ConvertOptions {
1998 vlm_endpoint: Some("http://127.0.0.1:9/v1".into()),
1999 ..Default::default()
2000 };
2001 assert!(matches!(stray.vlm_options(), Ok(None)));
2002 let full = ConvertOptions {
2003 pipeline: Some("vlm".into()),
2004 vlm_endpoint: Some("http://example.com/v1".into()),
2005 vlm_model: Some("m".into()),
2006 vlm_api_key: Some("sk-test".into()),
2007 vlm_prompt: Some("Read the page.".into()),
2008 vlm_max_tokens: Some(512),
2009 pages: Some("2-5".into()),
2010 ..Default::default()
2011 };
2012 let v = full.vlm_options().unwrap().expect("resolves");
2013 assert_eq!(v.endpoint, "http://example.com/v1");
2014 assert_eq!(v.model, "m");
2015 assert_eq!(v.api_key.as_deref(), Some("sk-test"));
2016 assert_eq!(v.prompt.as_deref(), Some("Read the page."));
2017 assert_eq!(v.max_tokens, 512);
2018 assert_eq!(v.page_range, Some((2, 5)));
2019 let e = ConvertOptions {
2020 vlm_max_tokens: Some(0),
2021 ..full.clone()
2022 }
2023 .vlm_options()
2024 .unwrap_err();
2025 assert_eq!(e.field, "vlm_max_tokens");
2026 if std::env::var_os("DOCLING_RS_VLM_MODEL").is_none() {
2029 let e = ConvertOptions {
2030 vlm_model: None,
2031 ..full
2032 }
2033 .vlm_options()
2034 .unwrap_err();
2035 assert_eq!(e.field, "vlm_model");
2036 assert!(e.message.contains("set vlm_model"), "{e}");
2037 assert!(
2038 e.cli_message().contains("--vlm-model"),
2039 "{}",
2040 e.cli_message()
2041 );
2042 assert!(e.message.contains("DOCLING_RS_VLM_MODEL"), "{e}");
2043 }
2044 }
2045
2046 #[test]
2052 fn ocr_spellings_and_password_parse() {
2053 let both: ConvertOptions = serde_json::from_str(
2054 r#"{"no_ocr": false, "skip_ocr": true, "text_layer_only": true,
2055 "pdf_password": "1234"}"#,
2056 )
2057 .expect("both spellings parse");
2058 assert!(both.ocr_disabled());
2059 assert_eq!(both.text_layer_only, Some(TextLayerOnly::On));
2060 assert_eq!(both.password.as_deref(), Some("1234"));
2061 let neutral: ConvertOptions = serde_json::from_str(r#"{"password": "x"}"#).unwrap();
2062 assert_eq!(neutral.password.as_deref(), Some("x"));
2063 let new: ConvertOptions = serde_json::from_str(r#"{"no_ocr": true}"#).unwrap();
2064 assert!(new.ocr_disabled());
2065 assert!(!ConvertOptions::default().ocr_disabled());
2066 let merged = new.merge_over(both);
2068 assert!(merged.ocr_disabled());
2069 assert_eq!(merged.password.as_deref(), Some("1234"));
2070 }
2071
2072 #[test]
2076 fn text_layer_only_reads_bools_and_auto() {
2077 let typed = |v: serde_json::Value| -> Option<TextLayerOnly> {
2078 serde_json::from_value::<ConvertOptions>(serde_json::json!({ "text_layer_only": v }))
2079 .unwrap()
2080 .text_layer_only
2081 };
2082 assert_eq!(typed(serde_json::json!(true)), Some(TextLayerOnly::On));
2083 assert_eq!(typed(serde_json::json!(false)), Some(TextLayerOnly::Off));
2084 assert_eq!(typed(serde_json::json!("auto")), Some(TextLayerOnly::Auto));
2085 assert_eq!(typed(serde_json::json!("AUTO")), Some(TextLayerOnly::Auto));
2086 assert_eq!(typed(serde_json::json!(null)), None);
2087 let err = serde_json::from_value::<ConvertOptions>(
2088 serde_json::json!({ "text_layer_only": "sometimes" }),
2089 )
2090 .unwrap_err()
2091 .to_string();
2092 assert!(err.contains("true, false, auto"), "{err}");
2093
2094 let loose = |v: serde_json::Value| {
2095 ConvertOptions::from_loose_json(
2096 serde_json::json!({ "text_layer_only": v })
2097 .as_object()
2098 .unwrap()
2099 .clone(),
2100 )
2101 };
2102 assert_eq!(
2103 loose(serde_json::json!("auto")).unwrap().text_layer_only,
2104 Some(TextLayerOnly::Auto)
2105 );
2106 assert_eq!(
2107 loose(serde_json::json!("yes")).unwrap().text_layer_only,
2108 Some(TextLayerOnly::On)
2109 );
2110 assert_eq!(
2111 loose(serde_json::json!(1)).unwrap().text_layer_only,
2112 Some(TextLayerOnly::On)
2113 );
2114 assert_eq!(
2115 loose(serde_json::json!("off")).unwrap().text_layer_only,
2116 Some(TextLayerOnly::Off)
2117 );
2118 assert!(loose(serde_json::json!("maybe")).is_err());
2119
2120 let auto = ConvertOptions {
2123 text_layer_only: Some(TextLayerOnly::Auto),
2124 ..Default::default()
2125 };
2126 let merged = merge_options(auto.clone(), ConvertOptions::default());
2127 assert_eq!(merged.text_layer_only, Some(TextLayerOnly::Auto));
2128 assert_eq!(
2129 serde_json::to_value(&auto).unwrap()["text_layer_only"],
2130 serde_json::json!("auto")
2131 );
2132 let on = ConvertOptions {
2133 text_layer_only: Some(TextLayerOnly::On),
2134 ..Default::default()
2135 };
2136 assert_eq!(
2137 serde_json::to_value(&on).unwrap()["text_layer_only"],
2138 serde_json::json!(true)
2139 );
2140 assert_eq!("auto".parse::<TextLayerOnly>(), Ok(TextLayerOnly::Auto));
2141 assert_eq!(TextLayerOnly::from(true), TextLayerOnly::On);
2142 assert_eq!(TextLayerOnly::Auto.to_string(), "auto");
2143 }
2144}