cloudiful_docling_convert/api/
source.rs

1use std::collections::BTreeMap;
2
3use serde::Serialize;
4use serde_json::Value;
5
6use crate::document::{ChunkerKind, ChunkingOptions, InputKind, PipelineKind};
7
8use super::docling::DoclingConvertRequest;
9
10#[derive(Debug, Clone, Serialize)]
11#[serde(tag = "kind")]
12pub enum HttpSource {
13    #[serde(rename = "http")]
14    Http {
15        url: String,
16        #[serde(default)]
17        headers: BTreeMap<String, Value>,
18    },
19}
20
21impl HttpSource {
22    pub fn new(url: impl Into<String>) -> Self {
23        Self::Http {
24            url: url.into(),
25            headers: BTreeMap::new(),
26        }
27    }
28}
29
30#[derive(Debug, Clone, Serialize)]
31#[serde(tag = "kind")]
32pub enum TargetKind {
33    #[serde(rename = "inbody")]
34    Inbody {},
35    #[serde(rename = "zip")]
36    Zip {},
37}
38
39#[derive(Debug, Clone, Serialize)]
40pub(crate) struct ConvertDocumentsOptions {
41    pub from_formats: Vec<String>,
42    pub to_formats: Vec<String>,
43    #[serde(skip_serializing_if = "Option::is_none")]
44    pub pipeline: Option<String>,
45    #[serde(skip_serializing_if = "Option::is_none")]
46    pub page_range: Option<[u64; 2]>,
47}
48
49#[derive(Debug, Clone, Serialize)]
50pub(crate) struct ConvertSourcesRequest {
51    pub options: ConvertDocumentsOptions,
52    pub sources: Vec<HttpSource>,
53    pub target: TargetKind,
54}
55
56#[derive(Debug, Clone, Serialize)]
57pub(crate) struct ChunkSourcesRequest {
58    pub convert_options: ConvertDocumentsOptions,
59    pub sources: Vec<HttpSource>,
60    pub include_converted_doc: bool,
61    pub target: TargetKind,
62    pub chunking_options: ChunkingRequestOptions,
63}
64
65#[derive(Debug, Clone, Serialize)]
66#[serde(tag = "chunker")]
67pub(crate) enum ChunkingRequestOptions {
68    #[serde(rename = "hybrid")]
69    Hybrid {
70        use_markdown_tables: bool,
71        use_markdown_images: bool,
72        image_placeholder: String,
73        include_raw_text: bool,
74        max_tokens: Option<u32>,
75        tokenizer: String,
76        merge_peers: bool,
77    },
78    #[serde(rename = "hierarchical")]
79    Hierarchical {
80        use_markdown_tables: bool,
81        use_markdown_images: bool,
82        image_placeholder: String,
83        include_raw_text: bool,
84    },
85}
86
87pub(crate) fn convert_options(
88    input_kind: InputKind,
89    request: &DoclingConvertRequest,
90) -> ConvertDocumentsOptions {
91    ConvertDocumentsOptions {
92        from_formats: vec![input_kind.from_formats_value().to_string()],
93        to_formats: request
94            .output_formats
95            .iter()
96            .map(|format| format.as_api_value().to_string())
97            .collect(),
98        pipeline: request.pipeline.map(|pipeline| pipeline.to_string()),
99        page_range: request
100            .page_range
101            .map(|(start, end)| [start as u64, end as u64]),
102    }
103}
104
105pub(crate) fn target_for(request: &DoclingConvertRequest) -> TargetKind {
106    if request
107        .output_formats
108        .iter()
109        .any(|format| format.is_archive())
110    {
111        TargetKind::Zip {}
112    } else {
113        TargetKind::Inbody {}
114    }
115}
116
117pub(crate) fn chunking_options(
118    chunker: ChunkerKind,
119    options: &ChunkingOptions,
120) -> crate::error::Result<ChunkingRequestOptions> {
121    match chunker {
122        ChunkerKind::Hybrid => {
123            let defaults = ChunkingOptions::hybrid_defaults();
124            Ok(ChunkingRequestOptions::Hybrid {
125                use_markdown_tables: options.use_markdown_tables,
126                use_markdown_images: options.use_markdown_images,
127                image_placeholder: if options.image_placeholder.is_empty() {
128                    defaults.image_placeholder
129                } else {
130                    options.image_placeholder.clone()
131                },
132                include_raw_text: options.include_raw_text,
133                max_tokens: options.max_tokens,
134                tokenizer: options
135                    .tokenizer
136                    .clone()
137                    .unwrap_or_else(|| defaults.tokenizer.unwrap_or_default()),
138                merge_peers: options.merge_peers,
139            })
140        }
141        ChunkerKind::Hierarchical => Ok(ChunkingRequestOptions::Hierarchical {
142            use_markdown_tables: options.use_markdown_tables,
143            use_markdown_images: options.use_markdown_images,
144            image_placeholder: if options.image_placeholder.is_empty() {
145                ChunkingOptions::hierarchical_defaults().image_placeholder
146            } else {
147                options.image_placeholder.clone()
148            },
149            include_raw_text: options.include_raw_text,
150        }),
151        ChunkerKind::None => Err(crate::error::PdfConvertError::validation_error(
152            "chunker",
153            "chunking options require hybrid or hierarchical",
154        )),
155    }
156}
157
158pub(crate) fn source_request(
159    url: impl Into<String>,
160    input_kind: InputKind,
161    request: &DoclingConvertRequest,
162) -> ConvertSourcesRequest {
163    ConvertSourcesRequest {
164        options: convert_options(input_kind, request),
165        sources: vec![HttpSource::new(url)],
166        target: target_for(request),
167    }
168}
169
170pub(crate) fn chunk_source_request(
171    url: impl Into<String>,
172    input_kind: InputKind,
173    request: &DoclingConvertRequest,
174) -> crate::error::Result<ChunkSourcesRequest> {
175    Ok(ChunkSourcesRequest {
176        convert_options: convert_options(input_kind, request),
177        sources: vec![HttpSource::new(url)],
178        include_converted_doc: false,
179        target: target_for(request),
180        chunking_options: chunking_options(request.chunker, &request.chunking)?,
181    })
182}
183
184impl std::fmt::Display for PipelineKind {
185    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
186        let value = match self {
187            PipelineKind::Legacy => "legacy",
188            PipelineKind::Standard => "standard",
189            PipelineKind::Vlm => "vlm",
190            PipelineKind::Asr => "asr",
191        };
192        f.write_str(value)
193    }
194}
cloudiful_docling_convert/api/source.rs

cloudiful_docling_convert/api/
source.rs
