Skip to main content

docling/
vlm.rs

1//! VLM pipeline (issue #77) — remote OpenAI-compatible vision endpoint.
2//!
3//! The counterpart of docling's `VlmPipeline` in its remote form
4//! (`ApiVlmOptions`): each PDF page is rendered to an image, sent to an
5//! OpenAI-compatible `chat/completions` endpoint (LM Studio, Ollama, vLLM, or
6//! a hosted service) together with a DocLang-eliciting prompt, and the
7//! returned markup is parsed into a [`DoclingDocument`] — DocTags answers
8//! (granite-docling-class models) via `docling_core::doctags`' tolerant
9//! parser (#152), DocLang XML via the existing reader (`backend::doclang`),
10//! and untagged prose via a line-per-paragraph fallback. Local ONNX
11//! inference of a docling VLM is a later enhancement — this module
12//! deliberately contains no model code, just the request loop.
13//!
14//! HTTP goes over `ureq`, the crate's existing blocking client
15//! (`fetch-images` pulls the same one, keeping a single HTTP stack in the
16//! graph — the converter is synchronous, so an async client would only add a
17//! runtime). Transient failures (transport errors, 408/429, 5xx) retry with
18//! exponential backoff. A page whose request still fails, or whose answer was
19//! cut off (`finish_reason` `length` / `content_filter`), is recorded as a
20//! page error and the result is a `PartialSuccess` — docling's
21//! `VlmPipeline._determine_status` (docling#4561, docling#4778; #762, #763);
22//! only a document where no page produced output fails.
23
24use std::io::Cursor;
25use std::time::Duration;
26
27use docling_core::DoclingDocument;
28
29use crate::backend::doclang::DoclangBackend;
30use crate::backend::DeclarativeBackend;
31use crate::error::ConversionError;
32use crate::format::InputFormat;
33use crate::result::{ConversionResult, ConversionStatus, ErrorItem};
34use crate::source::SourceDocument;
35
36/// Configuration for the remote VLM conversion. Everything has an env-var
37/// fallback so `--pipeline vlm` works without repeating flags:
38/// `DOCLING_RS_VLM_ENDPOINT`, `DOCLING_RS_VLM_MODEL`, `DOCLING_RS_VLM_PROMPT`,
39/// `DOCLING_RS_VLM_API_KEY`.
40#[derive(Debug, Clone)]
41pub struct VlmOptions {
42    /// Base URL of the OpenAI-compatible server (`http://localhost:11434/v1`)
43    /// or the full `…/chat/completions` URL — the suffix is appended when
44    /// missing, so both spellings work.
45    pub endpoint: String,
46    /// Model name as the server knows it (e.g. `granite-docling`).
47    pub model: String,
48    /// The instruction sent with every page image. Defaults to docling's
49    /// DocLang-eliciting prompt ([`DEFAULT_VLM_PROMPT`]).
50    pub prompt: Option<String>,
51    /// Bearer token, if the endpoint wants one. Local servers don't.
52    pub api_key: Option<String>,
53    /// 1-based inclusive page window (`--pages` composes with the VLM
54    /// pipeline exactly like with the ML one).
55    pub page_range: Option<(usize, usize)>,
56    /// `max_tokens` for each completion. A dense page of DocLang easily runs
57    /// long; the default (8192) fits every corpus page with headroom.
58    pub max_tokens: usize,
59}
60
61/// docling's page-conversion instruction for its DocLang-emitting VLMs.
62pub const DEFAULT_VLM_PROMPT: &str = "Convert this page to docling.";
63
64/// Unlimited-OCR's official model-card prompt (docling#4037), kept verbatim:
65/// the model answers other phrasings — generic "convert to markdown" included —
66/// with an empty completion and finish_reason "stop".
67pub const UNLIMITED_OCR_PROMPT: &str = "<image>document parsing.";
68
69/// Chandra's layout prompt (docling's `CHANDRA_OCR_LAYOUT_PROMPT`), verbatim.
70pub const CHANDRA_PROMPT: &str = concat!(
71    "OCR this image to HTML, arranged as layout blocks. Each layout block should be ",
72    "a div with the data-bbox attribute representing the bounding box of the block in ",
73    "x0 y0 x1 y1 format. Bboxes are normalized 0-1000. The data-label attribute is ",
74    "the label for the block.\n\n",
75    "Use the following labels:\n",
76    "- Caption\n- Footnote\n- Equation-Block\n- List-Group\n- Page-Header\n",
77    "- Page-Footer\n- Image\n- Section-Header\n- Table\n- Text\n- Complex-Block\n",
78    "- Code-Block\n- Form\n- Table-Of-Contents\n- Figure\n- Chemical-Block\n",
79    "- Diagram\n- Bibliography\n- Blank-Page\n\n",
80    "Only use these tags ['math', 'br', 'i', 'b', 'u', 'del', 'sup', 'sub', 'table', 'tr', 'td', ",
81    "'p', 'th', 'div', 'pre', 'h1', 'h2', 'h3', 'h4', 'h5', 'ul', 'ol', 'li', ",
82    "'input', 'a', 'span', 'img', 'hr', 'tbody', 'small', 'caption', 'strong', ",
83    "'thead', 'big', 'code', 'chem'], ",
84    "and these attributes ['class', 'colspan', 'rowspan', 'display', 'checked', 'type', 'border', ",
85    "'value', 'style', 'href', 'alt', 'align', 'data-bbox', 'data-label'].\n\n",
86    "Guidelines:\n",
87    "* Inline math: Surround math with <math>...</math> tags. Math expressions ",
88    "should be rendered in KaTeX-compatible LaTeX. Use display for block math.\n",
89    "* Tables: Use colspan and rowspan attributes to match table structure.\n",
90    "* Formatting: Maintain consistent formatting with the image, including spacing, ",
91    "indentation, subscripts/superscripts, and special characters.\n",
92    "* Images: Include a description of any images in the alt attribute of an <img> tag. ",
93    "Do not fill out the src property. Describe in detail inside the div tag. ",
94    "Also convert charts to high fidelity data, and convert diagrams to mermaid.\n",
95    "* Forms: Mark checkboxes and radio buttons properly.\n",
96    "* Text: join lines together properly into paragraphs using <p>...</p> tags. ",
97    "Use <br> tags for line breaks within paragraphs, but only when absolutely ",
98    "necessary to maintain meaning.\n",
99    "* Chemistry: Use <chem>...</chem> tags for chemical formulas with reactive SMILES.\n",
100    "* Lists: Preserve indents and proper list markers.\n",
101    "* Use the simplest possible HTML structure that accurately represents the content ",
102    "of the block.\n",
103    "* Make sure the text is accurate and easy for a human to read and interpret. ",
104    "Reading order should be correct and natural."
105);
106
107/// The default prompt for a model, by name (#322): the known grammars ship
108/// their official prompts, everything else keeps the DocLang-eliciting
109/// default. `DOCLING_RS_VLM_PROMPT` / the `prompt` option override all of it.
110fn default_prompt_for(model: &str) -> &'static str {
111    let lower = model.to_ascii_lowercase();
112    if lower.contains("unlimited") {
113        UNLIMITED_OCR_PROMPT
114    } else if lower.contains("chandra") {
115        CHANDRA_PROMPT
116    } else {
117        DEFAULT_VLM_PROMPT
118    }
119}
120
121impl VlmOptions {
122    /// Build options from explicit values, falling back to the
123    /// `DOCLING_RS_VLM_*` environment. Endpoint and model are required —
124    /// there is no sensible default server to talk to.
125    pub fn resolve(
126        endpoint: Option<String>,
127        model: Option<String>,
128    ) -> Result<Self, ConversionError> {
129        let env = docling_core::env::nonempty;
130        let endpoint = endpoint
131            .or_else(|| env("DOCLING_RS_VLM_ENDPOINT"))
132            .ok_or_else(|| {
133                ConversionError::Parse(
134                    "vlm: no endpoint (pass --vlm-endpoint or set DOCLING_RS_VLM_ENDPOINT)".into(),
135                )
136            })?;
137        let model = model
138            .or_else(|| env("DOCLING_RS_VLM_MODEL"))
139            .ok_or_else(|| {
140                ConversionError::Parse(
141                    "vlm: no model (pass --vlm-model or set DOCLING_RS_VLM_MODEL)".into(),
142                )
143            })?;
144        Ok(Self {
145            endpoint,
146            model,
147            prompt: env("DOCLING_RS_VLM_PROMPT"),
148            api_key: env("DOCLING_RS_VLM_API_KEY"),
149            page_range: None,
150            max_tokens: 8192,
151        })
152    }
153
154    fn url(&self) -> String {
155        chat_url(&self.endpoint)
156    }
157}
158
159pub use crate::options::{PictureDescriptionMode, CAPTION_PROMPT, OCR_PROMPT};
160
161/// The picture-description pass (#670): every picture of a document — a
162/// DOCX/PPTX/XLSX/HTML embedded image, a PDF figure crop, a sampled video
163/// frame — goes to an OpenAI-compatible vision endpoint and the answer
164/// becomes its description annotation (`meta.description` in docling JSON,
165/// printed after the caption in Markdown). MarkItDown's `llm_client` /
166/// `llm_model` / `llm_prompt` and its markitdown-ocr plugin, on every format
167/// and surface. Endpoint, model and key fall back to the `DOCLING_RS_VLM_*`
168/// environment like the page pipeline's; the prompt does not
169/// (`DOCLING_RS_VLM_PROMPT` asks for a page's markup, not a caption).
170#[derive(Debug, Clone)]
171pub struct PictureDescriptionOptions {
172    /// The server's `/v1` base or the full `…/chat/completions` URL.
173    pub endpoint: String,
174    /// Model name as the server knows it.
175    pub model: String,
176    /// Bearer token, if the endpoint wants one.
177    pub api_key: Option<String>,
178    /// The instruction sent with every picture; `None` = the mode's prompt.
179    pub prompt: Option<String>,
180    /// Caption or OCR.
181    pub mode: PictureDescriptionMode,
182    /// Pictures in flight at once (default 1; the endpoint's rate limit sets
183    /// the ceiling — 429s retry with backoff).
184    pub concurrency: usize,
185}
186
187impl PictureDescriptionOptions {
188    /// Build options from explicit values, the endpoint, model and key
189    /// falling back to the `DOCLING_RS_VLM_*` environment. Endpoint and
190    /// model are required.
191    pub fn resolve(
192        endpoint: Option<String>,
193        model: Option<String>,
194    ) -> Result<Self, ConversionError> {
195        let env = docling_core::env::nonempty;
196        let endpoint = endpoint
197            .or_else(|| env("DOCLING_RS_VLM_ENDPOINT"))
198            .ok_or_else(|| {
199                ConversionError::Parse(
200                    "picture description: no endpoint (pass --picture-description-endpoint or set \
201                     DOCLING_RS_VLM_ENDPOINT)"
202                        .into(),
203                )
204            })?;
205        let model = model
206            .or_else(|| env("DOCLING_RS_VLM_MODEL"))
207            .ok_or_else(|| {
208                ConversionError::Parse(
209                    "picture description: no model (pass --picture-description-model or set \
210                     DOCLING_RS_VLM_MODEL)"
211                        .into(),
212                )
213            })?;
214        Ok(Self {
215            endpoint,
216            model,
217            api_key: env("DOCLING_RS_VLM_API_KEY"),
218            prompt: None,
219            mode: PictureDescriptionMode::default(),
220            concurrency: 1,
221        })
222    }
223
224    /// The prompt in force: the explicit one, else the mode's.
225    pub fn prompt(&self) -> &str {
226        self.prompt.as_deref().unwrap_or(self.mode.default_prompt())
227    }
228
229    /// `created_by` of the descriptions this produces: the model name.
230    pub fn provenance(&self) -> String {
231        self.model.clone()
232    }
233
234    /// Ask the endpoint about one image (`mimetype` as the data URI says
235    /// it). The retry policy is the page loop's; an empty answer is `None`,
236    /// the way MarkItDown reads a blank completion as "no text".
237    pub fn describe(
238        &self,
239        agent: &ureq::Agent,
240        mimetype: &str,
241        image: &[u8],
242    ) -> Result<Option<String>, RequestError> {
243        let req = ImageRequest {
244            url: chat_url(&self.endpoint),
245            model: &self.model,
246            api_key: self.api_key.as_deref(),
247            prompt: self.prompt(),
248            max_tokens: None,
249            keep_special: false,
250        };
251        // A cut-off description is still the model's best answer about the
252        // picture; only the page pipeline reports `finish_reason`.
253        let text = request_image(agent, &req, mimetype, image)?.text;
254        let text = text.trim();
255        Ok((!text.is_empty()).then(|| text.to_string()))
256    }
257}
258
259/// Why a request produced no answer: the endpoint rejected this image (a
260/// 4xx, a malformed body — the next picture may fare better), or it could
261/// not be reached at all (transport errors, 5xx and 408/429 exhausted their
262/// retries, our own timeout — the next picture will not fare better).
263#[derive(Debug, Clone, PartialEq, Eq)]
264pub enum RequestError {
265    Rejected(String),
266    Unavailable(String),
267}
268
269impl std::fmt::Display for RequestError {
270    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
271        match self {
272            Self::Rejected(m) | Self::Unavailable(m) => f.write_str(m),
273        }
274    }
275}
276
277impl From<RequestError> for String {
278    fn from(e: RequestError) -> String {
279        e.to_string()
280    }
281}
282
283/// The `chat/completions` URL for an endpoint given as the `/v1` base or in
284/// full.
285fn chat_url(endpoint: &str) -> String {
286    let base = endpoint.trim_end_matches('/');
287    if base.ends_with("/chat/completions") {
288        base.to_string()
289    } else {
290        format!("{base}/chat/completions")
291    }
292}
293
294/// Convert a PDF or image through the remote VLM: the document alone, see
295/// [`convert_vlm_result`] for the pages that failed or came back cut off.
296pub fn convert_vlm(
297    source: &SourceDocument,
298    opts: &VlmOptions,
299) -> Result<DoclingDocument, ConversionError> {
300    convert_vlm_result(source, opts).map(|r| r.document)
301}
302
303/// Convert a PDF or image through the remote VLM. PDF pages render via
304/// pdfium at the ML pipeline's scale; a standalone image is sent as-is (it is
305/// its own page).
306///
307/// A page whose request fails (a 4xx, retries exhausted, our timeout) is left
308/// out and recorded as `VLM inference failed: …`; a page the server cut off
309/// (`finish_reason` `length` — `max_tokens` — or `content_filter`) keeps its
310/// partial text and is recorded as `VLM output incomplete (stop_reason=…)`.
311/// Either makes the result a [`ConversionStatus::PartialSuccess`] whose
312/// errors carry the page numbers — docling's `VlmPipeline._determine_status`
313/// (docling#4561, docling#4778). Only when every page failed is it an error.
314pub fn convert_vlm_result(
315    source: &SourceDocument,
316    opts: &VlmOptions,
317) -> Result<ConversionResult, ConversionError> {
318    let agent = agent();
319    let mut fragments: Vec<String> = Vec::new();
320    let mut errors: Vec<ErrorItem> = Vec::new();
321    // (page, reason) of the pages that produced no output.
322    let mut failed: Vec<(usize, String)> = Vec::new();
323    let mut pages = 0usize;
324    match source.format {
325        InputFormat::Pdf => {
326            // 1-based window → 0-based inclusive, validated like Pipeline::pages.
327            let total = docling_pdf::pdfium_backend::page_count(&source.bytes, None)
328                .map_err(|e| ConversionError::Parse(format!("vlm: open pdf: {e}")))?;
329            let range = match opts.page_range {
330                Some((first, last)) => {
331                    if first == 0 || last < first {
332                        return Err(ConversionError::Parse(format!(
333                            "invalid page range {first}-{last} (pages are 1-based, first <= last)"
334                        )));
335                    }
336                    if first > total {
337                        return Err(ConversionError::Parse(format!(
338                            "page range {first}-{last} is outside the document ({total} page(s))"
339                        )));
340                    }
341                    Some((first - 1, last.min(total) - 1))
342                }
343                None => None,
344            };
345            // A page that fails is recorded and the walk goes on: the pages
346            // around it are as good as they were (docling#4561).
347            docling_pdf::pdfium_backend::for_each_page::<docling_pdf::PdfError, _>(
348                &source.bytes,
349                None,
350                true,  // render page images — they are the whole input here
351                false, // the text layer never reaches the model: skip its decode
352                range,
353                |i, _total, page| {
354                    pages += 1;
355                    let step =
356                        encode_png(&page.image).and_then(|png| request_page(&agent, opts, &png));
357                    take_page(i + 1, step, &mut fragments, &mut errors, &mut failed);
358                    Ok(())
359                },
360            )
361            .map_err(pdf_err)?;
362        }
363        InputFormat::Image => {
364            // The image file is already the page; no re-encode, no pdfium.
365            pages = 1;
366            let step = request_page(&agent, opts, &source.bytes);
367            take_page(1, step, &mut fragments, &mut errors, &mut failed);
368        }
369        other => {
370            return Err(ConversionError::Parse(format!(
371                "vlm pipeline converts PDF and image inputs (got {other:?})"
372            )));
373        }
374    }
375    // No page produced output: a failed conversion, not a partial result
376    // (docling's `ConversionStatus.FAILURE`).
377    if pages > 0 && failed.len() == pages {
378        let (page, reason) = &failed[0];
379        let more = match pages {
380            1 => String::new(),
381            n => format!(" (all {n} pages failed)"),
382        };
383        return Err(ConversionError::Parse(format!(
384            "vlm: page {page}: {reason}{more}"
385        )));
386    }
387    // Two grammars come back from the wire. DocTags (granite-docling-class
388    // models: loc tokens, unclosed OTSL markers) goes to docling-core's
389    // dedicated tolerant parser, which keeps geometry and span structure
390    // (#152). Proper DocLang XML — or plain prose — takes the DocLang
391    // reader path a `.dclg` file would. One DocTags-shaped page routes the
392    // whole document: mixing grammars page-to-page is model misbehavior,
393    // and the DocTags parser degrades to paragraphs on non-DocTags input.
394    // The model-specific grammars go first (#322): their token shapes are
395    // unambiguous, and each parser shares the tolerance contract — hostile
396    // output degrades to text, never to an error.
397    let doc = if fragments
398        .iter()
399        .any(|f| crate::backend::looks_like_chandra(f))
400    {
401        // Chandra layout HTML (`<div data-bbox=… data-label=…>`).
402        crate::backend::chandra_pages(&source.name, &fragments)
403    } else if fragments.iter().any(|f| {
404        crate::backend::is_unlimited_ocr_markdown(f) || crate::backend::is_deepseek_markdown(f)
405    }) {
406        // Unlimited-OCR grounding output labels blocks the way DeepSeek-OCR
407        // does (docling#3944): normalize its `<|det|>label [x,y,x,y]<|/det|>`
408        // shape into the DeepSeek one and share that parser. Native
409        // DeepSeek-shaped responses take the same path directly.
410        let joined = fragments
411            .iter()
412            .map(|f| crate::backend::normalize_unlimited(f))
413            .collect::<Vec<_>>()
414            .join("\n");
415        crate::backend::deepseek_text(&source.name, &joined)
416    } else if fragments.iter().any(|f| looks_like_doctags(f)) {
417        let mut doc = docling_core::doctags::parse_pages(fragments.iter().map(String::as_str));
418        doc.name = source.name.clone();
419        doc
420    } else {
421        parse_doclang_fragments(&source.name, &fragments)
422    };
423    if doc.nodes.is_empty() {
424        // Some request succeeded, so this is a content problem, not a
425        // transport one: the model answered with nothing our reader keeps.
426        // An empty stdout with exit 0 buried that; say it loudly instead.
427        return Err(ConversionError::Parse(
428            "vlm: the model's responses contained no parseable DocLang/DocTags/Chandra/\
429             grounding blocks \
430             (set DOCLING_RS_VLM_DEBUG=1 to print raw responses; a generic VLM may need \
431             a DOCLING_RS_VLM_PROMPT that describes the expected markup)"
432                .into(),
433        ));
434    }
435    let status = if errors.is_empty() {
436        ConversionStatus::Success
437    } else {
438        ConversionStatus::PartialSuccess
439    };
440    Ok(ConversionResult {
441        document: doc,
442        status,
443        input_name: source.name.clone(),
444        format: source.format,
445        errors,
446        redaction: None,
447    })
448}
449
450/// File one page's answer: its markup into `fragments`, a cut-off answer
451/// also as a page error, a failed request as a page error and in `failed`.
452fn take_page(
453    page_no: usize,
454    step: Result<Completion, String>,
455    fragments: &mut Vec<String>,
456    errors: &mut Vec<ErrorItem>,
457    failed: &mut Vec<(usize, String)>,
458) {
459    match step {
460        Ok(completion) => {
461            if let Some(reason) = completion.incomplete() {
462                errors.push(ErrorItem::vlm_page(
463                    page_no,
464                    format!("VLM output incomplete (stop_reason={reason})."),
465                ));
466            }
467            fragments.push(strip_wrappers(&completion.text));
468        }
469        Err(e) => {
470            errors.push(ErrorItem::vlm_page(
471                page_no,
472                format!("VLM inference failed: {e}."),
473            ));
474            failed.push((page_no, e));
475        }
476    }
477}
478
479fn pdf_err(e: docling_pdf::PdfError) -> ConversionError {
480    ConversionError::with_source("pdf", e)
481}
482
483pub(crate) fn agent() -> ureq::Agent {
484    // A VLM can chew on a dense page for minutes — and a CPU-served endpoint
485    // for many more: #311's measurement clocked a routine page at 587 s on a
486    // desktop CPU, a hair under this default. `DOCLING_RS_VLM_TIMEOUT`
487    // (seconds) raises the per-request cap for such endpoints; retries make a
488    // too-small value expensive (4 attempts each grind to the cap while the
489    // server keeps generating for a client that already hung up).
490    let timeout = docling_core::env::parse("DOCLING_RS_VLM_TIMEOUT").unwrap_or(600);
491    ureq::Agent::config_builder()
492        .timeout_connect(Some(Duration::from_secs(10)))
493        .timeout_global(Some(Duration::from_secs(timeout)))
494        // Keep non-2xx as inspectable responses for the retry decision.
495        .http_status_as_error(false)
496        .build()
497        .into()
498}
499
500/// POST one page image, return the model's answer. Retries transport errors,
501/// 408/429 and 5xx with exponential backoff (2s/4s/8s); other statuses and a
502/// malformed body fail immediately.
503fn request_page(
504    agent: &ureq::Agent,
505    opts: &VlmOptions,
506    image: &[u8],
507) -> Result<Completion, String> {
508    // Unlimited-OCR's grounding markers are special tokens: without this vLLM
509    // extension flag the server strips them from the completion and the page
510    // comes back as flat text (docling#3944's api_overrides).
511    let req = ImageRequest {
512        url: opts.url(),
513        model: &opts.model,
514        api_key: opts.api_key.as_deref(),
515        prompt: opts
516            .prompt
517            .as_deref()
518            .unwrap_or(default_prompt_for(&opts.model)),
519        max_tokens: Some(opts.max_tokens),
520        keep_special: opts.model.to_ascii_lowercase().contains("unlimited"),
521    };
522    request_image(agent, &req, "image/png", image).map_err(String::from)
523}
524
525/// A model's answer: `choices[0].message.content` and the choice's
526/// `finish_reason`.
527#[derive(Debug)]
528struct Completion {
529    text: String,
530    finish_reason: Option<String>,
531}
532
533impl Completion {
534    /// The stop reason of an answer the server cut off — docling's
535    /// `_map_stop_reason`: `length` (`max_tokens` reached) and
536    /// `content_filter`; every other reason is a complete answer.
537    fn incomplete(&self) -> Option<&str> {
538        self.finish_reason
539            .as_deref()
540            .filter(|r| matches!(*r, "length" | "content_filter"))
541    }
542}
543
544/// One image in, one completion out — what the page loop and the picture
545/// description pass both send.
546struct ImageRequest<'a> {
547    url: String,
548    model: &'a str,
549    api_key: Option<&'a str>,
550    prompt: &'a str,
551    /// `max_tokens` when set; omitted from the body otherwise (the server's
552    /// default, as MarkItDown sends it).
553    max_tokens: Option<usize>,
554    /// vLLM's `skip_special_tokens: false`.
555    keep_special: bool,
556}
557
558/// POST `image` (a data URI of `mimetype`) with the request's prompt and
559/// return `choices[0]`'s content and finish reason. Transport errors, 408/429 and 5xx
560/// retry with exponential backoff (2s/4s/8s) and come back as
561/// [`RequestError::Unavailable`] once exhausted; other statuses and a
562/// malformed body are [`RequestError::Rejected`] at once.
563fn request_image(
564    agent: &ureq::Agent,
565    req: &ImageRequest<'_>,
566    mimetype: &str,
567    image: &[u8],
568) -> Result<Completion, RequestError> {
569    let data_uri = format!(
570        "data:{mimetype};base64,{}",
571        docling_core::base64::encode(image)
572    );
573    let mut body = serde_json::json!({
574        "model": req.model,
575        // Deterministic-ish output: sampling noise only hurts a structured
576        // markup task (docling's ApiVlmOptions ships temperature 0 too).
577        "temperature": 0,
578        "messages": [{
579            "role": "user",
580            "content": [
581                { "type": "text", "text": req.prompt },
582                { "type": "image_url", "image_url": { "url": data_uri } },
583            ],
584        }],
585    });
586    if let Some(n) = req.max_tokens {
587        body["max_tokens"] = serde_json::Value::from(n);
588    }
589    if req.keep_special {
590        body["skip_special_tokens"] = serde_json::Value::Bool(false);
591    }
592    // DOCLING_RS_VLM_EXTRA_BODY: a JSON object merged into the request at the
593    // top level — the escape hatch for server-specific knobs the OpenAI shape
594    // doesn't cover. The motivating case: vLLM's "skip_special_tokens": false,
595    // without which servers detokenize away granite-docling's DocTags
596    // structure tokens and only loc tokens + bare text survive.
597    if let Some(extra) = docling_core::env::nonempty("DOCLING_RS_VLM_EXTRA_BODY") {
598        match serde_json::from_str::<serde_json::Value>(&extra) {
599            Ok(serde_json::Value::Object(map)) => {
600                for (k, v) in map {
601                    body[k] = v;
602                }
603            }
604            _ => {
605                return Err(RequestError::Rejected(
606                    "DOCLING_RS_VLM_EXTRA_BODY is not a JSON object; fix or unset it".into(),
607                ))
608            }
609        }
610    }
611    let url = &req.url;
612    let mut delay = Duration::from_secs(2);
613    let mut last_err = String::new();
614    for attempt in 0..4 {
615        if attempt > 0 {
616            std::thread::sleep(delay);
617            delay *= 2;
618        }
619        let mut post = agent.post(url).header("content-type", "application/json");
620        if let Some(key) = req.api_key {
621            post = post.header("authorization", &format!("Bearer {key}"));
622        }
623        // Hand-serialized body: the crate pulls ureq without its `json`
624        // feature (fetch-images doesn't need it), and one to_string keeps it
625        // that way.
626        let payload = serde_json::to_string(&body).expect("static json shape");
627        match post.send(payload.as_bytes()) {
628            Ok(mut resp) => {
629                let status = resp.status().as_u16();
630                let text = resp
631                    .body_mut()
632                    .read_to_string()
633                    .map_err(|e| RequestError::Unavailable(format!("{url}: read response: {e}")))?;
634                if status == 408 || status == 429 || status >= 500 {
635                    // Keep a body snippet: for OpenAI-style servers the real
636                    // reason (insufficient_quota vs. a plain rate limit)
637                    // lives there, and hiding it made give-ups undiagnosable.
638                    last_err = format!(
639                        "{url}: HTTP {status} (attempt {}): {}",
640                        attempt + 1,
641                        text.replace(['\n', '\r'], " ")
642                            .chars()
643                            .take(300)
644                            .collect::<String>()
645                    );
646                    continue;
647                }
648                if status != 200 {
649                    return Err(RequestError::Rejected(format!(
650                        "{url}: HTTP {status}: {}",
651                        text.chars().take(300).collect::<String>()
652                    )));
653                }
654                let parsed: serde_json::Value = serde_json::from_str(&text).map_err(|e| {
655                    RequestError::Rejected(format!("{url}: malformed JSON response: {e}"))
656                })?;
657                let content = parsed["choices"][0]["message"]["content"]
658                    .as_str()
659                    .map(str::to_string)
660                    .ok_or_else(|| {
661                        RequestError::Rejected(format!(
662                            "{url}: no choices[0].message.content in response"
663                        ))
664                    });
665                let finish_reason = parsed["choices"][0]["finish_reason"]
666                    .as_str()
667                    .map(str::to_string);
668                if docling_core::env::flag("DOCLING_RS_VLM_DEBUG") {
669                    match &content {
670                        Ok(c) => eprintln!(
671                            "vlm: raw model response ({} chars, finish_reason {}):\n{c}\n---",
672                            c.len(),
673                            finish_reason.as_deref().unwrap_or("unset")
674                        ),
675                        Err(_) => eprintln!("vlm: raw endpoint body:\n{text}\n---"),
676                    }
677                }
678                return content.map(|text| Completion {
679                    text,
680                    finish_reason,
681                });
682            }
683            // Our own timeout is not retryable: the model is deterministic,
684            // so the same page grinds to the same timeout again — and against
685            // a single-threaded server the retry queues *behind* the orphaned
686            // generation it abandoned, multiplying the wasted wall-clock ×4
687            // (#311's GPU run: a 3300 s page vs. a 1800 s cap). Raise
688            // DOCLING_RS_VLM_TIMEOUT instead; the error says so.
689            Err(ureq::Error::Timeout(_)) => {
690                return Err(RequestError::Unavailable(format!(
691                    "{url}: request exceeded the {}s cap (DOCLING_RS_VLM_TIMEOUT raises \
692                     it; the server may still be finishing the abandoned generation)",
693                    docling_core::env::parse("DOCLING_RS_VLM_TIMEOUT").unwrap_or(600_u64)
694                )));
695            }
696            Err(e) => {
697                last_err = format!("{url}: {e} (attempt {})", attempt + 1);
698            }
699        }
700    }
701    Err(RequestError::Unavailable(format!(
702        "giving up after 4 attempts: {last_err}"
703    )))
704}
705
706/// Is this fragment DocTags rather than DocLang? Granite-class models always
707/// carry `loc_` tokens; the OTSL/section-header vocabularies clinch it when a
708/// (hypothetical) model omits locations.
709fn looks_like_doctags(fragment: &str) -> bool {
710    fragment.contains("<loc_")
711        || fragment.contains("<otsl>")
712        || fragment.contains("<fcel>")
713        || fragment.contains("<section_header_level_")
714}
715
716/// Assemble the non-DocTags fragments (proper DocLang XML, or the prose a model
717/// emits when it ignores the markup instruction) into a document.
718///
719/// The DocLang reader is a *strict* XML parser: a misbehaving model that leaves
720/// a `<heading>`/`<text>` unclosed or nests a stray root produces markup it
721/// rejects outright (`expected 'heading' tag, not 'doclang' …`), which — when
722/// propagated — fails the entire conversion on one bad page. The DocTags path
723/// never does that (#152: hostile model output, best-effort document out), so
724/// neither should this one: on a parse error, salvage the fragments through the
725/// tolerant DocTags parser, which degrades broken/unknown tags to text and
726/// never errors. Well-formed DocLang still takes the strict reader, keeping its
727/// heading levels and table structure.
728fn parse_doclang_fragments(name: &str, fragments: &[String]) -> DoclingDocument {
729    let body: Vec<String> = fragments.iter().map(|f| prose_fallback(f)).collect();
730    let xml = format!("<doclang version=\"0.7\">\n{}\n</doclang>", body.join("\n"));
731    let synthetic = SourceDocument::from_bytes(name, InputFormat::XmlDoclang, xml.into_bytes());
732    let mut doc = match DoclangBackend.convert(&synthetic) {
733        Ok(doc) => doc,
734        Err(_) => docling_core::doctags::parse_pages(fragments.iter().map(String::as_str)),
735    };
736    doc.name = name.to_string();
737    doc
738}
739
740/// DocLang-path fallback for a model that ignored the markup instruction
741/// entirely (plain prose / Markdown, no tags): one `<text>` per non-empty
742/// line, instead of silently dropping the page's content.
743fn prose_fallback(fragment: &str) -> String {
744    if !fragment.contains('<') && !fragment.trim().is_empty() {
745        return fragment
746            .lines()
747            .filter(|l| !l.trim().is_empty())
748            .map(|l| format!("<text>{}</text>", l.trim()))
749            .collect::<Vec<_>>()
750            .join("\n");
751    }
752    fragment.to_string()
753}
754
755/// Strip the wrappers models put around their answer: Markdown code fences
756/// and a `<doclang …>`/`<doctag>` document root (the body keeps its grammar —
757/// DocTags routing happens later on the stripped fragment).
758fn strip_wrappers(response: &str) -> String {
759    let mut text = response.trim();
760    // ```xml … ``` / ``` … ``` fences.
761    if let Some(rest) = text.strip_prefix("```") {
762        let rest = rest.split_once('\n').map(|(_, r)| r).unwrap_or(rest);
763        text = rest
764            .rsplit_once("```")
765            .map(|(r, _)| r)
766            .unwrap_or(rest)
767            .trim();
768    }
769    // Unwrap a <doclang>/<doctag> root down to its children.
770    for root in ["doclang", "doctag", "doctags"] {
771        let open = format!("<{root}");
772        if let Some(start) = text.find(&open) {
773            if let Some(gt) = text[start..].find('>') {
774                let inner_start = start + gt + 1;
775                let close = format!("</{root}>");
776                let inner_end = text.rfind(&close).unwrap_or(text.len());
777                if inner_start <= inner_end {
778                    return text[inner_start..inner_end].trim().to_string();
779                }
780            }
781        }
782    }
783    text.to_string()
784}
785
786/// PNG-encode a rendered page (the wire format every OpenAI-compatible
787/// server accepts as a data URI).
788fn encode_png(image: &image::RgbImage) -> Result<Vec<u8>, String> {
789    let mut out = Vec::new();
790    image
791        .write_to(&mut Cursor::new(&mut out), image::ImageFormat::Png)
792        .map_err(|e| format!("encode page image: {e}"))?;
793    Ok(out)
794}
795
796#[cfg(test)]
797mod tests {
798    use super::{default_prompt_for, CHANDRA_PROMPT, DEFAULT_VLM_PROMPT, UNLIMITED_OCR_PROMPT};
799    use super::{looks_like_doctags, parse_doclang_fragments, prose_fallback, strip_wrappers};
800
801    /// A misbehaving model can emit malformed DocLang — here an unclosed
802    /// `<heading>` — that the strict XML reader rejects with
803    /// `expected 'heading' tag, not 'doclang'`. That hard error dropped the
804    /// whole `normal_4pages` fixture from the #153 corpus run; the fallback
805    /// must salvage the page's text instead of failing the conversion.
806    #[test]
807    fn malformed_doclang_degrades_instead_of_erroring() {
808        let frag = "<heading level=\"1\">Unclosed title\n<text>Body line.</text>".to_string();
809        let md = parse_doclang_fragments("normal_4pages.pdf", &[frag]).export_to_markdown();
810        // Best-effort salvage via the tolerant DocTags parser: the text that
811        // the strict reader would have thrown away with the whole page survives.
812        assert!(md.contains("Body line."), "md: {md:?}");
813        assert!(md.contains("Unclosed title"), "md: {md:?}");
814    }
815
816    /// Well-formed DocLang still takes the strict reader, which keeps the
817    /// heading *level* (the tolerant fallback would flatten it to text).
818    #[test]
819    fn wellformed_doclang_keeps_structure() {
820        let frag = "<heading level=\"2\">Sec</heading>\n<text>Para.</text>".to_string();
821        let md = parse_doclang_fragments("d.pdf", &[frag]).export_to_markdown();
822        assert!(md.contains("## Sec"), "md: {md:?}");
823        assert!(md.contains("Para."), "md: {md:?}");
824    }
825
826    #[test]
827    fn wrapper_stripping() {
828        // Bare fragment passes through.
829        assert_eq!(strip_wrappers("<text>hi</text>"), "<text>hi</text>");
830        // Fenced answer is unwrapped.
831        assert_eq!(
832            strip_wrappers("```xml\n<text>hi</text>\n```"),
833            "<text>hi</text>"
834        );
835        // A full document root is stripped down to its body.
836        assert_eq!(
837            strip_wrappers("<doclang version=\"0.7\"><text>hi</text></doclang>"),
838            "<text>hi</text>"
839        );
840        assert_eq!(
841            strip_wrappers("<doctag><text>hi</text></doctag>"),
842            "<text>hi</text>"
843        );
844        // Prose around a root element: the root wins anywhere.
845        assert_eq!(
846            strip_wrappers("Here you go:\n<doclang><heading level=\"1\">T</heading></doclang>"),
847            "<heading level=\"1\">T</heading>"
848        );
849    }
850
851    /// Routing: granite-style DocTags goes to the docling-core parser (whose
852    /// own tests pin the full markup semantics); DocLang and prose do not.
853    #[test]
854    fn doctags_routing() {
855        assert!(looks_like_doctags(
856            "<text><loc_1><loc_2><loc_3><loc_4>Body</text>"
857        ));
858        assert!(looks_like_doctags("<otsl><ched>A<nl></otsl>"));
859        assert!(looks_like_doctags(
860            "<section_header_level_1>T</section_header_level_1>"
861        ));
862        assert!(!looks_like_doctags("<heading level=\"2\">T</heading>"));
863        assert!(!looks_like_doctags("Just prose, no markup."));
864    }
865
866    /// End-to-end through the same assembly the pipeline uses: the exact
867    /// shape live granite-docling emits (from the #77 bring-up) renders with
868    /// docling-parity structure.
869    #[test]
870    fn doctags_end_to_end_markdown() {
871        let page = "<doctag><picture><loc_15><loc_10><loc_240><loc_60><other></picture>\
872<section_header_level_1><loc_57><loc_70><loc_420><loc_78>Optimized Table Tokenization</section_header_level_1>\
873<text><loc_57><loc_84><loc_420><loc_98>Body with A &amp; B & C.</text>\
874<unordered_list><list_item><loc_60><loc_100><loc_420><loc_108>First item</list_item></unordered_list>\
875<otsl><loc_57><loc_120><loc_420><loc_160><caption><loc_57><loc_115><loc_420><loc_119>Table 1. HPO results.</caption><ched>Col A<ched>Col B<nl><fcel>1<fcel>2<nl></otsl>\
876<page_footer><loc_57><loc_280><loc_420><loc_288>7</page_footer></doctag>";
877        let fragment = strip_wrappers(page);
878        assert!(looks_like_doctags(&fragment));
879        let md = docling_core::doctags::parse(&fragment).export_to_markdown();
880        // Section level 1 → "##" (docling parity; bare "#" is the title).
881        assert!(md.contains("## Optimized Table Tokenization"), "md: {md:?}");
882        assert!(!md.contains("### Optimized"), "md: {md:?}");
883        // The in-otsl caption becomes a paragraph before the table.
884        assert!(
885            md.find("Table 1. HPO results.").unwrap() < md.find("Col A").unwrap(),
886            "caption must precede the table: {md:?}"
887        );
888        assert!(md.contains("Body with A & B & C."), "md: {md:?}");
889        assert!(md.contains("- First item"), "md: {md:?}");
890        assert!(md.contains("Col A |"), "md: {md:?}");
891        // Furniture (page footer) stays out of the Markdown body.
892        assert!(!md.contains("\n7\n"), "md: {md:?}");
893    }
894
895    #[test]
896    fn prose_fallback_wraps_untagged_lines() {
897        assert_eq!(
898            prose_fallback("First line.\n\nSecond line."),
899            "<text>First line.</text>\n<text>Second line.</text>"
900        );
901        // Tagged content is left for the DocLang reader untouched.
902        assert_eq!(prose_fallback("<text>hi</text>"), "<text>hi</text>");
903    }
904    /// #322: known model names get their official prompts; everything else
905    /// keeps the DocLang default.
906    #[test]
907    fn model_names_pick_their_official_prompts() {
908        assert_eq!(default_prompt_for("unlimited-ocr"), UNLIMITED_OCR_PROMPT);
909        assert_eq!(
910            default_prompt_for("baidu/Unlimited-OCR"),
911            UNLIMITED_OCR_PROMPT
912        );
913        assert_eq!(default_prompt_for("chandra-ocr-2"), CHANDRA_PROMPT);
914        assert_eq!(default_prompt_for("granite-docling"), DEFAULT_VLM_PROMPT);
915        assert!(CHANDRA_PROMPT.starts_with("OCR this image to HTML"));
916        assert_eq!(UNLIMITED_OCR_PROMPT, "<image>document parsing.");
917    }
918}