Struct Pipeline 

pub struct Pipeline { /* private fields */ }

A reusable PDF pipeline. The primary worker runs its models on every core, so a single-page / small / image / METS input is converted at full intra-op speed with no pool to load. A document with enough pages instead fans out across a pool of narrower workers processed concurrently. Both load lazily and are cached for reuse, so a one-shot conversion only pays for what it uses.

Implementations

impl Pipeline

pub fn new() -> Result<Self, PdfError>

Construct the pipeline. Models load lazily on first use (full-intra primary for serial inputs, the helper pool for multi-page PDFs), so nothing is loaded that a given document doesn’t need.

Examples found in repository

examples/snapshot.rs (line 50)

fn main() {
    let mut args = std::env::args().skip(1);
    let root = PathBuf::from(args.next().expect("usage: snapshot <root> <outdir>"));
    let outdir = PathBuf::from(args.next().expect("usage: snapshot <root> <outdir>"));

    let mut pdfs = Vec::new();
    find_pdfs(&root, &mut pdfs);

    let mut pipeline = Pipeline::new().expect("load pipeline");
    let (mut ok, mut err) = (0u32, 0u32);
    for pdf in &pdfs {
        let rel = pdf.strip_prefix(&root).unwrap_or(pdf);
        let name = pdf.file_name().unwrap().to_string_lossy().to_string();
        let md = match std::fs::read(pdf)
            .map_err(|e| format!("read: {e}"))
            .and_then(|bytes| {
                let ext = pdf.extension().and_then(|e| e.to_str()).unwrap_or("");
                let result = if ext == "gz" {
                    fleischwolf_pdf::convert_mets_gbs(&bytes, &name)
                } else if IMAGE_EXTS.contains(&ext) {
                    pipeline.convert_image(&bytes, &name)
                } else {
                    pipeline.convert(&bytes, None, &name)
                };
                result
                    .map(|d| d.export_to_markdown())
                    .map_err(|e| e.to_string())
            }) {
            Ok(md) => {
                ok += 1;
                md
            }
            Err(e) => {
                err += 1;
                eprintln!("ERR {}: {e}", rel.display());
                format!("ERROR: {e}\n")
            }
        };
        // Groundtruth naming keeps the source extension: `<file>.<ext>.md`.
        let mut dest = outdir.join(rel).into_os_string();
        dest.push(".md");
        let dest = PathBuf::from(dest);
        std::fs::create_dir_all(dest.parent().unwrap()).expect("mkdir");
        std::fs::write(&dest, md).expect("write snapshot");
    }
    eprintln!("snapshots: {} ok, {} error, {} total", ok, err, pdfs.len());
}

pub fn no_table_former(self, disable: bool) -> Self

Skip loading and running the TableFormer table-structure model. Table regions still get emitted, but reconstructed geometrically from cell positions instead of via the ONNX model’s predicted structure — faster (no model load, no per-table inference) at the cost of table fidelity. No effect if a worker is already loaded; set this before the first conversion.

pub fn convert( &mut self, bytes: &[u8], password: Option<&str>, name: &str, ) -> Result<DoclingDocument, PdfError>

Convert a PDF (bytes) to a DoclingDocument. A document with fewer than parallel_min pages (or a pool size of 1) streams through the full-intra primary; a larger one renders on this thread (pdfium is not thread-safe) and fans the pages out across the worker pool, reassembled in page order so the output is byte-identical to the serial path.

Examples found in repository

examples/snapshot.rs (line 64)

fn main() {
    let mut args = std::env::args().skip(1);
    let root = PathBuf::from(args.next().expect("usage: snapshot <root> <outdir>"));
    let outdir = PathBuf::from(args.next().expect("usage: snapshot <root> <outdir>"));

    let mut pdfs = Vec::new();
    find_pdfs(&root, &mut pdfs);

    let mut pipeline = Pipeline::new().expect("load pipeline");
    let (mut ok, mut err) = (0u32, 0u32);
    for pdf in &pdfs {
        let rel = pdf.strip_prefix(&root).unwrap_or(pdf);
        let name = pdf.file_name().unwrap().to_string_lossy().to_string();
        let md = match std::fs::read(pdf)
            .map_err(|e| format!("read: {e}"))
            .and_then(|bytes| {
                let ext = pdf.extension().and_then(|e| e.to_str()).unwrap_or("");
                let result = if ext == "gz" {
                    fleischwolf_pdf::convert_mets_gbs(&bytes, &name)
                } else if IMAGE_EXTS.contains(&ext) {
                    pipeline.convert_image(&bytes, &name)
                } else {
                    pipeline.convert(&bytes, None, &name)
                };
                result
                    .map(|d| d.export_to_markdown())
                    .map_err(|e| e.to_string())
            }) {
            Ok(md) => {
                ok += 1;
                md
            }
            Err(e) => {
                err += 1;
                eprintln!("ERR {}: {e}", rel.display());
                format!("ERROR: {e}\n")
            }
        };
        // Groundtruth naming keeps the source extension: `<file>.<ext>.md`.
        let mut dest = outdir.join(rel).into_os_string();
        dest.push(".md");
        let dest = PathBuf::from(dest);
        std::fs::create_dir_all(dest.parent().unwrap()).expect("mkdir");
        std::fs::write(&dest, md).expect("write snapshot");
    }
    eprintln!("snapshots: {} ok, {} error, {} total", ok, err, pdfs.len());
}

pub fn convert_streaming<F>( &mut self, bytes: &[u8], password: Option<&str>, name: &str, emit: F, ) -> Result<(), PdfError>

where F: FnMut(Vec<Node>, Vec<(String, String)>) -> Result<(), PdfError>,

Convert a PDF in streaming mode: emit is called with each finalized, in-document-order batch of nodes (and that span’s recovered links) as pages complete, so a caller can serialize Markdown page by page instead of waiting for the whole document. The batches are exactly the buffered convert’s nodes, split at safe block boundaries by [assemble::StreamAssembler] — the parallel path reorders pages back into document order before emitting, so the output is identical regardless of worker scheduling.

emit runs on the calling thread (never a worker), so it needn’t be Send and its backpressure throttles the whole pipeline. Returning Err from emit aborts the conversion with that error.

pub fn convert_image( &mut self, bytes: &[u8], name: &str, ) -> Result<DoclingDocument, PdfError>

Convert a standalone image (PNG/JPEG/TIFF/WebP/…) as a single page — docling routes images through the same layout+OCR pipeline as a PDF page.

Examples found in repository

examples/snapshot.rs (line 62)

fn main() {
    let mut args = std::env::args().skip(1);
    let root = PathBuf::from(args.next().expect("usage: snapshot <root> <outdir>"));
    let outdir = PathBuf::from(args.next().expect("usage: snapshot <root> <outdir>"));

    let mut pdfs = Vec::new();
    find_pdfs(&root, &mut pdfs);

    let mut pipeline = Pipeline::new().expect("load pipeline");
    let (mut ok, mut err) = (0u32, 0u32);
    for pdf in &pdfs {
        let rel = pdf.strip_prefix(&root).unwrap_or(pdf);
        let name = pdf.file_name().unwrap().to_string_lossy().to_string();
        let md = match std::fs::read(pdf)
            .map_err(|e| format!("read: {e}"))
            .and_then(|bytes| {
                let ext = pdf.extension().and_then(|e| e.to_str()).unwrap_or("");
                let result = if ext == "gz" {
                    fleischwolf_pdf::convert_mets_gbs(&bytes, &name)
                } else if IMAGE_EXTS.contains(&ext) {
                    pipeline.convert_image(&bytes, &name)
                } else {
                    pipeline.convert(&bytes, None, &name)
                };
                result
                    .map(|d| d.export_to_markdown())
                    .map_err(|e| e.to_string())
            }) {
            Ok(md) => {
                ok += 1;
                md
            }
            Err(e) => {
                err += 1;
                eprintln!("ERR {}: {e}", rel.display());
                format!("ERROR: {e}\n")
            }
        };
        // Groundtruth naming keeps the source extension: `<file>.<ext>.md`.
        let mut dest = outdir.join(rel).into_os_string();
        dest.push(".md");
        let dest = PathBuf::from(dest);
        std::fs::create_dir_all(dest.parent().unwrap()).expect("mkdir");
        std::fs::write(&dest, md).expect("write snapshot");
    }
    eprintln!("snapshots: {} ok, {} error, {} total", ok, err, pdfs.len());
}