docling 1.0.0

DocumentConverter and format backends for docling.rs (a Rust port of docling).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55

//! Chunk-record JSON export shared by the CLI (`--to chunks`) and the HTTP
//! server (`to=chunks`): the hierarchical chunker's records always, plus the
//! hybrid chunker's when a tokenizer is available — `DOCLING_CHUNK_TOKENIZER`,
//! or `.models/chunk/tokenizer.json` as populated by
//! `scripts/install/download_dependencies.sh` (requires the `chunking` build
//! feature; `DOCLING_CHUNK_MAX_TOKENS` overrides the default budget of 256).

use docling_core::chunker::{contextualize, DocChunk, HierarchicalChunker};
use docling_core::DoclingDocument;

fn records(chunks: &[DocChunk]) -> serde_json::Value {
    serde_json::Value::Array(
        chunks
            .iter()
            .map(|c| {
                serde_json::json!({
                    "text": c.text,
                    "headings": c.headings,
                    "doc_items": c.doc_items.iter().map(|i| i.self_ref.clone()).collect::<Vec<_>>(),
                    "contextualize": contextualize(c),
                })
            })
            .collect(),
    )
}

/// The chunk records for `document` as a JSON object
/// (`{"hierarchical": [...], "hybrid": [...]?}`). Tokenizer problems don't
/// fail the export — the hybrid records are skipped and the problem is
/// reported through `warn`.
pub fn chunk_records(
    document: &DoclingDocument,
    warn: &mut dyn FnMut(String),
) -> serde_json::Value {
    let hierarchical = HierarchicalChunker.chunk(document);
    #[cfg_attr(not(feature = "chunking"), allow(unused_mut, unused_variables))]
    let mut out = serde_json::json!({ "hierarchical": records(&hierarchical) });

    #[cfg(feature = "chunking")]
    if let Some(tok_path) = docling_core::env::nonempty("DOCLING_CHUNK_TOKENIZER")
        .or_else(|| docling_core::chunker::resolve_tokenizer_path(None).ok())
    {
        let max_tokens = docling_core::env::parse("DOCLING_CHUNK_MAX_TOKENS").unwrap_or(256);
        match docling_core::chunker::HuggingFaceTokenizer::from_file(&tok_path, max_tokens) {
            Ok(tok) => {
                let hybrid = docling_core::chunker::HybridChunker::new(tok).chunk(document);
                out["hybrid"] = records(&hybrid);
            }
            Err(e) => warn(e.to_string()),
        }
    }
    #[cfg(not(feature = "chunking"))]
    let _ = warn;
    out
}