xberg 1.0.2

High-performance document intelligence library for Rust. Extract text, metadata, and structured data from PDFs, Office documents, images, and 98 formats and 306 programming languages via tree-sitter code intelligence with async/sync APIs.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

//! Vendored from <https://github.com/benbrandt/text-splitter> @ v0.30.1
//! MIT, © 2023 Benjamin Brandt. See `ATTRIBUTIONS.md` for full provenance.
//!
//! Trimmed: dropped `code` (tree-sitter) and `tiktoken-rs` features and their
//! files; rebuilt against `tokenizers 0.23`. The `markdown` splitter is always
//! compiled because `pulldown-cmark` is already a non-optional xberg
//! dependency.

#![allow(
    clippy::pedantic,
    clippy::cargo,
    clippy::missing_errors_doc,
    clippy::missing_panics_doc,
    missing_docs,
    rust_2018_idioms
)]

mod chunk_size;
mod splitter;
mod trim;

pub(crate) use chunk_size::{Characters, ChunkCapacity, ChunkConfig, ChunkSizer};
pub(crate) use splitter::{MarkdownSplitter, TextSplitter};