text-splitter 0.30.1

Split text into semantic chunks, up to a desired chunk size. Supports calculating length by characters and tokens, and is callable from Rust and Python.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

#![doc = include_str!("../README.md")]
#![cfg_attr(docsrs, feature(doc_auto_cfg, doc_cfg))]

mod chunk_size;
mod splitter;
mod trim;

pub use chunk_size::{
    Characters, ChunkCapacity, ChunkCapacityError, ChunkConfig, ChunkConfigError, ChunkSizer,
};
#[cfg(feature = "markdown")]
pub use splitter::MarkdownSplitter;
pub use splitter::{ChunkCharIndex, TextSplitter};
#[cfg(feature = "code")]
pub use splitter::{CodeSplitter, CodeSplitterError};