Skip to main content

alith_core/splitting/
markdown.rs

1use super::SplitError;
2use text_splitter::{ChunkConfig, MarkdownSplitter};
3
4/// Generate a list of chunks from a given markdown text.
5#[inline]
6pub fn split_markdown(
7    text: &str,
8    size: usize,
9    overlap: usize,
10    trim: bool,
11) -> Result<Vec<String>, SplitError> {
12    let chunk_config = ChunkConfig::new(size)
13        .with_overlap(overlap)?
14        .with_trim(trim);
15    Ok(MarkdownSplitter::new(chunk_config)
16        .chunks(text)
17        .map(|x| x.to_string())
18        .collect())
19}