doc_extractor 0.1.1

Extract text chunks from common document formats.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

use text_splitter::ChunkConfig;

/**
 * modified from https://github.com/Abraxas-365/langchain-rust/tree/v4.6.0/src/text_splitter
 */
use super::*;

#[derive(Debug, Clone)]
pub struct TokenSplitter {
  splitter_options: SplitterOptions,
}

impl Default for TokenSplitter {
  fn default() -> Self {
    TokenSplitter::new(SplitterOptions::default())
  }
}

impl TokenSplitter {
  pub fn new(options: SplitterOptions) -> TokenSplitter {
    TokenSplitter {
      splitter_options: options,
    }
  }
}

impl TextSplitter for TokenSplitter {
  fn split_text(&self, text: &str) -> Result<Vec<String>, TextSplitterError> {
    let chunk_config = ChunkConfig::try_from(&self.splitter_options)?;
    Ok(
      text_splitter::TextSplitter::new(chunk_config)
        .chunks(text)
        .map(|x| x.to_string())
        .collect(),
    )
  }
}