doc_extractor 0.1.0

Extract text chunks from common document formats.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

/**
 * modified from https://github.com/Abraxas-365/langchain-rust/tree/v4.6.0/src/text_splitter
 */
use text_splitter::ChunkConfigError;
use thiserror::Error;

#[derive(Error, Debug)]
pub enum TextSplitterError {
  #[error("Empty input text")]
  EmptyInputText,

  #[error("Mismatch metadata and text")]
  MetadataTextMismatch,

  #[error("Tokenizer not found")]
  TokenizerNotFound,

  #[error("Tokenizer creation failed due to invalid tokenizer")]
  InvalidTokenizer,

  #[error("Tokenizer creation failed due to invalid model")]
  InvalidModel,

  #[error("Invalid chunk overlap and size")]
  InvalidSplitterOptions,

  #[error("Error: {0}")]
  OtherError(String),
}

impl From<ChunkConfigError> for TextSplitterError {
  fn from(_: ChunkConfigError) -> Self {
    Self::InvalidSplitterOptions
  }
}