#![forbid(unsafe_code)]
#![deny(missing_docs)]
use std::borrow::Cow;
use std::collections::HashMap;
use std::fmt;
use std::sync::Arc;
type CustomLengthFunction = dyn Fn(&str) -> usize + Send + Sync;
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct Chunk {
pub page_content: String,
pub start_index: Option<usize>,
pub end_index: Option<usize>,
}
impl Chunk {
pub fn new(page_content: impl Into<String>, start_index: Option<usize>) -> Self {
Self {
page_content: page_content.into(),
start_index,
end_index: None,
}
}
pub fn with_range(
page_content: impl Into<String>,
start_index: usize,
end_index: usize,
) -> Self {
Self {
page_content: page_content.into(),
start_index: Some(start_index),
end_index: Some(end_index),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct Document {
pub page_content: String,
pub metadata: HashMap<String, String>,
}
impl Document {
pub fn new(page_content: impl Into<String>) -> Self {
Self {
page_content: page_content.into(),
metadata: HashMap::new(),
}
}
pub fn with_metadata(
page_content: impl Into<String>,
metadata: HashMap<String, String>,
) -> Self {
Self {
page_content: page_content.into(),
metadata,
}
}
}
impl From<Chunk> for Document {
fn from(chunk: Chunk) -> Self {
let mut metadata = HashMap::with_capacity(2);
if let Some(start_index) = chunk.start_index {
metadata.insert("start_index".to_owned(), start_index.to_string());
}
if let Some(end_index) = chunk.end_index {
metadata.insert("end_index".to_owned(), end_index.to_string());
}
Self {
page_content: chunk.page_content,
metadata,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum LengthFunction {
#[default]
CharCount,
ByteCount,
}
impl LengthFunction {
#[inline]
pub fn len(self, text: &str) -> usize {
match self {
Self::CharCount => text.chars().count(),
Self::ByteCount => text.len(),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum KeepSeparator {
None,
#[default]
Start,
End,
}
impl KeepSeparator {
pub const fn from_bool(keep: bool) -> Self {
if keep { Self::Start } else { Self::None }
}
pub const fn is_keep(self) -> bool {
!matches!(self, Self::None)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SplitterError {
InvalidChunkSize {
chunk_size: usize,
},
ChunkOverlapExceedsChunkSize {
chunk_overlap: usize,
chunk_size: usize,
},
InvalidRegex {
pattern: String,
message: String,
},
}
impl fmt::Display for SplitterError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::InvalidChunkSize { chunk_size } => {
write!(f, "chunk_size must be greater than zero (got {chunk_size})")
}
Self::ChunkOverlapExceedsChunkSize {
chunk_overlap,
chunk_size,
} => write!(
f,
"chunk_overlap ({chunk_overlap}) must not exceed chunk_size ({chunk_size})"
),
Self::InvalidRegex { pattern, message } => {
write!(f, "invalid separator regex {pattern:?}: {message}")
}
}
}
}
impl std::error::Error for SplitterError {}
pub trait SplitterStrategy {
fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk>;
fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
self.split(splitter, text)
.into_iter()
.map(|chunk| chunk.page_content)
.collect()
}
}
#[derive(Debug, Clone, Copy, Default)]
pub struct SequentialStrategy;
impl SplitterStrategy for SequentialStrategy {
fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk> {
splitter
.split_raw(text, Span::whole(text))
.into_iter()
.map(|raw| raw.into_chunk(splitter.add_start_index))
.collect()
}
fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
splitter
.split_raw(text, Span::whole(text))
.into_iter()
.map(|raw| raw.content)
.collect()
}
}
#[cfg(feature = "rayon")]
#[derive(Debug, Clone, Copy, Default)]
pub struct ParallelStrategy;
#[cfg(feature = "rayon")]
impl SplitterStrategy for ParallelStrategy {
fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk> {
splitter
.split_raw_par(text, Span::whole(text))
.into_iter()
.map(|raw| raw.into_chunk(splitter.add_start_index))
.collect()
}
fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
splitter
.split_raw_par(text, Span::whole(text))
.into_iter()
.map(|raw| raw.content)
.collect()
}
}
#[derive(Clone)]
pub struct RecursiveCharacterTextSplitter {
separators: Vec<Separator>,
chunk_size: usize,
chunk_overlap: usize,
add_start_index: bool,
strip_whitespace: bool,
keep_separator: KeepSeparator,
length_function: LengthFunction,
custom_length_function: Option<Arc<CustomLengthFunction>>,
strategy: Arc<dyn SplitterStrategy + Send + Sync>,
}
impl fmt::Debug for RecursiveCharacterTextSplitter {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RecursiveCharacterTextSplitter")
.field(
"separators",
&self
.separators
.iter()
.map(|separator| separator.pattern.as_ref())
.collect::<Vec<_>>(),
)
.field("chunk_size", &self.chunk_size)
.field("chunk_overlap", &self.chunk_overlap)
.field("add_start_index", &self.add_start_index)
.field("strip_whitespace", &self.strip_whitespace)
.field("keep_separator", &self.keep_separator)
.field("length_function", &self.length_function)
.field(
"custom_length_function",
&self.custom_length_function.is_some(),
)
.finish_non_exhaustive()
}
}
impl Default for RecursiveCharacterTextSplitter {
fn default() -> Self {
Self::new()
}
}
impl RecursiveCharacterTextSplitter {
#[must_use]
pub fn new() -> Self {
Self {
separators: Vec::new(),
chunk_size: 4000,
chunk_overlap: 200,
add_start_index: false,
strip_whitespace: true,
keep_separator: KeepSeparator::Start,
length_function: LengthFunction::CharCount,
custom_length_function: None,
strategy: Arc::new(SequentialStrategy),
}
.with_separators(["\n\n", "\n", " ", ""])
}
#[must_use]
pub fn with_separators<I, S>(mut self, separators: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<Cow<'static, str>>,
{
self.separators = normalize_separators(
separators
.into_iter()
.map(|separator| Separator::literal(separator))
.collect(),
);
self
}
#[cfg(feature = "regex")]
pub fn with_regex_separators<I, S>(mut self, separators: I) -> Result<Self, SplitterError>
where
I: IntoIterator<Item = S>,
S: Into<Cow<'static, str>>,
{
let mut compiled = Vec::new();
for separator in separators {
compiled.push(Separator::regex(separator.into())?);
}
self.separators = normalize_separators(compiled);
Ok(self)
}
#[cfg(feature = "language")]
pub fn with_language(mut self, language: Language) -> Result<Self, SplitterError> {
let mut compiled = Vec::new();
for separator in language.separators() {
compiled.push(Separator::regex((*separator).into())?);
}
self.separators = normalize_separators(compiled);
Ok(self)
}
#[must_use]
pub fn with_chunk_size(mut self, size: usize) -> Self {
self.chunk_size = size;
self
}
#[must_use]
pub fn with_chunk_overlap(mut self, overlap: usize) -> Self {
self.chunk_overlap = overlap;
self
}
#[must_use]
pub fn with_add_start_index(mut self, add: bool) -> Self {
self.add_start_index = add;
self
}
#[must_use]
pub fn with_strip_whitespace(mut self, strip: bool) -> Self {
self.strip_whitespace = strip;
self
}
#[must_use]
pub fn with_keep_separator(mut self, keep: bool) -> Self {
self.keep_separator = KeepSeparator::from_bool(keep);
self
}
#[must_use]
pub fn with_keep_separator_mode(mut self, mode: KeepSeparator) -> Self {
self.keep_separator = mode;
self
}
#[must_use]
pub fn with_byte_length_function(mut self) -> Self {
self.length_function = LengthFunction::ByteCount;
self.custom_length_function = None;
self
}
#[must_use]
pub fn with_char_length_function(mut self) -> Self {
self.length_function = LengthFunction::CharCount;
self.custom_length_function = None;
self
}
#[must_use]
pub fn with_length_function<F>(mut self, length_function: F) -> Self
where
F: Fn(&str) -> usize + Send + Sync + 'static,
{
self.custom_length_function = Some(Arc::new(length_function));
self
}
#[must_use]
pub fn with_strategy<S>(self) -> Self
where
S: SplitterStrategy + Default + Send + Sync + 'static,
{
self.with_strategy_instance(S::default())
}
#[must_use]
pub fn with_strategy_instance<S>(mut self, strategy: S) -> Self
where
S: SplitterStrategy + Send + Sync + 'static,
{
self.strategy = Arc::new(strategy);
self
}
pub fn separators(&self) -> impl ExactSizeIterator<Item = &str> + '_ {
self.separators
.iter()
.map(|separator| separator.pattern.as_ref())
}
pub const fn chunk_size(&self) -> usize {
self.chunk_size
}
pub const fn chunk_overlap(&self) -> usize {
self.chunk_overlap
}
pub const fn add_start_index(&self) -> bool {
self.add_start_index
}
pub const fn strip_whitespace(&self) -> bool {
self.strip_whitespace
}
pub const fn keep_separator(&self) -> KeepSeparator {
self.keep_separator
}
pub fn validate(&self) -> Result<(), SplitterError> {
if self.chunk_size == 0 {
return Err(SplitterError::InvalidChunkSize {
chunk_size: self.chunk_size,
});
}
if self.chunk_overlap > self.chunk_size {
return Err(SplitterError::ChunkOverlapExceedsChunkSize {
chunk_overlap: self.chunk_overlap,
chunk_size: self.chunk_size,
});
}
Ok(())
}
#[must_use]
pub fn split_text(&self, text: &str) -> Vec<Chunk> {
self.try_split_text(text)
.unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
}
pub fn try_split_text(&self, text: &str) -> Result<Vec<Chunk>, SplitterError> {
self.validate()?;
Ok(self.strategy.split(self, text))
}
#[must_use]
pub fn split_text_content(&self, text: &str) -> Vec<String> {
self.try_split_text_content(text)
.unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
}
pub fn try_split_text_content(&self, text: &str) -> Result<Vec<String>, SplitterError> {
self.validate()?;
Ok(self.strategy.split_content(self, text))
}
#[cfg(feature = "rayon")]
#[must_use]
pub fn split_text_par(&self, text: &str) -> Vec<Chunk> {
self.try_split_text_par(text)
.unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
}
#[cfg(feature = "rayon")]
pub fn try_split_text_par(&self, text: &str) -> Result<Vec<Chunk>, SplitterError> {
self.validate()?;
Ok(self
.split_raw_par(text, Span::whole(text))
.into_iter()
.map(|raw| raw.into_chunk(self.add_start_index))
.collect())
}
#[cfg(feature = "rayon")]
#[must_use]
pub fn split_text_content_par(&self, text: &str) -> Vec<String> {
self.try_split_text_content_par(text)
.unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
}
#[cfg(feature = "rayon")]
pub fn try_split_text_content_par(&self, text: &str) -> Result<Vec<String>, SplitterError> {
self.validate()?;
Ok(self
.split_raw_par(text, Span::whole(text))
.into_iter()
.map(|raw| raw.content)
.collect())
}
#[cfg(feature = "rayon")]
pub const fn is_parallel(&self) -> bool {
true
}
#[cfg(not(feature = "rayon"))]
pub const fn is_parallel(&self) -> bool {
false
}
#[must_use]
pub fn split_documents(&self, documents: &[Document]) -> Vec<Document> {
self.try_split_documents(documents)
.unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
}
pub fn try_split_documents(
&self,
documents: &[Document],
) -> Result<Vec<Document>, SplitterError> {
self.validate()?;
let mut output = Vec::new();
for document in documents {
for chunk in self.strategy.split(self, &document.page_content) {
let mut metadata = document.metadata.clone();
if let Some(start_index) = chunk.start_index {
metadata.insert("start_index".to_owned(), start_index.to_string());
}
if let Some(end_index) = chunk.end_index {
metadata.insert("end_index".to_owned(), end_index.to_string());
}
output.push(Document {
page_content: chunk.page_content,
metadata,
});
}
}
Ok(output)
}
#[must_use]
pub fn create_documents(&self, texts: &[&str]) -> Vec<Document> {
self.try_create_documents(texts)
.unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
}
pub fn try_create_documents(&self, texts: &[&str]) -> Result<Vec<Document>, SplitterError> {
self.validate()?;
let mut output = Vec::new();
for text in texts {
output.extend(
self.strategy
.split(self, text)
.into_iter()
.map(Document::from),
);
}
Ok(output)
}
#[cfg(feature = "language")]
pub fn markdown() -> Result<Self, SplitterError> {
Self::new().with_language(Language::Markdown)
}
#[inline]
fn measure(&self, text: &str) -> usize {
match &self.custom_length_function {
Some(length_function) => length_function(text),
None => self.length_function.len(text),
}
}
#[inline]
fn merge_separator<'a>(&self, separator: &'a Separator) -> &'a str {
if self.keep_separator.is_keep() {
""
} else {
separator.pattern.as_ref()
}
}
fn find_separator<'a>(
&self,
text: &str,
span: Span,
separators: &'a [Separator],
) -> (&'a Separator, &'a [Separator]) {
let haystack = span.slice(text);
for (index, separator) in separators.iter().enumerate() {
if separator.is_empty() || separator.is_match(haystack) {
return (separator, &separators[index + 1..]);
}
}
if let Some(last) = separators.last() {
(last, &[])
} else {
(&CHAR_FALLBACK, &[])
}
}
fn split_raw(&self, text: &str, span: Span) -> Vec<RawChunk> {
self.split_raw_with(text, span, &self.separators)
}
fn split_raw_with(&self, text: &str, span: Span, separators: &[Separator]) -> Vec<RawChunk> {
if span.is_empty() {
return Vec::new();
}
let (separator, remaining) = self.find_separator(text, span, separators);
let splits = self.split_spans(text, span, separator);
let merge_separator = self.merge_separator(separator);
let mut final_chunks = Vec::new();
let mut good_splits: Vec<Span> = Vec::new();
for split in splits {
if self.measure(split.slice(text)) < self.chunk_size {
good_splits.push(split);
} else {
if !good_splits.is_empty() {
final_chunks.extend(self.merge_spans(text, &good_splits, merge_separator));
good_splits.clear();
}
if remaining.is_empty() {
final_chunks.push(self.direct_chunk(text, split));
} else {
final_chunks.extend(self.split_raw_with(text, split, remaining));
}
}
}
if !good_splits.is_empty() {
final_chunks.extend(self.merge_spans(text, &good_splits, merge_separator));
}
final_chunks
}
#[cfg(feature = "rayon")]
fn split_raw_par(&self, text: &str, span: Span) -> Vec<RawChunk> {
self.split_raw_par_with(text, span, &self.separators)
}
#[cfg(feature = "rayon")]
fn split_raw_par_with(
&self,
text: &str,
span: Span,
separators: &[Separator],
) -> Vec<RawChunk> {
use rayon::prelude::*;
if span.is_empty() {
return Vec::new();
}
let (separator, remaining) = self.find_separator(text, span, separators);
let splits = self.split_spans(text, span, separator);
let merge_separator = self.merge_separator(separator);
let mut final_chunks = Vec::new();
let mut good_splits: Vec<Span> = Vec::new();
let mut deferred: Vec<Span> = Vec::new();
let mut segments: Vec<ParallelSegment> = Vec::new();
for split in splits {
if self.measure(split.slice(text)) < self.chunk_size {
good_splits.push(split);
} else {
if !good_splits.is_empty() {
segments.push(ParallelSegment::Ready(self.merge_spans(
text,
&good_splits,
merge_separator,
)));
good_splits.clear();
}
if remaining.is_empty() {
segments.push(ParallelSegment::Ready(vec![self.direct_chunk(text, split)]));
} else {
deferred.push(split);
segments.push(ParallelSegment::Deferred);
}
}
}
if !good_splits.is_empty() {
segments.push(ParallelSegment::Ready(self.merge_spans(
text,
&good_splits,
merge_separator,
)));
}
if deferred.is_empty() {
for segment in segments {
if let ParallelSegment::Ready(chunks) = segment {
final_chunks.extend(chunks);
}
}
return final_chunks;
}
let results: Vec<Vec<RawChunk>> = if deferred.len() == 1 {
vec![self.split_raw_par_with(text, deferred[0], remaining)]
} else {
deferred
.par_iter()
.map(|&split| self.split_raw_par_with(text, split, remaining))
.collect()
};
let mut results = results.into_iter();
for segment in segments {
match segment {
ParallelSegment::Ready(chunks) => final_chunks.extend(chunks),
ParallelSegment::Deferred => {
if let Some(chunks) = results.next() {
final_chunks.extend(chunks);
}
}
}
}
final_chunks
}
fn split_spans(&self, text: &str, span: Span, separator: &Separator) -> Vec<Span> {
let haystack = span.slice(text);
if separator.is_empty() {
let mut spans = Vec::with_capacity(haystack.chars().count());
let mut start = span.start;
for character in haystack.chars() {
let end = start + character.len_utf8();
spans.push(Span::new(start, end));
start = end;
}
return spans;
}
match self.keep_separator {
KeepSeparator::None => {
let mut spans = Vec::new();
let mut previous_end = 0usize;
separator.for_each_match(haystack, |match_start, match_end| {
if match_start > previous_end {
spans.push(Span::new(
span.start + previous_end,
span.start + match_start,
));
}
previous_end = match_end;
});
if previous_end < span.len() {
spans.push(Span::new(span.start + previous_end, span.end));
}
spans
}
KeepSeparator::Start => {
let mut spans = Vec::new();
let mut previous_start: Option<usize> = None;
let mut first = true;
separator.for_each_match(haystack, |match_start, _match_end| {
if first {
if match_start > 0 {
spans.push(Span::new(span.start, span.start + match_start));
}
first = false;
} else if let Some(previous) = previous_start {
if match_start > previous {
spans.push(Span::new(span.start + previous, span.start + match_start));
}
}
previous_start = Some(match_start);
});
if let Some(previous) = previous_start {
if span.len() > previous {
spans.push(Span::new(span.start + previous, span.end));
}
}
if spans.is_empty() {
spans.push(span);
}
spans
}
KeepSeparator::End => {
let mut spans = Vec::new();
let mut previous_end = 0usize;
separator.for_each_match(haystack, |_match_start, match_end| {
if match_end > previous_end {
spans.push(Span::new(span.start + previous_end, span.start + match_end));
}
previous_end = match_end;
});
if previous_end < span.len() {
spans.push(Span::new(span.start + previous_end, span.end));
}
if spans.is_empty() {
spans.push(span);
}
spans
}
}
}
fn merge_spans(&self, text: &str, splits: &[Span], separator: &str) -> Vec<RawChunk> {
if splits.is_empty() {
return Vec::new();
}
let separator_len = self.measure(separator);
let mut docs = Vec::new();
let mut current_start = 0usize;
let mut total = 0usize;
for (index, split) in splits.iter().enumerate() {
let split_len = self.measure(split.slice(text));
let current_len = index - current_start;
let separator_before = if current_len > 0 { separator_len } else { 0 };
if total
.saturating_add(split_len)
.saturating_add(separator_before)
> self.chunk_size
&& current_len > 0
{
if let Some(chunk) =
self.make_merged_chunk(text, &splits[current_start..index], separator)
{
docs.push(chunk);
}
while current_start < index {
let current_len = index - current_start;
let need_trim = total > self.chunk_overlap
|| (total
.saturating_add(split_len)
.saturating_add(separator_len)
> self.chunk_size
&& total > 0);
if !need_trim {
break;
}
let first_len = self.measure(splits[current_start].slice(text));
let separators_in_total = if current_len > 1 { separator_len } else { 0 };
total = total.saturating_sub(first_len.saturating_add(separators_in_total));
current_start += 1;
}
}
let current_len = index - current_start;
total = total.saturating_add(split_len);
if current_len > 0 {
total = total.saturating_add(separator_len);
}
}
if current_start < splits.len() {
if let Some(chunk) = self.make_merged_chunk(text, &splits[current_start..], separator) {
docs.push(chunk);
}
}
docs
}
fn make_merged_chunk(&self, text: &str, spans: &[Span], separator: &str) -> Option<RawChunk> {
let first = spans.first()?;
let last = spans.last()?;
let span_start = first.start;
let span_end = last.end;
let mut content = if self.keep_separator.is_keep() {
text[span_start..span_end].to_owned()
} else {
let mut content = String::with_capacity(span_end - span_start);
for (index, span) in spans.iter().enumerate() {
if index > 0 {
content.push_str(separator);
}
content.push_str(span.slice(text));
}
content
};
let (start_index, end_index) = if self.strip_whitespace {
let trimmed = content.trim();
if trimmed.is_empty() {
return None;
}
let leading = content.len() - content.trim_start().len();
let trailing = content.len() - content.trim_end().len();
let indices = if !self.add_start_index {
(span_start, span_end)
} else if self.keep_separator.is_keep() {
(span_start + leading, span_end - trailing)
} else {
self.trimmed_source_bounds(text, span_start, span_end)
};
content.drain(..leading);
content.truncate(content.len() - trailing);
indices
} else {
if content.is_empty() {
return None;
}
(span_start, span_end)
};
Some(RawChunk {
content,
start: start_index,
end: end_index,
})
}
fn trimmed_source_bounds(&self, text: &str, start: usize, end: usize) -> (usize, usize) {
let slice = &text[start..end];
let first = slice
.char_indices()
.find(|(_, character)| !character.is_whitespace());
let last = slice
.char_indices()
.rev()
.find(|(_, character)| !character.is_whitespace());
match (first, last) {
(Some((first_index, _)), Some((last_index, last_character))) => (
start + first_index,
start + last_index + last_character.len_utf8(),
),
_ => (start, end),
}
}
fn direct_chunk(&self, text: &str, span: Span) -> RawChunk {
RawChunk {
content: span.slice(text).to_owned(),
start: span.start,
end: span.end,
}
}
}
#[cfg(feature = "language")]
pub fn markdown_text_splitter() -> Result<RecursiveCharacterTextSplitter, SplitterError> {
RecursiveCharacterTextSplitter::new().with_language(Language::Markdown)
}
#[derive(Clone, Debug)]
struct Separator {
pattern: Cow<'static, str>,
#[cfg(feature = "regex")]
regex: Option<regex::Regex>,
}
impl Separator {
fn literal(pattern: impl Into<Cow<'static, str>>) -> Self {
#[cfg(feature = "regex")]
{
Self {
pattern: pattern.into(),
regex: None,
}
}
#[cfg(not(feature = "regex"))]
{
Self {
pattern: pattern.into(),
}
}
}
#[cfg(feature = "regex")]
fn regex(pattern: Cow<'static, str>) -> Result<Self, SplitterError> {
if pattern.is_empty() {
return Ok(Self::literal(pattern));
}
let regex = regex::Regex::new(&pattern).map_err(|error| SplitterError::InvalidRegex {
pattern: pattern.to_string(),
message: error.to_string(),
})?;
if regex.is_match("") {
return Err(SplitterError::InvalidRegex {
pattern: pattern.to_string(),
message: "the pattern matches the empty string, which is not a valid split point"
.to_owned(),
});
}
Ok(Self {
pattern,
regex: Some(regex),
})
}
fn is_empty(&self) -> bool {
self.pattern.is_empty()
}
fn is_match(&self, haystack: &str) -> bool {
#[cfg(feature = "regex")]
if let Some(regex) = &self.regex {
return regex.is_match(haystack);
}
haystack.contains(self.pattern.as_ref())
}
fn for_each_match(&self, haystack: &str, mut callback: impl FnMut(usize, usize)) {
#[cfg(feature = "regex")]
if let Some(regex) = &self.regex {
for matched in regex.find_iter(haystack) {
if matched.start() < matched.end() {
callback(matched.start(), matched.end());
}
}
return;
}
let pattern = self.pattern.as_ref();
if pattern.is_empty() {
return;
}
for (index, _) in haystack.match_indices(pattern) {
callback(index, index + pattern.len());
}
}
}
const CHAR_FALLBACK: Separator = Separator {
pattern: Cow::Borrowed(""),
#[cfg(feature = "regex")]
regex: None,
};
fn normalize_separators(mut separators: Vec<Separator>) -> Vec<Separator> {
match separators.iter().position(Separator::is_empty) {
Some(index) => separators.truncate(index + 1),
None => separators.push(Separator::literal("")),
}
separators
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct Span {
start: usize,
end: usize,
}
impl Span {
const fn new(start: usize, end: usize) -> Self {
Self { start, end }
}
fn whole(text: &str) -> Self {
Self::new(0, text.len())
}
fn len(self) -> usize {
self.end - self.start
}
fn is_empty(self) -> bool {
self.start >= self.end
}
fn slice(self, text: &str) -> &str {
&text[self.start..self.end]
}
}
#[derive(Debug)]
struct RawChunk {
content: String,
start: usize,
end: usize,
}
impl RawChunk {
fn into_chunk(self, include_indices: bool) -> Chunk {
Chunk {
page_content: self.content,
start_index: include_indices.then_some(self.start),
end_index: include_indices.then_some(self.end),
}
}
}
#[cfg(feature = "rayon")]
enum ParallelSegment {
Ready(Vec<RawChunk>),
Deferred,
}
#[cfg(feature = "language")]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum Language {
C,
Cpp,
Go,
Java,
Kotlin,
JavaScript,
TypeScript,
Php,
Proto,
Python,
Rst,
Ruby,
Elixir,
Rust,
Scala,
Swift,
Markdown,
Latex,
Html,
Sol,
CSharp,
Cobol,
Lua,
Perl,
Haskell,
PowerShell,
VisualBasic6,
}
#[cfg(feature = "language")]
impl Language {
pub const fn separators(self) -> &'static [&'static str] {
match self {
Self::C | Self::Cpp => &[
r"\nclass ",
r"\nvoid ",
r"\nint ",
r"\nfloat ",
r"\ndouble ",
r"\nif ",
r"\nfor ",
r"\nwhile ",
r"\nswitch ",
r"\ncase ",
r"\n\n",
r"\n",
" ",
"",
],
Self::Go => &[
r"\nfunc ",
r"\nvar ",
r"\nconst ",
r"\ntype ",
r"\nif ",
r"\nfor ",
r"\nswitch ",
r"\ncase ",
r"\n\n",
r"\n",
" ",
"",
],
Self::Java => &[
r"\nclass ",
r"\npublic ",
r"\nprotected ",
r"\nprivate ",
r"\nstatic ",
r"\nif ",
r"\nfor ",
r"\nwhile ",
r"\nswitch ",
r"\ncase ",
r"\n\n",
r"\n",
" ",
"",
],
Self::Kotlin => &[
r"\nclass ",
r"\npublic ",
r"\nprotected ",
r"\nprivate ",
r"\ninternal ",
r"\ncompanion ",
r"\nfun ",
r"\nval ",
r"\nvar ",
r"\nif ",
r"\nfor ",
r"\nwhile ",
r"\nwhen ",
r"\ncase ",
r"\nelse ",
r"\n\n",
r"\n",
" ",
"",
],
Self::JavaScript => &[
r"\nfunction ",
r"\nconst ",
r"\nlet ",
r"\nvar ",
r"\nclass ",
r"\nif ",
r"\nfor ",
r"\nwhile ",
r"\nswitch ",
r"\ncase ",
r"\ndefault ",
r"\n\n",
r"\n",
" ",
"",
],
Self::TypeScript => &[
r"\nenum ",
r"\ninterface ",
r"\nnamespace ",
r"\ntype ",
r"\nclass ",
r"\nfunction ",
r"\nconst ",
r"\nlet ",
r"\nvar ",
r"\nif ",
r"\nfor ",
r"\nwhile ",
r"\nswitch ",
r"\ncase ",
r"\ndefault ",
r"\n\n",
r"\n",
" ",
"",
],
Self::Php => &[
r"\nfunction ",
r"\nclass ",
r"\nif ",
r"\nforeach ",
r"\nwhile ",
r"\ndo ",
r"\nswitch ",
r"\ncase ",
r"\n\n",
r"\n",
" ",
"",
],
Self::Proto => &[
r"\nmessage ",
r"\nservice ",
r"\nenum ",
r"\noption ",
r"\nimport ",
r"\nsyntax ",
r"\n\n",
r"\n",
" ",
"",
],
Self::Python => &[r"\nclass ", r"\ndef ", r"\n\tdef ", r"\n\n", r"\n", " ", ""],
Self::Rst => &[
r"\n=+\n",
r"\n-+\n",
r"\n\*+\n",
r"\n\n\.\. .*\n\n",
r"\n\n",
r"\n",
" ",
"",
],
Self::Ruby => &[
r"\ndef ",
r"\nclass ",
r"\nif ",
r"\nunless ",
r"\nwhile ",
r"\nfor ",
r"\ndo ",
r"\nbegin ",
r"\nrescue ",
r"\n\n",
r"\n",
" ",
"",
],
Self::Elixir => &[
r"\ndef ",
r"\ndefp ",
r"\ndefmodule ",
r"\ndefprotocol ",
r"\ndefmacro ",
r"\ndefmacrop ",
r"\nif ",
r"\nunless ",
r"\nwhile ",
r"\ncase ",
r"\ncond ",
r"\nwith ",
r"\nfor ",
r"\ndo ",
r"\n\n",
r"\n",
" ",
"",
],
Self::Rust => &[
r"\nfn ",
r"\nconst ",
r"\nlet ",
r"\nif ",
r"\nwhile ",
r"\nfor ",
r"\nloop ",
r"\nmatch ",
r"\n\n",
r"\n",
" ",
"",
],
Self::Scala => &[
r"\nclass ",
r"\nobject ",
r"\ndef ",
r"\nval ",
r"\nvar ",
r"\nif ",
r"\nfor ",
r"\nwhile ",
r"\nmatch ",
r"\ncase ",
r"\n\n",
r"\n",
" ",
"",
],
Self::Swift => &[
r"\nfunc ",
r"\nclass ",
r"\nstruct ",
r"\nenum ",
r"\nif ",
r"\nfor ",
r"\nwhile ",
r"\ndo ",
r"\nswitch ",
r"\ncase ",
r"\n\n",
r"\n",
" ",
"",
],
Self::Markdown => &[
r"\n#{1,6} ",
"```\n",
r"\n\*\*\*+\n",
r"\n---+\n",
r"\n___+\n",
r"\n\n",
r"\n",
" ",
"",
],
Self::Latex => &[
r"\n\\chapter\{",
r"\n\\section\{",
r"\n\\subsection\{",
r"\n\\subsubsection\{",
r"\n\\begin\{enumerate\}",
r"\n\\begin\{itemize\}",
r"\n\\begin\{description\}",
r"\n\\begin\{list\}",
r"\n\\begin\{quote\}",
r"\n\\begin\{quotation\}",
r"\n\\begin\{verse\}",
r"\n\\begin\{verbatim\}",
r"\n\\begin\{align\}",
r"\$\$",
r"\$",
" ",
"",
],
Self::Html => &[
"<body", "<div", "<p", "<br", "<li", "<h1", "<h2", "<h3", "<h4", "<h5", "<h6",
"<span", "<table", "<tr", "<td", "<th", "<ul", "<ol", "<header", "<footer", "<nav",
"<head", "<style", "<script", "<meta", "<title", "",
],
Self::Sol => &[
r"\npragma ",
r"\nusing ",
r"\ncontract ",
r"\ninterface ",
r"\nlibrary ",
r"\nconstructor ",
r"\ntype ",
r"\nfunction ",
r"\nevent ",
r"\nmodifier ",
r"\nerror ",
r"\nstruct ",
r"\nenum ",
r"\nif ",
r"\nfor ",
r"\nwhile ",
r"\ndo while ",
r"\nassembly ",
r"\n\n",
r"\n",
" ",
"",
],
Self::CSharp => &[
r"\ninterface ",
r"\nenum ",
r"\nimplements ",
r"\ndelegate ",
r"\nevent ",
r"\nclass ",
r"\nabstract ",
r"\npublic ",
r"\nprotected ",
r"\nprivate ",
r"\nstatic ",
r"\nreturn ",
r"\nif ",
r"\ncontinue ",
r"\nfor ",
r"\nforeach ",
r"\nwhile ",
r"\nswitch ",
r"\nbreak ",
r"\ncase ",
r"\nelse ",
r"\ntry ",
r"\nthrow ",
r"\nfinally ",
r"\ncatch ",
r"\n\n",
r"\n",
" ",
"",
],
Self::Cobol => &[
r"\nIDENTIFICATION DIVISION.",
r"\nENVIRONMENT DIVISION.",
r"\nDATA DIVISION.",
r"\nPROCEDURE DIVISION.",
r"\nWORKING-STORAGE SECTION.",
r"\nLINKAGE SECTION.",
r"\nFILE SECTION.",
r"\nINPUT-OUTPUT SECTION.",
r"\nOPEN ",
r"\nCLOSE ",
r"\nREAD ",
r"\nWRITE ",
r"\nIF ",
r"\nELSE ",
r"\nMOVE ",
r"\nPERFORM ",
r"\nUNTIL ",
r"\nVARYING ",
r"\nACCEPT ",
r"\nDISPLAY ",
r"\nSTOP RUN.",
r"\n",
" ",
"",
],
Self::Lua => &[
r"\nlocal ",
r"\nfunction ",
r"\nif ",
r"\nfor ",
r"\nwhile ",
r"\nrepeat ",
r"\n\n",
r"\n",
" ",
"",
],
Self::Perl => &[
r"\nsub ",
r"\npackage ",
r"\nuse ",
r"\nif ",
r"\nwhile ",
r"\nfor ",
r"\nforeach ",
r"\n\n",
r"\n",
" ",
"",
],
Self::Haskell => &[
r"\nmain :: ",
r"\nmain = ",
r"\nlet ",
r"\nin ",
r"\ndo ",
r"\nwhere ",
r"\n:: ",
r"\n= ",
r"\ndata ",
r"\nnewtype ",
r"\ntype ",
r"\nmodule ",
r"\nimport ",
r"\nqualified ",
r"\nimport qualified ",
r"\nclass ",
r"\ninstance ",
r"\ncase ",
r"\n\| ",
r"\n= \{",
r"\n, ",
r"\n\n",
r"\n",
" ",
"",
],
Self::PowerShell => &[
r"\nfunction ",
r"\nparam ",
r"\nif ",
r"\nforeach ",
r"\nfor ",
r"\nwhile ",
r"\nswitch ",
r"\nclass ",
r"\ntry ",
r"\ncatch ",
r"\nfinally ",
r"\n\n",
r"\n",
" ",
"",
],
Self::VisualBasic6 => &[
r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Sub\s+",
r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Function\s+",
r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Property\s+(?:Get|Let|Set)\s+",
r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Type\s+",
r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Enum\s+",
r"\nIf\s+",
r"\nElseIf\s+",
r"\nElse\s+",
r"\nSelect\s+Case\s+",
r"\nCase\s+",
r"\nFor\s+",
r"\nDo\s+",
r"\nWhile\s+",
r"\nWith\s+",
r"\n\n",
r"\n",
" ",
"",
],
}
}
}
#[cfg(feature = "language")]
impl fmt::Display for Language {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let name = match self {
Self::C => "C",
Self::Cpp => "C++",
Self::Go => "Go",
Self::Java => "Java",
Self::Kotlin => "Kotlin",
Self::JavaScript => "JavaScript",
Self::TypeScript => "TypeScript",
Self::Php => "PHP",
Self::Proto => "Protocol Buffers",
Self::Python => "Python",
Self::Rst => "reStructuredText",
Self::Ruby => "Ruby",
Self::Elixir => "Elixir",
Self::Rust => "Rust",
Self::Scala => "Scala",
Self::Swift => "Swift",
Self::Markdown => "Markdown",
Self::Latex => "LaTeX",
Self::Html => "HTML",
Self::Sol => "Solidity",
Self::CSharp => "C#",
Self::Cobol => "COBOL",
Self::Lua => "Lua",
Self::Perl => "Perl",
Self::Haskell => "Haskell",
Self::PowerShell => "PowerShell",
Self::VisualBasic6 => "Visual Basic 6",
};
f.write_str(name)
}
}