1#![forbid(unsafe_code)]
64#![deny(missing_docs)]
65
66use std::borrow::Cow;
67use std::collections::HashMap;
68use std::fmt;
69use std::sync::Arc;
70
71type CustomLengthFunction = dyn Fn(&str) -> usize + Send + Sync;
72
73#[derive(Debug, Clone, PartialEq, Eq)]
75#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
76pub struct Chunk {
77 pub page_content: String,
79 pub start_index: Option<usize>,
86 pub end_index: Option<usize>,
93}
94
95impl Chunk {
96 pub fn new(page_content: impl Into<String>, start_index: Option<usize>) -> Self {
101 Self {
102 page_content: page_content.into(),
103 start_index,
104 end_index: None,
105 }
106 }
107
108 pub fn with_range(
110 page_content: impl Into<String>,
111 start_index: usize,
112 end_index: usize,
113 ) -> Self {
114 Self {
115 page_content: page_content.into(),
116 start_index: Some(start_index),
117 end_index: Some(end_index),
118 }
119 }
120}
121
122#[derive(Debug, Clone, PartialEq, Eq, Default)]
127#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
128pub struct Document {
129 pub page_content: String,
131 pub metadata: HashMap<String, String>,
133}
134
135impl Document {
136 pub fn new(page_content: impl Into<String>) -> Self {
138 Self {
139 page_content: page_content.into(),
140 metadata: HashMap::new(),
141 }
142 }
143
144 pub fn with_metadata(
146 page_content: impl Into<String>,
147 metadata: HashMap<String, String>,
148 ) -> Self {
149 Self {
150 page_content: page_content.into(),
151 metadata,
152 }
153 }
154}
155
156impl From<Chunk> for Document {
157 fn from(chunk: Chunk) -> Self {
160 let mut metadata = HashMap::with_capacity(2);
161 if let Some(start_index) = chunk.start_index {
162 metadata.insert("start_index".to_owned(), start_index.to_string());
163 }
164 if let Some(end_index) = chunk.end_index {
165 metadata.insert("end_index".to_owned(), end_index.to_string());
166 }
167 Self {
168 page_content: chunk.page_content,
169 metadata,
170 }
171 }
172}
173
174#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
176pub enum LengthFunction {
177 #[default]
179 CharCount,
180 ByteCount,
182}
183
184impl LengthFunction {
185 #[inline]
187 pub fn len(self, text: &str) -> usize {
188 match self {
189 Self::CharCount => text.chars().count(),
190 Self::ByteCount => text.len(),
191 }
192 }
193}
194
195#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
199pub enum KeepSeparator {
200 None,
202 #[default]
204 Start,
205 End,
207}
208
209impl KeepSeparator {
210 pub const fn from_bool(keep: bool) -> Self {
212 if keep { Self::Start } else { Self::None }
213 }
214
215 pub const fn is_keep(self) -> bool {
217 !matches!(self, Self::None)
218 }
219}
220
221#[derive(Debug, Clone, PartialEq, Eq)]
224pub enum SplitterError {
225 InvalidChunkSize {
227 chunk_size: usize,
229 },
230 ChunkOverlapExceedsChunkSize {
232 chunk_overlap: usize,
234 chunk_size: usize,
236 },
237 InvalidRegex {
239 pattern: String,
241 message: String,
243 },
244}
245
246impl fmt::Display for SplitterError {
247 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
248 match self {
249 Self::InvalidChunkSize { chunk_size } => {
250 write!(f, "chunk_size must be greater than zero (got {chunk_size})")
251 }
252 Self::ChunkOverlapExceedsChunkSize {
253 chunk_overlap,
254 chunk_size,
255 } => write!(
256 f,
257 "chunk_overlap ({chunk_overlap}) must not exceed chunk_size ({chunk_size})"
258 ),
259 Self::InvalidRegex { pattern, message } => {
260 write!(f, "invalid separator regex {pattern:?}: {message}")
261 }
262 }
263 }
264}
265
266impl std::error::Error for SplitterError {}
267
268pub trait SplitterStrategy {
280 fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk>;
282
283 fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
289 self.split(splitter, text)
290 .into_iter()
291 .map(|chunk| chunk.page_content)
292 .collect()
293 }
294}
295
296#[derive(Debug, Clone, Copy, Default)]
298pub struct SequentialStrategy;
299
300impl SplitterStrategy for SequentialStrategy {
301 fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk> {
302 splitter
303 .split_raw(text, Span::whole(text))
304 .into_iter()
305 .map(|raw| raw.into_chunk(splitter.add_start_index))
306 .collect()
307 }
308
309 fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
310 splitter
311 .split_raw(text, Span::whole(text))
312 .into_iter()
313 .map(|raw| raw.content)
314 .collect()
315 }
316}
317
318#[cfg(feature = "rayon")]
324#[derive(Debug, Clone, Copy, Default)]
325pub struct ParallelStrategy;
326
327#[cfg(feature = "rayon")]
328impl SplitterStrategy for ParallelStrategy {
329 fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk> {
330 splitter
331 .split_raw_par(text, Span::whole(text))
332 .into_iter()
333 .map(|raw| raw.into_chunk(splitter.add_start_index))
334 .collect()
335 }
336
337 fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
338 splitter
339 .split_raw_par(text, Span::whole(text))
340 .into_iter()
341 .map(|raw| raw.content)
342 .collect()
343 }
344}
345
346#[derive(Clone)]
350pub struct RecursiveCharacterTextSplitter {
351 separators: Vec<Separator>,
352 chunk_size: usize,
353 chunk_overlap: usize,
354 add_start_index: bool,
355 strip_whitespace: bool,
356 keep_separator: KeepSeparator,
357 length_function: LengthFunction,
358 custom_length_function: Option<Arc<CustomLengthFunction>>,
359 strategy: Arc<dyn SplitterStrategy + Send + Sync>,
360}
361
362impl fmt::Debug for RecursiveCharacterTextSplitter {
363 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
364 f.debug_struct("RecursiveCharacterTextSplitter")
365 .field(
366 "separators",
367 &self
368 .separators
369 .iter()
370 .map(|separator| separator.pattern.as_ref())
371 .collect::<Vec<_>>(),
372 )
373 .field("chunk_size", &self.chunk_size)
374 .field("chunk_overlap", &self.chunk_overlap)
375 .field("add_start_index", &self.add_start_index)
376 .field("strip_whitespace", &self.strip_whitespace)
377 .field("keep_separator", &self.keep_separator)
378 .field("length_function", &self.length_function)
379 .field(
380 "custom_length_function",
381 &self.custom_length_function.is_some(),
382 )
383 .finish_non_exhaustive()
384 }
385}
386
387impl Default for RecursiveCharacterTextSplitter {
388 fn default() -> Self {
389 Self::new()
390 }
391}
392
393impl RecursiveCharacterTextSplitter {
394 #[must_use]
402 pub fn new() -> Self {
403 Self {
404 separators: Vec::new(),
405 chunk_size: 4000,
406 chunk_overlap: 200,
407 add_start_index: false,
408 strip_whitespace: true,
409 keep_separator: KeepSeparator::Start,
410 length_function: LengthFunction::CharCount,
411 custom_length_function: None,
412 strategy: Arc::new(SequentialStrategy),
413 }
414 .with_separators(["\n\n", "\n", " ", ""])
415 }
416
417 #[must_use]
435 pub fn with_separators<I, S>(mut self, separators: I) -> Self
436 where
437 I: IntoIterator<Item = S>,
438 S: Into<Cow<'static, str>>,
439 {
440 self.separators = normalize_separators(
441 separators
442 .into_iter()
443 .map(|separator| Separator::literal(separator))
444 .collect(),
445 );
446 self
447 }
448
449 #[cfg(feature = "regex")]
463 pub fn with_regex_separators<I, S>(mut self, separators: I) -> Result<Self, SplitterError>
464 where
465 I: IntoIterator<Item = S>,
466 S: Into<Cow<'static, str>>,
467 {
468 let mut compiled = Vec::new();
469 for separator in separators {
470 compiled.push(Separator::regex(separator.into())?);
471 }
472 self.separators = normalize_separators(compiled);
473 Ok(self)
474 }
475
476 #[cfg(feature = "language")]
498 pub fn with_language(mut self, language: Language) -> Result<Self, SplitterError> {
499 let mut compiled = Vec::new();
501 for separator in language.separators() {
502 compiled.push(Separator::regex((*separator).into())?);
503 }
504 self.separators = normalize_separators(compiled);
505 Ok(self)
506 }
507
508 #[must_use]
510 pub fn with_chunk_size(mut self, size: usize) -> Self {
511 self.chunk_size = size;
512 self
513 }
514
515 #[must_use]
521 pub fn with_chunk_overlap(mut self, overlap: usize) -> Self {
522 self.chunk_overlap = overlap;
523 self
524 }
525
526 #[must_use]
528 pub fn with_add_start_index(mut self, add: bool) -> Self {
529 self.add_start_index = add;
530 self
531 }
532
533 #[must_use]
535 pub fn with_strip_whitespace(mut self, strip: bool) -> Self {
536 self.strip_whitespace = strip;
537 self
538 }
539
540 #[must_use]
545 pub fn with_keep_separator(mut self, keep: bool) -> Self {
546 self.keep_separator = KeepSeparator::from_bool(keep);
547 self
548 }
549
550 #[must_use]
552 pub fn with_keep_separator_mode(mut self, mode: KeepSeparator) -> Self {
553 self.keep_separator = mode;
554 self
555 }
556
557 #[must_use]
561 pub fn with_byte_length_function(mut self) -> Self {
562 self.length_function = LengthFunction::ByteCount;
563 self.custom_length_function = None;
564 self
565 }
566
567 #[must_use]
571 pub fn with_char_length_function(mut self) -> Self {
572 self.length_function = LengthFunction::CharCount;
573 self.custom_length_function = None;
574 self
575 }
576
577 #[must_use]
593 pub fn with_length_function<F>(mut self, length_function: F) -> Self
594 where
595 F: Fn(&str) -> usize + Send + Sync + 'static,
596 {
597 self.custom_length_function = Some(Arc::new(length_function));
598 self
599 }
600
601 #[must_use]
610 pub fn with_strategy<S>(self) -> Self
611 where
612 S: SplitterStrategy + Default + Send + Sync + 'static,
613 {
614 self.with_strategy_instance(S::default())
615 }
616
617 #[must_use]
624 pub fn with_strategy_instance<S>(mut self, strategy: S) -> Self
625 where
626 S: SplitterStrategy + Send + Sync + 'static,
627 {
628 self.strategy = Arc::new(strategy);
629 self
630 }
631
632 pub fn separators(&self) -> impl ExactSizeIterator<Item = &str> + '_ {
635 self.separators
636 .iter()
637 .map(|separator| separator.pattern.as_ref())
638 }
639
640 pub const fn chunk_size(&self) -> usize {
642 self.chunk_size
643 }
644
645 pub const fn chunk_overlap(&self) -> usize {
647 self.chunk_overlap
648 }
649
650 pub const fn add_start_index(&self) -> bool {
652 self.add_start_index
653 }
654
655 pub const fn strip_whitespace(&self) -> bool {
657 self.strip_whitespace
658 }
659
660 pub const fn keep_separator(&self) -> KeepSeparator {
662 self.keep_separator
663 }
664
665 pub fn validate(&self) -> Result<(), SplitterError> {
673 if self.chunk_size == 0 {
674 return Err(SplitterError::InvalidChunkSize {
675 chunk_size: self.chunk_size,
676 });
677 }
678 if self.chunk_overlap > self.chunk_size {
679 return Err(SplitterError::ChunkOverlapExceedsChunkSize {
680 chunk_overlap: self.chunk_overlap,
681 chunk_size: self.chunk_size,
682 });
683 }
684 Ok(())
685 }
686
687 #[must_use]
694 pub fn split_text(&self, text: &str) -> Vec<Chunk> {
695 self.try_split_text(text)
696 .unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
697 }
698
699 pub fn try_split_text(&self, text: &str) -> Result<Vec<Chunk>, SplitterError> {
706 self.validate()?;
707 Ok(self.strategy.split(self, text))
708 }
709
710 #[must_use]
717 pub fn split_text_content(&self, text: &str) -> Vec<String> {
718 self.try_split_text_content(text)
719 .unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
720 }
721
722 pub fn try_split_text_content(&self, text: &str) -> Result<Vec<String>, SplitterError> {
729 self.validate()?;
730 Ok(self.strategy.split_content(self, text))
731 }
732
733 #[cfg(feature = "rayon")]
743 #[must_use]
744 pub fn split_text_par(&self, text: &str) -> Vec<Chunk> {
745 self.try_split_text_par(text)
746 .unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
747 }
748
749 #[cfg(feature = "rayon")]
755 pub fn try_split_text_par(&self, text: &str) -> Result<Vec<Chunk>, SplitterError> {
756 self.validate()?;
757 Ok(self
758 .split_raw_par(text, Span::whole(text))
759 .into_iter()
760 .map(|raw| raw.into_chunk(self.add_start_index))
761 .collect())
762 }
763
764 #[cfg(feature = "rayon")]
772 #[must_use]
773 pub fn split_text_content_par(&self, text: &str) -> Vec<String> {
774 self.try_split_text_content_par(text)
775 .unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
776 }
777
778 #[cfg(feature = "rayon")]
784 pub fn try_split_text_content_par(&self, text: &str) -> Result<Vec<String>, SplitterError> {
785 self.validate()?;
786 Ok(self
787 .split_raw_par(text, Span::whole(text))
788 .into_iter()
789 .map(|raw| raw.content)
790 .collect())
791 }
792
793 #[cfg(feature = "rayon")]
795 pub const fn is_parallel(&self) -> bool {
796 true
797 }
798
799 #[cfg(not(feature = "rayon"))]
802 pub const fn is_parallel(&self) -> bool {
803 false
804 }
805
806 #[must_use]
817 pub fn split_documents(&self, documents: &[Document]) -> Vec<Document> {
818 self.try_split_documents(documents)
819 .unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
820 }
821
822 pub fn try_split_documents(
828 &self,
829 documents: &[Document],
830 ) -> Result<Vec<Document>, SplitterError> {
831 self.validate()?;
832 let mut output = Vec::new();
833 for document in documents {
834 for chunk in self.strategy.split(self, &document.page_content) {
835 let mut metadata = document.metadata.clone();
836 if let Some(start_index) = chunk.start_index {
837 metadata.insert("start_index".to_owned(), start_index.to_string());
838 }
839 if let Some(end_index) = chunk.end_index {
840 metadata.insert("end_index".to_owned(), end_index.to_string());
841 }
842 output.push(Document {
843 page_content: chunk.page_content,
844 metadata,
845 });
846 }
847 }
848 Ok(output)
849 }
850
851 #[must_use]
858 pub fn create_documents(&self, texts: &[&str]) -> Vec<Document> {
859 self.try_create_documents(texts)
860 .unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
861 }
862
863 pub fn try_create_documents(&self, texts: &[&str]) -> Result<Vec<Document>, SplitterError> {
869 self.validate()?;
870 let mut output = Vec::new();
871 for text in texts {
872 output.extend(
873 self.strategy
874 .split(self, text)
875 .into_iter()
876 .map(Document::from),
877 );
878 }
879 Ok(output)
880 }
881
882 #[cfg(feature = "language")]
891 pub fn markdown() -> Result<Self, SplitterError> {
892 Self::new().with_language(Language::Markdown)
893 }
894
895 #[inline]
896 fn measure(&self, text: &str) -> usize {
897 match &self.custom_length_function {
898 Some(length_function) => length_function(text),
899 None => self.length_function.len(text),
900 }
901 }
902
903 #[inline]
904 fn merge_separator<'a>(&self, separator: &'a Separator) -> &'a str {
905 if self.keep_separator.is_keep() {
906 ""
907 } else {
908 separator.pattern.as_ref()
909 }
910 }
911
912 fn find_separator<'a>(
915 &self,
916 text: &str,
917 span: Span,
918 separators: &'a [Separator],
919 ) -> (&'a Separator, &'a [Separator]) {
920 let haystack = span.slice(text);
921 for (index, separator) in separators.iter().enumerate() {
922 if separator.is_empty() || separator.is_match(haystack) {
923 return (separator, &separators[index + 1..]);
924 }
925 }
926
927 if let Some(last) = separators.last() {
931 (last, &[])
932 } else {
933 (&CHAR_FALLBACK, &[])
934 }
935 }
936
937 fn split_raw(&self, text: &str, span: Span) -> Vec<RawChunk> {
938 self.split_raw_with(text, span, &self.separators)
939 }
940
941 fn split_raw_with(&self, text: &str, span: Span, separators: &[Separator]) -> Vec<RawChunk> {
942 if span.is_empty() {
943 return Vec::new();
944 }
945
946 let (separator, remaining) = self.find_separator(text, span, separators);
947 let splits = self.split_spans(text, span, separator);
948 let merge_separator = self.merge_separator(separator);
949
950 let mut final_chunks = Vec::new();
951 let mut good_splits: Vec<Span> = Vec::new();
952
953 for split in splits {
954 if self.measure(split.slice(text)) < self.chunk_size {
955 good_splits.push(split);
956 } else {
957 if !good_splits.is_empty() {
958 final_chunks.extend(self.merge_spans(text, &good_splits, merge_separator));
959 good_splits.clear();
960 }
961
962 if remaining.is_empty() {
963 final_chunks.push(self.direct_chunk(text, split));
964 } else {
965 final_chunks.extend(self.split_raw_with(text, split, remaining));
966 }
967 }
968 }
969
970 if !good_splits.is_empty() {
971 final_chunks.extend(self.merge_spans(text, &good_splits, merge_separator));
972 }
973
974 final_chunks
975 }
976
977 #[cfg(feature = "rayon")]
978 fn split_raw_par(&self, text: &str, span: Span) -> Vec<RawChunk> {
979 self.split_raw_par_with(text, span, &self.separators)
980 }
981
982 #[cfg(feature = "rayon")]
983 fn split_raw_par_with(
984 &self,
985 text: &str,
986 span: Span,
987 separators: &[Separator],
988 ) -> Vec<RawChunk> {
989 use rayon::prelude::*;
990
991 if span.is_empty() {
992 return Vec::new();
993 }
994
995 let (separator, remaining) = self.find_separator(text, span, separators);
996 let splits = self.split_spans(text, span, separator);
997 let merge_separator = self.merge_separator(separator);
998
999 let mut final_chunks = Vec::new();
1000 let mut good_splits: Vec<Span> = Vec::new();
1001 let mut deferred: Vec<Span> = Vec::new();
1002 let mut segments: Vec<ParallelSegment> = Vec::new();
1003
1004 for split in splits {
1005 if self.measure(split.slice(text)) < self.chunk_size {
1006 good_splits.push(split);
1007 } else {
1008 if !good_splits.is_empty() {
1009 segments.push(ParallelSegment::Ready(self.merge_spans(
1010 text,
1011 &good_splits,
1012 merge_separator,
1013 )));
1014 good_splits.clear();
1015 }
1016
1017 if remaining.is_empty() {
1018 segments.push(ParallelSegment::Ready(vec![self.direct_chunk(text, split)]));
1019 } else {
1020 deferred.push(split);
1021 segments.push(ParallelSegment::Deferred);
1022 }
1023 }
1024 }
1025
1026 if !good_splits.is_empty() {
1027 segments.push(ParallelSegment::Ready(self.merge_spans(
1028 text,
1029 &good_splits,
1030 merge_separator,
1031 )));
1032 }
1033
1034 if deferred.is_empty() {
1035 for segment in segments {
1036 if let ParallelSegment::Ready(chunks) = segment {
1037 final_chunks.extend(chunks);
1038 }
1039 }
1040 return final_chunks;
1041 }
1042
1043 let results: Vec<Vec<RawChunk>> = if deferred.len() == 1 {
1044 vec![self.split_raw_par_with(text, deferred[0], remaining)]
1045 } else {
1046 deferred
1047 .par_iter()
1048 .map(|&split| self.split_raw_par_with(text, split, remaining))
1049 .collect()
1050 };
1051
1052 let mut results = results.into_iter();
1053 for segment in segments {
1054 match segment {
1055 ParallelSegment::Ready(chunks) => final_chunks.extend(chunks),
1056 ParallelSegment::Deferred => {
1057 if let Some(chunks) = results.next() {
1058 final_chunks.extend(chunks);
1059 }
1060 }
1061 }
1062 }
1063
1064 final_chunks
1065 }
1066
1067 fn split_spans(&self, text: &str, span: Span, separator: &Separator) -> Vec<Span> {
1069 let haystack = span.slice(text);
1070
1071 if separator.is_empty() {
1072 let mut spans = Vec::with_capacity(haystack.chars().count());
1073 let mut start = span.start;
1074 for character in haystack.chars() {
1075 let end = start + character.len_utf8();
1076 spans.push(Span::new(start, end));
1077 start = end;
1078 }
1079 return spans;
1080 }
1081
1082 match self.keep_separator {
1083 KeepSeparator::None => {
1084 let mut spans = Vec::new();
1085 let mut previous_end = 0usize;
1086 separator.for_each_match(haystack, |match_start, match_end| {
1087 if match_start > previous_end {
1088 spans.push(Span::new(
1089 span.start + previous_end,
1090 span.start + match_start,
1091 ));
1092 }
1093 previous_end = match_end;
1094 });
1095 if previous_end < span.len() {
1096 spans.push(Span::new(span.start + previous_end, span.end));
1097 }
1098 spans
1099 }
1100 KeepSeparator::Start => {
1101 let mut spans = Vec::new();
1102 let mut previous_start: Option<usize> = None;
1103 let mut first = true;
1104 separator.for_each_match(haystack, |match_start, _match_end| {
1105 if first {
1106 if match_start > 0 {
1107 spans.push(Span::new(span.start, span.start + match_start));
1108 }
1109 first = false;
1110 } else if let Some(previous) = previous_start {
1111 if match_start > previous {
1112 spans.push(Span::new(span.start + previous, span.start + match_start));
1113 }
1114 }
1115 previous_start = Some(match_start);
1116 });
1117 if let Some(previous) = previous_start {
1118 if span.len() > previous {
1119 spans.push(Span::new(span.start + previous, span.end));
1120 }
1121 }
1122 if spans.is_empty() {
1124 spans.push(span);
1125 }
1126 spans
1127 }
1128 KeepSeparator::End => {
1129 let mut spans = Vec::new();
1130 let mut previous_end = 0usize;
1131 separator.for_each_match(haystack, |_match_start, match_end| {
1132 if match_end > previous_end {
1133 spans.push(Span::new(span.start + previous_end, span.start + match_end));
1134 }
1135 previous_end = match_end;
1136 });
1137 if previous_end < span.len() {
1138 spans.push(Span::new(span.start + previous_end, span.end));
1139 }
1140 if spans.is_empty() {
1142 spans.push(span);
1143 }
1144 spans
1145 }
1146 }
1147 }
1148
1149 fn merge_spans(&self, text: &str, splits: &[Span], separator: &str) -> Vec<RawChunk> {
1151 if splits.is_empty() {
1152 return Vec::new();
1153 }
1154
1155 let separator_len = self.measure(separator);
1156 let mut docs = Vec::new();
1157 let mut current_start = 0usize;
1158 let mut total = 0usize;
1159
1160 for (index, split) in splits.iter().enumerate() {
1161 let split_len = self.measure(split.slice(text));
1162 let current_len = index - current_start;
1163 let separator_before = if current_len > 0 { separator_len } else { 0 };
1164
1165 if total
1166 .saturating_add(split_len)
1167 .saturating_add(separator_before)
1168 > self.chunk_size
1169 && current_len > 0
1170 {
1171 if let Some(chunk) =
1172 self.make_merged_chunk(text, &splits[current_start..index], separator)
1173 {
1174 docs.push(chunk);
1175 }
1176
1177 while current_start < index {
1179 let current_len = index - current_start;
1180 let need_trim = total > self.chunk_overlap
1181 || (total
1182 .saturating_add(split_len)
1183 .saturating_add(separator_len)
1184 > self.chunk_size
1185 && total > 0);
1186 if !need_trim {
1187 break;
1188 }
1189
1190 let first_len = self.measure(splits[current_start].slice(text));
1191 let separators_in_total = if current_len > 1 { separator_len } else { 0 };
1192 total = total.saturating_sub(first_len.saturating_add(separators_in_total));
1193 current_start += 1;
1194 }
1195 }
1196
1197 let current_len = index - current_start;
1198 total = total.saturating_add(split_len);
1199 if current_len > 0 {
1200 total = total.saturating_add(separator_len);
1201 }
1202 }
1203
1204 if current_start < splits.len() {
1205 if let Some(chunk) = self.make_merged_chunk(text, &splits[current_start..], separator) {
1206 docs.push(chunk);
1207 }
1208 }
1209
1210 docs
1211 }
1212
1213 fn make_merged_chunk(&self, text: &str, spans: &[Span], separator: &str) -> Option<RawChunk> {
1215 let first = spans.first()?;
1216 let last = spans.last()?;
1217 let span_start = first.start;
1218 let span_end = last.end;
1219
1220 let mut content = if self.keep_separator.is_keep() {
1221 text[span_start..span_end].to_owned()
1223 } else {
1224 let mut content = String::with_capacity(span_end - span_start);
1225 for (index, span) in spans.iter().enumerate() {
1226 if index > 0 {
1227 content.push_str(separator);
1228 }
1229 content.push_str(span.slice(text));
1230 }
1231 content
1232 };
1233
1234 let (start_index, end_index) = if self.strip_whitespace {
1235 let trimmed = content.trim();
1236 if trimmed.is_empty() {
1237 return None;
1238 }
1239
1240 let leading = content.len() - content.trim_start().len();
1241 let trailing = content.len() - content.trim_end().len();
1242
1243 let indices = if !self.add_start_index {
1244 (span_start, span_end)
1245 } else if self.keep_separator.is_keep() {
1246 (span_start + leading, span_end - trailing)
1248 } else {
1249 self.trimmed_source_bounds(text, span_start, span_end)
1253 };
1254
1255 content.drain(..leading);
1256 content.truncate(content.len() - trailing);
1257 indices
1258 } else {
1259 if content.is_empty() {
1260 return None;
1261 }
1262 (span_start, span_end)
1263 };
1264
1265 Some(RawChunk {
1266 content,
1267 start: start_index,
1268 end: end_index,
1269 })
1270 }
1271
1272 fn trimmed_source_bounds(&self, text: &str, start: usize, end: usize) -> (usize, usize) {
1275 let slice = &text[start..end];
1276 let first = slice
1277 .char_indices()
1278 .find(|(_, character)| !character.is_whitespace());
1279 let last = slice
1280 .char_indices()
1281 .rev()
1282 .find(|(_, character)| !character.is_whitespace());
1283
1284 match (first, last) {
1285 (Some((first_index, _)), Some((last_index, last_character))) => (
1286 start + first_index,
1287 start + last_index + last_character.len_utf8(),
1288 ),
1289 _ => (start, end),
1294 }
1295 }
1296
1297 fn direct_chunk(&self, text: &str, span: Span) -> RawChunk {
1298 RawChunk {
1299 content: span.slice(text).to_owned(),
1300 start: span.start,
1301 end: span.end,
1302 }
1303 }
1304}
1305
1306#[cfg(feature = "language")]
1315pub fn markdown_text_splitter() -> Result<RecursiveCharacterTextSplitter, SplitterError> {
1316 RecursiveCharacterTextSplitter::new().with_language(Language::Markdown)
1317}
1318
1319#[derive(Clone, Debug)]
1321struct Separator {
1322 pattern: Cow<'static, str>,
1323 #[cfg(feature = "regex")]
1324 regex: Option<regex::Regex>,
1325}
1326
1327impl Separator {
1328 fn literal(pattern: impl Into<Cow<'static, str>>) -> Self {
1329 #[cfg(feature = "regex")]
1330 {
1331 Self {
1332 pattern: pattern.into(),
1333 regex: None,
1334 }
1335 }
1336 #[cfg(not(feature = "regex"))]
1337 {
1338 Self {
1339 pattern: pattern.into(),
1340 }
1341 }
1342 }
1343
1344 #[cfg(feature = "regex")]
1345 fn regex(pattern: Cow<'static, str>) -> Result<Self, SplitterError> {
1346 if pattern.is_empty() {
1347 return Ok(Self::literal(pattern));
1348 }
1349
1350 let regex = regex::Regex::new(&pattern).map_err(|error| SplitterError::InvalidRegex {
1351 pattern: pattern.to_string(),
1352 message: error.to_string(),
1353 })?;
1354
1355 if regex.is_match("") {
1356 return Err(SplitterError::InvalidRegex {
1357 pattern: pattern.to_string(),
1358 message: "the pattern matches the empty string, which is not a valid split point"
1359 .to_owned(),
1360 });
1361 }
1362
1363 Ok(Self {
1364 pattern,
1365 regex: Some(regex),
1366 })
1367 }
1368
1369 fn is_empty(&self) -> bool {
1370 self.pattern.is_empty()
1371 }
1372
1373 fn is_match(&self, haystack: &str) -> bool {
1374 #[cfg(feature = "regex")]
1375 if let Some(regex) = &self.regex {
1376 return regex.is_match(haystack);
1377 }
1378
1379 haystack.contains(self.pattern.as_ref())
1380 }
1381
1382 fn for_each_match(&self, haystack: &str, mut callback: impl FnMut(usize, usize)) {
1383 #[cfg(feature = "regex")]
1384 if let Some(regex) = &self.regex {
1385 for matched in regex.find_iter(haystack) {
1386 if matched.start() < matched.end() {
1387 callback(matched.start(), matched.end());
1388 }
1389 }
1390 return;
1391 }
1392
1393 let pattern = self.pattern.as_ref();
1394 if pattern.is_empty() {
1395 return;
1396 }
1397 for (index, _) in haystack.match_indices(pattern) {
1398 callback(index, index + pattern.len());
1399 }
1400 }
1401}
1402
1403const CHAR_FALLBACK: Separator = Separator {
1404 pattern: Cow::Borrowed(""),
1405 #[cfg(feature = "regex")]
1406 regex: None,
1407};
1408
1409fn normalize_separators(mut separators: Vec<Separator>) -> Vec<Separator> {
1410 match separators.iter().position(Separator::is_empty) {
1411 Some(index) => separators.truncate(index + 1),
1416 None => separators.push(Separator::literal("")),
1417 }
1418 separators
1419}
1420
1421#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1423struct Span {
1424 start: usize,
1425 end: usize,
1426}
1427
1428impl Span {
1429 const fn new(start: usize, end: usize) -> Self {
1430 Self { start, end }
1431 }
1432
1433 fn whole(text: &str) -> Self {
1434 Self::new(0, text.len())
1435 }
1436
1437 fn len(self) -> usize {
1438 self.end - self.start
1439 }
1440
1441 fn is_empty(self) -> bool {
1442 self.start >= self.end
1443 }
1444
1445 fn slice(self, text: &str) -> &str {
1446 &text[self.start..self.end]
1447 }
1448}
1449
1450#[derive(Debug)]
1452struct RawChunk {
1453 content: String,
1454 start: usize,
1455 end: usize,
1456}
1457
1458impl RawChunk {
1459 fn into_chunk(self, include_indices: bool) -> Chunk {
1460 Chunk {
1461 page_content: self.content,
1462 start_index: include_indices.then_some(self.start),
1463 end_index: include_indices.then_some(self.end),
1464 }
1465 }
1466}
1467
1468#[cfg(feature = "rayon")]
1469enum ParallelSegment {
1470 Ready(Vec<RawChunk>),
1471 Deferred,
1472}
1473
1474#[cfg(feature = "language")]
1482#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1483#[non_exhaustive]
1484pub enum Language {
1485 C,
1487 Cpp,
1489 Go,
1491 Java,
1493 Kotlin,
1495 JavaScript,
1497 TypeScript,
1499 Php,
1501 Proto,
1503 Python,
1505 Rst,
1507 Ruby,
1509 Elixir,
1511 Rust,
1513 Scala,
1515 Swift,
1517 Markdown,
1519 Latex,
1521 Html,
1523 Sol,
1525 CSharp,
1527 Cobol,
1529 Lua,
1531 Perl,
1533 Haskell,
1535 PowerShell,
1537 VisualBasic6,
1539}
1540
1541#[cfg(feature = "language")]
1542impl Language {
1543 pub const fn separators(self) -> &'static [&'static str] {
1545 match self {
1546 Self::C | Self::Cpp => &[
1547 r"\nclass ",
1548 r"\nvoid ",
1549 r"\nint ",
1550 r"\nfloat ",
1551 r"\ndouble ",
1552 r"\nif ",
1553 r"\nfor ",
1554 r"\nwhile ",
1555 r"\nswitch ",
1556 r"\ncase ",
1557 r"\n\n",
1558 r"\n",
1559 " ",
1560 "",
1561 ],
1562 Self::Go => &[
1563 r"\nfunc ",
1564 r"\nvar ",
1565 r"\nconst ",
1566 r"\ntype ",
1567 r"\nif ",
1568 r"\nfor ",
1569 r"\nswitch ",
1570 r"\ncase ",
1571 r"\n\n",
1572 r"\n",
1573 " ",
1574 "",
1575 ],
1576 Self::Java => &[
1577 r"\nclass ",
1578 r"\npublic ",
1579 r"\nprotected ",
1580 r"\nprivate ",
1581 r"\nstatic ",
1582 r"\nif ",
1583 r"\nfor ",
1584 r"\nwhile ",
1585 r"\nswitch ",
1586 r"\ncase ",
1587 r"\n\n",
1588 r"\n",
1589 " ",
1590 "",
1591 ],
1592 Self::Kotlin => &[
1593 r"\nclass ",
1594 r"\npublic ",
1595 r"\nprotected ",
1596 r"\nprivate ",
1597 r"\ninternal ",
1598 r"\ncompanion ",
1599 r"\nfun ",
1600 r"\nval ",
1601 r"\nvar ",
1602 r"\nif ",
1603 r"\nfor ",
1604 r"\nwhile ",
1605 r"\nwhen ",
1606 r"\ncase ",
1607 r"\nelse ",
1608 r"\n\n",
1609 r"\n",
1610 " ",
1611 "",
1612 ],
1613 Self::JavaScript => &[
1614 r"\nfunction ",
1615 r"\nconst ",
1616 r"\nlet ",
1617 r"\nvar ",
1618 r"\nclass ",
1619 r"\nif ",
1620 r"\nfor ",
1621 r"\nwhile ",
1622 r"\nswitch ",
1623 r"\ncase ",
1624 r"\ndefault ",
1625 r"\n\n",
1626 r"\n",
1627 " ",
1628 "",
1629 ],
1630 Self::TypeScript => &[
1631 r"\nenum ",
1632 r"\ninterface ",
1633 r"\nnamespace ",
1634 r"\ntype ",
1635 r"\nclass ",
1636 r"\nfunction ",
1637 r"\nconst ",
1638 r"\nlet ",
1639 r"\nvar ",
1640 r"\nif ",
1641 r"\nfor ",
1642 r"\nwhile ",
1643 r"\nswitch ",
1644 r"\ncase ",
1645 r"\ndefault ",
1646 r"\n\n",
1647 r"\n",
1648 " ",
1649 "",
1650 ],
1651 Self::Php => &[
1652 r"\nfunction ",
1653 r"\nclass ",
1654 r"\nif ",
1655 r"\nforeach ",
1656 r"\nwhile ",
1657 r"\ndo ",
1658 r"\nswitch ",
1659 r"\ncase ",
1660 r"\n\n",
1661 r"\n",
1662 " ",
1663 "",
1664 ],
1665 Self::Proto => &[
1666 r"\nmessage ",
1667 r"\nservice ",
1668 r"\nenum ",
1669 r"\noption ",
1670 r"\nimport ",
1671 r"\nsyntax ",
1672 r"\n\n",
1673 r"\n",
1674 " ",
1675 "",
1676 ],
1677 Self::Python => &[r"\nclass ", r"\ndef ", r"\n\tdef ", r"\n\n", r"\n", " ", ""],
1678 Self::Rst => &[
1679 r"\n=+\n",
1680 r"\n-+\n",
1681 r"\n\*+\n",
1682 r"\n\n\.\. .*\n\n",
1683 r"\n\n",
1684 r"\n",
1685 " ",
1686 "",
1687 ],
1688 Self::Ruby => &[
1689 r"\ndef ",
1690 r"\nclass ",
1691 r"\nif ",
1692 r"\nunless ",
1693 r"\nwhile ",
1694 r"\nfor ",
1695 r"\ndo ",
1696 r"\nbegin ",
1697 r"\nrescue ",
1698 r"\n\n",
1699 r"\n",
1700 " ",
1701 "",
1702 ],
1703 Self::Elixir => &[
1704 r"\ndef ",
1705 r"\ndefp ",
1706 r"\ndefmodule ",
1707 r"\ndefprotocol ",
1708 r"\ndefmacro ",
1709 r"\ndefmacrop ",
1710 r"\nif ",
1711 r"\nunless ",
1712 r"\nwhile ",
1713 r"\ncase ",
1714 r"\ncond ",
1715 r"\nwith ",
1716 r"\nfor ",
1717 r"\ndo ",
1718 r"\n\n",
1719 r"\n",
1720 " ",
1721 "",
1722 ],
1723 Self::Rust => &[
1724 r"\nfn ",
1725 r"\nconst ",
1726 r"\nlet ",
1727 r"\nif ",
1728 r"\nwhile ",
1729 r"\nfor ",
1730 r"\nloop ",
1731 r"\nmatch ",
1732 r"\n\n",
1733 r"\n",
1734 " ",
1735 "",
1736 ],
1737 Self::Scala => &[
1738 r"\nclass ",
1739 r"\nobject ",
1740 r"\ndef ",
1741 r"\nval ",
1742 r"\nvar ",
1743 r"\nif ",
1744 r"\nfor ",
1745 r"\nwhile ",
1746 r"\nmatch ",
1747 r"\ncase ",
1748 r"\n\n",
1749 r"\n",
1750 " ",
1751 "",
1752 ],
1753 Self::Swift => &[
1754 r"\nfunc ",
1755 r"\nclass ",
1756 r"\nstruct ",
1757 r"\nenum ",
1758 r"\nif ",
1759 r"\nfor ",
1760 r"\nwhile ",
1761 r"\ndo ",
1762 r"\nswitch ",
1763 r"\ncase ",
1764 r"\n\n",
1765 r"\n",
1766 " ",
1767 "",
1768 ],
1769 Self::Markdown => &[
1770 r"\n#{1,6} ",
1771 "```\n",
1772 r"\n\*\*\*+\n",
1773 r"\n---+\n",
1774 r"\n___+\n",
1775 r"\n\n",
1776 r"\n",
1777 " ",
1778 "",
1779 ],
1780 Self::Latex => &[
1781 r"\n\\chapter\{",
1782 r"\n\\section\{",
1783 r"\n\\subsection\{",
1784 r"\n\\subsubsection\{",
1785 r"\n\\begin\{enumerate\}",
1786 r"\n\\begin\{itemize\}",
1787 r"\n\\begin\{description\}",
1788 r"\n\\begin\{list\}",
1789 r"\n\\begin\{quote\}",
1790 r"\n\\begin\{quotation\}",
1791 r"\n\\begin\{verse\}",
1792 r"\n\\begin\{verbatim\}",
1793 r"\n\\begin\{align\}",
1794 r"\$\$",
1795 r"\$",
1796 " ",
1797 "",
1798 ],
1799 Self::Html => &[
1800 "<body", "<div", "<p", "<br", "<li", "<h1", "<h2", "<h3", "<h4", "<h5", "<h6",
1801 "<span", "<table", "<tr", "<td", "<th", "<ul", "<ol", "<header", "<footer", "<nav",
1802 "<head", "<style", "<script", "<meta", "<title", "",
1803 ],
1804 Self::Sol => &[
1805 r"\npragma ",
1806 r"\nusing ",
1807 r"\ncontract ",
1808 r"\ninterface ",
1809 r"\nlibrary ",
1810 r"\nconstructor ",
1811 r"\ntype ",
1812 r"\nfunction ",
1813 r"\nevent ",
1814 r"\nmodifier ",
1815 r"\nerror ",
1816 r"\nstruct ",
1817 r"\nenum ",
1818 r"\nif ",
1819 r"\nfor ",
1820 r"\nwhile ",
1821 r"\ndo while ",
1822 r"\nassembly ",
1823 r"\n\n",
1824 r"\n",
1825 " ",
1826 "",
1827 ],
1828 Self::CSharp => &[
1829 r"\ninterface ",
1830 r"\nenum ",
1831 r"\nimplements ",
1832 r"\ndelegate ",
1833 r"\nevent ",
1834 r"\nclass ",
1835 r"\nabstract ",
1836 r"\npublic ",
1837 r"\nprotected ",
1838 r"\nprivate ",
1839 r"\nstatic ",
1840 r"\nreturn ",
1841 r"\nif ",
1842 r"\ncontinue ",
1843 r"\nfor ",
1844 r"\nforeach ",
1845 r"\nwhile ",
1846 r"\nswitch ",
1847 r"\nbreak ",
1848 r"\ncase ",
1849 r"\nelse ",
1850 r"\ntry ",
1851 r"\nthrow ",
1852 r"\nfinally ",
1853 r"\ncatch ",
1854 r"\n\n",
1855 r"\n",
1856 " ",
1857 "",
1858 ],
1859 Self::Cobol => &[
1860 r"\nIDENTIFICATION DIVISION.",
1861 r"\nENVIRONMENT DIVISION.",
1862 r"\nDATA DIVISION.",
1863 r"\nPROCEDURE DIVISION.",
1864 r"\nWORKING-STORAGE SECTION.",
1865 r"\nLINKAGE SECTION.",
1866 r"\nFILE SECTION.",
1867 r"\nINPUT-OUTPUT SECTION.",
1868 r"\nOPEN ",
1869 r"\nCLOSE ",
1870 r"\nREAD ",
1871 r"\nWRITE ",
1872 r"\nIF ",
1873 r"\nELSE ",
1874 r"\nMOVE ",
1875 r"\nPERFORM ",
1876 r"\nUNTIL ",
1877 r"\nVARYING ",
1878 r"\nACCEPT ",
1879 r"\nDISPLAY ",
1880 r"\nSTOP RUN.",
1881 r"\n",
1882 " ",
1883 "",
1884 ],
1885 Self::Lua => &[
1886 r"\nlocal ",
1887 r"\nfunction ",
1888 r"\nif ",
1889 r"\nfor ",
1890 r"\nwhile ",
1891 r"\nrepeat ",
1892 r"\n\n",
1893 r"\n",
1894 " ",
1895 "",
1896 ],
1897 Self::Perl => &[
1898 r"\nsub ",
1899 r"\npackage ",
1900 r"\nuse ",
1901 r"\nif ",
1902 r"\nwhile ",
1903 r"\nfor ",
1904 r"\nforeach ",
1905 r"\n\n",
1906 r"\n",
1907 " ",
1908 "",
1909 ],
1910 Self::Haskell => &[
1911 r"\nmain :: ",
1912 r"\nmain = ",
1913 r"\nlet ",
1914 r"\nin ",
1915 r"\ndo ",
1916 r"\nwhere ",
1917 r"\n:: ",
1918 r"\n= ",
1919 r"\ndata ",
1920 r"\nnewtype ",
1921 r"\ntype ",
1922 r"\nmodule ",
1923 r"\nimport ",
1924 r"\nqualified ",
1925 r"\nimport qualified ",
1926 r"\nclass ",
1927 r"\ninstance ",
1928 r"\ncase ",
1929 r"\n\| ",
1930 r"\n= \{",
1931 r"\n, ",
1932 r"\n\n",
1933 r"\n",
1934 " ",
1935 "",
1936 ],
1937 Self::PowerShell => &[
1938 r"\nfunction ",
1939 r"\nparam ",
1940 r"\nif ",
1941 r"\nforeach ",
1942 r"\nfor ",
1943 r"\nwhile ",
1944 r"\nswitch ",
1945 r"\nclass ",
1946 r"\ntry ",
1947 r"\ncatch ",
1948 r"\nfinally ",
1949 r"\n\n",
1950 r"\n",
1951 " ",
1952 "",
1953 ],
1954 Self::VisualBasic6 => &[
1955 r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Sub\s+",
1956 r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Function\s+",
1957 r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Property\s+(?:Get|Let|Set)\s+",
1958 r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Type\s+",
1959 r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Enum\s+",
1960 r"\nIf\s+",
1961 r"\nElseIf\s+",
1962 r"\nElse\s+",
1963 r"\nSelect\s+Case\s+",
1964 r"\nCase\s+",
1965 r"\nFor\s+",
1966 r"\nDo\s+",
1967 r"\nWhile\s+",
1968 r"\nWith\s+",
1969 r"\n\n",
1970 r"\n",
1971 " ",
1972 "",
1973 ],
1974 }
1975 }
1976}
1977
1978#[cfg(feature = "language")]
1979impl fmt::Display for Language {
1980 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1981 let name = match self {
1982 Self::C => "C",
1983 Self::Cpp => "C++",
1984 Self::Go => "Go",
1985 Self::Java => "Java",
1986 Self::Kotlin => "Kotlin",
1987 Self::JavaScript => "JavaScript",
1988 Self::TypeScript => "TypeScript",
1989 Self::Php => "PHP",
1990 Self::Proto => "Protocol Buffers",
1991 Self::Python => "Python",
1992 Self::Rst => "reStructuredText",
1993 Self::Ruby => "Ruby",
1994 Self::Elixir => "Elixir",
1995 Self::Rust => "Rust",
1996 Self::Scala => "Scala",
1997 Self::Swift => "Swift",
1998 Self::Markdown => "Markdown",
1999 Self::Latex => "LaTeX",
2000 Self::Html => "HTML",
2001 Self::Sol => "Solidity",
2002 Self::CSharp => "C#",
2003 Self::Cobol => "COBOL",
2004 Self::Lua => "Lua",
2005 Self::Perl => "Perl",
2006 Self::Haskell => "Haskell",
2007 Self::PowerShell => "PowerShell",
2008 Self::VisualBasic6 => "Visual Basic 6",
2009 };
2010 f.write_str(name)
2011 }
2012}