Skip to main content

recursive_text_splitter/
lib.rs

1//! A high-performance, dependency-free recursive character text splitter.
2//!
3//! This crate is a Rust port of Python LangChain's
4//! `RecursiveCharacterTextSplitter`. It tries each separator in priority order,
5//! splits on the first viable one, and recursively splits oversized pieces with
6//! the remaining separators. Small pieces are merged with configurable overlap.
7//!
8//! # Example
9//!
10//! ```
11//! use recursive_text_splitter::RecursiveCharacterTextSplitter;
12//!
13//! let splitter = RecursiveCharacterTextSplitter::new()
14//!     .with_separators(vec!["\n\n", "\n", " ", ""])
15//!     .with_chunk_size(20)
16//!     .with_chunk_overlap(5);
17//!
18//! let chunks = splitter.split_text_content(
19//!     "First paragraph.\n\nSecond paragraph, which is a little longer.",
20//! );
21//!
22//! assert!(!chunks.is_empty());
23//! ```
24//!
25//! # Guarantees
26//!
27//! * No `unsafe` code, and no panics while splitting arbitrary UTF-8 input
28//!   with a valid configuration.
29//! * With the built-in length functions, every chunk fits within `chunk_size`
30//!   whenever the text can be divided into indivisible units (Unicode scalar
31//!   values) that are small enough.
32//! * `start_index` and `end_index` are **byte offsets** into the input, so they
33//!   can be used directly with Rust slicing. They are valid UTF-8 boundaries
34//!   and are non-decreasing across the returned chunks.
35//! * The splitter is `Send + Sync` and does not mutate shared state.
36//!
37//! # Overlap semantics
38//!
39//! [`RecursiveCharacterTextSplitter::with_chunk_overlap`] follows Python
40//! LangChain's merge algorithm. Overlap is a *target* for the suffix retained
41//! from the previous chunk, not a strict lower or upper bound on the source
42//! ranges. When the overlap is large relative to `chunk_size`, consecutive
43//! chunks can start at the same offset, contain the same text (duplicates), or
44//! be nested. This matches Python's output. Lower the overlap if your RAG
45//! pipeline requires strictly advancing chunks.
46//!
47//! Invalid configuration is reported by [`SplitterError`]. The infallible
48//! convenience methods panic with a descriptive message for invalid
49//! configuration; use the `try_*` variants when configuration is not trusted.
50//!
51//! # Feature flags
52//!
53//! * `rayon` — enables [`ParallelStrategy`].
54//! * `regex` — enables regex separators via
55//!   [`RecursiveCharacterTextSplitter::with_regex_separators`].
56//! * `language` — enables [`Language`] presets (implies `regex`).
57//! * `serde` — derives `Serialize`/`Deserialize` for [`Chunk`] and [`Document`].
58//!
59//! The `rig` and `rig-types` features are deprecated no-ops. They are kept so
60//! existing `Cargo.toml` files continue to resolve; mapping chunks to `rig`'s
61//! document type is a few lines of ordinary Rust (see the README).
62
63#![forbid(unsafe_code)]
64#![deny(missing_docs)]
65
66use std::borrow::Cow;
67use std::collections::HashMap;
68use std::fmt;
69use std::sync::Arc;
70
71type CustomLengthFunction = dyn Fn(&str) -> usize + Send + Sync;
72
73/// A chunk of text produced by the splitter.
74#[derive(Debug, Clone, PartialEq, Eq)]
75#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
76pub struct Chunk {
77    /// The text content of this chunk.
78    pub page_content: String,
79    /// Byte offset of the first byte of this chunk's source region.
80    ///
81    /// Only populated when [`RecursiveCharacterTextSplitter::with_add_start_index`]
82    /// is enabled. For chunks produced with `keep_separator = None`, consecutive
83    /// separators may be collapsed in `page_content`; the reported range still
84    /// covers the corresponding region of the source.
85    pub start_index: Option<usize>,
86    /// Byte offset just past the last byte of this chunk's source region.
87    ///
88    /// Only populated when [`RecursiveCharacterTextSplitter::with_add_start_index`]
89    /// is enabled. It is an exact end position when separators are kept; with
90    /// `keep_separator = None` it may include removed separators (see
91    /// [`Chunk::start_index`]).
92    pub end_index: Option<usize>,
93}
94
95impl Chunk {
96    /// Creates a chunk with optional start metadata.
97    ///
98    /// `end_index` is left unset. Use [`Chunk::with_range`] when the exact
99    /// source range is known.
100    pub fn new(page_content: impl Into<String>, start_index: Option<usize>) -> Self {
101        Self {
102            page_content: page_content.into(),
103            start_index,
104            end_index: None,
105        }
106    }
107
108    /// Creates a chunk with an exact byte range in the source text.
109    pub fn with_range(
110        page_content: impl Into<String>,
111        start_index: usize,
112        end_index: usize,
113    ) -> Self {
114        Self {
115            page_content: page_content.into(),
116            start_index: Some(start_index),
117            end_index: Some(end_index),
118        }
119    }
120}
121
122/// A document with text content and string metadata.
123///
124/// This mirrors the shape of Python LangChain's `Document` and is convenient
125/// for RAG pipelines that attach metadata to each chunk.
126#[derive(Debug, Clone, PartialEq, Eq, Default)]
127#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
128pub struct Document {
129    /// The text content of the document.
130    pub page_content: String,
131    /// Arbitrary string metadata associated with the document.
132    pub metadata: HashMap<String, String>,
133}
134
135impl Document {
136    /// Creates a document with empty metadata.
137    pub fn new(page_content: impl Into<String>) -> Self {
138        Self {
139            page_content: page_content.into(),
140            metadata: HashMap::new(),
141        }
142    }
143
144    /// Creates a document with the supplied metadata.
145    pub fn with_metadata(
146        page_content: impl Into<String>,
147        metadata: HashMap<String, String>,
148    ) -> Self {
149        Self {
150            page_content: page_content.into(),
151            metadata,
152        }
153    }
154}
155
156impl From<Chunk> for Document {
157    /// Converts a [`Chunk`] into a [`Document`], storing byte offsets as
158    /// `start_index` and `end_index` metadata when present.
159    fn from(chunk: Chunk) -> Self {
160        let mut metadata = HashMap::with_capacity(2);
161        if let Some(start_index) = chunk.start_index {
162            metadata.insert("start_index".to_owned(), start_index.to_string());
163        }
164        if let Some(end_index) = chunk.end_index {
165            metadata.insert("end_index".to_owned(), end_index.to_string());
166        }
167        Self {
168            page_content: chunk.page_content,
169            metadata,
170        }
171    }
172}
173
174/// How to measure the length of a piece of text.
175#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
176pub enum LengthFunction {
177    /// Count Unicode scalar values (characters).
178    #[default]
179    CharCount,
180    /// Count UTF-8 bytes.
181    ByteCount,
182}
183
184impl LengthFunction {
185    /// Measures `text` according to this length function.
186    #[inline]
187    pub fn len(self, text: &str) -> usize {
188        match self {
189            Self::CharCount => text.chars().count(),
190            Self::ByteCount => text.len(),
191        }
192    }
193}
194
195/// Where to keep separators when splitting.
196///
197/// Python's `keep_separator=True` is equivalent to [`KeepSeparator::Start`].
198#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
199pub enum KeepSeparator {
200    /// Remove separators from the pieces before merging.
201    None,
202    /// Attach each separator to the start of the following piece.
203    #[default]
204    Start,
205    /// Attach each separator to the end of the preceding piece.
206    End,
207}
208
209impl KeepSeparator {
210    /// Converts the Python-style boolean into a [`KeepSeparator`] mode.
211    pub const fn from_bool(keep: bool) -> Self {
212        if keep { Self::Start } else { Self::None }
213    }
214
215    /// Returns `true` when separators are retained.
216    pub const fn is_keep(self) -> bool {
217        !matches!(self, Self::None)
218    }
219}
220
221/// Errors produced by invalid splitter configuration or invalid regex
222/// separators.
223#[derive(Debug, Clone, PartialEq, Eq)]
224pub enum SplitterError {
225    /// `chunk_size` must be greater than zero.
226    InvalidChunkSize {
227        /// The offending chunk size.
228        chunk_size: usize,
229    },
230    /// `chunk_overlap` must not be greater than `chunk_size`.
231    ChunkOverlapExceedsChunkSize {
232        /// The offending overlap.
233        chunk_overlap: usize,
234        /// The configured chunk size.
235        chunk_size: usize,
236    },
237    /// A separator regex failed to compile or is unsafe for splitting.
238    InvalidRegex {
239        /// The offending pattern.
240        pattern: String,
241        /// A description of the problem.
242        message: String,
243    },
244}
245
246impl fmt::Display for SplitterError {
247    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
248        match self {
249            Self::InvalidChunkSize { chunk_size } => {
250                write!(f, "chunk_size must be greater than zero (got {chunk_size})")
251            }
252            Self::ChunkOverlapExceedsChunkSize {
253                chunk_overlap,
254                chunk_size,
255            } => write!(
256                f,
257                "chunk_overlap ({chunk_overlap}) must not exceed chunk_size ({chunk_size})"
258            ),
259            Self::InvalidRegex { pattern, message } => {
260                write!(f, "invalid separator regex {pattern:?}: {message}")
261            }
262        }
263    }
264}
265
266impl std::error::Error for SplitterError {}
267
268/// A strategy for turning text into chunks.
269///
270/// The default is [`SequentialStrategy`]. With the `rayon` feature,
271/// [`ParallelStrategy`] is also available. Custom strategies can be installed
272/// with [`RecursiveCharacterTextSplitter::with_strategy_instance`].
273///
274/// Implementations that want the built-in recursive algorithm should delegate
275/// to [`SequentialStrategy::split`]. Do not call
276/// [`RecursiveCharacterTextSplitter::split_text`] from inside
277/// [`SplitterStrategy::split`]: that method dispatches back to the installed
278/// strategy and would recurse indefinitely.
279pub trait SplitterStrategy {
280    /// Splits `text` into chunks.
281    fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk>;
282
283    /// Splits `text` into chunk contents.
284    ///
285    /// The default implementation calls [`SplitterStrategy::split`] and
286    /// discards metadata. Strategies can override it to avoid allocating
287    /// [`Chunk`] values.
288    fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
289        self.split(splitter, text)
290            .into_iter()
291            .map(|chunk| chunk.page_content)
292            .collect()
293    }
294}
295
296/// Default single-threaded splitting strategy.
297#[derive(Debug, Clone, Copy, Default)]
298pub struct SequentialStrategy;
299
300impl SplitterStrategy for SequentialStrategy {
301    fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk> {
302        splitter
303            .split_raw(text, Span::whole(text))
304            .into_iter()
305            .map(|raw| raw.into_chunk(splitter.add_start_index))
306            .collect()
307    }
308
309    fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
310        splitter
311            .split_raw(text, Span::whole(text))
312            .into_iter()
313            .map(|raw| raw.content)
314            .collect()
315    }
316}
317
318/// Parallel splitting strategy powered by `rayon`.
319///
320/// Requires the `rayon` feature. The output is byte-for-byte identical to
321/// [`SequentialStrategy`]; only the order in which independent recursive
322/// branches are evaluated changes.
323#[cfg(feature = "rayon")]
324#[derive(Debug, Clone, Copy, Default)]
325pub struct ParallelStrategy;
326
327#[cfg(feature = "rayon")]
328impl SplitterStrategy for ParallelStrategy {
329    fn split(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<Chunk> {
330        splitter
331            .split_raw_par(text, Span::whole(text))
332            .into_iter()
333            .map(|raw| raw.into_chunk(splitter.add_start_index))
334            .collect()
335    }
336
337    fn split_content(&self, splitter: &RecursiveCharacterTextSplitter, text: &str) -> Vec<String> {
338        splitter
339            .split_raw_par(text, Span::whole(text))
340            .into_iter()
341            .map(|raw| raw.content)
342            .collect()
343    }
344}
345
346/// A recursive character text splitter.
347///
348/// See the [crate documentation](crate) for examples and guarantees.
349#[derive(Clone)]
350pub struct RecursiveCharacterTextSplitter {
351    separators: Vec<Separator>,
352    chunk_size: usize,
353    chunk_overlap: usize,
354    add_start_index: bool,
355    strip_whitespace: bool,
356    keep_separator: KeepSeparator,
357    length_function: LengthFunction,
358    custom_length_function: Option<Arc<CustomLengthFunction>>,
359    strategy: Arc<dyn SplitterStrategy + Send + Sync>,
360}
361
362impl fmt::Debug for RecursiveCharacterTextSplitter {
363    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
364        f.debug_struct("RecursiveCharacterTextSplitter")
365            .field(
366                "separators",
367                &self
368                    .separators
369                    .iter()
370                    .map(|separator| separator.pattern.as_ref())
371                    .collect::<Vec<_>>(),
372            )
373            .field("chunk_size", &self.chunk_size)
374            .field("chunk_overlap", &self.chunk_overlap)
375            .field("add_start_index", &self.add_start_index)
376            .field("strip_whitespace", &self.strip_whitespace)
377            .field("keep_separator", &self.keep_separator)
378            .field("length_function", &self.length_function)
379            .field(
380                "custom_length_function",
381                &self.custom_length_function.is_some(),
382            )
383            .finish_non_exhaustive()
384    }
385}
386
387impl Default for RecursiveCharacterTextSplitter {
388    fn default() -> Self {
389        Self::new()
390    }
391}
392
393impl RecursiveCharacterTextSplitter {
394    /// Creates a splitter with LangChain-compatible defaults.
395    ///
396    /// Default separators: `["\n\n", "\n", " ", ""]`
397    ///
398    /// Default chunk size: `4000` characters
399    ///
400    /// Default chunk overlap: `200` characters
401    #[must_use]
402    pub fn new() -> Self {
403        Self {
404            separators: Vec::new(),
405            chunk_size: 4000,
406            chunk_overlap: 200,
407            add_start_index: false,
408            strip_whitespace: true,
409            keep_separator: KeepSeparator::Start,
410            length_function: LengthFunction::CharCount,
411            custom_length_function: None,
412            strategy: Arc::new(SequentialStrategy),
413        }
414        .with_separators(["\n\n", "\n", " ", ""])
415    }
416
417    /// Sets the separators, tried in priority order.
418    ///
419    /// An empty separator (`""`) is appended automatically when missing. This
420    /// guarantees that the splitter can always fall back to Unicode scalar
421    /// values and honour `chunk_size`, unlike Python LangChain, which can emit
422    /// oversized chunks when the final separator does not match. As in Python,
423    /// separators after the first empty separator are ignored.
424    ///
425    /// Both borrowed and owned separator strings are accepted:
426    ///
427    /// ```
428    /// use recursive_text_splitter::RecursiveCharacterTextSplitter;
429    ///
430    /// let dynamic = vec![String::from("\n\n"), String::from("\n"), String::from("")];
431    /// let splitter = RecursiveCharacterTextSplitter::new().with_separators(dynamic);
432    /// assert_eq!(splitter.chunk_size(), 4000);
433    /// ```
434    #[must_use]
435    pub fn with_separators<I, S>(mut self, separators: I) -> Self
436    where
437        I: IntoIterator<Item = S>,
438        S: Into<Cow<'static, str>>,
439    {
440        self.separators = normalize_separators(
441            separators
442                .into_iter()
443                .map(|separator| Separator::literal(separator))
444                .collect(),
445        );
446        self
447    }
448
449    /// Sets regex separators, tried in priority order.
450    ///
451    /// The patterns use the Rust [`regex`](https://docs.rs/regex) syntax, which
452    /// guarantees linear-time matching and therefore cannot suffer from
453    /// catastrophic backtracking. Patterns that match the empty string are
454    /// rejected because they do not describe meaningful split points.
455    ///
456    /// Requires the `regex` feature.
457    ///
458    /// # Errors
459    ///
460    /// Returns [`SplitterError::InvalidRegex`] if a pattern is not valid Rust
461    /// regex syntax or can match the empty string.
462    #[cfg(feature = "regex")]
463    pub fn with_regex_separators<I, S>(mut self, separators: I) -> Result<Self, SplitterError>
464    where
465        I: IntoIterator<Item = S>,
466        S: Into<Cow<'static, str>>,
467    {
468        let mut compiled = Vec::new();
469        for separator in separators {
470            compiled.push(Separator::regex(separator.into())?);
471        }
472        self.separators = normalize_separators(compiled);
473        Ok(self)
474    }
475
476    /// Configures separators for a programming or markup language.
477    ///
478    /// Requires the `language` feature. The preset patterns are regular
479    /// expressions compatible with the `regex` crate.
480    ///
481    /// ```
482    /// # #[cfg(feature = "language")] {
483    /// use recursive_text_splitter::{Language, RecursiveCharacterTextSplitter};
484    ///
485    /// let splitter = RecursiveCharacterTextSplitter::new()
486    ///     .with_language(Language::Markdown)
487    ///     .unwrap()
488    ///     .with_chunk_size(500);
489    /// # let _ = splitter;
490    /// # }
491    /// ```
492    ///
493    /// # Errors
494    ///
495    /// Returns [`SplitterError::InvalidRegex`] if one of the preset patterns is
496    /// not valid Rust regex syntax.
497    #[cfg(feature = "language")]
498    pub fn with_language(mut self, language: Language) -> Result<Self, SplitterError> {
499        // Compile each pattern, then delegate to the shared normalization path.
500        let mut compiled = Vec::new();
501        for separator in language.separators() {
502            compiled.push(Separator::regex((*separator).into())?);
503        }
504        self.separators = normalize_separators(compiled);
505        Ok(self)
506    }
507
508    /// Sets the maximum chunk size, measured by the configured length function.
509    #[must_use]
510    pub fn with_chunk_size(mut self, size: usize) -> Self {
511        self.chunk_size = size;
512        self
513    }
514
515    /// Sets the number of units to overlap between adjacent chunks.
516    ///
517    /// The value is a target, not a hard guarantee: Python-compatible merging
518    /// can produce duplicate or nested chunks when the overlap is large
519    /// relative to `chunk_size`. See the [crate documentation](crate#overlap-semantics).
520    #[must_use]
521    pub fn with_chunk_overlap(mut self, overlap: usize) -> Self {
522        self.chunk_overlap = overlap;
523        self
524    }
525
526    /// Enables or disables `start_index`/`end_index` byte offsets on chunks.
527    #[must_use]
528    pub fn with_add_start_index(mut self, add: bool) -> Self {
529        self.add_start_index = add;
530        self
531    }
532
533    /// Enables or disables trimming whitespace from chunk edges.
534    #[must_use]
535    pub fn with_strip_whitespace(mut self, strip: bool) -> Self {
536        self.strip_whitespace = strip;
537        self
538    }
539
540    /// Sets whether separators are kept (the Python boolean API).
541    ///
542    /// `true` is equivalent to [`KeepSeparator::Start`], `false` to
543    /// [`KeepSeparator::None`].
544    #[must_use]
545    pub fn with_keep_separator(mut self, keep: bool) -> Self {
546        self.keep_separator = KeepSeparator::from_bool(keep);
547        self
548    }
549
550    /// Sets where separators are kept in the output.
551    #[must_use]
552    pub fn with_keep_separator_mode(mut self, mode: KeepSeparator) -> Self {
553        self.keep_separator = mode;
554        self
555    }
556
557    /// Measures chunks in UTF-8 bytes instead of Unicode scalar values.
558    ///
559    /// This clears any custom length function.
560    #[must_use]
561    pub fn with_byte_length_function(mut self) -> Self {
562        self.length_function = LengthFunction::ByteCount;
563        self.custom_length_function = None;
564        self
565    }
566
567    /// Measures chunks in Unicode scalar values (the default).
568    ///
569    /// This clears any custom length function.
570    #[must_use]
571    pub fn with_char_length_function(mut self) -> Self {
572        self.length_function = LengthFunction::CharCount;
573        self.custom_length_function = None;
574        self
575    }
576
577    /// Uses a custom function to measure text length, for example a tokenizer.
578    ///
579    /// The function must be thread-safe because the splitter is `Send + Sync`.
580    /// Custom length functions override the built-in character/byte counters.
581    ///
582    /// ```
583    /// use recursive_text_splitter::RecursiveCharacterTextSplitter;
584    ///
585    /// let splitter = RecursiveCharacterTextSplitter::new()
586    ///     .with_length_function(|text| text.split_whitespace().count())
587    ///     .with_chunk_size(10)
588    ///     .with_chunk_overlap(0);
589    ///
590    /// assert!(!splitter.split_text_content("one two three").is_empty());
591    /// ```
592    #[must_use]
593    pub fn with_length_function<F>(mut self, length_function: F) -> Self
594    where
595        F: Fn(&str) -> usize + Send + Sync + 'static,
596    {
597        self.custom_length_function = Some(Arc::new(length_function));
598        self
599    }
600
601    /// Sets the splitting strategy from its [`Default`] value.
602    ///
603    /// ```
604    /// use recursive_text_splitter::{RecursiveCharacterTextSplitter, SequentialStrategy};
605    ///
606    /// let splitter = RecursiveCharacterTextSplitter::new()
607    ///     .with_strategy::<SequentialStrategy>();
608    /// ```
609    #[must_use]
610    pub fn with_strategy<S>(self) -> Self
611    where
612        S: SplitterStrategy + Default + Send + Sync + 'static,
613    {
614        self.with_strategy_instance(S::default())
615    }
616
617    /// Installs a specific strategy instance.
618    ///
619    /// Use this instead of [`Self::with_strategy`] when the strategy carries
620    /// configuration. Custom strategies should delegate to
621    /// [`SequentialStrategy`] or implement splitting directly; see
622    /// [`SplitterStrategy`] for details.
623    #[must_use]
624    pub fn with_strategy_instance<S>(mut self, strategy: S) -> Self
625    where
626        S: SplitterStrategy + Send + Sync + 'static,
627    {
628        self.strategy = Arc::new(strategy);
629        self
630    }
631
632    /// Returns the configured separators, including the implicit character
633    /// fallback.
634    pub fn separators(&self) -> impl ExactSizeIterator<Item = &str> + '_ {
635        self.separators
636            .iter()
637            .map(|separator| separator.pattern.as_ref())
638    }
639
640    /// Returns the configured chunk size.
641    pub const fn chunk_size(&self) -> usize {
642        self.chunk_size
643    }
644
645    /// Returns the configured chunk overlap.
646    pub const fn chunk_overlap(&self) -> usize {
647        self.chunk_overlap
648    }
649
650    /// Returns whether chunk byte offsets are enabled.
651    pub const fn add_start_index(&self) -> bool {
652        self.add_start_index
653    }
654
655    /// Returns whether whitespace is stripped from chunk edges.
656    pub const fn strip_whitespace(&self) -> bool {
657        self.strip_whitespace
658    }
659
660    /// Returns the configured separator placement mode.
661    pub const fn keep_separator(&self) -> KeepSeparator {
662        self.keep_separator
663    }
664
665    /// Validates the configuration.
666    ///
667    /// # Errors
668    ///
669    /// Returns [`SplitterError::InvalidChunkSize`] when `chunk_size` is zero,
670    /// and [`SplitterError::ChunkOverlapExceedsChunkSize`] when the overlap is
671    /// larger than the chunk size.
672    pub fn validate(&self) -> Result<(), SplitterError> {
673        if self.chunk_size == 0 {
674            return Err(SplitterError::InvalidChunkSize {
675                chunk_size: self.chunk_size,
676            });
677        }
678        if self.chunk_overlap > self.chunk_size {
679            return Err(SplitterError::ChunkOverlapExceedsChunkSize {
680                chunk_overlap: self.chunk_overlap,
681                chunk_size: self.chunk_size,
682            });
683        }
684        Ok(())
685    }
686
687    /// Splits text into chunks.
688    ///
689    /// # Panics
690    ///
691    /// Panics if the configuration is invalid. Use [`Self::try_split_text`] for
692    /// a fallible variant.
693    #[must_use]
694    pub fn split_text(&self, text: &str) -> Vec<Chunk> {
695        self.try_split_text(text)
696            .unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
697    }
698
699    /// Splits text into chunks, returning configuration errors instead of
700    /// panicking.
701    ///
702    /// # Errors
703    ///
704    /// Returns a [`SplitterError`] if the configuration is invalid.
705    pub fn try_split_text(&self, text: &str) -> Result<Vec<Chunk>, SplitterError> {
706        self.validate()?;
707        Ok(self.strategy.split(self, text))
708    }
709
710    /// Splits text into chunk contents without metadata.
711    ///
712    /// # Panics
713    ///
714    /// Panics if the configuration is invalid. Use
715    /// [`Self::try_split_text_content`] for a fallible variant.
716    #[must_use]
717    pub fn split_text_content(&self, text: &str) -> Vec<String> {
718        self.try_split_text_content(text)
719            .unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
720    }
721
722    /// Splits text into contents, returning configuration errors instead of
723    /// panicking.
724    ///
725    /// # Errors
726    ///
727    /// Returns a [`SplitterError`] if the configuration is invalid.
728    pub fn try_split_text_content(&self, text: &str) -> Result<Vec<String>, SplitterError> {
729        self.validate()?;
730        Ok(self.strategy.split_content(self, text))
731    }
732
733    /// Parallel version of [`Self::split_text`].
734    ///
735    /// Requires the `rayon` feature. The result is identical to
736    /// [`Self::split_text`].
737    ///
738    /// # Panics
739    ///
740    /// Panics if the configuration is invalid. Use
741    /// [`Self::try_split_text_par`] for a fallible variant.
742    #[cfg(feature = "rayon")]
743    #[must_use]
744    pub fn split_text_par(&self, text: &str) -> Vec<Chunk> {
745        self.try_split_text_par(text)
746            .unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
747    }
748
749    /// Fallible parallel version of [`Self::split_text`].
750    ///
751    /// # Errors
752    ///
753    /// Returns a [`SplitterError`] if the configuration is invalid.
754    #[cfg(feature = "rayon")]
755    pub fn try_split_text_par(&self, text: &str) -> Result<Vec<Chunk>, SplitterError> {
756        self.validate()?;
757        Ok(self
758            .split_raw_par(text, Span::whole(text))
759            .into_iter()
760            .map(|raw| raw.into_chunk(self.add_start_index))
761            .collect())
762    }
763
764    /// Parallel version of [`Self::split_text_content`].
765    ///
766    /// Requires the `rayon` feature.
767    ///
768    /// # Panics
769    ///
770    /// Panics if the configuration is invalid.
771    #[cfg(feature = "rayon")]
772    #[must_use]
773    pub fn split_text_content_par(&self, text: &str) -> Vec<String> {
774        self.try_split_text_content_par(text)
775            .unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
776    }
777
778    /// Fallible parallel version of [`Self::split_text_content`].
779    ///
780    /// # Errors
781    ///
782    /// Returns a [`SplitterError`] if the configuration is invalid.
783    #[cfg(feature = "rayon")]
784    pub fn try_split_text_content_par(&self, text: &str) -> Result<Vec<String>, SplitterError> {
785        self.validate()?;
786        Ok(self
787            .split_raw_par(text, Span::whole(text))
788            .into_iter()
789            .map(|raw| raw.content)
790            .collect())
791    }
792
793    /// Returns whether the `rayon` feature is enabled.
794    #[cfg(feature = "rayon")]
795    pub const fn is_parallel(&self) -> bool {
796        true
797    }
798
799    /// Returns whether the `rayon` feature is enabled (always `false` in this
800    /// build).
801    #[cfg(not(feature = "rayon"))]
802    pub const fn is_parallel(&self) -> bool {
803        false
804    }
805
806    /// Splits documents, preserving their metadata.
807    ///
808    /// When [`Self::with_add_start_index`] is enabled, `start_index` and
809    /// `end_index` are inserted into each chunk document's metadata as decimal
810    /// strings.
811    ///
812    /// # Panics
813    ///
814    /// Panics if the configuration is invalid. Use
815    /// [`Self::try_split_documents`] for a fallible variant.
816    #[must_use]
817    pub fn split_documents(&self, documents: &[Document]) -> Vec<Document> {
818        self.try_split_documents(documents)
819            .unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
820    }
821
822    /// Fallible version of [`Self::split_documents`].
823    ///
824    /// # Errors
825    ///
826    /// Returns a [`SplitterError`] if the configuration is invalid.
827    pub fn try_split_documents(
828        &self,
829        documents: &[Document],
830    ) -> Result<Vec<Document>, SplitterError> {
831        self.validate()?;
832        let mut output = Vec::new();
833        for document in documents {
834            for chunk in self.strategy.split(self, &document.page_content) {
835                let mut metadata = document.metadata.clone();
836                if let Some(start_index) = chunk.start_index {
837                    metadata.insert("start_index".to_owned(), start_index.to_string());
838                }
839                if let Some(end_index) = chunk.end_index {
840                    metadata.insert("end_index".to_owned(), end_index.to_string());
841                }
842                output.push(Document {
843                    page_content: chunk.page_content,
844                    metadata,
845                });
846            }
847        }
848        Ok(output)
849    }
850
851    /// Creates documents from plain texts with empty metadata.
852    ///
853    /// # Panics
854    ///
855    /// Panics if the configuration is invalid. Use
856    /// [`Self::try_create_documents`] for a fallible variant.
857    #[must_use]
858    pub fn create_documents(&self, texts: &[&str]) -> Vec<Document> {
859        self.try_create_documents(texts)
860            .unwrap_or_else(|error| panic!("invalid splitter configuration: {error}"))
861    }
862
863    /// Fallible version of [`Self::create_documents`].
864    ///
865    /// # Errors
866    ///
867    /// Returns a [`SplitterError`] if the configuration is invalid.
868    pub fn try_create_documents(&self, texts: &[&str]) -> Result<Vec<Document>, SplitterError> {
869        self.validate()?;
870        let mut output = Vec::new();
871        for text in texts {
872            output.extend(
873                self.strategy
874                    .split(self, text)
875                    .into_iter()
876                    .map(Document::from),
877            );
878        }
879        Ok(output)
880    }
881
882    /// Creates a Markdown-aware splitter.
883    ///
884    /// Requires the `language` feature.
885    ///
886    /// # Errors
887    ///
888    /// Returns [`SplitterError::InvalidRegex`] if a preset pattern cannot be
889    /// compiled.
890    #[cfg(feature = "language")]
891    pub fn markdown() -> Result<Self, SplitterError> {
892        Self::new().with_language(Language::Markdown)
893    }
894
895    #[inline]
896    fn measure(&self, text: &str) -> usize {
897        match &self.custom_length_function {
898            Some(length_function) => length_function(text),
899            None => self.length_function.len(text),
900        }
901    }
902
903    #[inline]
904    fn merge_separator<'a>(&self, separator: &'a Separator) -> &'a str {
905        if self.keep_separator.is_keep() {
906            ""
907        } else {
908            separator.pattern.as_ref()
909        }
910    }
911
912    /// Returns the first separator that matches the span, plus the remaining
913    /// separators.
914    fn find_separator<'a>(
915        &self,
916        text: &str,
917        span: Span,
918        separators: &'a [Separator],
919    ) -> (&'a Separator, &'a [Separator]) {
920        let haystack = span.slice(text);
921        for (index, separator) in separators.iter().enumerate() {
922            if separator.is_empty() || separator.is_match(haystack) {
923                return (separator, &separators[index + 1..]);
924            }
925        }
926
927        // `separators` is normalized to contain an empty fallback, so this is
928        // unreachable in practice. Keep a defensive fallback instead of
929        // panicking if a future invariant changes.
930        if let Some(last) = separators.last() {
931            (last, &[])
932        } else {
933            (&CHAR_FALLBACK, &[])
934        }
935    }
936
937    fn split_raw(&self, text: &str, span: Span) -> Vec<RawChunk> {
938        self.split_raw_with(text, span, &self.separators)
939    }
940
941    fn split_raw_with(&self, text: &str, span: Span, separators: &[Separator]) -> Vec<RawChunk> {
942        if span.is_empty() {
943            return Vec::new();
944        }
945
946        let (separator, remaining) = self.find_separator(text, span, separators);
947        let splits = self.split_spans(text, span, separator);
948        let merge_separator = self.merge_separator(separator);
949
950        let mut final_chunks = Vec::new();
951        let mut good_splits: Vec<Span> = Vec::new();
952
953        for split in splits {
954            if self.measure(split.slice(text)) < self.chunk_size {
955                good_splits.push(split);
956            } else {
957                if !good_splits.is_empty() {
958                    final_chunks.extend(self.merge_spans(text, &good_splits, merge_separator));
959                    good_splits.clear();
960                }
961
962                if remaining.is_empty() {
963                    final_chunks.push(self.direct_chunk(text, split));
964                } else {
965                    final_chunks.extend(self.split_raw_with(text, split, remaining));
966                }
967            }
968        }
969
970        if !good_splits.is_empty() {
971            final_chunks.extend(self.merge_spans(text, &good_splits, merge_separator));
972        }
973
974        final_chunks
975    }
976
977    #[cfg(feature = "rayon")]
978    fn split_raw_par(&self, text: &str, span: Span) -> Vec<RawChunk> {
979        self.split_raw_par_with(text, span, &self.separators)
980    }
981
982    #[cfg(feature = "rayon")]
983    fn split_raw_par_with(
984        &self,
985        text: &str,
986        span: Span,
987        separators: &[Separator],
988    ) -> Vec<RawChunk> {
989        use rayon::prelude::*;
990
991        if span.is_empty() {
992            return Vec::new();
993        }
994
995        let (separator, remaining) = self.find_separator(text, span, separators);
996        let splits = self.split_spans(text, span, separator);
997        let merge_separator = self.merge_separator(separator);
998
999        let mut final_chunks = Vec::new();
1000        let mut good_splits: Vec<Span> = Vec::new();
1001        let mut deferred: Vec<Span> = Vec::new();
1002        let mut segments: Vec<ParallelSegment> = Vec::new();
1003
1004        for split in splits {
1005            if self.measure(split.slice(text)) < self.chunk_size {
1006                good_splits.push(split);
1007            } else {
1008                if !good_splits.is_empty() {
1009                    segments.push(ParallelSegment::Ready(self.merge_spans(
1010                        text,
1011                        &good_splits,
1012                        merge_separator,
1013                    )));
1014                    good_splits.clear();
1015                }
1016
1017                if remaining.is_empty() {
1018                    segments.push(ParallelSegment::Ready(vec![self.direct_chunk(text, split)]));
1019                } else {
1020                    deferred.push(split);
1021                    segments.push(ParallelSegment::Deferred);
1022                }
1023            }
1024        }
1025
1026        if !good_splits.is_empty() {
1027            segments.push(ParallelSegment::Ready(self.merge_spans(
1028                text,
1029                &good_splits,
1030                merge_separator,
1031            )));
1032        }
1033
1034        if deferred.is_empty() {
1035            for segment in segments {
1036                if let ParallelSegment::Ready(chunks) = segment {
1037                    final_chunks.extend(chunks);
1038                }
1039            }
1040            return final_chunks;
1041        }
1042
1043        let results: Vec<Vec<RawChunk>> = if deferred.len() == 1 {
1044            vec![self.split_raw_par_with(text, deferred[0], remaining)]
1045        } else {
1046            deferred
1047                .par_iter()
1048                .map(|&split| self.split_raw_par_with(text, split, remaining))
1049                .collect()
1050        };
1051
1052        let mut results = results.into_iter();
1053        for segment in segments {
1054            match segment {
1055                ParallelSegment::Ready(chunks) => final_chunks.extend(chunks),
1056                ParallelSegment::Deferred => {
1057                    if let Some(chunks) = results.next() {
1058                        final_chunks.extend(chunks);
1059                    }
1060                }
1061            }
1062        }
1063
1064        final_chunks
1065    }
1066
1067    /// Splits a span by a separator, producing source spans.
1068    fn split_spans(&self, text: &str, span: Span, separator: &Separator) -> Vec<Span> {
1069        let haystack = span.slice(text);
1070
1071        if separator.is_empty() {
1072            let mut spans = Vec::with_capacity(haystack.chars().count());
1073            let mut start = span.start;
1074            for character in haystack.chars() {
1075                let end = start + character.len_utf8();
1076                spans.push(Span::new(start, end));
1077                start = end;
1078            }
1079            return spans;
1080        }
1081
1082        match self.keep_separator {
1083            KeepSeparator::None => {
1084                let mut spans = Vec::new();
1085                let mut previous_end = 0usize;
1086                separator.for_each_match(haystack, |match_start, match_end| {
1087                    if match_start > previous_end {
1088                        spans.push(Span::new(
1089                            span.start + previous_end,
1090                            span.start + match_start,
1091                        ));
1092                    }
1093                    previous_end = match_end;
1094                });
1095                if previous_end < span.len() {
1096                    spans.push(Span::new(span.start + previous_end, span.end));
1097                }
1098                spans
1099            }
1100            KeepSeparator::Start => {
1101                let mut spans = Vec::new();
1102                let mut previous_start: Option<usize> = None;
1103                let mut first = true;
1104                separator.for_each_match(haystack, |match_start, _match_end| {
1105                    if first {
1106                        if match_start > 0 {
1107                            spans.push(Span::new(span.start, span.start + match_start));
1108                        }
1109                        first = false;
1110                    } else if let Some(previous) = previous_start {
1111                        if match_start > previous {
1112                            spans.push(Span::new(span.start + previous, span.start + match_start));
1113                        }
1114                    }
1115                    previous_start = Some(match_start);
1116                });
1117                if let Some(previous) = previous_start {
1118                    if span.len() > previous {
1119                        spans.push(Span::new(span.start + previous, span.end));
1120                    }
1121                }
1122                // Defensive: a non-matching separator must not erase the span.
1123                if spans.is_empty() {
1124                    spans.push(span);
1125                }
1126                spans
1127            }
1128            KeepSeparator::End => {
1129                let mut spans = Vec::new();
1130                let mut previous_end = 0usize;
1131                separator.for_each_match(haystack, |_match_start, match_end| {
1132                    if match_end > previous_end {
1133                        spans.push(Span::new(span.start + previous_end, span.start + match_end));
1134                    }
1135                    previous_end = match_end;
1136                });
1137                if previous_end < span.len() {
1138                    spans.push(Span::new(span.start + previous_end, span.end));
1139                }
1140                // Defensive: a non-matching separator must not erase the span.
1141                if spans.is_empty() {
1142                    spans.push(span);
1143                }
1144                spans
1145            }
1146        }
1147    }
1148
1149    /// Merges adjacent spans with overlap, mirroring Python's `_merge_splits`.
1150    fn merge_spans(&self, text: &str, splits: &[Span], separator: &str) -> Vec<RawChunk> {
1151        if splits.is_empty() {
1152            return Vec::new();
1153        }
1154
1155        let separator_len = self.measure(separator);
1156        let mut docs = Vec::new();
1157        let mut current_start = 0usize;
1158        let mut total = 0usize;
1159
1160        for (index, split) in splits.iter().enumerate() {
1161            let split_len = self.measure(split.slice(text));
1162            let current_len = index - current_start;
1163            let separator_before = if current_len > 0 { separator_len } else { 0 };
1164
1165            if total
1166                .saturating_add(split_len)
1167                .saturating_add(separator_before)
1168                > self.chunk_size
1169                && current_len > 0
1170            {
1171                if let Some(chunk) =
1172                    self.make_merged_chunk(text, &splits[current_start..index], separator)
1173                {
1174                    docs.push(chunk);
1175                }
1176
1177                // Trim from the front until the overlap constraint is met.
1178                while current_start < index {
1179                    let current_len = index - current_start;
1180                    let need_trim = total > self.chunk_overlap
1181                        || (total
1182                            .saturating_add(split_len)
1183                            .saturating_add(separator_len)
1184                            > self.chunk_size
1185                            && total > 0);
1186                    if !need_trim {
1187                        break;
1188                    }
1189
1190                    let first_len = self.measure(splits[current_start].slice(text));
1191                    let separators_in_total = if current_len > 1 { separator_len } else { 0 };
1192                    total = total.saturating_sub(first_len.saturating_add(separators_in_total));
1193                    current_start += 1;
1194                }
1195            }
1196
1197            let current_len = index - current_start;
1198            total = total.saturating_add(split_len);
1199            if current_len > 0 {
1200                total = total.saturating_add(separator_len);
1201            }
1202        }
1203
1204        if current_start < splits.len() {
1205            if let Some(chunk) = self.make_merged_chunk(text, &splits[current_start..], separator) {
1206                docs.push(chunk);
1207            }
1208        }
1209
1210        docs
1211    }
1212
1213    /// Builds a chunk from adjacent spans, preserving source offsets.
1214    fn make_merged_chunk(&self, text: &str, spans: &[Span], separator: &str) -> Option<RawChunk> {
1215        let first = spans.first()?;
1216        let last = spans.last()?;
1217        let span_start = first.start;
1218        let span_end = last.end;
1219
1220        let mut content = if self.keep_separator.is_keep() {
1221            // Kept separators make the spans contiguous in the source.
1222            text[span_start..span_end].to_owned()
1223        } else {
1224            let mut content = String::with_capacity(span_end - span_start);
1225            for (index, span) in spans.iter().enumerate() {
1226                if index > 0 {
1227                    content.push_str(separator);
1228                }
1229                content.push_str(span.slice(text));
1230            }
1231            content
1232        };
1233
1234        let (start_index, end_index) = if self.strip_whitespace {
1235            let trimmed = content.trim();
1236            if trimmed.is_empty() {
1237                return None;
1238            }
1239
1240            let leading = content.len() - content.trim_start().len();
1241            let trailing = content.len() - content.trim_end().len();
1242
1243            let indices = if !self.add_start_index {
1244                (span_start, span_end)
1245            } else if self.keep_separator.is_keep() {
1246                // The content is an exact source slice.
1247                (span_start + leading, span_end - trailing)
1248            } else {
1249                // Separator normalization can change byte counts; locate the
1250                // trimmed boundaries in the source instead of doing arithmetic
1251                // on the normalized content.
1252                self.trimmed_source_bounds(text, span_start, span_end)
1253            };
1254
1255            content.drain(..leading);
1256            content.truncate(content.len() - trailing);
1257            indices
1258        } else {
1259            if content.is_empty() {
1260                return None;
1261            }
1262            (span_start, span_end)
1263        };
1264
1265        Some(RawChunk {
1266            content,
1267            start: start_index,
1268            end: end_index,
1269        })
1270    }
1271
1272    /// Returns the byte range of the first and last non-whitespace characters
1273    /// in `text[start..end]`.
1274    fn trimmed_source_bounds(&self, text: &str, start: usize, end: usize) -> (usize, usize) {
1275        let slice = &text[start..end];
1276        let first = slice
1277            .char_indices()
1278            .find(|(_, character)| !character.is_whitespace());
1279        let last = slice
1280            .char_indices()
1281            .rev()
1282            .find(|(_, character)| !character.is_whitespace());
1283
1284        match (first, last) {
1285            (Some((first_index, _)), Some((last_index, last_character))) => (
1286                start + first_index,
1287                start + last_index + last_character.len_utf8(),
1288            ),
1289            // The joined content can contain non-whitespace taken from a regex
1290            // separator pattern even when every source byte in the range is
1291            // whitespace. Report the whole source range rather than an empty
1292            // range for non-empty content.
1293            _ => (start, end),
1294        }
1295    }
1296
1297    fn direct_chunk(&self, text: &str, span: Span) -> RawChunk {
1298        RawChunk {
1299            content: span.slice(text).to_owned(),
1300            start: span.start,
1301            end: span.end,
1302        }
1303    }
1304}
1305
1306/// Creates a Markdown-configured splitter.
1307///
1308/// Requires the `language` feature.
1309///
1310/// # Errors
1311///
1312/// Returns [`SplitterError::InvalidRegex`] if a preset pattern cannot be
1313/// compiled.
1314#[cfg(feature = "language")]
1315pub fn markdown_text_splitter() -> Result<RecursiveCharacterTextSplitter, SplitterError> {
1316    RecursiveCharacterTextSplitter::new().with_language(Language::Markdown)
1317}
1318
1319/// A separator used internally by the splitter.
1320#[derive(Clone, Debug)]
1321struct Separator {
1322    pattern: Cow<'static, str>,
1323    #[cfg(feature = "regex")]
1324    regex: Option<regex::Regex>,
1325}
1326
1327impl Separator {
1328    fn literal(pattern: impl Into<Cow<'static, str>>) -> Self {
1329        #[cfg(feature = "regex")]
1330        {
1331            Self {
1332                pattern: pattern.into(),
1333                regex: None,
1334            }
1335        }
1336        #[cfg(not(feature = "regex"))]
1337        {
1338            Self {
1339                pattern: pattern.into(),
1340            }
1341        }
1342    }
1343
1344    #[cfg(feature = "regex")]
1345    fn regex(pattern: Cow<'static, str>) -> Result<Self, SplitterError> {
1346        if pattern.is_empty() {
1347            return Ok(Self::literal(pattern));
1348        }
1349
1350        let regex = regex::Regex::new(&pattern).map_err(|error| SplitterError::InvalidRegex {
1351            pattern: pattern.to_string(),
1352            message: error.to_string(),
1353        })?;
1354
1355        if regex.is_match("") {
1356            return Err(SplitterError::InvalidRegex {
1357                pattern: pattern.to_string(),
1358                message: "the pattern matches the empty string, which is not a valid split point"
1359                    .to_owned(),
1360            });
1361        }
1362
1363        Ok(Self {
1364            pattern,
1365            regex: Some(regex),
1366        })
1367    }
1368
1369    fn is_empty(&self) -> bool {
1370        self.pattern.is_empty()
1371    }
1372
1373    fn is_match(&self, haystack: &str) -> bool {
1374        #[cfg(feature = "regex")]
1375        if let Some(regex) = &self.regex {
1376            return regex.is_match(haystack);
1377        }
1378
1379        haystack.contains(self.pattern.as_ref())
1380    }
1381
1382    fn for_each_match(&self, haystack: &str, mut callback: impl FnMut(usize, usize)) {
1383        #[cfg(feature = "regex")]
1384        if let Some(regex) = &self.regex {
1385            for matched in regex.find_iter(haystack) {
1386                if matched.start() < matched.end() {
1387                    callback(matched.start(), matched.end());
1388                }
1389            }
1390            return;
1391        }
1392
1393        let pattern = self.pattern.as_ref();
1394        if pattern.is_empty() {
1395            return;
1396        }
1397        for (index, _) in haystack.match_indices(pattern) {
1398            callback(index, index + pattern.len());
1399        }
1400    }
1401}
1402
1403const CHAR_FALLBACK: Separator = Separator {
1404    pattern: Cow::Borrowed(""),
1405    #[cfg(feature = "regex")]
1406    regex: None,
1407};
1408
1409fn normalize_separators(mut separators: Vec<Separator>) -> Vec<Separator> {
1410    match separators.iter().position(Separator::is_empty) {
1411        // Python stops at the first empty separator and never considers later
1412        // entries, so discard them. Keeping them would make the defensive
1413        // separator fallback select a non-matching trailing separator and
1414        // silently drop text.
1415        Some(index) => separators.truncate(index + 1),
1416        None => separators.push(Separator::literal("")),
1417    }
1418    separators
1419}
1420
1421/// A byte range in the source text.
1422#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1423struct Span {
1424    start: usize,
1425    end: usize,
1426}
1427
1428impl Span {
1429    const fn new(start: usize, end: usize) -> Self {
1430        Self { start, end }
1431    }
1432
1433    fn whole(text: &str) -> Self {
1434        Self::new(0, text.len())
1435    }
1436
1437    fn len(self) -> usize {
1438        self.end - self.start
1439    }
1440
1441    fn is_empty(self) -> bool {
1442        self.start >= self.end
1443    }
1444
1445    fn slice(self, text: &str) -> &str {
1446        &text[self.start..self.end]
1447    }
1448}
1449
1450/// An internal chunk that always carries its source range.
1451#[derive(Debug)]
1452struct RawChunk {
1453    content: String,
1454    start: usize,
1455    end: usize,
1456}
1457
1458impl RawChunk {
1459    fn into_chunk(self, include_indices: bool) -> Chunk {
1460        Chunk {
1461            page_content: self.content,
1462            start_index: include_indices.then_some(self.start),
1463            end_index: include_indices.then_some(self.end),
1464        }
1465    }
1466}
1467
1468#[cfg(feature = "rayon")]
1469enum ParallelSegment {
1470    Ready(Vec<RawChunk>),
1471    Deferred,
1472}
1473
1474/// Language presets for [`RecursiveCharacterTextSplitter::with_language`].
1475///
1476/// The separator lists follow Python LangChain's
1477/// `RecursiveCharacterTextSplitter.from_language`. Patterns use the Rust
1478/// `regex` crate syntax. Lookahead-based presets (Visual Basic 6) are
1479/// approximated because the `regex` crate intentionally does not support
1480/// lookaround.
1481#[cfg(feature = "language")]
1482#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1483#[non_exhaustive]
1484pub enum Language {
1485    /// C.
1486    C,
1487    /// C++.
1488    Cpp,
1489    /// Go.
1490    Go,
1491    /// Java.
1492    Java,
1493    /// Kotlin.
1494    Kotlin,
1495    /// JavaScript.
1496    JavaScript,
1497    /// TypeScript.
1498    TypeScript,
1499    /// PHP.
1500    Php,
1501    /// Protocol Buffers.
1502    Proto,
1503    /// Python.
1504    Python,
1505    /// reStructuredText.
1506    Rst,
1507    /// Ruby.
1508    Ruby,
1509    /// Elixir.
1510    Elixir,
1511    /// Rust.
1512    Rust,
1513    /// Scala.
1514    Scala,
1515    /// Swift.
1516    Swift,
1517    /// Markdown.
1518    Markdown,
1519    /// LaTeX.
1520    Latex,
1521    /// HTML.
1522    Html,
1523    /// Solidity.
1524    Sol,
1525    /// C#.
1526    CSharp,
1527    /// COBOL.
1528    Cobol,
1529    /// Lua.
1530    Lua,
1531    /// Perl.
1532    Perl,
1533    /// Haskell.
1534    Haskell,
1535    /// PowerShell.
1536    PowerShell,
1537    /// Visual Basic 6.
1538    VisualBasic6,
1539}
1540
1541#[cfg(feature = "language")]
1542impl Language {
1543    /// Returns the regex separators for this language.
1544    pub const fn separators(self) -> &'static [&'static str] {
1545        match self {
1546            Self::C | Self::Cpp => &[
1547                r"\nclass ",
1548                r"\nvoid ",
1549                r"\nint ",
1550                r"\nfloat ",
1551                r"\ndouble ",
1552                r"\nif ",
1553                r"\nfor ",
1554                r"\nwhile ",
1555                r"\nswitch ",
1556                r"\ncase ",
1557                r"\n\n",
1558                r"\n",
1559                " ",
1560                "",
1561            ],
1562            Self::Go => &[
1563                r"\nfunc ",
1564                r"\nvar ",
1565                r"\nconst ",
1566                r"\ntype ",
1567                r"\nif ",
1568                r"\nfor ",
1569                r"\nswitch ",
1570                r"\ncase ",
1571                r"\n\n",
1572                r"\n",
1573                " ",
1574                "",
1575            ],
1576            Self::Java => &[
1577                r"\nclass ",
1578                r"\npublic ",
1579                r"\nprotected ",
1580                r"\nprivate ",
1581                r"\nstatic ",
1582                r"\nif ",
1583                r"\nfor ",
1584                r"\nwhile ",
1585                r"\nswitch ",
1586                r"\ncase ",
1587                r"\n\n",
1588                r"\n",
1589                " ",
1590                "",
1591            ],
1592            Self::Kotlin => &[
1593                r"\nclass ",
1594                r"\npublic ",
1595                r"\nprotected ",
1596                r"\nprivate ",
1597                r"\ninternal ",
1598                r"\ncompanion ",
1599                r"\nfun ",
1600                r"\nval ",
1601                r"\nvar ",
1602                r"\nif ",
1603                r"\nfor ",
1604                r"\nwhile ",
1605                r"\nwhen ",
1606                r"\ncase ",
1607                r"\nelse ",
1608                r"\n\n",
1609                r"\n",
1610                " ",
1611                "",
1612            ],
1613            Self::JavaScript => &[
1614                r"\nfunction ",
1615                r"\nconst ",
1616                r"\nlet ",
1617                r"\nvar ",
1618                r"\nclass ",
1619                r"\nif ",
1620                r"\nfor ",
1621                r"\nwhile ",
1622                r"\nswitch ",
1623                r"\ncase ",
1624                r"\ndefault ",
1625                r"\n\n",
1626                r"\n",
1627                " ",
1628                "",
1629            ],
1630            Self::TypeScript => &[
1631                r"\nenum ",
1632                r"\ninterface ",
1633                r"\nnamespace ",
1634                r"\ntype ",
1635                r"\nclass ",
1636                r"\nfunction ",
1637                r"\nconst ",
1638                r"\nlet ",
1639                r"\nvar ",
1640                r"\nif ",
1641                r"\nfor ",
1642                r"\nwhile ",
1643                r"\nswitch ",
1644                r"\ncase ",
1645                r"\ndefault ",
1646                r"\n\n",
1647                r"\n",
1648                " ",
1649                "",
1650            ],
1651            Self::Php => &[
1652                r"\nfunction ",
1653                r"\nclass ",
1654                r"\nif ",
1655                r"\nforeach ",
1656                r"\nwhile ",
1657                r"\ndo ",
1658                r"\nswitch ",
1659                r"\ncase ",
1660                r"\n\n",
1661                r"\n",
1662                " ",
1663                "",
1664            ],
1665            Self::Proto => &[
1666                r"\nmessage ",
1667                r"\nservice ",
1668                r"\nenum ",
1669                r"\noption ",
1670                r"\nimport ",
1671                r"\nsyntax ",
1672                r"\n\n",
1673                r"\n",
1674                " ",
1675                "",
1676            ],
1677            Self::Python => &[r"\nclass ", r"\ndef ", r"\n\tdef ", r"\n\n", r"\n", " ", ""],
1678            Self::Rst => &[
1679                r"\n=+\n",
1680                r"\n-+\n",
1681                r"\n\*+\n",
1682                r"\n\n\.\. .*\n\n",
1683                r"\n\n",
1684                r"\n",
1685                " ",
1686                "",
1687            ],
1688            Self::Ruby => &[
1689                r"\ndef ",
1690                r"\nclass ",
1691                r"\nif ",
1692                r"\nunless ",
1693                r"\nwhile ",
1694                r"\nfor ",
1695                r"\ndo ",
1696                r"\nbegin ",
1697                r"\nrescue ",
1698                r"\n\n",
1699                r"\n",
1700                " ",
1701                "",
1702            ],
1703            Self::Elixir => &[
1704                r"\ndef ",
1705                r"\ndefp ",
1706                r"\ndefmodule ",
1707                r"\ndefprotocol ",
1708                r"\ndefmacro ",
1709                r"\ndefmacrop ",
1710                r"\nif ",
1711                r"\nunless ",
1712                r"\nwhile ",
1713                r"\ncase ",
1714                r"\ncond ",
1715                r"\nwith ",
1716                r"\nfor ",
1717                r"\ndo ",
1718                r"\n\n",
1719                r"\n",
1720                " ",
1721                "",
1722            ],
1723            Self::Rust => &[
1724                r"\nfn ",
1725                r"\nconst ",
1726                r"\nlet ",
1727                r"\nif ",
1728                r"\nwhile ",
1729                r"\nfor ",
1730                r"\nloop ",
1731                r"\nmatch ",
1732                r"\n\n",
1733                r"\n",
1734                " ",
1735                "",
1736            ],
1737            Self::Scala => &[
1738                r"\nclass ",
1739                r"\nobject ",
1740                r"\ndef ",
1741                r"\nval ",
1742                r"\nvar ",
1743                r"\nif ",
1744                r"\nfor ",
1745                r"\nwhile ",
1746                r"\nmatch ",
1747                r"\ncase ",
1748                r"\n\n",
1749                r"\n",
1750                " ",
1751                "",
1752            ],
1753            Self::Swift => &[
1754                r"\nfunc ",
1755                r"\nclass ",
1756                r"\nstruct ",
1757                r"\nenum ",
1758                r"\nif ",
1759                r"\nfor ",
1760                r"\nwhile ",
1761                r"\ndo ",
1762                r"\nswitch ",
1763                r"\ncase ",
1764                r"\n\n",
1765                r"\n",
1766                " ",
1767                "",
1768            ],
1769            Self::Markdown => &[
1770                r"\n#{1,6} ",
1771                "```\n",
1772                r"\n\*\*\*+\n",
1773                r"\n---+\n",
1774                r"\n___+\n",
1775                r"\n\n",
1776                r"\n",
1777                " ",
1778                "",
1779            ],
1780            Self::Latex => &[
1781                r"\n\\chapter\{",
1782                r"\n\\section\{",
1783                r"\n\\subsection\{",
1784                r"\n\\subsubsection\{",
1785                r"\n\\begin\{enumerate\}",
1786                r"\n\\begin\{itemize\}",
1787                r"\n\\begin\{description\}",
1788                r"\n\\begin\{list\}",
1789                r"\n\\begin\{quote\}",
1790                r"\n\\begin\{quotation\}",
1791                r"\n\\begin\{verse\}",
1792                r"\n\\begin\{verbatim\}",
1793                r"\n\\begin\{align\}",
1794                r"\$\$",
1795                r"\$",
1796                " ",
1797                "",
1798            ],
1799            Self::Html => &[
1800                "<body", "<div", "<p", "<br", "<li", "<h1", "<h2", "<h3", "<h4", "<h5", "<h6",
1801                "<span", "<table", "<tr", "<td", "<th", "<ul", "<ol", "<header", "<footer", "<nav",
1802                "<head", "<style", "<script", "<meta", "<title", "",
1803            ],
1804            Self::Sol => &[
1805                r"\npragma ",
1806                r"\nusing ",
1807                r"\ncontract ",
1808                r"\ninterface ",
1809                r"\nlibrary ",
1810                r"\nconstructor ",
1811                r"\ntype ",
1812                r"\nfunction ",
1813                r"\nevent ",
1814                r"\nmodifier ",
1815                r"\nerror ",
1816                r"\nstruct ",
1817                r"\nenum ",
1818                r"\nif ",
1819                r"\nfor ",
1820                r"\nwhile ",
1821                r"\ndo while ",
1822                r"\nassembly ",
1823                r"\n\n",
1824                r"\n",
1825                " ",
1826                "",
1827            ],
1828            Self::CSharp => &[
1829                r"\ninterface ",
1830                r"\nenum ",
1831                r"\nimplements ",
1832                r"\ndelegate ",
1833                r"\nevent ",
1834                r"\nclass ",
1835                r"\nabstract ",
1836                r"\npublic ",
1837                r"\nprotected ",
1838                r"\nprivate ",
1839                r"\nstatic ",
1840                r"\nreturn ",
1841                r"\nif ",
1842                r"\ncontinue ",
1843                r"\nfor ",
1844                r"\nforeach ",
1845                r"\nwhile ",
1846                r"\nswitch ",
1847                r"\nbreak ",
1848                r"\ncase ",
1849                r"\nelse ",
1850                r"\ntry ",
1851                r"\nthrow ",
1852                r"\nfinally ",
1853                r"\ncatch ",
1854                r"\n\n",
1855                r"\n",
1856                " ",
1857                "",
1858            ],
1859            Self::Cobol => &[
1860                r"\nIDENTIFICATION DIVISION.",
1861                r"\nENVIRONMENT DIVISION.",
1862                r"\nDATA DIVISION.",
1863                r"\nPROCEDURE DIVISION.",
1864                r"\nWORKING-STORAGE SECTION.",
1865                r"\nLINKAGE SECTION.",
1866                r"\nFILE SECTION.",
1867                r"\nINPUT-OUTPUT SECTION.",
1868                r"\nOPEN ",
1869                r"\nCLOSE ",
1870                r"\nREAD ",
1871                r"\nWRITE ",
1872                r"\nIF ",
1873                r"\nELSE ",
1874                r"\nMOVE ",
1875                r"\nPERFORM ",
1876                r"\nUNTIL ",
1877                r"\nVARYING ",
1878                r"\nACCEPT ",
1879                r"\nDISPLAY ",
1880                r"\nSTOP RUN.",
1881                r"\n",
1882                " ",
1883                "",
1884            ],
1885            Self::Lua => &[
1886                r"\nlocal ",
1887                r"\nfunction ",
1888                r"\nif ",
1889                r"\nfor ",
1890                r"\nwhile ",
1891                r"\nrepeat ",
1892                r"\n\n",
1893                r"\n",
1894                " ",
1895                "",
1896            ],
1897            Self::Perl => &[
1898                r"\nsub ",
1899                r"\npackage ",
1900                r"\nuse ",
1901                r"\nif ",
1902                r"\nwhile ",
1903                r"\nfor ",
1904                r"\nforeach ",
1905                r"\n\n",
1906                r"\n",
1907                " ",
1908                "",
1909            ],
1910            Self::Haskell => &[
1911                r"\nmain :: ",
1912                r"\nmain = ",
1913                r"\nlet ",
1914                r"\nin ",
1915                r"\ndo ",
1916                r"\nwhere ",
1917                r"\n:: ",
1918                r"\n= ",
1919                r"\ndata ",
1920                r"\nnewtype ",
1921                r"\ntype ",
1922                r"\nmodule ",
1923                r"\nimport ",
1924                r"\nqualified ",
1925                r"\nimport qualified ",
1926                r"\nclass ",
1927                r"\ninstance ",
1928                r"\ncase ",
1929                r"\n\| ",
1930                r"\n= \{",
1931                r"\n, ",
1932                r"\n\n",
1933                r"\n",
1934                " ",
1935                "",
1936            ],
1937            Self::PowerShell => &[
1938                r"\nfunction ",
1939                r"\nparam ",
1940                r"\nif ",
1941                r"\nforeach ",
1942                r"\nfor ",
1943                r"\nwhile ",
1944                r"\nswitch ",
1945                r"\nclass ",
1946                r"\ntry ",
1947                r"\ncatch ",
1948                r"\nfinally ",
1949                r"\n\n",
1950                r"\n",
1951                " ",
1952                "",
1953            ],
1954            Self::VisualBasic6 => &[
1955                r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Sub\s+",
1956                r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Function\s+",
1957                r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Property\s+(?:Get|Let|Set)\s+",
1958                r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Type\s+",
1959                r"\n(?:Public\s+|Private\s+|Friend\s+|Global\s+|Static\s+)?Enum\s+",
1960                r"\nIf\s+",
1961                r"\nElseIf\s+",
1962                r"\nElse\s+",
1963                r"\nSelect\s+Case\s+",
1964                r"\nCase\s+",
1965                r"\nFor\s+",
1966                r"\nDo\s+",
1967                r"\nWhile\s+",
1968                r"\nWith\s+",
1969                r"\n\n",
1970                r"\n",
1971                " ",
1972                "",
1973            ],
1974        }
1975    }
1976}
1977
1978#[cfg(feature = "language")]
1979impl fmt::Display for Language {
1980    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1981        let name = match self {
1982            Self::C => "C",
1983            Self::Cpp => "C++",
1984            Self::Go => "Go",
1985            Self::Java => "Java",
1986            Self::Kotlin => "Kotlin",
1987            Self::JavaScript => "JavaScript",
1988            Self::TypeScript => "TypeScript",
1989            Self::Php => "PHP",
1990            Self::Proto => "Protocol Buffers",
1991            Self::Python => "Python",
1992            Self::Rst => "reStructuredText",
1993            Self::Ruby => "Ruby",
1994            Self::Elixir => "Elixir",
1995            Self::Rust => "Rust",
1996            Self::Scala => "Scala",
1997            Self::Swift => "Swift",
1998            Self::Markdown => "Markdown",
1999            Self::Latex => "LaTeX",
2000            Self::Html => "HTML",
2001            Self::Sol => "Solidity",
2002            Self::CSharp => "C#",
2003            Self::Cobol => "COBOL",
2004            Self::Lua => "Lua",
2005            Self::Perl => "Perl",
2006            Self::Haskell => "Haskell",
2007            Self::PowerShell => "PowerShell",
2008            Self::VisualBasic6 => "Visual Basic 6",
2009        };
2010        f.write_str(name)
2011    }
2012}