#[cfg(feature = "python")]
use pyo3::prelude::*;
mod backend;
mod block;
mod chunk_types;
mod chunker;
mod errors;
mod file;
mod markdown;
mod normalization;
mod parallel;
#[cfg(feature = "python")]
mod python;
mod split_engine;
pub use backend::{Backend, Tokenizer};
pub use block::BlockKind;
pub use block::{Block, Headers};
pub use chunk_types::{context_prefix, Chunk, ChunkConfig};
pub use chunker::SmartChunker;
pub use chunker::{smart_chunk, smart_chunk_mapped};
pub use errors::ChipError;
pub use file::{read_text_file, FileChunker, FLAT_FILE_MAX_BYTES};
pub use markdown::list_parser::{parse_list_items, ListItem, ListMarker};
pub use markdown::BlockParser;
pub use normalization::{NormBase, NormMap, Normalization, OrigSpan};
#[cfg(feature = "python")]
pub use python::{TextChipper, TextChunk, TextStreamIter};
pub use split_engine::{split_by_tokens, CharCursor, Origin};
#[cfg(feature = "mimalloc")]
#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
#[cfg(feature = "python")]
#[pymodule]
fn text_chipper(m: &Bound<'_, PyModule>) -> PyResult<()> {
python::register(m)
}
#[cfg(test)]
#[path = "tests/unit.rs"]
mod tests;
#[cfg(test)]
#[path = "tests/merge_margin.rs"]
mod merge_margin_tests;
#[cfg(test)]
#[cfg_attr(not(feature = "diagnostics"), allow(dead_code, unused_imports))]
#[path = "tests/oracle_list.rs"]
mod oracle_tests;
#[cfg(test)]
#[cfg_attr(not(feature = "diagnostics"), allow(dead_code, unused_imports))]
#[path = "tests/oracle_block.rs"]
mod oracle_block_tests;
#[cfg(test)]
#[cfg_attr(not(feature = "diagnostics"), allow(dead_code, unused_imports))]
#[path = "tests/parallel.rs"]
mod parallel_tests;
#[cfg(test)]
#[cfg_attr(not(feature = "diagnostics"), allow(dead_code, unused_imports))]
#[path = "tests/robustness.rs"]
mod robustness_tests;
#[cfg(all(test, feature = "diagnostics"))]
#[path = "tests/release_suite.rs"]
mod release_suite;