Struct TextProcessor 

pub struct TextProcessor { /* private fields */ }

Text processing utilities for chunking and preprocessing

Implementations

impl TextProcessor

pub fn new(chunk_size: usize, chunk_overlap: usize) -> Result<Self>

Create a new text processor

pub fn with_parallel_processing( chunk_size: usize, chunk_overlap: usize, parallel_processor: ParallelProcessor, ) -> Result<Self>

Create a new text processor with parallel processing support

pub fn chunk_text_hierarchical( &self, document: &Document, ) -> Result<Vec<TextChunk>>

Split text into chunks with overlap using hierarchical boundary preservation

pub fn chunk_text(&self, document: &Document) -> Result<Vec<TextChunk>>

Split text into chunks with overlap (legacy method)

pub fn chunk_text_with_enrichment( &self, document: &Document, enricher: &mut ChunkEnricher, ) -> Result<Vec<TextChunk>>

Chunk text and enrich with semantic metadata

pub fn chunk_text_hierarchical_with_enrichment( &self, document: &Document, enricher: &mut ChunkEnricher, ) -> Result<Vec<TextChunk>>

Chunk text hierarchically and enrich with semantic metadata

pub fn create_default_enricher(document: &Document) -> ChunkEnricher

Create a default enricher for document processing

pub fn chunk_and_enrich(&self, document: &Document) -> Result<Vec<TextChunk>>

Convenience method: chunk and enrich with auto-detected format

pub fn chunk_hierarchical_and_enrich( &self, document: &Document, ) -> Result<Vec<TextChunk>>

Convenience method: chunk hierarchically and enrich with auto-detected format

pub fn chunk_with_strategy( &self, document: &Document, strategy: &dyn ChunkingStrategy, ) -> Result<Vec<TextChunk>>

Chunk text using any strategy that implements ChunkingStrategy trait

This method provides a flexible way to use different chunking approaches while maintaining the same interface.

Arguments

• document - The document to chunk
• strategy - Any type implementing ChunkingStrategy

Returns

A vector of TextChunk objects

Examples

use graphrag_core::text::{TextProcessor, HierarchicalChunkingStrategy};

let processor = TextProcessor::new(1000, 100)?;
let strategy = HierarchicalChunkingStrategy::new(500, 50, document.id.clone());
let chunks = processor.chunk_with_strategy(&document, &strategy)?;

pub fn clean_text(&self, text: &str) -> String

Clean and normalize text

pub fn extract_sentences(&self, text: &str) -> Vec<String>

Extract sentences from text

pub fn word_count(&self, text: &str) -> usize

Count words in text

pub fn batch_chunk_documents( &self, documents: Vec<Document>, ) -> Result<Vec<Vec<TextChunk>>>

Process multiple documents in parallel

pub fn batch_extract_keywords( &self, texts: &[&str], max_keywords: usize, ) -> Vec<Vec<String>>

Parallel extraction of keywords from multiple texts

pub fn batch_extract_sentences(&self, texts: &[&str]) -> Vec<Vec<String>>

Parallel sentence extraction from multiple texts

pub fn batch_clean_text(&self, texts: &[&str]) -> Vec<String>

Parallel text cleaning for multiple texts

pub fn extract_keywords(&self, text: &str, max_keywords: usize) -> Vec<String>

Extract keywords using simple frequency analysis

pub fn get_performance_stats(&self) -> (usize, Duration)

Get performance statistics

pub fn average_processing_time(&self) -> Duration

Get average processing time per operation

pub fn reset_performance_stats(&mut self)

Reset performance monitoring statistics

pub fn get_parallel_stats(&self) -> Option<ParallelStatistics>

Get parallel processing statistics if available

Trait Implementations

impl Debug for TextProcessor

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.