scirs2_text/
lib.rs

1#![allow(clippy::manual_strip)]
2#![allow(clippy::needless_range_loop)]
3#![allow(clippy::if_same_then_else)]
4#![allow(clippy::cloned_ref_to_slice_refs)]
5#![allow(dead_code)]
6//! # SciRS2 Text - Natural Language Processing
7//!
8//! **scirs2-text** provides comprehensive text processing and NLP capabilities,
9//! offering tokenization, TF-IDF vectorization, word embeddings, sentiment analysis,
10//! topic modeling, and text classification with SIMD acceleration and parallel processing.
11//!
12//! ## 🎯 Key Features
13//!
14//! - **Tokenization**: Word, sentence, N-gram, BPE, regex tokenizers
15//! - **Vectorization**: TF-IDF, count vectorizers, word embeddings
16//! - **Text Processing**: Stemming, lemmatization, normalization, stopword removal
17//! - **Embeddings**: Word2Vec (Skip-gram, CBOW), GloVe loading
18//! - **Similarity**: Cosine, Jaccard, Levenshtein, phonetic algorithms
19//! - **NLP**: Sentiment analysis, topic modeling (LDA), text classification
20//! - **Performance**: SIMD operations, parallel processing, sparse matrices
21//!
22//! ## 📦 Module Overview
23//!
24//! | SciRS2 Module | Python Equivalent | Description |
25//! |---------------|-------------------|-------------|
26//! | `tokenize` | `nltk.tokenize` | Text tokenization utilities |
27//! | `vectorize` | `sklearn.feature_extraction.text.TfidfVectorizer` | TF-IDF and count vectorization |
28//! | `embeddings` | `gensim.models.Word2Vec` | Word embeddings (Word2Vec) |
29//! | `sentiment` | `nltk.sentiment` | Sentiment analysis |
30//! | `topic_modeling` | `sklearn.decomposition.LatentDirichletAllocation` | Topic modeling (LDA) |
31//! | `stemming` | `nltk.stem` | Stemming and lemmatization |
32//!
33//! ## 🚀 Quick Start
34//!
35//! ```toml
36//! [dependencies]
37//! scirs2-text = "0.6.3"
38//! ```
39//!
40//! ```rust,no_run
41//! use scirs2_text::{tokenize::WordTokenizer, vectorize::TfidfVectorizer, Tokenizer, Vectorizer};
42//!
43//! // Tokenization
44//! let tokenizer = WordTokenizer::default();
45//! let tokens = tokenizer.tokenize("Hello, world!").unwrap();
46//!
47//! // TF-IDF vectorization
48//! let docs = vec!["Hello world", "Good morning world"];
49//! let mut vectorizer = TfidfVectorizer::new(false, true, Some("l2".to_string()));
50//! let matrix = vectorizer.fit_transform(&docs).unwrap();
51//! ```
52//!
53//! ## 🔒 Version: 0.6.3 (July 22, 2026)
54//!
55//! ## Quick Start
56//!
57//! ```rust
58//! use scirs2_text::{
59//!     tokenize::WordTokenizer,
60//!     vectorize::TfidfVectorizer,
61//!     sentiment::LexiconSentimentAnalyzer,
62//!     Tokenizer, Vectorizer
63//! };
64//!
65//! // Basic tokenization
66//! let tokenizer = WordTokenizer::default();
67//! let tokens = tokenizer.tokenize("Hello, world! This is a test.").unwrap();
68//!
69//! // TF-IDF vectorization
70//! let documents = vec![
71//!     "The quick brown fox jumps over the lazy dog",
72//!     "A quick brown dog outpaces a quick fox",
73//!     "The lazy dog sleeps all day"
74//! ];
75//! let mut vectorizer = TfidfVectorizer::new(false, true, Some("l2".to_string()));
76//! let matrix = vectorizer.fit_transform(&documents).unwrap();
77//!
78//! // Sentiment analysis
79//! let analyzer = LexiconSentimentAnalyzer::with_basiclexicon();
80//! let sentiment = analyzer.analyze("I love this library!").unwrap();
81//! println!("Sentiment: {:?}", sentiment.sentiment);
82//! ```
83//!
84//! ## Architecture
85//!
86//! The module is organized into focused sub-modules:
87//!
88//! - [`tokenize`]: Text tokenization utilities
89//! - [`vectorize`]: Document vectorization and TF-IDF
90//! - [`embeddings`]: Word embedding training and utilities
91//! - [`sentiment`]: Sentiment analysis tools
92//! - [`topic_modeling`]: Topic modeling with LDA
93//! - [`string_metrics`]: String similarity and distance metrics
94//! - [`preprocess`]: Text cleaning and normalization
95//! - [`stemming`]: Stemming and lemmatization
96//! - [`parallel`]: Parallel processing utilities
97//! - [`simd_ops`]: SIMD-accelerated operations
98//!
99//! ## Performance
100//!
101//! SciRS2 Text is designed for high performance:
102//!
103//! - SIMD acceleration for string operations
104//! - Parallel processing for large document collections
105//! - Memory-efficient sparse matrix representations
106//! - Zero-copy string processing where possible
107//! - Optimized algorithms with complexity guarantees
108
109#![warn(missing_docs)]
110
111/// Batch tokenization with padding and attention masks.
112pub mod batch_tokenizer;
113pub mod classification;
114pub mod cleansing;
115pub mod distance;
116pub mod domain_processors;
117pub mod embeddings;
118pub mod enhanced_vectorize;
119pub mod error;
120pub mod evaluation;
121/// GPT-2 byte-level BPE tokenizer.
122pub mod gpt_bpe;
123pub mod huggingface_compat;
124pub mod information_extraction;
125pub mod language_model;
126/// Statistical language models (Unigram, Bigram, N-gram with Kneser-Ney).
127pub mod language_models;
128pub mod lemmatization;
129pub mod ml_integration;
130pub mod ml_sentiment;
131pub mod model_registry;
132pub mod multilingual;
133pub mod neural_architectures;
134pub mod parallel;
135pub mod paraphrasing;
136pub mod performance;
137pub mod pipeline;
138pub mod pos_tagging;
139pub mod preprocess;
140pub mod semantic_similarity;
141/// SentencePiece Unigram Language Model tokenizer.
142pub mod sentencepiece;
143pub mod sentiment;
144pub mod simd_ops;
145pub mod sparse;
146pub mod sparse_vectorize;
147pub mod spelling;
148pub mod stemming;
149pub mod streaming;
150pub mod string_metrics;
151pub mod summarization;
152pub mod text_coordinator;
153pub mod text_statistics;
154pub mod token_filter;
155pub mod tokenize;
156pub mod tokenizer;
157pub mod topic_coherence;
158pub mod topic_modeling;
159pub mod transformer;
160pub mod utils;
161pub mod vectorize;
162pub mod visualization;
163pub mod vocabulary;
164pub mod weighted_distance;
165
166// New text processing modules
167pub mod keyword_extraction;
168pub mod language_detection;
169pub mod named_entity_recognition;
170pub mod text_similarity;
171pub mod text_summarization;
172
173// BERT fine-tuning
174pub mod bert_finetune;
175// Cross-lingual NER and transliteration
176pub mod crosslingual;
177// Correlated topic model
178pub mod ctm;
179// Dynamic topic model
180pub mod dtm;
181// Hierarchical Dirichlet Process topic model
182pub mod hdp;
183// Sentence embeddings
184pub mod sentence_embeddings;
185// Semantic similarity with embedding-based search
186pub mod similarity;
187// Advanced tokenization algorithms (BPE, WordPiece, language-agnostic Unicode)
188pub mod tokenization;
189// Tokenizer implementations (HuggingFace, byte-level BPE, Unicode)
190pub mod tokenizers;
191// Transliteration
192pub mod transliteration;
193// Topic modelling (HDP automatic topic selection)
194pub mod topic;
195
196// Additional text processing modules
197pub mod abstractive_summary;
198pub mod advanced_classification;
199pub mod advanced_distance;
200pub mod alignment;
201pub mod bpe_tokenizer;
202pub mod coreference;
203pub mod dialog;
204pub mod discourse;
205pub mod doc_similarity;
206pub mod event_extraction;
207pub mod keywords;
208pub mod multilingual_ext;
209pub mod ner;
210pub mod pos_tagging_original;
211// The `pos_tagging_original_tests` suite is compiled and run as a child of
212// `pos_tagging_original` via the `#[cfg(test)] #[path = "pos_tagging_original_tests.rs"]`
213// include at the bottom of that module. It must NOT be declared here as a top-level
214// crate module: its `use super::*;` relies on the `pos_tagging_original` items
215// (`MorphologicalAnalyzer`, `ContextualDisambiguator`, and the private `PosTagger`
216// fields/methods) which are not re-exported at the crate root, and the crate root
217// instead re-exports the refactored `pos_tagging` versions.
218pub mod question_answering;
219pub mod regex_lite;
220pub mod segmentation;
221pub mod summarize_advanced;
222pub mod text_classification;
223pub mod text_preprocess;
224pub mod topic_model;
225
226// Re-export commonly used items
227pub use classification::{
228    cross_validate_nb, BernoulliNaiveBayes, CrossValidationResult, FeatureHasher, FoldResult,
229    MultiLabelClassifier, MultiLabelPrediction, MultinomialNaiveBayes, TextClassificationMetrics,
230    TextClassificationPipeline, TextDataset, TextFeatureSelector, TfidfCosineClassifier,
231};
232pub use cleansing::{
233    expand_contractions, normalize_currencies, normalize_numbers, normalize_ordinals,
234    normalize_percentages, normalize_unicode, normalize_whitespace, remove_accents, replace_emails,
235    replace_urls, strip_html_tags, AdvancedTextCleaner,
236};
237pub use distance::{cosine_similarity, jaccard_similarity, levenshtein_distance};
238pub use domain_processors::{
239    Domain, DomainProcessorConfig, FinancialTextProcessor, LegalTextProcessor,
240    MedicalTextProcessor, NewsTextProcessor, PatentTextProcessor, ProcessedDomainText,
241    ScientificTextProcessor, SocialMediaTextProcessor, UnifiedDomainProcessor,
242};
243pub use embeddings::{
244    embedding_cosine_similarity,
245    fasttext::{FastText, FastTextConfig},
246    glove::{CooccurrenceMatrix, GloVe, GloVeTrainer, GloVeTrainerConfig},
247    pairwise_similarity, Word2Vec, Word2VecAlgorithm, Word2VecConfig, WordEmbedding,
248};
249pub use enhanced_vectorize::{EnhancedCountVectorizer, EnhancedTfidfVectorizer};
250pub use error::{Result, TextError};
251pub use huggingface_compat::{
252    ClassificationResult, FeatureExtractionPipeline, FillMaskPipeline, FillMaskResult,
253    FormatConverter, HfConfig, HfEncodedInput, HfHub, HfModelAdapter, HfPipeline, HfTokenizer,
254    HfTokenizerConfig, QuestionAnsweringPipeline, QuestionAnsweringResult,
255    TextClassificationPipeline as HfTextClassificationPipeline, ZeroShotClassificationPipeline,
256};
257pub use information_extraction::{
258    AdvancedExtractedInformation, AdvancedExtractionPipeline, ConfidenceScorer, CoreferenceChain,
259    CoreferenceMention, CoreferenceResolver, DocumentInformationExtractor, DocumentSummary, Entity,
260    EntityCluster, EntityLinker, EntityType, Event, ExtractedInformation,
261    InformationExtractionPipeline, KeyPhraseExtractor, KnowledgeBaseEntry, LinkedEntity,
262    MentionType, PatternExtractor, Relation, RelationExtractor, RuleBasedNER,
263    StructuredDocumentInformation, TemporalExtractor, Topic,
264};
265pub use language_model::{NgramModel, SmoothingMethod};
266pub use lemmatization::{Lemmatizer, RuleBasedLemmatizer, WordNetLemmatizer};
267pub use ml_integration::{
268    BatchTextProcessor, FeatureExtractionMode, MLTextPreprocessor, TextFeatures, TextMLPipeline,
269};
270pub use ml_sentiment::{
271    ClassMetrics, EvaluationMetrics, MLSentimentAnalyzer, MLSentimentConfig, TrainingMetrics,
272};
273pub use model_registry::{
274    ModelMetadata, ModelRegistry, ModelType, PrebuiltModels, RegistrableModel,
275    SerializableModelData,
276};
277pub use multilingual::{
278    is_cjk_char, is_combining_mark, is_cyrillic, Language, LanguageDetectionResult,
279    LanguageDetector, MultilingualProcessor, ProcessedText, ScriptFamily, StopWords,
280    Transliterator as MultilingualTransliterator, UnicodeTokenizer, UnicodeTokenizerConfig,
281};
282pub use neural_architectures::{
283    ActivationFunction, AdditiveAttention, BiLSTM, CNNLSTMHybrid, Conv1D, CrossAttention, Dropout,
284    GRUCell, LSTMCell, LayerNorm as NeuralLayerNorm, MaxPool1D,
285    MultiHeadAttention as NeuralMultiHeadAttention, MultiScaleCNN, PositionwiseFeedForward,
286    ResidualBlock1D, SelfAttention, TextCNN,
287};
288pub use parallel::{
289    ParallelCorpusProcessor, ParallelTextProcessor, ParallelTokenizer, ParallelVectorizer,
290};
291pub use paraphrasing::{ParaphraseConfig, ParaphraseResult, ParaphraseStrategy, Paraphraser};
292pub use performance::{
293    AdvancedPerformanceMonitor, DetailedPerformanceReport, OptimizationRecommendation,
294    PerformanceSummary, PerformanceThresholds,
295};
296pub use pipeline::{
297    basic_pipeline, lemmatization_pipeline, ngram_pipeline, stemming_pipeline, BatchProcessor,
298    NlpPipeline, PipelineBuilder, PipelineStep,
299};
300pub use pos_tagging::{
301    PosAwareLemmatizer, PosTagResult, PosTagger, PosTaggerConfig, PosTaggingResult,
302};
303pub use preprocess::{BasicNormalizer, BasicTextCleaner, TextCleaner, TextNormalizer};
304pub use semantic_similarity::{
305    LcsSimilarity, SemanticSimilarityEnsemble, SoftCosineSimilarity, WeightedJaccard,
306    WordMoversDistance,
307};
308pub use sentiment::{
309    aggregate_sentiment, analyze_and_aggregate, AggregatedSentiment, AspectSentiment,
310    AspectSentimentAnalyzer, LexiconSentimentAnalyzer, NaiveBayesSentiment,
311    RuleBasedSentimentAnalyzer, Sentiment, SentimentLexicon, SentimentResult, SentimentRules,
312    SentimentWordCounts, VaderResult, VaderSentimentAnalyzer,
313};
314pub use simd_ops::{
315    AdvancedSIMDTextProcessor, SimdEditDistance, SimdStringOps, SimdTextAnalyzer,
316    TextProcessingResult,
317};
318pub use sparse::{CsrMatrix, DokMatrix, SparseMatrixBuilder, SparseVector};
319pub use sparse_vectorize::{
320    sparse_cosine_similarity, MemoryStats, SparseCountVectorizer, SparseTfidfVectorizer,
321};
322pub use spelling::{
323    DictionaryCorrector, DictionaryCorrectorConfig, EditOp, ErrorModel, NGramModel,
324    SpellingCorrector, StatisticalCorrector, StatisticalCorrectorConfig,
325};
326pub use stemming::{
327    LancasterStemmer, LemmatizerConfig, PorterStemmer, PosTag, RuleLemmatizer,
328    RuleLemmatizerBuilder, SimpleLemmatizer, SnowballStemmer, Stemmer,
329};
330pub use streaming::{
331    AdvancedStreamingMetrics, AdvancedStreamingProcessor, ChunkedCorpusReader, MemoryMappedCorpus,
332    ProgressTracker, StreamingTextProcessor, StreamingVectorizer,
333};
334pub use string_metrics::{
335    AlignmentResult, DamerauLevenshteinMetric, Metaphone, NeedlemanWunsch, Nysiis,
336    PhoneticAlgorithm, SmithWaterman, Soundex, StringMetric,
337};
338pub use summarization::{CentroidSummarizer, KeywordExtractor, TextRank};
339pub use text_coordinator::{
340    AdvancedBatchClassificationResult, AdvancedSemanticSimilarityResult, AdvancedTextConfig,
341    AdvancedTextCoordinator, AdvancedTextResult, AdvancedTopicModelingResult,
342};
343pub use text_statistics::{ReadabilityMetrics, TextMetrics, TextStatistics};
344pub use token_filter::{
345    CompositeFilter, CustomFilter, FrequencyFilter, LengthFilter, RegexFilter, StopwordsFilter,
346    TokenFilter,
347};
348pub use tokenize::{
349    bpe::{BpeConfig, BpeTokenizer, BpeVocabulary},
350    CharacterTokenizer, NgramTokenizer, RegexTokenizer, SentenceTokenizer, Tokenizer,
351    WhitespaceTokenizer, WordTokenizer,
352};
353pub use tokenizer::{
354    BPETokenizer, SimpleCharTokenizer, SimpleWhitespaceTokenizer, TransformerTokenizer,
355    WordPieceTokenizer,
356};
357pub use topic_coherence::{TopicCoherence, TopicDiversity};
358pub use topic_modeling::{
359    LatentDirichletAllocation, LdaBuilder, LdaConfig, LdaLearningMethod, Topic as LdaTopic,
360};
361pub use transformer::{
362    FeedForward, LayerNorm, MultiHeadAttention, PositionalEncoding, TokenEmbedding,
363    TransformerConfig, TransformerDecoder, TransformerDecoderLayer, TransformerEncoder,
364    TransformerEncoderLayer, TransformerModel,
365};
366pub use vectorize::{CountVectorizer, TfidfVectorizer, Vectorizer};
367pub use visualization::{
368    AttentionVisualizer, Color, ColorScheme, EmbeddingVisualizer, SentimentVisualizer,
369    TextAnalyticsDashboard, TopicVisualizer, VisualizationConfig, WordCloud,
370};
371pub use vocabulary::Vocabulary;
372pub use weighted_distance::{
373    DamerauLevenshteinWeights, LevenshteinWeights, WeightedDamerauLevenshtein, WeightedLevenshtein,
374    WeightedStringMetric,
375};
376
377// Re-exports for new modules
378pub use keyword_extraction::{
379    extract_keywords, Keyword, KeywordMethod, RakeKeywordExtractor, TextRankKeywordExtractor,
380    TfIdfKeywordExtractor,
381};
382pub use language_detection::{
383    detect_language, detect_language_with_strategy, DetectedLanguage, DetectionStrategy,
384    LanguageDetectionOutput,
385};
386pub use named_entity_recognition::{extract_entities, NerEntity, NerEntityType, NerPatternConfig};
387pub use text_similarity::{
388    bm25_score, char_ngram_jaccard_similarity, edit_distance_similarity, jaccard_token_similarity,
389    text_similarity, tfidf_cosine_similarity, Bm25Config, Bm25Scorer, SimilarityMethod,
390    SimilarityResult, TfIdfCosineSimilarity,
391};
392pub use text_summarization::{
393    score_position, score_textrank, score_tfidf, summarize, ScoredSentence, SummarizationMethod,
394};
395
396// HDP topic model
397pub use hdp::{HdpConfig, HdpModel, HdpResult};
398
399// New sentence encoder (USE-style, word-level)
400pub use sentence_embeddings::{
401    SentenceEncoder, SentenceEncoderConfig, SentenceEncoderPooling, SimCSELoss, SimCSETrainer,
402};
403
404// topic module re-exports (HDP with automatic topic selection)
405pub use topic::hdp::{
406    Hdp, HdpConfig as HdpAutoConfig, HdpState, HdpTopicConfig, HdpTopicModel, TopicError,
407};
408
409// tokenizers module (Unicode tokenizer from tokenizers crate kept accessible via path)
410pub use embeddings::sentence_encoder::{
411    PoolingStrategy as SentencePoolingStrategy, SemanticSimilarity as EmbeddingSearch,
412    SentenceEncoder as ProjSentenceEncoder, SimCseConfig, SimCseTrainer as ProjSimCseTrainer,
413};
414
415// Transliteration: trait + concrete transliterators
416pub use transliteration::{
417    CyrillicScheme, CyrillicTransliterator, HepburnTransliterator, PinyinStyle,
418    PinyinTransliterator, Transliterator,
419};
420
421// Neural NLP: transformer encoder, attention viz, BERT classifier, neural NER
422pub mod neural_nlp;
423pub use neural_nlp::{
424    AttentionHeatmap, AttentionVisualization, BertClassifier, BertClassifierConfig, NerTag,
425    NeuralNer, NeuralNerConfig, TransformerEncoderConfig as NeuralEncoderConfig,
426    TransformerTextEncoder,
427};
scirs2_text/lib.rs

scirs2_text/
lib.rs
