libgrammstein 0.1.0

Hybrid language model (N-gram + Embeddings) for WFST text correction
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

//! Corpus processing with streaming readers.
//!
//! This module provides traits and implementations for reading text corpora
//! in a memory-efficient streaming fashion.
//!
//! # Supported Formats
//!
//! - **Wikipedia**: XML dump format with bz2 compression (streaming)
//! - **Project Gutenberg**: Plain text files with boilerplate stripping
//! - **Plaintext**: Generic text files or directories
//!
//! # Example
//!
//! ```ignore
//! use libgrammstein::corpus::{CorpusReader, PlaintextReader};
//!
//! let reader = PlaintextReader::from_file("corpus.txt")?;
//! for sentence in reader.sentences() {
//!     println!("{}", sentence);
//! }
//! ```

mod dedup;
mod gutenberg;
mod normalizer;
mod plaintext;
mod prefetch;
mod preprocessing;
mod quality;
mod reader;
mod tokenizer;
mod wikipedia;

#[cfg(feature = "subword")]
mod subword;

pub use dedup::{DeduplicationMode, DeduplicationStats, Deduplicator, DeduplicatorBuilder};
pub use gutenberg::GutenbergReader;
pub use normalizer::Normalizer;
pub use plaintext::{LineIterator, PlaintextReader};
pub use prefetch::{PrefetchBatchIterator, PrefetchConfig, PrefetchingReader};
pub use preprocessing::{
    tokens, PreprocessingPipeline, PreprocessingPipelineBuilder, TextPreprocessor,
    TextPreprocessorBuilder, UnicodeNorm,
};
pub use quality::{
    QualityFilter, QualityFilterBuilder, QualityMetrics, QualityStats, RejectionReason,
};
pub use reader::{CorpusReader, Document};
pub use tokenizer::Tokenizer;
pub use wikipedia::{WikipediaConfig, WikipediaReader};

#[cfg(feature = "http-corpus")]
pub use wikipedia::LoadStrategy;

#[cfg(feature = "subword")]
pub use subword::{special_tokens, BpeConfig, SubwordError, SubwordTokenizer, TokenizeExt};