Struct StatisticalCorrector 

pub struct StatisticalCorrector { /* private fields */ }

Statistical spelling corrector

Implementations

impl StatisticalCorrector

pub fn new(config: StatisticalCorrectorConfig) -> Self

Create a new statistical spelling corrector with the given configuration

Examples found in repository

examples/statistical_spelling_demo.rs (line 272)

fn performance_test(
    dict_corrector: &DictionaryCorrector,
    stat_corrector: &StatisticalCorrector,
) -> Result<(), Box<dyn std::error::Error>> {
    println!("\n=== Performance Test ===\n");

    // Create test text with a mix of correct and incorrect words
    let testtext = TEXT_WITH_CONTEXT_MISSPELLINGS.repeat(10);

    // Measure dictionary _corrector performance
    let start = Instant::now();
    let _ = dict_corrector.correcttext(&testtext)?;
    let dict_time = start.elapsed();

    // Measure statistical _corrector performance
    let start = Instant::now();
    let _ = stat_corrector.correcttext(&testtext)?;
    let stat_time = start.elapsed();

    // Create a non-contextual statistical _corrector for comparison
    let non_context_config = StatisticalCorrectorConfig {
        use_context: false,
        ..Default::default()
    };
    let mut non_context_corrector = StatisticalCorrector::new(non_context_config);

    // Add training data to ensure fair comparison
    train_language_model(&mut non_context_corrector);
    add_example_words(&mut non_context_corrector);

    // Measure non-contextual statistical _corrector performance
    let start = Instant::now();
    let _ = non_context_corrector.correcttext(&testtext)?;
    let non_context_time = start.elapsed();

    println!(
        "Performance comparison on text with {} characters:",
        testtext.len()
    );
    println!("  - Dictionary corrector: {dict_time:?}");
    println!("  - Statistical _corrector (without context): {non_context_time:?}");
    println!("  - Statistical _corrector (with context): {stat_time:?}");

    Ok(())
}

// Demonstrate different configurations for statistical correction
#[allow(dead_code)]
fn configuration_demo() -> Result<(), Box<dyn std::error::Error>> {
    println!("\n=== Configuration Options Demo ===\n");

    // Create configurations with different settings
    let configs = [
        ("Default", StatisticalCorrectorConfig::default()),
        (
            "Conservative (max_edit_distance=1)",
            StatisticalCorrectorConfig {
                max_edit_distance: 1,
                ..Default::default()
            },
        ),
        (
            "Aggressive (max_edit_distance=3)",
            StatisticalCorrectorConfig {
                max_edit_distance: 3,
                ..Default::default()
            },
        ),
        (
            "Language model focused (weight=0.9)",
            StatisticalCorrectorConfig {
                language_model_weight: 0.9,
                edit_distance_weight: 0.1,
                ..Default::default()
            },
        ),
        (
            "Edit distance focused (weight=0.9)",
            StatisticalCorrectorConfig {
                language_model_weight: 0.1,
                edit_distance_weight: 0.9,
                ..Default::default()
            },
        ),
        (
            "No context",
            StatisticalCorrectorConfig {
                use_context: false,
                ..Default::default()
            },
        ),
    ];

    // Sample misspelled words with varied edit distances
    let test_cases = [
        "recieve",     // Should be "receive"
        "accidant",    // Could be "accident" or "accidental" depending on max_edit_distance
        "programing",  // Single-m, should be "programming"
        "thier",       // Common misspelling of "their"
        "complements", // Could be "compliments" depending on context
    ];

    // Test each configuration
    for (name, config) in &configs {
        let mut corrector = StatisticalCorrector::new(config.clone());

        // Train the model to ensure consistent behavior
        train_language_model(&mut corrector);
        add_example_words(&mut corrector);

        println!("{name} configuration:");
        println!("  max_editdistance: {}", config.max_edit_distance);
        println!("  language_modelweight: {}", config.language_model_weight);
        println!("  edit_distanceweight: {}", config.edit_distance_weight);
        println!("  usecontext: {}", config.use_context);

        println!("\n  Correction examples:");
        for word in &test_cases {
            let corrected = corrector.correct(word)?;
            println!("    {word} -> {corrected}");
        }

        // Show a context example if context is enabled
        if config.use_context {
            let context_example = "Going to the bnk to deposit money. The river bnk was muddy.";
            let corrected = corrector.correcttext(context_example)?;
            println!("\n  Context example:");
            println!("    Before: {context_example}");
            println!("    After:  {corrected}");
        }

        println!("\n{:-<60}", "");
    }

    Ok(())
}

// Demonstrate the error model (noisy channel model)
#[allow(dead_code)]
fn noise_model_demo() -> Result<(), Box<dyn std::error::Error>> {
    println!("\n=== Error Model Demo ===\n");

    // Create different error models with varying error type probabilities
    let models = [
        ("Default", ErrorModel::default()),
        ("Deletion-heavy", ErrorModel::new(0.7, 0.1, 0.1, 0.1)),
        ("Insertion-heavy", ErrorModel::new(0.1, 0.7, 0.1, 0.1)),
        ("Substitution-heavy", ErrorModel::new(0.1, 0.1, 0.7, 0.1)),
        ("Transposition-heavy", ErrorModel::new(0.1, 0.1, 0.1, 0.7)),
    ];

    // Test cases for different error types
    let test_pairs = [
        ("recieve", "receive"),        // Transposition (i and e)
        ("acheive", "achieve"),        // Transposition (i and e)
        ("languge", "language"),       // Deletion (missing 'a')
        ("programing", "programming"), // Deletion (missing 'm')
        ("probblem", "problem"),       // Insertion (extra 'b')
        ("committe", "committee"),     // Insertion (missing 'e')
        ("definately", "definitely"),  // Substitution ('a' instead of 'i')
        ("seperate", "separate"),      // Substitution ('e' instead of 'a')
    ];

    // Test each error model
    println!(
        "{:<20} {:<12} {:<12} {:<12} {:<12} {:<12}",
        "Model", "Delete Prob", "Insert Prob", "Subst Prob", "Transp Prob", "Example"
    );
    println!("{:-<80}", "");

    for (name, model) in &models {
        // Pick one example to show
        let (typo, correct) = test_pairs[0];
        let probability = model.error_probability(typo, correct);

        println!(
            "{:<20} {:<12.2} {:<12.2} {:<12.2} {:<12.2} {:<12.4}",
            name,
            model.p_deletion,
            model.p_insertion,
            model.p_substitution,
            model.p_transposition,
            probability
        );
    }

    println!("\nError probabilities for different error types (using default model):");

    let default_model = ErrorModel::default();

    for (typo, correct) in &test_pairs {
        let prob = default_model.error_probability(typo, correct);
        println!("{typo:<12} -> {correct:<12}: {prob:.6}");
    }

    println!("\nImpact on correction with custom error model:");

    // Create a statistical corrector with a custom error model
    let custom_config = StatisticalCorrectorConfig {
        language_model_weight: 0.3,
        edit_distance_weight: 0.7,
        ..Default::default()
    };

    let mut custom_corrector = StatisticalCorrector::new(custom_config);
    train_language_model(&mut custom_corrector);
    add_example_words(&mut custom_corrector);

    // Create a transposition-heavy error model (good for common spelling errors)
    let transposition_model = ErrorModel::new(0.1, 0.1, 0.1, 0.7);
    custom_corrector.set_error_model(transposition_model);

    // Test some examples
    println!("\nCorrecting text with transposition-heavy error model:");
    let testtext = "I recieved a mesage about thier acheivements.";
    let corrected = custom_corrector.correcttext(testtext)?;

    println!("Before: {testtext}");
    println!("After:  {corrected}");

    Ok(())
}

pub fn from_dictionary_corrector(dictcorrector: &DictionaryCorrector) -> Self

Create a statistical corrector from a base dictionary corrector

pub fn add_corpus_file<P: AsRef<Path>>(&mut self, path: P) -> Result<()>

Add a corpus file to train the language model

pub fn add_trainingtext(&mut self, text: &str)

Add text to train the language model

Examples found in repository

examples/statistical_spelling_demo.rs (line 74)

fn train_language_model(corrector: &mut StatisticalCorrector) {
    println!("Training language model with sample text...");

    // Add sample training text
    corrector.add_trainingtext(SAMPLE_TRAINING_TEXT);

    // Add more specialized training examples for context disambiguation
    let additional_examples = [
        // Bank context examples
        "I went to the bank to deposit money.",
        "The bank is open until 5pm.",
        "She works at the bank downtown.",
        "I need to check my bank account.",
        // River bank context examples
        "We sat on the bank of the river.",
        "The river bank was covered with flowers.",
        "They fished from the bank of the lake.",
        "The boat was tied to the bank.",
        // Homophone examples for there/their/they're
        "There is a book on the table.",
        "Their house is very beautiful.",
        "They're going to the movies tonight.",
        "There was a problem with the system.",
        "Their car broke down yesterday.",
        "They're planning a vacation next month.",
        // Complement/compliment examples
        "He received many compliments on his presentation.",
        "She gave him a compliment about his new haircut.",
        "Red and green are complementary colors.",
        "This wine complements the meal perfectly.",
        // Message examples
        "I received your message yesterday.",
        "Please send me a message when you arrive.",
        "The message was unclear and confusing.",
        "She left a message on my voicemail.",
    ];

    for example in &additional_examples {
        corrector.add_trainingtext(example);
    }

    println!(
        "Language model trained with {} words vocabulary\n",
        corrector.vocabulary_size()
    );
}

pub fn set_language_model(&mut self, model: NGramModel)

Set the language model

pub fn set_error_model(&mut self, model: ErrorModel)

Set the error model

Examples found in repository

examples/statistical_spelling_demo.rs (line 458)

fn noise_model_demo() -> Result<(), Box<dyn std::error::Error>> {
    println!("\n=== Error Model Demo ===\n");

    // Create different error models with varying error type probabilities
    let models = [
        ("Default", ErrorModel::default()),
        ("Deletion-heavy", ErrorModel::new(0.7, 0.1, 0.1, 0.1)),
        ("Insertion-heavy", ErrorModel::new(0.1, 0.7, 0.1, 0.1)),
        ("Substitution-heavy", ErrorModel::new(0.1, 0.1, 0.7, 0.1)),
        ("Transposition-heavy", ErrorModel::new(0.1, 0.1, 0.1, 0.7)),
    ];

    // Test cases for different error types
    let test_pairs = [
        ("recieve", "receive"),        // Transposition (i and e)
        ("acheive", "achieve"),        // Transposition (i and e)
        ("languge", "language"),       // Deletion (missing 'a')
        ("programing", "programming"), // Deletion (missing 'm')
        ("probblem", "problem"),       // Insertion (extra 'b')
        ("committe", "committee"),     // Insertion (missing 'e')
        ("definately", "definitely"),  // Substitution ('a' instead of 'i')
        ("seperate", "separate"),      // Substitution ('e' instead of 'a')
    ];

    // Test each error model
    println!(
        "{:<20} {:<12} {:<12} {:<12} {:<12} {:<12}",
        "Model", "Delete Prob", "Insert Prob", "Subst Prob", "Transp Prob", "Example"
    );
    println!("{:-<80}", "");

    for (name, model) in &models {
        // Pick one example to show
        let (typo, correct) = test_pairs[0];
        let probability = model.error_probability(typo, correct);

        println!(
            "{:<20} {:<12.2} {:<12.2} {:<12.2} {:<12.2} {:<12.4}",
            name,
            model.p_deletion,
            model.p_insertion,
            model.p_substitution,
            model.p_transposition,
            probability
        );
    }

    println!("\nError probabilities for different error types (using default model):");

    let default_model = ErrorModel::default();

    for (typo, correct) in &test_pairs {
        let prob = default_model.error_probability(typo, correct);
        println!("{typo:<12} -> {correct:<12}: {prob:.6}");
    }

    println!("\nImpact on correction with custom error model:");

    // Create a statistical corrector with a custom error model
    let custom_config = StatisticalCorrectorConfig {
        language_model_weight: 0.3,
        edit_distance_weight: 0.7,
        ..Default::default()
    };

    let mut custom_corrector = StatisticalCorrector::new(custom_config);
    train_language_model(&mut custom_corrector);
    add_example_words(&mut custom_corrector);

    // Create a transposition-heavy error model (good for common spelling errors)
    let transposition_model = ErrorModel::new(0.1, 0.1, 0.1, 0.7);
    custom_corrector.set_error_model(transposition_model);

    // Test some examples
    println!("\nCorrecting text with transposition-heavy error model:");
    let testtext = "I recieved a mesage about thier acheivements.";
    let corrected = custom_corrector.correcttext(testtext)?;

    println!("Before: {testtext}");
    println!("After:  {corrected}");

    Ok(())
}

pub fn set_metric<M: StringMetric + Send + Sync + 'static>(&mut self, metric: M)

Set the string metric to use for similarity calculations

pub fn set_config(&mut self, config: StatisticalCorrectorConfig)

Set the configuration

pub fn correct_sentence(&self, sentence: &str) -> Result<String>

Correct a sentence using a context-aware approach

pub fn add_word(&mut self, word: &str, frequency: usize)

Add a word to the dictionary

Examples found in repository

examples/statistical_spelling_demo.rs (line 146)

fn add_example_words(corrector: &mut StatisticalCorrector) {
    // Add specific words to the dictionary
    let word_frequencies = [
        // Common misspelled words
        ("bank", 100),
        ("river", 100),
        ("deposit", 100),
        ("money", 100),
        ("received", 100),
        ("message", 100),
        ("meeting", 100),
        ("compliments", 100),
        ("complements", 100),
        ("work", 100),
        ("there", 100),
        ("their", 100),
        ("they're", 100),
        ("was", 100),
        ("problem", 100),
        ("computer", 100),
        ("museum", 100),
        ("display", 100),
        ("historical", 100),
        ("artifacts", 100),
    ];

    for (word, freq) in &word_frequencies {
        corrector.add_word(word, *freq);
    }
}

pub fn remove_word(&mut self, word: &str)

Remove a word from the dictionary

pub fn dictionary_size(&self) -> usize

Get the total number of words in the dictionary

Examples found in repository

examples/statistical_spelling_demo.rs (line 190)

fn compare_correctors(
    dict_corrector: &DictionaryCorrector,
    stat_corrector: &StatisticalCorrector,
) -> Result<(), Box<dyn std::error::Error>> {
    println!("=== Dictionary vs. Statistical Correction ===\n");

    // Define test cases with known misspellings
    let test_cases = [
        ("recieve", "receive"),
        ("mesage", "message"),
        ("bnk", "bank"),
        ("thier", "their"),
        ("complements", "compliments"), // Can be correct in some contexts
        ("artefacts", "artifacts"),
        ("disply", "display"),
        ("definately", "definitely"),
    ];

    println!(
        "{:<15} {:<15} {:<15}",
        "Misspelled", "Dictionary", "Statistical"
    );
    println!("{:-<45}", "");

    for (misspelled, _expected) in &test_cases {
        let dict_correction = dict_corrector.correct(misspelled)?;
        let stat_correction = stat_corrector.correct(misspelled)?;

        println!("{misspelled:<15} {dict_correction:<15} {stat_correction:<15}");
    }

    println!("\nDictionary sizes:");
    println!(
        "  - Dictionary _corrector: {} words",
        dict_corrector.dictionary_size()
    );
    println!(
        "  - Statistical _corrector: {} words (+ {} in language model)",
        stat_corrector.dictionary_size(),
        stat_corrector.vocabulary_size()
    );

    Ok(())
}

pub fn vocabulary_size(&self) -> usize

Get the vocabulary size of the language model

Examples found in repository

examples/statistical_spelling_demo.rs (line 113)

fn train_language_model(corrector: &mut StatisticalCorrector) {
    println!("Training language model with sample text...");

    // Add sample training text
    corrector.add_trainingtext(SAMPLE_TRAINING_TEXT);

    // Add more specialized training examples for context disambiguation
    let additional_examples = [
        // Bank context examples
        "I went to the bank to deposit money.",
        "The bank is open until 5pm.",
        "She works at the bank downtown.",
        "I need to check my bank account.",
        // River bank context examples
        "We sat on the bank of the river.",
        "The river bank was covered with flowers.",
        "They fished from the bank of the lake.",
        "The boat was tied to the bank.",
        // Homophone examples for there/their/they're
        "There is a book on the table.",
        "Their house is very beautiful.",
        "They're going to the movies tonight.",
        "There was a problem with the system.",
        "Their car broke down yesterday.",
        "They're planning a vacation next month.",
        // Complement/compliment examples
        "He received many compliments on his presentation.",
        "She gave him a compliment about his new haircut.",
        "Red and green are complementary colors.",
        "This wine complements the meal perfectly.",
        // Message examples
        "I received your message yesterday.",
        "Please send me a message when you arrive.",
        "The message was unclear and confusing.",
        "She left a message on my voicemail.",
    ];

    for example in &additional_examples {
        corrector.add_trainingtext(example);
    }

    println!(
        "Language model trained with {} words vocabulary\n",
        corrector.vocabulary_size()
    );
}

// Function to add specific words for consistent example behavior
#[allow(dead_code)]
fn add_example_words(corrector: &mut StatisticalCorrector) {
    // Add specific words to the dictionary
    let word_frequencies = [
        // Common misspelled words
        ("bank", 100),
        ("river", 100),
        ("deposit", 100),
        ("money", 100),
        ("received", 100),
        ("message", 100),
        ("meeting", 100),
        ("compliments", 100),
        ("complements", 100),
        ("work", 100),
        ("there", 100),
        ("their", 100),
        ("they're", 100),
        ("was", 100),
        ("problem", 100),
        ("computer", 100),
        ("museum", 100),
        ("display", 100),
        ("historical", 100),
        ("artifacts", 100),
    ];

    for (word, freq) in &word_frequencies {
        corrector.add_word(word, *freq);
    }
}

// Compare dictionary-based and statistical spelling correction
#[allow(dead_code)]
fn compare_correctors(
    dict_corrector: &DictionaryCorrector,
    stat_corrector: &StatisticalCorrector,
) -> Result<(), Box<dyn std::error::Error>> {
    println!("=== Dictionary vs. Statistical Correction ===\n");

    // Define test cases with known misspellings
    let test_cases = [
        ("recieve", "receive"),
        ("mesage", "message"),
        ("bnk", "bank"),
        ("thier", "their"),
        ("complements", "compliments"), // Can be correct in some contexts
        ("artefacts", "artifacts"),
        ("disply", "display"),
        ("definately", "definitely"),
    ];

    println!(
        "{:<15} {:<15} {:<15}",
        "Misspelled", "Dictionary", "Statistical"
    );
    println!("{:-<45}", "");

    for (misspelled, _expected) in &test_cases {
        let dict_correction = dict_corrector.correct(misspelled)?;
        let stat_correction = stat_corrector.correct(misspelled)?;

        println!("{misspelled:<15} {dict_correction:<15} {stat_correction:<15}");
    }

    println!("\nDictionary sizes:");
    println!(
        "  - Dictionary _corrector: {} words",
        dict_corrector.dictionary_size()
    );
    println!(
        "  - Statistical _corrector: {} words (+ {} in language model)",
        stat_corrector.dictionary_size(),
        stat_corrector.vocabulary_size()
    );

    Ok(())
}

Trait Implementations

impl Clone for StatisticalCorrector

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for StatisticalCorrector

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for StatisticalCorrector

fn default() -> Self

Returns the “default value” for a type.

impl SpellingCorrector for StatisticalCorrector

fn correct(&self, word: &str) -> Result<String>

Correct a potentially misspelled word

fn get_suggestions(&self, word: &str, limit: usize) -> Result<Vec<String>>

Get a list of suggestions for a potentially misspelled word

fn is_correct(&self, word: &str) -> bool

Check if a word is spelled correctly

fn correcttext(&self, text: &str) -> Result<String>

Correct all words in a text