snowstem 0.1.0

//! Replays the shipped conformance fixture, one tsv per language.
//!
//! Each line of `tests/data/<name>.tsv` holds a word and the stem the
//! Snowball reference computes for it, sampled from the BSD-covered
//! official test vocabularies or built from this crate's own word
//! lists where a vocabulary carries another license. Every line must
//! match byte for byte through both `stem` and `stem_into`. CI replays
//! the full vocabularies with the same check before a release.

use std::borrow::Cow;
use std::fs;
use std::path::Path;

use snowstem::{Algorithm, Stemmer};

fn pairs(name: &str) -> Vec<(String, String)> {
    let path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .join("tests/data")
        .join(format!("{name}.tsv"));
    let text = fs::read_to_string(&path).unwrap();
    let pairs: Vec<(String, String)> = text
        .lines()
        .map(|line| {
            let (word, stem) = line.split_once('\t').unwrap();
            (word.to_owned(), stem.to_owned())
        })
        .collect();
    assert!(pairs.len() >= 400, "fixture {name} lost most of its lines");
    pairs
}

fn replay(algorithm: Algorithm, name: &str) {
    let stemmer = Stemmer::create(algorithm);
    let mut buffer = String::new();
    for (word, expected) in pairs(name) {
        let stem = stemmer.stem(&word);
        assert_eq!(stem, *expected, "{name}: stem({word:?})");
        if word == *expected && word.len() <= 64 {
            assert!(
                matches!(stem, Cow::Borrowed(_)),
                "{name}: stem({word:?}) copied a word that is its own stem"
            );
        }
        stemmer.stem_into(&word, &mut buffer);
        assert_eq!(buffer, *expected, "{name}: stem_into({word:?})");
    }
}

macro_rules! vocabulary {
    ($($test:ident => $algorithm:ident, $file:literal;)*) => {
        $(
            #[test]
            fn $test() {
                replay(Algorithm::$algorithm, $file);
            }
        )*
    };
}

vocabulary! {
    vocabulary_arabic => Arabic, "arabic";
    vocabulary_czech => Czech, "czech";
    vocabulary_danish => Danish, "danish";
    vocabulary_dutch => Dutch, "dutch";
    vocabulary_dutch_porter => DutchPorter, "dutch_porter";
    vocabulary_english => English, "english";
    vocabulary_finnish => Finnish, "finnish";
    vocabulary_french => French, "french";
    vocabulary_german => German, "german";
    vocabulary_greek => Greek, "greek";
    vocabulary_hungarian => Hungarian, "hungarian";
    vocabulary_italian => Italian, "italian";
    vocabulary_norwegian => Norwegian, "norwegian";
    vocabulary_polish => Polish, "polish";
    vocabulary_portuguese => Portuguese, "portuguese";
    vocabulary_romanian => Romanian, "romanian";
    vocabulary_russian => Russian, "russian";
    vocabulary_spanish => Spanish, "spanish";
    vocabulary_swedish => Swedish, "swedish";
    vocabulary_tamil => Tamil, "tamil";
    vocabulary_turkish => Turkish, "turkish";
}

/// The headline defect this crate exists to fix: "internal" must not
/// collapse into the "intern" family.
#[test]
fn english_internal() {
    let stemmer = Stemmer::create(Algorithm::English);
    assert_eq!(stemmer.stem("internal"), "internal");
}

#[test]
fn english_international() {
    let stemmer = Stemmer::create(Algorithm::English);
    assert_eq!(stemmer.stem("international"), "internat");
}

/// "evening" sits on the English exception list and keeps its form.
#[test]
fn english_evening() {
    let stemmer = Stemmer::create(Algorithm::English);
    assert_eq!(stemmer.stem("evening"), "evening");
}

/// Double-consonant undoubling: "added" stems to "add", not "ad".
#[test]
fn english_added() {
    let stemmer = Stemmer::create(Algorithm::English);
    assert_eq!(stemmer.stem("added"), "add");
}

/// A short Greek word must stem to a stem, never to the empty string.
#[test]
fn greek_isa() {
    let stemmer = Stemmer::create(Algorithm::Greek);
    assert_eq!(stemmer.stem("ίσα"), "ισ");
}

/// A trailing period after an Arabic letter stays in place.
#[test]
fn arabic_trailing_period() {
    let stemmer = Stemmer::create(Algorithm::Arabic);
    assert_eq!(stemmer.stem("د."), "د.");
}

/// The two Dutch algorithms are distinct: the current one and the
/// pre-3.0 one disagree on a large share of the vocabulary.
#[test]
fn dutch_variants_differ() {
    let current = Stemmer::create(Algorithm::Dutch);
    let porter = Stemmer::create(Algorithm::DutchPorter);
    let differing = pairs("dutch")
        .iter()
        .filter(|(word, _)| current.stem(word) != porter.stem(word))
        .count();
    assert!(differing > 50, "only {differing} words differ");
}

#[test]
fn empty_word_is_borrowed() {
    let stemmer = Stemmer::create(Algorithm::English);
    assert!(matches!(stemmer.stem(""), Cow::Borrowed("")));
}

#[test]
fn algorithm_names_round_trip() {
    for name in [
        "arabic",
        "czech",
        "danish",
        "dutch",
        "dutch_porter",
        "english",
        "finnish",
        "french",
        "german",
        "greek",
        "hungarian",
        "italian",
        "norwegian",
        "polish",
        "portuguese",
        "romanian",
        "russian",
        "spanish",
        "swedish",
        "tamil",
        "turkish",
    ] {
        let algorithm: Algorithm = name.parse().unwrap();
        assert_eq!(algorithm.to_string(), name);
    }
    assert!("klingon".parse::<Algorithm>().is_err());
    assert!("English".parse::<Algorithm>().is_err());
}