candding 0.1.0

use super::*;

/// Weights come from the module's own `saturation`/`bm25_hash`, so this checks the stems, counts and `doc_len` only.
fn expected(pairs: &[(&str, u32)], doc_len: u32) -> SparseEmbedding {
  SparseEmbedding::from_pairs(
    pairs
      .iter()
      .map(|&(stem, count)| (bm25_hash(stem), term_frequency::saturation(count, doc_len))),
  )
}

/// 9 raw tokens, three stopwords dropped, 6 survivors once each.
#[test]
fn short_fixture_matches_the_reference() {
  let stopwords = test_stopwords();
  let got = embed_document("The quick brown fox jumps over the lazy dog.", &stopwords);
  let want = expected(
    &[
      ("brown", 1),
      ("dog", 1),
      ("fox", 1),
      ("jump", 1),
      ("lazi", 1),
      ("quick", 1),
    ],
    6,
  );
  assert_eq!(got, want);
}

#[test]
fn empty_and_whitespace_fixtures_produce_empty_rows() {
  let stopwords = test_stopwords();
  assert!(embed_document("", &stopwords).is_empty());
  assert!(embed_document("   \n\t  ", &stopwords).is_empty());
}

/// 18 raw tokens, `i`/`i`/`as` dropped, 15 survivors; `f32`, `xs` and `0` occur twice.
#[test]
fn code_fixture_matches_the_reference_including_repeated_tokens() {
  let stopwords = test_stopwords();
  let text = "fn main() { let xs: Vec<f32> = (0..10).map(|i| i as f32 * 0.5).collect(); println!(\"{:?}\", xs); }";
  let got = embed_document(text, &stopwords);
  let want = expected(
    &[
      ("0", 2),
      ("10", 1),
      ("5", 1),
      ("collect", 1),
      ("f32", 2),
      ("fn", 1),
      ("let", 1),
      ("main", 1),
      ("map", 1),
      ("println", 1),
      ("vec", 1),
      ("xs", 2),
    ],
    15,
  );
  assert_eq!(got, want);
}

/// `to` drops as a stopword and the other four stem (`embeddings` -> `embed`) in a four-token document.
#[test]
fn batch_a_fixture_matches_the_reference() {
  let stopwords = test_stopwords();
  let got = embed_document("Embeddings map text to vectors.", &stopwords);
  let want = expected(&[("embed", 1), ("map", 1), ("text", 1), ("vector", 1)], 4);
  assert_eq!(got, want);
}

#[test]
fn batch_c_fixture_matches_the_reference() {
  let stopwords = test_stopwords();
  let got = embed_document("Short.", &stopwords);
  let want = expected(&[("short", 1)], 1);
  assert_eq!(got, want);
}

/// A lone `_` is the one reachable punctuation case: the tokenizer keeps it as a word character.
#[test]
fn a_lone_underscore_is_dropped_by_the_punctuation_filter() {
  let stopwords = test_stopwords();
  let got = embed_document("hello _ world", &stopwords);
  let want = expected(&[("hello", 1), ("world", 1)], 2);
  assert_eq!(got, want);
}

/// `token_max_length` is inclusive at 40; the trailing "y" stems to "i" under Snowball English.
#[test]
fn the_length_cap_keeps_exactly_forty_characters_and_drops_forty_one() {
  let stopwords = test_stopwords();
  let text = format!("a {} b {} c", "x".repeat(41), "y".repeat(40));
  let got = embed_document(&text, &stopwords);
  // "a" is a stopword and the 41-character run is cut; the rest survive once each in a three-token document.
  let want = expected(
    &[
      ("b", 1),
      ("c", 1),
      ("yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyi", 1),
    ],
    3,
  );
  assert_eq!(got, want);
}

/// `SimpleTokenizer.tokenize` lowercases before this module sees a token.
#[test]
fn stemming_and_stopwords_are_case_insensitive() {
  let stopwords = test_stopwords();
  let got = embed_document("The RUNNING Runner ran QUICKLY", &stopwords);
  let want = expected(&[("quick", 1), ("ran", 1), ("run", 1), ("runner", 1)], 4);
  assert_eq!(got, want);
}

/// The apostrophe becomes a space, and `"s"` is itself in the English stopword list.
#[test]
fn a_possessive_apostrophe_splits_into_two_tokens_and_the_fragment_is_a_real_stopword() {
  let stopwords = test_stopwords();
  assert!(stopwords.contains("s"));
  let got = embed_document("the model's parameters", &stopwords);
  let want = expected(&[("model", 1), ("paramet", 1)], 2);
  assert_eq!(got, want);
}

/// All 8 raw tokens survive; the Japanese clause stays one token, the reference's own limitation for unspaced scripts.
#[test]
fn multilingual_fixture_reaches_no_stopword_or_punctuation_path_and_keeps_every_token() {
  let stopwords = test_stopwords();
  let text = "Árvíztűrő tükörfúrógép — 東京タワーは高いです。Ça va très bien, merci!";
  let got = embed_document(text, &stopwords);
  let want = expected(
    &[
      ("árvíztűrő", 1),
      ("tükörfúrógép", 1),
      ("東京タワーは高いです", 1),
      ("ça", 1),
      ("va", 1),
      ("très", 1),
      ("bien", 1),
      ("merci", 1),
    ],
    8,
  );
  assert_eq!(got, want);
}

/// The repeated `"lazy"` would change a passage weight; the query row never counts.
#[test]
fn query_path_hashes_unique_stems_at_a_uniform_weight() {
  let stopwords = test_stopwords();
  let got = embed_query("dog dog cat", &stopwords);
  let want = SparseEmbedding::from_pairs([(bm25_hash("dog"), 1.0f32), (bm25_hash("cat"), 1.0f32)]);
  assert_eq!(got, want);
  assert_eq!(
    got.len(),
    2,
    "a repeated stem contributes one entry, not two"
  );
}

#[test]
fn query_path_shares_the_same_filters_as_the_document_path() {
  let stopwords = test_stopwords();
  let got = embed_query("the _ dog", &stopwords);
  let want = SparseEmbedding::from_pairs([(bm25_hash("dog"), 1.0f32)]);
  assert_eq!(got, want);
}

#[test]
fn a_stem_repeated_across_a_document_shares_one_weight_scaled_by_saturation() {
  let stopwords = test_stopwords();
  let got = embed_document("alpha beta alpha gamma alpha", &stopwords);
  let want = expected(&[("alpha", 3), ("beta", 1), ("gamma", 1)], 5);
  assert_eq!(got, want);
  assert_ne!(
    term_frequency::saturation(3, 5),
    term_frequency::saturation(1, 5),
    "a thrice-repeated stem must not collapse to the same weight as a once-only one"
  );
}