candding 0.1.0

//! `Qdrant/bm25`'s text splitting: `remove_non_alphanumeric`, then lowercase-and-split. `is_alphanumeric` or `_`
//! stands in for Python's Unicode `\w`, which may differ at the margins; every fixture tokenizes identically.

/// Word characters and whitespace survive; everything else becomes a space. Runs once, before `tokenize`.
pub(crate) fn remove_non_alphanumeric(text: &str) -> String {
  text
    .chars()
    .map(|c| {
      if is_word(c) || c.is_whitespace() {
        c
      } else {
        ' '
      }
    })
    .collect()
}

/// `SimpleTokenizer.tokenize`: lowercase, blank every non-word character including whitespace, split on whitespace.
pub(crate) fn tokenize(text: &str) -> Vec<String> {
  text
    .to_lowercase()
    .chars()
    .map(|c| if is_word(c) { c } else { ' ' })
    .collect::<String>()
    .split_whitespace()
    .map(str::to_string)
    .collect()
}

/// Stands in for Python's `\w`.
fn is_word(c: char) -> bool {
  c.is_alphanumeric() || c == '_'
}

#[cfg(test)]
mod tests {
  use super::*;

  /// Matches the reference's two regexes: the sentence becomes nine lowercase words, the period a separator.
  #[test]
  fn tokenizes_a_short_english_sentence() {
    let cleaned = remove_non_alphanumeric("The quick brown fox jumps over the lazy dog.");
    let tokens = tokenize(&cleaned);
    assert_eq!(
      tokens,
      vec![
        "the", "quick", "brown", "fox", "jumps", "over", "the", "lazy", "dog"
      ]
    );
  }

  /// `_` is `\w`, so a lone underscore is its own token here; the punctuation filter drops it later.
  #[test]
  fn a_lone_underscore_is_its_own_token() {
    let cleaned = remove_non_alphanumeric("hello _ world");
    assert_eq!(tokenize(&cleaned), vec!["hello", "_", "world"]);
  }

  /// Unlike `bm42`'s WordPiece input, this regex erases a bare symbol before a token forms around it.
  #[test]
  fn bare_punctuation_and_symbols_vanish_rather_than_becoming_tokens() {
    let text = "fn main() { let xs: Vec<f32> = (0..10).map(|i| i as f32 * 0.5).collect(); }";
    let cleaned = remove_non_alphanumeric(text);
    let tokens = tokenize(&cleaned);
    for symbol in ["<", ">", "=", "(", ")", "{", "}", ":", ";", "!", ".", ","] {
      assert!(
        !tokens.contains(&symbol.to_string()),
        "{symbol} in {tokens:?}"
      );
    }
    assert_eq!(
      tokens,
      vec![
        "fn", "main", "let", "xs", "vec", "f32", "0", "10", "map", "i", "i", "as", "f32", "0", "5",
        "collect"
      ]
    );
  }

  /// The reference keeps `"東京タワーは高いです"` as one token: unspaced scripts are its own limitation.
  #[test]
  fn a_whitespace_free_script_is_one_undivided_token() {
    let cleaned = remove_non_alphanumeric("東京タワーは高いです。");
    assert_eq!(tokenize(&cleaned), vec!["東京タワーは高いです"]);
  }

  #[test]
  fn accented_latin_words_lowercase_and_tokenize_as_single_words() {
    let cleaned = remove_non_alphanumeric("Árvíztűrő tükörfúrógép");
    assert_eq!(tokenize(&cleaned), vec!["árvíztűrő", "tükörfúrógép"]);
  }

  #[test]
  fn empty_and_whitespace_only_text_tokenizes_to_nothing() {
    assert!(tokenize(&remove_non_alphanumeric("")).is_empty());
    assert!(tokenize(&remove_non_alphanumeric("   \n\t  ")).is_empty());
  }

  #[test]
  fn mixed_whitespace_runs_collapse_between_tokens() {
    let cleaned = remove_non_alphanumeric("a \t\n  b");
    assert_eq!(tokenize(&cleaned), vec!["a", "b"]);
  }
}