candding 0.1.0

//! A parsed one-word-per-line stopword list. miniCOIL's and BM42's lists are different files; only the
//! format and this type are shared.

use std::collections::HashSet;

/// A parsed stopword list: one lowercase word per line, membership checked by exact string match.
#[derive(Debug)]
pub struct Stopwords(HashSet<String>);

impl Stopwords {
  /// One word per line; blank lines and surrounding whitespace are tolerated.
  pub fn parse(text: &str) -> Self {
    Self(
      text
        .lines()
        .map(str::trim)
        .filter(|line| !line.is_empty())
        .map(str::to_string)
        .collect(),
    )
  }

  pub fn contains(&self, word: &str) -> bool {
    self.0.contains(word)
  }

  pub fn len(&self) -> usize {
    self.0.len()
  }

  pub fn is_empty(&self) -> bool {
    self.0.is_empty()
  }
}

#[cfg(test)]
mod tests {
  use super::*;

  #[test]
  fn parses_one_word_per_line() {
    let stopwords = Stopwords::parse("the\na\nin\n");
    assert_eq!(stopwords.len(), 3);
    assert!(stopwords.contains("the"));
    assert!(stopwords.contains("a"));
    assert!(stopwords.contains("in"));
    assert!(!stopwords.contains("fruit"));
  }

  /// An excerpt of `Qdrant/minicoil-v1`'s `stopwords.txt`, with the "in" and "a" its README drops from "bat in a cave".
  #[test]
  fn parses_a_real_excerpt_of_minicoils_own_stopwords_file() {
    let stopwords = Stopwords::parse("i\nme\nmy\nin\na\nthe\nwon\nwouldn\n");
    assert!(stopwords.contains("in"));
    assert!(stopwords.contains("a"));
    assert_eq!(stopwords.len(), 8);
  }

  #[test]
  fn blank_lines_and_surrounding_whitespace_are_ignored() {
    let stopwords = Stopwords::parse("the\n\n  a  \n\nin\n");
    assert_eq!(stopwords.len(), 3);
    assert!(stopwords.contains("a"));
  }

  #[test]
  fn an_empty_file_has_no_stopwords() {
    assert!(Stopwords::parse("").is_empty());
  }
}