Struct CosineSearcher

pub struct CosineSearcher { /* private fields */ }

Searcher for all pairs of similar documents in the Cosine space.

Approach

The search steps consist of

1. Extracts features from documents, where a feature is a tfidf-weighted vector representation of character or word ngrams.
2. Convert the features into binary sketches through the simplified simhash.
3. Search for similar sketches in the Hamming space using ChunkedJoiner.

Examples

use find_simdoc::tfidf::{Idf, Tf};
use find_simdoc::CosineSearcher;

let documents = vec![
    "Welcome to Jimbocho, the town of books and curry!",
    "Welcome to Jimbocho, the city of books and curry!",
    "We welcome you to Jimbocho, the town of books and curry.",
    "Welcome to the town of books and curry, Jimbocho!",
];

// Creates a searcher for word unigrams (with random seed value 42).
let searcher = CosineSearcher::new(1, Some(' '), Some(42)).unwrap();
// Creates a term frequency (TF) weighter.
let tf = Tf::new();
// Creates a inverse document frequency (IDF) weighter.
let idf = Idf::new()
    .build(documents.iter().clone(), searcher.config())
    .unwrap();
// Builds the database of binary sketches converted from input documents,
let searcher = searcher
    // with the TF weighter and
    .tf(Some(tf))
    // the IDF weighter,
    .idf(Some(idf))
    // where binary sketches are in the Hamming space of 10*64 dimensions.
    .build_sketches_in_parallel(documents.iter(), 10)
    .unwrap();

// Searches all similar pairs within radius 0.25.
let results = searcher.search_similar_pairs(0.25);
// A result consists of the left-side id, the right-side id, and their distance.
assert_eq!(results, vec![(0, 1, 0.1296875), (0, 3, 0.24375)]);

Implementations

impl CosineSearcher

pub fn new( window_size: usize, delimiter: Option<char>, seed: Option<u64>, ) -> Result<Self>

Creates an instance.

Arguments

• window_size - Window size for w-shingling in feature extraction (must be more than 0).
• delimiter - Delimiter for recognizing words as tokens in feature extraction. If None, characters are used for tokens.
• seed - Seed value for random values.

Examples found in repository

examples/find_cosine.rs (line 13)

fn main() {
    let documents = vec![
        "Welcome to Jimbocho, the town of books and curry!",
        "Welcome to Jimbocho, the city of books and curry!",
        "We welcome you to Jimbocho, the town of books and curry.",
        "Welcome to the town of books and curry, Jimbocho!",
    ];

    // Creates a searcher for word unigrams (with random seed value 42).
    let searcher = CosineSearcher::new(1, Some(' '), Some(42)).unwrap();
    // Creates a term frequency (TF) weighter.
    let tf = Tf::new();
    // Creates a inverse document frequency (IDF) weighter.
    let idf = Idf::new()
        .build(documents.iter().clone(), searcher.config())
        .unwrap();
    // Builds the database of binary sketches converted from input documents,
    let searcher = searcher
        // with the TF weighter and
        .tf(Some(tf))
        // the IDF weighter,
        .idf(Some(idf))
        // where binary sketches are in the Hamming space of 10*64 dimensions.
        .build_sketches_in_parallel(documents.iter(), 10)
        .unwrap();

    // Searches all similar pairs within radius 0.25.
    let results = searcher.search_similar_pairs(0.25);
    // A result consists of the left-side id, the right-side id, and their distance.
    assert_eq!(results, vec![(0, 1, 0.1671875), (0, 3, 0.246875)]);
}

pub const fn shows_progress(self, yes: bool) -> Self

Shows the progress via the standard error output?

pub fn tf(self, tf: Option<Tf>) -> Self

Sets the scheme of TF weighting.

Examples found in repository

examples/find_cosine.rs (line 23)

fn main() {
    let documents = vec![
        "Welcome to Jimbocho, the town of books and curry!",
        "Welcome to Jimbocho, the city of books and curry!",
        "We welcome you to Jimbocho, the town of books and curry.",
        "Welcome to the town of books and curry, Jimbocho!",
    ];

    // Creates a searcher for word unigrams (with random seed value 42).
    let searcher = CosineSearcher::new(1, Some(' '), Some(42)).unwrap();
    // Creates a term frequency (TF) weighter.
    let tf = Tf::new();
    // Creates a inverse document frequency (IDF) weighter.
    let idf = Idf::new()
        .build(documents.iter().clone(), searcher.config())
        .unwrap();
    // Builds the database of binary sketches converted from input documents,
    let searcher = searcher
        // with the TF weighter and
        .tf(Some(tf))
        // the IDF weighter,
        .idf(Some(idf))
        // where binary sketches are in the Hamming space of 10*64 dimensions.
        .build_sketches_in_parallel(documents.iter(), 10)
        .unwrap();

    // Searches all similar pairs within radius 0.25.
    let results = searcher.search_similar_pairs(0.25);
    // A result consists of the left-side id, the right-side id, and their distance.
    assert_eq!(results, vec![(0, 1, 0.1671875), (0, 3, 0.246875)]);
}

pub fn idf(self, idf: Option<Idf<u64>>) -> Self

Sets the scheme of IDF weighting.

Examples found in repository

examples/find_cosine.rs (line 25)

fn main() {
    let documents = vec![
        "Welcome to Jimbocho, the town of books and curry!",
        "Welcome to Jimbocho, the city of books and curry!",
        "We welcome you to Jimbocho, the town of books and curry.",
        "Welcome to the town of books and curry, Jimbocho!",
    ];

    // Creates a searcher for word unigrams (with random seed value 42).
    let searcher = CosineSearcher::new(1, Some(' '), Some(42)).unwrap();
    // Creates a term frequency (TF) weighter.
    let tf = Tf::new();
    // Creates a inverse document frequency (IDF) weighter.
    let idf = Idf::new()
        .build(documents.iter().clone(), searcher.config())
        .unwrap();
    // Builds the database of binary sketches converted from input documents,
    let searcher = searcher
        // with the TF weighter and
        .tf(Some(tf))
        // the IDF weighter,
        .idf(Some(idf))
        // where binary sketches are in the Hamming space of 10*64 dimensions.
        .build_sketches_in_parallel(documents.iter(), 10)
        .unwrap();

    // Searches all similar pairs within radius 0.25.
    let results = searcher.search_similar_pairs(0.25);
    // A result consists of the left-side id, the right-side id, and their distance.
    assert_eq!(results, vec![(0, 1, 0.1671875), (0, 3, 0.246875)]);
}

pub fn build_sketches<I, D>( self, documents: I, num_chunks: usize, ) -> Result<Self>

where I: IntoIterator<Item = D>, D: AsRef<str>,

Builds the database of sketches from input documents.

Arguments

• documents - List of documents (must not include an empty string).
• num_chunks - Number of chunks of sketches, indicating that the number of dimensions in the Hamming space is num_chunks*64.

pub fn build_sketches_in_parallel<I, D>( self, documents: I, num_chunks: usize, ) -> Result<Self>

where I: Iterator<Item = D> + Send, D: AsRef<str> + Send,

Builds the database of sketches from input documents in parallel.

Arguments

• documents - List of documents (must not include an empty string).
• num_chunks - Number of chunks of sketches, indicating that the number of dimensions in the Hamming space is num_chunks*64.

Notes

The progress is not printed even if shows_progress = true.

Examples found in repository

examples/find_cosine.rs (line 27)

fn main() {
    let documents = vec![
        "Welcome to Jimbocho, the town of books and curry!",
        "Welcome to Jimbocho, the city of books and curry!",
        "We welcome you to Jimbocho, the town of books and curry.",
        "Welcome to the town of books and curry, Jimbocho!",
    ];

    // Creates a searcher for word unigrams (with random seed value 42).
    let searcher = CosineSearcher::new(1, Some(' '), Some(42)).unwrap();
    // Creates a term frequency (TF) weighter.
    let tf = Tf::new();
    // Creates a inverse document frequency (IDF) weighter.
    let idf = Idf::new()
        .build(documents.iter().clone(), searcher.config())
        .unwrap();
    // Builds the database of binary sketches converted from input documents,
    let searcher = searcher
        // with the TF weighter and
        .tf(Some(tf))
        // the IDF weighter,
        .idf(Some(idf))
        // where binary sketches are in the Hamming space of 10*64 dimensions.
        .build_sketches_in_parallel(documents.iter(), 10)
        .unwrap();

    // Searches all similar pairs within radius 0.25.
    let results = searcher.search_similar_pairs(0.25);
    // A result consists of the left-side id, the right-side id, and their distance.
    assert_eq!(results, vec![(0, 1, 0.1671875), (0, 3, 0.246875)]);
}

pub fn search_similar_pairs(&self, radius: f64) -> Vec<(usize, usize, f64)>

Searches for all pairs of similar documents within an input radius, returning triplets of the left-side id, the right-side id, and their distance.

Examples found in repository

examples/find_cosine.rs (line 31)

fn main() {
    let documents = vec![
        "Welcome to Jimbocho, the town of books and curry!",
        "Welcome to Jimbocho, the city of books and curry!",
        "We welcome you to Jimbocho, the town of books and curry.",
        "Welcome to the town of books and curry, Jimbocho!",
    ];

    // Creates a searcher for word unigrams (with random seed value 42).
    let searcher = CosineSearcher::new(1, Some(' '), Some(42)).unwrap();
    // Creates a term frequency (TF) weighter.
    let tf = Tf::new();
    // Creates a inverse document frequency (IDF) weighter.
    let idf = Idf::new()
        .build(documents.iter().clone(), searcher.config())
        .unwrap();
    // Builds the database of binary sketches converted from input documents,
    let searcher = searcher
        // with the TF weighter and
        .tf(Some(tf))
        // the IDF weighter,
        .idf(Some(idf))
        // where binary sketches are in the Hamming space of 10*64 dimensions.
        .build_sketches_in_parallel(documents.iter(), 10)
        .unwrap();

    // Searches all similar pairs within radius 0.25.
    let results = searcher.search_similar_pairs(0.25);
    // A result consists of the left-side id, the right-side id, and their distance.
    assert_eq!(results, vec![(0, 1, 0.1671875), (0, 3, 0.246875)]);
}

pub fn len(&self) -> usize

Gets the number of input documents.

pub fn is_empty(&self) -> bool

Checks if the database is empty.

pub fn memory_in_bytes(&self) -> usize

Gets the memory usage in bytes.

pub const fn config(&self) -> &FeatureConfig

Gets the configure of feature extraction.

Examples found in repository

examples/find_cosine.rs (line 18)

fn main() {
    let documents = vec![
        "Welcome to Jimbocho, the town of books and curry!",
        "Welcome to Jimbocho, the city of books and curry!",
        "We welcome you to Jimbocho, the town of books and curry.",
        "Welcome to the town of books and curry, Jimbocho!",
    ];

    // Creates a searcher for word unigrams (with random seed value 42).
    let searcher = CosineSearcher::new(1, Some(' '), Some(42)).unwrap();
    // Creates a term frequency (TF) weighter.
    let tf = Tf::new();
    // Creates a inverse document frequency (IDF) weighter.
    let idf = Idf::new()
        .build(documents.iter().clone(), searcher.config())
        .unwrap();
    // Builds the database of binary sketches converted from input documents,
    let searcher = searcher
        // with the TF weighter and
        .tf(Some(tf))
        // the IDF weighter,
        .idf(Some(idf))
        // where binary sketches are in the Hamming space of 10*64 dimensions.
        .build_sketches_in_parallel(documents.iter(), 10)
        .unwrap();

    // Searches all similar pairs within radius 0.25.
    let results = searcher.search_similar_pairs(0.25);
    // A result consists of the left-side id, the right-side id, and their distance.
    assert_eq!(results, vec![(0, 1, 0.1671875), (0, 3, 0.246875)]);
}