Struct TFIDFVectorizer 

pub struct TFIDFVectorizer<N = f32, K = String, E = DefaultTFIDFEngine>
where
    N: Num + Copy + Into<f64> + Send + Sync,
    E: TFIDFEngine<N> + Send + Sync,
    K: Clone + Send + Sync,
{
    pub documents: Vec<TFVector<N, K>>,
    pub token_dim_sample: IndexSet<Box<str>, RandomState>,
    pub corpus_ref: Arc<Corpus>,
    pub idf: IDFVector<N>,
    /* private fields */
}

Fields

documents: Vec<TFVector<N, K>>

Document’s TF Vector

token_dim_sample: IndexSet<Box<str>, RandomState>

TF Vector’s token dimension sample

corpus_ref: Arc<Corpus>

Corpus reference

idf: IDFVector<N>

IDF Vector

Implementations

impl<N, K, E> TFIDFVectorizer<N, K, E>

where K: Clone + Sync + Send + PartialEq, N: Num + Copy + Into<f64> + DeNormalizer + Send + Sync, E: TFIDFEngine<N> + Send + Sync,

pub fn similarity( &mut self, freq: &TokenFrequency, algorithm: &SimilarityAlgorithm, ) -> Hits<K>

Calculate similarity scores based on query token frequency Uses the specified similarity algorithm Calls update_idf() to use the latest IDF vector

Examples found in repository

examples/basic.rs (line 26)

fn main() {
    // build corpus
    let corpus = Arc::new(Corpus::new());

    // add documents
    let mut freq1 = TokenFrequency::new();
    freq1.add_tokens(&["rust", "高速", "並列", "rust"]);
    let mut freq2 = TokenFrequency::new();
    freq2.add_tokens(&["rust", "柔軟", "安全", "rust"]);

    // build query
    let mut vectorizer: TFIDFVectorizer<u16> = TFIDFVectorizer::new(corpus);    
    vectorizer.add_doc("doc1".to_string(), &freq1);
    vectorizer.add_doc("doc2".to_string(), &freq2);
    vectorizer.del_doc(&"doc1".to_string());
    vectorizer.add_doc("doc3".to_string(), &freq1);

    // similarity search
    let mut query_tokens = TokenFrequency::new();
    query_tokens.add_tokens(&["rust", "高速"]);
    let algorithm = SimilarityAlgorithm::CosineSimilarity;
    let mut result = vectorizer.similarity(&query_tokens, &algorithm);
    result.sort_by_score();

    // print result
    result.list.iter().for_each(|(k, s, l)| {
        println!("doc: {}, score: {}, length: {}", k, s, l);
    });
    // debug
    println!("result count: {}", result.list.len());
    println!("{:?}", vectorizer);
}

pub fn similarity_uncheck_idf( &self, freq: &TokenFrequency, algorithm: &SimilarityAlgorithm, ) -> Hits<K>

Calculate similarity scores based on query token frequency Uses the specified similarity algorithm Does not check the IDF vector (can be called with immutable reference) Call update_idf() manually if needed

impl<N, K, E> TFIDFVectorizer<N, K, E>

where N: Num + Copy + Into<f64> + Send + Sync, E: TFIDFEngine<N> + Send + Sync, K: Clone + Send + Sync,

pub fn new(corpus_ref: Arc<Corpus>) -> Self

Create a new TFIDFVectorizer instance

Examples found in repository

examples/basic.rs (line 16)

fn main() {
    // build corpus
    let corpus = Arc::new(Corpus::new());

    // add documents
    let mut freq1 = TokenFrequency::new();
    freq1.add_tokens(&["rust", "高速", "並列", "rust"]);
    let mut freq2 = TokenFrequency::new();
    freq2.add_tokens(&["rust", "柔軟", "安全", "rust"]);

    // build query
    let mut vectorizer: TFIDFVectorizer<u16> = TFIDFVectorizer::new(corpus);    
    vectorizer.add_doc("doc1".to_string(), &freq1);
    vectorizer.add_doc("doc2".to_string(), &freq2);
    vectorizer.del_doc(&"doc1".to_string());
    vectorizer.add_doc("doc3".to_string(), &freq1);

    // similarity search
    let mut query_tokens = TokenFrequency::new();
    query_tokens.add_tokens(&["rust", "高速"]);
    let algorithm = SimilarityAlgorithm::CosineSimilarity;
    let mut result = vectorizer.similarity(&query_tokens, &algorithm);
    result.sort_by_score();

    // print result
    result.list.iter().for_each(|(k, s, l)| {
        println!("doc: {}, score: {}, length: {}", k, s, l);
    });
    // debug
    println!("result count: {}", result.list.len());
    println!("{:?}", vectorizer);
}

pub fn set_corpus_ref(&mut self, corpus_ref: Arc<Corpus>)

set corpus reference and recalculate idf

pub fn update_idf(&mut self)

Corpusに変更があればIDFを再計算する

impl<N, K, E> TFIDFVectorizer<N, K, E>

where N: Num + Copy + Into<f64> + Send + Sync, E: TFIDFEngine<N> + Send + Sync, K: PartialEq + Clone + Send + Sync,

pub fn add_doc(&mut self, doc_id: K, doc: &TokenFrequency)

Add a document The immediately referenced Corpus is also updated

Examples found in repository

examples/basic.rs (line 17)

fn main() {
    // build corpus
    let corpus = Arc::new(Corpus::new());

    // add documents
    let mut freq1 = TokenFrequency::new();
    freq1.add_tokens(&["rust", "高速", "並列", "rust"]);
    let mut freq2 = TokenFrequency::new();
    freq2.add_tokens(&["rust", "柔軟", "安全", "rust"]);

    // build query
    let mut vectorizer: TFIDFVectorizer<u16> = TFIDFVectorizer::new(corpus);    
    vectorizer.add_doc("doc1".to_string(), &freq1);
    vectorizer.add_doc("doc2".to_string(), &freq2);
    vectorizer.del_doc(&"doc1".to_string());
    vectorizer.add_doc("doc3".to_string(), &freq1);

    // similarity search
    let mut query_tokens = TokenFrequency::new();
    query_tokens.add_tokens(&["rust", "高速"]);
    let algorithm = SimilarityAlgorithm::CosineSimilarity;
    let mut result = vectorizer.similarity(&query_tokens, &algorithm);
    result.sort_by_score();

    // print result
    result.list.iter().for_each(|(k, s, l)| {
        println!("doc: {}, score: {}, length: {}", k, s, l);
    });
    // debug
    println!("result count: {}", result.list.len());
    println!("{:?}", vectorizer);
}

pub fn del_doc(&mut self, doc_id: &K)

where K: PartialEq,

Examples found in repository

examples/basic.rs (line 19)

fn main() {
    // build corpus
    let corpus = Arc::new(Corpus::new());

    // add documents
    let mut freq1 = TokenFrequency::new();
    freq1.add_tokens(&["rust", "高速", "並列", "rust"]);
    let mut freq2 = TokenFrequency::new();
    freq2.add_tokens(&["rust", "柔軟", "安全", "rust"]);

    // build query
    let mut vectorizer: TFIDFVectorizer<u16> = TFIDFVectorizer::new(corpus);    
    vectorizer.add_doc("doc1".to_string(), &freq1);
    vectorizer.add_doc("doc2".to_string(), &freq2);
    vectorizer.del_doc(&"doc1".to_string());
    vectorizer.add_doc("doc3".to_string(), &freq1);

    // similarity search
    let mut query_tokens = TokenFrequency::new();
    query_tokens.add_tokens(&["rust", "高速"]);
    let algorithm = SimilarityAlgorithm::CosineSimilarity;
    let mut result = vectorizer.similarity(&query_tokens, &algorithm);
    result.sort_by_score();

    // print result
    result.list.iter().for_each(|(k, s, l)| {
        println!("doc: {}, score: {}, length: {}", k, s, l);
    });
    // debug
    println!("result count: {}", result.list.len());
    println!("{:?}", vectorizer);
}

pub fn get_tf(&self, key: &K) -> Option<&TFVector<N, K>>

where K: PartialEq,

Get TFVector by document ID

pub fn get_tf_into_token_freq(&self, key: &K) -> Option<TokenFrequency>

Get TokenFrequency by document ID If quantized, there may be some error Words not included in the corpus are ignored

pub fn contains_doc(&self, key: &K) -> bool

where K: PartialEq,

Check if a document with the given ID exists

pub fn contains_token(&self, token: &str) -> bool

Check if the token exists in the token dimension sample

pub fn contains_tokens_from_freq(&self, freq: &TokenFrequency) -> bool

Check if all tokens in the given TokenFrequency exist in the token dimension sample

pub fn doc_num(&self) -> usize

Trait Implementations

impl<N, K, E> Clone for TFIDFVectorizer<N, K, E>

where N: Num + Copy + Into<f64> + Send + Sync + Clone, E: TFIDFEngine<N> + Send + Sync + Clone, K: Clone + Send + Sync + Clone,

fn clone(&self) -> TFIDFVectorizer<N, K, E>

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<N, K, E> Debug for TFIDFVectorizer<N, K, E>

where N: Num + Copy + Into<f64> + Send + Sync + Debug, E: TFIDFEngine<N> + Send + Sync + Debug, K: Clone + Send + Sync + Debug,

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<N, K, E> Serialize for TFIDFVectorizer<N, K, E>

where N: Num + Copy + Serialize + Into<f64> + Send + Sync, K: Serialize + Clone + Send + Sync, E: TFIDFEngine<N>,

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

where S: Serializer,

Serialize TFIDFVectorizer. This struct contains references, so they are excluded from serialization. Use TFIDFData for deserialization.