Struct TokenFrequency

Source

pub struct TokenFrequency { /* private fields */ }

Expand description

TokenFrequency struct Manages the frequency of token occurrences. Counts the number of times each token appears.

§Examples

use crate::tf_idf_vectorizer::vectorizer::token::TokenFrequency;
let mut token_freq = TokenFrequency::new();
token_freq.add_token("token1");
token_freq.add_token("token2");
token_freq.add_token("token1");

assert_eq!(token_freq.token_count("token1"), 2);

Implementations§

Source §

impl TokenFrequency

Implementation for adding and removing tokens

Source

pub fn new() -> Self

Create a new TokenFrequency

Examples found in repository ?

examples/basic.rs (line 10)

5fn main() {
6    // build corpus
7    let corpus = Arc::new(Corpus::new());
8
9    // add documents
10    let mut freq1 = TokenFrequency::new();
11    freq1.add_tokens(&["rust", "高速", "並列", "rust"]);
12    let mut freq2 = TokenFrequency::new();
13    freq2.add_tokens(&["rust", "柔軟", "安全", "rust"]);
14
15    // build query
16    let mut vectorizer: TFIDFVectorizer<u16> = TFIDFVectorizer::new(corpus);    
17    vectorizer.add_doc("doc1".to_string(), &freq1);
18    vectorizer.add_doc("doc2".to_string(), &freq2);
19    vectorizer.del_doc(&"doc1".to_string());
20    vectorizer.add_doc("doc3".to_string(), &freq1);
21
22    // similarity search
23    let mut query_tokens = TokenFrequency::new();
24    query_tokens.add_tokens(&["rust", "高速"]);
25    let algorithm = SimilarityAlgorithm::CosineSimilarity;
26    let mut result = vectorizer.similarity(&query_tokens, &algorithm);
27    result.sort_by_score();
28
29    // print result
30    result.list.iter().for_each(|(k, s, l)| {
31        println!("doc: {}, score: {}, length: {}", k, s, l);
32    });
33    // debug
34    println!("result count: {}", result.list.len());
35    println!("{:?}", vectorizer);
36}

Source

pub fn add_token(&mut self, token: &str) -> &mut Self

Add a token

§Arguments

token - Token to add

Source

pub fn add_tokens<T>(&mut self, tokens: &[T]) -> &mut Self
where T: AsRef<str>,

Add multiple tokens

§Arguments

tokens - Slice of tokens to add

Examples found in repository ?

examples/basic.rs (line 11)

5fn main() {
6    // build corpus
7    let corpus = Arc::new(Corpus::new());
8
9    // add documents
10    let mut freq1 = TokenFrequency::new();
11    freq1.add_tokens(&["rust", "高速", "並列", "rust"]);
12    let mut freq2 = TokenFrequency::new();
13    freq2.add_tokens(&["rust", "柔軟", "安全", "rust"]);
14
15    // build query
16    let mut vectorizer: TFIDFVectorizer<u16> = TFIDFVectorizer::new(corpus);    
17    vectorizer.add_doc("doc1".to_string(), &freq1);
18    vectorizer.add_doc("doc2".to_string(), &freq2);
19    vectorizer.del_doc(&"doc1".to_string());
20    vectorizer.add_doc("doc3".to_string(), &freq1);
21
22    // similarity search
23    let mut query_tokens = TokenFrequency::new();
24    query_tokens.add_tokens(&["rust", "高速"]);
25    let algorithm = SimilarityAlgorithm::CosineSimilarity;
26    let mut result = vectorizer.similarity(&query_tokens, &algorithm);
27    result.sort_by_score();
28
29    // print result
30    result.list.iter().for_each(|(k, s, l)| {
31        println!("doc: {}, score: {}, length: {}", k, s, l);
32    });
33    // debug
34    println!("result count: {}", result.list.len());
35    println!("{:?}", vectorizer);
36}