Struct TokenFrequency 

pub struct TokenFrequency { /* private fields */ }

TokenFrequency struct Manages the frequency of token occurrences. Counts the number of times each token appears.

Examples

use crate::tf_idf_vectorizer::vectorizer::token::TokenFrequency;
let mut token_freq = TokenFrequency::new();
token_freq.add_token("token1");
token_freq.add_token("token2");
token_freq.add_token("token1");

assert_eq!(token_freq.token_count("token1"), 2);

Implementations

impl TokenFrequency

Implementation for adding and removing tokens

pub fn new() -> Self

Create a new TokenFrequency

Examples found in repository

examples/basic.rs (line 10)

fn main() {
    // build corpus
    let corpus = Arc::new(Corpus::new());

    // add documents
    let mut freq1 = TokenFrequency::new();
    freq1.add_tokens(&["rust", "高速", "並列", "rust"]);
    let mut freq2 = TokenFrequency::new();
    freq2.add_tokens(&["rust", "柔軟", "安全", "rust"]);

    // build query
    let mut vectorizer: TFIDFVectorizer<u16> = TFIDFVectorizer::new(corpus);    
    vectorizer.add_doc("doc1".to_string(), &freq1);
    vectorizer.add_doc("doc2".to_string(), &freq2);

    // similarity search
    let mut query_tokens = TokenFrequency::new();
    query_tokens.add_tokens(&["rust", "高速"]);
    let algorithm = SimilarityAlgorithm::CosineSimilarity;
    let mut result = vectorizer.similarity(&query_tokens, &algorithm);
    result.sort_by_score();

    // print result
    result.list.iter().for_each(|(k, s, l)| {
        println!("doc: {}, score: {}, length: {}", k, s, l);
    });
    // debug
    println!("result count: {}", result.list.len());
    println!("{:?}", vectorizer);
}

pub fn add_token(&mut self, token: &str) -> &mut Self

Add a token

Arguments

• token - Token to add

pub fn add_tokens<T>(&mut self, tokens: &[T]) -> &mut Self

where T: AsRef<str>,

Add multiple tokens

Arguments

• tokens - Slice of tokens to add

Examples found in repository

examples/basic.rs (line 11)

fn main() {
    // build corpus
    let corpus = Arc::new(Corpus::new());

    // add documents
    let mut freq1 = TokenFrequency::new();
    freq1.add_tokens(&["rust", "高速", "並列", "rust"]);
    let mut freq2 = TokenFrequency::new();
    freq2.add_tokens(&["rust", "柔軟", "安全", "rust"]);

    // build query
    let mut vectorizer: TFIDFVectorizer<u16> = TFIDFVectorizer::new(corpus);    
    vectorizer.add_doc("doc1".to_string(), &freq1);
    vectorizer.add_doc("doc2".to_string(), &freq2);

    // similarity search
    let mut query_tokens = TokenFrequency::new();
    query_tokens.add_tokens(&["rust", "高速"]);
    let algorithm = SimilarityAlgorithm::CosineSimilarity;
    let mut result = vectorizer.similarity(&query_tokens, &algorithm);
    result.sort_by_score();

    // print result
    result.list.iter().for_each(|(k, s, l)| {
        println!("doc: {}, score: {}, length: {}", k, s, l);
    });
    // debug
    println!("result count: {}", result.list.len());
    println!("{:?}", vectorizer);
}

pub fn sub_token(&mut self, token: &str) -> &mut Self

Subtract a token

Arguments

• token - Token to subtract

pub fn sub_tokens<T>(&mut self, tokens: &[T]) -> &mut Self

where T: AsRef<str>,

Subtract multiple tokens

Arguments

• tokens - Slice of tokens to subtract

pub fn set_token_count(&mut self, token: &str, count: u64) -> &mut Self

Set the occurrence count for a token

Arguments

• token - Token
• count - Occurrence count

pub fn add_tokens_from_freq(&mut self, other: &TokenFrequency) -> &mut Self

Merge with another TokenFrequency

Arguments

• other - Another TokenFrequency to merge with

pub fn scale(&mut self, scalar: f64) -> &mut Self

Scale the token counts by a scalar

Arguments

• scalar - Scalar to scale by

impl TokenFrequency

Implementation for retrieving information from TokenFrequency

pub fn token_count_vector(&self) -> Vec<(String, u64)>

Get a vector of all tokens and their counts

Returns

• Vec<(String, u64)> - Vector of tokens and their counts

pub fn token_count_vector_ref_str(&self) -> Vec<(&str, u64)>

Get a vector of all tokens and their counts (as &str)

Returns

• Vec<(&str, u64)> - Vector of tokens and their counts

pub fn token_count_hashmap_ref_str(&self) -> HashMap<&str, u64, RandomState>

Get a hashmap of all tokens and their counts (as &str)

Returns

• HashMap<&str, u64> - HashMap of tokens and their counts

pub fn token_sum(&self) -> u64

Get the total count of all tokens

Returns

• u64 - Total token count

pub fn token_count(&self, token: &str) -> u64

Get the occurrence count for a specific token

Arguments

• token - Token

Returns

• u64 - Occurrence count for the token

pub fn most_frequent_tokens_vector(&self) -> Vec<(String, u64)>

Get the most frequent tokens If multiple tokens have the same count, all are returned

Returns

• Vec<(String, u64)> - Vector of most frequent tokens and their counts

pub fn most_frequent_token_count(&self) -> u64

Get the count of the most frequent token

Returns

• u64 - Count of the most frequent token

pub fn contains_token(&self, token: &str) -> bool

Check if a token exists

Arguments

• token - Token

Returns

• bool - true if the token exists, false otherwise

pub fn token_set(&self) -> Vec<String>

Get the set of tokens

Returns

• Vec<String> - Set of tokens

pub fn token_set_ref_str(&self) -> Vec<&str>

Get the set of tokens (as &str)

Returns

• Vec<&str> - Set of tokens

pub fn token_hashset(&self) -> HashSet<String, RandomState>

Get the set of tokens as a HashSet

Returns

• HashSet<String> - Set of tokens

pub fn token_hashset_ref_str(&self) -> HashSet<&str, RandomState>

Get the set of tokens as a HashSet (as &str)

Returns

• HashSet<&str> - Set of tokens

pub fn token_num(&self) -> usize

Get the number of unique tokens

Returns

• usize - Number of unique tokens

pub fn remove_stop_tokens(&mut self, stop_tokens: &[&str]) -> u64

Remove stop tokens

Arguments

• stop_tokens - Slice of stop tokens to remove

Returns

• u64 - Total count of removed tokens

pub fn remove_tokens_by<F>(&mut self, condition: F) -> u64

where F: Fn(&str, &u64) -> bool,

Remove tokens by a condition

Arguments

• condition - Closure to determine which tokens to remove

Returns

• u64 - Total count of removed tokens

pub fn sorted_frequency_vector(&self) -> Vec<(String, u64)>

Get a vector of tokens sorted by frequency (descending)

Returns

• Vec<(String, u64)> - Vector of tokens sorted by frequency

pub fn sorted_dict_order_vector(&self) -> Vec<(String, u64)>

Get a vector of tokens sorted by dictionary order (ascending)

Returns

• Vec<(String, u64)> - Vector of tokens sorted by dictionary order

pub fn unique_token_ratio(&self) -> f64

Calculate the diversity of tokens 1.0 indicates complete diversity, 0.0 indicates no diversity

Returns

• f64 - Diversity of tokens

pub fn probability_vector(&self) -> Vec<(String, f64)>

Get the probability distribution P(token) (owned String version) Returns an empty vector if total is 0

pub fn probability_vector_ref_str(&self) -> Vec<(&str, f64)>

Get the probability distribution P(token) (as &str) Returns an empty vector if total is 0

pub fn probability(&self, token: &str) -> f64

Get the probability P(token) for a specific token Returns 0.0 if total is 0

pub fn clear(&mut self)

Reset all counts

pub fn shrink_to_fit(&mut self)

Shrink internal storage to fit current size

Trait Implementations

impl Clone for TokenFrequency

fn clone(&self) -> TokenFrequency

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TokenFrequency

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for TokenFrequency

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl<T> From<&[T]> for TokenFrequency

where T: AsRef<str>,

fn from(tokens: &[T]) -> Self

Converts to this type from the input type.

impl Serialize for TokenFrequency

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.