Struct TFIDFVectorizer 

pub struct TFIDFVectorizer<N = f32, K = String, E = DefaultTFIDFEngine>
where
    N: Num + Copy,
    E: TFIDFEngine<N>,
{
    pub documents: Vec<TFVector<N, K>>,
    pub token_dim_sample: IndexSet<String, RandomState>,
    pub corpus_ref: Arc<Corpus>,
    pub idf: IDFVector<N>,
    /* private fields */
}

Fields

documents: Vec<TFVector<N, K>>

Document’s TF Vector

token_dim_sample: IndexSet<String, RandomState>

TF Vector’s token dimension sample

corpus_ref: Arc<Corpus>

Corpus reference

idf: IDFVector<N>

IDF Vector

Implementations

impl<N, K, E> TFIDFVectorizer<N, K, E>

where K: Clone, N: Num + Copy + Into<f64> + DeNormalizer, E: TFIDFEngine<N>,

pub fn similarity( &mut self, freq: &TokenFrequency, algorithm: &SimilarityAlgorithm, ) -> Hits<K>

Calculate similarity scores based on query token frequency Uses the specified similarity algorithm Calls update_idf() to use the latest IDF vector

Examples found in repository

examples/basic.rs (line 24)

fn main() {
    // build corpus
    let corpus = Arc::new(Corpus::new());

    // add documents
    let mut freq1 = TokenFrequency::new();
    freq1.add_tokens(&["rust", "高速", "並列", "rust"]);
    let mut freq2 = TokenFrequency::new();
    freq2.add_tokens(&["rust", "柔軟", "安全", "rust"]);

    // build query
    let mut vectorizer: TFIDFVectorizer<u16> = TFIDFVectorizer::new(corpus);    
    vectorizer.add_doc("doc1".to_string(), &freq1);
    vectorizer.add_doc("doc2".to_string(), &freq2);

    // similarity search
    let mut query_tokens = TokenFrequency::new();
    query_tokens.add_tokens(&["rust", "高速"]);
    let algorithm = SimilarityAlgorithm::CosineSimilarity;
    let mut result = vectorizer.similarity(&query_tokens, &algorithm);
    result.sort_by_score();

    // print result
    result.list.iter().for_each(|(k, s, l)| {
        println!("doc: {}, score: {}, length: {}", k, s, l);
    });
    // debug
    println!("result count: {}", result.list.len());
    println!("{:?}", vectorizer);
}

pub fn similarity_uncheck_idf( &self, freq: &TokenFrequency, algorithm: &SimilarityAlgorithm, ) -> Hits<K>

Calculate similarity scores based on query token frequency Uses the specified similarity algorithm Does not check the IDF vector (can be called with immutable reference) Call update_idf() manually if needed

impl<N, K, E> TFIDFVectorizer<N, K, E>

where N: Num + Copy, E: TFIDFEngine<N>,

pub fn new(corpus_ref: Arc<Corpus>) -> Self

Create a new TFIDFVectorizer instance

Examples found in repository

examples/basic.rs (line 16)

fn main() {
    // build corpus
    let corpus = Arc::new(Corpus::new());

    // add documents
    let mut freq1 = TokenFrequency::new();
    freq1.add_tokens(&["rust", "高速", "並列", "rust"]);
    let mut freq2 = TokenFrequency::new();
    freq2.add_tokens(&["rust", "柔軟", "安全", "rust"]);

    // build query
    let mut vectorizer: TFIDFVectorizer<u16> = TFIDFVectorizer::new(corpus);    
    vectorizer.add_doc("doc1".to_string(), &freq1);
    vectorizer.add_doc("doc2".to_string(), &freq2);

    // similarity search
    let mut query_tokens = TokenFrequency::new();
    query_tokens.add_tokens(&["rust", "高速"]);
    let algorithm = SimilarityAlgorithm::CosineSimilarity;
    let mut result = vectorizer.similarity(&query_tokens, &algorithm);
    result.sort_by_score();

    // print result
    result.list.iter().for_each(|(k, s, l)| {
        println!("doc: {}, score: {}, length: {}", k, s, l);
    });
    // debug
    println!("result count: {}", result.list.len());
    println!("{:?}", vectorizer);
}

pub fn set_corpus_ref(&mut self, corpus_ref: Arc<Corpus>)

set corpus reference and recalculate idf

pub fn update_idf(&mut self)

Corpusに変更があればIDFを再計算する

impl<N, K, E> TFIDFVectorizer<N, K, E>

where N: Num + Copy, E: TFIDFEngine<N>,

pub fn add_doc(&mut self, doc_id: K, doc: &TokenFrequency)

Add a document The immediately referenced Corpus is also updated

Examples found in repository

examples/basic.rs (line 17)

fn main() {
    // build corpus
    let corpus = Arc::new(Corpus::new());

    // add documents
    let mut freq1 = TokenFrequency::new();
    freq1.add_tokens(&["rust", "高速", "並列", "rust"]);
    let mut freq2 = TokenFrequency::new();
    freq2.add_tokens(&["rust", "柔軟", "安全", "rust"]);

    // build query
    let mut vectorizer: TFIDFVectorizer<u16> = TFIDFVectorizer::new(corpus);    
    vectorizer.add_doc("doc1".to_string(), &freq1);
    vectorizer.add_doc("doc2".to_string(), &freq2);

    // similarity search
    let mut query_tokens = TokenFrequency::new();
    query_tokens.add_tokens(&["rust", "高速"]);
    let algorithm = SimilarityAlgorithm::CosineSimilarity;
    let mut result = vectorizer.similarity(&query_tokens, &algorithm);
    result.sort_by_score();

    // print result
    result.list.iter().for_each(|(k, s, l)| {
        println!("doc: {}, score: {}, length: {}", k, s, l);
    });
    // debug
    println!("result count: {}", result.list.len());
    println!("{:?}", vectorizer);
}

pub fn get_tf(&self, key: &K) -> Option<&TFVector<N, K>>

where K: PartialEq,

Get TFVector by document ID

pub fn get_tf_into_token_freq(&self, key: &K) -> Option<TokenFrequency>

where K: PartialEq, N: Into<f64>,

Get TokenFrequency by document ID If quantized, there may be some error Words not included in the corpus are ignored

pub fn contains_doc(&self, key: &K) -> bool

where K: PartialEq,

Check if a document with the given ID exists

pub fn contains_token(&self, token: &str) -> bool

Check if the token exists in the token dimension sample

pub fn contains_tokens_from_freq(&self, freq: &TokenFrequency) -> bool

Check if all tokens in the given TokenFrequency exist in the token dimension sample

Trait Implementations

impl<N, K: Debug, E> Debug for TFIDFVectorizer<N, K, E>

where N: Num + Copy + Debug, E: TFIDFEngine<N> + Debug,

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<N, K, E> Serialize for TFIDFVectorizer<N, K, E>

where N: Num + Copy + Serialize, K: Serialize, E: TFIDFEngine<N>,

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

where S: Serializer,

Serialize TFIDFVectorizer. This struct contains references, so they are excluded from serialization. Use TFIDFData for deserialization.