Struct TFIDFVectorizer 

pub struct TFIDFVectorizer<N = f32, K = String, E = DefaultTFIDFEngine>
where
    N: Num + Copy + Into<f64> + Send + Sync,
    E: TFIDFEngine<N, K> + Send + Sync,
    K: Clone + Send + Sync + Eq + Hash,
{
    pub documents: IndexMap<KeyRc<K>, TFVector<N>>,
    pub token_dim_rev_index: IndexMap<Box<str>, Vec<KeyRc<K>>>,
    pub corpus_ref: Arc<Corpus>,
    pub idf_cache: IDFVector,
    /* private fields */
}

Fields

documents: IndexMap<KeyRc<K>, TFVector<N>>

Document’s TF Vector

token_dim_rev_index: IndexMap<Box<str>, Vec<KeyRc<K>>>

TF Vector’s token dimension sample and reverse index

corpus_ref: Arc<Corpus>

Corpus reference

idf_cache: IDFVector

IDF Vector

Implementations

impl<N, K, E> TFIDFVectorizer<N, K, E>

where K: Clone + Sync + Send + PartialEq + Eq + Hash, N: Num + Copy + Into<f64> + DeNormalizer + Send + Sync, E: TFIDFEngine<N, K> + Send + Sync,

pub fn similarity( &mut self, freq: &TokenFrequency, algorithm: &SimilarityAlgorithm, ) -> Hits<K>

Calculate similarity scores based on query token frequency Uses the specified similarity algorithm Calls update_idf() to use the latest IDF vector

pub fn similarity_uncheck_idf( &self, freq: &TokenFrequency, algorithm: &SimilarityAlgorithm, ) -> Hits<K>

Calculate similarity scores based on query token frequency Uses the specified similarity algorithm Does not check the IDF vector (can be called with immutable reference) Call update_idf() manually if needed

pub fn similarity_full_scan( &mut self, freq: &TokenFrequency, algorithm: &SimilarityAlgorithm, ) -> Hits<K>

pub fn similarity_full_scan_uncheck_idf( &self, freq: &TokenFrequency, algorithm: &SimilarityAlgorithm, ) -> Hits<K>

impl<N, K, E> TFIDFVectorizer<N, K, E>

where N: Num + Copy + Into<f64> + Send + Sync, E: TFIDFEngine<N, K> + Send + Sync, K: Clone + Send + Sync + Eq + Hash,

pub fn new(corpus_ref: Arc<Corpus>) -> Self

Create a new TFIDFVectorizer instance

pub fn set_corpus_ref(&mut self, corpus_ref: Arc<Corpus>)

set corpus reference and recalculate idf

pub fn update_idf(&mut self)

Corpusに変更があればIDFを再計算する

impl<N, K, E> TFIDFVectorizer<N, K, E>

where N: Num + Copy + Into<f64> + Send + Sync, E: TFIDFEngine<N, K> + Send + Sync, K: PartialEq + Clone + Send + Sync + Eq + Hash,

pub fn add_doc(&mut self, key: K, doc: &TokenFrequency)

Add a document The immediately referenced Corpus is also updated

pub fn del_doc(&mut self, key: &K)

where K: PartialEq,

pub fn get_tf(&self, key: &K) -> Option<&TFVector<N>>

where K: Eq + Hash,

Get TFVector by document ID

pub fn get_tf_into_token_freq(&self, key: &K) -> Option<TokenFrequency>

Get TokenFrequency by document ID If quantized, there may be some error Words not included in the corpus are ignored

pub fn contains_doc(&self, key: &K) -> bool

where K: PartialEq,

Check if a document with the given ID exists

pub fn contains_token(&self, token: &str) -> bool

Check if the token exists in the token dimension sample

pub fn contains_tokens_from_freq(&self, freq: &TokenFrequency) -> bool

Check if all tokens in the given TokenFrequency exist in the token dimension sample

pub fn doc_num(&self) -> usize

Trait Implementations

impl<N, K, E> Clone for TFIDFVectorizer<N, K, E>

where N: Num + Copy + Into<f64> + Send + Sync + Clone, E: TFIDFEngine<N, K> + Send + Sync + Clone, K: Clone + Send + Sync + Eq + Hash + Clone,

fn clone(&self) -> TFIDFVectorizer<N, K, E>

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<N, K, E> Debug for TFIDFVectorizer<N, K, E>

where N: Num + Copy + Into<f64> + Send + Sync + Debug, E: TFIDFEngine<N, K> + Send + Sync + Debug, K: Clone + Send + Sync + Eq + Hash + Debug,

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<N, K, E> Serialize for TFIDFVectorizer<N, K, E>

where N: Num + Copy + Serialize + Into<f64> + Send + Sync, K: Serialize + Clone + Send + Sync + Eq + Hash, E: TFIDFEngine<N, K>,

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

where S: Serializer,

Serialize TFIDFVectorizer. This struct contains references, so they are excluded from serialization. Use TFIDFData for deserialization.