Struct TFIDFVectorizer 

pub struct TFIDFVectorizer<'a, N = f32, K = String, E = DefaultTFIDFEngine, C = DefaultCompare>
where
    N: Num + Copy,
    E: TFIDFEngine<N>,
    C: Compare<N>,
{
    pub documents: Vec<TFVector<N, K>>,
    pub token_dim_sample: Vec<String>,
    pub token_dim_set: HashSet<String>,
    pub corpus_ref: &'a Corpus,
    pub idf: IDFVector<N>,
    /* private fields */
}

Fields

documents: Vec<TFVector<N, K>>

ドキュメントのTFベクトル

token_dim_sample: Vec<String>

TFベクトルのトークンの次元サンプル

token_dim_set: HashSet<String>

高速存在判定用の語彙セット (token_dim_sample と常に同期)

corpus_ref: &'a Corpus

コーパスの参照

idf: IDFVector<N>

IDFベクトル

Implementations

impl<'a, N, K, E> TFIDFVectorizer<'a, N, K, E>

where K: Clone, N: Num + Copy, E: TFIDFEngine<N>, DefaultCompare: Compare<N>,

pub fn similarity(&self, query: SimilarityQuery) -> Hits<K>

Examples found in repository

examples/basic.rs (line 17)

fn main() {
    let corpus = Corpus::new();
    let mut freq1 = TokenFrequency::new();
    freq1.add_tokens(&["rust", "高速", "並列", "rust"]);
    let mut freq2 = TokenFrequency::new();
    freq2.add_tokens(&["rust", "柔軟", "安全", "rust"]);

    let mut vectorizer: TFIDFVectorizer<u16> = TFIDFVectorizer::new(&corpus);    
    vectorizer.add_doc("doc1".to_string(), &freq1);
    vectorizer.add_doc("doc2".to_string(), &freq2);

    let mut query_tokens = TokenFrequency::new();
    query_tokens.add_tokens(&["rust", "高速"]);
    let query = SimilarityQuery::CosineSimilarity(query_tokens);
    let mut result = vectorizer.similarity(query);
    result.sort_by_score();

    result.list.iter().for_each(|(k, s)| {
        println!("doc: {}, score: {}", k, s);
    });    
}

impl<'a, N, K, E, C> TFIDFVectorizer<'a, N, K, E, C>

where N: Num + Copy, E: TFIDFEngine<N>, C: Compare<N>,

pub fn new(corpus_ref: &'a Corpus) -> Self

Create a new TFIDFVectorizer instance

Examples found in repository

examples/basic.rs (line 10)

fn main() {
    let corpus = Corpus::new();
    let mut freq1 = TokenFrequency::new();
    freq1.add_tokens(&["rust", "高速", "並列", "rust"]);
    let mut freq2 = TokenFrequency::new();
    freq2.add_tokens(&["rust", "柔軟", "安全", "rust"]);

    let mut vectorizer: TFIDFVectorizer<u16> = TFIDFVectorizer::new(&corpus);    
    vectorizer.add_doc("doc1".to_string(), &freq1);
    vectorizer.add_doc("doc2".to_string(), &freq2);

    let mut query_tokens = TokenFrequency::new();
    query_tokens.add_tokens(&["rust", "高速"]);
    let query = SimilarityQuery::CosineSimilarity(query_tokens);
    let mut result = vectorizer.similarity(query);
    result.sort_by_score();

    result.list.iter().for_each(|(k, s)| {
        println!("doc: {}, score: {}", k, s);
    });    
}

pub fn set_corpus_ref(&mut self, corpus_ref: &'a Corpus)

Corpusを指定する

pub fn update_idf(&mut self)

Corpusに変更があればIDFを再計算する

impl<'a, N, K, E, C> TFIDFVectorizer<'a, N, K, E, C>

where N: Num + Copy, E: TFIDFEngine<N>, C: Compare<N>,

pub fn add_doc(&mut self, doc_id: K, doc: &TokenFrequency)

ドキュメントを追加します 即時参照されているCorpusも更新されます

Examples found in repository

examples/basic.rs (line 11)

fn main() {
    let corpus = Corpus::new();
    let mut freq1 = TokenFrequency::new();
    freq1.add_tokens(&["rust", "高速", "並列", "rust"]);
    let mut freq2 = TokenFrequency::new();
    freq2.add_tokens(&["rust", "柔軟", "安全", "rust"]);

    let mut vectorizer: TFIDFVectorizer<u16> = TFIDFVectorizer::new(&corpus);    
    vectorizer.add_doc("doc1".to_string(), &freq1);
    vectorizer.add_doc("doc2".to_string(), &freq2);

    let mut query_tokens = TokenFrequency::new();
    query_tokens.add_tokens(&["rust", "高速"]);
    let query = SimilarityQuery::CosineSimilarity(query_tokens);
    let mut result = vectorizer.similarity(query);
    result.sort_by_score();

    result.list.iter().for_each(|(k, s)| {
        println!("doc: {}, score: {}", k, s);
    });    
}

Trait Implementations

impl<'a, N, K: Debug, E, C> Debug for TFIDFVectorizer<'a, N, K, E, C>

where N: Num + Copy + Debug, E: TFIDFEngine<N> + Debug, C: Compare<N> + Debug,

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'a, N, K, E, C> Serialize for TFIDFVectorizer<'a, N, K, E, C>

where N: Num + Copy + Serialize, K: Serialize, E: TFIDFEngine<N>, C: Compare<N>,

fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

where S: Serializer,

TFIDFVectorizerをシリアライズします これは参照を含んでるため、それを除外したものになります。 デシリアライズするにはTFIDFDataを使用してください。