Struct rnltk::document::DocumentTermFrequencies
source · pub struct DocumentTermFrequencies {
pub document_term_frequencies: GenericMatrix,
}
Expand description
Struct for holding the matrix of document_term_frequencies
Fields§
§document_term_frequencies: GenericMatrix
Implementations§
source§impl DocumentTermFrequencies
impl DocumentTermFrequencies
sourcepub fn new(document_term_frequencies: GenericMatrix) -> Self
pub fn new(document_term_frequencies: GenericMatrix) -> Self
Creates new instance of DocumentTermFrequencies from a DMatrix
.
Examples
use rnltk::document::DocumentTermFrequencies;
use nalgebra::DMatrix;
let term_frequencies = DMatrix::from_row_slice(11, 4, &[1., 0., 0., 0.,
0., 1., 0., 0.,
0., 0., 1., 1.,
1., 0., 0., 0.,
1., 0., 0., 0.,
2., 0., 0., 0.,
0., 0., 0., 1.,
0., 1., 0., 0.,
0., 0., 0., 1.,
0., 0., 1., 0.,
1., 0., 0., 0.,]);
let document_term_frequencies: DocumentTermFrequencies = DocumentTermFrequencies::new(term_frequencies);
Examples found in repository?
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
fn main() {
let document1 = "It is a far, far better thing I do, than I have ever done";
let document2 = "Call me Ishmael";
let document3 = "Is this a dagger I see before me?";
let document4 = "O happy dagger";
let documents = vec![document1, document2, document3, document4];
let stop_words = token::get_stop_words();
let token_config = token::TokenConfig {
remove_stop_words: true,
stem: true,
stop_words
};
let documents_term_frequencies = token::get_term_frequencies_from_sentences_configurable(&documents, token_config);
let mut all_term_frequencies: Vec<f64> = vec![];
documents_term_frequencies.iter().for_each(|term_frequencies| {
all_term_frequencies.extend(term_frequencies.values().into_iter());
});
let nrows = documents_term_frequencies[0].values().len();
let ncols = documents.len();
let document_term_frequencies = DMatrix::from_vec(nrows, ncols, all_term_frequencies);
let document_term_frequency_matrix = document::DocumentTermFrequencies::new(document_term_frequencies);
let tfidf_matrix = document_term_frequency_matrix.get_tfidf_from_term_frequencies();
let cosine_similarity = tfidf_matrix.get_cosine_similarity_from_tfidf();
let cosine_similarity_matrix = cosine_similarity.get_cosine_similarity_matrix();
println!("COSINE SIMILARITY MATRIX");
for row_index in 0..ncols {
println!(
"Document {} {:.2} {:.2} {:.2} {:.2}",
row_index + 1,
&cosine_similarity_matrix[(row_index, 0)],
&cosine_similarity_matrix[(row_index, 1)],
&cosine_similarity_matrix[(row_index, 2)],
&cosine_similarity_matrix[(row_index, 3)]
)
}
println!(" Document 1 Document 2 Document 3 Document 4");
println!("\n-----------------------------\n");
let lsa_cosine_similarity = tfidf_matrix.get_lsa_cosine_similarity_from_tfidf(2).unwrap();
let lsa_cosine_similarity_matrix = lsa_cosine_similarity.get_lsa_cosine_similarity_matrix();
println!("LSA COSINE SIMILARITY MATRIX");
for row_index in 0..ncols {
println!(
"Document {} {:.2} {:.2} {:.2} {:.2}",
row_index + 1,
&lsa_cosine_similarity_matrix[(row_index, 0)],
&lsa_cosine_similarity_matrix[(row_index, 1)],
&lsa_cosine_similarity_matrix[(row_index, 2)],
&lsa_cosine_similarity_matrix[(row_index, 3)]
)
}
println!(" Document 1 Document 2 Document 3 Document 4");
}
sourcepub fn get_tfidf_from_term_frequencies(&self) -> TfidfMatrix
pub fn get_tfidf_from_term_frequencies(&self) -> TfidfMatrix
Gets the Term Frequency–Inverse Document Frequency (TF-IDF) matrix of the
DocumentTermFrequencies
’s document_term_frequencies
.
Creating a TF-IDF matrix takes place over two steps. The first step is applying a weight, \(w_{i,j}\), for every term, \(t_i\), in the document, \(D_j\). \(w_{i,j}\) is defined as \(tf_{i,j} \times idf_i\), where \(tf_{i,j}\) is the number of occurrences of \(t_i\) in \(D_j\), and \(idf_i\) is the log of inverse fraction of documents \(n_i\) that contain at least one occurrence of \(t_i, idf_i = ln(n / n_i)\). The second step takes the weighted matrix and then normalizes each document vector in order to remove the influence of document length.
Examples
use rnltk::document::DocumentTermFrequencies;
use rnltk::sample_data;
let document_term_frequencies: DocumentTermFrequencies = DocumentTermFrequencies::new(sample_data::get_term_frequencies());
let tfidf_matrix = document_term_frequencies.get_tfidf_from_term_frequencies();
Examples found in repository?
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
fn main() {
let document1 = "It is a far, far better thing I do, than I have ever done";
let document2 = "Call me Ishmael";
let document3 = "Is this a dagger I see before me?";
let document4 = "O happy dagger";
let documents = vec![document1, document2, document3, document4];
let stop_words = token::get_stop_words();
let token_config = token::TokenConfig {
remove_stop_words: true,
stem: true,
stop_words
};
let documents_term_frequencies = token::get_term_frequencies_from_sentences_configurable(&documents, token_config);
let mut all_term_frequencies: Vec<f64> = vec![];
documents_term_frequencies.iter().for_each(|term_frequencies| {
all_term_frequencies.extend(term_frequencies.values().into_iter());
});
let nrows = documents_term_frequencies[0].values().len();
let ncols = documents.len();
let document_term_frequencies = DMatrix::from_vec(nrows, ncols, all_term_frequencies);
let document_term_frequency_matrix = document::DocumentTermFrequencies::new(document_term_frequencies);
let tfidf_matrix = document_term_frequency_matrix.get_tfidf_from_term_frequencies();
let cosine_similarity = tfidf_matrix.get_cosine_similarity_from_tfidf();
let cosine_similarity_matrix = cosine_similarity.get_cosine_similarity_matrix();
println!("COSINE SIMILARITY MATRIX");
for row_index in 0..ncols {
println!(
"Document {} {:.2} {:.2} {:.2} {:.2}",
row_index + 1,
&cosine_similarity_matrix[(row_index, 0)],
&cosine_similarity_matrix[(row_index, 1)],
&cosine_similarity_matrix[(row_index, 2)],
&cosine_similarity_matrix[(row_index, 3)]
)
}
println!(" Document 1 Document 2 Document 3 Document 4");
println!("\n-----------------------------\n");
let lsa_cosine_similarity = tfidf_matrix.get_lsa_cosine_similarity_from_tfidf(2).unwrap();
let lsa_cosine_similarity_matrix = lsa_cosine_similarity.get_lsa_cosine_similarity_matrix();
println!("LSA COSINE SIMILARITY MATRIX");
for row_index in 0..ncols {
println!(
"Document {} {:.2} {:.2} {:.2} {:.2}",
row_index + 1,
&lsa_cosine_similarity_matrix[(row_index, 0)],
&lsa_cosine_similarity_matrix[(row_index, 1)],
&lsa_cosine_similarity_matrix[(row_index, 2)],
&lsa_cosine_similarity_matrix[(row_index, 3)]
)
}
println!(" Document 1 Document 2 Document 3 Document 4");
}
Trait Implementations§
source§impl Clone for DocumentTermFrequencies
impl Clone for DocumentTermFrequencies
source§fn clone(&self) -> DocumentTermFrequencies
fn clone(&self) -> DocumentTermFrequencies
1.0.0 · source§fn clone_from(&mut self, source: &Self)
fn clone_from(&mut self, source: &Self)
source
. Read moreAuto Trait Implementations§
impl RefUnwindSafe for DocumentTermFrequencies
impl Send for DocumentTermFrequencies
impl Sync for DocumentTermFrequencies
impl Unpin for DocumentTermFrequencies
impl UnwindSafe for DocumentTermFrequencies
Blanket Implementations§
source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere T: ?Sized,
source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
§impl<SS, SP> SupersetOf<SS> for SPwhere
SS: SubsetOf<SP>,
impl<SS, SP> SupersetOf<SS> for SPwhere SS: SubsetOf<SP>,
§fn to_subset(&self) -> Option<SS>
fn to_subset(&self) -> Option<SS>
self
from the equivalent element of its
superset. Read more§fn is_in_subset(&self) -> bool
fn is_in_subset(&self) -> bool
self
is actually part of its subset T
(and can be converted to it).§fn to_subset_unchecked(&self) -> SS
fn to_subset_unchecked(&self) -> SS
self.to_subset
but without any property checks. Always succeeds.§fn from_subset(element: &SS) -> SP
fn from_subset(element: &SS) -> SP
self
to the equivalent element of its superset.