pub struct TokenFrequency { /* private fields */ }Expand description
TokenFrequency 構造体
Implementations§
Source§impl TokenFrequency
Tokenの追加、削除の実装
impl TokenFrequency
Tokenの追加、削除の実装
pub fn new() -> Self
Sourcepub fn add_tokens<T>(&mut self, tokens: &[T]) -> &mut Self
pub fn add_tokens<T>(&mut self, tokens: &[T]) -> &mut Self
Sourcepub fn set_token_count(&mut self, token: &str, count: u32) -> &mut Self
👎Deprecated: countに0を指定した場合、token_numはそれを1つのユニークなtokenとしてカウントします。
このメソッドは、token_numのカウントを不正にする可能性があるため、非推奨です
pub fn set_token_count(&mut self, token: &str, count: u32) -> &mut Self
Source§impl TokenFrequency
TF-calculationの実装
impl TokenFrequency
TF-calculationの実装
Source§impl TokenFrequency
IDF-calculationの実装
impl TokenFrequency
IDF-calculationの実装
Sourcepub fn idf_vector<N>(&self, total_doc_count: u64) -> Vec<(String, N)>
pub fn idf_vector<N>(&self, total_doc_count: u64) -> Vec<(String, N)>
Sourcepub fn idf_vector_ref_str<N>(&self, total_doc_count: u64) -> Vec<(&str, N)>
pub fn idf_vector_ref_str<N>(&self, total_doc_count: u64) -> Vec<(&str, N)>
Sourcepub fn idf_hashmap<N>(&self, total_doc_count: u64) -> HashMap<String, N>
pub fn idf_hashmap<N>(&self, total_doc_count: u64) -> HashMap<String, N>
Sourcepub fn idf_hashmap_ref_str<N>(&self, total_doc_count: u64) -> HashMap<&str, N>
pub fn idf_hashmap_ref_str<N>(&self, total_doc_count: u64) -> HashMap<&str, N>
Source§impl TokenFrequency
TokenFrequencyの情報を取得するための実装
impl TokenFrequency
TokenFrequencyの情報を取得するための実装
Sourcepub fn token_count_vector(&self) -> Vec<(String, u32)>
pub fn token_count_vector(&self) -> Vec<(String, u32)>
Sourcepub fn token_count_vector_ref_str(&self) -> Vec<(&str, u32)>
pub fn token_count_vector_ref_str(&self) -> Vec<(&str, u32)>
Sourcepub fn token_count_hashmap_ref_str(&self) -> HashMap<&str, u32>
pub fn token_count_hashmap_ref_str(&self) -> HashMap<&str, u32>
Sourcepub fn token_total_count(&self) -> u64
pub fn token_total_count(&self) -> u64
Sourcepub fn token_count(&self, token: &str) -> u32
pub fn token_count(&self, token: &str) -> u32
Sourcepub fn most_frequent_tokens_vector(&self) -> Vec<(String, u32)>
pub fn most_frequent_tokens_vector(&self) -> Vec<(String, u32)>
もっとも多く出現したtokenを取得します 同じ出現回数のtokenが複数ある場合は、すべてのtokenを取得します
§Returns
Vec<(String, u32)>- トークンとその出現回数のベクタ
Sourcepub fn most_frequent_token_count(&self) -> u32
pub fn most_frequent_token_count(&self) -> u32
Sourcepub fn contains_token(&self, token: &str) -> bool
pub fn contains_token(&self, token: &str) -> bool
Sourcepub fn token_set_ref_str(&self) -> Vec<&str>
pub fn token_set_ref_str(&self) -> Vec<&str>
Sourcepub fn token_hashset(&self) -> HashSet<String>
pub fn token_hashset(&self) -> HashSet<String>
Sourcepub fn token_hashset_ref_str(&self) -> HashSet<&str>
pub fn token_hashset_ref_str(&self) -> HashSet<&str>
Sourcepub fn remove_tokens_by_condition<F>(&mut self, condition: F) -> u64
pub fn remove_tokens_by_condition<F>(&mut self, condition: F) -> u64
Sourcepub fn sorted_frequency_vector(&self) -> Vec<(String, u32)>
pub fn sorted_frequency_vector(&self) -> Vec<(String, u32)>
Sourcepub fn sorted_dict_order_vector(&self) -> Vec<(String, u32)>
pub fn sorted_dict_order_vector(&self) -> Vec<(String, u32)>
Sourcepub fn unique_token_ratio(&self) -> f64
pub fn unique_token_ratio(&self) -> f64
Trait Implementations§
Source§impl Clone for TokenFrequency
impl Clone for TokenFrequency
Source§fn clone(&self) -> TokenFrequency
fn clone(&self) -> TokenFrequency
Returns a copy of the value. Read more
1.0.0 · Source§fn clone_from(&mut self, source: &Self)
fn clone_from(&mut self, source: &Self)
Performs copy-assignment from
source. Read moreSource§impl Debug for TokenFrequency
impl Debug for TokenFrequency
Source§impl<'de> Deserialize<'de> for TokenFrequency
impl<'de> Deserialize<'de> for TokenFrequency
Source§fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where
__D: Deserializer<'de>,
fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>where
__D: Deserializer<'de>,
Deserialize this value from the given Serde deserializer. Read more
Auto Trait Implementations§
impl Freeze for TokenFrequency
impl RefUnwindSafe for TokenFrequency
impl Send for TokenFrequency
impl Sync for TokenFrequency
impl Unpin for TokenFrequency
impl UnwindSafe for TokenFrequency
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
Source§impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> CloneToUninit for Twhere
T: Clone,
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more