Struct tongrams::trie_count_lm::TrieCountLm

pub struct TrieCountLm<T, V, A> where
    T: TrieArray,
    V: Vocabulary,
    A: RankArray,  { /* private fields */ }

Elias-Fano trie for indexing N-grams with their frequency counts.

Implementations

impl<T, V, A> TrieCountLm<T, V, A> where
    T: TrieArray,
    V: Vocabulary,
    A: RankArray, 

pub fn from_files<P>(filepaths: &[P], fmt: GramsFileFormats) -> Result<Self> where
    P: AsRef<Path>, 

Builds the index from N-gram counts files.

Arguments

• filepaths: Paths of N-gram counts files that should be sorted by N = 1, 2, …
• fmt: File format.

pub fn from_plain_files<P>(filepaths: &[P]) -> Result<Self> where
    P: AsRef<Path>, 

Builds the index from N-gram counts files in a plain text format.

Arguments

• filepaths: Paths of N-gram counts files that should be sorted by N = 1, 2, …

pub fn from_gz_files<P>(filepaths: &[P]) -> Result<Self> where
    P: AsRef<Path>, 

Builds the index from N-gram counts files in a gzip compressed format.

Arguments

• filepaths: Paths of N-gram counts files that should be sorted by N = 1, 2, …

pub fn from_deflate_files<P>(filepaths: &[P]) -> Result<Self> where
    P: AsRef<Path>, 

Builds the index from N-gram counts files in a deflate compressed format.

Arguments

• filepaths: Paths of N-gram counts files that should be sorted by N = 1, 2, …

pub fn from_zlib_files<P>(filepaths: &[P]) -> Result<Self> where
    P: AsRef<Path>, 

Builds the index from N-gram counts files in a zlib compressed format.

Arguments

• filepaths: Paths of N-gram counts files that should be sorted by N = 1, 2, …

pub fn serialize_into<W>(&self, writer: W) -> Result<usize> where
    W: Write, 

Serializes the index into the writer.

pub fn deserialize_from<R>(reader: R) -> Result<Self> where
    R: Read, 

Deserializes the index from the reader.

pub fn size_in_bytes(&self) -> usize

Gets the number of bytes to serialize the index.

pub fn memory_statistics(&self) -> Value

Gets breakdowns of memory usages for components.

pub fn lookuper(&self) -> TrieCountLmLookuper<'_, T, V, A>

Makes the lookuper.

pub fn num_orders(&self) -> usize

Gets the maximum of N.

pub fn num_grams(&self) -> usize

Gets the number of stored grams.

Trait Implementations

impl<T: Debug, V: Debug, A: Debug> Debug for TrieCountLm<T, V, A> where
    T: TrieArray,
    V: Vocabulary,
    A: RankArray, 

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<T: Default, V: Default, A: Default> Default for TrieCountLm<T, V, A> where
    T: TrieArray,
    V: Vocabulary,
    A: RankArray, 

fn default() -> TrieCountLm<T, V, A>

Returns the “default value” for a type.