libgrammstein 0.1.0

Hybrid language model (N-gram + Embeddings) for WFST text correction
1
2
3
4
5
6
7
8
9
10
11
12
13
14

//! Language support module.
//!
//! This module provides language-related types and utilities for multi-language
//! model training and organization.

mod detection;
mod registry;
mod tag;
mod tokenizer;

pub use detection::{detect_from_sentences, detect_language, LanguageDetectionError};
pub use registry::{ModelEntry, ModelRegistry};
pub use tag::{wikipedia_dump_url, LanguageTag, LanguageTagError, WIKIPEDIA_URLS};
pub use tokenizer::{create_tokenizer, CharacterTokenizer, Tokenizer, WhitespaceTokenizer};