use std::sync::Arc;
use lling_llang::backend::{LatticeBackend, VocabId};
use rustc_hash::FxHashMap;
use libdictenstein::Dictionary;
#[derive(Clone)]
pub struct DictionaryBackend<D>
where
D: Dictionary + Clone + Send + Sync,
D::Node: Send + Sync,
{
dictionary: D,
word_to_id: FxHashMap<Arc<str>, VocabId>,
id_to_word: Vec<Arc<str>>,
}
impl<D> DictionaryBackend<D>
where
D: Dictionary + Clone + Send + Sync,
D::Node: Send + Sync,
{
pub fn new(dictionary: D) -> Self {
Self {
dictionary,
word_to_id: FxHashMap::default(),
id_to_word: Vec::new(),
}
}
pub fn with_vocabulary<I>(dictionary: D, terms: I) -> Self
where
I: IntoIterator<Item = String>,
{
let mut backend = Self::new(dictionary);
for term in terms {
backend.intern(&term);
}
backend
}
pub fn dictionary(&self) -> &D {
&self.dictionary
}
pub fn dictionary_mut(&mut self) -> &mut D {
&mut self.dictionary
}
pub fn into_dictionary(self) -> D {
self.dictionary
}
}
impl<D> LatticeBackend for DictionaryBackend<D>
where
D: Dictionary + Clone + Send + Sync,
D::Node: Send + Sync,
{
fn intern(&mut self, word: &str) -> VocabId {
if let Some(&id) = self.word_to_id.get(word) {
return id;
}
let id = self.id_to_word.len() as VocabId;
let word_arc: Arc<str> = word.into();
self.word_to_id.insert(word_arc.clone(), id);
self.id_to_word.push(word_arc);
id
}
fn lookup(&self, id: VocabId) -> Option<&str> {
self.id_to_word.get(id as usize).map(|s| s.as_ref())
}
fn vocab_size(&self) -> usize {
self.id_to_word.len()
}
fn contains(&self, word: &str) -> bool {
self.word_to_id.contains_key(word) || self.dictionary.contains(word)
}
fn get_id(&self, word: &str) -> Option<VocabId> {
self.word_to_id.get(word).copied()
}
fn iter(&self) -> impl Iterator<Item = (VocabId, &str)> {
self.id_to_word
.iter()
.enumerate()
.map(|(i, s)| (i as VocabId, s.as_ref()))
}
fn supports_sharing(&self) -> bool {
false
}
}
#[cfg(test)]
mod tests {
use super::*;
use libdictenstein::dynamic_dawg::char::DynamicDawgChar;
#[test]
fn test_dictionary_backend_new() {
let dict = DynamicDawgChar::<()>::from_terms(vec!["hello", "world"]);
let backend = DictionaryBackend::new(dict);
assert_eq!(backend.vocab_size(), 0); }
#[test]
fn test_dictionary_backend_intern() {
let dict = DynamicDawgChar::<()>::from_terms(vec!["hello", "world"]);
let mut backend = DictionaryBackend::new(dict);
let id1 = backend.intern("hello");
let id2 = backend.intern("world");
let id3 = backend.intern("hello");
assert_eq!(id1, id3); assert_ne!(id1, id2); assert_eq!(backend.vocab_size(), 2);
}
#[test]
fn test_dictionary_backend_lookup() {
let dict = DynamicDawgChar::<()>::from_terms(vec!["hello", "world"]);
let mut backend = DictionaryBackend::new(dict);
let id = backend.intern("hello");
assert_eq!(backend.lookup(id), Some("hello"));
assert_eq!(backend.lookup(999), None);
}
#[test]
fn test_dictionary_backend_contains() {
let dict = DynamicDawgChar::<()>::from_terms(vec!["hello", "world"]);
let backend = DictionaryBackend::new(dict);
assert!(backend.contains("hello"));
assert!(backend.contains("world"));
assert!(!backend.contains("missing"));
}
#[test]
fn test_dictionary_backend_get_id() {
let dict = DynamicDawgChar::<()>::from_terms(vec!["hello", "world"]);
let mut backend = DictionaryBackend::new(dict);
assert_eq!(backend.get_id("hello"), None);
let id = backend.intern("hello");
assert_eq!(backend.get_id("hello"), Some(id));
}
#[test]
fn test_dictionary_backend_iter() {
let dict = DynamicDawgChar::<()>::from_terms(vec!["hello", "world"]);
let mut backend = DictionaryBackend::new(dict);
backend.intern("hello");
backend.intern("world");
let entries: Vec<_> = backend.iter().collect();
assert_eq!(entries.len(), 2);
}
#[test]
fn test_dictionary_backend_with_vocabulary() {
let dict = DynamicDawgChar::<()>::from_terms(vec!["hello", "world", "test"]);
let terms = vec!["hello".to_string(), "world".to_string()];
let backend = DictionaryBackend::with_vocabulary(dict, terms);
assert_eq!(backend.vocab_size(), 2);
assert!(backend.get_id("hello").is_some());
assert!(backend.get_id("world").is_some());
assert_eq!(backend.get_id("test"), None); }
#[test]
fn test_dictionary_backend_clone() {
let dict = DynamicDawgChar::<()>::from_terms(vec!["hello", "world"]);
let mut backend = DictionaryBackend::new(dict);
backend.intern("hello");
let cloned = backend.clone();
assert_eq!(cloned.vocab_size(), 1);
assert_eq!(cloned.get_id("hello"), backend.get_id("hello"));
}
}