pub type VocabId = u32;
pub trait LatticeBackend: Clone + Send + Sync {
fn intern(&mut self, word: &str) -> VocabId;
fn lookup(&self, id: VocabId) -> Option<&str>;
fn supports_sharing(&self) -> bool {
false
}
fn vocab_size(&self) -> usize;
fn contains(&self, word: &str) -> bool;
fn get_id(&self, word: &str) -> Option<VocabId>;
fn iter(&self) -> impl Iterator<Item = (VocabId, &str)>;
}
pub trait SharingBackend: LatticeBackend {
fn shares_storage_with(&self, other: &Self) -> bool;
fn fork(&self) -> Self;
}
#[cfg(test)]
mod tests {
use super::*;
use crate::backend::HashMapBackend;
#[test]
fn test_intern_returns_same_id() {
let mut backend = HashMapBackend::new();
let id1 = backend.intern("hello");
let id2 = backend.intern("hello");
assert_eq!(id1, id2);
}
#[test]
fn test_different_words_different_ids() {
let mut backend = HashMapBackend::new();
let id1 = backend.intern("hello");
let id2 = backend.intern("world");
assert_ne!(id1, id2);
}
#[test]
fn test_lookup_valid_id() {
let mut backend = HashMapBackend::new();
let id = backend.intern("hello");
assert_eq!(backend.lookup(id), Some("hello"));
}
#[test]
fn test_lookup_invalid_id() {
let backend = HashMapBackend::new();
assert_eq!(backend.lookup(999), None);
}
#[test]
fn test_vocab_size() {
let mut backend = HashMapBackend::new();
assert_eq!(backend.vocab_size(), 0);
backend.intern("hello");
assert_eq!(backend.vocab_size(), 1);
backend.intern("world");
assert_eq!(backend.vocab_size(), 2);
backend.intern("hello"); assert_eq!(backend.vocab_size(), 2);
}
#[test]
fn test_contains() {
let mut backend = HashMapBackend::new();
assert!(!backend.contains("hello"));
backend.intern("hello");
assert!(backend.contains("hello"));
assert!(!backend.contains("world"));
}
#[test]
fn test_get_id() {
let mut backend = HashMapBackend::new();
assert_eq!(backend.get_id("hello"), None);
let id = backend.intern("hello");
assert_eq!(backend.get_id("hello"), Some(id));
}
#[test]
fn test_iter() {
let mut backend = HashMapBackend::new();
let id1 = backend.intern("hello");
let id2 = backend.intern("world");
let mut entries: Vec<_> = backend.iter().collect();
entries.sort_by_key(|(id, _)| *id);
assert_eq!(entries.len(), 2);
assert_eq!(entries[0], (id1, "hello"));
assert_eq!(entries[1], (id2, "world"));
}
}