1pub mod beam;
8pub mod context;
9pub mod lm;
10pub mod prediction;
11pub mod trainer;
12pub mod zero_copy;
13
14pub use beam::BeamSearchDecoder;
15pub use context::ContextScorer;
16pub use lm::LanguageModel;
17pub use prediction::NextWordPredictor;
18pub use trainer::{
19 train_files_streaming, CorpusTrainer, TrainedLanguageModelData, TrainingConfig,
20};
21pub use zero_copy::ZeroCopyLanguageModel;
22
23#[cfg(test)]
24mod tests {
25 use super::*;
26
27 #[test]
28 fn test_full_ai_pipeline() {
29 let lm = LanguageModel::new();
30 let scorer = ContextScorer::new();
31 let predictor = NextWordPredictor::new();
32 let decoder = BeamSearchDecoder::new();
33
34 let p = lm.score_candidate(Some("আমি"), Some("ভাত"), "খাচ্ছি");
36 assert!(p > -2.0);
37
38 let cands = vec!["পড়া".to_string(), "পরা".to_string()];
40 let reranked = scorer.rank_candidates(&["শার্ট"], &cands);
41 assert_eq!(reranked[0], "পরা");
42
43 let next_words = predictor.predict_next(&["বাংলাদেশ", "একটি"], 3);
45 assert!(!next_words.is_empty());
46
47 let seq = vec![
49 vec!["আমি".to_string()],
50 vec!["বই".to_string()],
51 vec!["পরা".to_string(), "পড়া".to_string()],
52 ];
53 let path = decoder.decode(&seq);
54 assert_eq!(path, vec!["আমি", "বই", "পড়া"]);
55 }
56}