language_model/
language_model.rs1use rusting_brain::{Optimizer, Precision, Sampler, Schedule, TokenBatch, TransformerLm};
24
25const WEIGHTS: &str = "lm.rbw";
26const VOCABULARY: &str = "lm.vocab";
27
28const SEQ_LEN: usize = 64;
29const STRIDE: usize = 8;
30const MICRO_BATCH: usize = 8;
31const ACCUMULATE: usize = 2;
32const STEPS: usize = 600;
33const WARMUP: usize = 60;
34const PEAK_LR: f32 = 3e-3;
35const MAX_GRAD_NORM: f32 = 1.0;
36
37const CORPUS: &str = "\
38ownership moves, borrows do not. a value has exactly one owner, and when the \
39owner goes out of scope the value is dropped. a reference borrows the value \
40without taking ownership, and the compiler checks that no reference outlives \
41the value it points at. shared references are read only and there may be many. \
42a mutable reference is exclusive and there may be only one. these two rules \
43together are what make data races impossible to write, not merely unlikely. \
44the borrow checker is not your enemy. it is a colleague who has read the code \
45more carefully than you have. when it rejects a program it is telling you that \
46two parts of the program disagree about who owns a value, and that the \
47disagreement would have been a crash. the fix is almost never to fight it. the \
48fix is to decide who owns the value and say so.";
49
50struct CharTokenizer {
54 vocabulary: Vec<char>,
55}
56
57impl CharTokenizer {
58 fn fit(text: &str) -> Self {
59 let mut vocabulary: Vec<char> = text.chars().collect();
60 vocabulary.sort_unstable();
61 vocabulary.dedup();
62 Self { vocabulary }
63 }
64
65 fn load(path: &str) -> std::io::Result<Self> {
66 Ok(Self {
67 vocabulary: std::fs::read_to_string(path)?.chars().collect(),
68 })
69 }
70
71 fn save(&self, path: &str) -> std::io::Result<()> {
72 std::fs::write(path, self.vocabulary.iter().collect::<String>())
73 }
74
75 fn encode(&self, text: &str) -> Vec<u32> {
76 text.chars()
77 .filter_map(|c| self.vocabulary.iter().position(|&v| v == c))
78 .map(|index| index as u32)
79 .collect()
80 }
81
82 fn decode(&self, ids: &[u32]) -> String {
83 ids.iter()
84 .filter_map(|&id| self.vocabulary.get(id as usize))
85 .collect()
86 }
87
88 fn len(&self) -> usize {
89 self.vocabulary.len()
90 }
91}
92
93fn train(moe: bool) -> Result<(), Box<dyn std::error::Error>> {
94 let tokenizer = CharTokenizer::fit(CORPUS);
95 let ids = tokenizer.encode(CORPUS);
96
97 let split = ids.len() * 85 / 100;
102 let window = |ids: &[u32]| -> Vec<Vec<u32>> {
103 ids.windows(SEQ_LEN)
104 .step_by(STRIDE)
105 .map(<[u32]>::to_vec)
106 .collect()
107 };
108 let windows = window(&ids[..split]);
109 let held_out = TokenBatch::new(&window(&ids[split..]))?;
110
111 let mut builder = TransformerLm::builder()
112 .vocab_size(tokenizer.len())
113 .d_model(128)
114 .n_layers(4)
115 .heads(4, 2, 32)
116 .d_ff(256)
117 .max_seq_len(4 * SEQ_LEN)
121 .optimizer(Optimizer::adam(PEAK_LR))
122 .seed(42);
123
124 builder = if moe {
125 builder
126 .moe_d_ff(64)
127 .experts(8, 2)
128 .moe_layers(1..4)
129 .shared_expert(true)
130 } else {
131 builder.moe_layers([])
132 };
133
134 let mut model = builder.build()?;
135
136 println!(
141 "{} training windows of {SEQ_LEN} tokens, {} held out, vocabulary {}",
142 windows.len(),
143 held_out.batch(),
144 tokenizer.len()
145 );
146 println!("{}\n", model.parameter_counts());
147 println!(" step lr loss ppl aux |g| val");
148
149 let schedule = Schedule::warmup_cosine(PEAK_LR, WARMUP, STEPS);
153
154 let mut cursor = 0;
155 for step in 0..STEPS {
156 let rate = schedule.rate(step);
159 model.optimizer.set_learning_rate(rate);
160
161 model.zero_grad();
164 let (mut lm, mut aux) = (0.0, 0.0);
165 for _ in 0..ACCUMULATE {
166 let micro: Vec<Vec<u32>> = (0..MICRO_BATCH)
167 .map(|_| {
168 let window = windows[cursor % windows.len()].clone();
169 cursor += 1;
170 window
171 })
172 .collect();
173 let loss = model.accumulate_step(&TokenBatch::new(µ)?)?;
174 lm += loss.lm_loss;
175 aux += loss.auxiliary_loss;
176 }
177 let norm = model.step_clipped(1.0 / ACCUMULATE as f32, MAX_GRAD_NORM)?;
186
187 if step == 0 || (step + 1) % 100 == 0 {
188 let lm = lm / ACCUMULATE as f32;
189 let validation = model.evaluate(&held_out)?;
193 println!(
194 "{:>5} {rate:.5} {lm:.4} {:>6.2} {:.4} {norm:.3} {:.4}",
195 step + 1,
196 lm.exp(),
197 aux / ACCUMULATE as f32,
198 validation.lm_loss
199 );
200 }
201 }
202
203 #[cfg(feature = "cuda")]
206 model.sync_from_device()?;
207 model.save_bin(WEIGHTS, Precision::F32)?;
208 model.save_optimizer_state(format!("{WEIGHTS}.opt"))?;
209 tokenizer.save(VOCABULARY)?;
210
211 println!("\nwrote {WEIGHTS}, {WEIGHTS}.opt and {VOCABULARY}");
212 println!("now: cargo run --release --example language_model -- generate \"the borrow\"");
213 Ok(())
214}
215
216fn generate(prompt: &str, new_tokens: usize) -> Result<(), Box<dyn std::error::Error>> {
217 let tokenizer = CharTokenizer::load(VOCABULARY)
218 .map_err(|_| format!("{VOCABULARY} not found - run `language_model -- train` first"))?;
219 let model = TransformerLm::load_bin(WEIGHTS)?;
220
221 let ids = tokenizer.encode(prompt);
222 if ids.is_empty() {
223 return Err("the prompt has no characters this model was trained on".into());
224 }
225
226 for (temperature, top_k) in [(0.2, 8), (0.8, 8)] {
233 let mut sampler = Sampler::temperature(temperature, Some(42)).top_k(top_k);
236
237 print!("T={temperature:.1} k={top_k}\n {}", tokenizer.decode(&ids));
241 model.generate_with(&ids, new_tokens, &mut sampler, |id| {
242 print!("{}", tokenizer.decode(&[id]));
243 let _ = std::io::Write::flush(&mut std::io::stdout());
244 true
245 })?;
246 println!("\n");
247 }
248
249 Ok(())
250}
251
252fn main() -> Result<(), Box<dyn std::error::Error>> {
253 let args: Vec<String> = std::env::args().skip(1).collect();
254 match args.first().map(String::as_str) {
255 Some("train") => train(args.iter().any(|a| a == "--moe")),
256 Some("generate") => {
257 let prompt = args
258 .get(1)
259 .map(String::as_str)
260 .unwrap_or("the borrow checker");
261 generate(prompt, 120)
262 }
263 _ => {
264 eprintln!("usage:");
265 eprintln!(" language_model -- train [--moe]");
266 eprintln!(" language_model -- generate \"<prompt>\"");
267 Ok(())
268 }
269 }
270}