Skip to main content

language_model/
language_model.rs

1//! Trains a character-level transformer language model, then generates from it.
2//!
3//! ```bash
4//! cargo run --release --example language_model -- train
5//! cargo run --release --example language_model -- train --moe
6//! cargo run --release --example language_model -- generate "the borrow checker"
7//! ```
8//!
9//! `train` writes `lm.rbw` and `lm.vocab`; `generate` reads them back, so the
10//! two halves run in separate processes and the checkpoint is really exercised.
11//!
12//! The corpus is a paragraph and the model is 0.6M parameters, so the held-out
13//! loss printed beside the training loss turns back up within a hundred steps:
14//! there is nothing here to generalize to and memorizing is the only thing left
15//! to do. That gap is the point of printing it.
16//!
17//! Character-level because the crate ships no tokenizer: tokenization is a
18//! text problem rather than a neural-network one, and the `tokenizers` crate
19//! already solves it. Swap a BPE vocabulary in and nothing else here changes.
20//!
21//! Tutorial chapters 14 to 16 explain every decision in this file.
22
23use rusting_brain::{Optimizer, Precision, Sampler, Schedule, TokenBatch, TransformerLm};
24
25const WEIGHTS: &str = "lm.rbw";
26const VOCABULARY: &str = "lm.vocab";
27
28const SEQ_LEN: usize = 64;
29const STRIDE: usize = 8;
30const MICRO_BATCH: usize = 8;
31const ACCUMULATE: usize = 2;
32const STEPS: usize = 600;
33const WARMUP: usize = 60;
34const PEAK_LR: f32 = 3e-3;
35const MAX_GRAD_NORM: f32 = 1.0;
36
37const CORPUS: &str = "\
38ownership moves, borrows do not. a value has exactly one owner, and when the \
39owner goes out of scope the value is dropped. a reference borrows the value \
40without taking ownership, and the compiler checks that no reference outlives \
41the value it points at. shared references are read only and there may be many. \
42a mutable reference is exclusive and there may be only one. these two rules \
43together are what make data races impossible to write, not merely unlikely. \
44the borrow checker is not your enemy. it is a colleague who has read the code \
45more carefully than you have. when it rejects a program it is telling you that \
46two parts of the program disagree about who owns a value, and that the \
47disagreement would have been a crash. the fix is almost never to fight it. the \
48fix is to decide who owns the value and say so.";
49
50/// One token per distinct character. The vocabulary is saved beside the
51/// weights, because a checkpoint and the tokenizer that produced its ids are
52/// only meaningful together.
53struct CharTokenizer {
54    vocabulary: Vec<char>,
55}
56
57impl CharTokenizer {
58    fn fit(text: &str) -> Self {
59        let mut vocabulary: Vec<char> = text.chars().collect();
60        vocabulary.sort_unstable();
61        vocabulary.dedup();
62        Self { vocabulary }
63    }
64
65    fn load(path: &str) -> std::io::Result<Self> {
66        Ok(Self {
67            vocabulary: std::fs::read_to_string(path)?.chars().collect(),
68        })
69    }
70
71    fn save(&self, path: &str) -> std::io::Result<()> {
72        std::fs::write(path, self.vocabulary.iter().collect::<String>())
73    }
74
75    fn encode(&self, text: &str) -> Vec<u32> {
76        text.chars()
77            .filter_map(|c| self.vocabulary.iter().position(|&v| v == c))
78            .map(|index| index as u32)
79            .collect()
80    }
81
82    fn decode(&self, ids: &[u32]) -> String {
83        ids.iter()
84            .filter_map(|&id| self.vocabulary.get(id as usize))
85            .collect()
86    }
87
88    fn len(&self) -> usize {
89        self.vocabulary.len()
90    }
91}
92
93fn train(moe: bool) -> Result<(), Box<dyn std::error::Error>> {
94    let tokenizer = CharTokenizer::fit(CORPUS);
95    let ids = tokenizer.encode(CORPUS);
96
97    // The held-out split is a slice of the text, not a sample of the windows:
98    // windows overlap, so holding out every n-th one would put most of a
99    // validation window inside a training window and report a loss that means
100    // nothing.
101    let split = ids.len() * 85 / 100;
102    let window = |ids: &[u32]| -> Vec<Vec<u32>> {
103        ids.windows(SEQ_LEN)
104            .step_by(STRIDE)
105            .map(<[u32]>::to_vec)
106            .collect()
107    };
108    let windows = window(&ids[..split]);
109    let held_out = TokenBatch::new(&window(&ids[split..]))?;
110
111    let mut builder = TransformerLm::builder()
112        .vocab_size(tokenizer.len())
113        .d_model(128)
114        .n_layers(4)
115        .heads(4, 2, 32)
116        .d_ff(256)
117        // Generation runs past the training window, and the limit is baked
118        // into the checkpoint, so it is set for the longest sequence the model
119        // will ever see rather than the longest it trains on.
120        .max_seq_len(4 * SEQ_LEN)
121        .optimizer(Optimizer::adam(PEAK_LR))
122        .seed(42);
123
124    builder = if moe {
125        builder
126            .moe_d_ff(64)
127            .experts(8, 2)
128            .moe_layers(1..4)
129            .shared_expert(true)
130    } else {
131        builder.moe_layers([])
132    };
133
134    let mut model = builder.build()?;
135
136    // On a GPU, these two lines and nothing else:
137    // model.set_mixed_precision(true);
138    // model.to_cuda(0, 9_000)?;
139
140    println!(
141        "{} training windows of {SEQ_LEN} tokens, {} held out, vocabulary {}",
142        windows.len(),
143        held_out.batch(),
144        tokenizer.len()
145    );
146    println!("{}\n", model.parameter_counts());
147    println!(" step      lr     loss    ppl     aux    |g|      val");
148
149    // Warmup then cosine decay: full-size steps before Adam's moments have
150    // settled knock the model somewhere it takes a long time to leave, and
151    // full-size steps at the end stop it settling at all.
152    let schedule = Schedule::warmup_cosine(PEAK_LR, WARMUP, STEPS);
153
154    let mut cursor = 0;
155    for step in 0..STEPS {
156        // The optimizer is read fresh at every step, so a schedule is one
157        // assignment and there is nothing to register.
158        let rate = schedule.rate(step);
159        model.optimizer.set_learning_rate(rate);
160
161        // Gradient accumulation: the effective batch is not bounded by the
162        // memory one forward pass needs.
163        model.zero_grad();
164        let (mut lm, mut aux) = (0.0, 0.0);
165        for _ in 0..ACCUMULATE {
166            let micro: Vec<Vec<u32>> = (0..MICRO_BATCH)
167                .map(|_| {
168                    let window = windows[cursor % windows.len()].clone();
169                    cursor += 1;
170                    window
171                })
172                .collect();
173            let loss = model.accumulate_step(&TokenBatch::new(&micro)?)?;
174            lm += loss.lm_loss;
175            aux += loss.auxiliary_loss;
176        }
177        // The averaging belongs on the step: Adam normalizes by the gradient's
178        // own second moment, so scaling every accumulation identically would
179        // cancel out and change nothing.
180        //
181        // Clipping is not an optimization. A single batch whose gradient is an
182        // order of magnitude larger than usual moves the weights far enough to
183        // undo thousands of steps, and the norm printed below is the warning
184        // that it happened.
185        let norm = model.step_clipped(1.0 / ACCUMULATE as f32, MAX_GRAD_NORM)?;
186
187        if step == 0 || (step + 1) % 100 == 0 {
188            let lm = lm / ACCUMULATE as f32;
189            // Training loss falls whether or not the model is learning anything
190            // general. The held-out loss is what says which, and the step where
191            // it turns back up is the step to have stopped at.
192            let validation = model.evaluate(&held_out)?;
193            println!(
194                "{:>5}  {rate:.5}  {lm:.4}  {:>6.2}  {:.4}  {norm:.3}  {:.4}",
195                step + 1,
196                lm.exp(),
197                aux / ACCUMULATE as f32,
198                validation.lm_loss
199            );
200        }
201    }
202
203    // F32, not Q8: int8 rounding is invisible for inference and shows up as a
204    // step in the loss curve of a run that resumes from the file.
205    #[cfg(feature = "cuda")]
206    model.sync_from_device()?;
207    model.save_bin(WEIGHTS, Precision::F32)?;
208    model.save_optimizer_state(format!("{WEIGHTS}.opt"))?;
209    tokenizer.save(VOCABULARY)?;
210
211    println!("\nwrote {WEIGHTS}, {WEIGHTS}.opt and {VOCABULARY}");
212    println!("now: cargo run --release --example language_model -- generate \"the borrow\"");
213    Ok(())
214}
215
216fn generate(prompt: &str, new_tokens: usize) -> Result<(), Box<dyn std::error::Error>> {
217    let tokenizer = CharTokenizer::load(VOCABULARY)
218        .map_err(|_| format!("{VOCABULARY} not found - run `language_model -- train` first"))?;
219    let model = TransformerLm::load_bin(WEIGHTS)?;
220
221    let ids = tokenizer.encode(prompt);
222    if ids.is_empty() {
223        return Err("the prompt has no characters this model was trained on".into());
224    }
225
226    // Two temperatures from the same prompt, because the difference is the
227    // whole point: low is repetitive and safe, high is varied and wrong more
228    // often. Top-k matters as much as the temperature: without it the thirty
229    // thousand individually-impossible tokens carry enough probability between
230    // them that one is eventually drawn, and a single wrong token derails
231    // everything after it.
232    for (temperature, top_k) in [(0.2, 8), (0.8, 8)] {
233        // A fixed seed, so re-running the example on the same checkpoint prints
234        // the same text; pass `None` for a different sample each run.
235        let mut sampler = Sampler::temperature(temperature, Some(42)).top_k(top_k);
236
237        // Printed as it decodes rather than at the end: each token costs a
238        // forward pass, and on a real model that is several seconds of nothing.
239        // `flush`, because stdout is line buffered and none of this is a line.
240        print!("T={temperature:.1} k={top_k}\n  {}", tokenizer.decode(&ids));
241        model.generate_with(&ids, new_tokens, &mut sampler, |id| {
242            print!("{}", tokenizer.decode(&[id]));
243            let _ = std::io::Write::flush(&mut std::io::stdout());
244            true
245        })?;
246        println!("\n");
247    }
248
249    Ok(())
250}
251
252fn main() -> Result<(), Box<dyn std::error::Error>> {
253    let args: Vec<String> = std::env::args().skip(1).collect();
254    match args.first().map(String::as_str) {
255        Some("train") => train(args.iter().any(|a| a == "--moe")),
256        Some("generate") => {
257            let prompt = args
258                .get(1)
259                .map(String::as_str)
260                .unwrap_or("the borrow checker");
261            generate(prompt, 120)
262        }
263        _ => {
264            eprintln!("usage:");
265            eprintln!("  language_model -- train [--moe]");
266            eprintln!("  language_model -- generate \"<prompt>\"");
267            Ok(())
268        }
269    }
270}