1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
//! Packing several sequences into one activation matrix.
//!
//! Every module in the transformer stack works on a `[rows, width]` matrix
//! whose rows are tokens. One sequence per call makes every matmul as tall as
//! the sequence, which is far too small to keep a GPU busy and far too small to
//! amortize a kernel launch. A batch is therefore packed into a single matrix
//! of `batch * seq_len` rows, and the two places where rows are *not*
//! independent - the causal mask and the next-token shift in the loss - use the
//! [`Layout`] to find the sequence a row belongs to.
//!
//! Sequences are right-padded to the longest one in the batch. Padding is free
//! for attention (a pad row sits after every real token of its sequence, and
//! the causal mask already stops a real token from seeing it) but not for the
//! router or the loss, which is what [`Layout::valid`] is for. Equal-length
//! sequences - the usual case, since language-model corpora are chunked to a
//! fixed context - carry no padding at all.
use crate::network::NetworkError;
/// A right-padded batch of token id sequences.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TokenBatch {
ids: Vec<u32>,
lengths: Vec<usize>,
seq_len: usize,
valid: Vec<bool>,
supervised: Option<Vec<bool>>,
}
impl TokenBatch {
/// Packs `sequences` into `batch * seq_len` rows, padding short ones.
///
/// Each sequence needs at least two tokens, because a next-token loss over
/// one token predicts nothing.
pub fn new<S: AsRef<[u32]>>(sequences: &[S]) -> Result<Self, NetworkError> {
if sequences.is_empty() {
return Err(NetworkError::EmptyDataset);
}
let lengths: Vec<usize> = sequences.iter().map(|s| s.as_ref().len()).collect();
if let Some(short) = lengths.iter().find(|&&length| length < 2) {
return Err(NetworkError::InvalidConfig(format!(
"a causal language-modelling loss needs at least two tokens per sequence, got {short}"
)));
}
let seq_len = lengths.iter().copied().max().unwrap_or(0);
let mut ids = vec![0u32; sequences.len() * seq_len];
let mut valid = vec![false; sequences.len() * seq_len];
for (index, sequence) in sequences.iter().enumerate() {
let sequence = sequence.as_ref();
let base = index * seq_len;
ids[base..base + sequence.len()].copy_from_slice(sequence);
valid[base..base + sequence.len()].fill(true);
}
Ok(Self {
ids,
lengths,
seq_len,
valid,
supervised: None,
})
}
/// Packs prompt-and-response pairs and flags the response spans, which is
/// the batch supervised fine-tuning wants.
///
/// Each pair is concatenated into one sequence, and only the response
/// tokens count toward the loss: the prompt conditions the model without
/// being learned. Building the flat mask
/// [`with_loss_mask`](TokenBatch::with_loss_mask) takes is otherwise the
/// caller's job, and getting the padding or the row order wrong there
/// trains on the prompt without saying so.
///
/// Append whatever end-of-turn token the model should learn to emit to the
/// response side; nothing is added here.
///
/// ```
/// # use rusting_brain::TokenBatch;
/// // "<user>2+2<assistant>" -> "4<end>"
/// let batch = TokenBatch::supervised(&[(vec![1, 5, 2], vec![9, 3])])?;
/// assert_eq!(batch.predicted(), 2); // the two response tokens
/// # Ok::<(), Box<dyn std::error::Error>>(())
/// ```
pub fn supervised<P: AsRef<[u32]>, C: AsRef<[u32]>>(
pairs: &[(P, C)],
) -> Result<Self, NetworkError> {
let sequences: Vec<Vec<u32>> = pairs
.iter()
.map(|(prompt, response)| [prompt.as_ref(), response.as_ref()].concat())
.collect();
let batch = Self::new(&sequences)?;
let mut mask = vec![false; batch.rows()];
for (index, (prompt, response)) in pairs.iter().enumerate() {
let (prompt, response) = (prompt.as_ref().len(), response.as_ref().len());
if response == 0 {
return Err(NetworkError::InvalidConfig(format!(
"pair {index} has an empty response, so there is nothing to learn from it"
)));
}
let base = index * batch.seq_len() + prompt;
mask[base..base + response].fill(true);
}
batch.with_loss_mask(&mask)
}
/// Restricts the loss to the token positions flagged `true`.
///
/// `mask` holds one flag per row of [`TokenBatch::ids`] - `batch *
/// seq_len` flags, padding included - and a flag marks its token as a
/// *target*: when `mask[i]` is true the position that predicts token `i`
/// contributes to the loss and to the gradient, and when it is false that
/// position contributes exactly zero to both. This is the same convention
/// as a Hugging Face `labels` tensor, where masked entries are `-100`.
///
/// The first token of a sequence is never a target - nothing precedes it -
/// so its flag is ignored, and padding stays excluded whatever the mask
/// says.
///
/// Masking changes the loss alone. Masked rows still run through the
/// forward pass and still serve as context for the rows that do count,
/// which is what makes this usable for supervised fine-tuning: flag the
/// response span and the instruction span in front of it conditions the
/// model without being learned.
pub fn with_loss_mask(mut self, mask: &[bool]) -> Result<Self, NetworkError> {
if mask.len() != self.ids.len() {
return Err(NetworkError::InvalidConfig(format!(
"a loss mask covers {} token positions, not {}",
mask.len(),
self.ids.len()
)));
}
self.supervised = Some(mask.to_vec());
if self.predicted() == 0 {
return Err(NetworkError::InvalidConfig(
"a loss mask left no position to predict".into(),
));
}
Ok(self)
}
/// Whether row `row` predicts a token that counts toward the loss.
///
/// False for padding, for the last position of a sequence, and for a
/// position whose target a loss mask excluded.
pub fn predicts(&self, row: usize) -> bool {
if row % self.seq_len + 1 >= self.lengths[row / self.seq_len] {
return false;
}
self.supervised
.as_ref()
.is_none_or(|supervised| supervised[row + 1])
}
/// The same batch with different token ids, which is how a masked
/// language model swaps corrupted tokens in without repacking.
///
/// Lengths, padding and any loss mask are kept, so only what the model
/// reads changes.
pub(crate) fn replacing_ids(mut self, ids: &[u32]) -> Result<Self, NetworkError> {
if ids.len() != self.ids.len() {
return Err(NetworkError::InvalidConfig(format!(
"a batch of {} rows cannot take {} ids",
self.ids.len(),
ids.len()
)));
}
self.ids.copy_from_slice(ids);
Ok(self)
}
/// Number of sequences.
pub fn batch(&self) -> usize {
self.lengths.len()
}
/// Rows per sequence, which is the longest sequence in the batch.
pub fn seq_len(&self) -> usize {
self.seq_len
}
/// Total rows, padding included.
pub fn rows(&self) -> usize {
self.ids.len()
}
/// Padded ids, `[batch * seq_len]`.
pub fn ids(&self) -> &[u32] {
&self.ids
}
pub fn lengths(&self) -> &[usize] {
&self.lengths
}
/// Whether any row is padding. Equal-length batches are the fast case.
pub fn is_padded(&self) -> bool {
self.lengths.iter().any(|&length| length != self.seq_len)
}
/// How many positions a next-token loss covers.
pub fn predicted(&self) -> usize {
match self.supervised {
None => self.lengths.iter().map(|length| length - 1).sum(),
Some(_) => (0..self.rows()).filter(|&row| self.predicts(row)).count(),
}
}
/// The row-to-sequence map every batched module needs.
pub fn layout(&self) -> Layout<'_> {
Layout {
seq_len: Some(self.seq_len),
valid: self.is_padded().then_some(&self.valid),
}
}
}
/// How the rows of a packed matrix map onto sequences.
///
/// The default - no `seq_len`, no mask - is a single unpadded sequence, which
/// is what every module did before batching existed.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct Layout<'a> {
/// Rows per sequence. `None` means the whole matrix is one sequence.
pub seq_len: Option<usize>,
/// Per-row padding mask. `None` means every row is a real token.
pub valid: Option<&'a [bool]>,
}
impl<'a> Layout<'a> {
/// One sequence of `rows` tokens, no padding.
pub fn single(rows: usize) -> Self {
Self {
seq_len: Some(rows),
valid: None,
}
}
/// Rows per sequence, resolving the "one sequence" default against the
/// matrix actually being processed.
pub fn seq_len(&self, rows: usize) -> usize {
self.seq_len.unwrap_or(rows).max(1)
}
/// Whether row `row` holds a real token.
pub fn is_valid(&self, row: usize) -> bool {
self.valid.is_none_or(|mask| mask[row])
}
/// Errors unless `rows` splits evenly into sequences and any mask covers
/// every row.
pub fn check(&self, rows: usize) -> Result<(), NetworkError> {
let seq_len = self.seq_len(rows);
if !rows.is_multiple_of(seq_len) {
return Err(NetworkError::InvalidConfig(format!(
"{rows} rows do not divide into sequences of {seq_len}"
)));
}
match self.valid {
Some(mask) if mask.len() != rows => Err(NetworkError::InvalidConfig(format!(
"padding mask covers {} rows, not {rows}",
mask.len()
))),
_ => Ok(()),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn equal_length_sequences_need_no_padding() {
let batch = TokenBatch::new(&[[1u32, 2, 3], [4, 5, 6]]).unwrap();
assert_eq!(batch.rows(), 6);
assert_eq!(batch.seq_len(), 3);
assert!(!batch.is_padded());
assert_eq!(batch.layout().valid, None);
assert_eq!(batch.predicted(), 4);
}
#[test]
fn supervised_pairs_flag_the_response_and_nothing_else() {
// Two pairs of different lengths, so padding is in play: prompts of 2
// and 3 tokens, responses of 3 and 1.
let batch =
TokenBatch::supervised(&[(vec![1u32, 2], vec![7u32, 8, 9]), (vec![3, 4, 5], vec![6])])
.unwrap();
assert_eq!(batch.seq_len(), 5);
assert_eq!(batch.ids(), &[1, 2, 7, 8, 9, 3, 4, 5, 6, 0]);
// Four response tokens, and the position before each one predicts it.
assert_eq!(batch.predicted(), 4);
assert!(batch.predicts(1)); // predicts 7, the first response token
assert!(!batch.predicts(0)); // predicts 2, still the prompt
assert!(batch.predicts(7)); // predicts 6
assert!(!batch.predicts(8)); // the response is over, the rest is padding
// The same pairs hand-masked agree with the helper.
let by_hand = TokenBatch::new(&[vec![1u32, 2, 7, 8, 9], vec![3, 4, 5, 6]])
.unwrap()
.with_loss_mask(&[
false, false, true, true, true, false, false, false, true, false,
])
.unwrap();
assert_eq!(batch, by_hand);
// An empty response is named rather than left to fail as an empty mask.
assert!(TokenBatch::supervised(&[(vec![1u32, 2], vec![])]).is_err());
}
#[test]
fn a_short_sequence_is_padded_on_the_right_and_masked() {
let batch = TokenBatch::new(&[vec![1u32, 2, 3, 4], vec![5, 6]]).unwrap();
assert_eq!(batch.ids(), &[1, 2, 3, 4, 5, 6, 0, 0]);
assert!(batch.is_padded());
let layout = batch.layout();
assert!(layout.is_valid(5));
assert!(!layout.is_valid(6));
assert_eq!(batch.predicted(), 3 + 1);
}
#[test]
fn a_loss_mask_counts_only_the_flagged_targets() {
let batch = TokenBatch::new(&[[1u32, 2, 3, 4]])
.unwrap()
// Flag the last two tokens: rows 1 and 2 predict them.
.with_loss_mask(&[false, false, true, true])
.unwrap();
assert_eq!(batch.predicted(), 2);
assert!(!batch.predicts(0));
assert!(batch.predicts(1));
assert!(batch.predicts(2));
// Nothing follows the last token, whatever the mask says.
assert!(!batch.predicts(3));
}
#[test]
fn a_loss_mask_never_resurrects_padding() {
let batch = TokenBatch::new(&[vec![1u32, 2, 3, 4], vec![5, 6]])
.unwrap()
.with_loss_mask(&[true; 8])
.unwrap();
// Same count as the unmasked batch: rows 5, 6 and 7 of the short
// sequence are padding or its final token.
assert_eq!(batch.predicted(), 4);
assert!(batch.predicts(4));
assert!(!batch.predicts(5));
assert!(!batch.predicts(6));
}
#[test]
fn a_loss_mask_of_the_wrong_length_or_all_false_is_rejected() {
let batch = || TokenBatch::new(&[[1u32, 2, 3, 4]]).unwrap();
assert!(batch().with_loss_mask(&[true; 3]).is_err());
assert!(batch().with_loss_mask(&[false; 4]).is_err());
// Flagging only the first token leaves nothing to predict: nothing
// precedes it.
assert!(
batch()
.with_loss_mask(&[true, false, false, false])
.is_err()
);
}
#[test]
fn a_one_token_sequence_is_rejected() {
assert!(TokenBatch::new(&[vec![1u32]]).is_err());
assert!(TokenBatch::new::<Vec<u32>>(&[]).is_err());
}
#[test]
fn the_default_layout_is_one_sequence() {
let layout = Layout::default();
assert_eq!(layout.seq_len(7), 7);
assert!(layout.is_valid(6));
assert!(layout.check(7).is_ok());
}
#[test]
fn a_layout_that_does_not_divide_the_rows_is_rejected() {
assert!(Layout::single(3).check(7).is_err());
let mask = [true, true];
let layout = Layout {
seq_len: Some(2),
valid: Some(&mask),
};
assert!(layout.check(4).is_err());
}
}