1#![deny(missing_docs)]
14#![doc = include_str!("../README.md")]
15use rusttorch_core::{Device, RustTorchError, Tensor};
16use rusttorch_data::{
17 BatchSource, BatchSourceCheckpoint, Collate, CollateError, Dataset, DefaultConvert,
18 DistributedConfiguration, DistributedSampler, DistributedSamplerState, MemoryFootprint,
19 PinMemory, ResourceLimits, Sampler, SamplerCheckpoint,
20};
21use serde::{Deserialize, Serialize};
22use std::{fs::File, path::Path};
23pub use tokenizers;
25#[derive(Debug, thiserror::Error)]
27#[non_exhaustive]
28pub enum TextError {
29 #[error("tokenizer: {0}")]
31 Tokenizer(#[source] tokenizers::Error),
32 #[error("{0}")]
34 Invalid(String),
35 #[error(transparent)]
37 Io(#[from] std::io::Error),
38 #[error(transparent)]
40 Limit(#[from] RustTorchError),
41 #[error(transparent)]
43 Tensor(#[from] tch::TchError),
44}
45pub type Result<T> = std::result::Result<T, TextError>;
47#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
49pub struct EncodedSequence {
50 pub ids: Vec<u32>,
52 pub type_ids: Vec<u32>,
54 pub special_tokens_mask: Vec<u32>,
56}
57impl EncodedSequence {
58 pub fn from_ids(ids: Vec<u32>) -> Self {
60 let n = ids.len();
61 Self {
62 ids,
63 type_ids: vec![0; n],
64 special_tokens_mask: vec![0; n],
65 }
66 }
67 fn validate(&self, limits: ResourceLimits) -> Result<()> {
68 limits.check("tokens", self.ids.len(), limits.max_tokens)?;
69 if self.ids.len() != self.type_ids.len()
70 || self.ids.len() != self.special_tokens_mask.len()
71 || self.special_tokens_mask.iter().any(|&v| v > 1)
72 {
73 return Err(TextError::Invalid(
74 "sequence fields must have equal length and binary special-token mask".into(),
75 ));
76 }
77 Ok(())
78 }
79}
80impl MemoryFootprint for EncodedSequence {
81 fn resident_bytes(&self) -> usize {
82 self.ids
83 .capacity()
84 .saturating_add(self.type_ids.capacity())
85 .saturating_add(self.special_tokens_mask.capacity())
86 .saturating_mul(4)
87 }
88}
89impl DefaultConvert for EncodedSequence {
90 type Output = Self;
91 fn default_convert(self) -> std::result::Result<Self, CollateError> {
92 Ok(self)
93 }
94}
95#[derive(Clone, Copy, Debug)]
99pub enum Truncation {
100 Reject(usize),
102 Right(usize),
104}
105#[derive(Clone)]
107pub struct HfTokenizer {
108 inner: tokenizers::Tokenizer,
109 limits: ResourceLimits,
110 truncation: Truncation,
111 add_special_tokens: bool,
112}
113impl HfTokenizer {
114 pub fn new(
117 mut inner: tokenizers::Tokenizer,
118 truncation: Truncation,
119 add_special_tokens: bool,
120 limits: ResourceLimits,
121 ) -> Result<Self> {
122 let n = match truncation {
123 Truncation::Reject(n) | Truncation::Right(n) => n,
124 };
125 if n == 0 {
126 return Err(TextError::Invalid("token limit must be positive".into()));
127 }
128 limits.check("token policy", n, limits.max_tokens)?;
129 inner.with_padding(None);
130 inner.with_truncation(None).map_err(TextError::Tokenizer)?;
131 Ok(Self {
132 inner,
133 limits,
134 truncation,
135 add_special_tokens,
136 })
137 }
138 pub fn from_file(
140 path: impl AsRef<Path>,
141 truncation: Truncation,
142 add_special_tokens: bool,
143 limits: ResourceLimits,
144 ) -> Result<Self> {
145 let bytes = limits.read_encoded(File::open(path)?)?;
146 Self::new(
147 tokenizers::Tokenizer::from_bytes(bytes).map_err(TextError::Tokenizer)?,
148 truncation,
149 add_special_tokens,
150 limits,
151 )
152 }
153 pub fn inner(&self) -> &tokenizers::Tokenizer {
155 &self.inner
156 }
157 pub fn vocabulary_size(&self) -> usize {
159 self.inner.get_vocab_size(true)
160 }
161 pub fn encode(&self, text: &str) -> Result<EncodedSequence> {
163 self.limits
164 .check("text bytes", text.len(), self.limits.max_string_bytes)?;
165 let e = self
166 .inner
167 .encode(text, self.add_special_tokens)
168 .map_err(TextError::Tokenizer)?;
169 self.limits
170 .check("encoded tokens", e.len(), self.limits.max_tokens)?;
171 let n = match self.truncation {
172 Truncation::Reject(n) => {
173 if e.len() > n {
174 return Err(TextError::Invalid(
175 "sequence exceeds truncation policy".into(),
176 ));
177 }
178 e.len()
179 }
180 Truncation::Right(n) => e.len().min(n),
181 };
182 Ok(EncodedSequence {
183 ids: e.get_ids()[..n].to_vec(),
184 type_ids: e.get_type_ids()[..n].to_vec(),
185 special_tokens_mask: e.get_special_tokens_mask()[..n].to_vec(),
186 })
187 }
188}
189pub struct TextDataset {
191 texts: Vec<String>,
192 tokenizer: HfTokenizer,
193}
194impl TextDataset {
195 pub fn new(texts: Vec<String>, tokenizer: HfTokenizer) -> Result<Self> {
197 tokenizer
198 .limits
199 .check("text records", texts.len(), tokenizer.limits.max_records)?;
200 for t in &texts {
201 tokenizer
202 .limits
203 .check("text bytes", t.len(), tokenizer.limits.max_string_bytes)?;
204 }
205 Ok(Self { texts, tokenizer })
206 }
207 pub fn lengths(&self) -> Result<Vec<usize>> {
209 self.texts
210 .iter()
211 .map(|t| self.tokenizer.encode(t).map(|s| s.ids.len()))
212 .collect()
213 }
214}
215impl Dataset for TextDataset {
216 fn get_batch_with_context(
217 &self,
218 indices: &[usize],
219 context: &rusttorch_data::WorkerContext,
220 ) -> Result<Vec<Self::Sample>> {
221 indices
222 .iter()
223 .map(|&index| {
224 context.check().map_err(|_| {
225 TextError::Invalid("sample loading cancelled or timed out".into())
226 })?;
227 self.get(index)
228 })
229 .collect()
230 }
231
232 type Sample = EncodedSequence;
233 type Error = TextError;
234 fn len(&self) -> usize {
235 self.texts.len()
236 }
237 fn get(&self, index: usize) -> Result<EncodedSequence> {
238 self.tokenizer.encode(
239 self.texts
240 .get(index)
241 .ok_or_else(|| TextError::Invalid("text index out of range".into()))?,
242 )
243 }
244}
245#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
247pub enum Padding {
248 Longest {
250 pad_id: u32,
252 },
253 Fixed {
255 pad_id: u32,
257 length: usize,
259 },
260}
261#[derive(Debug)]
263pub struct TextBatch {
264 pub input_ids: Tensor,
266 pub attention_mask: Tensor,
268 pub token_type_ids: Tensor,
270 pub special_tokens_mask: Tensor,
272 pub lengths: Vec<usize>,
274}
275impl MemoryFootprint for TextBatch {
276 fn resident_bytes(&self) -> usize {
277 self.input_ids
278 .resident_bytes()
279 .saturating_add(self.attention_mask.resident_bytes())
280 .saturating_add(self.token_type_ids.resident_bytes())
281 .saturating_add(self.special_tokens_mask.resident_bytes())
282 .saturating_add(
283 self.lengths
284 .capacity()
285 .saturating_mul(std::mem::size_of::<usize>()),
286 )
287 }
288}
289impl PinMemory for TextBatch {
290 fn pin_memory(mut self, d: Device) -> rusttorch_core::Result<Self> {
291 self.input_ids = self.input_ids.pin_memory(d)?;
292 self.attention_mask = self.attention_mask.pin_memory(d)?;
293 self.token_type_ids = self.token_type_ids.pin_memory(d)?;
294 self.special_tokens_mask = self.special_tokens_mask.pin_memory(d)?;
295 Ok(self)
296 }
297}
298#[derive(Clone, Copy, Debug)]
300pub struct TextCollator {
301 padding: Padding,
302 limits: ResourceLimits,
303}
304impl TextCollator {
305 pub fn new(padding: Padding, limits: ResourceLimits) -> Result<Self> {
307 if let Padding::Fixed { length, .. } = padding {
308 limits.check("padding width", length, limits.max_tokens)?;
309 }
310 Ok(Self { padding, limits })
311 }
312}
313impl Collate<EncodedSequence> for TextCollator {
314 type Batch = TextBatch;
315 type Error = TextError;
316 fn collate(&mut self, samples: Vec<EncodedSequence>) -> Result<TextBatch> {
317 if samples.is_empty() {
318 return Err(TextError::Invalid(
319 "cannot collate an empty text batch".into(),
320 ));
321 }
322 self.limits
323 .check("text batch records", samples.len(), self.limits.max_records)?;
324 for s in &samples {
325 s.validate(self.limits)?;
326 }
327 let longest = samples.iter().map(|s| s.ids.len()).max().unwrap_or(0);
328 let (pad, width) = match self.padding {
329 Padding::Longest { pad_id } => (pad_id, longest),
330 Padding::Fixed { pad_id, length } => (pad_id, length),
331 };
332 if longest > width {
333 return Err(TextError::Invalid("sample exceeds padding width".into()));
334 }
335 let n = self.limits.tensor_elements(&[samples.len(), width])?;
336 self.limits
337 .check("padded tokens", n, self.limits.max_tokens)?;
338 self.limits.check(
339 "text batch bytes",
340 n.checked_mul(32)
341 .and_then(|bytes| {
342 samples
343 .len()
344 .checked_mul(std::mem::size_of::<usize>())
345 .and_then(|lengths| bytes.checked_add(lengths))
346 })
347 .ok_or_else(|| TextError::Invalid("text batch byte overflow".into()))?,
348 self.limits.max_decoded_bytes,
349 )?;
350 let mut ids = vec![pad as i64; n];
351 let mut masks = vec![0i64; n];
352 let mut types = vec![0i64; n];
353 let mut specials = vec![0i64; n];
354 let lengths = samples.iter().map(|s| s.ids.len()).collect();
355 for (row, s) in samples.iter().enumerate() {
356 for i in 0..s.ids.len() {
357 let j = row * width + i;
358 ids[j] = s.ids[i] as i64;
359 masks[j] = 1;
360 types[j] = s.type_ids[i] as i64;
361 specials[j] = s.special_tokens_mask[i] as i64;
362 }
363 }
364 let shape = [samples.len() as i64, width as i64];
365 Ok(TextBatch {
366 input_ids: Tensor::f_from_slice(&ids)?.f_reshape(shape)?,
367 attention_mask: Tensor::f_from_slice(&masks)?.f_reshape(shape)?,
368 token_type_ids: Tensor::f_from_slice(&types)?.f_reshape(shape)?,
369 special_tokens_mask: Tensor::f_from_slice(&specials)?.f_reshape(shape)?,
370 lengths,
371 })
372 }
373}
374impl rusttorch_data::Checkpointable for TextCollator {
375 type State = (Padding, ResourceLimits);
376 fn save_state(&self) -> Self::State {
377 (self.padding, self.limits)
378 }
379 fn validate_state(&self, s: &Self::State) -> rusttorch_core::Result<()> {
380 if *s != self.save_state() {
381 return Err(invalid("text collator policy differs"));
382 }
383 Ok(())
384 }
385 fn load_validated(&mut self, _: &Self::State) {}
386}
387
388#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
391pub struct TokenBudgetConfig {
392 pub max_tokens: usize,
394 pub max_samples: usize,
396 pub replicas: usize,
398 pub rank: usize,
400 pub shuffle: bool,
402 pub seed: u64,
404 pub drop_rank_tail: bool,
406}
407impl Default for TokenBudgetConfig {
408 fn default() -> Self {
409 Self {
410 max_tokens: 4096,
411 max_samples: 64,
412 replicas: 1,
413 rank: 0,
414 shuffle: false,
415 seed: 0,
416 drop_rank_tail: false,
417 }
418 }
419}
420pub struct TokenBudgetSampler {
424 lengths: Vec<usize>,
425 config: TokenBudgetConfig,
426 sampler: DistributedSampler,
427}
428#[derive(Clone, Serialize, Deserialize)]
430pub struct TokenBudgetState {
431 version: u32,
432 lengths: Vec<usize>,
433 config: TokenBudgetConfig,
434 sampler: DistributedSamplerState,
435 next_batch: u64,
436 next_sample: u64,
437}
438fn invalid(reason: &str) -> RustTorchError {
439 RustTorchError::InvalidConfiguration {
440 field: "token budget",
441 reason: reason.into(),
442 }
443}
444impl TokenBudgetSampler {
445 pub fn new(
447 lengths: Vec<usize>,
448 config: TokenBudgetConfig,
449 limits: ResourceLimits,
450 ) -> rusttorch_core::Result<Self> {
451 limits.check("token lengths", lengths.len(), limits.max_records)?;
452 limits.check("token budget", config.max_tokens, limits.max_tokens)?;
453 if config.max_tokens == 0
454 || config.max_samples == 0
455 || lengths.iter().any(|&n| n > config.max_tokens)
456 {
457 return Err(invalid(
458 "positive budgets required and every sequence must fit",
459 ));
460 }
461 let sampler = DistributedSampler::new(
462 lengths.len(),
463 config.replicas,
464 config.rank,
465 config.shuffle,
466 config.seed,
467 config.drop_rank_tail,
468 )?;
469 Ok(Self {
470 lengths,
471 config,
472 sampler,
473 })
474 }
475 fn group(&self, indices: impl Iterator<Item = usize>) -> Vec<Vec<usize>> {
476 let mut batches = Vec::new();
477 let mut batch = Vec::new();
478 let mut longest = 0;
479 for index in indices {
480 let next = longest.max(self.lengths[index]);
481 if !batch.is_empty()
482 && (batch.len() == self.config.max_samples
483 || next
484 .checked_mul(batch.len() + 1)
485 .is_none_or(|n| n > self.config.max_tokens))
486 {
487 batches.push(std::mem::take(&mut batch));
488 longest = 0;
489 }
490 longest = longest.max(self.lengths[index]);
491 batch.push(index);
492 }
493 if !batch.is_empty() {
494 batches.push(batch);
495 }
496 batches
497 }
498 fn boundary(&self, batch: u64, sample: u64, epoch: u64) -> rusttorch_core::Result<()> {
499 let mut sampler = DistributedSampler::new(
500 self.lengths.len(),
501 self.config.replicas,
502 self.config.rank,
503 self.config.shuffle,
504 self.config.seed,
505 self.config.drop_rank_tail,
506 )?;
507 sampler.set_epoch(epoch);
508 let batches = self.group(sampler.iter());
509 let b = usize::try_from(batch).map_err(|_| invalid("batch position overflow"))?;
510 if b > batches.len() || batches[..b].iter().map(Vec::len).sum::<usize>() as u64 != sample {
511 return Err(invalid("checkpoint is not a token-budget batch boundary"));
512 }
513 Ok(())
514 }
515}
516impl BatchSource for TokenBudgetSampler {
517 type Iter = std::vec::IntoIter<Vec<usize>>;
518 fn iter(&self) -> Self::Iter {
519 self.group(self.sampler.iter()).into_iter()
520 }
521 fn exact_len(&self) -> Option<usize> {
522 Some(self.group(self.sampler.iter()).len())
523 }
524 fn epoch(&self) -> u64 {
525 self.sampler.epoch()
526 }
527 fn set_epoch(&mut self, epoch: u64) {
528 self.sampler.set_epoch(epoch);
529 }
530}
531impl BatchSourceCheckpoint for TokenBudgetSampler {
532 type State = TokenBudgetState;
533 fn kind(&self) -> String {
534 "token_budget/distributed/v1".into()
535 }
536 fn checkpoint_kind_from_state(_: &Self::State) -> String {
537 "token_budget/distributed/v1".into()
538 }
539 fn checkpoint_state(&self, b: u64, s: u64) -> rusttorch_core::Result<Self::State> {
540 self.boundary(b, s, self.sampler.epoch())?;
541 Ok(TokenBudgetState {
542 version: 1,
543 lengths: self.lengths.clone(),
544 config: self.config.clone(),
545 sampler: self.sampler.checkpoint_state(s)?,
546 next_batch: b,
547 next_sample: s,
548 })
549 }
550 fn validate_checkpoint_state(
551 &self,
552 state: &Self::State,
553 epoch: u64,
554 b: u64,
555 s: u64,
556 ) -> rusttorch_core::Result<()> {
557 if state.version != 1
558 || state.lengths != self.lengths
559 || state.config != self.config
560 || state.next_batch != b
561 || state.next_sample != s
562 {
563 return Err(invalid("checkpoint token-budget configuration differs"));
564 }
565 self.sampler
566 .validate_checkpoint_state(&state.sampler, epoch, s)?;
567 self.boundary(b, s, epoch)
568 }
569 fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter {
570 let indices = self
571 .sampler
572 .restore_checkpoint_iter_validated(&state.sampler);
573 self.group(indices).into_iter()
574 }
575 fn distributed_configuration(&self) -> Option<DistributedConfiguration> {
576 self.sampler.distributed_configuration()
577 }
578 fn checkpoint_distributed_configuration_from_state(
579 state: &Self::State,
580 ) -> rusttorch_core::Result<Option<DistributedConfiguration>> {
581 DistributedSampler::checkpoint_distributed_configuration_from_state(&state.sampler)
582 }
583}