Skip to main content

rusttorch_text/
lib.rs

1//! Tokenize local text and produce padded model inputs with explicit policies.
2//!
3//! ```
4//! use rusttorch_text::{EncodedSequence, Padding, TextCollator};
5//! use rusttorch_data::Collate;
6//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
7//! let sequences = vec![EncodedSequence::from_ids(vec![2, 3]), EncodedSequence::from_ids(vec![4])];
8//! let batch = TextCollator::new(Padding::Longest { pad_id: 0 }, Default::default())?.collate(sequences)?;
9//! assert_eq!(batch.input_ids.size(), [2, 2]);
10//! assert_eq!(Vec::<Vec<i64>>::try_from(&batch.attention_mask)?, [[1,1],[1,0]]);
11//! # Ok(()) }
12//! ```
13#![deny(missing_docs)]
14#![doc = include_str!("../README.md")]
15use rusttorch_core::{Device, RustTorchError, Tensor};
16use rusttorch_data::{
17    BatchSource, BatchSourceCheckpoint, Collate, CollateError, Dataset, DefaultConvert,
18    DistributedConfiguration, DistributedSampler, DistributedSamplerState, MemoryFootprint,
19    PinMemory, ResourceLimits, Sampler, SamplerCheckpoint,
20};
21use serde::{Deserialize, Serialize};
22use std::{fs::File, path::Path};
23/// Advanced Hugging Face models, normalizers and post-processors.
24pub use tokenizers;
25/// Text processing failure with preserved native source chains.
26#[derive(Debug, thiserror::Error)]
27#[non_exhaustive]
28pub enum TextError {
29    /// Tokenizer validation/encoding failure.
30    #[error("tokenizer: {0}")]
31    Tokenizer(#[source] tokenizers::Error),
32    /// Invalid input or policy.
33    #[error("{0}")]
34    Invalid(String),
35    /// Local tokenizer file failure.
36    #[error(transparent)]
37    Io(#[from] std::io::Error),
38    /// Resource/configuration failure.
39    #[error(transparent)]
40    Limit(#[from] RustTorchError),
41    /// Tensor allocation/conversion failure.
42    #[error(transparent)]
43    Tensor(#[from] tch::TchError),
44}
45/// Result returned by text operations.
46pub type Result<T> = std::result::Result<T, TextError>;
47/// One unpadded sequence, including post-processor special tokens.
48#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
49pub struct EncodedSequence {
50    /// Vocabulary IDs.
51    pub ids: Vec<u32>,
52    /// Segment/type identifiers aligned to IDs.
53    pub type_ids: Vec<u32>,
54    /// One for special tokens, zero for ordinary tokens.
55    pub special_tokens_mask: Vec<u32>,
56}
57impl EncodedSequence {
58    /// Constructs a single-segment sequence with no marked special tokens.
59    pub fn from_ids(ids: Vec<u32>) -> Self {
60        let n = ids.len();
61        Self {
62            ids,
63            type_ids: vec![0; n],
64            special_tokens_mask: vec![0; n],
65        }
66    }
67    fn validate(&self, limits: ResourceLimits) -> Result<()> {
68        limits.check("tokens", self.ids.len(), limits.max_tokens)?;
69        if self.ids.len() != self.type_ids.len()
70            || self.ids.len() != self.special_tokens_mask.len()
71            || self.special_tokens_mask.iter().any(|&v| v > 1)
72        {
73            return Err(TextError::Invalid(
74                "sequence fields must have equal length and binary special-token mask".into(),
75            ));
76        }
77        Ok(())
78    }
79}
80impl MemoryFootprint for EncodedSequence {
81    fn resident_bytes(&self) -> usize {
82        self.ids
83            .capacity()
84            .saturating_add(self.type_ids.capacity())
85            .saturating_add(self.special_tokens_mask.capacity())
86            .saturating_mul(4)
87    }
88}
89impl DefaultConvert for EncodedSequence {
90    type Output = Self;
91    fn default_convert(self) -> std::result::Result<Self, CollateError> {
92        Ok(self)
93    }
94}
95/// Explicit right-truncation policy. Truncation occurs after special-token
96/// insertion and can remove trailing special tokens; use tokenizer-native
97/// postprocessing when a model requires preserving a closing separator.
98#[derive(Clone, Copy, Debug)]
99pub enum Truncation {
100    /// Reject sequences longer than this limit.
101    Reject(usize),
102    /// Keep at most the first specified number of tokens.
103    Right(usize),
104}
105/// Adapter over a locally configured Hugging Face tokenizer.
106#[derive(Clone)]
107pub struct HfTokenizer {
108    inner: tokenizers::Tokenizer,
109    limits: ResourceLimits,
110    truncation: Truncation,
111    add_special_tokens: bool,
112}
113impl HfTokenizer {
114    /// Creates an adapter, disabling native padding/truncation so the specified
115    /// RustTorch policies are the only length transformations.
116    pub fn new(
117        mut inner: tokenizers::Tokenizer,
118        truncation: Truncation,
119        add_special_tokens: bool,
120        limits: ResourceLimits,
121    ) -> Result<Self> {
122        let n = match truncation {
123            Truncation::Reject(n) | Truncation::Right(n) => n,
124        };
125        if n == 0 {
126            return Err(TextError::Invalid("token limit must be positive".into()));
127        }
128        limits.check("token policy", n, limits.max_tokens)?;
129        inner.with_padding(None);
130        inner.with_truncation(None).map_err(TextError::Tokenizer)?;
131        Ok(Self {
132            inner,
133            limits,
134            truncation,
135            add_special_tokens,
136        })
137    }
138    /// Loads bounded local tokenizer JSON. No model or vocabulary is downloaded.
139    pub fn from_file(
140        path: impl AsRef<Path>,
141        truncation: Truncation,
142        add_special_tokens: bool,
143        limits: ResourceLimits,
144    ) -> Result<Self> {
145        let bytes = limits.read_encoded(File::open(path)?)?;
146        Self::new(
147            tokenizers::Tokenizer::from_bytes(bytes).map_err(TextError::Tokenizer)?,
148            truncation,
149            add_special_tokens,
150            limits,
151        )
152    }
153    /// Read-only access to vocabulary and advanced tokenizer configuration.
154    pub fn inner(&self) -> &tokenizers::Tokenizer {
155        &self.inner
156    }
157    /// Number of vocabulary entries, including added tokens.
158    pub fn vocabulary_size(&self) -> usize {
159        self.inner.get_vocab_size(true)
160    }
161    /// Encodes one bounded UTF-8 sample with deterministic length policy.
162    pub fn encode(&self, text: &str) -> Result<EncodedSequence> {
163        self.limits
164            .check("text bytes", text.len(), self.limits.max_string_bytes)?;
165        let e = self
166            .inner
167            .encode(text, self.add_special_tokens)
168            .map_err(TextError::Tokenizer)?;
169        self.limits
170            .check("encoded tokens", e.len(), self.limits.max_tokens)?;
171        let n = match self.truncation {
172            Truncation::Reject(n) => {
173                if e.len() > n {
174                    return Err(TextError::Invalid(
175                        "sequence exceeds truncation policy".into(),
176                    ));
177                }
178                e.len()
179            }
180            Truncation::Right(n) => e.len().min(n),
181        };
182        Ok(EncodedSequence {
183            ids: e.get_ids()[..n].to_vec(),
184            type_ids: e.get_type_ids()[..n].to_vec(),
185            special_tokens_mask: e.get_special_tokens_mask()[..n].to_vec(),
186        })
187    }
188}
189/// Local strings encoded lazily during map-style loading.
190pub struct TextDataset {
191    texts: Vec<String>,
192    tokenizer: HfTokenizer,
193}
194impl TextDataset {
195    /// Validates total record count and per-string byte limits before iteration.
196    pub fn new(texts: Vec<String>, tokenizer: HfTokenizer) -> Result<Self> {
197        tokenizer
198            .limits
199            .check("text records", texts.len(), tokenizer.limits.max_records)?;
200        for t in &texts {
201            tokenizer
202                .limits
203                .check("text bytes", t.len(), tokenizer.limits.max_string_bytes)?;
204        }
205        Ok(Self { texts, tokenizer })
206    }
207    /// Measures encoded lengths using exactly the same policy as [`Dataset::get`].
208    pub fn lengths(&self) -> Result<Vec<usize>> {
209        self.texts
210            .iter()
211            .map(|t| self.tokenizer.encode(t).map(|s| s.ids.len()))
212            .collect()
213    }
214}
215impl Dataset for TextDataset {
216    fn get_batch_with_context(
217        &self,
218        indices: &[usize],
219        context: &rusttorch_data::WorkerContext,
220    ) -> Result<Vec<Self::Sample>> {
221        indices
222            .iter()
223            .map(|&index| {
224                context.check().map_err(|_| {
225                    TextError::Invalid("sample loading cancelled or timed out".into())
226                })?;
227                self.get(index)
228            })
229            .collect()
230    }
231
232    type Sample = EncodedSequence;
233    type Error = TextError;
234    fn len(&self) -> usize {
235        self.texts.len()
236    }
237    fn get(&self, index: usize) -> Result<EncodedSequence> {
238        self.tokenizer.encode(
239            self.texts
240                .get(index)
241                .ok_or_else(|| TextError::Invalid("text index out of range".into()))?,
242        )
243    }
244}
245/// Right-padding policy; long sequences are rejected, never silently truncated.
246#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
247pub enum Padding {
248    /// Pad to the longest sample in the batch; all-empty batches have width zero.
249    Longest {
250        /// Vocabulary padding ID.
251        pad_id: u32,
252    },
253    /// Pad every batch to exactly this width.
254    Fixed {
255        /// Vocabulary padding ID.
256        pad_id: u32,
257        /// Required token width.
258        length: usize,
259    },
260}
261/// Model-ready integer tensors and original unpadded lengths.
262#[derive(Debug)]
263pub struct TextBatch {
264    /// Padded `[batch,length]` Int64 vocabulary IDs.
265    pub input_ids: Tensor,
266    /// Int64 mask: one for valid tokens, zero for padding.
267    pub attention_mask: Tensor,
268    /// Padded Int64 segment IDs.
269    pub token_type_ids: Tensor,
270    /// Int64 mask for original special tokens; padded positions are zero.
271    pub special_tokens_mask: Tensor,
272    /// Original token lengths.
273    pub lengths: Vec<usize>,
274}
275impl MemoryFootprint for TextBatch {
276    fn resident_bytes(&self) -> usize {
277        self.input_ids
278            .resident_bytes()
279            .saturating_add(self.attention_mask.resident_bytes())
280            .saturating_add(self.token_type_ids.resident_bytes())
281            .saturating_add(self.special_tokens_mask.resident_bytes())
282            .saturating_add(
283                self.lengths
284                    .capacity()
285                    .saturating_mul(std::mem::size_of::<usize>()),
286            )
287    }
288}
289impl PinMemory for TextBatch {
290    fn pin_memory(mut self, d: Device) -> rusttorch_core::Result<Self> {
291        self.input_ids = self.input_ids.pin_memory(d)?;
292        self.attention_mask = self.attention_mask.pin_memory(d)?;
293        self.token_type_ids = self.token_type_ids.pin_memory(d)?;
294        self.special_tokens_mask = self.special_tokens_mask.pin_memory(d)?;
295        Ok(self)
296    }
297}
298/// Stateless deterministic collator for encoded sequences.
299#[derive(Clone, Copy, Debug)]
300pub struct TextCollator {
301    padding: Padding,
302    limits: ResourceLimits,
303}
304impl TextCollator {
305    /// Validates the fixed padding width against the token ceiling.
306    pub fn new(padding: Padding, limits: ResourceLimits) -> Result<Self> {
307        if let Padding::Fixed { length, .. } = padding {
308            limits.check("padding width", length, limits.max_tokens)?;
309        }
310        Ok(Self { padding, limits })
311    }
312}
313impl Collate<EncodedSequence> for TextCollator {
314    type Batch = TextBatch;
315    type Error = TextError;
316    fn collate(&mut self, samples: Vec<EncodedSequence>) -> Result<TextBatch> {
317        if samples.is_empty() {
318            return Err(TextError::Invalid(
319                "cannot collate an empty text batch".into(),
320            ));
321        }
322        self.limits
323            .check("text batch records", samples.len(), self.limits.max_records)?;
324        for s in &samples {
325            s.validate(self.limits)?;
326        }
327        let longest = samples.iter().map(|s| s.ids.len()).max().unwrap_or(0);
328        let (pad, width) = match self.padding {
329            Padding::Longest { pad_id } => (pad_id, longest),
330            Padding::Fixed { pad_id, length } => (pad_id, length),
331        };
332        if longest > width {
333            return Err(TextError::Invalid("sample exceeds padding width".into()));
334        }
335        let n = self.limits.tensor_elements(&[samples.len(), width])?;
336        self.limits
337            .check("padded tokens", n, self.limits.max_tokens)?;
338        self.limits.check(
339            "text batch bytes",
340            n.checked_mul(32)
341                .and_then(|bytes| {
342                    samples
343                        .len()
344                        .checked_mul(std::mem::size_of::<usize>())
345                        .and_then(|lengths| bytes.checked_add(lengths))
346                })
347                .ok_or_else(|| TextError::Invalid("text batch byte overflow".into()))?,
348            self.limits.max_decoded_bytes,
349        )?;
350        let mut ids = vec![pad as i64; n];
351        let mut masks = vec![0i64; n];
352        let mut types = vec![0i64; n];
353        let mut specials = vec![0i64; n];
354        let lengths = samples.iter().map(|s| s.ids.len()).collect();
355        for (row, s) in samples.iter().enumerate() {
356            for i in 0..s.ids.len() {
357                let j = row * width + i;
358                ids[j] = s.ids[i] as i64;
359                masks[j] = 1;
360                types[j] = s.type_ids[i] as i64;
361                specials[j] = s.special_tokens_mask[i] as i64;
362            }
363        }
364        let shape = [samples.len() as i64, width as i64];
365        Ok(TextBatch {
366            input_ids: Tensor::f_from_slice(&ids)?.f_reshape(shape)?,
367            attention_mask: Tensor::f_from_slice(&masks)?.f_reshape(shape)?,
368            token_type_ids: Tensor::f_from_slice(&types)?.f_reshape(shape)?,
369            special_tokens_mask: Tensor::f_from_slice(&specials)?.f_reshape(shape)?,
370            lengths,
371        })
372    }
373}
374impl rusttorch_data::Checkpointable for TextCollator {
375    type State = (Padding, ResourceLimits);
376    fn save_state(&self) -> Self::State {
377        (self.padding, self.limits)
378    }
379    fn validate_state(&self, s: &Self::State) -> rusttorch_core::Result<()> {
380        if *s != self.save_state() {
381            return Err(invalid("text collator policy differs"));
382        }
383        Ok(())
384    }
385    fn load_validated(&mut self, _: &Self::State) {}
386}
387
388/// Token-budget and distributed-rank policy. Padded token cost is
389/// `batch_size * longest_sequence`, and every sequence must fit by itself.
390#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
391pub struct TokenBudgetConfig {
392    /// Maximum padded tokens in a batch.
393    pub max_tokens: usize,
394    /// Maximum samples, including empty sequences, in a batch.
395    pub max_samples: usize,
396    /// Total distributed ranks.
397    pub replicas: usize,
398    /// This rank, starting at zero.
399    pub rank: usize,
400    /// Shuffle with the supplied seed and epoch.
401    pub shuffle: bool,
402    /// Base permutation seed.
403    pub seed: u64,
404    /// Drop the incomplete distributed tail instead of padding rank indices.
405    pub drop_rank_tail: bool,
406}
407impl Default for TokenBudgetConfig {
408    fn default() -> Self {
409        Self {
410            max_tokens: 4096,
411            max_samples: 64,
412            replicas: 1,
413            rank: 0,
414            shuffle: false,
415            seed: 0,
416            drop_rank_tail: false,
417        }
418    }
419}
420/// Checkpointable variable-size batches over measured sequence lengths.
421/// Distributed ranks receive equal sample counts, but can receive different
422/// batch counts; synchronized training must account for uneven batch counts.
423pub struct TokenBudgetSampler {
424    lengths: Vec<usize>,
425    config: TokenBudgetConfig,
426    sampler: DistributedSampler,
427}
428/// Versioned token-budget checkpoint containing immutable sequence lengths.
429#[derive(Clone, Serialize, Deserialize)]
430pub struct TokenBudgetState {
431    version: u32,
432    lengths: Vec<usize>,
433    config: TokenBudgetConfig,
434    sampler: DistributedSamplerState,
435    next_batch: u64,
436    next_sample: u64,
437}
438fn invalid(reason: &str) -> RustTorchError {
439    RustTorchError::InvalidConfiguration {
440        field: "token budget",
441        reason: reason.into(),
442    }
443}
444impl TokenBudgetSampler {
445    /// Validates positive budgets and every measured length before batching.
446    pub fn new(
447        lengths: Vec<usize>,
448        config: TokenBudgetConfig,
449        limits: ResourceLimits,
450    ) -> rusttorch_core::Result<Self> {
451        limits.check("token lengths", lengths.len(), limits.max_records)?;
452        limits.check("token budget", config.max_tokens, limits.max_tokens)?;
453        if config.max_tokens == 0
454            || config.max_samples == 0
455            || lengths.iter().any(|&n| n > config.max_tokens)
456        {
457            return Err(invalid(
458                "positive budgets required and every sequence must fit",
459            ));
460        }
461        let sampler = DistributedSampler::new(
462            lengths.len(),
463            config.replicas,
464            config.rank,
465            config.shuffle,
466            config.seed,
467            config.drop_rank_tail,
468        )?;
469        Ok(Self {
470            lengths,
471            config,
472            sampler,
473        })
474    }
475    fn group(&self, indices: impl Iterator<Item = usize>) -> Vec<Vec<usize>> {
476        let mut batches = Vec::new();
477        let mut batch = Vec::new();
478        let mut longest = 0;
479        for index in indices {
480            let next = longest.max(self.lengths[index]);
481            if !batch.is_empty()
482                && (batch.len() == self.config.max_samples
483                    || next
484                        .checked_mul(batch.len() + 1)
485                        .is_none_or(|n| n > self.config.max_tokens))
486            {
487                batches.push(std::mem::take(&mut batch));
488                longest = 0;
489            }
490            longest = longest.max(self.lengths[index]);
491            batch.push(index);
492        }
493        if !batch.is_empty() {
494            batches.push(batch);
495        }
496        batches
497    }
498    fn boundary(&self, batch: u64, sample: u64, epoch: u64) -> rusttorch_core::Result<()> {
499        let mut sampler = DistributedSampler::new(
500            self.lengths.len(),
501            self.config.replicas,
502            self.config.rank,
503            self.config.shuffle,
504            self.config.seed,
505            self.config.drop_rank_tail,
506        )?;
507        sampler.set_epoch(epoch);
508        let batches = self.group(sampler.iter());
509        let b = usize::try_from(batch).map_err(|_| invalid("batch position overflow"))?;
510        if b > batches.len() || batches[..b].iter().map(Vec::len).sum::<usize>() as u64 != sample {
511            return Err(invalid("checkpoint is not a token-budget batch boundary"));
512        }
513        Ok(())
514    }
515}
516impl BatchSource for TokenBudgetSampler {
517    type Iter = std::vec::IntoIter<Vec<usize>>;
518    fn iter(&self) -> Self::Iter {
519        self.group(self.sampler.iter()).into_iter()
520    }
521    fn exact_len(&self) -> Option<usize> {
522        Some(self.group(self.sampler.iter()).len())
523    }
524    fn epoch(&self) -> u64 {
525        self.sampler.epoch()
526    }
527    fn set_epoch(&mut self, epoch: u64) {
528        self.sampler.set_epoch(epoch);
529    }
530}
531impl BatchSourceCheckpoint for TokenBudgetSampler {
532    type State = TokenBudgetState;
533    fn kind(&self) -> String {
534        "token_budget/distributed/v1".into()
535    }
536    fn checkpoint_kind_from_state(_: &Self::State) -> String {
537        "token_budget/distributed/v1".into()
538    }
539    fn checkpoint_state(&self, b: u64, s: u64) -> rusttorch_core::Result<Self::State> {
540        self.boundary(b, s, self.sampler.epoch())?;
541        Ok(TokenBudgetState {
542            version: 1,
543            lengths: self.lengths.clone(),
544            config: self.config.clone(),
545            sampler: self.sampler.checkpoint_state(s)?,
546            next_batch: b,
547            next_sample: s,
548        })
549    }
550    fn validate_checkpoint_state(
551        &self,
552        state: &Self::State,
553        epoch: u64,
554        b: u64,
555        s: u64,
556    ) -> rusttorch_core::Result<()> {
557        if state.version != 1
558            || state.lengths != self.lengths
559            || state.config != self.config
560            || state.next_batch != b
561            || state.next_sample != s
562        {
563            return Err(invalid("checkpoint token-budget configuration differs"));
564        }
565        self.sampler
566            .validate_checkpoint_state(&state.sampler, epoch, s)?;
567        self.boundary(b, s, epoch)
568    }
569    fn restore_checkpoint_iter_validated(&mut self, state: &Self::State) -> Self::Iter {
570        let indices = self
571            .sampler
572            .restore_checkpoint_iter_validated(&state.sampler);
573        self.group(indices).into_iter()
574    }
575    fn distributed_configuration(&self) -> Option<DistributedConfiguration> {
576        self.sampler.distributed_configuration()
577    }
578    fn checkpoint_distributed_configuration_from_state(
579        state: &Self::State,
580    ) -> rusttorch_core::Result<Option<DistributedConfiguration>> {
581        DistributedSampler::checkpoint_distributed_configuration_from_state(&state.sampler)
582    }
583}