Skip to main content

rusttorch_data/
limits.rs

1//! Shared finite limits for decoding and parsing untrusted local data.
2
3use std::io::{self, Read};
4
5use rusttorch_core::{Result, RustTorchError};
6use serde::{Deserialize, Serialize};
7
8/// Resource ceilings applied before allocation whenever metadata permits.
9///
10/// These bound logical input/output payloads, not native decoder RSS. Loader
11/// queue bytes remain a separate [`crate::MemoryFootprint`] budget.
12///
13/// ```
14/// use rusttorch_data::ResourceLimits;
15/// let limits = ResourceLimits::default();
16/// limits.check("tokens", 128, limits.max_tokens)?;
17/// assert!(limits.check("tokens", usize::MAX, limits.max_tokens).is_err());
18/// # Ok::<(), rusttorch_core::RustTorchError>(())
19/// ```
20#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
21pub struct ResourceLimits {
22    /// Maximum encoded bytes per local file or bounded input (64 MiB).
23    pub max_encoded_bytes: usize,
24    /// Maximum decoded sample or explicitly bounded batch payload (256 MiB).
25    pub max_decoded_bytes: usize,
26    /// Maximum elements in one tensor (64 million).
27    pub max_tensor_elements: usize,
28    /// Maximum image width or height (16,384).
29    pub max_image_dimension: usize,
30    /// Maximum records in a dataset or explicitly bounded batch (1 million).
31    pub max_records: usize,
32    /// Maximum fields per tabular record (4,096).
33    pub max_columns: usize,
34    /// Maximum UTF-8 bytes per field or text sample (1 MiB).
35    pub max_string_bytes: usize,
36    /// Maximum tokens per sequence or padded batch (1 million).
37    pub max_tokens: usize,
38    /// Maximum Arrow record-batch or Parquet row-group payload (256 MiB).
39    pub max_columnar_bytes: usize,
40    /// Maximum media duration in whole seconds (one hour).
41    pub max_media_seconds: u64,
42    /// Maximum streams per media container (32).
43    pub max_streams: usize,
44}
45
46impl Default for ResourceLimits {
47    fn default() -> Self {
48        Self {
49            max_encoded_bytes: 64 * 1024 * 1024,
50            max_decoded_bytes: 256 * 1024 * 1024,
51            max_tensor_elements: 64 * 1024 * 1024,
52            max_image_dimension: 16_384,
53            max_records: 1_000_000,
54            max_columns: 4_096,
55            max_string_bytes: 1024 * 1024,
56            max_tokens: 1_000_000,
57            max_columnar_bytes: 256 * 1024 * 1024,
58            max_media_seconds: 3_600,
59            max_streams: 32,
60        }
61    }
62}
63
64impl ResourceLimits {
65    /// Explicitly disables logical ceilings; integer overflow is still rejected.
66    pub const fn unlimited() -> Self {
67        Self {
68            max_encoded_bytes: usize::MAX,
69            max_decoded_bytes: usize::MAX,
70            max_tensor_elements: usize::MAX,
71            max_image_dimension: usize::MAX,
72            max_records: usize::MAX,
73            max_columns: usize::MAX,
74            max_string_bytes: usize::MAX,
75            max_tokens: usize::MAX,
76            max_columnar_bytes: usize::MAX,
77            max_media_seconds: u64::MAX,
78            max_streams: usize::MAX,
79        }
80    }
81
82    /// Rejects a value above its named ceiling.
83    pub fn check(&self, field: &'static str, actual: usize, maximum: usize) -> Result<()> {
84        if actual > maximum {
85            return Err(RustTorchError::InvalidConfiguration {
86                field,
87                reason: format!("{actual} exceeds resource limit {maximum}"),
88            });
89        }
90        Ok(())
91    }
92
93    /// Checks multiplication and the resulting tensor-element ceiling.
94    pub fn tensor_elements(&self, dimensions: &[usize]) -> Result<usize> {
95        for &dimension in dimensions {
96            self.check("tensor dimension", dimension, i64::MAX as usize)?;
97        }
98        let elements = dimensions
99            .iter()
100            .try_fold(1usize, |n, &d| n.checked_mul(d))
101            .ok_or_else(|| RustTorchError::InvalidConfiguration {
102                field: "tensor elements",
103                reason: "shape product overflows usize".into(),
104            })?;
105        self.check(
106            "tensor elements",
107            elements,
108            self.max_tensor_elements.min(i64::MAX as usize),
109        )?;
110        Ok(elements)
111    }
112
113    /// Reads at most the encoded ceiling plus one sentinel byte, then rejects
114    /// oversized input. The limit is enforced even if file metadata changes.
115    pub fn read_encoded(&self, reader: impl Read) -> io::Result<Vec<u8>> {
116        let mut bytes = Vec::new();
117        reader
118            .take((self.max_encoded_bytes as u64).saturating_add(1))
119            .read_to_end(&mut bytes)?;
120        if bytes.len() > self.max_encoded_bytes {
121            return Err(io::Error::new(
122                io::ErrorKind::InvalidData,
123                "encoded resource limit exceeded",
124            ));
125        }
126        Ok(bytes)
127    }
128}
129
130#[cfg(test)]
131mod tests {
132    use super::*;
133    #[test]
134    fn rejects_oversize_inputs_and_shape_overflow() {
135        let limits = ResourceLimits {
136            max_encoded_bytes: 2,
137            max_tensor_elements: 8,
138            ..Default::default()
139        };
140        assert_eq!(limits.read_encoded(&b"ab"[..]).unwrap(), b"ab");
141        assert!(limits.read_encoded(&b"abc"[..]).is_err());
142        assert!(limits.tensor_elements(&[3, 3]).is_err());
143        assert!(
144            ResourceLimits::unlimited()
145                .tensor_elements(&[usize::MAX, 2])
146                .is_err()
147        );
148    }
149}