Skip to main content

rusttorch_tabular/
lib.rs

1//! Stream local records, fit preprocessing on training data, and reuse that
2//! immutable state for validation and inference.
3//!
4//! ```
5//! use rusttorch_tabular::{CsvRecords, Column, FittedPreprocessor, MissingValue, Schema};
6//! use rusttorch_data::ResourceLimits;
7//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
8//! let limits = ResourceLimits::default();
9//! let schema = Schema::new(vec![Column::Numeric { name: "age".into(), missing: MissingValue::Reject }])?;
10//! let training = CsvRecords::new("age\n20\n40\n".as_bytes(), limits)?;
11//! let fitted = FittedPreprocessor::fit(schema, training, limits)?;
12//! let state = fitted.to_json()?;
13//! let validation = CsvRecords::new("age\n30\n".as_bytes(), limits)?;
14//! let restored = FittedPreprocessor::from_json(&state, limits)?;
15//! let sample = restored.transform(&validation.into_iter().next().unwrap()?)?;
16//! assert_eq!(Vec::<f32>::try_from(&sample.numeric)?, [0.]);
17//! # Ok(()) }
18//! ```
19#![deny(missing_docs)]
20#![doc = include_str!("../README.md")]
21use rusttorch_core::{Device, Tensor};
22use rusttorch_data::{
23    CollateError, DefaultCollate, DefaultConvert, MemoryFootprint, PinMemory, ResourceLimits,
24};
25use serde::{Deserialize, Serialize};
26use std::{
27    collections::{BTreeMap, BTreeSet},
28    io::{BufRead, BufReader, Read},
29};
30/// Parsing, schema, fitted-state or tensor failure.
31#[derive(Debug, thiserror::Error)]
32#[non_exhaustive]
33pub enum TabularError {
34    /// CSV syntax/type error with record position retained.
35    #[error(transparent)]
36    Csv(#[from] csv::Error),
37    /// JSON syntax error with line and column retained.
38    #[error(transparent)]
39    Json(#[from] serde_json::Error),
40    /// Local reader failed.
41    #[error(transparent)]
42    Io(#[from] std::io::Error),
43    /// Invalid schema, cell or fitted state.
44    #[error("{0}")]
45    Invalid(String),
46    /// Finite input/output limit exceeded.
47    #[error(transparent)]
48    Limit(#[from] rusttorch_core::RustTorchError),
49    /// LibTorch failure.
50    #[error(transparent)]
51    Tensor(#[from] tch::TchError),
52    /// Arrow reader or schema error.
53    #[cfg(feature = "arrow")]
54    #[error(transparent)]
55    Arrow(#[from] arrow_schema::ArrowError),
56    /// Parquet reader error.
57    #[cfg(feature = "parquet")]
58    #[error(transparent)]
59    Parquet(#[from] parquet::errors::ParquetError),
60}
61/// Result returned by tabular operations.
62pub type Result<T> = std::result::Result<T, TabularError>;
63/// One named row. Empty/missing cells are `None`; numbers remain decimal strings
64/// until the selected schema parses them. Nested JSON values are rejected.
65#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
66pub struct Row {
67    /// Column names and optional UTF-8 cell values.
68    pub fields: BTreeMap<String, Option<String>>,
69}
70impl Row {
71    fn validate(&self, limits: ResourceLimits) -> Result<()> {
72        limits.check("row columns", self.fields.len(), limits.max_columns)?;
73        for (name, value) in &self.fields {
74            limits.check("column name", name.len(), limits.max_string_bytes)?;
75            if let Some(v) = value {
76                limits.check("field bytes", v.len(), limits.max_string_bytes)?;
77            }
78        }
79        Ok(())
80    }
81}
82impl MemoryFootprint for Row {
83    fn resident_bytes(&self) -> usize {
84        self.fields.resident_bytes()
85    }
86}
87impl DefaultConvert for Row {
88    type Output = Self;
89    fn default_convert(self) -> std::result::Result<Self, CollateError> {
90        Ok(self)
91    }
92}
93/// Streaming CSV reader with mandatory unique headers and finite total input.
94/// Quoted separators/newlines are handled by the established `csv` parser.
95pub struct CsvRecords<R: Read> {
96    reader: csv::Reader<std::io::Take<R>>,
97    headers: Vec<String>,
98    limits: ResourceLimits,
99    count: usize,
100    done: bool,
101}
102impl<R: Read> CsvRecords<R> {
103    /// Creates a reader; a zero-byte/duplicate-header input is rejected.
104    pub fn new(reader: R, limits: ResourceLimits) -> Result<Self> {
105        let mut reader = csv::ReaderBuilder::new()
106            .from_reader(reader.take((limits.max_encoded_bytes as u64).saturating_add(1)));
107        let headers = reader
108            .headers()?
109            .iter()
110            .map(str::to_owned)
111            .collect::<Vec<_>>();
112        limits.check("CSV columns", headers.len(), limits.max_columns)?;
113        let unique = headers.iter().collect::<BTreeSet<_>>();
114        if headers.is_empty()
115            || unique.len() != headers.len()
116            || headers.iter().any(String::is_empty)
117        {
118            return Err(TabularError::Invalid(
119                "CSV requires nonempty unique headers".into(),
120            ));
121        }
122        for h in &headers {
123            limits.check("CSV header bytes", h.len(), limits.max_string_bytes)?;
124        }
125        Ok(Self {
126            reader,
127            headers,
128            limits,
129            count: 0,
130            done: false,
131        })
132    }
133}
134impl<R: Read> Iterator for CsvRecords<R> {
135    type Item = Result<Row>;
136    fn next(&mut self) -> Option<Self::Item> {
137        if self.done {
138            return None;
139        }
140        let result = (|| {
141            let mut record = csv::StringRecord::new();
142            let has = self.reader.read_record(&mut record)?;
143            self.limits.check(
144                "CSV encoded bytes",
145                usize::try_from(self.reader.position().byte()).unwrap_or(usize::MAX),
146                self.limits.max_encoded_bytes,
147            )?;
148            if !has {
149                return Ok(None);
150            }
151            self.count = self
152                .count
153                .checked_add(1)
154                .ok_or_else(|| TabularError::Invalid("record counter overflow".into()))?;
155            self.limits
156                .check("CSV records", self.count, self.limits.max_records)?;
157            let row = Row {
158                fields: self
159                    .headers
160                    .iter()
161                    .cloned()
162                    .zip(record.iter().map(|s| {
163                        if s.is_empty() {
164                            None
165                        } else {
166                            Some(s.to_owned())
167                        }
168                    }))
169                    .collect(),
170            };
171            row.validate(self.limits)?;
172            Ok(Some(row))
173        })();
174        match result {
175            Ok(Some(row)) => Some(Ok(row)),
176            Ok(None) => {
177                self.done = true;
178                None
179            }
180            Err(e) => {
181                self.done = true;
182                Some(Err(e))
183            }
184        }
185    }
186}
187/// Bounded JSON Lines reader. Each line is one flat object; blank lines fail.
188pub struct JsonLines<R: BufRead> {
189    reader: R,
190    limits: ResourceLimits,
191    line: usize,
192    encoded: usize,
193    done: bool,
194}
195impl<R: BufRead> JsonLines<R> {
196    /// Creates a lazy reader without consuming input.
197    pub fn new(reader: R, limits: ResourceLimits) -> Self {
198        Self {
199            reader,
200            limits,
201            line: 0,
202            encoded: 0,
203            done: false,
204        }
205    }
206}
207impl<R: BufRead> Iterator for JsonLines<R> {
208    type Item = Result<Row>;
209    fn next(&mut self) -> Option<Self::Item> {
210        if self.done {
211            return None;
212        }
213        let result = (|| {
214            let mut bytes = Vec::new();
215            let remaining = self.limits.max_encoded_bytes.saturating_sub(self.encoded);
216            let ceiling = remaining
217                .min(self.limits.max_string_bytes)
218                .saturating_add(1);
219            let n = (&mut self.reader)
220                .take(ceiling as u64)
221                .read_until(b'\n', &mut bytes)?;
222            if n == 0 {
223                return Ok(None);
224            }
225            self.encoded = self
226                .encoded
227                .checked_add(n)
228                .ok_or_else(|| TabularError::Invalid("encoded byte count overflow".into()))?;
229            self.limits.check(
230                "JSONL encoded bytes",
231                self.encoded,
232                self.limits.max_encoded_bytes,
233            )?;
234            self.limits
235                .check("JSONL line bytes", n, self.limits.max_string_bytes)?;
236            self.line += 1;
237            self.limits
238                .check("JSONL records", self.line, self.limits.max_records)?;
239            let object: BTreeMap<String, serde_json::Value> = serde_json::from_slice(&bytes)?;
240            let fields=object.into_iter().map(|(key,v)|{let value=match v{serde_json::Value::Null=>None,serde_json::Value::String(v)=>Some(v),serde_json::Value::Number(v)=>Some(v.to_string()),_=>return Err(TabularError::Invalid(format!("JSONL line {} column {key}: expected scalar string, number or null",self.line)))};Ok((key,value))}).collect::<Result<_>>()?;
241            let row = Row { fields };
242            row.validate(self.limits)?;
243            Ok(Some(row))
244        })();
245        match result {
246            Ok(Some(row)) => Some(Ok(row)),
247            Ok(None) => {
248                self.done = true;
249                None
250            }
251            Err(e) => {
252                self.done = true;
253                Some(Err(e))
254            }
255        }
256    }
257}
258/// Missing numeric value behavior.
259#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
260pub enum MissingValue {
261    /// Fail on missing numeric cells.
262    Reject,
263    /// Substitute the training mean, yielding normalized zero.
264    Mean,
265}
266/// One model input column; numeric and categorical output orders follow schema order.
267#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
268pub enum Column {
269    /// Numeric feature standardized with training population variance.
270    Numeric {
271        /// Exact input column name.
272        name: String,
273        /// Missing-value policy.
274        missing: MissingValue,
275    },
276    /// Categorical feature with sorted training vocabulary and Int64 output.
277    Categorical {
278        /// Exact input column name.
279        name: String,
280        /// Reserve ID zero for missing/unknown values; otherwise reject.
281        unknown_zero: bool,
282    },
283}
284impl Column {
285    fn name(&self) -> &str {
286        match self {
287            Self::Numeric { name, .. } | Self::Categorical { name, .. } => name,
288        }
289    }
290}
291/// Nonempty schema with unique column names. Extra input fields are ignored.
292#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
293pub struct Schema {
294    columns: Vec<Column>,
295}
296impl<'de> Deserialize<'de> for Schema {
297    fn deserialize<D: serde::Deserializer<'de>>(
298        deserializer: D,
299    ) -> std::result::Result<Self, D::Error> {
300        #[derive(Deserialize)]
301        struct Fields {
302            columns: Vec<Column>,
303        }
304        let fields = Fields::deserialize(deserializer)?;
305        Self::new(fields.columns).map_err(serde::de::Error::custom)
306    }
307}
308impl Schema {
309    /// Validates unique nonempty names.
310    pub fn new(columns: Vec<Column>) -> Result<Self> {
311        let names = columns.iter().map(Column::name).collect::<BTreeSet<_>>();
312        if columns.is_empty() || names.len() != columns.len() || names.contains("") {
313            return Err(TabularError::Invalid(
314                "schema requires unique nonempty column names".into(),
315            ));
316        }
317        Ok(Self { columns })
318    }
319    /// Columns in model input order.
320    pub fn columns(&self) -> &[Column] {
321        &self.columns
322    }
323}
324#[derive(Clone, Debug, Serialize, Deserialize)]
325enum FittedColumn {
326    Numeric { mean: f64, scale: f64 },
327    Categorical { values: Vec<String> },
328}
329/// Immutable fitted training statistics and categorical vocabularies.
330#[derive(Clone, Debug, Serialize)]
331pub struct FittedPreprocessor {
332    version: u32,
333    schema: Schema,
334    columns: Vec<FittedColumn>,
335    #[serde(skip)]
336    limits: ResourceLimits,
337}
338#[derive(Deserialize)]
339struct SerializedFit {
340    version: u32,
341    schema: Schema,
342    columns: Vec<FittedColumn>,
343}
344impl FittedPreprocessor {
345    /// Fits once over training rows. Numeric variance uses a stable online update;
346    /// validation/test transformation never modifies these statistics.
347    pub fn fit(
348        schema: Schema,
349        rows: impl IntoIterator<Item = Result<Row>>,
350        limits: ResourceLimits,
351    ) -> Result<Self> {
352        limits.check("schema columns", schema.columns.len(), limits.max_columns)?;
353        for column in &schema.columns {
354            limits.check("column name", column.name().len(), limits.max_string_bytes)?;
355        }
356        let mut stats = vec![(0usize, 0f64, 0f64); schema.columns.len()];
357        let mut vocab = vec![BTreeSet::new(); schema.columns.len()];
358        let mut count = 0;
359        let mut vocabulary_bytes = 0usize;
360        for row in rows {
361            let row = row?;
362            row.validate(limits)?;
363            count += 1;
364            limits.check("training records", count, limits.max_records)?;
365            for (i, c) in schema.columns.iter().enumerate() {
366                let value = row.fields.get(c.name()).and_then(Option::as_deref);
367                match c {
368                    Column::Numeric { missing, .. } => {
369                        let Some(value) = value else {
370                            if *missing == MissingValue::Reject {
371                                return Err(TabularError::Invalid(format!(
372                                    "missing numeric column {}",
373                                    c.name()
374                                )));
375                            }
376                            continue;
377                        };
378                        let x = parse_number(value, c.name())?;
379                        let (n, mean, m2) = &mut stats[i];
380                        *n += 1;
381                        let d = x - *mean;
382                        *mean += d / (*n as f64);
383                        *m2 += d * (x - *mean);
384                        if !mean.is_finite() || !m2.is_finite() {
385                            return Err(TabularError::Invalid(
386                                "training statistics overflow".into(),
387                            ));
388                        }
389                    }
390                    Column::Categorical { unknown_zero, .. } => {
391                        if let Some(v) = value {
392                            if !vocab[i].contains(v) {
393                                vocabulary_bytes =
394                                    vocabulary_bytes.checked_add(v.len()).ok_or_else(|| {
395                                        TabularError::Invalid("vocabulary size overflow".into())
396                                    })?;
397                                limits.check(
398                                    "fitted vocabulary bytes",
399                                    vocabulary_bytes,
400                                    limits.max_decoded_bytes,
401                                )?;
402                                limits.check(
403                                    "vocabulary size",
404                                    vocab[i].len().saturating_add(1),
405                                    limits.max_records,
406                                )?;
407                                vocab[i].insert(v.to_owned());
408                            }
409                        } else if !unknown_zero {
410                            return Err(TabularError::Invalid(format!(
411                                "missing category {}",
412                                c.name()
413                            )));
414                        }
415                    }
416                }
417            }
418        }
419        if count == 0 {
420            return Err(TabularError::Invalid(
421                "cannot fit empty training input".into(),
422            ));
423        }
424        let columns = schema
425            .columns
426            .iter()
427            .enumerate()
428            .map(|(i, c)| match c {
429                Column::Numeric { .. } => {
430                    let (n, mean, m2) = stats[i];
431                    if n == 0 {
432                        return Err(TabularError::Invalid(format!(
433                            "no numeric training values for {}",
434                            c.name()
435                        )));
436                    }
437                    let scale = (m2 / (n as f64)).max(0.).sqrt();
438                    Ok(FittedColumn::Numeric {
439                        mean,
440                        scale: if scale == 0. { 1. } else { scale },
441                    })
442                }
443                Column::Categorical { .. } => Ok(FittedColumn::Categorical {
444                    values: std::mem::take(&mut vocab[i]).into_iter().collect(),
445                }),
446            })
447            .collect::<Result<_>>()?;
448        Ok(Self {
449            version: 1,
450            schema,
451            columns,
452            limits,
453        })
454    }
455    /// Transforms one row using stored statistics; does not fit or mutate state.
456    pub fn transform(&self, row: &Row) -> Result<TabularSample> {
457        row.validate(self.limits)?;
458        let mut numeric = Vec::new();
459        let mut categorical = Vec::new();
460        for (c, f) in self.schema.columns.iter().zip(&self.columns) {
461            let v = row.fields.get(c.name()).and_then(Option::as_deref);
462            match (c, f) {
463                (Column::Numeric { missing, .. }, FittedColumn::Numeric { mean, scale }) => {
464                    let x = match v {
465                        Some(v) => parse_number(v, c.name())?,
466                        None if *missing == MissingValue::Mean => *mean,
467                        None => {
468                            return Err(TabularError::Invalid(format!(
469                                "missing numeric column {}",
470                                c.name()
471                            )));
472                        }
473                    };
474                    let normalized = ((x - mean) / scale) as f32;
475                    if !normalized.is_finite() {
476                        return Err(TabularError::Invalid(
477                            "normalized feature is not finite Float".into(),
478                        ));
479                    }
480                    numeric.push(normalized);
481                }
482                (
483                    Column::Categorical { unknown_zero, .. },
484                    FittedColumn::Categorical { values },
485                ) => {
486                    let id = v
487                        .and_then(|v| values.binary_search_by(|s| s.as_str().cmp(v)).ok())
488                        .map(|i| i as i64 + 1);
489                    categorical.push(match id {
490                        Some(id) => id,
491                        None if *unknown_zero => 0,
492                        None => {
493                            return Err(TabularError::Invalid(format!(
494                                "unknown or missing category {}",
495                                c.name()
496                            )));
497                        }
498                    });
499                }
500                _ => {
501                    return Err(TabularError::Invalid(
502                        "fitted column type differs from schema".into(),
503                    ));
504                }
505            }
506        }
507        self.limits
508            .tensor_elements(&[numeric.len().saturating_add(categorical.len())])?;
509        Ok(TabularSample {
510            numeric: Tensor::f_from_slice(&numeric)?,
511            categorical: Tensor::f_from_slice(&categorical)?,
512        })
513    }
514    /// Serializes a versioned immutable fitted state.
515    pub fn to_json(&self) -> Result<String> {
516        Ok(serde_json::to_string(self)?)
517    }
518    /// Restores state after validating sizes, finite statistics, column kinds,
519    /// sorted unique vocabularies and supported schema version.
520    pub fn from_json(json: &str, limits: ResourceLimits) -> Result<Self> {
521        limits.check("fitted state bytes", json.len(), limits.max_encoded_bytes)?;
522        let state: SerializedFit = serde_json::from_str(json)?;
523        let schema = Schema::new(state.schema.columns)?;
524        limits.check("fitted columns", schema.columns.len(), limits.max_columns)?;
525        if state.version != 1 || state.columns.len() != schema.columns.len() {
526            return Err(TabularError::Invalid(
527                "unsupported fitted state version or column count".into(),
528            ));
529        }
530        let mut vocabulary_bytes = 0usize;
531        for (c, f) in schema.columns.iter().zip(&state.columns) {
532            limits.check("column name", c.name().len(), limits.max_string_bytes)?;
533            match (c, f) {
534                (Column::Numeric { .. }, FittedColumn::Numeric { mean, scale })
535                    if mean.is_finite() && scale.is_finite() && *scale > 0. => {}
536                (Column::Categorical { .. }, FittedColumn::Categorical { values }) => {
537                    limits.check("vocabulary size", values.len(), limits.max_records)?;
538                    if values.windows(2).any(|w| w[0] >= w[1]) {
539                        return Err(TabularError::Invalid(
540                            "vocabulary must be sorted and unique".into(),
541                        ));
542                    }
543                    for v in values {
544                        vocabulary_bytes =
545                            vocabulary_bytes.checked_add(v.len()).ok_or_else(|| {
546                                TabularError::Invalid("vocabulary size overflow".into())
547                            })?;
548                        limits.check(
549                            "fitted vocabulary bytes",
550                            vocabulary_bytes,
551                            limits.max_decoded_bytes,
552                        )?;
553                        limits.check("category bytes", v.len(), limits.max_string_bytes)?;
554                    }
555                }
556                _ => {
557                    return Err(TabularError::Invalid(
558                        "invalid fitted statistics or column kind".into(),
559                    ));
560                }
561            }
562        }
563        Ok(Self {
564            version: 1,
565            schema,
566            columns: state.columns,
567            limits,
568        })
569    }
570    /// Lazily converts a record iterator into model inputs suitable for
571    /// `rusttorch_data::batches` or a sharded `StreamDataLoader` source.
572    pub fn transform_rows<'a, I>(
573        &'a self,
574        rows: I,
575    ) -> impl Iterator<Item = Result<TabularSample>> + 'a
576    where
577        I: IntoIterator<Item = Result<Row>> + 'a,
578        I::IntoIter: 'a,
579    {
580        rows.into_iter().map(|row| self.transform(&row?))
581    }
582}
583fn parse_number(value: &str, name: &str) -> Result<f64> {
584    let x = value
585        .parse::<f64>()
586        .map_err(|_| TabularError::Invalid(format!("column {name}: invalid number {value:?}")))?;
587    if !x.is_finite() {
588        return Err(TabularError::Invalid(format!(
589            "column {name}: non-finite number"
590        )));
591    }
592    Ok(x)
593}
594/// One model-ready row, with separate floating-point and categorical inputs.
595#[derive(Debug)]
596pub struct TabularSample {
597    /// Float standardized features, shape `[numeric_columns]`.
598    pub numeric: Tensor,
599    /// Int64 category IDs, shape `[categorical_columns]`.
600    pub categorical: Tensor,
601}
602/// Tabular model batch with one leading sample dimension.
603#[derive(Debug)]
604pub struct TabularBatch {
605    /// Float `[batch,numeric_columns]` features.
606    pub numeric: Tensor,
607    /// Int64 `[batch,categorical_columns]` IDs.
608    pub categorical: Tensor,
609}
610impl DefaultCollate for TabularSample {
611    type Batch = TabularBatch;
612    fn default_collate(samples: Vec<Self>) -> std::result::Result<TabularBatch, CollateError> {
613        let (n, c): (Vec<_>, Vec<_>) = samples
614            .into_iter()
615            .map(|s| (s.numeric, s.categorical))
616            .unzip();
617        Ok(TabularBatch {
618            numeric: Tensor::default_collate(n)?,
619            categorical: Tensor::default_collate(c)?,
620        })
621    }
622}
623impl DefaultConvert for TabularSample {
624    type Output = Self;
625    fn default_convert(self) -> std::result::Result<Self, CollateError> {
626        Ok(self)
627    }
628}
629impl MemoryFootprint for TabularSample {
630    fn resident_bytes(&self) -> usize {
631        self.numeric
632            .resident_bytes()
633            .saturating_add(self.categorical.resident_bytes())
634    }
635}
636impl MemoryFootprint for TabularBatch {
637    fn resident_bytes(&self) -> usize {
638        self.numeric
639            .resident_bytes()
640            .saturating_add(self.categorical.resident_bytes())
641    }
642}
643impl PinMemory for TabularSample {
644    fn pin_memory(mut self, d: Device) -> rusttorch_core::Result<Self> {
645        self.numeric = self.numeric.pin_memory(d)?;
646        self.categorical = self.categorical.pin_memory(d)?;
647        Ok(self)
648    }
649}
650impl PinMemory for TabularBatch {
651    fn pin_memory(mut self, d: Device) -> rusttorch_core::Result<Self> {
652        self.numeric = self.numeric.pin_memory(d)?;
653        self.categorical = self.categorical.pin_memory(d)?;
654        Ok(self)
655    }
656}
657
658/// Reads a local JSON Lines file through a buffered bounded iterator.
659pub fn open_jsonl(
660    path: impl AsRef<std::path::Path>,
661    limits: ResourceLimits,
662) -> Result<JsonLines<BufReader<std::fs::File>>> {
663    Ok(JsonLines::new(
664        BufReader::new(std::fs::File::open(path)?),
665        limits,
666    ))
667}
668/// Reads a local CSV file through a buffered bounded iterator.
669pub fn open_csv(
670    path: impl AsRef<std::path::Path>,
671    limits: ResourceLimits,
672) -> Result<CsvRecords<BufReader<std::fs::File>>> {
673    CsvRecords::new(BufReader::new(std::fs::File::open(path)?), limits)
674}
675
676#[cfg(feature = "arrow")]
677mod columnar;
678#[cfg(feature = "parquet")]
679pub use columnar::open_parquet;
680#[cfg(feature = "arrow")]
681pub use columnar::{ArrowRows, open_arrow};
682
683/// Finite immutable rows transformed lazily for ordinary worker DataLoader use.
684/// For datasets larger than memory, use CSV/JSONL/Arrow streaming iterators.
685pub struct TabularDataset {
686    rows: Vec<Row>,
687    preprocessing: FittedPreprocessor,
688}
689impl TabularDataset {
690    /// Validates row counts and field limits using the fitted state's policy.
691    pub fn new(rows: Vec<Row>, preprocessing: FittedPreprocessor) -> Result<Self> {
692        preprocessing.limits.check(
693            "tabular records",
694            rows.len(),
695            preprocessing.limits.max_records,
696        )?;
697        let mut bytes = 0usize;
698        for row in &rows {
699            row.validate(preprocessing.limits)?;
700            bytes = bytes.saturating_add(row.resident_bytes());
701        }
702        preprocessing.limits.check(
703            "retained tabular bytes",
704            bytes,
705            preprocessing.limits.max_decoded_bytes,
706        )?;
707        Ok(Self {
708            rows,
709            preprocessing,
710        })
711    }
712}
713impl rusttorch_data::Dataset for TabularDataset {
714    fn get_batch_with_context(
715        &self,
716        indices: &[usize],
717        context: &rusttorch_data::WorkerContext,
718    ) -> Result<Vec<Self::Sample>> {
719        indices
720            .iter()
721            .map(|&index| {
722                context.check().map_err(|_| {
723                    TabularError::Invalid("sample loading cancelled or timed out".into())
724                })?;
725                self.get(index)
726            })
727            .collect()
728    }
729    type Sample = TabularSample;
730    type Error = TabularError;
731    fn len(&self) -> usize {
732        self.rows.len()
733    }
734    fn get(&self, index: usize) -> Result<TabularSample> {
735        self.preprocessing.transform(
736            self.rows
737                .get(index)
738                .ok_or_else(|| TabularError::Invalid("tabular index out of range".into()))?,
739        )
740    }
741}