1#![deny(missing_docs)]
20#![doc = include_str!("../README.md")]
21use rusttorch_core::{Device, Tensor};
22use rusttorch_data::{
23 CollateError, DefaultCollate, DefaultConvert, MemoryFootprint, PinMemory, ResourceLimits,
24};
25use serde::{Deserialize, Serialize};
26use std::{
27 collections::{BTreeMap, BTreeSet},
28 io::{BufRead, BufReader, Read},
29};
30#[derive(Debug, thiserror::Error)]
32#[non_exhaustive]
33pub enum TabularError {
34 #[error(transparent)]
36 Csv(#[from] csv::Error),
37 #[error(transparent)]
39 Json(#[from] serde_json::Error),
40 #[error(transparent)]
42 Io(#[from] std::io::Error),
43 #[error("{0}")]
45 Invalid(String),
46 #[error(transparent)]
48 Limit(#[from] rusttorch_core::RustTorchError),
49 #[error(transparent)]
51 Tensor(#[from] tch::TchError),
52 #[cfg(feature = "arrow")]
54 #[error(transparent)]
55 Arrow(#[from] arrow_schema::ArrowError),
56 #[cfg(feature = "parquet")]
58 #[error(transparent)]
59 Parquet(#[from] parquet::errors::ParquetError),
60}
61pub type Result<T> = std::result::Result<T, TabularError>;
63#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
66pub struct Row {
67 pub fields: BTreeMap<String, Option<String>>,
69}
70impl Row {
71 fn validate(&self, limits: ResourceLimits) -> Result<()> {
72 limits.check("row columns", self.fields.len(), limits.max_columns)?;
73 for (name, value) in &self.fields {
74 limits.check("column name", name.len(), limits.max_string_bytes)?;
75 if let Some(v) = value {
76 limits.check("field bytes", v.len(), limits.max_string_bytes)?;
77 }
78 }
79 Ok(())
80 }
81}
82impl MemoryFootprint for Row {
83 fn resident_bytes(&self) -> usize {
84 self.fields.resident_bytes()
85 }
86}
87impl DefaultConvert for Row {
88 type Output = Self;
89 fn default_convert(self) -> std::result::Result<Self, CollateError> {
90 Ok(self)
91 }
92}
93pub struct CsvRecords<R: Read> {
96 reader: csv::Reader<std::io::Take<R>>,
97 headers: Vec<String>,
98 limits: ResourceLimits,
99 count: usize,
100 done: bool,
101}
102impl<R: Read> CsvRecords<R> {
103 pub fn new(reader: R, limits: ResourceLimits) -> Result<Self> {
105 let mut reader = csv::ReaderBuilder::new()
106 .from_reader(reader.take((limits.max_encoded_bytes as u64).saturating_add(1)));
107 let headers = reader
108 .headers()?
109 .iter()
110 .map(str::to_owned)
111 .collect::<Vec<_>>();
112 limits.check("CSV columns", headers.len(), limits.max_columns)?;
113 let unique = headers.iter().collect::<BTreeSet<_>>();
114 if headers.is_empty()
115 || unique.len() != headers.len()
116 || headers.iter().any(String::is_empty)
117 {
118 return Err(TabularError::Invalid(
119 "CSV requires nonempty unique headers".into(),
120 ));
121 }
122 for h in &headers {
123 limits.check("CSV header bytes", h.len(), limits.max_string_bytes)?;
124 }
125 Ok(Self {
126 reader,
127 headers,
128 limits,
129 count: 0,
130 done: false,
131 })
132 }
133}
134impl<R: Read> Iterator for CsvRecords<R> {
135 type Item = Result<Row>;
136 fn next(&mut self) -> Option<Self::Item> {
137 if self.done {
138 return None;
139 }
140 let result = (|| {
141 let mut record = csv::StringRecord::new();
142 let has = self.reader.read_record(&mut record)?;
143 self.limits.check(
144 "CSV encoded bytes",
145 usize::try_from(self.reader.position().byte()).unwrap_or(usize::MAX),
146 self.limits.max_encoded_bytes,
147 )?;
148 if !has {
149 return Ok(None);
150 }
151 self.count = self
152 .count
153 .checked_add(1)
154 .ok_or_else(|| TabularError::Invalid("record counter overflow".into()))?;
155 self.limits
156 .check("CSV records", self.count, self.limits.max_records)?;
157 let row = Row {
158 fields: self
159 .headers
160 .iter()
161 .cloned()
162 .zip(record.iter().map(|s| {
163 if s.is_empty() {
164 None
165 } else {
166 Some(s.to_owned())
167 }
168 }))
169 .collect(),
170 };
171 row.validate(self.limits)?;
172 Ok(Some(row))
173 })();
174 match result {
175 Ok(Some(row)) => Some(Ok(row)),
176 Ok(None) => {
177 self.done = true;
178 None
179 }
180 Err(e) => {
181 self.done = true;
182 Some(Err(e))
183 }
184 }
185 }
186}
187pub struct JsonLines<R: BufRead> {
189 reader: R,
190 limits: ResourceLimits,
191 line: usize,
192 encoded: usize,
193 done: bool,
194}
195impl<R: BufRead> JsonLines<R> {
196 pub fn new(reader: R, limits: ResourceLimits) -> Self {
198 Self {
199 reader,
200 limits,
201 line: 0,
202 encoded: 0,
203 done: false,
204 }
205 }
206}
207impl<R: BufRead> Iterator for JsonLines<R> {
208 type Item = Result<Row>;
209 fn next(&mut self) -> Option<Self::Item> {
210 if self.done {
211 return None;
212 }
213 let result = (|| {
214 let mut bytes = Vec::new();
215 let remaining = self.limits.max_encoded_bytes.saturating_sub(self.encoded);
216 let ceiling = remaining
217 .min(self.limits.max_string_bytes)
218 .saturating_add(1);
219 let n = (&mut self.reader)
220 .take(ceiling as u64)
221 .read_until(b'\n', &mut bytes)?;
222 if n == 0 {
223 return Ok(None);
224 }
225 self.encoded = self
226 .encoded
227 .checked_add(n)
228 .ok_or_else(|| TabularError::Invalid("encoded byte count overflow".into()))?;
229 self.limits.check(
230 "JSONL encoded bytes",
231 self.encoded,
232 self.limits.max_encoded_bytes,
233 )?;
234 self.limits
235 .check("JSONL line bytes", n, self.limits.max_string_bytes)?;
236 self.line += 1;
237 self.limits
238 .check("JSONL records", self.line, self.limits.max_records)?;
239 let object: BTreeMap<String, serde_json::Value> = serde_json::from_slice(&bytes)?;
240 let fields=object.into_iter().map(|(key,v)|{let value=match v{serde_json::Value::Null=>None,serde_json::Value::String(v)=>Some(v),serde_json::Value::Number(v)=>Some(v.to_string()),_=>return Err(TabularError::Invalid(format!("JSONL line {} column {key}: expected scalar string, number or null",self.line)))};Ok((key,value))}).collect::<Result<_>>()?;
241 let row = Row { fields };
242 row.validate(self.limits)?;
243 Ok(Some(row))
244 })();
245 match result {
246 Ok(Some(row)) => Some(Ok(row)),
247 Ok(None) => {
248 self.done = true;
249 None
250 }
251 Err(e) => {
252 self.done = true;
253 Some(Err(e))
254 }
255 }
256 }
257}
258#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)]
260pub enum MissingValue {
261 Reject,
263 Mean,
265}
266#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
268pub enum Column {
269 Numeric {
271 name: String,
273 missing: MissingValue,
275 },
276 Categorical {
278 name: String,
280 unknown_zero: bool,
282 },
283}
284impl Column {
285 fn name(&self) -> &str {
286 match self {
287 Self::Numeric { name, .. } | Self::Categorical { name, .. } => name,
288 }
289 }
290}
291#[derive(Clone, Debug, Serialize, PartialEq, Eq)]
293pub struct Schema {
294 columns: Vec<Column>,
295}
296impl<'de> Deserialize<'de> for Schema {
297 fn deserialize<D: serde::Deserializer<'de>>(
298 deserializer: D,
299 ) -> std::result::Result<Self, D::Error> {
300 #[derive(Deserialize)]
301 struct Fields {
302 columns: Vec<Column>,
303 }
304 let fields = Fields::deserialize(deserializer)?;
305 Self::new(fields.columns).map_err(serde::de::Error::custom)
306 }
307}
308impl Schema {
309 pub fn new(columns: Vec<Column>) -> Result<Self> {
311 let names = columns.iter().map(Column::name).collect::<BTreeSet<_>>();
312 if columns.is_empty() || names.len() != columns.len() || names.contains("") {
313 return Err(TabularError::Invalid(
314 "schema requires unique nonempty column names".into(),
315 ));
316 }
317 Ok(Self { columns })
318 }
319 pub fn columns(&self) -> &[Column] {
321 &self.columns
322 }
323}
324#[derive(Clone, Debug, Serialize, Deserialize)]
325enum FittedColumn {
326 Numeric { mean: f64, scale: f64 },
327 Categorical { values: Vec<String> },
328}
329#[derive(Clone, Debug, Serialize)]
331pub struct FittedPreprocessor {
332 version: u32,
333 schema: Schema,
334 columns: Vec<FittedColumn>,
335 #[serde(skip)]
336 limits: ResourceLimits,
337}
338#[derive(Deserialize)]
339struct SerializedFit {
340 version: u32,
341 schema: Schema,
342 columns: Vec<FittedColumn>,
343}
344impl FittedPreprocessor {
345 pub fn fit(
348 schema: Schema,
349 rows: impl IntoIterator<Item = Result<Row>>,
350 limits: ResourceLimits,
351 ) -> Result<Self> {
352 limits.check("schema columns", schema.columns.len(), limits.max_columns)?;
353 for column in &schema.columns {
354 limits.check("column name", column.name().len(), limits.max_string_bytes)?;
355 }
356 let mut stats = vec![(0usize, 0f64, 0f64); schema.columns.len()];
357 let mut vocab = vec![BTreeSet::new(); schema.columns.len()];
358 let mut count = 0;
359 let mut vocabulary_bytes = 0usize;
360 for row in rows {
361 let row = row?;
362 row.validate(limits)?;
363 count += 1;
364 limits.check("training records", count, limits.max_records)?;
365 for (i, c) in schema.columns.iter().enumerate() {
366 let value = row.fields.get(c.name()).and_then(Option::as_deref);
367 match c {
368 Column::Numeric { missing, .. } => {
369 let Some(value) = value else {
370 if *missing == MissingValue::Reject {
371 return Err(TabularError::Invalid(format!(
372 "missing numeric column {}",
373 c.name()
374 )));
375 }
376 continue;
377 };
378 let x = parse_number(value, c.name())?;
379 let (n, mean, m2) = &mut stats[i];
380 *n += 1;
381 let d = x - *mean;
382 *mean += d / (*n as f64);
383 *m2 += d * (x - *mean);
384 if !mean.is_finite() || !m2.is_finite() {
385 return Err(TabularError::Invalid(
386 "training statistics overflow".into(),
387 ));
388 }
389 }
390 Column::Categorical { unknown_zero, .. } => {
391 if let Some(v) = value {
392 if !vocab[i].contains(v) {
393 vocabulary_bytes =
394 vocabulary_bytes.checked_add(v.len()).ok_or_else(|| {
395 TabularError::Invalid("vocabulary size overflow".into())
396 })?;
397 limits.check(
398 "fitted vocabulary bytes",
399 vocabulary_bytes,
400 limits.max_decoded_bytes,
401 )?;
402 limits.check(
403 "vocabulary size",
404 vocab[i].len().saturating_add(1),
405 limits.max_records,
406 )?;
407 vocab[i].insert(v.to_owned());
408 }
409 } else if !unknown_zero {
410 return Err(TabularError::Invalid(format!(
411 "missing category {}",
412 c.name()
413 )));
414 }
415 }
416 }
417 }
418 }
419 if count == 0 {
420 return Err(TabularError::Invalid(
421 "cannot fit empty training input".into(),
422 ));
423 }
424 let columns = schema
425 .columns
426 .iter()
427 .enumerate()
428 .map(|(i, c)| match c {
429 Column::Numeric { .. } => {
430 let (n, mean, m2) = stats[i];
431 if n == 0 {
432 return Err(TabularError::Invalid(format!(
433 "no numeric training values for {}",
434 c.name()
435 )));
436 }
437 let scale = (m2 / (n as f64)).max(0.).sqrt();
438 Ok(FittedColumn::Numeric {
439 mean,
440 scale: if scale == 0. { 1. } else { scale },
441 })
442 }
443 Column::Categorical { .. } => Ok(FittedColumn::Categorical {
444 values: std::mem::take(&mut vocab[i]).into_iter().collect(),
445 }),
446 })
447 .collect::<Result<_>>()?;
448 Ok(Self {
449 version: 1,
450 schema,
451 columns,
452 limits,
453 })
454 }
455 pub fn transform(&self, row: &Row) -> Result<TabularSample> {
457 row.validate(self.limits)?;
458 let mut numeric = Vec::new();
459 let mut categorical = Vec::new();
460 for (c, f) in self.schema.columns.iter().zip(&self.columns) {
461 let v = row.fields.get(c.name()).and_then(Option::as_deref);
462 match (c, f) {
463 (Column::Numeric { missing, .. }, FittedColumn::Numeric { mean, scale }) => {
464 let x = match v {
465 Some(v) => parse_number(v, c.name())?,
466 None if *missing == MissingValue::Mean => *mean,
467 None => {
468 return Err(TabularError::Invalid(format!(
469 "missing numeric column {}",
470 c.name()
471 )));
472 }
473 };
474 let normalized = ((x - mean) / scale) as f32;
475 if !normalized.is_finite() {
476 return Err(TabularError::Invalid(
477 "normalized feature is not finite Float".into(),
478 ));
479 }
480 numeric.push(normalized);
481 }
482 (
483 Column::Categorical { unknown_zero, .. },
484 FittedColumn::Categorical { values },
485 ) => {
486 let id = v
487 .and_then(|v| values.binary_search_by(|s| s.as_str().cmp(v)).ok())
488 .map(|i| i as i64 + 1);
489 categorical.push(match id {
490 Some(id) => id,
491 None if *unknown_zero => 0,
492 None => {
493 return Err(TabularError::Invalid(format!(
494 "unknown or missing category {}",
495 c.name()
496 )));
497 }
498 });
499 }
500 _ => {
501 return Err(TabularError::Invalid(
502 "fitted column type differs from schema".into(),
503 ));
504 }
505 }
506 }
507 self.limits
508 .tensor_elements(&[numeric.len().saturating_add(categorical.len())])?;
509 Ok(TabularSample {
510 numeric: Tensor::f_from_slice(&numeric)?,
511 categorical: Tensor::f_from_slice(&categorical)?,
512 })
513 }
514 pub fn to_json(&self) -> Result<String> {
516 Ok(serde_json::to_string(self)?)
517 }
518 pub fn from_json(json: &str, limits: ResourceLimits) -> Result<Self> {
521 limits.check("fitted state bytes", json.len(), limits.max_encoded_bytes)?;
522 let state: SerializedFit = serde_json::from_str(json)?;
523 let schema = Schema::new(state.schema.columns)?;
524 limits.check("fitted columns", schema.columns.len(), limits.max_columns)?;
525 if state.version != 1 || state.columns.len() != schema.columns.len() {
526 return Err(TabularError::Invalid(
527 "unsupported fitted state version or column count".into(),
528 ));
529 }
530 let mut vocabulary_bytes = 0usize;
531 for (c, f) in schema.columns.iter().zip(&state.columns) {
532 limits.check("column name", c.name().len(), limits.max_string_bytes)?;
533 match (c, f) {
534 (Column::Numeric { .. }, FittedColumn::Numeric { mean, scale })
535 if mean.is_finite() && scale.is_finite() && *scale > 0. => {}
536 (Column::Categorical { .. }, FittedColumn::Categorical { values }) => {
537 limits.check("vocabulary size", values.len(), limits.max_records)?;
538 if values.windows(2).any(|w| w[0] >= w[1]) {
539 return Err(TabularError::Invalid(
540 "vocabulary must be sorted and unique".into(),
541 ));
542 }
543 for v in values {
544 vocabulary_bytes =
545 vocabulary_bytes.checked_add(v.len()).ok_or_else(|| {
546 TabularError::Invalid("vocabulary size overflow".into())
547 })?;
548 limits.check(
549 "fitted vocabulary bytes",
550 vocabulary_bytes,
551 limits.max_decoded_bytes,
552 )?;
553 limits.check("category bytes", v.len(), limits.max_string_bytes)?;
554 }
555 }
556 _ => {
557 return Err(TabularError::Invalid(
558 "invalid fitted statistics or column kind".into(),
559 ));
560 }
561 }
562 }
563 Ok(Self {
564 version: 1,
565 schema,
566 columns: state.columns,
567 limits,
568 })
569 }
570 pub fn transform_rows<'a, I>(
573 &'a self,
574 rows: I,
575 ) -> impl Iterator<Item = Result<TabularSample>> + 'a
576 where
577 I: IntoIterator<Item = Result<Row>> + 'a,
578 I::IntoIter: 'a,
579 {
580 rows.into_iter().map(|row| self.transform(&row?))
581 }
582}
583fn parse_number(value: &str, name: &str) -> Result<f64> {
584 let x = value
585 .parse::<f64>()
586 .map_err(|_| TabularError::Invalid(format!("column {name}: invalid number {value:?}")))?;
587 if !x.is_finite() {
588 return Err(TabularError::Invalid(format!(
589 "column {name}: non-finite number"
590 )));
591 }
592 Ok(x)
593}
594#[derive(Debug)]
596pub struct TabularSample {
597 pub numeric: Tensor,
599 pub categorical: Tensor,
601}
602#[derive(Debug)]
604pub struct TabularBatch {
605 pub numeric: Tensor,
607 pub categorical: Tensor,
609}
610impl DefaultCollate for TabularSample {
611 type Batch = TabularBatch;
612 fn default_collate(samples: Vec<Self>) -> std::result::Result<TabularBatch, CollateError> {
613 let (n, c): (Vec<_>, Vec<_>) = samples
614 .into_iter()
615 .map(|s| (s.numeric, s.categorical))
616 .unzip();
617 Ok(TabularBatch {
618 numeric: Tensor::default_collate(n)?,
619 categorical: Tensor::default_collate(c)?,
620 })
621 }
622}
623impl DefaultConvert for TabularSample {
624 type Output = Self;
625 fn default_convert(self) -> std::result::Result<Self, CollateError> {
626 Ok(self)
627 }
628}
629impl MemoryFootprint for TabularSample {
630 fn resident_bytes(&self) -> usize {
631 self.numeric
632 .resident_bytes()
633 .saturating_add(self.categorical.resident_bytes())
634 }
635}
636impl MemoryFootprint for TabularBatch {
637 fn resident_bytes(&self) -> usize {
638 self.numeric
639 .resident_bytes()
640 .saturating_add(self.categorical.resident_bytes())
641 }
642}
643impl PinMemory for TabularSample {
644 fn pin_memory(mut self, d: Device) -> rusttorch_core::Result<Self> {
645 self.numeric = self.numeric.pin_memory(d)?;
646 self.categorical = self.categorical.pin_memory(d)?;
647 Ok(self)
648 }
649}
650impl PinMemory for TabularBatch {
651 fn pin_memory(mut self, d: Device) -> rusttorch_core::Result<Self> {
652 self.numeric = self.numeric.pin_memory(d)?;
653 self.categorical = self.categorical.pin_memory(d)?;
654 Ok(self)
655 }
656}
657
658pub fn open_jsonl(
660 path: impl AsRef<std::path::Path>,
661 limits: ResourceLimits,
662) -> Result<JsonLines<BufReader<std::fs::File>>> {
663 Ok(JsonLines::new(
664 BufReader::new(std::fs::File::open(path)?),
665 limits,
666 ))
667}
668pub fn open_csv(
670 path: impl AsRef<std::path::Path>,
671 limits: ResourceLimits,
672) -> Result<CsvRecords<BufReader<std::fs::File>>> {
673 CsvRecords::new(BufReader::new(std::fs::File::open(path)?), limits)
674}
675
676#[cfg(feature = "arrow")]
677mod columnar;
678#[cfg(feature = "parquet")]
679pub use columnar::open_parquet;
680#[cfg(feature = "arrow")]
681pub use columnar::{ArrowRows, open_arrow};
682
683pub struct TabularDataset {
686 rows: Vec<Row>,
687 preprocessing: FittedPreprocessor,
688}
689impl TabularDataset {
690 pub fn new(rows: Vec<Row>, preprocessing: FittedPreprocessor) -> Result<Self> {
692 preprocessing.limits.check(
693 "tabular records",
694 rows.len(),
695 preprocessing.limits.max_records,
696 )?;
697 let mut bytes = 0usize;
698 for row in &rows {
699 row.validate(preprocessing.limits)?;
700 bytes = bytes.saturating_add(row.resident_bytes());
701 }
702 preprocessing.limits.check(
703 "retained tabular bytes",
704 bytes,
705 preprocessing.limits.max_decoded_bytes,
706 )?;
707 Ok(Self {
708 rows,
709 preprocessing,
710 })
711 }
712}
713impl rusttorch_data::Dataset for TabularDataset {
714 fn get_batch_with_context(
715 &self,
716 indices: &[usize],
717 context: &rusttorch_data::WorkerContext,
718 ) -> Result<Vec<Self::Sample>> {
719 indices
720 .iter()
721 .map(|&index| {
722 context.check().map_err(|_| {
723 TabularError::Invalid("sample loading cancelled or timed out".into())
724 })?;
725 self.get(index)
726 })
727 .collect()
728 }
729 type Sample = TabularSample;
730 type Error = TabularError;
731 fn len(&self) -> usize {
732 self.rows.len()
733 }
734 fn get(&self, index: usize) -> Result<TabularSample> {
735 self.preprocessing.transform(
736 self.rows
737 .get(index)
738 .ok_or_else(|| TabularError::Invalid("tabular index out of range".into()))?,
739 )
740 }
741}