Skip to main content

rusttorch_tabular/
columnar.rs

1use super::{Result, Row, TabularError};
2use arrow_array::{
3    Array, Float32Array, Float64Array, Int32Array, Int64Array, LargeStringArray, RecordBatch,
4    StringArray,
5};
6use arrow_schema::DataType;
7use rusttorch_data::ResourceLimits;
8use std::{fs::File, path::Path};
9/// Lazy row iterator over Arrow record batches. Supports Utf8/LargeUtf8,
10/// Int32/Int64 and Float32/Float64 columns; nested/dictionary types fail explicitly.
11pub struct ArrowRows<I> {
12    batches: I,
13    batch: Option<RecordBatch>,
14    row: usize,
15    total: usize,
16    limits: ResourceLimits,
17    done: bool,
18}
19impl<I> ArrowRows<I> {
20    /// Adapts a batch reader and enforces per-batch bytes, fields and row limits.
21    pub fn new(batches: I, limits: ResourceLimits) -> Self {
22        Self {
23            batches,
24            batch: None,
25            row: 0,
26            total: 0,
27            limits,
28            done: false,
29        }
30    }
31}
32impl<I> Iterator for ArrowRows<I>
33where
34    I: Iterator<Item = std::result::Result<RecordBatch, arrow_schema::ArrowError>>,
35{
36    type Item = Result<Row>;
37    fn next(&mut self) -> Option<Self::Item> {
38        if self.done {
39            return None;
40        }
41        let result = (|| {
42            loop {
43                if self.batch.as_ref().is_none_or(|b| self.row >= b.num_rows()) {
44                    let Some(batch) = self.batches.next() else {
45                        return Ok(None);
46                    };
47                    let batch = batch?;
48                    self.limits.check(
49                        "Arrow columns",
50                        batch.num_columns(),
51                        self.limits.max_columns,
52                    )?;
53                    self.limits.check(
54                        "Arrow batch bytes",
55                        batch.get_array_memory_size(),
56                        self.limits.max_columnar_bytes,
57                    )?;
58                    if batch.num_rows() > self.limits.max_records.saturating_sub(self.total) {
59                        return Err(TabularError::Invalid("Arrow row limit exceeded".into()));
60                    }
61                    let schema = batch.schema();
62                    let mut names = std::collections::BTreeSet::new();
63                    for f in schema.fields() {
64                        self.limits.check(
65                            "Arrow column name",
66                            f.name().len(),
67                            self.limits.max_string_bytes,
68                        )?;
69                        if f.name().is_empty() || !names.insert(f.name()) {
70                            return Err(TabularError::Invalid(
71                                "Arrow column names must be nonempty and unique".into(),
72                            ));
73                        }
74                        if !matches!(
75                            f.data_type(),
76                            DataType::Utf8
77                                | DataType::LargeUtf8
78                                | DataType::Int32
79                                | DataType::Int64
80                                | DataType::Float32
81                                | DataType::Float64
82                        ) {
83                            return Err(TabularError::Invalid(format!(
84                                "unsupported Arrow type {}",
85                                f.data_type()
86                            )));
87                        }
88                    }
89                    self.row = 0;
90                    self.batch = Some(batch);
91                    continue;
92                }
93                let batch = self.batch.as_ref().expect("batch assigned above");
94                let fields = batch
95                    .schema()
96                    .fields()
97                    .iter()
98                    .zip(batch.columns())
99                    .map(|(field, a)| {
100                        let value = if a.is_null(self.row) {
101                            None
102                        } else {
103                            Some(match a.data_type() {
104                                DataType::Utf8 => a
105                                    .as_any()
106                                    .downcast_ref::<StringArray>()
107                                    .expect("Arrow Utf8 type invariant")
108                                    .value(self.row)
109                                    .to_owned(),
110                                DataType::LargeUtf8 => a
111                                    .as_any()
112                                    .downcast_ref::<LargeStringArray>()
113                                    .expect("Arrow LargeUtf8 invariant")
114                                    .value(self.row)
115                                    .to_owned(),
116                                DataType::Int32 => a
117                                    .as_any()
118                                    .downcast_ref::<Int32Array>()
119                                    .expect("Arrow Int32 invariant")
120                                    .value(self.row)
121                                    .to_string(),
122                                DataType::Int64 => a
123                                    .as_any()
124                                    .downcast_ref::<Int64Array>()
125                                    .expect("Arrow Int64 invariant")
126                                    .value(self.row)
127                                    .to_string(),
128                                DataType::Float32 => a
129                                    .as_any()
130                                    .downcast_ref::<Float32Array>()
131                                    .expect("Arrow Float32 invariant")
132                                    .value(self.row)
133                                    .to_string(),
134                                DataType::Float64 => a
135                                    .as_any()
136                                    .downcast_ref::<Float64Array>()
137                                    .expect("Arrow Float64 invariant")
138                                    .value(self.row)
139                                    .to_string(),
140                                _ => unreachable!("validated Arrow datatype"),
141                            })
142                        };
143                        (field.name().clone(), value)
144                    })
145                    .collect();
146                self.row += 1;
147                self.total += 1;
148                let row = Row { fields };
149                row.validate(self.limits)?;
150                return Ok(Some(row));
151            }
152        })();
153        match result {
154            Ok(Some(row)) => Some(Ok(row)),
155            Ok(None) => {
156                self.done = true;
157                None
158            }
159            Err(e) => {
160                self.done = true;
161                Some(Err(e))
162            }
163        }
164    }
165}
166fn bounded_file(path: &Path, limits: ResourceLimits) -> Result<File> {
167    let file = File::open(path)?;
168    let n = usize::try_from(file.metadata()?.len()).unwrap_or(usize::MAX);
169    limits.check("columnar encoded bytes", n, limits.max_encoded_bytes)?;
170    Ok(file)
171}
172/// Opens an Arrow IPC file (file framing, not stream framing). The file size and
173/// each decoded record batch are checked; internal decoder allocation is upstream-owned.
174pub fn open_arrow(
175    path: impl AsRef<Path>,
176    limits: ResourceLimits,
177) -> Result<ArrowRows<arrow_ipc::reader::FileReader<File>>> {
178    let reader =
179        arrow_ipc::reader::FileReader::try_new(bounded_file(path.as_ref(), limits)?, None)?;
180    Ok(ArrowRows::new(reader, limits))
181}
182/// Opens local Parquet and rejects oversized declared uncompressed row groups
183/// before decoding. Uses batches of at most 1,024 rows. Compression codecs are
184/// not enabled by default; unsupported encodings return a Parquet error.
185#[cfg(feature = "parquet")]
186pub fn open_parquet(
187    path: impl AsRef<Path>,
188    limits: ResourceLimits,
189) -> Result<ArrowRows<parquet::arrow::arrow_reader::ParquetRecordBatchReader>> {
190    use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
191    let builder = ParquetRecordBatchReaderBuilder::try_new(bounded_file(path.as_ref(), limits)?)?;
192    let mut total = 0usize;
193    for group in builder.metadata().row_groups() {
194        let bytes = usize::try_from(group.total_byte_size())
195            .map_err(|_| TabularError::Invalid("invalid Parquet row-group byte count".into()))?;
196        limits.check("Parquet row-group bytes", bytes, limits.max_columnar_bytes)?;
197        let rows = usize::try_from(group.num_rows())
198            .map_err(|_| TabularError::Invalid("invalid Parquet row count".into()))?;
199        total = total
200            .checked_add(rows)
201            .ok_or_else(|| TabularError::Invalid("Parquet row count overflow".into()))?;
202        limits.check("Parquet rows", total, limits.max_records)?;
203    }
204    let reader = builder
205        .with_batch_size(1024.min(limits.max_records.max(1)))
206        .build()?;
207    Ok(ArrowRows::new(reader, limits))
208}