rusttorch_tabular/
columnar.rs1use super::{Result, Row, TabularError};
2use arrow_array::{
3 Array, Float32Array, Float64Array, Int32Array, Int64Array, LargeStringArray, RecordBatch,
4 StringArray,
5};
6use arrow_schema::DataType;
7use rusttorch_data::ResourceLimits;
8use std::{fs::File, path::Path};
9pub struct ArrowRows<I> {
12 batches: I,
13 batch: Option<RecordBatch>,
14 row: usize,
15 total: usize,
16 limits: ResourceLimits,
17 done: bool,
18}
19impl<I> ArrowRows<I> {
20 pub fn new(batches: I, limits: ResourceLimits) -> Self {
22 Self {
23 batches,
24 batch: None,
25 row: 0,
26 total: 0,
27 limits,
28 done: false,
29 }
30 }
31}
32impl<I> Iterator for ArrowRows<I>
33where
34 I: Iterator<Item = std::result::Result<RecordBatch, arrow_schema::ArrowError>>,
35{
36 type Item = Result<Row>;
37 fn next(&mut self) -> Option<Self::Item> {
38 if self.done {
39 return None;
40 }
41 let result = (|| {
42 loop {
43 if self.batch.as_ref().is_none_or(|b| self.row >= b.num_rows()) {
44 let Some(batch) = self.batches.next() else {
45 return Ok(None);
46 };
47 let batch = batch?;
48 self.limits.check(
49 "Arrow columns",
50 batch.num_columns(),
51 self.limits.max_columns,
52 )?;
53 self.limits.check(
54 "Arrow batch bytes",
55 batch.get_array_memory_size(),
56 self.limits.max_columnar_bytes,
57 )?;
58 if batch.num_rows() > self.limits.max_records.saturating_sub(self.total) {
59 return Err(TabularError::Invalid("Arrow row limit exceeded".into()));
60 }
61 let schema = batch.schema();
62 let mut names = std::collections::BTreeSet::new();
63 for f in schema.fields() {
64 self.limits.check(
65 "Arrow column name",
66 f.name().len(),
67 self.limits.max_string_bytes,
68 )?;
69 if f.name().is_empty() || !names.insert(f.name()) {
70 return Err(TabularError::Invalid(
71 "Arrow column names must be nonempty and unique".into(),
72 ));
73 }
74 if !matches!(
75 f.data_type(),
76 DataType::Utf8
77 | DataType::LargeUtf8
78 | DataType::Int32
79 | DataType::Int64
80 | DataType::Float32
81 | DataType::Float64
82 ) {
83 return Err(TabularError::Invalid(format!(
84 "unsupported Arrow type {}",
85 f.data_type()
86 )));
87 }
88 }
89 self.row = 0;
90 self.batch = Some(batch);
91 continue;
92 }
93 let batch = self.batch.as_ref().expect("batch assigned above");
94 let fields = batch
95 .schema()
96 .fields()
97 .iter()
98 .zip(batch.columns())
99 .map(|(field, a)| {
100 let value = if a.is_null(self.row) {
101 None
102 } else {
103 Some(match a.data_type() {
104 DataType::Utf8 => a
105 .as_any()
106 .downcast_ref::<StringArray>()
107 .expect("Arrow Utf8 type invariant")
108 .value(self.row)
109 .to_owned(),
110 DataType::LargeUtf8 => a
111 .as_any()
112 .downcast_ref::<LargeStringArray>()
113 .expect("Arrow LargeUtf8 invariant")
114 .value(self.row)
115 .to_owned(),
116 DataType::Int32 => a
117 .as_any()
118 .downcast_ref::<Int32Array>()
119 .expect("Arrow Int32 invariant")
120 .value(self.row)
121 .to_string(),
122 DataType::Int64 => a
123 .as_any()
124 .downcast_ref::<Int64Array>()
125 .expect("Arrow Int64 invariant")
126 .value(self.row)
127 .to_string(),
128 DataType::Float32 => a
129 .as_any()
130 .downcast_ref::<Float32Array>()
131 .expect("Arrow Float32 invariant")
132 .value(self.row)
133 .to_string(),
134 DataType::Float64 => a
135 .as_any()
136 .downcast_ref::<Float64Array>()
137 .expect("Arrow Float64 invariant")
138 .value(self.row)
139 .to_string(),
140 _ => unreachable!("validated Arrow datatype"),
141 })
142 };
143 (field.name().clone(), value)
144 })
145 .collect();
146 self.row += 1;
147 self.total += 1;
148 let row = Row { fields };
149 row.validate(self.limits)?;
150 return Ok(Some(row));
151 }
152 })();
153 match result {
154 Ok(Some(row)) => Some(Ok(row)),
155 Ok(None) => {
156 self.done = true;
157 None
158 }
159 Err(e) => {
160 self.done = true;
161 Some(Err(e))
162 }
163 }
164 }
165}
166fn bounded_file(path: &Path, limits: ResourceLimits) -> Result<File> {
167 let file = File::open(path)?;
168 let n = usize::try_from(file.metadata()?.len()).unwrap_or(usize::MAX);
169 limits.check("columnar encoded bytes", n, limits.max_encoded_bytes)?;
170 Ok(file)
171}
172pub fn open_arrow(
175 path: impl AsRef<Path>,
176 limits: ResourceLimits,
177) -> Result<ArrowRows<arrow_ipc::reader::FileReader<File>>> {
178 let reader =
179 arrow_ipc::reader::FileReader::try_new(bounded_file(path.as_ref(), limits)?, None)?;
180 Ok(ArrowRows::new(reader, limits))
181}
182#[cfg(feature = "parquet")]
186pub fn open_parquet(
187 path: impl AsRef<Path>,
188 limits: ResourceLimits,
189) -> Result<ArrowRows<parquet::arrow::arrow_reader::ParquetRecordBatchReader>> {
190 use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
191 let builder = ParquetRecordBatchReaderBuilder::try_new(bounded_file(path.as_ref(), limits)?)?;
192 let mut total = 0usize;
193 for group in builder.metadata().row_groups() {
194 let bytes = usize::try_from(group.total_byte_size())
195 .map_err(|_| TabularError::Invalid("invalid Parquet row-group byte count".into()))?;
196 limits.check("Parquet row-group bytes", bytes, limits.max_columnar_bytes)?;
197 let rows = usize::try_from(group.num_rows())
198 .map_err(|_| TabularError::Invalid("invalid Parquet row count".into()))?;
199 total = total
200 .checked_add(rows)
201 .ok_or_else(|| TabularError::Invalid("Parquet row count overflow".into()))?;
202 limits.check("Parquet rows", total, limits.max_records)?;
203 }
204 let reader = builder
205 .with_batch_size(1024.min(limits.max_records.max(1)))
206 .build()?;
207 Ok(ArrowRows::new(reader, limits))
208}