1#[cfg(feature = "mmap")]
2use std::fs::File;
3#[cfg(feature = "mmap")]
4use std::io::{Read, Seek, SeekFrom};
5use std::ops::Range;
6use std::path::Path;
7use std::sync::Arc;
8
9use arrow::record_batch::RecordBatch;
10
11use crate::arrow_export::apply_compare_filter;
12use crate::consumer::CollectingConsumer;
13use crate::decoder::{split_points_to_ranges, RecordParser, Splitter};
14use crate::engine::TableBuilder;
15use crate::input::InputBuffer;
16use crate::plan::ExecutionPlan;
17use crate::Result;
18
19#[derive(Clone, Copy, Debug)]
21pub struct MemoryBudget {
22 bytes: usize,
23}
24
25impl MemoryBudget {
26 pub fn new(bytes: usize) -> Self {
27 Self { bytes }
28 }
29
30 pub fn bytes(&self) -> usize {
31 self.bytes
32 }
33}
34
35const MAX_SPLIT_CHUNKS: usize = 100_000;
42
43pub struct BoundedExecutor {
45 budget: MemoryBudget,
46}
47
48impl BoundedExecutor {
49 pub fn new(budget: MemoryBudget) -> Self {
50 Self { budget }
51 }
52
53 fn plan_chunks(
56 &self,
57 bytes: &[u8],
58 splitter: &dyn Splitter,
59 ) -> (Vec<Range<usize>>, usize, usize) {
60 let bytes_per_row = splitter.estimate_bytes_per_row(bytes).max(1);
61 let total_rows_est = bytes.len() / bytes_per_row;
62 let rows_per_batch = (self.budget.bytes() / bytes_per_row)
63 .max(1)
64 .min(total_rows_est.max(1));
65
66 let num_batches = (total_rows_est / rows_per_batch).max(1);
67 let capped = num_batches.min(MAX_SPLIT_CHUNKS);
68 let split_points = splitter.find_split_points(bytes, capped);
69 let chunks = split_points_to_ranges(&split_points, bytes.len());
70 (chunks, rows_per_batch, bytes_per_row)
71 }
72
73 pub fn run_bytes_stream<P, C>(
79 &self,
80 bytes: &[u8],
81 splitter: &dyn Splitter,
82 parser: P,
83 plan: Arc<ExecutionPlan>,
84 consumer: &mut C,
85 ) -> Result<()>
86 where
87 P: RecordParser + Clone + Send + Sync,
88 C: crate::consumer::BatchConsumer,
89 {
90 if bytes.is_empty() {
91 return Ok(());
92 }
93
94 let (chunks, rows_per_batch, bytes_per_row) = self.plan_chunks(bytes, splitter);
95
96 let mut batch_engine = TableBuilder::with_plan(bytes_per_row.max(64), Arc::clone(&plan));
97 let mut rows_in_batch = 0usize;
98
99 for chunk in &chunks {
100 let chunk_bytes = &bytes[chunk.start..chunk.end];
101 let mut chunk_engine = TableBuilder::with_plan(
102 (chunk.len() / bytes_per_row.max(512)).max(64),
103 Arc::clone(&plan),
104 );
105 parser.validate(chunk_bytes)?;
106 parser.parse_chunk_generic(chunk_bytes, &mut chunk_engine)?;
107
108 let chunk_rows = chunk_engine.num_rows();
109 batch_engine.extend(chunk_engine)?;
110 rows_in_batch += chunk_rows;
111
112 while rows_in_batch >= rows_per_batch
114 || batch_engine.bytes_used() >= self.budget.bytes()
115 {
116 if batch_engine.num_rows() == 0 {
117 break;
118 }
119 let n = rows_per_batch.min(batch_engine.num_rows());
120 let n = if batch_engine.bytes_used() >= self.budget.bytes()
122 && batch_engine.num_rows() > 1
123 {
124 let est = (batch_engine.num_rows() as f64 * self.budget.bytes() as f64
125 / batch_engine.bytes_used() as f64) as usize;
126 est.clamp(1, n)
127 } else {
128 n
129 };
130 let mut to_consume = batch_engine.split_off(n);
131 let mut batch = to_consume.finish()?;
132 if let Some(ref filter) = plan.filter {
133 batch = apply_compare_filter(batch, filter)?;
134 }
135 consumer.consume(batch)?;
136 rows_in_batch = rows_in_batch.saturating_sub(n);
137 }
138 }
139
140 if batch_engine.num_rows() > 0 {
141 let mut batch = batch_engine.finish()?;
142 if let Some(ref filter) = plan.filter {
143 batch = apply_compare_filter(batch, filter)?;
144 }
145 consumer.consume(batch)?;
146 }
147
148 Ok(())
149 }
150
151 pub fn run_bytes<P>(
157 &self,
158 bytes: &[u8],
159 splitter: &dyn Splitter,
160 parser: P,
161 plan: Arc<ExecutionPlan>,
162 ) -> Result<Vec<RecordBatch>>
163 where
164 P: RecordParser + Clone + Send + Sync,
165 {
166 let batches = Vec::new();
167 let mut consumer = CollectingConsumer(batches);
168 self.run_bytes_stream(bytes, splitter, parser, plan, &mut consumer)?;
169 Ok(consumer.0)
170 }
171
172 pub fn run_stream<P, C>(
178 &self,
179 path: &Path,
180 splitter: &dyn Splitter,
181 parser: P,
182 plan: Arc<ExecutionPlan>,
183 prefault: bool,
184 consumer: &mut C,
185 ) -> Result<()>
186 where
187 P: RecordParser + Clone + Send + Sync,
188 C: crate::consumer::BatchConsumer,
189 {
190 let use_mmap = cfg!(feature = "mmap");
191 let input = InputBuffer::open(path, use_mmap, prefault)?;
192
193 #[cfg(feature = "mmap")]
194 if matches!(input, InputBuffer::Mmap(_)) {
195 return self.run_mapped_stream(path, input, splitter, parser, plan, consumer);
196 }
197
198 self.run_bytes_stream(input.as_slice(), splitter, parser, plan, consumer)
199 }
200
201 pub fn run<P>(
212 &self,
213 path: &Path,
214 splitter: &dyn Splitter,
215 parser: P,
216 plan: Arc<ExecutionPlan>,
217 prefault: bool,
218 ) -> Result<Vec<RecordBatch>>
219 where
220 P: RecordParser + Clone + Send + Sync,
221 {
222 let batches = Vec::new();
223 let mut consumer = CollectingConsumer(batches);
224 self.run_stream(path, splitter, parser, plan, prefault, &mut consumer)?;
225 Ok(consumer.0)
226 }
227
228 #[cfg(feature = "mmap")]
231 fn run_mapped_stream<P, C>(
232 &self,
233 path: &Path,
234 input: InputBuffer,
235 splitter: &dyn Splitter,
236 parser: P,
237 plan: Arc<ExecutionPlan>,
238 consumer: &mut C,
239 ) -> Result<()>
240 where
241 P: RecordParser + Clone + Send + Sync,
242 C: crate::consumer::BatchConsumer,
243 {
244 let bytes = input.as_slice();
245 if bytes.is_empty() {
246 return Ok(());
247 }
248
249 let (chunks, rows_per_batch, bytes_per_row) = self.plan_chunks(bytes, splitter);
250 drop(input);
251
252 let mut batch_engine = TableBuilder::with_plan(bytes_per_row.max(64), Arc::clone(&plan));
253 let mut rows_in_batch = 0usize;
254
255 let mut file = File::open(path)?;
256 let max_chunk = chunks.iter().map(|r| r.len()).max().unwrap_or(0);
258 let mut chunk_buf = Vec::with_capacity(max_chunk);
259
260 for chunk in &chunks {
261 let chunk_len = chunk.len();
262 chunk_buf.resize(chunk_len, 0);
263 file.seek(SeekFrom::Start(chunk.start as u64))?;
264 file.read_exact(&mut chunk_buf)?;
265 let mut chunk_engine = TableBuilder::with_plan(
266 (chunk.len() / bytes_per_row.max(512)).max(64),
267 Arc::clone(&plan),
268 );
269 parser.validate(&chunk_buf)?;
270 parser.parse_chunk_generic(&chunk_buf, &mut chunk_engine)?;
271
272 let chunk_rows = chunk_engine.num_rows();
273 batch_engine.extend(chunk_engine)?;
274 rows_in_batch += chunk_rows;
275
276 while rows_in_batch >= rows_per_batch
278 || batch_engine.bytes_used() >= self.budget.bytes()
279 {
280 if batch_engine.num_rows() == 0 {
281 break;
282 }
283 let n = rows_per_batch.min(batch_engine.num_rows());
284 let n = if batch_engine.bytes_used() >= self.budget.bytes()
286 && batch_engine.num_rows() > 1
287 {
288 let est = (batch_engine.num_rows() as f64 * self.budget.bytes() as f64
289 / batch_engine.bytes_used() as f64) as usize;
290 est.clamp(1, n)
291 } else {
292 n
293 };
294 let mut to_consume = batch_engine.split_off(n);
295 let mut batch = to_consume.finish()?;
296 if let Some(ref filter) = plan.filter {
297 batch = apply_compare_filter(batch, filter)?;
298 }
299 consumer.consume(batch)?;
300 rows_in_batch = rows_in_batch.saturating_sub(n);
301 }
302 }
303
304 if batch_engine.num_rows() > 0 {
305 let mut batch = batch_engine.finish()?;
306 if let Some(ref filter) = plan.filter {
307 batch = apply_compare_filter(batch, filter)?;
308 }
309 consumer.consume(batch)?;
310 }
311
312 Ok(())
313 }
314
315 #[cfg(feature = "mmap")]
318 #[expect(dead_code, reason = "legacy path kept for reference")]
319 fn run_mapped<P>(
320 &self,
321 path: &Path,
322 input: InputBuffer,
323 splitter: &dyn Splitter,
324 parser: P,
325 plan: Arc<ExecutionPlan>,
326 ) -> Result<Vec<RecordBatch>>
327 where
328 P: RecordParser + Clone + Send + Sync,
329 {
330 let batches = Vec::new();
331 let mut consumer = CollectingConsumer(batches);
332 self.run_mapped_stream(path, input, splitter, parser, plan, &mut consumer)?;
333 Ok(consumer.0)
334 }
335}
336
337#[expect(dead_code, reason = "legacy path kept for reference")]
341fn apply_plan_filter(batches: &mut Vec<RecordBatch>, plan: &ExecutionPlan) -> Result<()> {
342 if let Some(ref filter) = plan.filter {
343 for batch in batches {
344 *batch = apply_compare_filter(
345 std::mem::replace(batch, RecordBatch::new_empty(batch.schema())),
346 filter,
347 )?;
348 }
349 }
350 Ok(())
351}