1#[cfg(feature = "mmap")]
2use std::fs::File;
3#[cfg(feature = "mmap")]
4use std::io::{Read, Seek, SeekFrom};
5use std::ops::Range;
6use std::path::Path;
7use std::sync::Arc;
8
9use arrow::record_batch::RecordBatch;
10
11use crate::arrow_export::apply_compare_filter;
12use crate::consumer::CollectingConsumer;
13use crate::decoder::{split_points_to_ranges, RecordParser, Splitter};
14use crate::engine::TableBuilder;
15use crate::input::InputBuffer;
16use crate::plan::ExecutionPlan;
17use crate::Result;
18
19#[derive(Clone, Copy, Debug)]
21pub struct MemoryBudget {
22 bytes: usize,
23}
24
25impl MemoryBudget {
26 pub fn new(bytes: usize) -> Self {
27 Self { bytes }
28 }
29
30 pub fn bytes(&self) -> usize {
31 self.bytes
32 }
33}
34
35pub const MAX_SPLIT_CHUNKS: usize = 100_000;
42
43pub struct BoundedExecutor {
45 budget: MemoryBudget,
46 split_cap: usize,
47}
48
49impl BoundedExecutor {
50 pub fn new(budget: MemoryBudget) -> Self {
51 Self {
52 budget,
53 split_cap: MAX_SPLIT_CHUNKS,
54 }
55 }
56
57 pub fn with_split_cap(mut self, cap: usize) -> Self {
61 self.split_cap = cap;
62 self
63 }
64
65 fn plan_chunks(
68 &self,
69 bytes: &[u8],
70 splitter: &dyn Splitter,
71 ) -> (Vec<Range<usize>>, usize, usize) {
72 let bytes_per_row = splitter.estimate_bytes_per_row(bytes).max(1);
73 let total_rows_est = bytes.len() / bytes_per_row;
74 let rows_per_batch = (self.budget.bytes() / bytes_per_row)
75 .max(1)
76 .min(total_rows_est.max(1));
77
78 let num_batches = (total_rows_est / rows_per_batch).max(1);
79 let capped = num_batches.min(self.split_cap);
80 let split_points = splitter.find_split_points(bytes, capped);
81 let chunks = split_points_to_ranges(&split_points, bytes.len());
82 (chunks, rows_per_batch, bytes_per_row)
83 }
84
85 pub fn run_bytes_stream<P, C>(
91 &self,
92 bytes: &[u8],
93 splitter: &dyn Splitter,
94 parser: P,
95 plan: Arc<ExecutionPlan>,
96 consumer: &mut C,
97 ) -> Result<()>
98 where
99 P: RecordParser + Clone + Send + Sync,
100 C: crate::consumer::BatchConsumer,
101 {
102 if bytes.is_empty() {
103 return Ok(());
104 }
105
106 let (chunks, rows_per_batch, bytes_per_row) = self.plan_chunks(bytes, splitter);
107
108 let mut batch_engine = TableBuilder::with_plan(bytes_per_row.max(64), Arc::clone(&plan));
109 let mut rows_in_batch = 0usize;
110
111 for chunk in &chunks {
112 let chunk_bytes = &bytes[chunk.start..chunk.end];
113 let mut chunk_engine = TableBuilder::with_plan(
114 (chunk.len() / bytes_per_row.max(512)).max(64),
115 Arc::clone(&plan),
116 );
117 parser.validate(chunk_bytes)?;
118 parser.parse_chunk_generic(chunk_bytes, &mut chunk_engine)?;
119
120 let chunk_rows = chunk_engine.num_rows();
121 batch_engine.extend(chunk_engine)?;
122 rows_in_batch += chunk_rows;
123
124 while rows_in_batch >= rows_per_batch
126 || batch_engine.bytes_used() >= self.budget.bytes()
127 {
128 if batch_engine.num_rows() == 0 {
129 break;
130 }
131 let n = rows_per_batch.min(batch_engine.num_rows());
132 let n = if batch_engine.bytes_used() >= self.budget.bytes()
134 && batch_engine.num_rows() > 1
135 {
136 let est = (batch_engine.num_rows() as f64 * self.budget.bytes() as f64
137 / batch_engine.bytes_used() as f64) as usize;
138 est.clamp(1, n)
139 } else {
140 n
141 };
142 let mut to_consume = batch_engine.split_off(n);
143 let mut batch = to_consume.finish()?;
144 if let Some(ref filter) = plan.filter {
145 batch = apply_compare_filter(batch, filter)?;
146 }
147 consumer.consume(batch)?;
148 rows_in_batch = rows_in_batch.saturating_sub(n);
149 }
150 }
151
152 if let Some(err) = batch_engine.unknown_error.take() {
156 return Err(crate::Error::Merge(err));
157 }
158 if let Some(err) = batch_engine.strict_error.take() {
159 return Err(err);
160 }
161
162 if batch_engine.num_rows() > 0 {
163 let mut batch = batch_engine.finish()?;
164 if let Some(ref filter) = plan.filter {
165 batch = apply_compare_filter(batch, filter)?;
166 }
167 consumer.consume(batch)?;
168 }
169
170 Ok(())
171 }
172
173 pub fn run_bytes<P>(
179 &self,
180 bytes: &[u8],
181 splitter: &dyn Splitter,
182 parser: P,
183 plan: Arc<ExecutionPlan>,
184 ) -> Result<Vec<RecordBatch>>
185 where
186 P: RecordParser + Clone + Send + Sync,
187 {
188 let batches = Vec::new();
189 let mut consumer = CollectingConsumer(batches);
190 self.run_bytes_stream(bytes, splitter, parser, plan, &mut consumer)?;
191 Ok(consumer.0)
192 }
193
194 pub fn run_stream<P, C>(
200 &self,
201 path: &Path,
202 splitter: &dyn Splitter,
203 parser: P,
204 plan: Arc<ExecutionPlan>,
205 prefault: bool,
206 consumer: &mut C,
207 ) -> Result<()>
208 where
209 P: RecordParser + Clone + Send + Sync,
210 C: crate::consumer::BatchConsumer,
211 {
212 let use_mmap = cfg!(feature = "mmap");
213 let input = InputBuffer::open(path, use_mmap, prefault)?;
214
215 #[cfg(feature = "mmap")]
216 if matches!(input, InputBuffer::Mmap(_)) {
217 return self.run_mapped_stream(path, input, splitter, parser, plan, consumer);
218 }
219
220 self.run_bytes_stream(input.as_slice(), splitter, parser, plan, consumer)
221 }
222
223 pub fn run<P>(
234 &self,
235 path: &Path,
236 splitter: &dyn Splitter,
237 parser: P,
238 plan: Arc<ExecutionPlan>,
239 prefault: bool,
240 ) -> Result<Vec<RecordBatch>>
241 where
242 P: RecordParser + Clone + Send + Sync,
243 {
244 let batches = Vec::new();
245 let mut consumer = CollectingConsumer(batches);
246 self.run_stream(path, splitter, parser, plan, prefault, &mut consumer)?;
247 Ok(consumer.0)
248 }
249
250 #[cfg(feature = "mmap")]
253 fn run_mapped_stream<P, C>(
254 &self,
255 path: &Path,
256 input: InputBuffer,
257 splitter: &dyn Splitter,
258 parser: P,
259 plan: Arc<ExecutionPlan>,
260 consumer: &mut C,
261 ) -> Result<()>
262 where
263 P: RecordParser + Clone + Send + Sync,
264 C: crate::consumer::BatchConsumer,
265 {
266 let bytes = input.as_slice();
267 if bytes.is_empty() {
268 return Ok(());
269 }
270
271 let (chunks, rows_per_batch, bytes_per_row) = self.plan_chunks(bytes, splitter);
272 drop(input);
273
274 let mut batch_engine = TableBuilder::with_plan(bytes_per_row.max(64), Arc::clone(&plan));
275 let mut rows_in_batch = 0usize;
276
277 let mut file = File::open(path)?;
278 let max_chunk = chunks.iter().map(|r| r.len()).max().unwrap_or(0);
280 let mut chunk_buf = Vec::with_capacity(max_chunk);
281
282 for chunk in &chunks {
283 let chunk_len = chunk.len();
284 chunk_buf.resize(chunk_len, 0);
285 file.seek(SeekFrom::Start(chunk.start as u64))?;
286 file.read_exact(&mut chunk_buf)?;
287 let mut chunk_engine = TableBuilder::with_plan(
288 (chunk.len() / bytes_per_row.max(512)).max(64),
289 Arc::clone(&plan),
290 );
291 parser.validate(&chunk_buf)?;
292 parser.parse_chunk_generic(&chunk_buf, &mut chunk_engine)?;
293
294 let chunk_rows = chunk_engine.num_rows();
295 batch_engine.extend(chunk_engine)?;
296 rows_in_batch += chunk_rows;
297
298 while rows_in_batch >= rows_per_batch
300 || batch_engine.bytes_used() >= self.budget.bytes()
301 {
302 if batch_engine.num_rows() == 0 {
303 break;
304 }
305 let n = rows_per_batch.min(batch_engine.num_rows());
306 let n = if batch_engine.bytes_used() >= self.budget.bytes()
308 && batch_engine.num_rows() > 1
309 {
310 let est = (batch_engine.num_rows() as f64 * self.budget.bytes() as f64
311 / batch_engine.bytes_used() as f64) as usize;
312 est.clamp(1, n)
313 } else {
314 n
315 };
316 let mut to_consume = batch_engine.split_off(n);
317 let mut batch = to_consume.finish()?;
318 if let Some(ref filter) = plan.filter {
319 batch = apply_compare_filter(batch, filter)?;
320 }
321 consumer.consume(batch)?;
322 rows_in_batch = rows_in_batch.saturating_sub(n);
323 }
324 }
325
326 if let Some(err) = batch_engine.unknown_error.take() {
330 return Err(crate::Error::Merge(err));
331 }
332 if let Some(err) = batch_engine.strict_error.take() {
333 return Err(err);
334 }
335
336 if batch_engine.num_rows() > 0 {
337 let mut batch = batch_engine.finish()?;
338 if let Some(ref filter) = plan.filter {
339 batch = apply_compare_filter(batch, filter)?;
340 }
341 consumer.consume(batch)?;
342 }
343
344 Ok(())
345 }
346
347 #[cfg(feature = "mmap")]
350 #[expect(dead_code, reason = "legacy path kept for reference")]
351 fn run_mapped<P>(
352 &self,
353 path: &Path,
354 input: InputBuffer,
355 splitter: &dyn Splitter,
356 parser: P,
357 plan: Arc<ExecutionPlan>,
358 ) -> Result<Vec<RecordBatch>>
359 where
360 P: RecordParser + Clone + Send + Sync,
361 {
362 let batches = Vec::new();
363 let mut consumer = CollectingConsumer(batches);
364 self.run_mapped_stream(path, input, splitter, parser, plan, &mut consumer)?;
365 Ok(consumer.0)
366 }
367}
368
369#[expect(dead_code, reason = "legacy path kept for reference")]
373fn apply_plan_filter(batches: &mut Vec<RecordBatch>, plan: &ExecutionPlan) -> Result<()> {
374 if let Some(ref filter) = plan.filter {
375 for batch in batches {
376 *batch = apply_compare_filter(
377 std::mem::replace(batch, RecordBatch::new_empty(batch.schema())),
378 filter,
379 )?;
380 }
381 }
382 Ok(())
383}
384
385#[cfg(test)]
386mod tests {
387 use super::*;
388 use crate::decoder::Splitter;
389 use crate::ExecutionPlan;
390
391 struct NewlineSplitter;
392
393 impl Splitter for NewlineSplitter {
394 fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
395 if from == 0 && !bytes.is_empty() {
396 return Some(0);
397 }
398 bytes[from.saturating_sub(1)..]
399 .iter()
400 .position(|&b| b == b'\n')
401 .map(|i| from + i + 1)
402 .filter(|&p| p < bytes.len())
403 }
404
405 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
406 let newlines = sample.iter().filter(|&&b| b == b'\n').count().max(1);
407 (sample.len() / newlines).max(1)
408 }
409 }
410
411 fn rows(n: usize) -> Vec<u8> {
412 "abcdefghij\n".repeat(n).into_bytes()
413 }
414
415 #[test]
416 fn test_split_cap_limits_chunk_count() {
417 let data = rows(10_000);
418 let budget = MemoryBudget::new(1024);
419
420 let default_chunks = BoundedExecutor::new(budget)
421 .plan_chunks(&data, &NewlineSplitter)
422 .0
423 .len();
424 assert!(
425 default_chunks > 2,
426 "uncapped run must split: {default_chunks}"
427 );
428
429 for cap in [1, 2, 3] {
430 let chunks = BoundedExecutor::new(budget)
431 .with_split_cap(cap)
432 .plan_chunks(&data, &NewlineSplitter)
433 .0;
434 assert!(
435 chunks.len() <= cap.max(1),
436 "cap {cap} must bound chunk count, got {}",
437 chunks.len()
438 );
439 }
440 }
441
442 #[test]
443 fn test_default_split_cap_is_max_split_chunks() {
444 assert_eq!(
445 BoundedExecutor::new(MemoryBudget::new(1)).split_cap,
446 MAX_SPLIT_CHUNKS
447 );
448 let plan = ExecutionPlan::new().with_max_split_chunks(7);
449 assert_eq!(plan.max_split_chunks, Some(7));
450 assert_eq!(ExecutionPlan::new().max_split_chunks, None);
451 }
452}