1use std::fs::File;
2use std::io::{Read, Seek, SeekFrom};
3use std::path::Path;
4
5use arrow::record_batch::RecordBatch;
6
7use crate::arrow_export::apply_compare_filter;
8use crate::decoder::{split_points_to_ranges, RecordParser, Splitter};
9use crate::engine::TableBuilder;
10use crate::input::InputBuffer;
11use crate::plan::ExecutionPlan;
12use crate::Result;
13
14#[derive(Clone, Copy, Debug)]
16pub struct MemoryBudget {
17 bytes: usize,
18}
19
20impl MemoryBudget {
21 pub fn new(bytes: usize) -> Self {
22 Self { bytes }
23 }
24
25 pub fn bytes(&self) -> usize {
26 self.bytes
27 }
28}
29
30pub struct BoundedExecutor {
32 budget: MemoryBudget,
33}
34
35impl BoundedExecutor {
36 pub fn new(budget: MemoryBudget) -> Self {
37 Self { budget }
38 }
39
40 pub fn run<P>(
45 &self,
46 path: &Path,
47 splitter: &dyn Splitter,
48 parser: P,
49 plan: ExecutionPlan,
50 prefault: bool,
51 ) -> Result<Vec<RecordBatch>>
52 where
53 P: RecordParser + Clone + Send + Sync,
54 {
55 let use_mmap = cfg!(feature = "mmap");
56 let input = InputBuffer::open(path, use_mmap, prefault)?;
57 let bytes = input.as_slice();
58 let file_len = bytes.len();
59
60 let bytes_per_row = splitter.estimate_bytes_per_row(bytes).max(1);
61 let total_rows_est = file_len / bytes_per_row;
62 let rows_per_batch = (self.budget.bytes() / bytes_per_row)
63 .max(1)
64 .min(total_rows_est.max(1));
65
66 let num_batches = (total_rows_est / rows_per_batch).max(1);
67 let split_points = splitter.find_split_points(bytes, num_batches.min(64));
68 let chunks = split_points_to_ranges(&split_points, file_len);
69 drop(input);
70
71 if file_len == 0 {
72 return Ok(Vec::new());
73 }
74
75 let mut batches = Vec::new();
76 let mut batch_engine = TableBuilder::with_plan(bytes_per_row.max(64), plan.clone());
77 let mut rows_in_batch = 0usize;
78
79 let mut file = File::open(path)?;
80 for chunk in &chunks {
81 let chunk_len = chunk.len();
82 let mut chunk_buf = vec![0u8; chunk_len];
83 file.seek(SeekFrom::Start(chunk.start as u64))?;
84 file.read_exact(&mut chunk_buf)?;
85
86 let mut chunk_engine = TableBuilder::with_plan((chunk_len / 512).max(64), plan.clone());
87 parser.validate(&chunk_buf)?;
88 parser.parse_chunk(&chunk_buf, &mut chunk_engine)?;
89
90 let chunk_rows = chunk_engine.num_rows();
91 batch_engine.extend(chunk_engine)?;
92 rows_in_batch += chunk_rows;
93
94 if rows_in_batch >= rows_per_batch {
95 let batch = batch_engine.finish()?;
96 batches.push(batch);
97 batch_engine.reset();
98 rows_in_batch = 0;
99 }
100 }
101
102 if batch_engine.num_rows() > 0 {
103 let batch = batch_engine.finish()?;
104 batches.push(batch);
105 }
106
107 if let Some(ref filter) = plan.filter {
108 for batch in &mut batches {
109 *batch = apply_compare_filter(
110 std::mem::replace(batch, RecordBatch::new_empty(batch.schema())),
111 filter,
112 )?;
113 }
114 }
115
116 Ok(batches)
117 }
118}