Skip to main content

rypipe_core/
bounded.rs

1#[cfg(feature = "mmap")]
2use std::fs::File;
3#[cfg(feature = "mmap")]
4use std::io::{Read, Seek, SeekFrom};
5use std::ops::Range;
6use std::path::Path;
7use std::sync::Arc;
8
9use arrow::record_batch::RecordBatch;
10
11use crate::arrow_export::apply_compare_filter;
12use crate::consumer::CollectingConsumer;
13use crate::decoder::{split_points_to_ranges, RecordParser, Splitter};
14use crate::engine::TableBuilder;
15use crate::input::InputBuffer;
16use crate::plan::ExecutionPlan;
17use crate::Result;
18
19/// A memory budget expressed in bytes.
20#[derive(Clone, Copy, Debug)]
21pub struct MemoryBudget {
22    bytes: usize,
23}
24
25impl MemoryBudget {
26    pub fn new(bytes: usize) -> Self {
27        Self { bytes }
28    }
29
30    pub fn bytes(&self) -> usize {
31        self.bytes
32    }
33}
34
35/// Internal safeguard: never request more than this many split points, so a
36/// pathological row-size estimate cannot explode per-chunk overhead. The
37/// batch count is otherwise derived from the budget, input size, and
38/// estimated bytes per row; batches may still exceed the budget when the
39/// required count exceeds this cap. Increased for 64KB streaming (50GB/64KB
40/// ≈ 800k batches); still bounded by file scan cost.
41const MAX_SPLIT_CHUNKS: usize = 100_000;
42
43/// Parse an input in bounded batches to stay within a memory budget.
44pub struct BoundedExecutor {
45    budget: MemoryBudget,
46}
47
48impl BoundedExecutor {
49    pub fn new(budget: MemoryBudget) -> Self {
50        Self { budget }
51    }
52
53    /// Derive chunk ranges and batch sizing from an in-memory sample of the
54    /// input. Returns `(chunk_ranges, rows_per_batch, bytes_per_row)`.
55    fn plan_chunks(
56        &self,
57        bytes: &[u8],
58        splitter: &dyn Splitter,
59    ) -> (Vec<Range<usize>>, usize, usize) {
60        let bytes_per_row = splitter.estimate_bytes_per_row(bytes).max(1);
61        let total_rows_est = bytes.len() / bytes_per_row;
62        let rows_per_batch = (self.budget.bytes() / bytes_per_row)
63            .max(1)
64            .min(total_rows_est.max(1));
65
66        let num_batches = (total_rows_est / rows_per_batch).max(1);
67        let capped = num_batches.min(MAX_SPLIT_CHUNKS);
68        let split_points = splitter.find_split_points(bytes, capped);
69        let chunks = split_points_to_ranges(&split_points, bytes.len());
70        (chunks, rows_per_batch, bytes_per_row)
71    }
72
73    /// Parse an in-memory byte slice in bounded batches, calling `consumer` per batch.
74    ///
75    /// Chunks are sliced directly from `bytes`; no file I/O occurs. This is
76    /// the streaming entry point for adapters holding decompressed data.
77    /// Peak memory is `budget + batch`.
78    pub fn run_bytes_stream<P, C>(
79        &self,
80        bytes: &[u8],
81        splitter: &dyn Splitter,
82        parser: P,
83        plan: Arc<ExecutionPlan>,
84        consumer: &mut C,
85    ) -> Result<()>
86    where
87        P: RecordParser + Clone + Send + Sync,
88        C: crate::consumer::BatchConsumer,
89    {
90        if bytes.is_empty() {
91            return Ok(());
92        }
93
94        let (chunks, rows_per_batch, bytes_per_row) = self.plan_chunks(bytes, splitter);
95
96        let mut batch_engine = TableBuilder::with_plan(bytes_per_row.max(64), Arc::clone(&plan));
97        let mut rows_in_batch = 0usize;
98
99        for chunk in &chunks {
100            let chunk_bytes = &bytes[chunk.start..chunk.end];
101            let mut chunk_engine = TableBuilder::with_plan(
102                (chunk.len() / bytes_per_row.max(512)).max(64),
103                Arc::clone(&plan),
104            );
105            parser.validate(chunk_bytes)?;
106            parser.parse_chunk_generic(chunk_bytes, &mut chunk_engine)?;
107
108            let chunk_rows = chunk_engine.num_rows();
109            batch_engine.extend(chunk_engine)?;
110            rows_in_batch += chunk_rows;
111
112            // Dynamic sizing: flush when either row count or actual bytes_used exceeds budget.
113            while rows_in_batch >= rows_per_batch
114                || batch_engine.bytes_used() >= self.budget.bytes()
115            {
116                if batch_engine.num_rows() == 0 {
117                    break;
118                }
119                let n = rows_per_batch.min(batch_engine.num_rows());
120                // If bytes_used is the trigger, estimate a smaller n to stay under budget
121                let n = if batch_engine.bytes_used() >= self.budget.bytes()
122                    && batch_engine.num_rows() > 1
123                {
124                    let est = (batch_engine.num_rows() as f64 * self.budget.bytes() as f64
125                        / batch_engine.bytes_used() as f64) as usize;
126                    est.clamp(1, n)
127                } else {
128                    n
129                };
130                let mut to_consume = batch_engine.split_off(n);
131                let mut batch = to_consume.finish()?;
132                if let Some(ref filter) = plan.filter {
133                    batch = apply_compare_filter(batch, filter)?;
134                }
135                consumer.consume(batch)?;
136                rows_in_batch = rows_in_batch.saturating_sub(n);
137            }
138        }
139
140        if batch_engine.num_rows() > 0 {
141            let mut batch = batch_engine.finish()?;
142            if let Some(ref filter) = plan.filter {
143                batch = apply_compare_filter(batch, filter)?;
144            }
145            consumer.consume(batch)?;
146        }
147
148        Ok(())
149    }
150
151    /// Parse an in-memory byte slice in bounded batches, returning one
152    /// `RecordBatch` per batch.
153    ///
154    /// Chunks are sliced directly from `bytes`; no file I/O occurs. This is
155    /// the entry point for adapters holding decompressed or streamed-in data.
156    pub fn run_bytes<P>(
157        &self,
158        bytes: &[u8],
159        splitter: &dyn Splitter,
160        parser: P,
161        plan: Arc<ExecutionPlan>,
162    ) -> Result<Vec<RecordBatch>>
163    where
164        P: RecordParser + Clone + Send + Sync,
165    {
166        let batches = Vec::new();
167        let mut consumer = CollectingConsumer(batches);
168        self.run_bytes_stream(bytes, splitter, parser, plan, &mut consumer)?;
169        Ok(consumer.0)
170    }
171
172    /// Parse `path` in batches, calling `consumer` per batch (streaming).
173    ///
174    /// Compressed inputs are decompressed up front and served via
175    /// `run_bytes_stream`. Uncompressed inputs with `mmap` use the seek-based
176    /// path with a reusable chunk buffer to keep RSS constant.
177    pub fn run_stream<P, C>(
178        &self,
179        path: &Path,
180        splitter: &dyn Splitter,
181        parser: P,
182        plan: Arc<ExecutionPlan>,
183        prefault: bool,
184        consumer: &mut C,
185    ) -> Result<()>
186    where
187        P: RecordParser + Clone + Send + Sync,
188        C: crate::consumer::BatchConsumer,
189    {
190        let use_mmap = cfg!(feature = "mmap");
191        let input = InputBuffer::open(path, use_mmap, prefault)?;
192
193        #[cfg(feature = "mmap")]
194        if matches!(input, InputBuffer::Mmap(_)) {
195            return self.run_mapped_stream(path, input, splitter, parser, plan, consumer);
196        }
197
198        self.run_bytes_stream(input.as_slice(), splitter, parser, plan, consumer)
199    }
200
201    /// Parse `path` in batches, returning one `RecordBatch` per batch.
202    ///
203    /// The caller is responsible for concatenating the batches if a single
204    /// table is desired.
205    ///
206    /// Compressed inputs (see [`InputBuffer::open`]) are decompressed up
207    /// front and served from memory via [`BoundedExecutor::run_bytes`].
208    /// Uncompressed inputs on platforms with the `mmap` feature retain the
209    /// seek-based path: the mapping is dropped after planning and each chunk
210    /// is read from disk on demand, keeping resident memory low.
211    pub fn run<P>(
212        &self,
213        path: &Path,
214        splitter: &dyn Splitter,
215        parser: P,
216        plan: Arc<ExecutionPlan>,
217        prefault: bool,
218    ) -> Result<Vec<RecordBatch>>
219    where
220        P: RecordParser + Clone + Send + Sync,
221    {
222        let batches = Vec::new();
223        let mut consumer = CollectingConsumer(batches);
224        self.run_stream(path, splitter, parser, plan, prefault, &mut consumer)?;
225        Ok(consumer.0)
226    }
227
228    /// Streaming path for mapped inputs: plan against the mapping, drop it,
229    /// then read each chunk with a reusable buffer.
230    #[cfg(feature = "mmap")]
231    fn run_mapped_stream<P, C>(
232        &self,
233        path: &Path,
234        input: InputBuffer,
235        splitter: &dyn Splitter,
236        parser: P,
237        plan: Arc<ExecutionPlan>,
238        consumer: &mut C,
239    ) -> Result<()>
240    where
241        P: RecordParser + Clone + Send + Sync,
242        C: crate::consumer::BatchConsumer,
243    {
244        let bytes = input.as_slice();
245        if bytes.is_empty() {
246            return Ok(());
247        }
248
249        let (chunks, rows_per_batch, bytes_per_row) = self.plan_chunks(bytes, splitter);
250        drop(input);
251
252        let mut batch_engine = TableBuilder::with_plan(bytes_per_row.max(64), Arc::clone(&plan));
253        let mut rows_in_batch = 0usize;
254
255        let mut file = File::open(path)?;
256        // Reusable buffer sized to the largest chunk to avoid per-chunk alloc.
257        let max_chunk = chunks.iter().map(|r| r.len()).max().unwrap_or(0);
258        let mut chunk_buf = Vec::with_capacity(max_chunk);
259
260        for chunk in &chunks {
261            let chunk_len = chunk.len();
262            chunk_buf.resize(chunk_len, 0);
263            file.seek(SeekFrom::Start(chunk.start as u64))?;
264            file.read_exact(&mut chunk_buf)?;
265            let mut chunk_engine = TableBuilder::with_plan(
266                (chunk.len() / bytes_per_row.max(512)).max(64),
267                Arc::clone(&plan),
268            );
269            parser.validate(&chunk_buf)?;
270            parser.parse_chunk_generic(&chunk_buf, &mut chunk_engine)?;
271
272            let chunk_rows = chunk_engine.num_rows();
273            batch_engine.extend(chunk_engine)?;
274            rows_in_batch += chunk_rows;
275
276            // Dynamic sizing: flush when either row count or actual bytes_used exceeds budget.
277            while rows_in_batch >= rows_per_batch
278                || batch_engine.bytes_used() >= self.budget.bytes()
279            {
280                if batch_engine.num_rows() == 0 {
281                    break;
282                }
283                let n = rows_per_batch.min(batch_engine.num_rows());
284                // If bytes_used is the trigger, estimate a smaller n to stay under budget
285                let n = if batch_engine.bytes_used() >= self.budget.bytes()
286                    && batch_engine.num_rows() > 1
287                {
288                    let est = (batch_engine.num_rows() as f64 * self.budget.bytes() as f64
289                        / batch_engine.bytes_used() as f64) as usize;
290                    est.clamp(1, n)
291                } else {
292                    n
293                };
294                let mut to_consume = batch_engine.split_off(n);
295                let mut batch = to_consume.finish()?;
296                if let Some(ref filter) = plan.filter {
297                    batch = apply_compare_filter(batch, filter)?;
298                }
299                consumer.consume(batch)?;
300                rows_in_batch = rows_in_batch.saturating_sub(n);
301            }
302        }
303
304        if batch_engine.num_rows() > 0 {
305            let mut batch = batch_engine.finish()?;
306            if let Some(ref filter) = plan.filter {
307                batch = apply_compare_filter(batch, filter)?;
308            }
309            consumer.consume(batch)?;
310        }
311
312        Ok(())
313    }
314
315    /// Legacy path for mapped inputs: plan against the mapping, drop it, then
316    /// read each chunk from the file with `seek` + `read_exact`.
317    #[cfg(feature = "mmap")]
318    #[expect(dead_code, reason = "legacy path kept for reference")]
319    fn run_mapped<P>(
320        &self,
321        path: &Path,
322        input: InputBuffer,
323        splitter: &dyn Splitter,
324        parser: P,
325        plan: Arc<ExecutionPlan>,
326    ) -> Result<Vec<RecordBatch>>
327    where
328        P: RecordParser + Clone + Send + Sync,
329    {
330        let batches = Vec::new();
331        let mut consumer = CollectingConsumer(batches);
332        self.run_mapped_stream(path, input, splitter, parser, plan, &mut consumer)?;
333        Ok(consumer.0)
334    }
335}
336
337/// Post-assembly safety net: re-apply pure column-comparison plans with
338/// Arrow kernels. Per-row evaluation during parse is authoritative, so trees
339/// involving `Or`, `Not`, `Equal`, or `NotEqual` pass through untouched.
340#[expect(dead_code, reason = "legacy path kept for reference")]
341fn apply_plan_filter(batches: &mut Vec<RecordBatch>, plan: &ExecutionPlan) -> Result<()> {
342    if let Some(ref filter) = plan.filter {
343        for batch in batches {
344            *batch = apply_compare_filter(
345                std::mem::replace(batch, RecordBatch::new_empty(batch.schema())),
346                filter,
347            )?;
348        }
349    }
350    Ok(())
351}