Skip to main content

rypipe_core/
bounded.rs

1use std::fs::File;
2use std::io::{Read, Seek, SeekFrom};
3use std::path::Path;
4
5use arrow::record_batch::RecordBatch;
6
7use crate::arrow_export::apply_compare_filter;
8use crate::decoder::{split_points_to_ranges, RecordParser, Splitter};
9use crate::engine::TableBuilder;
10use crate::input::InputBuffer;
11use crate::plan::ExecutionPlan;
12use crate::Result;
13
14/// A memory budget expressed in bytes.
15#[derive(Clone, Copy, Debug)]
16pub struct MemoryBudget {
17    bytes: usize,
18}
19
20impl MemoryBudget {
21    pub fn new(bytes: usize) -> Self {
22        Self { bytes }
23    }
24
25    pub fn bytes(&self) -> usize {
26        self.bytes
27    }
28}
29
30/// Parse a file in bounded batches to stay within a memory budget.
31pub struct BoundedExecutor {
32    budget: MemoryBudget,
33}
34
35impl BoundedExecutor {
36    pub fn new(budget: MemoryBudget) -> Self {
37        Self { budget }
38    }
39
40    /// Parse `path` in batches, returning one `RecordBatch` per batch.
41    ///
42    /// The caller is responsible for concatenating the batches if a single
43    /// table is desired.
44    pub fn run<P>(
45        &self,
46        path: &Path,
47        splitter: &dyn Splitter,
48        parser: P,
49        plan: ExecutionPlan,
50        prefault: bool,
51    ) -> Result<Vec<RecordBatch>>
52    where
53        P: RecordParser + Clone + Send + Sync,
54    {
55        let use_mmap = cfg!(feature = "mmap");
56        let input = InputBuffer::open(path, use_mmap, prefault)?;
57        let bytes = input.as_slice();
58        let file_len = bytes.len();
59
60        let bytes_per_row = splitter.estimate_bytes_per_row(bytes).max(1);
61        let total_rows_est = file_len / bytes_per_row;
62        let rows_per_batch = (self.budget.bytes() / bytes_per_row)
63            .max(1)
64            .min(total_rows_est.max(1));
65
66        let num_batches = (total_rows_est / rows_per_batch).max(1);
67        let split_points = splitter.find_split_points(bytes, num_batches.min(64));
68        let chunks = split_points_to_ranges(&split_points, file_len);
69        drop(input);
70
71        if file_len == 0 {
72            return Ok(Vec::new());
73        }
74
75        let mut batches = Vec::new();
76        let mut batch_engine = TableBuilder::with_plan(bytes_per_row.max(64), plan.clone());
77        let mut rows_in_batch = 0usize;
78
79        let mut file = File::open(path)?;
80        for chunk in &chunks {
81            let chunk_len = chunk.len();
82            let mut chunk_buf = vec![0u8; chunk_len];
83            file.seek(SeekFrom::Start(chunk.start as u64))?;
84            file.read_exact(&mut chunk_buf)?;
85
86            let mut chunk_engine = TableBuilder::with_plan((chunk_len / 512).max(64), plan.clone());
87            parser.validate(&chunk_buf)?;
88            parser.parse_chunk(&chunk_buf, &mut chunk_engine)?;
89
90            let chunk_rows = chunk_engine.num_rows();
91            batch_engine.extend(chunk_engine)?;
92            rows_in_batch += chunk_rows;
93
94            if rows_in_batch >= rows_per_batch {
95                let batch = batch_engine.finish()?;
96                batches.push(batch);
97                batch_engine.reset();
98                rows_in_batch = 0;
99            }
100        }
101
102        if batch_engine.num_rows() > 0 {
103            let batch = batch_engine.finish()?;
104            batches.push(batch);
105        }
106
107        if let Some(ref filter) = plan.filter {
108            for batch in &mut batches {
109                *batch = apply_compare_filter(
110                    std::mem::replace(batch, RecordBatch::new_empty(batch.schema())),
111                    filter,
112                )?;
113            }
114        }
115
116        Ok(batches)
117    }
118}