Skip to main content

rypipe_core/
parallel.rs

1use std::panic::{catch_unwind, AssertUnwindSafe};
2
3use arrow::record_batch::RecordBatch;
4use rayon::prelude::*;
5
6use crate::arrow_export::apply_compare_filter;
7use crate::decoder::{split_points_to_ranges, RecordParser, Splitter};
8use crate::engine::TableBuilder;
9use crate::merge::engines_to_record_batches;
10use crate::plan::ExecutionPlan;
11use crate::Result;
12
13/// Parallel executor that splits an input byte slice into chunks, parses each
14/// chunk independently, and returns the resulting Arrow record batches.
15pub struct ParallelExecutor;
16
17impl ParallelExecutor {
18    /// Parse `bytes` in parallel using `splitter` and `parser`.
19    ///
20    /// * Fast path (no `auto_dict` and no `Compare` filter): each chunk is
21    ///   exported as its own `RecordBatch` and a vector of batches is returned.
22    /// * Merge path: chunk builders are merged sequentially so that
23    ///   auto-dictionary upgrades see the full cardinality, then a single
24    ///   `RecordBatch` is returned inside the vector.
25    pub fn parse<P>(
26        bytes: &[u8],
27        splitter: &dyn Splitter,
28        parser: P,
29        plan: ExecutionPlan,
30        num_chunks: usize,
31    ) -> Result<Vec<RecordBatch>>
32    where
33        P: RecordParser + Clone + Send + Sync,
34    {
35        let split_points = splitter.find_split_points(bytes, num_chunks);
36        let ranges = split_points_to_ranges(&split_points, bytes.len());
37
38        let results: Vec<Result<TableBuilder>> = ranges
39            .into_par_iter()
40            .map(|range| {
41                catch_unwind(AssertUnwindSafe(|| {
42                    let est = if !range.is_empty() {
43                        (range.len() / 512).max(64)
44                    } else {
45                        64
46                    };
47                    let mut sink = TableBuilder::with_plan(est, plan.clone());
48                    parser.validate(&bytes[range.clone()])?;
49                    parser.parse_chunk(&bytes[range.clone()], &mut sink)?;
50                    Ok(sink)
51                }))
52                .unwrap_or_else(|_| {
53                    Err(crate::Error::Merge(
54                        "worker panicked during parallel parse".to_string(),
55                    ))
56                })
57            })
58            .collect();
59
60        let engines: Vec<TableBuilder> = results.into_iter().collect::<Result<_>>()?;
61
62        if engines.is_empty() {
63            return Ok(Vec::new());
64        }
65
66        let plan = engines[0].plan.clone();
67
68        // Fast path: no auto_dict and no post-reduce Compare filter.
69        if !plan.auto_dict
70            && !matches!(
71                plan.filter,
72                Some(crate::plan::FilterPredicate::Compare { .. })
73            )
74        {
75            return engines_to_record_batches(engines, &plan);
76        }
77
78        // Merge path.
79        let mut merged = TableBuilder::with_plan(engines.len().max(64) * 512, plan.clone());
80        for engine in engines {
81            merged.extend(engine)?;
82        }
83        let batch = merged.finish()?;
84        if let Some(ref filter) = plan.filter {
85            return Ok(vec![apply_compare_filter(batch, filter)?]);
86        }
87        Ok(vec![batch])
88    }
89}