1use std::panic::{catch_unwind, AssertUnwindSafe};
2
3use arrow::record_batch::RecordBatch;
4use rayon::prelude::*;
5
6use crate::arrow_export::apply_compare_filter;
7use crate::decoder::{split_points_to_ranges, RecordParser, Splitter};
8use crate::engine::TableBuilder;
9use crate::merge::engines_to_record_batches;
10use crate::plan::ExecutionPlan;
11use crate::Result;
12
13pub struct ParallelExecutor;
16
17impl ParallelExecutor {
18 pub fn parse<P>(
26 bytes: &[u8],
27 splitter: &dyn Splitter,
28 parser: P,
29 plan: ExecutionPlan,
30 num_chunks: usize,
31 ) -> Result<Vec<RecordBatch>>
32 where
33 P: RecordParser + Clone + Send + Sync,
34 {
35 let split_points = splitter.find_split_points(bytes, num_chunks);
36 let ranges = split_points_to_ranges(&split_points, bytes.len());
37
38 let results: Vec<Result<TableBuilder>> = ranges
39 .into_par_iter()
40 .map(|range| {
41 catch_unwind(AssertUnwindSafe(|| {
42 let est = if !range.is_empty() {
43 (range.len() / 512).max(64)
44 } else {
45 64
46 };
47 let mut sink = TableBuilder::with_plan(est, plan.clone());
48 parser.validate(&bytes[range.clone()])?;
49 parser.parse_chunk(&bytes[range.clone()], &mut sink)?;
50 Ok(sink)
51 }))
52 .unwrap_or_else(|_| {
53 Err(crate::Error::Merge(
54 "worker panicked during parallel parse".to_string(),
55 ))
56 })
57 })
58 .collect();
59
60 let engines: Vec<TableBuilder> = results.into_iter().collect::<Result<_>>()?;
61
62 if engines.is_empty() {
63 return Ok(Vec::new());
64 }
65
66 let plan = engines[0].plan.clone();
67
68 if !plan.auto_dict
70 && !matches!(
71 plan.filter,
72 Some(crate::plan::FilterPredicate::Compare { .. })
73 )
74 {
75 return engines_to_record_batches(engines, &plan);
76 }
77
78 let mut merged = TableBuilder::with_plan(engines.len().max(64) * 512, plan.clone());
80 for engine in engines {
81 merged.extend(engine)?;
82 }
83 let batch = merged.finish()?;
84 if let Some(ref filter) = plan.filter {
85 return Ok(vec![apply_compare_filter(batch, filter)?]);
86 }
87 Ok(vec![batch])
88 }
89}