use std::panic::{catch_unwind, AssertUnwindSafe};
use arrow::record_batch::RecordBatch;
use rayon::prelude::*;
use crate::arrow_export::apply_compare_filter;
use crate::decoder::{split_points_to_ranges, RecordParser, Splitter};
use crate::engine::TableBuilder;
use crate::merge::engines_to_record_batches;
use crate::plan::ExecutionPlan;
use crate::Result;
pub struct ParallelExecutor;
impl ParallelExecutor {
pub fn parse<P>(
bytes: &[u8],
splitter: &dyn Splitter,
parser: P,
plan: ExecutionPlan,
num_chunks: usize,
) -> Result<Vec<RecordBatch>>
where
P: RecordParser + Clone + Send + Sync,
{
let split_points = splitter.find_split_points(bytes, num_chunks);
let ranges = split_points_to_ranges(&split_points, bytes.len());
let results: Vec<Result<TableBuilder>> = ranges
.into_par_iter()
.map(|range| {
catch_unwind(AssertUnwindSafe(|| {
let est = if !range.is_empty() {
(range.len() / 512).max(64)
} else {
64
};
let mut sink = TableBuilder::with_plan(est, plan.clone());
parser.validate(&bytes[range.clone()])?;
parser.parse_chunk(&bytes[range.clone()], &mut sink)?;
Ok(sink)
}))
.unwrap_or_else(|_| {
Err(crate::Error::Merge(
"worker panicked during parallel parse".to_string(),
))
})
})
.collect();
let engines: Vec<TableBuilder> = results.into_iter().collect::<Result<_>>()?;
if engines.is_empty() {
return Ok(Vec::new());
}
let plan = engines[0].plan.clone();
if !plan.auto_dict
&& !matches!(
plan.filter,
Some(crate::plan::FilterPredicate::Compare { .. })
)
{
return engines_to_record_batches(engines, &plan);
}
let mut merged = TableBuilder::with_plan(engines.len().max(64) * 512, plan.clone());
for engine in engines {
merged.extend(engine)?;
}
let batch = merged.finish()?;
if let Some(ref filter) = plan.filter {
return Ok(vec![apply_compare_filter(batch, filter)?]);
}
Ok(vec![batch])
}
}