use crate::dataset::Dataset;
use crate::engine;
use crate::model::{Projection, QueryBatch, QueryOptions, QuerySource};
use crate::{ParqkitError, Result};
use arrow::datatypes::{Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use parquet::arrow::arrow_reader::{ParquetRecordBatchReader, ParquetRecordBatchReaderBuilder};
use parquet::arrow::ProjectionMask;
use std::collections::BTreeSet;
use std::fmt;
use std::fs::File;
use std::iter::FusedIterator;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct QueryStream {
sources: Vec<QuerySource>,
plans: Vec<SourcePlan>,
options: QueryOptions,
next_source: usize,
active: Option<ActiveSource>,
finished: bool,
}
impl QueryStream {
pub(crate) fn try_new(dataset: &Dataset, options: QueryOptions) -> Result<Self> {
validate_options(&options)?;
let plans = dataset
.paths()
.map(|path| SourcePlan::new(path, &options.projection))
.collect::<Result<Vec<_>>>()?;
let sources = plans
.iter()
.map(|plan| QuerySource {
path: plan.path.clone(),
schema: Arc::clone(&plan.schema),
})
.collect();
Ok(Self {
sources,
plans,
options,
next_source: 0,
active: None,
finished: false,
})
}
pub fn sources(&self) -> &[QuerySource] {
&self.sources
}
pub fn compatible_schema(&self) -> Result<SchemaRef> {
let first = self.sources.first().ok_or(ParqkitError::NoInputFiles)?;
for source in self.sources.iter().skip(1) {
if source.schema.as_ref() != first.schema.as_ref() {
return Err(ParqkitError::SchemaMismatch {
file1: first.path.display().to_string(),
file2: source.path.display().to_string(),
details: "Projected query schemas differ".to_string(),
});
}
}
Ok(Arc::clone(&first.schema))
}
fn open_next_source(&mut self) -> Result<bool> {
let Some(plan) = self.plans.get(self.next_source) else {
return Ok(false);
};
let source_index = self.next_source;
self.next_source += 1;
self.active = Some(ActiveSource::open(source_index, plan, &self.options)?);
Ok(true)
}
}
impl Iterator for QueryStream {
type Item = Result<QueryBatch>;
fn next(&mut self) -> Option<Self::Item> {
if self.finished {
return None;
}
loop {
if let Some(active) = self.active.as_mut() {
match active.reader.next() {
Some(Ok(batch)) => {
let batch = match active.project_batch(batch) {
Ok(batch) => batch,
Err(error) => {
self.active = None;
self.finished = true;
return Some(Err(error));
}
};
return Some(Ok(QueryBatch {
source_index: active.source_index,
path: active.path.clone(),
batch,
}));
}
Some(Err(error)) => {
let error = ParqkitError::corrupted(&active.path, error);
self.active = None;
self.finished = true;
return Some(Err(error));
}
None => self.active = None,
}
}
match self.open_next_source() {
Ok(true) => {}
Ok(false) => {
self.finished = true;
return None;
}
Err(error) => {
self.finished = true;
return Some(Err(error));
}
}
}
}
}
impl FusedIterator for QueryStream {}
impl fmt::Debug for QueryStream {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("QueryStream")
.field("sources", &self.sources)
.field("options", &self.options)
.field("next_source", &self.next_source)
.field("has_active_source", &self.active.is_some())
.field("finished", &self.finished)
.finish()
}
}
struct SourcePlan {
path: PathBuf,
schema: SchemaRef,
}
impl SourcePlan {
fn new(path: &Path, projection: &Projection) -> Result<Self> {
let builder = engine::parquet::reader_builder(path)?;
let projection = ProjectionPlan::new(path, &builder, projection)?;
Ok(Self {
path: path.to_path_buf(),
schema: projection.schema,
})
}
}
struct ProjectionPlan {
schema: SchemaRef,
projection_mask: ProjectionMask,
output_order: Option<Vec<usize>>,
}
impl ProjectionPlan {
fn new(
path: &Path,
builder: &ParquetRecordBatchReaderBuilder<File>,
projection: &Projection,
) -> Result<Self> {
match projection {
Projection::All => Ok(Self {
schema: engine::parquet::schema_without_custom_metadata(builder.schema()),
projection_mask: ProjectionMask::all(),
output_order: None,
}),
Projection::Columns(columns) => {
let root_indices = resolve_columns(path, builder.schema(), columns)?;
let mut file_order = root_indices.clone();
file_order.sort_unstable();
let output_order = root_indices
.iter()
.map(|root_index| {
file_order.binary_search(root_index).map_err(|_| {
ParqkitError::invalid_query(
"projected column could not be resolved in reader schema",
)
})
})
.collect::<Result<Vec<_>>>()?;
let output_order = (output_order != (0..output_order.len()).collect::<Vec<_>>())
.then_some(output_order);
let fields = root_indices
.iter()
.map(|index| Arc::clone(&builder.schema().fields()[*index]))
.collect::<Vec<_>>();
let schema = Arc::new(Schema::new(fields));
Ok(Self {
schema: engine::parquet::schema_without_custom_metadata(&schema),
projection_mask: ProjectionMask::roots(builder.parquet_schema(), root_indices),
output_order,
})
}
}
}
}
struct ActiveSource {
source_index: usize,
path: PathBuf,
schema: SchemaRef,
output_order: Option<Vec<usize>>,
reader: ParquetRecordBatchReader,
}
impl ActiveSource {
fn open(source_index: usize, planned: &SourcePlan, options: &QueryOptions) -> Result<Self> {
let builder = engine::parquet::reader_builder(&planned.path)?;
let current = ProjectionPlan::new(&planned.path, &builder, &options.projection)?;
if current.schema.as_ref() != planned.schema.as_ref() {
return Err(ParqkitError::invalid_metadata(
&planned.path,
"projected schema changed after query planning",
));
}
let mut builder = builder
.with_projection(current.projection_mask)
.with_batch_size(options.batch_size);
if let Some(limit) = options.limit {
builder = builder.with_limit(limit);
}
let reader = builder
.build()
.map_err(|error| ParqkitError::from_read(&planned.path, error))?;
Ok(Self {
source_index,
path: planned.path.clone(),
schema: Arc::clone(&planned.schema),
output_order: current.output_order,
reader,
})
}
fn project_batch(&self, batch: RecordBatch) -> Result<RecordBatch> {
let batch = if let Some(output_order) = self.output_order.as_ref() {
batch.project(output_order).map_err(|error| {
ParqkitError::invalid_metadata(
&self.path,
format!("could not order projected columns: {error}"),
)
})?
} else {
batch
};
engine::parquet::batch_with_schema(&self.path, batch, &self.schema)
}
}
fn validate_options(options: &QueryOptions) -> Result<()> {
if options.batch_size == 0 {
return Err(ParqkitError::invalid_query(
"query batch size must be greater than zero",
));
}
if matches!(&options.projection, Projection::Columns(columns) if columns.is_empty()) {
return Err(ParqkitError::invalid_query(
"query projection must contain at least one column",
));
}
Ok(())
}
fn resolve_columns(path: &Path, schema: &SchemaRef, columns: &[String]) -> Result<Vec<usize>> {
let mut seen = BTreeSet::new();
let mut indices = Vec::with_capacity(columns.len());
for column in columns {
if !seen.insert(column) {
return Err(ParqkitError::invalid_query(format!(
"duplicate projected column: {column}"
)));
}
let index = schema
.index_of(column)
.map_err(|_| ParqkitError::column_not_found(path, column))?;
indices.push(index);
}
Ok(indices)
}