use super::{Cmp, Lens, Pred, ScanStats};
use crate::fold::Fold;
use crate::part::Part;
use crate::store::ReadStore;
use crate::types::AttrValue;
use anyhow::Result;
use arrow::datatypes::SchemaRef;
use datafusion::catalog::{Session, TableProvider};
use datafusion::common::Result as DfResult;
use datafusion::datasource::TableType;
use datafusion::error::DataFusionError;
use datafusion::execution::{SendableRecordBatchStream, TaskContext};
use datafusion::logical_expr::{Operator, TableProviderFilterPushDown};
use datafusion::physical_expr::EquivalenceProperties;
use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
use datafusion::physical_plan::memory::MemoryStream;
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
use datafusion::physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
};
use datafusion::prelude::SessionContext;
use datafusion::scalar::ScalarValue;
use futures::stream;
use std::fmt;
use std::sync::{Arc, Mutex};
pub struct TurndbTable {
parts: Vec<Arc<Part>>,
fold: Arc<Fold>,
lens: Arc<Lens>,
stats: Arc<Mutex<ScanStats>>,
}
impl fmt::Debug for TurndbTable {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "TurndbTable({} parts)", self.parts.len())
}
}
impl TurndbTable {
pub fn new(parts: Vec<Arc<Part>>, fold: Arc<Fold>) -> Result<TurndbTable> {
let lens = Arc::new(Lens::new(&parts)?);
Ok(TurndbTable { parts, fold, lens, stats: Arc::new(Mutex::new(ScanStats::default())) })
}
pub fn context(store: ReadStore, name: &str) -> Result<(SessionContext, Arc<TurndbTable>)> {
let ctx = SessionContext::new();
let table = TurndbTable::register(store, &ctx, name)?;
Ok((ctx, table))
}
pub fn register(
store: ReadStore,
ctx: &SessionContext,
name: &str,
) -> Result<Arc<TurndbTable>> {
let (fold, parts) = store.into_parts();
let t = Arc::new(TurndbTable::new(parts, fold)?);
ctx.register_table(name, t.clone())?;
Ok(t)
}
pub fn stats(&self) -> ScanStats {
*self.stats.lock().unwrap()
}
pub fn reset_stats(&self) {
*self.stats.lock().unwrap() = ScanStats::default();
}
}
#[async_trait::async_trait]
impl TableProvider for TurndbTable {
fn schema(&self) -> SchemaRef {
self.lens.schema()
}
fn table_type(&self) -> TableType {
TableType::Base
}
fn supports_filters_pushdown(
&self,
filters: &[&datafusion::prelude::Expr],
) -> DfResult<Vec<TableProviderFilterPushDown>> {
Ok(filters
.iter()
.map(|f| match to_pred(f, &self.lens) {
Some(_) => TableProviderFilterPushDown::Inexact,
None => TableProviderFilterPushDown::Unsupported,
})
.collect())
}
async fn scan(
&self,
_state: &dyn Session,
projection: Option<&Vec<usize>>,
filters: &[datafusion::prelude::Expr],
_limit: Option<usize>,
) -> DfResult<Arc<dyn ExecutionPlan>> {
let full: Vec<usize> = (0..self.lens.schema().fields().len()).collect();
let proj = projection.cloned().unwrap_or(full);
let preds: Vec<Pred> = filters.iter().filter_map(|f| to_pred(f, &self.lens)).collect();
Ok(Arc::new(TurndbExec::try_new(
self.parts.clone(),
self.fold.clone(),
self.lens.clone(),
proj,
preds,
self.stats.clone(),
)?))
}
}
fn to_pred(e: &datafusion::prelude::Expr, lens: &Lens) -> Option<Pred> {
use datafusion::prelude::Expr;
let Expr::BinaryExpr(b) = e else { return None };
let (name, op, lit) = match (&*b.left, &*b.right) {
(Expr::Column(c), Expr::Literal(v, _)) => (&c.name, b.op, v),
(Expr::Literal(v, _), Expr::Column(c)) => (&c.name, flip(b.op)?, v),
_ => return None,
};
let field = lens.schema().index_of(name).ok()?;
let op = match op {
Operator::Eq => Cmp::Eq,
Operator::NotEq => Cmp::Ne,
Operator::Lt => Cmp::Lt,
Operator::LtEq => Cmp::LtEq,
Operator::Gt => Cmp::Gt,
Operator::GtEq => Cmp::GtEq,
_ => return None,
};
let val = scalar_to_attr(lit)?;
Some(Pred { field, op, val })
}
fn scalar_to_attr(v: &ScalarValue) -> Option<AttrValue> {
Some(match v {
ScalarValue::Dictionary(_, inner) => return scalar_to_attr(inner),
ScalarValue::Utf8(Some(s))
| ScalarValue::LargeUtf8(Some(s))
| ScalarValue::Utf8View(Some(s)) => AttrValue::Str(s.clone()),
ScalarValue::Int64(Some(i)) => AttrValue::Int(*i),
ScalarValue::Int32(Some(i)) => AttrValue::Int(*i as i64),
ScalarValue::Int16(Some(i)) => AttrValue::Int(*i as i64),
ScalarValue::Int8(Some(i)) => AttrValue::Int(*i as i64),
ScalarValue::UInt32(Some(i)) => AttrValue::Int(*i as i64),
ScalarValue::UInt64(Some(i)) => AttrValue::UInt(*i),
ScalarValue::Binary(Some(bytes)) | ScalarValue::LargeBinary(Some(bytes)) => {
AttrValue::Bytes(bytes.clone())
}
ScalarValue::TimestampNanosecond(Some(ns), _) => AttrValue::TimestampNs(*ns),
ScalarValue::Float64(Some(f)) => AttrValue::Float(*f),
ScalarValue::Float32(Some(f)) => AttrValue::Float(*f as f64),
ScalarValue::Boolean(Some(b)) => AttrValue::Bool(*b),
_ => return None,
})
}
fn flip(op: Operator) -> Option<Operator> {
Some(match op {
Operator::Eq => Operator::Eq,
Operator::NotEq => Operator::NotEq,
Operator::Lt => Operator::Gt,
Operator::LtEq => Operator::GtEq,
Operator::Gt => Operator::Lt,
Operator::GtEq => Operator::LtEq,
_ => return None,
})
}
struct TurndbExec {
parts: Vec<Arc<Part>>,
fold: Arc<Fold>,
lens: Arc<Lens>,
projection: Vec<usize>,
preds: Vec<Pred>,
stats: Arc<Mutex<ScanStats>>,
schema: SchemaRef,
props: Arc<PlanProperties>,
fetch: Option<usize>,
}
impl TurndbExec {
fn try_new(
parts: Vec<Arc<Part>>,
fold: Arc<Fold>,
lens: Arc<Lens>,
projection: Vec<usize>,
preds: Vec<Pred>,
stats: Arc<Mutex<ScanStats>>,
) -> DfResult<TurndbExec> {
let full = lens.schema();
let fields: Vec<_> = projection.iter().map(|&i| full.field(i).clone()).collect();
let schema: SchemaRef = Arc::new(arrow::datatypes::Schema::new(fields));
let props = PlanProperties::new(
EquivalenceProperties::new(schema.clone()),
Partitioning::UnknownPartitioning(parts.len().max(1)),
EmissionType::Incremental,
Boundedness::Bounded,
);
Ok(TurndbExec {
parts,
fold,
lens,
projection,
preds,
stats,
schema,
props: Arc::new(props),
fetch: None,
})
}
}
impl fmt::Debug for TurndbExec {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "TurndbExec({} parts)", self.parts.len())
}
}
impl DisplayAs for TurndbExec {
fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let cols: Vec<&str> = self.schema.fields().iter().map(|x| x.name().as_str()).collect();
write!(f, "TurndbExec: parts={}, projection=[{}]", self.parts.len(), cols.join(", "))
}
}
impl ExecutionPlan for TurndbExec {
fn name(&self) -> &str {
"TurndbExec"
}
fn properties(&self) -> &Arc<PlanProperties> {
&self.props
}
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
vec![]
}
fn fetch(&self) -> Option<usize> {
self.fetch
}
fn with_fetch(&self, limit: Option<usize>) -> Option<Arc<dyn ExecutionPlan>> {
Some(Arc::new(TurndbExec {
parts: self.parts.clone(),
fold: self.fold.clone(),
lens: self.lens.clone(),
projection: self.projection.clone(),
preds: self.preds.clone(),
stats: self.stats.clone(),
schema: self.schema.clone(),
props: self.props.clone(),
fetch: limit,
}))
}
fn with_new_children(
self: Arc<Self>,
_c: Vec<Arc<dyn ExecutionPlan>>,
) -> DfResult<Arc<dyn ExecutionPlan>> {
Ok(self)
}
fn execute(
&self,
partition: usize,
_ctx: Arc<TaskContext>,
) -> DfResult<SendableRecordBatchStream> {
let Some(part) = self.parts.get(partition) else {
return Ok(Box::pin(MemoryStream::try_new(vec![], self.schema.clone(), None)?));
};
let wants_content = self.lens.projection_reads_content(&self.projection);
let fold = wants_content.then_some(&self.fold);
let scan = self
.lens
.scan(part, fold, &self.projection, &self.preds)
.map_err(|e| DataFusionError::External(e.into()))?
.with_fetch(self.fetch);
let s = stream::unfold(
Some((scan, self.stats.clone(), ScanStats::default())),
move |st| async move {
let (mut scan, shared, prev) = st?;
match scan.next_batch() {
Ok(Some(b)) => {
let now = scan.stats();
shared.lock().unwrap().add(now.since(prev));
Some((Ok(b), Some((scan, shared, now))))
}
Ok(None) => {
shared.lock().unwrap().add(scan.stats().since(prev));
None
}
Err(e) => Some((Err(DataFusionError::External(e.into())), None)),
}
},
);
Ok(Box::pin(RecordBatchStreamAdapter::new(self.schema.clone(), s)))
}
}