use super::table::TurndbTable;
use super::ScanStats;
use crate::store::ReadStore;
use anyhow::{bail, Context, Result};
use arrow::datatypes::SchemaRef;
use arrow::ipc::writer::StreamWriter;
use arrow::record_batch::RecordBatch;
use datafusion::error::DataFusionError;
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
use datafusion::physical_plan::SendableRecordBatchStream;
use datafusion::prelude::{SQLOptions, SessionConfig, SessionContext};
use datafusion::scalar::ScalarValue;
use futures::StreamExt;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
pub const TABLE_NAME: &str = "records";
pub const DEFAULT_MEMORY_BYTES: usize = 256 << 20;
pub const DEFAULT_AGGREGATE_MEMORY_BYTES: usize = 1 << 30;
#[derive(Clone)]
pub struct SqlBudget {
inner: Arc<SqlBudgetInner>,
}
struct SqlBudgetInner {
limit: usize,
reserved: AtomicUsize,
}
#[derive(Debug)]
pub struct SqlBudgetExhausted {
pub requested: usize,
pub available: usize,
pub limit: usize,
}
impl std::fmt::Display for SqlBudgetExhausted {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"SQL query requests {} execution-memory bytes but only {} of the shared {}-byte budget are available",
self.requested, self.available, self.limit
)
}
}
impl std::error::Error for SqlBudgetExhausted {}
struct SqlReservation {
budget: SqlBudget,
bytes: usize,
}
impl Drop for SqlReservation {
fn drop(&mut self) {
self.budget.inner.reserved.fetch_sub(self.bytes, Ordering::AcqRel);
}
}
impl SqlBudget {
pub fn new(limit: usize) -> Result<SqlBudget> {
if limit == 0 {
bail!("aggregate SQL memory limit must be greater than zero");
}
Ok(SqlBudget { inner: Arc::new(SqlBudgetInner { limit, reserved: AtomicUsize::new(0) }) })
}
pub fn limit(&self) -> usize {
self.inner.limit
}
pub fn reserved(&self) -> usize {
self.inner.reserved.load(Ordering::Acquire)
}
fn reserve(&self, bytes: usize) -> Result<SqlReservation> {
let mut current = self.reserved();
loop {
let available = self.inner.limit.saturating_sub(current);
if bytes > available {
return Err(SqlBudgetExhausted {
requested: bytes,
available,
limit: self.inner.limit,
}
.into());
}
match self.inner.reserved.compare_exchange_weak(
current,
current + bytes,
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => return Ok(SqlReservation { budget: self.clone(), bytes }),
Err(observed) => current = observed,
}
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SqlErrorClass {
InvalidArgument,
ResourceExhausted,
Unsupported,
Io,
Internal,
}
pub fn classify_error(error: &anyhow::Error) -> SqlErrorClass {
if error.chain().any(|cause| cause.downcast_ref::<SqlBudgetExhausted>().is_some()) {
return SqlErrorClass::ResourceExhausted;
}
error
.chain()
.find_map(|cause| cause.downcast_ref::<DataFusionError>())
.map(classify_datafusion)
.unwrap_or(SqlErrorClass::Internal)
}
fn classify_datafusion(error: &DataFusionError) -> SqlErrorClass {
match error {
DataFusionError::Context(_, source) => classify_datafusion(source),
DataFusionError::External(source) => source
.downcast_ref::<DataFusionError>()
.map(classify_datafusion)
.unwrap_or(SqlErrorClass::Internal),
DataFusionError::SQL(..)
| DataFusionError::Plan(_)
| DataFusionError::SchemaError(..)
| DataFusionError::Execution(_)
| DataFusionError::Configuration(_) => SqlErrorClass::InvalidArgument,
DataFusionError::ResourcesExhausted(_) => SqlErrorClass::ResourceExhausted,
DataFusionError::NotImplemented(_) => SqlErrorClass::Unsupported,
DataFusionError::IoError(_) => SqlErrorClass::Io,
_ => SqlErrorClass::Internal,
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct SqlOptions {
pub max_memory_bytes: usize,
}
impl Default for SqlOptions {
fn default() -> Self {
SqlOptions { max_memory_bytes: DEFAULT_MEMORY_BYTES }
}
}
#[derive(Clone, Debug, PartialEq)]
pub enum SqlValue {
Null,
String(String),
Int(i64),
UInt(u64),
Float(f64),
Bool(bool),
Binary(Vec<u8>),
TimestampNs(i64),
}
impl From<SqlValue> for ScalarValue {
fn from(value: SqlValue) -> ScalarValue {
match value {
SqlValue::Null => ScalarValue::Null,
SqlValue::String(value) => ScalarValue::Utf8(Some(value)),
SqlValue::Int(value) => ScalarValue::Int64(Some(value)),
SqlValue::UInt(value) => ScalarValue::UInt64(Some(value)),
SqlValue::Float(value) => ScalarValue::Float64(Some(value)),
SqlValue::Bool(value) => ScalarValue::Boolean(Some(value)),
SqlValue::Binary(value) => ScalarValue::Binary(Some(value)),
SqlValue::TimestampNs(value) => {
ScalarValue::TimestampNanosecond(Some(value), Some("UTC".into()))
}
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SqlBatch {
pub ipc: Vec<u8>,
pub rows: usize,
}
pub struct SqlQuery {
stream: Option<SendableRecordBatchStream>,
schema_ipc: Vec<u8>,
table: std::sync::Arc<TurndbTable>,
finished: bool,
reservation: Option<SqlReservation>,
planning_duration_ns: u64,
execution_duration_ns: u64,
}
impl SqlQuery {
pub async fn open(
store: ReadStore,
sql: &str,
params: Vec<SqlValue>,
options: SqlOptions,
) -> Result<SqlQuery> {
Self::open_inner(store, sql, params, options, None).await
}
pub async fn open_with_budget(
store: ReadStore,
sql: &str,
params: Vec<SqlValue>,
options: SqlOptions,
budget: &SqlBudget,
) -> Result<SqlQuery> {
Self::open_inner(store, sql, params, options, Some(budget)).await
}
async fn open_inner(
store: ReadStore,
sql: &str,
params: Vec<SqlValue>,
options: SqlOptions,
budget: Option<&SqlBudget>,
) -> Result<SqlQuery> {
let planning_started = std::time::Instant::now();
if sql.trim().is_empty() {
bail!("SQL query must not be empty");
}
if options.max_memory_bytes == 0 {
bail!("SQL max_memory_bytes must be greater than zero");
}
let reservation =
budget.map(|budget| budget.reserve(options.max_memory_bytes)).transpose()?;
let runtime = RuntimeEnvBuilder::new()
.with_memory_limit(options.max_memory_bytes, 1.0)
.build_arc()
.context("build bounded SQL runtime")?;
let ctx = SessionContext::new_with_config_rt(SessionConfig::new(), runtime);
let table =
TurndbTable::register(store, &ctx, TABLE_NAME).context("register TurnDB SQL table")?;
let read_only = SQLOptions::new()
.with_allow_ddl(false)
.with_allow_dml(false)
.with_allow_statements(false);
let mut frame =
ctx.sql_with_options(sql, read_only).await.context("plan read-only TurnDB SQL")?;
frame = frame
.with_param_values(params.into_iter().map(ScalarValue::from).collect::<Vec<_>>())
.context("bind TurnDB SQL parameters")?;
let stream = frame.execute_stream().await.context("start TurnDB SQL execution")?;
let schema_ipc = encode_schema(&stream.schema()).context("encode SQL result schema")?;
let planning_duration_ns =
u64::try_from(planning_started.elapsed().as_nanos()).unwrap_or(u64::MAX);
Ok(SqlQuery {
stream: Some(stream),
schema_ipc,
table,
finished: false,
reservation,
planning_duration_ns,
execution_duration_ns: 0,
})
}
pub fn schema_ipc(&self) -> &[u8] {
&self.schema_ipc
}
pub async fn next(&mut self) -> Result<Option<SqlBatch>> {
if self.finished {
return Ok(None);
}
let started = std::time::Instant::now();
let result = self.next_inner().await;
let elapsed = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX);
self.execution_duration_ns = self.execution_duration_ns.saturating_add(elapsed);
result
}
async fn next_inner(&mut self) -> Result<Option<SqlBatch>> {
let batch = match self
.stream
.as_mut()
.expect("unfinished SQL query retains its stream")
.next()
.await
.transpose()
.context("execute TurnDB SQL batch")
{
Ok(Some(batch)) => batch,
Ok(None) => {
self.finished = true;
self.stream = None;
self.reservation = None;
return Ok(None);
}
Err(error) => {
self.finished = true;
self.stream = None;
self.reservation = None;
return Err(error);
}
};
let rows = batch.num_rows();
let ipc = encode_batch(&batch).context("encode TurnDB SQL batch")?;
Ok(Some(SqlBatch { ipc, rows }))
}
pub fn stats(&self) -> ScanStats {
ScanStats {
planning_duration_ns: self.planning_duration_ns,
execution_duration_ns: self.execution_duration_ns,
..self.table.stats()
}
}
pub fn is_finished(&self) -> bool {
self.finished
}
}
fn encode_schema(schema: &SchemaRef) -> Result<Vec<u8>> {
let mut ipc = Vec::new();
{
let mut writer = StreamWriter::try_new(&mut ipc, schema)?;
writer.finish()?;
}
Ok(ipc)
}
fn encode_batch(batch: &RecordBatch) -> Result<Vec<u8>> {
let mut ipc = Vec::new();
{
let mut writer = StreamWriter::try_new(&mut ipc, &batch.schema())?;
writer.write(batch)?;
writer.finish()?;
}
Ok(ipc)
}