use std::{
collections::BTreeMap,
path::PathBuf,
sync::{Arc, Mutex},
};
use async_trait::async_trait;
use duckdb::{Connection, params_from_iter, types::ValueRef};
use super::{Backend, BackendCapabilities, Capability, PolyglotError};
const MAX_QUERY_ROWS: u32 = 10_000;
const MAX_SQL_BYTES: usize = 1024 * 1024;
const MAX_PARAMETERS: usize = 1_024;
const MAX_PARAMETER_BYTES: usize = 8 * 1024 * 1024;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum AnalyticsTimeUnit {
Second,
Millisecond,
Microsecond,
Nanosecond,
}
#[derive(Debug, Clone, PartialEq)]
#[non_exhaustive]
pub enum AnalyticsValue {
Null,
Boolean(bool),
Signed(i128),
Unsigned(u128),
Float(f64),
Text(String),
Bytes(Vec<u8>),
Decimal {
precision: u8,
scale: u8,
scaled: i128,
},
Timestamp {
unit: AnalyticsTimeUnit,
value: i64,
},
DateDays(i32),
Time {
unit: AnalyticsTimeUnit,
value: i64,
},
Interval {
months: i32,
days: i32,
nanos: i64,
},
Geometry(Vec<u8>),
}
#[derive(Debug, Clone, PartialEq)]
pub struct AnalyticsRow {
columns: BTreeMap<String, AnalyticsValue>,
}
impl AnalyticsRow {
pub fn get(&self, name: &str) -> Option<&AnalyticsValue> {
self.columns.get(name)
}
pub fn into_columns(self) -> BTreeMap<String, AnalyticsValue> {
self.columns
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct QueryLimit(u32);
impl QueryLimit {
pub fn new(limit: u32) -> Result<Self, PolyglotError> {
if limit == 0 || limit > MAX_QUERY_ROWS {
return Err(PolyglotError::InvalidIdentifier {
kind: "analytics query limit",
reason: "limit must be between 1 and 10000",
});
}
Ok(Self(limit))
}
pub const fn get(self) -> u32 {
self.0
}
}
#[async_trait]
pub trait AnalyticsRepository: Send + Sync {
async fn execute(
&self,
sql: impl Into<String> + Send,
parameters: Vec<AnalyticsValue>,
) -> Result<usize, PolyglotError>;
async fn query(
&self,
sql: impl Into<String> + Send,
parameters: Vec<AnalyticsValue>,
limit: QueryLimit,
) -> Result<Vec<AnalyticsRow>, PolyglotError>;
}
#[derive(Clone)]
pub struct DuckDbStore {
connection: Arc<Mutex<Connection>>,
}
impl DuckDbStore {
pub async fn in_memory() -> Result<Self, PolyglotError> {
let connection = tokio::task::spawn_blocking(Connection::open_in_memory)
.await
.map_err(worker_error)?
.map_err(|error| PolyglotError::driver("DuckDB", error))?;
Ok(Self {
connection: Arc::new(Mutex::new(connection)),
})
}
pub async fn open(path: impl Into<PathBuf> + Send + 'static) -> Result<Self, PolyglotError> {
let path = path.into();
let connection = tokio::task::spawn_blocking(move || Connection::open(path))
.await
.map_err(worker_error)?
.map_err(|error| PolyglotError::driver("DuckDB", error))?;
Ok(Self {
connection: Arc::new(Mutex::new(connection)),
})
}
pub const fn capabilities() -> BackendCapabilities {
BackendCapabilities::new(Backend::DuckDb, &[Capability::Analytics])
}
}
#[async_trait]
impl AnalyticsRepository for DuckDbStore {
async fn execute(
&self,
sql: impl Into<String> + Send,
parameters: Vec<AnalyticsValue>,
) -> Result<usize, PolyglotError> {
let connection = Arc::clone(&self.connection);
let sql = sql.into();
validate_analytics_request(&sql, ¶meters)?;
tokio::task::spawn_blocking(move || {
let connection = connection.lock().map_err(|_| PolyglotError::Worker {
backend: "DuckDB",
message: "connection lock was poisoned".to_owned(),
})?;
let values = parameters
.into_iter()
.map(to_duckdb_value)
.collect::<Result<Vec<_>, _>>()?;
connection
.execute(&sql, params_from_iter(values.iter()))
.map_err(|error| PolyglotError::driver("DuckDB", error))
})
.await
.map_err(worker_error)?
}
async fn query(
&self,
sql: impl Into<String> + Send,
parameters: Vec<AnalyticsValue>,
limit: QueryLimit,
) -> Result<Vec<AnalyticsRow>, PolyglotError> {
let connection = Arc::clone(&self.connection);
let sql = sql.into();
validate_analytics_request(&sql, ¶meters)?;
tokio::task::spawn_blocking(move || {
let connection = connection.lock().map_err(|_| PolyglotError::Worker {
backend: "DuckDB",
message: "connection lock was poisoned".to_owned(),
})?;
let values = parameters
.into_iter()
.map(to_duckdb_value)
.collect::<Result<Vec<_>, _>>()?;
let mut statement = connection
.prepare(&sql)
.map_err(|error| PolyglotError::driver("DuckDB", error))?;
let mut rows = statement
.query(params_from_iter(values.iter()))
.map_err(|error| PolyglotError::driver("DuckDB", error))?;
let Some(metadata) = rows.as_ref() else {
return Err(PolyglotError::Driver {
backend: "DuckDB",
message: "query returned no result metadata".to_owned(),
});
};
let column_names = metadata.column_names();
let mut materialized = Vec::new();
while materialized.len() < limit.get() as usize {
let Some(row) = rows
.next()
.map_err(|error| PolyglotError::driver("DuckDB", error))?
else {
break;
};
let mut columns = BTreeMap::new();
for (index, name) in column_names.iter().enumerate() {
if columns.contains_key(name) {
return Err(PolyglotError::Driver {
backend: "DuckDB",
message: format!("duplicate result column name: {name}"),
});
}
let value = row
.get_ref(index)
.map_err(|error| PolyglotError::driver("DuckDB", error))?;
columns.insert(name.clone(), from_duckdb_value(value)?);
}
materialized.push(AnalyticsRow { columns });
}
Ok(materialized)
})
.await
.map_err(worker_error)?
}
}
fn validate_analytics_request(
sql: &str,
parameters: &[AnalyticsValue],
) -> Result<(), PolyglotError> {
if sql.trim().is_empty() || sql.len() > MAX_SQL_BYTES {
return Err(PolyglotError::InvalidIdentifier {
kind: "DuckDB SQL statement",
reason: "SQL must contain between 1 byte and 1 MiB",
});
}
if parameters.len() > MAX_PARAMETERS {
return Err(PolyglotError::InvalidIdentifier {
kind: "DuckDB SQL parameters",
reason: "a statement accepts at most 1024 parameters",
});
}
let bytes = parameters.iter().try_fold(0usize, |total, value| {
total.checked_add(match value {
AnalyticsValue::Null => 0,
AnalyticsValue::Boolean(_) => 1,
AnalyticsValue::Signed(_)
| AnalyticsValue::Unsigned(_)
| AnalyticsValue::Decimal { .. }
| AnalyticsValue::Float(_)
| AnalyticsValue::Timestamp { .. }
| AnalyticsValue::Time { .. }
| AnalyticsValue::Interval { .. } => 16,
AnalyticsValue::DateDays(_) => 4,
AnalyticsValue::Text(value) => value.len(),
AnalyticsValue::Bytes(value) | AnalyticsValue::Geometry(value) => value.len(),
})
});
if bytes.is_none_or(|bytes| bytes > MAX_PARAMETER_BYTES) {
return Err(PolyglotError::InvalidIdentifier {
kind: "DuckDB SQL parameters",
reason: "parameter payload must not exceed 8 MiB",
});
}
Ok(())
}
fn to_duckdb_value(value: AnalyticsValue) -> Result<duckdb::types::Value, PolyglotError> {
use duckdb::types::{Decimal, Value};
match value {
AnalyticsValue::Null => Ok(Value::Null),
AnalyticsValue::Boolean(value) => Ok(Value::Boolean(value)),
AnalyticsValue::Signed(value) => Ok(Value::HugeInt(value)),
AnalyticsValue::Unsigned(value) => Ok(Value::UHugeInt(value)),
AnalyticsValue::Float(value) => Ok(Value::Double(value)),
AnalyticsValue::Text(value) => Ok(Value::Text(value)),
AnalyticsValue::Bytes(value) => Ok(Value::Blob(value)),
AnalyticsValue::Decimal {
precision,
scale,
scaled,
} => Decimal::new(precision, scale, scaled)
.map(Value::Decimal)
.map_err(|error| PolyglotError::UnsupportedValue {
backend: "DuckDB",
kind: error.to_string(),
}),
AnalyticsValue::Timestamp { unit, value } => {
Ok(Value::Timestamp(to_duckdb_time_unit(unit), value))
}
AnalyticsValue::DateDays(value) => Ok(Value::Date32(value)),
AnalyticsValue::Time { unit, value } => Ok(Value::Time64(to_duckdb_time_unit(unit), value)),
AnalyticsValue::Interval {
months,
days,
nanos,
} => Ok(Value::Interval {
months,
days,
nanos,
}),
AnalyticsValue::Geometry(value) => Ok(Value::Geometry(value)),
}
}
fn from_duckdb_value(value: ValueRef<'_>) -> Result<AnalyticsValue, PolyglotError> {
Ok(match value {
ValueRef::Null => AnalyticsValue::Null,
ValueRef::Boolean(value) => AnalyticsValue::Boolean(value),
ValueRef::TinyInt(value) => AnalyticsValue::Signed(i128::from(value)),
ValueRef::SmallInt(value) => AnalyticsValue::Signed(i128::from(value)),
ValueRef::Int(value) => AnalyticsValue::Signed(i128::from(value)),
ValueRef::BigInt(value) => AnalyticsValue::Signed(i128::from(value)),
ValueRef::HugeInt(value) => AnalyticsValue::Signed(value),
ValueRef::UTinyInt(value) => AnalyticsValue::Unsigned(u128::from(value)),
ValueRef::USmallInt(value) => AnalyticsValue::Unsigned(u128::from(value)),
ValueRef::UInt(value) => AnalyticsValue::Unsigned(u128::from(value)),
ValueRef::UBigInt(value) => AnalyticsValue::Unsigned(u128::from(value)),
ValueRef::UHugeInt(value) => AnalyticsValue::Unsigned(value),
ValueRef::Float(value) => AnalyticsValue::Float(f64::from(value)),
ValueRef::Double(value) => AnalyticsValue::Float(value),
ValueRef::Decimal(value) => AnalyticsValue::Decimal {
precision: value.width(),
scale: value.scale(),
scaled: value.value(),
},
ValueRef::Timestamp(unit, value) => AnalyticsValue::Timestamp {
unit: from_duckdb_time_unit(unit),
value,
},
ValueRef::Text(value) => AnalyticsValue::Text(
std::str::from_utf8(value)
.map_err(PolyglotError::serialization)?
.to_owned(),
),
ValueRef::Blob(value) => AnalyticsValue::Bytes(value.to_vec()),
ValueRef::Geometry(value) => AnalyticsValue::Geometry(value.to_vec()),
ValueRef::Date32(value) => AnalyticsValue::DateDays(value),
ValueRef::Time64(unit, value) => AnalyticsValue::Time {
unit: from_duckdb_time_unit(unit),
value,
},
ValueRef::Interval {
months,
days,
nanos,
} => AnalyticsValue::Interval {
months,
days,
nanos,
},
other => {
return Err(PolyglotError::UnsupportedValue {
backend: "DuckDB",
kind: format!("{:?}", other.data_type()),
});
}
})
}
fn to_duckdb_time_unit(unit: AnalyticsTimeUnit) -> duckdb::types::TimeUnit {
match unit {
AnalyticsTimeUnit::Second => duckdb::types::TimeUnit::Second,
AnalyticsTimeUnit::Millisecond => duckdb::types::TimeUnit::Millisecond,
AnalyticsTimeUnit::Microsecond => duckdb::types::TimeUnit::Microsecond,
AnalyticsTimeUnit::Nanosecond => duckdb::types::TimeUnit::Nanosecond,
}
}
fn from_duckdb_time_unit(unit: duckdb::types::TimeUnit) -> AnalyticsTimeUnit {
match unit {
duckdb::types::TimeUnit::Second => AnalyticsTimeUnit::Second,
duckdb::types::TimeUnit::Millisecond => AnalyticsTimeUnit::Millisecond,
duckdb::types::TimeUnit::Microsecond => AnalyticsTimeUnit::Microsecond,
duckdb::types::TimeUnit::Nanosecond => AnalyticsTimeUnit::Nanosecond,
}
}
fn worker_error(error: tokio::task::JoinError) -> PolyglotError {
PolyglotError::Worker {
backend: "DuckDB",
message: error.to_string(),
}
}
#[cfg(test)]
mod tests;