use std::sync::Arc;
use std::time::Instant;
use ballista::prelude::SessionContextExt;
use datafusion::arrow::array::RecordBatch;
use datafusion::arrow::datatypes::SchemaRef;
use datafusion::dataframe::DataFrame;
use datafusion::execution::SessionStateBuilder;
use datafusion::physical_plan::{ExecutionPlan, collect, displayable};
use datafusion::prelude::{SessionConfig, SessionContext};
use oxidelake_compute::{
GpuAggregateExec, GpuFilterExec, GpuHashJoinExec, GpuVectorDistanceExec, local_backend,
oxide_udfs,
};
use oxidelake_core::telemetry::{OperatorSnapshot, TelemetryHub, TierCapacity};
use oxidelake_core::{BackendKind, EngineError};
use oxidelake_planner::{HardwarePlacementRule, physical_optimizer_rules};
use oxidelake_storage::{
ParquetScanGuard, default_object_store, register_local_store, register_parquet_table,
with_gpu_batch_size, with_pruning,
};
use crate::cluster;
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum SessionMode {
Embedded {
target: BackendKind,
},
Cluster {
scheduler_url: String,
},
}
fn local_capacity() -> TierCapacity {
let Ok(backend) = local_backend() else {
return TierCapacity::default();
};
let Ok(info) = backend.memory_info() else {
return TierCapacity::default();
};
if backend.kind().is_gpu() {
TierCapacity {
device_bytes: Some(info.total_bytes),
host_bytes: None,
spill_on_query_path: false,
}
} else {
TierCapacity {
device_bytes: None,
host_bytes: Some(info.total_bytes),
spill_on_query_path: false,
}
}
}
impl std::fmt::Display for SessionMode {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
SessionMode::Embedded { target } => write!(f, "embedded/{target}"),
SessionMode::Cluster { scheduler_url } => write!(f, "cluster/{scheduler_url}"),
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct SessionOptions {
pub target: Option<BackendKind>,
pub batch_size: Option<usize>,
}
impl SessionOptions {
pub fn new() -> Self {
Self::default()
}
pub fn with_target(mut self, target: BackendKind) -> Self {
self.target = Some(target);
self
}
pub fn with_batch_size(mut self, rows: usize) -> Self {
self.batch_size = Some(rows);
self
}
fn apply(
&self,
config: SessionConfig,
target: Option<BackendKind>,
) -> Result<SessionConfig, EngineError> {
match self.batch_size {
Some(0) => Err(EngineError::plan(
"batch size must be at least 1 row (0 would make every query return nothing)",
)),
Some(rows) => Ok(config.with_batch_size(rows)),
None if target.is_some_and(BackendKind::is_gpu) => Ok(with_gpu_batch_size(config)),
None => Ok(config),
}
}
}
pub struct OxideSession {
ctx: SessionContext,
mode: SessionMode,
telemetry: Arc<TelemetryHub>,
}
impl OxideSession {
pub fn local() -> Result<Self, EngineError> {
Self::local_with_options(&SessionOptions::new())
}
pub fn local_with_target(target: BackendKind) -> Result<Self, EngineError> {
Self::local_with_options(&SessionOptions::new().with_target(target))
}
pub fn local_with_options(options: &SessionOptions) -> Result<Self, EngineError> {
let target = match options.target {
Some(target) => target,
None => local_backend()?.kind(),
};
let telemetry = TelemetryHub::new();
telemetry.set_capacity(local_capacity());
let rule = HardwarePlacementRule::new(target).with_telemetry(Arc::clone(&telemetry));
let config = options.apply(with_pruning(SessionConfig::new()), Some(target))?;
let state = SessionStateBuilder::new()
.with_default_features()
.with_config(config)
.with_physical_optimizer_rules(physical_optimizer_rules(rule))
.with_physical_optimizer_rule(Arc::new(ParquetScanGuard::new()))
.build();
let ctx = SessionContext::new_with_state(state);
register_local_store(&ctx, default_object_store());
for udf in oxide_udfs() {
ctx.register_udf(udf.as_ref().clone());
}
Ok(Self {
ctx,
mode: SessionMode::Embedded { target },
telemetry,
})
}
pub async fn connect(scheduler_url: &str) -> Result<Self, EngineError> {
Self::connect_with_options(scheduler_url, &SessionOptions::new()).await
}
pub async fn connect_with_options(
scheduler_url: &str,
options: &SessionOptions,
) -> Result<Self, EngineError> {
let config = cluster::session_config().with_option_extension(cluster::ClientOptions {
explicit_batch_size: options.batch_size.is_some(),
});
let config = options.apply(config, None)?;
let state = SessionStateBuilder::new()
.with_default_features()
.with_config(config)
.build();
let ctx = SessionContext::remote_with_state(scheduler_url, state).await?;
for udf in oxide_udfs() {
ctx.register_udf(udf.as_ref().clone());
}
Ok(Self {
ctx,
mode: SessionMode::Cluster {
scheduler_url: scheduler_url.to_owned(),
},
telemetry: TelemetryHub::new(),
})
}
pub fn mode(&self) -> &SessionMode {
&self.mode
}
pub fn ctx(&self) -> &SessionContext {
&self.ctx
}
pub fn telemetry(&self) -> &Arc<TelemetryHub> {
&self.telemetry
}
pub async fn sql(&self, query: &str) -> Result<DataFrame, EngineError> {
Ok(self.ctx.sql(query).await?)
}
pub async fn collect(&self, query: &str) -> Result<(SchemaRef, Vec<RecordBatch>), EngineError> {
let started = Instant::now();
let frame = self.sql(query).await?;
let schema = SchemaRef::from(frame.schema().clone());
let task_ctx = Arc::new(frame.task_ctx());
let plan = frame.create_physical_plan().await?;
let batches = collect(Arc::clone(&plan), task_ctx).await?;
let rows: usize = batches.iter().map(RecordBatch::num_rows).sum();
let operators = query_operators(&plan);
let fallback_batches: u64 = operators.iter().map(|o| o.fallback_batches).sum();
let accelerated = operators.iter().filter(|o| o.backend.is_gpu()).count();
tracing::info!(
mode = %self.mode,
rows,
batches = batches.len(),
elapsed_ms = started.elapsed().as_secs_f64() * 1_000.0,
gpu_operators = accelerated,
fallback_batches,
"query finished"
);
Ok((schema, batches))
}
pub async fn register_parquet(&self, name: &str, path: &str) -> Result<(), EngineError> {
register_parquet_table(&self.ctx, name, path).await
}
pub async fn explain(&self, query: &str) -> Result<String, EngineError> {
self.telemetry.clear_skips();
let plan = self.ctx.sql(query).await?.create_physical_plan().await?;
let mut text = displayable(plan.as_ref()).indent(true).to_string();
text.push_str(&self.placement_notes());
Ok(text)
}
fn placement_notes(&self) -> String {
let skips = self.telemetry.skips();
if skips.is_empty() {
return String::new();
}
let mut out = String::from("\nplacement notes (target ");
match &self.mode {
SessionMode::Embedded { target } => out.push_str(target.as_str()),
SessionMode::Cluster { .. } => out.push_str("on the scheduler"),
}
out.push_str("):\n");
for skip in skips {
out.push_str(&format!(" {}: {}\n", skip.node, skip.reason));
}
out
}
}
fn query_operators(plan: &Arc<dyn ExecutionPlan>) -> Vec<OperatorSnapshot> {
let mut found = Vec::new();
let mut pending = vec![Arc::clone(plan)];
while let Some(node) = pending.pop() {
let stats = if let Some(exec) = node.downcast_ref::<GpuFilterExec>() {
exec.operator_stats()
} else if let Some(exec) = node.downcast_ref::<GpuHashJoinExec>() {
exec.operator_stats()
} else if let Some(exec) = node.downcast_ref::<GpuAggregateExec>() {
exec.operator_stats()
} else if let Some(exec) = node.downcast_ref::<GpuVectorDistanceExec>() {
exec.operator_stats()
} else {
None
};
found.extend(stats);
pending.extend(node.children().into_iter().cloned());
}
found
}
impl std::fmt::Debug for OxideSession {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("OxideSession")
.field("mode", &self.mode)
.field("session_id", &self.ctx.session_id())
.finish_non_exhaustive()
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
use super::*;
#[tokio::test]
async fn embedded_session_runs_sql_and_reports_mode() {
let session = OxideSession::local().unwrap();
assert!(matches!(session.mode(), SessionMode::Embedded { .. }));
let batches = session
.sql("SELECT 1 + 1 AS two")
.await
.unwrap()
.collect()
.await
.unwrap();
assert_eq!(batches.len(), 1);
assert_eq!(batches[0].num_rows(), 1);
let text = session.explain("SELECT 1 + 1 AS two").await.unwrap();
assert!(text.contains("ProjectionExec"), "{text}");
}
fn batch_size(session: &OxideSession) -> usize {
session.ctx().state().config().batch_size()
}
#[test]
fn the_batch_size_option_reaches_the_session_configuration() {
let session =
OxideSession::local_with_options(&SessionOptions::new().with_batch_size(7)).unwrap();
assert_eq!(batch_size(&session), 7);
}
#[test]
fn the_target_picks_the_default_batch_size_and_the_option_overrides_it() {
let gpu = OxideSession::local_with_target(BackendKind::Cuda).unwrap();
assert_eq!(batch_size(&gpu), oxidelake_storage::GPU_BATCH_SIZE);
let cpu = OxideSession::local_with_target(BackendKind::CpuSimd).unwrap();
assert_eq!(batch_size(&cpu), SessionConfig::new().batch_size());
let forced = OxideSession::local_with_options(
&SessionOptions::new()
.with_target(BackendKind::Cuda)
.with_batch_size(1_024),
)
.unwrap();
assert_eq!(batch_size(&forced), 1_024);
}
#[test]
fn a_zero_batch_size_is_refused() {
let err = OxideSession::local_with_options(&SessionOptions::new().with_batch_size(0))
.unwrap_err()
.to_string();
assert!(err.contains("at least 1 row"), "{err}");
}
#[test]
fn a_session_reports_real_capacities_and_an_honest_spill_flag() {
let session = OxideSession::local().unwrap();
let capacity = session.telemetry().capacity();
assert!(
!capacity.spill_on_query_path,
"no operator registers with a SpillManager in 0.2"
);
let SessionMode::Embedded { target } = session.mode() else {
panic!("local() is embedded");
};
let reported = if target.is_gpu() {
capacity.device_bytes
} else {
capacity.host_bytes
};
assert!(
reported.is_some_and(|bytes| bytes > 0),
"{target} reported no memory: {capacity:?}"
);
let absent = if target.is_gpu() {
capacity.host_bytes
} else {
capacity.device_bytes
};
assert_eq!(absent, None);
}
#[derive(Clone, Default)]
struct Captured(Arc<std::sync::Mutex<Vec<u8>>>);
impl std::io::Write for Captured {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.lock().unwrap().extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
fn field(line: &str, name: &str) -> u64 {
line.split_whitespace()
.find_map(|part| part.strip_prefix(name)?.strip_prefix('='))
.unwrap_or_else(|| panic!("no {name} in {line}"))
.parse()
.unwrap()
}
#[tokio::test]
async fn the_query_line_counts_only_its_own_operators() {
let captured = Captured::default();
let writer = captured.clone();
let subscriber = tracing_subscriber::fmt()
.with_writer(move || writer.clone())
.with_ansi(false)
.with_max_level(tracing::Level::INFO)
.finish();
let _default = tracing::subscriber::set_default(subscriber);
let session = OxideSession::local_with_target(BackendKind::Cuda).unwrap();
session
.ctx()
.sql("CREATE TABLE a AS SELECT value AS k, value * 3 AS w FROM generate_series(0, 99)")
.await
.unwrap()
.collect()
.await
.unwrap();
let query = "SELECT x.k, y.w FROM (SELECT * FROM a WHERE k > 10) x \
JOIN (SELECT * FROM a WHERE w > 30) y ON x.k = y.k";
for _ in 0..2 {
session.collect(query).await.unwrap();
}
let text = String::from_utf8(captured.0.lock().unwrap().clone()).unwrap();
let lines: Vec<&str> = text
.lines()
.filter(|line| line.contains("query finished"))
.collect();
let [first, second] = lines[..] else {
panic!("expected two query lines:\n{text}");
};
let plan = session.explain(query).await.unwrap();
let planned = plan
.lines()
.filter(|line| line.trim_start().starts_with("Gpu"))
.count();
assert!(planned > 0, "nothing was lowered:\n{plan}");
assert_eq!(
field(first, "gpu_operators"),
planned as u64,
"{first}\n{plan}"
);
for name in ["gpu_operators", "fallback_batches", "rows"] {
assert_eq!(
field(second, name),
field(first, name),
"{name} of the second run:\n{first}\n{second}"
);
}
}
#[test]
fn an_embedded_session_checks_parquet_files_and_bounds_reads() {
let session = OxideSession::local().unwrap();
let state = session.ctx().state();
assert!(
state
.physical_optimizers()
.iter()
.any(|rule| rule.name() == "ParquetScanGuard"),
"no ParquetScanGuard in the physical optimizer rules"
);
let store = session
.ctx()
.runtime_env()
.object_store(datafusion::execution::object_store::ObjectStoreUrl::local_filesystem())
.unwrap();
assert!(
store.to_string().starts_with("BoundedReadStore("),
"{store}"
);
}
#[test]
fn the_default_options_are_the_plain_constructor() {
let plain = OxideSession::local().unwrap();
let built = OxideSession::local_with_options(&SessionOptions::default()).unwrap();
assert_eq!(plain.mode(), built.mode());
assert_eq!(batch_size(&plain), batch_size(&built));
}
}