1use std::sync::Arc;
4use std::time::Instant;
5
6use ballista::prelude::SessionContextExt;
7use datafusion::arrow::array::RecordBatch;
8use datafusion::arrow::datatypes::SchemaRef;
9use datafusion::dataframe::DataFrame;
10use datafusion::execution::SessionStateBuilder;
11use datafusion::physical_plan::{ExecutionPlan, collect, displayable};
12use datafusion::prelude::{SessionConfig, SessionContext};
13use oxidelake_compute::{
14 GpuAggregateExec, GpuFilterExec, GpuHashJoinExec, GpuVectorDistanceExec, local_backend,
15 oxide_udfs,
16};
17use oxidelake_core::telemetry::{OperatorSnapshot, TelemetryHub, TierCapacity};
18use oxidelake_core::{BackendKind, EngineError};
19use oxidelake_planner::{HardwarePlacementRule, physical_optimizer_rules};
20use oxidelake_storage::{
21 ParquetScanGuard, default_object_store, register_local_store, register_parquet_table,
22 with_gpu_batch_size, with_pruning,
23};
24
25use crate::cluster;
26
27#[derive(Debug, Clone, PartialEq, Eq)]
29#[non_exhaustive]
30pub enum SessionMode {
31 Embedded {
33 target: BackendKind,
35 },
36 Cluster {
38 scheduler_url: String,
40 },
41}
42
43fn local_capacity() -> TierCapacity {
57 let Ok(backend) = local_backend() else {
58 return TierCapacity::default();
59 };
60 let Ok(info) = backend.memory_info() else {
61 return TierCapacity::default();
62 };
63 if backend.kind().is_gpu() {
64 TierCapacity {
65 device_bytes: Some(info.total_bytes),
66 host_bytes: None,
67 spill_on_query_path: false,
68 }
69 } else {
70 TierCapacity {
71 device_bytes: None,
72 host_bytes: Some(info.total_bytes),
73 spill_on_query_path: false,
74 }
75 }
76}
77
78impl std::fmt::Display for SessionMode {
79 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
82 match self {
83 SessionMode::Embedded { target } => write!(f, "embedded/{target}"),
84 SessionMode::Cluster { scheduler_url } => write!(f, "cluster/{scheduler_url}"),
85 }
86 }
87}
88
89#[derive(Debug, Clone, Default, PartialEq, Eq)]
96#[non_exhaustive]
97pub struct SessionOptions {
98 pub target: Option<BackendKind>,
100 pub batch_size: Option<usize>,
105}
106
107impl SessionOptions {
108 pub fn new() -> Self {
110 Self::default()
111 }
112
113 pub fn with_target(mut self, target: BackendKind) -> Self {
115 self.target = Some(target);
116 self
117 }
118
119 pub fn with_batch_size(mut self, rows: usize) -> Self {
121 self.batch_size = Some(rows);
122 self
123 }
124
125 fn apply(
132 &self,
133 config: SessionConfig,
134 target: Option<BackendKind>,
135 ) -> Result<SessionConfig, EngineError> {
136 match self.batch_size {
137 Some(0) => Err(EngineError::plan(
138 "batch size must be at least 1 row (0 would make every query return nothing)",
139 )),
140 Some(rows) => Ok(config.with_batch_size(rows)),
141 None if target.is_some_and(BackendKind::is_gpu) => Ok(with_gpu_batch_size(config)),
142 None => Ok(config),
143 }
144 }
145}
146
147pub struct OxideSession {
149 ctx: SessionContext,
150 mode: SessionMode,
151 telemetry: Arc<TelemetryHub>,
152}
153
154impl OxideSession {
155 pub fn local() -> Result<Self, EngineError> {
159 Self::local_with_options(&SessionOptions::new())
160 }
161
162 pub fn local_with_target(target: BackendKind) -> Result<Self, EngineError> {
168 Self::local_with_options(&SessionOptions::new().with_target(target))
169 }
170
171 pub fn local_with_options(options: &SessionOptions) -> Result<Self, EngineError> {
173 let target = match options.target {
174 Some(target) => target,
175 None => local_backend()?.kind(),
176 };
177 let telemetry = TelemetryHub::new();
178 telemetry.set_capacity(local_capacity());
179 let rule = HardwarePlacementRule::new(target).with_telemetry(Arc::clone(&telemetry));
180 let config = options.apply(with_pruning(SessionConfig::new()), Some(target))?;
181 let state = SessionStateBuilder::new()
182 .with_default_features()
183 .with_config(config)
184 .with_physical_optimizer_rules(physical_optimizer_rules(rule))
185 .with_physical_optimizer_rule(Arc::new(ParquetScanGuard::new()))
186 .build();
187 let ctx = SessionContext::new_with_state(state);
188 register_local_store(&ctx, default_object_store());
189 for udf in oxide_udfs() {
190 ctx.register_udf(udf.as_ref().clone());
191 }
192 Ok(Self {
193 ctx,
194 mode: SessionMode::Embedded { target },
195 telemetry,
196 })
197 }
198
199 pub async fn connect(scheduler_url: &str) -> Result<Self, EngineError> {
204 Self::connect_with_options(scheduler_url, &SessionOptions::new()).await
205 }
206
207 pub async fn connect_with_options(
211 scheduler_url: &str,
212 options: &SessionOptions,
213 ) -> Result<Self, EngineError> {
214 let config = cluster::session_config().with_option_extension(cluster::ClientOptions {
220 explicit_batch_size: options.batch_size.is_some(),
221 });
222 let config = options.apply(config, None)?;
223 let state = SessionStateBuilder::new()
224 .with_default_features()
225 .with_config(config)
226 .build();
227 let ctx = SessionContext::remote_with_state(scheduler_url, state).await?;
228 for udf in oxide_udfs() {
229 ctx.register_udf(udf.as_ref().clone());
230 }
231 Ok(Self {
232 ctx,
233 mode: SessionMode::Cluster {
234 scheduler_url: scheduler_url.to_owned(),
235 },
236 telemetry: TelemetryHub::new(),
237 })
238 }
239
240 pub fn mode(&self) -> &SessionMode {
242 &self.mode
243 }
244
245 pub fn ctx(&self) -> &SessionContext {
247 &self.ctx
248 }
249
250 pub fn telemetry(&self) -> &Arc<TelemetryHub> {
259 &self.telemetry
260 }
261
262 pub async fn sql(&self, query: &str) -> Result<DataFrame, EngineError> {
267 Ok(self.ctx.sql(query).await?)
268 }
269
270 pub async fn collect(&self, query: &str) -> Result<(SchemaRef, Vec<RecordBatch>), EngineError> {
283 let started = Instant::now();
284 let frame = self.sql(query).await?;
285 let schema = SchemaRef::from(frame.schema().clone());
286 let task_ctx = Arc::new(frame.task_ctx());
289 let plan = frame.create_physical_plan().await?;
290 let batches = collect(Arc::clone(&plan), task_ctx).await?;
291 let rows: usize = batches.iter().map(RecordBatch::num_rows).sum();
292 let operators = query_operators(&plan);
293 let fallback_batches: u64 = operators.iter().map(|o| o.fallback_batches).sum();
294 let accelerated = operators.iter().filter(|o| o.backend.is_gpu()).count();
295 tracing::info!(
296 mode = %self.mode,
297 rows,
298 batches = batches.len(),
299 elapsed_ms = started.elapsed().as_secs_f64() * 1_000.0,
300 gpu_operators = accelerated,
301 fallback_batches,
302 "query finished"
303 );
304 Ok((schema, batches))
305 }
306
307 pub async fn register_parquet(&self, name: &str, path: &str) -> Result<(), EngineError> {
309 register_parquet_table(&self.ctx, name, path).await
310 }
311
312 pub async fn explain(&self, query: &str) -> Result<String, EngineError> {
316 self.telemetry.clear_skips();
317 let plan = self.ctx.sql(query).await?.create_physical_plan().await?;
318 let mut text = displayable(plan.as_ref()).indent(true).to_string();
319 text.push_str(&self.placement_notes());
320 Ok(text)
321 }
322
323 fn placement_notes(&self) -> String {
333 let skips = self.telemetry.skips();
334 if skips.is_empty() {
335 return String::new();
336 }
337 let mut out = String::from("\nplacement notes (target ");
338 match &self.mode {
339 SessionMode::Embedded { target } => out.push_str(target.as_str()),
340 SessionMode::Cluster { .. } => out.push_str("on the scheduler"),
341 }
342 out.push_str("):\n");
343 for skip in skips {
344 out.push_str(&format!(" {}: {}\n", skip.node, skip.reason));
345 }
346 out
347 }
348}
349
350fn query_operators(plan: &Arc<dyn ExecutionPlan>) -> Vec<OperatorSnapshot> {
358 let mut found = Vec::new();
359 let mut pending = vec![Arc::clone(plan)];
360 while let Some(node) = pending.pop() {
361 let stats = if let Some(exec) = node.downcast_ref::<GpuFilterExec>() {
362 exec.operator_stats()
363 } else if let Some(exec) = node.downcast_ref::<GpuHashJoinExec>() {
364 exec.operator_stats()
365 } else if let Some(exec) = node.downcast_ref::<GpuAggregateExec>() {
366 exec.operator_stats()
367 } else if let Some(exec) = node.downcast_ref::<GpuVectorDistanceExec>() {
368 exec.operator_stats()
369 } else {
370 None
371 };
372 found.extend(stats);
373 pending.extend(node.children().into_iter().cloned());
374 }
375 found
376}
377
378impl std::fmt::Debug for OxideSession {
379 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
380 f.debug_struct("OxideSession")
381 .field("mode", &self.mode)
382 .field("session_id", &self.ctx.session_id())
383 .finish_non_exhaustive()
384 }
385}
386
387#[cfg(test)]
388#[allow(clippy::unwrap_used, clippy::expect_used)]
389mod tests {
390 use super::*;
391
392 #[tokio::test]
393 async fn embedded_session_runs_sql_and_reports_mode() {
394 let session = OxideSession::local().unwrap();
395 assert!(matches!(session.mode(), SessionMode::Embedded { .. }));
396 let batches = session
397 .sql("SELECT 1 + 1 AS two")
398 .await
399 .unwrap()
400 .collect()
401 .await
402 .unwrap();
403 assert_eq!(batches.len(), 1);
404 assert_eq!(batches[0].num_rows(), 1);
405 let text = session.explain("SELECT 1 + 1 AS two").await.unwrap();
406 assert!(text.contains("ProjectionExec"), "{text}");
407 }
408
409 fn batch_size(session: &OxideSession) -> usize {
410 session.ctx().state().config().batch_size()
411 }
412
413 #[test]
417 fn the_batch_size_option_reaches_the_session_configuration() {
418 let session =
419 OxideSession::local_with_options(&SessionOptions::new().with_batch_size(7)).unwrap();
420 assert_eq!(batch_size(&session), 7);
421 }
422
423 #[test]
427 fn the_target_picks_the_default_batch_size_and_the_option_overrides_it() {
428 let gpu = OxideSession::local_with_target(BackendKind::Cuda).unwrap();
429 assert_eq!(batch_size(&gpu), oxidelake_storage::GPU_BATCH_SIZE);
430
431 let cpu = OxideSession::local_with_target(BackendKind::CpuSimd).unwrap();
432 assert_eq!(batch_size(&cpu), SessionConfig::new().batch_size());
433
434 let forced = OxideSession::local_with_options(
435 &SessionOptions::new()
436 .with_target(BackendKind::Cuda)
437 .with_batch_size(1_024),
438 )
439 .unwrap();
440 assert_eq!(batch_size(&forced), 1_024);
441 }
442
443 #[test]
444 fn a_zero_batch_size_is_refused() {
445 let err = OxideSession::local_with_options(&SessionOptions::new().with_batch_size(0))
446 .unwrap_err()
447 .to_string();
448 assert!(err.contains("at least 1 row"), "{err}");
449 }
450
451 #[test]
457 fn a_session_reports_real_capacities_and_an_honest_spill_flag() {
458 let session = OxideSession::local().unwrap();
459 let capacity = session.telemetry().capacity();
460 assert!(
461 !capacity.spill_on_query_path,
462 "no operator registers with a SpillManager in 0.2"
463 );
464 let SessionMode::Embedded { target } = session.mode() else {
465 panic!("local() is embedded");
466 };
467 let reported = if target.is_gpu() {
468 capacity.device_bytes
469 } else {
470 capacity.host_bytes
471 };
472 assert!(
475 reported.is_some_and(|bytes| bytes > 0),
476 "{target} reported no memory: {capacity:?}"
477 );
478 let absent = if target.is_gpu() {
481 capacity.host_bytes
482 } else {
483 capacity.device_bytes
484 };
485 assert_eq!(absent, None);
486 }
487
488 #[derive(Clone, Default)]
490 struct Captured(Arc<std::sync::Mutex<Vec<u8>>>);
491
492 impl std::io::Write for Captured {
493 fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
494 self.0.lock().unwrap().extend_from_slice(buf);
495 Ok(buf.len())
496 }
497
498 fn flush(&mut self) -> std::io::Result<()> {
499 Ok(())
500 }
501 }
502
503 fn field(line: &str, name: &str) -> u64 {
505 line.split_whitespace()
506 .find_map(|part| part.strip_prefix(name)?.strip_prefix('='))
507 .unwrap_or_else(|| panic!("no {name} in {line}"))
508 .parse()
509 .unwrap()
510 }
511
512 #[tokio::test]
518 async fn the_query_line_counts_only_its_own_operators() {
519 let captured = Captured::default();
520 let writer = captured.clone();
521 let subscriber = tracing_subscriber::fmt()
522 .with_writer(move || writer.clone())
523 .with_ansi(false)
524 .with_max_level(tracing::Level::INFO)
525 .finish();
526 let _default = tracing::subscriber::set_default(subscriber);
527
528 let session = OxideSession::local_with_target(BackendKind::Cuda).unwrap();
531 session
532 .ctx()
533 .sql("CREATE TABLE a AS SELECT value AS k, value * 3 AS w FROM generate_series(0, 99)")
534 .await
535 .unwrap()
536 .collect()
537 .await
538 .unwrap();
539 let query = "SELECT x.k, y.w FROM (SELECT * FROM a WHERE k > 10) x \
540 JOIN (SELECT * FROM a WHERE w > 30) y ON x.k = y.k";
541 for _ in 0..2 {
542 session.collect(query).await.unwrap();
543 }
544
545 let text = String::from_utf8(captured.0.lock().unwrap().clone()).unwrap();
546 let lines: Vec<&str> = text
547 .lines()
548 .filter(|line| line.contains("query finished"))
549 .collect();
550 let [first, second] = lines[..] else {
551 panic!("expected two query lines:\n{text}");
552 };
553 let plan = session.explain(query).await.unwrap();
555 let planned = plan
556 .lines()
557 .filter(|line| line.trim_start().starts_with("Gpu"))
558 .count();
559 assert!(planned > 0, "nothing was lowered:\n{plan}");
560 assert_eq!(
561 field(first, "gpu_operators"),
562 planned as u64,
563 "{first}\n{plan}"
564 );
565 for name in ["gpu_operators", "fallback_batches", "rows"] {
566 assert_eq!(
567 field(second, name),
568 field(first, name),
569 "{name} of the second run:\n{first}\n{second}"
570 );
571 }
572 }
573
574 #[test]
579 fn an_embedded_session_checks_parquet_files_and_bounds_reads() {
580 let session = OxideSession::local().unwrap();
581 let state = session.ctx().state();
582 assert!(
583 state
584 .physical_optimizers()
585 .iter()
586 .any(|rule| rule.name() == "ParquetScanGuard"),
587 "no ParquetScanGuard in the physical optimizer rules"
588 );
589 let store = session
590 .ctx()
591 .runtime_env()
592 .object_store(datafusion::execution::object_store::ObjectStoreUrl::local_filesystem())
593 .unwrap();
594 assert!(
595 store.to_string().starts_with("BoundedReadStore("),
596 "{store}"
597 );
598 }
599
600 #[test]
603 fn the_default_options_are_the_plain_constructor() {
604 let plain = OxideSession::local().unwrap();
605 let built = OxideSession::local_with_options(&SessionOptions::default()).unwrap();
606 assert_eq!(plain.mode(), built.mode());
607 assert_eq!(batch_size(&plain), batch_size(&built));
608 }
609}