Skip to main content

oxidelake_runtime/
session.rs

1//! The user-facing session: one query API over embedded and cluster execution.
2
3use std::sync::Arc;
4use std::time::Instant;
5
6use ballista::prelude::SessionContextExt;
7use datafusion::arrow::array::RecordBatch;
8use datafusion::arrow::datatypes::SchemaRef;
9use datafusion::dataframe::DataFrame;
10use datafusion::execution::SessionStateBuilder;
11use datafusion::physical_plan::{ExecutionPlan, collect, displayable};
12use datafusion::prelude::{SessionConfig, SessionContext};
13use oxidelake_compute::{
14    GpuAggregateExec, GpuFilterExec, GpuHashJoinExec, GpuVectorDistanceExec, local_backend,
15    oxide_udfs,
16};
17use oxidelake_core::telemetry::{OperatorSnapshot, TelemetryHub, TierCapacity};
18use oxidelake_core::{BackendKind, EngineError};
19use oxidelake_planner::{HardwarePlacementRule, physical_optimizer_rules};
20use oxidelake_storage::{
21    ParquetScanGuard, default_object_store, register_local_store, register_parquet_table,
22    with_gpu_batch_size, with_pruning,
23};
24
25use crate::cluster;
26
27/// Where a session executes.
28#[derive(Debug, Clone, PartialEq, Eq)]
29#[non_exhaustive]
30pub enum SessionMode {
31    /// In-process DataFusion with the placement rule targeting the local backend.
32    Embedded {
33        /// The backend detected on this machine.
34        target: BackendKind,
35    },
36    /// A Ballista cluster reached through its scheduler URL (`df://host:port`).
37    Cluster {
38        /// The scheduler URL.
39        scheduler_url: String,
40    },
41}
42
43/// What the local backend says its memory tiers hold (#25).
44///
45/// A GPU backend's `memory_info` describes the device; the CPU backend's
46/// describes host RAM. Neither is a guess, and a backend that cannot answer
47/// leaves the field `None` rather than contributing a number the dashboard
48/// would draw a gauge against.
49///
50/// `spill_on_query_path` is `false` and stays false until an operator
51/// registers a batch with a `SpillManager`. Nothing does in 0.2: the hash
52/// join's build side is probed by every batch, so demoting it would trade a
53/// bounded memory claim for an unbounded one in time, and a spilling
54/// aggregate needs the streaming aggregate first. Saying so is the honest
55/// half of this release; see `docs/architecture.md`.
56fn local_capacity() -> TierCapacity {
57    let Ok(backend) = local_backend() else {
58        return TierCapacity::default();
59    };
60    let Ok(info) = backend.memory_info() else {
61        return TierCapacity::default();
62    };
63    if backend.kind().is_gpu() {
64        TierCapacity {
65            device_bytes: Some(info.total_bytes),
66            host_bytes: None,
67            spill_on_query_path: false,
68        }
69    } else {
70        TierCapacity {
71            device_bytes: None,
72            host_bytes: Some(info.total_bytes),
73            spill_on_query_path: false,
74        }
75    }
76}
77
78impl std::fmt::Display for SessionMode {
79    /// `embedded/cuda` or `cluster/df://host:port` — one field in a log line
80    /// rather than two, because the second is only meaningful given the first.
81    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
82        match self {
83            SessionMode::Embedded { target } => write!(f, "embedded/{target}"),
84            SessionMode::Cluster { scheduler_url } => write!(f, "cluster/{scheduler_url}"),
85        }
86    }
87}
88
89/// Knobs a session is built with.
90///
91/// Every field has a default that matches the no-argument constructors, so
92/// `SessionOptions::default()` and [`OxideSession::local`] agree. The struct
93/// is `#[non_exhaustive]`: a knob added later is a minor release, not a
94/// broken build for anyone who used the builder methods.
95#[derive(Debug, Clone, Default, PartialEq, Eq)]
96#[non_exhaustive]
97pub struct SessionOptions {
98    /// Plan for this backend instead of the detected one (embedded only).
99    pub target: Option<BackendKind>,
100    /// Rows per record batch. `None` leaves DataFusion's default (8192), or
101    /// [`oxidelake_storage::GPU_BATCH_SIZE`] when the placement target is a GPU
102    /// (on a cluster, the scheduler's target). A size set here is kept on a
103    /// GPU target too, a cluster's included.
104    pub batch_size: Option<usize>,
105}
106
107impl SessionOptions {
108    /// Defaults: detected backend, default batch size.
109    pub fn new() -> Self {
110        Self::default()
111    }
112
113    /// Plans for `target` rather than the detected backend.
114    pub fn with_target(mut self, target: BackendKind) -> Self {
115        self.target = Some(target);
116        self
117    }
118
119    /// Sets the rows per record batch, overriding the CPU and GPU defaults.
120    pub fn with_batch_size(mut self, rows: usize) -> Self {
121        self.batch_size = Some(rows);
122        self
123    }
124
125    /// Applies the batch size to `config`, or the GPU default when the
126    /// placement target is a device and no size was asked for.
127    ///
128    /// A batch size of zero is rejected here rather than passed on: DataFusion
129    /// would take it and then produce no rows at all, which reads as an empty
130    /// result rather than as a bad flag.
131    fn apply(
132        &self,
133        config: SessionConfig,
134        target: Option<BackendKind>,
135    ) -> Result<SessionConfig, EngineError> {
136        match self.batch_size {
137            Some(0) => Err(EngineError::plan(
138                "batch size must be at least 1 row (0 would make every query return nothing)",
139            )),
140            Some(rows) => Ok(config.with_batch_size(rows)),
141            None if target.is_some_and(BackendKind::is_gpu) => Ok(with_gpu_batch_size(config)),
142            None => Ok(config),
143        }
144    }
145}
146
147/// An OxideLake session.
148pub struct OxideSession {
149    ctx: SessionContext,
150    mode: SessionMode,
151    telemetry: Arc<TelemetryHub>,
152}
153
154impl OxideSession {
155    /// Creates an embedded session: Parquet pruning on, the local object store
156    /// registered, the SQL UDFs, and the placement rule targeting the detected
157    /// backend.
158    pub fn local() -> Result<Self, EngineError> {
159        Self::local_with_options(&SessionOptions::new())
160    }
161
162    /// An embedded session whose placement rule targets `target` instead of
163    /// the detected backend. Placement is a planning decision: operators still
164    /// select the real local backend at `execute()` time and fall back to the
165    /// CPU reference, so planning for an absent GPU is safe — it is exactly
166    /// what cluster executors do with the scheduler's plans.
167    pub fn local_with_target(target: BackendKind) -> Result<Self, EngineError> {
168        Self::local_with_options(&SessionOptions::new().with_target(target))
169    }
170
171    /// An embedded session built with `options`.
172    pub fn local_with_options(options: &SessionOptions) -> Result<Self, EngineError> {
173        let target = match options.target {
174            Some(target) => target,
175            None => local_backend()?.kind(),
176        };
177        let telemetry = TelemetryHub::new();
178        telemetry.set_capacity(local_capacity());
179        let rule = HardwarePlacementRule::new(target).with_telemetry(Arc::clone(&telemetry));
180        let config = options.apply(with_pruning(SessionConfig::new()), Some(target))?;
181        let state = SessionStateBuilder::new()
182            .with_default_features()
183            .with_config(config)
184            .with_physical_optimizer_rules(physical_optimizer_rules(rule))
185            .with_physical_optimizer_rule(Arc::new(ParquetScanGuard::new()))
186            .build();
187        let ctx = SessionContext::new_with_state(state);
188        register_local_store(&ctx, default_object_store());
189        for udf in oxide_udfs() {
190            ctx.register_udf(udf.as_ref().clone());
191        }
192        Ok(Self {
193            ctx,
194            mode: SessionMode::Embedded { target },
195            telemetry,
196        })
197    }
198
199    /// Connects to a Ballista scheduler (`df://host:port`). The session carries
200    /// OxideLake's plan codec so `Gpu*Exec` nodes survive the trip to executors,
201    /// and the SQL UDFs so queries plan client-side; placement itself happens
202    /// on the scheduler.
203    pub async fn connect(scheduler_url: &str) -> Result<Self, EngineError> {
204        Self::connect_with_options(scheduler_url, &SessionOptions::new()).await
205    }
206
207    /// Connects to a Ballista scheduler with `options`. `target` is ignored:
208    /// on a cluster the scheduler's `OXIDE_CLUSTER_BACKEND` decides placement,
209    /// so a client-side target would be a knob that quietly does nothing.
210    pub async fn connect_with_options(
211        scheduler_url: &str,
212        options: &SessionOptions,
213    ) -> Result<Self, EngineError> {
214        // The scheduler picks the batch size a GPU cluster runs with, so it
215        // has to be told when this one is a choice rather than a default (#93).
216        // The rest is what every cluster process runs with, adaptive planning
217        // off included, so this client's own settings say what the scheduler
218        // will do with its query (#134).
219        let config = cluster::session_config().with_option_extension(cluster::ClientOptions {
220            explicit_batch_size: options.batch_size.is_some(),
221        });
222        let config = options.apply(config, None)?;
223        let state = SessionStateBuilder::new()
224            .with_default_features()
225            .with_config(config)
226            .build();
227        let ctx = SessionContext::remote_with_state(scheduler_url, state).await?;
228        for udf in oxide_udfs() {
229            ctx.register_udf(udf.as_ref().clone());
230        }
231        Ok(Self {
232            ctx,
233            mode: SessionMode::Cluster {
234                scheduler_url: scheduler_url.to_owned(),
235            },
236            telemetry: TelemetryHub::new(),
237        })
238    }
239
240    /// The execution mode.
241    pub fn mode(&self) -> &SessionMode {
242        &self.mode
243    }
244
245    /// The underlying DataFusion context.
246    pub fn ctx(&self) -> &SessionContext {
247        &self.ctx
248    }
249
250    /// The telemetry hub for this session.
251    ///
252    /// **Embedded sessions only.** A cluster session's operators run on
253    /// executors, in other processes; this hub is created for the shape of
254    /// the type and stays empty, so a snapshot of it is not "no work
255    /// happened" but "the work happened somewhere else" (#33). Each worker
256    /// reports into its own process-wide hub, which `oxide-worker
257    /// --metrics-port` exposes as Prometheus text.
258    pub fn telemetry(&self) -> &Arc<TelemetryHub> {
259        &self.telemetry
260    }
261
262    /// Plans a SQL statement into a lazily executed [`DataFrame`].
263    ///
264    /// Nothing runs here, so nothing is logged here: see [`Self::collect`]
265    /// for the one-line-per-query record.
266    pub async fn sql(&self, query: &str) -> Result<DataFrame, EngineError> {
267        Ok(self.ctx.sql(query).await?)
268    }
269
270    /// Runs `query` to completion and logs one `INFO` line describing it
271    /// (#33): the backend it was planned for, the rows it produced, how long
272    /// it took, and how many batches fell back to the CPU reference.
273    ///
274    /// This is the line an operator reads to answer "is the GPU being used
275    /// and how long did the query take" without attaching a dashboard. It is
276    /// on `collect` rather than on [`Self::sql`] because a `DataFrame` has
277    /// not run yet: a line logged at planning time could only report the
278    /// plan, and the interesting half is what the plan then did.
279    /// The schema comes back beside the batches because an empty result has
280    /// no batch to take it from, and a caller rendering CSV still has to
281    /// print the header.
282    pub async fn collect(&self, query: &str) -> Result<(SchemaRef, Vec<RecordBatch>), EngineError> {
283        let started = Instant::now();
284        let frame = self.sql(query).await?;
285        let schema = SchemaRef::from(frame.schema().clone());
286        // `DataFrame::collect` in its two steps, so the plan outlives the
287        // run: its `Gpu*Exec` nodes hold this query's counters.
288        let task_ctx = Arc::new(frame.task_ctx());
289        let plan = frame.create_physical_plan().await?;
290        let batches = collect(Arc::clone(&plan), task_ctx).await?;
291        let rows: usize = batches.iter().map(RecordBatch::num_rows).sum();
292        let operators = query_operators(&plan);
293        let fallback_batches: u64 = operators.iter().map(|o| o.fallback_batches).sum();
294        let accelerated = operators.iter().filter(|o| o.backend.is_gpu()).count();
295        tracing::info!(
296            mode = %self.mode,
297            rows,
298            batches = batches.len(),
299            elapsed_ms = started.elapsed().as_secs_f64() * 1_000.0,
300            gpu_operators = accelerated,
301            fallback_batches,
302            "query finished"
303        );
304        Ok((schema, batches))
305    }
306
307    /// Registers a Parquet file or directory as `name`.
308    pub async fn register_parquet(&self, name: &str, path: &str) -> Result<(), EngineError> {
309        register_parquet_table(&self.ctx, name, path).await
310    }
311
312    /// The indented physical plan for `query`, with placement tags in embedded
313    /// mode. In cluster mode this is the client-side plan (`DistributedQueryExec`);
314    /// the scheduler's plan is what carries the tags there.
315    pub async fn explain(&self, query: &str) -> Result<String, EngineError> {
316        self.telemetry.clear_skips();
317        let plan = self.ctx.sql(query).await?.create_physical_plan().await?;
318        let mut text = displayable(plan.as_ref()).indent(true).to_string();
319        text.push_str(&self.placement_notes());
320        Ok(text)
321    }
322
323    /// The placement rule's reasons for leaving nodes on the CPU (#32).
324    ///
325    /// A node the rule skipped is an ordinary DataFusion operator in the
326    /// plan above, indistinguishable from one that was never eligible. The
327    /// reason used to live only in a `debug!` line, which is no use to
328    /// someone reading a plan, so it is printed under it.
329    ///
330    /// Empty in cluster mode: the rule runs on the scheduler there, and this
331    /// process only planned the `DistributedQueryExec` wrapper.
332    fn placement_notes(&self) -> String {
333        let skips = self.telemetry.skips();
334        if skips.is_empty() {
335            return String::new();
336        }
337        let mut out = String::from("\nplacement notes (target ");
338        match &self.mode {
339            SessionMode::Embedded { target } => out.push_str(target.as_str()),
340            SessionMode::Cluster { .. } => out.push_str("on the scheduler"),
341        }
342        out.push_str("):\n");
343        for skip in skips {
344            out.push_str(&format!("  {}: {}\n", skip.node, skip.reason));
345        }
346        out
347    }
348}
349
350/// The counters of every `Gpu*Exec` in `plan` that executed, read from the
351/// nodes themselves: one query's operators and no other's (#102).
352///
353/// The session's hub holds every operator the session has run, so the
354/// `query finished` line, which summed it, logged twice the first run's
355/// numbers for the second run of the same query. Nor is a slice of the hub
356/// one query's: two queries running at once register into it interleaved.
357fn query_operators(plan: &Arc<dyn ExecutionPlan>) -> Vec<OperatorSnapshot> {
358    let mut found = Vec::new();
359    let mut pending = vec![Arc::clone(plan)];
360    while let Some(node) = pending.pop() {
361        let stats = if let Some(exec) = node.downcast_ref::<GpuFilterExec>() {
362            exec.operator_stats()
363        } else if let Some(exec) = node.downcast_ref::<GpuHashJoinExec>() {
364            exec.operator_stats()
365        } else if let Some(exec) = node.downcast_ref::<GpuAggregateExec>() {
366            exec.operator_stats()
367        } else if let Some(exec) = node.downcast_ref::<GpuVectorDistanceExec>() {
368            exec.operator_stats()
369        } else {
370            None
371        };
372        found.extend(stats);
373        pending.extend(node.children().into_iter().cloned());
374    }
375    found
376}
377
378impl std::fmt::Debug for OxideSession {
379    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
380        f.debug_struct("OxideSession")
381            .field("mode", &self.mode)
382            .field("session_id", &self.ctx.session_id())
383            .finish_non_exhaustive()
384    }
385}
386
387#[cfg(test)]
388#[allow(clippy::unwrap_used, clippy::expect_used)]
389mod tests {
390    use super::*;
391
392    #[tokio::test]
393    async fn embedded_session_runs_sql_and_reports_mode() {
394        let session = OxideSession::local().unwrap();
395        assert!(matches!(session.mode(), SessionMode::Embedded { .. }));
396        let batches = session
397            .sql("SELECT 1 + 1 AS two")
398            .await
399            .unwrap()
400            .collect()
401            .await
402            .unwrap();
403        assert_eq!(batches.len(), 1);
404        assert_eq!(batches[0].num_rows(), 1);
405        let text = session.explain("SELECT 1 + 1 AS two").await.unwrap();
406        assert!(text.contains("ProjectionExec"), "{text}");
407    }
408
409    fn batch_size(session: &OxideSession) -> usize {
410        session.ctx().state().config().batch_size()
411    }
412
413    /// `--batch-size` has to reach DataFusion's configuration: a flag that is
414    /// parsed and then dropped would pass every test that only compares query
415    /// results, because batch boundaries do not change them.
416    #[test]
417    fn the_batch_size_option_reaches_the_session_configuration() {
418        let session =
419            OxideSession::local_with_options(&SessionOptions::new().with_batch_size(7)).unwrap();
420        assert_eq!(batch_size(&session), 7);
421    }
422
423    /// Without an explicit size, a GPU-targeted session keeps the larger
424    /// default that amortises the host↔device round trip, and a CPU one keeps
425    /// DataFusion's. An explicit size overrides both.
426    #[test]
427    fn the_target_picks_the_default_batch_size_and_the_option_overrides_it() {
428        let gpu = OxideSession::local_with_target(BackendKind::Cuda).unwrap();
429        assert_eq!(batch_size(&gpu), oxidelake_storage::GPU_BATCH_SIZE);
430
431        let cpu = OxideSession::local_with_target(BackendKind::CpuSimd).unwrap();
432        assert_eq!(batch_size(&cpu), SessionConfig::new().batch_size());
433
434        let forced = OxideSession::local_with_options(
435            &SessionOptions::new()
436                .with_target(BackendKind::Cuda)
437                .with_batch_size(1_024),
438        )
439        .unwrap();
440        assert_eq!(batch_size(&forced), 1_024);
441    }
442
443    #[test]
444    fn a_zero_batch_size_is_refused() {
445        let err = OxideSession::local_with_options(&SessionOptions::new().with_batch_size(0))
446            .unwrap_err()
447            .to_string();
448        assert!(err.contains("at least 1 row"), "{err}");
449    }
450
451    /// The dashboard's memory gauges are drawn against what the backend says,
452    /// and the spill flag stays false until a query path registers a batch —
453    /// which nothing does in 0.2 (#25). Both halves matter: a capacity that
454    /// was a constant and a gauge that could never move made the dashboard
455    /// describe a bounded memory model the engine does not have.
456    #[test]
457    fn a_session_reports_real_capacities_and_an_honest_spill_flag() {
458        let session = OxideSession::local().unwrap();
459        let capacity = session.telemetry().capacity();
460        assert!(
461            !capacity.spill_on_query_path,
462            "no operator registers with a SpillManager in 0.2"
463        );
464        let SessionMode::Embedded { target } = session.mode() else {
465            panic!("local() is embedded");
466        };
467        let reported = if target.is_gpu() {
468            capacity.device_bytes
469        } else {
470            capacity.host_bytes
471        };
472        // The number comes from the machine, so the assertion is on its
473        // shape: present, and not the 4 GiB constant the panel used to draw.
474        assert!(
475            reported.is_some_and(|bytes| bytes > 0),
476            "{target} reported no memory: {capacity:?}"
477        );
478        // The other tier has no backend to ask, and says so rather than
479        // contributing a number.
480        let absent = if target.is_gpu() {
481            capacity.host_bytes
482        } else {
483            capacity.device_bytes
484        };
485        assert_eq!(absent, None);
486    }
487
488    /// Collects what a `tracing` subscriber writes.
489    #[derive(Clone, Default)]
490    struct Captured(Arc<std::sync::Mutex<Vec<u8>>>);
491
492    impl std::io::Write for Captured {
493        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
494            self.0.lock().unwrap().extend_from_slice(buf);
495            Ok(buf.len())
496        }
497
498        fn flush(&mut self) -> std::io::Result<()> {
499            Ok(())
500        }
501    }
502
503    /// `name=value` out of one formatted log line.
504    fn field(line: &str, name: &str) -> u64 {
505        line.split_whitespace()
506            .find_map(|part| part.strip_prefix(name)?.strip_prefix('='))
507            .unwrap_or_else(|| panic!("no {name} in {line}"))
508            .parse()
509            .unwrap()
510    }
511
512    /// The `query finished` line describes its own query (#102).
513    ///
514    /// It used to sum every operator the session's hub had ever registered,
515    /// so the second run of one query logged twice what the first did:
516    /// `gpu_operators=4 fallback_batches=4` after `2` and `2`.
517    #[tokio::test]
518    async fn the_query_line_counts_only_its_own_operators() {
519        let captured = Captured::default();
520        let writer = captured.clone();
521        let subscriber = tracing_subscriber::fmt()
522            .with_writer(move || writer.clone())
523            .with_ansi(false)
524            .with_max_level(tracing::Level::INFO)
525            .finish();
526        let _default = tracing::subscriber::set_default(subscriber);
527
528        // Two filters, each lowered to a `GpuFilterExec` on a GPU target and
529        // run on whatever backend this machine has.
530        let session = OxideSession::local_with_target(BackendKind::Cuda).unwrap();
531        session
532            .ctx()
533            .sql("CREATE TABLE a AS SELECT value AS k, value * 3 AS w FROM generate_series(0, 99)")
534            .await
535            .unwrap()
536            .collect()
537            .await
538            .unwrap();
539        let query = "SELECT x.k, y.w FROM (SELECT * FROM a WHERE k > 10) x \
540                     JOIN (SELECT * FROM a WHERE w > 30) y ON x.k = y.k";
541        for _ in 0..2 {
542            session.collect(query).await.unwrap();
543        }
544
545        let text = String::from_utf8(captured.0.lock().unwrap().clone()).unwrap();
546        let lines: Vec<&str> = text
547            .lines()
548            .filter(|line| line.contains("query finished"))
549            .collect();
550        let [first, second] = lines[..] else {
551            panic!("expected two query lines:\n{text}");
552        };
553        // The query's own operators are the `Gpu*Exec` nodes of its plan.
554        let plan = session.explain(query).await.unwrap();
555        let planned = plan
556            .lines()
557            .filter(|line| line.trim_start().starts_with("Gpu"))
558            .count();
559        assert!(planned > 0, "nothing was lowered:\n{plan}");
560        assert_eq!(
561            field(first, "gpu_operators"),
562            planned as u64,
563            "{first}\n{plan}"
564        );
565        for name in ["gpu_operators", "fallback_batches", "rows"] {
566            assert_eq!(
567                field(second, name),
568                field(first, name),
569                "{name} of the second run:\n{first}\n{second}"
570            );
571        }
572    }
573
574    /// An embedded session scans through the Parquet guard and a bounded
575    /// local store (#109). Lose either and a corrupt footer's lengths reach
576    /// the reader unchecked again, while every query on a valid file still
577    /// passes.
578    #[test]
579    fn an_embedded_session_checks_parquet_files_and_bounds_reads() {
580        let session = OxideSession::local().unwrap();
581        let state = session.ctx().state();
582        assert!(
583            state
584                .physical_optimizers()
585                .iter()
586                .any(|rule| rule.name() == "ParquetScanGuard"),
587            "no ParquetScanGuard in the physical optimizer rules"
588        );
589        let store = session
590            .ctx()
591            .runtime_env()
592            .object_store(datafusion::execution::object_store::ObjectStoreUrl::local_filesystem())
593            .unwrap();
594        assert!(
595            store.to_string().starts_with("BoundedReadStore("),
596            "{store}"
597        );
598    }
599
600    /// The no-argument constructors and `SessionOptions::default()` are the
601    /// same session, so the builder cannot drift away from the plain one.
602    #[test]
603    fn the_default_options_are_the_plain_constructor() {
604        let plain = OxideSession::local().unwrap();
605        let built = OxideSession::local_with_options(&SessionOptions::default()).unwrap();
606        assert_eq!(plain.mode(), built.mode());
607        assert_eq!(batch_size(&plain), batch_size(&built));
608    }
609}