oxidelake-runtime 0.3.0

OxideLake runtime: OxideSession, Ballista scheduler/executor wrappers and the oxide CLI
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
//! Cluster mode on Apache DataFusion Ballista (ADR-0013).
//!
//! OxideLake does not write a scheduler. This module builds the pieces Ballista
//! lets us plug in — a session builder that installs the placement rule for the
//! declared cluster capability, a physical codec that ships `Gpu*Exec` nodes,
//! and the executor function registry — and wires them into Ballista's
//! scheduler and executor processes, plus an in-process standalone cluster for
//! tests and the CLI.

use std::any::Any;
use std::future::Future;
use std::net::SocketAddr;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;

use ballista_core::extension::{
    SessionConfigExt, ballista_aggregate_functions, ballista_scalar_functions,
    ballista_window_functions,
};
use ballista_core::registry::BallistaFunctionRegistry;
use ballista_core::serde::protobuf::scheduler_grpc_client::SchedulerGrpcClient;
use ballista_core::serde::{
    BallistaCodec, BallistaLogicalExtensionCodec, BallistaPhysicalExtensionCodec,
};
use ballista_core::utils::{default_config_producer, default_session_builder};
use ballista_core::{ConfigProducer, RuntimeProducer};
use ballista_executor::executor_process::{ExecutorProcessConfig, start_executor_process};
use ballista_executor::new_standalone_executor_from_builder;
use ballista_scheduler::cluster::memory::InMemoryJobState;
use ballista_scheduler::cluster::{BallistaCluster, JobState};
use ballista_scheduler::config::SchedulerConfig;
use ballista_scheduler::scheduler_process::{create_scheduler, start_grpc_service};
use ballista_scheduler::scheduler_server::SessionBuilder;
use ballista_scheduler::standalone::new_standalone_scheduler_with_builder;
use ballista_scheduler::state::session_manager::share_file_statistics_cache;
use datafusion::config::{ConfigEntry, ConfigExtension, ExtensionOptions};
use datafusion::error::DataFusionError;
use datafusion::execution::SessionStateBuilder;
use datafusion::execution::object_store::ObjectStoreUrl;
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
use datafusion::execution::session_state::SessionState;
use datafusion::prelude::SessionConfig;
use datafusion_proto::physical_plan::PhysicalExtensionCodec;
use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode};
use oxidelake_compute::oxide_udfs;
use oxidelake_core::{BackendKind, EngineError};
use oxidelake_planner::codec::build_description;
use oxidelake_planner::{HardwarePlacementRule, OxidePhysicalCodec, physical_optimizer_rules};
use oxidelake_storage::{
    BoundedReadStore, ParquetScanGuard, default_object_store, with_gpu_batch_size, with_pruning,
};

use crate::admission::{
    REFUSED, SameBuildClusterState, failing_without_a_usable_worker, tagged_executor_id,
};
use crate::shutdown::{ShutdownSignal, Stopped, drain, refusing_while_draining, unfinished_jobs};
use crate::worker_loss::{ReportingJobState, WorkerLoss};

fn ballista_error(err: impl std::fmt::Display) -> EngineError {
    EngineError::execution(format!("ballista: {err}"))
}

/// The physical codec every OxideLake process installs: our `Gpu*Exec`
/// encoding on top of Ballista's own codec (shuffle nodes).
pub fn oxide_codec() -> Arc<dyn PhysicalExtensionCodec> {
    Arc::new(OxidePhysicalCodec::new(Arc::new(
        BallistaPhysicalExtensionCodec::default(),
    )))
}

/// Ballista's combined logical + physical codec with [`oxide_codec`] as the physical half.
pub fn ballista_codec() -> BallistaCodec {
    BallistaCodec::new(
        Arc::new(BallistaLogicalExtensionCodec::default()),
        oxide_codec(),
    )
}

/// Session configuration for every process: Ballista defaults, Parquet pruning
/// on, the OxideLake codec, and Ballista's static planner.
///
/// Ballista 55 plans adaptively by default, and its adaptive planner builds
/// its own session with Ballista's physical optimizer rules in place of the
/// session's, so the placement rules never run: a GPU-targeted cluster
/// executed no `Gpu*Exec` at all while `EXPLAIN`, which plans statically,
/// still showed them (#134). The adaptive planner has no hook to add a rule
/// to, so it stays off until it does, and [`build_session_state`] holds the
/// scheduler to that whatever a client sends.
pub fn session_config() -> SessionConfig {
    with_pruning(SessionConfig::new_with_ballista())
        .with_ballista_physical_extension_codec(oxide_codec())
        .with_ballista_adaptive_query_planner(false)
        .with_option_extension(ClientOptions::default())
}

/// What an OxideLake cluster client tells the scheduler beside DataFusion's
/// own settings (#93).
///
/// Ballista ships a client's whole configuration to the scheduler as
/// key/value pairs, and an extension registered at both ends travels with it
/// ([`session_config`] registers this one everywhere). The scheduler needs a
/// fact DataFusion's settings cannot carry: whether the batch size was a
/// choice. A GPU-targeted cluster raises an unchosen size to
/// [`GPU_BATCH_SIZE`](oxidelake_storage::GPU_BATCH_SIZE), and DataFusion's
/// default of 8192 is also a size someone can ask for, so the value alone
/// cannot say which it is.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub(crate) struct ClientOptions {
    /// The client set its batch size (`--batch-size`,
    /// `SessionOptions::with_batch_size`).
    pub(crate) explicit_batch_size: bool,
}

impl ClientOptions {
    const EXPLICIT_BATCH_SIZE: &'static str = "explicit_batch_size";
}

impl ExtensionOptions for ClientOptions {
    fn as_any(&self) -> &dyn Any {
        self
    }

    fn as_any_mut(&mut self) -> &mut dyn Any {
        self
    }

    fn cloned(&self) -> Box<dyn ExtensionOptions> {
        Box::new(self.clone())
    }

    fn set(&mut self, key: &str, value: &str) -> datafusion::error::Result<()> {
        if key != Self::EXPLICIT_BATCH_SIZE {
            return Err(DataFusionError::Configuration(format!(
                "no OxideLake setting `{}.{key}`",
                Self::PREFIX
            )));
        }
        self.explicit_batch_size = value.parse().map_err(|_| {
            DataFusionError::Configuration(format!(
                "`{}.{key}` takes true or false, not `{value}`",
                Self::PREFIX
            ))
        })?;
        Ok(())
    }

    // Prefixed, unlike the entries `extensions_options!` generates: Ballista
    // sends `entries()` to the scheduler as they are, and the scheduler's
    // `ConfigOptions::set` finds the extension by that prefix. An unprefixed
    // key arrives without a namespace and is dropped at `debug` level, so the
    // marker would silently never arrive.
    fn entries(&self) -> Vec<ConfigEntry> {
        vec![ConfigEntry {
            key: format!("{}.{}", Self::PREFIX, Self::EXPLICIT_BATCH_SIZE),
            value: Some(self.explicit_batch_size.to_string()),
            description: "The client chose its batch size, so a GPU-targeted \
                          cluster keeps it instead of raising it to the GPU default",
        }]
    }
}

impl ConfigExtension for ClientOptions {
    const PREFIX: &'static str = "oxidelake";
}

/// Whether `config`'s batch size is one the client asked for (#93): said so
/// explicitly, or set to anything but DataFusion's default (a `SET
/// datafusion.execution.batch_size`, or a client that predates the marker).
fn client_chose_batch_size(config: &SessionConfig) -> bool {
    config
        .options()
        .extensions
        .get::<ClientOptions>()
        .is_some_and(|client| client.explicit_batch_size)
        || config.batch_size() != SessionConfig::new().batch_size()
}

/// Ballista `ConfigProducer` yielding [`session_config`].
pub fn config_producer() -> ConfigProducer {
    Arc::new(session_config)
}

/// Builds the scheduler-side session state: Ballista's defaults plus the
/// placement rule targeting the declared cluster capability. This is where
/// `Gpu*Exec` nodes enter a distributed plan.
///
/// It is also where a cluster checks the Parquet files a query reads (#109).
/// Executors scan with DataFusion's own reader, so [`ParquetScanGuard`]
/// checks, while the scheduler plans, the metadata planning has just read
/// into this session's cache, and the local store is bounded like an
/// embedded session's, so that planning read cannot reserve memory for a
/// range a corrupt footer declares past the end of the file.
/// `config` is the client's (Ballista sends it with every query). A GPU
/// target raises its batch size to the GPU default unless the client chose
/// one, which it keeps (#93). Adaptive planning is turned off whatever the
/// client asked for, because it would drop the placement rule (#134; see
/// [`session_config`]).
pub fn build_session_state(
    config: SessionConfig,
    target: BackendKind,
) -> datafusion::error::Result<SessionState> {
    let mut scalar_functions = ballista_scalar_functions();
    scalar_functions.extend(oxide_udfs());
    // The scheduler picks its planner from the state built here, after the
    // client's settings were applied, so this is the one place a client's
    // `SET ballista.planner.adaptive.enabled = true` (or a stock Ballista
    // client, whose default it is) cannot win.
    let config = if config.ballista_adaptive_query_planner_enabled() {
        tracing::warn!(
            "adaptive query planning is not available on an OxideLake cluster: \
             it would drop the placement rules, so this query is planned statically (#134)"
        );
        config.with_ballista_adaptive_query_planner(false)
    } else {
        config
    };
    let config = if target.is_gpu() && !client_chose_batch_size(&config) {
        with_gpu_batch_size(config)
    } else {
        config
    };
    let runtime = Arc::new(RuntimeEnvBuilder::new().build()?);
    runtime.register_object_store(
        ObjectStoreUrl::local_filesystem().as_ref(),
        Arc::new(BoundedReadStore::new(default_object_store())),
    );
    let guard = ParquetScanGuard::new()
        .with_metadata_cache(runtime.cache_manager.get_file_metadata_cache());
    Ok(SessionStateBuilder::new()
        .with_default_features()
        .with_config(config)
        .with_runtime_env(runtime)
        .with_scalar_functions(scalar_functions)
        .with_aggregate_functions(ballista_aggregate_functions())
        .with_window_functions(ballista_window_functions())
        .with_physical_optimizer_rules(physical_optimizer_rules(HardwarePlacementRule::new(target)))
        .with_physical_optimizer_rule(Arc::new(guard))
        .build())
}

/// Ballista `SessionBuilder` wrapping [`build_session_state`].
pub fn session_builder(target: BackendKind) -> SessionBuilder {
    Arc::new(move |config: SessionConfig| build_session_state(config, target))
}

/// Ballista `RuntimeProducer` for executors.
pub fn runtime_producer() -> RuntimeProducer {
    Arc::new(|_config: &SessionConfig| Ok(Arc::new(RuntimeEnvBuilder::new().build()?)))
}

/// Functions registered on every executor: Ballista's defaults plus
/// OxideLake's SQL UDFs (`l2_distance`, `cosine_distance`), so a distance
/// projection that was not lowered to a `Gpu*Exec` still runs on workers.
pub fn function_registry() -> BallistaFunctionRegistry {
    let mut registry = BallistaFunctionRegistry::default();
    for udf in oxide_udfs() {
        registry.scalar_functions.insert(udf.name().to_owned(), udf);
    }
    registry
}

/// How many failed attempts a stage may have before its query fails, unless
/// told otherwise: one, so a stage whose shuffle input was lost with a
/// worker is never run again (#148).
///
/// Ballista's default is 4, and every retry rolls the stage back, which in
/// Ballista 55 can return some rows twice with exit 0 or never finish
/// (apache/datafusion-ballista#2677, unreleased after 55.0.0, and #2539,
/// #2698). At 1 the first failed fetch fails the query, naming the worker.
/// Only a failed fetch counts: Ballista rolls back a worker that is stopped,
/// or silent past its timeout, whatever this says, which is why
/// [`run_scheduler`] also fails a lost worker's queries itself.
pub const DEFAULT_STAGE_MAX_FAILURES: usize = 1;

/// Scheduler configuration with OxideLake's hooks installed, failing a query
/// whose shuffle input was lost with a worker
/// ([`DEFAULT_STAGE_MAX_FAILURES`]).
pub fn scheduler_config(bind_host: &str, port: u16, target: BackendKind) -> SchedulerConfig {
    SchedulerConfig {
        bind_host: bind_host.to_owned(),
        bind_port: port,
        override_session_builder: Some(session_builder(target)),
        override_config_producer: Some(config_producer()),
        override_physical_codec: Some(oxide_codec()),
        stage_max_failures: DEFAULT_STAGE_MAX_FAILURES,
        ..SchedulerConfig::default()
    }
}

/// How long a scheduler waits on a silent worker unless told otherwise:
/// Ballista's `executor_timeout_seconds`.
pub const DEFAULT_WORKER_TIMEOUT: Duration = Duration::from_secs(180);

/// Sets how long a worker may go without a heartbeat before `config`'s
/// scheduler presumes it lost and re-runs its tasks elsewhere (#31), in
/// whole seconds, at least one.
///
/// Until then a worker that died holding tasks holds its query too, unless a
/// shuffle fetch from it fails first. Ballista checks for silent workers on
/// a fixed 15-second tick, which would make a timeout shorter than that a
/// fiction; the tick here is a third of the timeout, never more than 15
/// seconds. A query left with no worker at all fails after the same timeout
/// again, as Ballista's own scheduler binary arranges it.
pub fn with_worker_timeout(mut config: SchedulerConfig, timeout: Duration) -> SchedulerConfig {
    let seconds = timeout.as_secs().max(1);
    config.executor_timeout_seconds = seconds;
    config.expire_dead_executor_interval_seconds = (seconds / 3).clamp(1, 15);
    config.no_executors_grace_period_seconds = seconds;
    config
}

type SchedulerServe = Pin<Box<dyn Future<Output = ballista_core::error::Result<()>> + Send>>;

/// A scheduler ready to serve, with what a shutdown needs to reach.
struct Scheduler {
    serve: SchedulerServe,
    jobs: Arc<dyn JobState>,
    draining: Arc<AtomicBool>,
}

impl Scheduler {
    fn new(config: SchedulerConfig) -> Result<Self, EngineError> {
        let address: SocketAddr = format!("{}:{}", config.bind_host, config.bind_port)
            .parse()
            .map_err(|e| EngineError::plan(format!("invalid scheduler bind address: {e}")))?;
        let draining = Arc::new(AtomicBool::new(false));
        let loss = Arc::new(WorkerLoss::default());
        let cluster = same_build_cluster(&config, Arc::clone(&draining), Arc::clone(&loss));
        let jobs = cluster.job_state();
        // What Ballista's `start_server` does, keeping hold of the scheduler
        // so a lost worker's queries can be failed through it (#148).
        let serve = async move {
            config.validate()?;
            tracing::info!(%address, "Ballista scheduler listening");
            let scheduler =
                create_scheduler::<LogicalPlanNode, PhysicalPlanNode>(cluster, Arc::new(config))
                    .await?;
            loss.attach(scheduler.clone());
            start_grpc_service(address, scheduler).await
        };
        Ok(Self {
            serve: Box::pin(serve),
            jobs,
            draining,
        })
    }
}

/// Runs a Ballista scheduler until it exits.
///
/// The scheduler admits only workers of its own build (#92): one whose plan
/// fingerprint differs is refused when it registers, and a query sent to a
/// cluster whose workers were all refused fails at once with the reason,
/// where Ballista would hold it for a worker that is never coming.
pub async fn run_scheduler(config: SchedulerConfig) -> Result<(), EngineError> {
    Scheduler::new(config)?.serve.await.map_err(ballista_error)
}

/// Runs a scheduler like [`run_scheduler`] until `signal`, then stops it
/// gracefully (#31): a query sent from then on fails at once saying the
/// scheduler is shutting down, and the queries already running get up to
/// `drain_timeout` to finish before this returns. A second signal stops it at
/// once. Ballista's scheduler keeps every query in memory, so whatever is
/// still running when this returns fails.
pub async fn run_scheduler_with_shutdown(
    config: SchedulerConfig,
    mut signal: ShutdownSignal,
    drain_timeout: Duration,
) -> Result<Stopped, EngineError> {
    let Scheduler {
        mut serve,
        jobs,
        draining,
    } = Scheduler::new(config)?;
    let first = tokio::select! {
        served = &mut serve => return served.map(|()| Stopped::Drained).map_err(ballista_error),
        first = signal.recv() => first,
    };
    draining.store(true, Ordering::SeqCst);
    tracing::info!(
        signal = first,
        running = unfinished_jobs(jobs.as_ref()).await,
        drain_timeout_s = drain_timeout.as_secs_f64(),
        "shutting down: refusing new queries and waiting for the running ones"
    );
    // The server keeps serving while the queries drain: their clients still
    // poll it for their status.
    tokio::select! {
        served = &mut serve => served.map(|()| Stopped::Drained).map_err(ballista_error),
        unfinished = drain(jobs.as_ref(), drain_timeout) => Ok(if unfinished == 0 {
            tracing::info!("every running query finished; scheduler stopped");
            Stopped::Drained
        } else {
            tracing::warn!(
                unfinished,
                drain_timeout_s = drain_timeout.as_secs_f64(),
                "the drain timeout passed with queries still running; they fail as this \
                 scheduler stops"
            );
            Stopped::Abandoned
        }),
        second = signal.recv() => {
            tracing::warn!(
                signal = second,
                unfinished = unfinished_jobs(jobs.as_ref()).await,
                "a second signal: stopping now, and the queries still running fail"
            );
            Ok(Stopped::Abandoned)
        }
    }
}

/// What `BallistaCluster::new_from_config` builds, in-memory cluster and job
/// state, with the cluster state admitting only workers of this build and
/// failing a lost worker's queries through `loss` (#148), the job state
/// saying why, and the session builder failing a query that no admitted
/// worker can run and, once `draining` is set, every new query (#31).
fn same_build_cluster(
    config: &SchedulerConfig,
    draining: Arc<AtomicBool>,
    loss: Arc<WorkerLoss>,
) -> BallistaCluster {
    let state = Arc::new(SameBuildClusterState::new(Arc::clone(&loss)));
    let session_builder = config
        .override_session_builder
        .clone()
        .unwrap_or_else(|| Arc::new(default_session_builder));
    let config_producer = config
        .override_config_producer
        .clone()
        .unwrap_or_else(|| Arc::new(default_config_producer));
    let jobs = InMemoryJobState::new(
        config.scheduler_id.clone(),
        share_file_statistics_cache(refusing_while_draining(
            failing_without_a_usable_worker(session_builder, Arc::clone(&state)),
            draining,
        )),
        config_producer,
    );
    BallistaCluster::new(state, Arc::new(ReportingJobState::new(jobs, loss)))
}

/// Executor configuration with OxideLake's hooks installed.
///
/// The executor id carries this build's plan fingerprint, which is what a
/// scheduler checks when the executor registers (#92).
pub fn executor_config(
    scheduler_host: &str,
    scheduler_port: u16,
    port: u16,
    grpc_port: u16,
    concurrent_tasks: Option<usize>,
    work_dir: Option<String>,
) -> ExecutorProcessConfig {
    let defaults = ExecutorProcessConfig::default();
    ExecutorProcessConfig {
        executor_id: tagged_executor_id(&defaults.executor_id),
        scheduler_host: scheduler_host.to_owned(),
        scheduler_port,
        port,
        grpc_port,
        // Ballista 55 renamed `concurrent_tasks` to `vcores`: still the number
        // of tasks this executor runs at once, and `0` still means every
        // available core (`start_executor_process` resolves it).
        vcores: concurrent_tasks.unwrap_or(defaults.vcores),
        work_dir,
        override_physical_codec: Some(oxide_codec()),
        override_function_registry: Some(Arc::new(function_registry())),
        override_config_producer: Some(config_producer()),
        ..defaults
    }
}

/// Runs a Ballista executor until it exits.
///
/// A scheduler of another build refuses it when it registers (#92), and this
/// returns that refusal with the executor's own build added, so the two
/// builds are named side by side in the worker's log.
pub async fn run_executor(config: ExecutorProcessConfig) -> Result<(), EngineError> {
    start_executor_process(Arc::new(config))
        .await
        .map_err(|err| {
            let text = err.to_string();
            if text.contains(REFUSED) {
                EngineError::execution(format!(
                    "ballista: {text}. This worker is {}",
                    build_description()
                ))
            } else {
                ballista_error(err)
            }
        })
}

/// Runs an executor like [`run_executor`], and once `signal` comes, gives it
/// at most `drain_timeout` to stop (#31).
///
/// Ballista's executor sees the same signal and stops gracefully on its own:
/// it takes no new tasks, tells the scheduler, which re-runs the tasks it was
/// running elsewhere, and waits for those tasks to finish. That wait has no
/// bound, so a task that does not finish would keep the worker alive until it
/// is killed. When the bound passes, or a second signal comes, this returns
/// [`Stopped::Abandoned`] and the caller should exit without waiting for the
/// tasks still running.
///
/// Ballista's executor starts listening only once it has registered, a
/// fraction of a second after the worker starts. A signal that arrives
/// before that is seen here alone: the worker then stops when the drain
/// timeout passes, still with exit 0, and the scheduler learns it is gone
/// from its missed heartbeats.
pub async fn run_executor_with_shutdown(
    config: ExecutorProcessConfig,
    mut signal: ShutdownSignal,
    drain_timeout: Duration,
) -> Result<Stopped, EngineError> {
    let executor = run_executor(config);
    tokio::pin!(executor);
    let first = tokio::select! {
        stopped = &mut executor => return stopped.map(|()| Stopped::Drained),
        first = signal.recv() => first,
    };
    tracing::info!(
        signal = first,
        drain_timeout_s = drain_timeout.as_secs_f64(),
        "stopping: taking no new tasks, and waiting for the running ones, which the scheduler \
         re-runs elsewhere"
    );
    tokio::select! {
        drained = tokio::time::timeout(drain_timeout, &mut executor) => match drained {
            Ok(stopped) => stopped.map(|()| Stopped::Drained),
            Err(_) => {
                tracing::warn!(
                    drain_timeout_s = drain_timeout.as_secs_f64(),
                    "the drain timeout passed with tasks still running: stopping without them"
                );
                Ok(Stopped::Abandoned)
            }
        },
        second = signal.recv() => {
            tracing::warn!(signal = second, "a second signal: stopping without the running tasks");
            Ok(Stopped::Abandoned)
        }
    }
}

/// Starts an in-process scheduler plus `executors` executors on ephemeral
/// localhost ports and returns the scheduler address. Everything runs on the
/// current tokio runtime and stops with it.
pub async fn start_standalone(
    target: BackendKind,
    executors: usize,
    concurrent_tasks: usize,
) -> Result<SocketAddr, EngineError> {
    let addr = new_standalone_scheduler_with_builder(
        session_builder(target),
        config_producer(),
        ballista_codec(),
    )
    .await
    .map_err(ballista_error)?;
    for _ in 0..executors.max(1) {
        let client = SchedulerGrpcClient::connect(format!("http://{addr}"))
            .await
            .map_err(ballista_error)?;
        new_standalone_executor_from_builder(
            client,
            concurrent_tasks.max(1),
            config_producer(),
            runtime_producer(),
            ballista_codec(),
            function_registry(),
        )
        .await
        .map_err(ballista_error)?;
    }
    tracing::info!(%addr, executors, "standalone Ballista cluster started");
    Ok(addr)
}

#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
    use super::*;

    /// A cluster's Parquet checks live on the scheduler (#109): its executors
    /// scan with DataFusion's own reader, so if the scheduler's session lost
    /// the guard, its metadata cache or the bounded store, nothing in the
    /// cluster would check a file at all.
    #[test]
    fn the_scheduler_session_checks_parquet_files_and_bounds_reads() {
        let state = build_session_state(session_config(), BackendKind::CpuSimd).unwrap();
        let guards: Vec<String> = state
            .physical_optimizers()
            .iter()
            .filter(|rule| rule.name() == "ParquetScanGuard")
            .map(|rule| format!("{rule:?}"))
            .collect();
        assert_eq!(guards.len(), 1, "{guards:?}");
        assert!(
            guards[0].contains("checks_metadata_cache: true"),
            "{}",
            guards[0]
        );
        let store = state
            .runtime_env()
            .object_store(ObjectStoreUrl::local_filesystem())
            .unwrap();
        assert!(
            store.to_string().starts_with("BoundedReadStore("),
            "{store}"
        );
    }

    use ballista_core::extension::SessionConfigHelperExt;
    use oxidelake_storage::GPU_BATCH_SIZE;

    /// The config the scheduler hands its session builder for a client's
    /// query: its own producer's, updated from the client's settings the way
    /// Ballista's `create_context` does it.
    fn as_the_scheduler_sees(client: &SessionConfig) -> SessionConfig {
        session_config().update_from_key_value_pair(&client.to_key_value_pairs())
    }

    fn client(explicit: bool, batch_size: usize) -> SessionConfig {
        session_config()
            .with_option_extension(ClientOptions {
                explicit_batch_size: explicit,
            })
            .with_batch_size(batch_size)
    }

    fn planned_batch_size(config: SessionConfig, target: BackendKind) -> usize {
        build_session_state(config, target)
            .unwrap()
            .config()
            .batch_size()
    }

    /// The marker survives Ballista's key/value trip to the scheduler. A key
    /// without its `oxidelake.` prefix would be dropped there at `debug`
    /// level, and every other assertion about the marker would then be
    /// testing a value the scheduler never receives.
    #[test]
    fn the_explicit_batch_size_marker_reaches_the_scheduler() {
        let seen = as_the_scheduler_sees(&client(true, 7));
        let options = seen.options().extensions.get::<ClientOptions>().unwrap();
        assert!(options.explicit_batch_size);
        assert_eq!(seen.batch_size(), 7);

        let seen = as_the_scheduler_sees(&client(false, 7));
        let options = seen.options().extensions.get::<ClientOptions>().unwrap();
        assert!(!options.explicit_batch_size);
    }

    /// A GPU cluster raises only a batch size nobody chose (#93).
    #[test]
    fn a_gpu_cluster_keeps_a_batch_size_the_client_chose() {
        let default = SessionConfig::new().batch_size();
        // Nobody chose: the GPU default.
        let seen = as_the_scheduler_sees(&client(false, default));
        assert_eq!(planned_batch_size(seen, BackendKind::Cuda), GPU_BATCH_SIZE);
        // Chosen, and equal to DataFusion's default: only the marker says so.
        let seen = as_the_scheduler_sees(&client(true, default));
        assert_eq!(planned_batch_size(seen, BackendKind::Cuda), default);
        // Not the default (`SET`, or a client from before the marker).
        let seen = as_the_scheduler_sees(&client(false, 7));
        assert_eq!(planned_batch_size(seen, BackendKind::Metal), 7);
        // A CPU cluster never raises it.
        let seen = as_the_scheduler_sees(&client(false, default));
        assert_eq!(planned_batch_size(seen, BackendKind::CpuSimd), default);
    }

    #[test]
    fn a_malformed_marker_is_refused_by_name() {
        let mut options = ClientOptions::default();
        let err = options.set("explicit_batch_size", "yes").unwrap_err();
        assert!(
            err.to_string().contains("oxidelake.explicit_batch_size"),
            "{err}"
        );
        let err = options.set("batch", "true").unwrap_err();
        assert!(err.to_string().contains("oxidelake.batch"), "{err}");
    }

    /// The scheduler plans statically whatever a client sends (#134):
    /// Ballista's adaptive planner would replace the placement rules.
    #[test]
    fn a_client_cannot_turn_adaptive_planning_back_on() {
        assert!(!session_config().ballista_adaptive_query_planner_enabled());
        // `SET ballista.planner.adaptive.enabled = true`, and a stock Ballista
        // client, whose default it is: both arrive at the session builder.
        let asked = session_config().with_ballista_adaptive_query_planner(true);
        let stock = SessionConfig::new_with_ballista();
        for client in [asked, stock] {
            let seen = as_the_scheduler_sees(&client);
            assert!(seen.ballista_adaptive_query_planner_enabled());
            for target in [BackendKind::CpuSimd, BackendKind::Cuda] {
                let state = build_session_state(seen.clone(), target).unwrap();
                assert!(!state.config().ballista_adaptive_query_planner_enabled());
            }
        }
    }
}