use std::any::Any;
use std::future::Future;
use std::net::SocketAddr;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use ballista_core::extension::{
SessionConfigExt, ballista_aggregate_functions, ballista_scalar_functions,
ballista_window_functions,
};
use ballista_core::registry::BallistaFunctionRegistry;
use ballista_core::serde::protobuf::scheduler_grpc_client::SchedulerGrpcClient;
use ballista_core::serde::{
BallistaCodec, BallistaLogicalExtensionCodec, BallistaPhysicalExtensionCodec,
};
use ballista_core::utils::{default_config_producer, default_session_builder};
use ballista_core::{ConfigProducer, RuntimeProducer};
use ballista_executor::executor_process::{ExecutorProcessConfig, start_executor_process};
use ballista_executor::new_standalone_executor_from_builder;
use ballista_scheduler::cluster::memory::InMemoryJobState;
use ballista_scheduler::cluster::{BallistaCluster, JobState};
use ballista_scheduler::config::SchedulerConfig;
use ballista_scheduler::scheduler_process::{create_scheduler, start_grpc_service};
use ballista_scheduler::scheduler_server::SessionBuilder;
use ballista_scheduler::standalone::new_standalone_scheduler_with_builder;
use ballista_scheduler::state::session_manager::share_file_statistics_cache;
use datafusion::config::{ConfigEntry, ConfigExtension, ExtensionOptions};
use datafusion::error::DataFusionError;
use datafusion::execution::SessionStateBuilder;
use datafusion::execution::object_store::ObjectStoreUrl;
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
use datafusion::execution::session_state::SessionState;
use datafusion::prelude::SessionConfig;
use datafusion_proto::physical_plan::PhysicalExtensionCodec;
use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode};
use oxidelake_compute::oxide_udfs;
use oxidelake_core::{BackendKind, EngineError};
use oxidelake_planner::codec::build_description;
use oxidelake_planner::{HardwarePlacementRule, OxidePhysicalCodec, physical_optimizer_rules};
use oxidelake_storage::{
BoundedReadStore, ParquetScanGuard, default_object_store, with_gpu_batch_size, with_pruning,
};
use crate::admission::{
REFUSED, SameBuildClusterState, failing_without_a_usable_worker, tagged_executor_id,
};
use crate::shutdown::{ShutdownSignal, Stopped, drain, refusing_while_draining, unfinished_jobs};
use crate::worker_loss::{ReportingJobState, WorkerLoss};
fn ballista_error(err: impl std::fmt::Display) -> EngineError {
EngineError::execution(format!("ballista: {err}"))
}
pub fn oxide_codec() -> Arc<dyn PhysicalExtensionCodec> {
Arc::new(OxidePhysicalCodec::new(Arc::new(
BallistaPhysicalExtensionCodec::default(),
)))
}
pub fn ballista_codec() -> BallistaCodec {
BallistaCodec::new(
Arc::new(BallistaLogicalExtensionCodec::default()),
oxide_codec(),
)
}
pub fn session_config() -> SessionConfig {
with_pruning(SessionConfig::new_with_ballista())
.with_ballista_physical_extension_codec(oxide_codec())
.with_ballista_adaptive_query_planner(false)
.with_option_extension(ClientOptions::default())
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub(crate) struct ClientOptions {
pub(crate) explicit_batch_size: bool,
}
impl ClientOptions {
const EXPLICIT_BATCH_SIZE: &'static str = "explicit_batch_size";
}
impl ExtensionOptions for ClientOptions {
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
fn cloned(&self) -> Box<dyn ExtensionOptions> {
Box::new(self.clone())
}
fn set(&mut self, key: &str, value: &str) -> datafusion::error::Result<()> {
if key != Self::EXPLICIT_BATCH_SIZE {
return Err(DataFusionError::Configuration(format!(
"no OxideLake setting `{}.{key}`",
Self::PREFIX
)));
}
self.explicit_batch_size = value.parse().map_err(|_| {
DataFusionError::Configuration(format!(
"`{}.{key}` takes true or false, not `{value}`",
Self::PREFIX
))
})?;
Ok(())
}
fn entries(&self) -> Vec<ConfigEntry> {
vec![ConfigEntry {
key: format!("{}.{}", Self::PREFIX, Self::EXPLICIT_BATCH_SIZE),
value: Some(self.explicit_batch_size.to_string()),
description: "The client chose its batch size, so a GPU-targeted \
cluster keeps it instead of raising it to the GPU default",
}]
}
}
impl ConfigExtension for ClientOptions {
const PREFIX: &'static str = "oxidelake";
}
fn client_chose_batch_size(config: &SessionConfig) -> bool {
config
.options()
.extensions
.get::<ClientOptions>()
.is_some_and(|client| client.explicit_batch_size)
|| config.batch_size() != SessionConfig::new().batch_size()
}
pub fn config_producer() -> ConfigProducer {
Arc::new(session_config)
}
pub fn build_session_state(
config: SessionConfig,
target: BackendKind,
) -> datafusion::error::Result<SessionState> {
let mut scalar_functions = ballista_scalar_functions();
scalar_functions.extend(oxide_udfs());
let config = if config.ballista_adaptive_query_planner_enabled() {
tracing::warn!(
"adaptive query planning is not available on an OxideLake cluster: \
it would drop the placement rules, so this query is planned statically (#134)"
);
config.with_ballista_adaptive_query_planner(false)
} else {
config
};
let config = if target.is_gpu() && !client_chose_batch_size(&config) {
with_gpu_batch_size(config)
} else {
config
};
let runtime = Arc::new(RuntimeEnvBuilder::new().build()?);
runtime.register_object_store(
ObjectStoreUrl::local_filesystem().as_ref(),
Arc::new(BoundedReadStore::new(default_object_store())),
);
let guard = ParquetScanGuard::new()
.with_metadata_cache(runtime.cache_manager.get_file_metadata_cache());
Ok(SessionStateBuilder::new()
.with_default_features()
.with_config(config)
.with_runtime_env(runtime)
.with_scalar_functions(scalar_functions)
.with_aggregate_functions(ballista_aggregate_functions())
.with_window_functions(ballista_window_functions())
.with_physical_optimizer_rules(physical_optimizer_rules(HardwarePlacementRule::new(target)))
.with_physical_optimizer_rule(Arc::new(guard))
.build())
}
pub fn session_builder(target: BackendKind) -> SessionBuilder {
Arc::new(move |config: SessionConfig| build_session_state(config, target))
}
pub fn runtime_producer() -> RuntimeProducer {
Arc::new(|_config: &SessionConfig| Ok(Arc::new(RuntimeEnvBuilder::new().build()?)))
}
pub fn function_registry() -> BallistaFunctionRegistry {
let mut registry = BallistaFunctionRegistry::default();
for udf in oxide_udfs() {
registry.scalar_functions.insert(udf.name().to_owned(), udf);
}
registry
}
pub const DEFAULT_STAGE_MAX_FAILURES: usize = 1;
pub fn scheduler_config(bind_host: &str, port: u16, target: BackendKind) -> SchedulerConfig {
SchedulerConfig {
bind_host: bind_host.to_owned(),
bind_port: port,
override_session_builder: Some(session_builder(target)),
override_config_producer: Some(config_producer()),
override_physical_codec: Some(oxide_codec()),
stage_max_failures: DEFAULT_STAGE_MAX_FAILURES,
..SchedulerConfig::default()
}
}
pub const DEFAULT_WORKER_TIMEOUT: Duration = Duration::from_secs(180);
pub fn with_worker_timeout(mut config: SchedulerConfig, timeout: Duration) -> SchedulerConfig {
let seconds = timeout.as_secs().max(1);
config.executor_timeout_seconds = seconds;
config.expire_dead_executor_interval_seconds = (seconds / 3).clamp(1, 15);
config.no_executors_grace_period_seconds = seconds;
config
}
type SchedulerServe = Pin<Box<dyn Future<Output = ballista_core::error::Result<()>> + Send>>;
struct Scheduler {
serve: SchedulerServe,
jobs: Arc<dyn JobState>,
draining: Arc<AtomicBool>,
}
impl Scheduler {
fn new(config: SchedulerConfig) -> Result<Self, EngineError> {
let address: SocketAddr = format!("{}:{}", config.bind_host, config.bind_port)
.parse()
.map_err(|e| EngineError::plan(format!("invalid scheduler bind address: {e}")))?;
let draining = Arc::new(AtomicBool::new(false));
let loss = Arc::new(WorkerLoss::default());
let cluster = same_build_cluster(&config, Arc::clone(&draining), Arc::clone(&loss));
let jobs = cluster.job_state();
let serve = async move {
config.validate()?;
tracing::info!(%address, "Ballista scheduler listening");
let scheduler =
create_scheduler::<LogicalPlanNode, PhysicalPlanNode>(cluster, Arc::new(config))
.await?;
loss.attach(scheduler.clone());
start_grpc_service(address, scheduler).await
};
Ok(Self {
serve: Box::pin(serve),
jobs,
draining,
})
}
}
pub async fn run_scheduler(config: SchedulerConfig) -> Result<(), EngineError> {
Scheduler::new(config)?.serve.await.map_err(ballista_error)
}
pub async fn run_scheduler_with_shutdown(
config: SchedulerConfig,
mut signal: ShutdownSignal,
drain_timeout: Duration,
) -> Result<Stopped, EngineError> {
let Scheduler {
mut serve,
jobs,
draining,
} = Scheduler::new(config)?;
let first = tokio::select! {
served = &mut serve => return served.map(|()| Stopped::Drained).map_err(ballista_error),
first = signal.recv() => first,
};
draining.store(true, Ordering::SeqCst);
tracing::info!(
signal = first,
running = unfinished_jobs(jobs.as_ref()).await,
drain_timeout_s = drain_timeout.as_secs_f64(),
"shutting down: refusing new queries and waiting for the running ones"
);
tokio::select! {
served = &mut serve => served.map(|()| Stopped::Drained).map_err(ballista_error),
unfinished = drain(jobs.as_ref(), drain_timeout) => Ok(if unfinished == 0 {
tracing::info!("every running query finished; scheduler stopped");
Stopped::Drained
} else {
tracing::warn!(
unfinished,
drain_timeout_s = drain_timeout.as_secs_f64(),
"the drain timeout passed with queries still running; they fail as this \
scheduler stops"
);
Stopped::Abandoned
}),
second = signal.recv() => {
tracing::warn!(
signal = second,
unfinished = unfinished_jobs(jobs.as_ref()).await,
"a second signal: stopping now, and the queries still running fail"
);
Ok(Stopped::Abandoned)
}
}
}
fn same_build_cluster(
config: &SchedulerConfig,
draining: Arc<AtomicBool>,
loss: Arc<WorkerLoss>,
) -> BallistaCluster {
let state = Arc::new(SameBuildClusterState::new(Arc::clone(&loss)));
let session_builder = config
.override_session_builder
.clone()
.unwrap_or_else(|| Arc::new(default_session_builder));
let config_producer = config
.override_config_producer
.clone()
.unwrap_or_else(|| Arc::new(default_config_producer));
let jobs = InMemoryJobState::new(
config.scheduler_id.clone(),
share_file_statistics_cache(refusing_while_draining(
failing_without_a_usable_worker(session_builder, Arc::clone(&state)),
draining,
)),
config_producer,
);
BallistaCluster::new(state, Arc::new(ReportingJobState::new(jobs, loss)))
}
pub fn executor_config(
scheduler_host: &str,
scheduler_port: u16,
port: u16,
grpc_port: u16,
concurrent_tasks: Option<usize>,
work_dir: Option<String>,
) -> ExecutorProcessConfig {
let defaults = ExecutorProcessConfig::default();
ExecutorProcessConfig {
executor_id: tagged_executor_id(&defaults.executor_id),
scheduler_host: scheduler_host.to_owned(),
scheduler_port,
port,
grpc_port,
vcores: concurrent_tasks.unwrap_or(defaults.vcores),
work_dir,
override_physical_codec: Some(oxide_codec()),
override_function_registry: Some(Arc::new(function_registry())),
override_config_producer: Some(config_producer()),
..defaults
}
}
pub async fn run_executor(config: ExecutorProcessConfig) -> Result<(), EngineError> {
start_executor_process(Arc::new(config))
.await
.map_err(|err| {
let text = err.to_string();
if text.contains(REFUSED) {
EngineError::execution(format!(
"ballista: {text}. This worker is {}",
build_description()
))
} else {
ballista_error(err)
}
})
}
pub async fn run_executor_with_shutdown(
config: ExecutorProcessConfig,
mut signal: ShutdownSignal,
drain_timeout: Duration,
) -> Result<Stopped, EngineError> {
let executor = run_executor(config);
tokio::pin!(executor);
let first = tokio::select! {
stopped = &mut executor => return stopped.map(|()| Stopped::Drained),
first = signal.recv() => first,
};
tracing::info!(
signal = first,
drain_timeout_s = drain_timeout.as_secs_f64(),
"stopping: taking no new tasks, and waiting for the running ones, which the scheduler \
re-runs elsewhere"
);
tokio::select! {
drained = tokio::time::timeout(drain_timeout, &mut executor) => match drained {
Ok(stopped) => stopped.map(|()| Stopped::Drained),
Err(_) => {
tracing::warn!(
drain_timeout_s = drain_timeout.as_secs_f64(),
"the drain timeout passed with tasks still running: stopping without them"
);
Ok(Stopped::Abandoned)
}
},
second = signal.recv() => {
tracing::warn!(signal = second, "a second signal: stopping without the running tasks");
Ok(Stopped::Abandoned)
}
}
}
pub async fn start_standalone(
target: BackendKind,
executors: usize,
concurrent_tasks: usize,
) -> Result<SocketAddr, EngineError> {
let addr = new_standalone_scheduler_with_builder(
session_builder(target),
config_producer(),
ballista_codec(),
)
.await
.map_err(ballista_error)?;
for _ in 0..executors.max(1) {
let client = SchedulerGrpcClient::connect(format!("http://{addr}"))
.await
.map_err(ballista_error)?;
new_standalone_executor_from_builder(
client,
concurrent_tasks.max(1),
config_producer(),
runtime_producer(),
ballista_codec(),
function_registry(),
)
.await
.map_err(ballista_error)?;
}
tracing::info!(%addr, executors, "standalone Ballista cluster started");
Ok(addr)
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
use super::*;
#[test]
fn the_scheduler_session_checks_parquet_files_and_bounds_reads() {
let state = build_session_state(session_config(), BackendKind::CpuSimd).unwrap();
let guards: Vec<String> = state
.physical_optimizers()
.iter()
.filter(|rule| rule.name() == "ParquetScanGuard")
.map(|rule| format!("{rule:?}"))
.collect();
assert_eq!(guards.len(), 1, "{guards:?}");
assert!(
guards[0].contains("checks_metadata_cache: true"),
"{}",
guards[0]
);
let store = state
.runtime_env()
.object_store(ObjectStoreUrl::local_filesystem())
.unwrap();
assert!(
store.to_string().starts_with("BoundedReadStore("),
"{store}"
);
}
use ballista_core::extension::SessionConfigHelperExt;
use oxidelake_storage::GPU_BATCH_SIZE;
fn as_the_scheduler_sees(client: &SessionConfig) -> SessionConfig {
session_config().update_from_key_value_pair(&client.to_key_value_pairs())
}
fn client(explicit: bool, batch_size: usize) -> SessionConfig {
session_config()
.with_option_extension(ClientOptions {
explicit_batch_size: explicit,
})
.with_batch_size(batch_size)
}
fn planned_batch_size(config: SessionConfig, target: BackendKind) -> usize {
build_session_state(config, target)
.unwrap()
.config()
.batch_size()
}
#[test]
fn the_explicit_batch_size_marker_reaches_the_scheduler() {
let seen = as_the_scheduler_sees(&client(true, 7));
let options = seen.options().extensions.get::<ClientOptions>().unwrap();
assert!(options.explicit_batch_size);
assert_eq!(seen.batch_size(), 7);
let seen = as_the_scheduler_sees(&client(false, 7));
let options = seen.options().extensions.get::<ClientOptions>().unwrap();
assert!(!options.explicit_batch_size);
}
#[test]
fn a_gpu_cluster_keeps_a_batch_size_the_client_chose() {
let default = SessionConfig::new().batch_size();
let seen = as_the_scheduler_sees(&client(false, default));
assert_eq!(planned_batch_size(seen, BackendKind::Cuda), GPU_BATCH_SIZE);
let seen = as_the_scheduler_sees(&client(true, default));
assert_eq!(planned_batch_size(seen, BackendKind::Cuda), default);
let seen = as_the_scheduler_sees(&client(false, 7));
assert_eq!(planned_batch_size(seen, BackendKind::Metal), 7);
let seen = as_the_scheduler_sees(&client(false, default));
assert_eq!(planned_batch_size(seen, BackendKind::CpuSimd), default);
}
#[test]
fn a_malformed_marker_is_refused_by_name() {
let mut options = ClientOptions::default();
let err = options.set("explicit_batch_size", "yes").unwrap_err();
assert!(
err.to_string().contains("oxidelake.explicit_batch_size"),
"{err}"
);
let err = options.set("batch", "true").unwrap_err();
assert!(err.to_string().contains("oxidelake.batch"), "{err}");
}
#[test]
fn a_client_cannot_turn_adaptive_planning_back_on() {
assert!(!session_config().ballista_adaptive_query_planner_enabled());
let asked = session_config().with_ballista_adaptive_query_planner(true);
let stock = SessionConfig::new_with_ballista();
for client in [asked, stock] {
let seen = as_the_scheduler_sees(&client);
assert!(seen.ballista_adaptive_query_planner_enabled());
for target in [BackendKind::CpuSimd, BackendKind::Cuda] {
let state = build_session_state(seen.clone(), target).unwrap();
assert!(!state.config().ballista_adaptive_query_planner_enabled());
}
}
}
}