use std::time::Duration;
use ballista_scheduler::config::SchedulerConfig;
use clap::Parser;
use oxidelake_core::BackendKind;
use oxidelake_runtime::cluster;
use oxidelake_runtime::exposure::{check_listen_address, warn_unless_loopback};
use oxidelake_runtime::shutdown::ShutdownSignal;
use tracing_subscriber::EnvFilter;
#[derive(Parser)]
#[command(
name = "oxide-scheduler",
version,
about = "OxideLake scheduler (Apache DataFusion Ballista)"
)]
struct Cli {
#[arg(long, default_value = "127.0.0.1")]
bind_host: String,
#[arg(long, default_value_t = 50050)]
port: u16,
#[arg(long)]
cluster_backend: Option<String>,
#[arg(long, value_name = "PORT")]
metrics_port: Option<u16>,
#[arg(long, default_value = "127.0.0.1", value_name = "HOST")]
metrics_host: String,
#[arg(long)]
allow_insecure_bind: bool,
#[arg(long, default_value_t = 4, value_name = "N", value_parser = at_least_one)]
task_max_failures: usize,
#[arg(
long,
default_value_t = cluster::DEFAULT_STAGE_MAX_FAILURES,
value_name = "N",
value_parser = at_least_one
)]
stage_max_failures: usize,
#[arg(
long,
default_value_t = cluster::DEFAULT_WORKER_TIMEOUT.as_secs(),
value_name = "SECS",
value_parser = clap::value_parser!(u64).range(1..)
)]
worker_timeout: u64,
#[arg(long, default_value_t = 20, value_name = "SECS")]
drain_timeout: u64,
}
fn at_least_one(value: &str) -> Result<usize, String> {
match value.parse::<usize>() {
Ok(0) => Err("must be at least 1 (1 means no retry)".to_owned()),
Ok(n) => Ok(n),
Err(e) => Err(e.to_string()),
}
}
fn config(cli: &Cli, target: BackendKind) -> SchedulerConfig {
let config = cluster::with_worker_timeout(
cluster::scheduler_config(&cli.bind_host, cli.port, target),
Duration::from_secs(cli.worker_timeout),
);
SchedulerConfig {
task_max_failures: cli.task_max_failures,
stage_max_failures: cli.stage_max_failures,
..config
}
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
tracing_subscriber::fmt()
.with_env_filter(EnvFilter::from_default_env())
.with_writer(std::io::stderr)
.with_ansi(std::io::IsTerminal::is_terminal(&std::io::stderr()))
.init();
let cli = Cli::parse();
let signal = ShutdownSignal::listen()?;
let exposure = check_listen_address("--bind-host", &cli.bind_host, cli.allow_insecure_bind)?;
if cli.metrics_port.is_some() {
check_listen_address("--metrics-host", &cli.metrics_host, cli.allow_insecure_bind)?;
}
warn_unless_loopback("the scheduler", &cli.bind_host, exposure);
let target: BackendKind = match &cli.cluster_backend {
Some(name) => name.parse()?,
None => oxidelake_planner::cluster_target_from_env()?,
};
tracing::info!(%target, "placement target for this cluster");
tracing::info!(
build = %oxidelake_planner::codec::build_description(),
"admitting workers of this build only"
);
serve_metrics(cli.metrics_port, &cli.metrics_host).await?;
cluster::run_scheduler_with_shutdown(
config(&cli, target),
signal,
Duration::from_secs(cli.drain_timeout),
)
.await?;
Ok(())
}
#[cfg(feature = "metrics")]
async fn serve_metrics(port: Option<u16>, host: &str) -> anyhow::Result<()> {
use std::net::ToSocketAddrs;
let Some(port) = port else { return Ok(()) };
let addr = (host, port)
.to_socket_addrs()?
.next()
.ok_or_else(|| anyhow::anyhow!("--metrics-host '{host}' resolved to no address"))?;
let hub = std::sync::Arc::clone(oxidelake_core::telemetry::TelemetryHub::global());
let (bound, server) = oxidelake_runtime::metrics::serve(addr, hub).await?;
tracing::info!(%bound, path = oxidelake_runtime::metrics::METRICS_PATH, "serving metrics");
tokio::spawn(server);
Ok(())
}
#[cfg(not(feature = "metrics"))]
async fn serve_metrics(port: Option<u16>, _host: &str) -> anyhow::Result<()> {
match port {
None => Ok(()),
Some(_) => anyhow::bail!(
"--metrics-port needs the `metrics` feature; rebuild with \
`cargo build -p oxidelake-runtime --features metrics`"
),
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
use super::*;
fn parse(args: &[&str]) -> Result<Cli, clap::Error> {
Cli::try_parse_from(std::iter::once("oxide-scheduler").chain(args.iter().copied()))
}
#[test]
fn the_retry_and_timeout_flags_reach_the_scheduler() {
let cli = parse(&[
"--task-max-failures",
"2",
"--stage-max-failures",
"1",
"--worker-timeout",
"9",
])
.unwrap();
let config = config(&cli, BackendKind::CpuSimd);
assert_eq!(config.task_max_failures, 2);
assert_eq!(config.stage_max_failures, 1);
assert_eq!(config.executor_timeout_seconds, 9);
assert_eq!(config.expire_dead_executor_interval_seconds, 3);
assert_eq!(config.no_executors_grace_period_seconds, 9);
}
#[test]
fn the_defaults_are_ballistas_but_a_lost_worker_is_not_retried() {
let ballista = SchedulerConfig::default();
let config = config(&parse(&[]).unwrap(), BackendKind::CpuSimd);
assert_eq!(config.task_max_failures, ballista.task_max_failures);
assert_eq!(ballista.stage_max_failures, 4, "Ballista's default moved");
assert_eq!(config.stage_max_failures, 1);
assert_eq!(
config.executor_timeout_seconds,
ballista.executor_timeout_seconds
);
assert_eq!(
config.expire_dead_executor_interval_seconds,
ballista.expire_dead_executor_interval_seconds
);
assert_eq!(
config.no_executors_grace_period_seconds,
ballista.no_executors_grace_period_seconds
);
}
#[test]
fn a_zero_retry_limit_or_timeout_is_refused() {
for flag in ["--task-max-failures", "--stage-max-failures"] {
let err = parse(&[flag, "0"]).err().expect(flag).to_string();
assert!(err.contains("1 means no retry"), "{err}");
}
assert!(parse(&["--worker-timeout", "0"]).is_err());
}
}