use std::io;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use ballista_core::serde::protobuf::job_status;
use ballista_scheduler::cluster::JobState;
use ballista_scheduler::scheduler_server::SessionBuilder;
use datafusion::error::DataFusionError;
use tokio::time::Instant;
pub(crate) const SHUTTING_DOWN: &str = "this scheduler is shutting down: it finishes the \
queries it is running and takes no new ones; send this query to another scheduler, or \
again once this one has restarted";
const LINGER: Duration = Duration::from_secs(1);
const POLL: Duration = Duration::from_millis(100);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum Stopped {
Drained,
Abandoned,
}
#[derive(Debug)]
pub struct ShutdownSignal {
#[cfg(unix)]
terminate: tokio::signal::unix::Signal,
#[cfg(unix)]
interrupt: tokio::signal::unix::Signal,
}
impl ShutdownSignal {
pub fn listen() -> io::Result<Self> {
#[cfg(unix)]
{
use tokio::signal::unix::{SignalKind, signal};
Ok(Self {
terminate: signal(SignalKind::terminate())?,
interrupt: signal(SignalKind::interrupt())?,
})
}
#[cfg(not(unix))]
{
Ok(Self {})
}
}
pub async fn recv(&mut self) -> &'static str {
#[cfg(unix)]
{
tokio::select! {
_ = self.terminate.recv() => "SIGTERM",
_ = self.interrupt.recv() => "SIGINT",
}
}
#[cfg(not(unix))]
{
match tokio::signal::ctrl_c().await {
Ok(()) => "Ctrl-C",
Err(_) => std::future::pending().await,
}
}
}
}
pub(crate) fn refusing_while_draining(
builder: SessionBuilder,
draining: Arc<AtomicBool>,
) -> SessionBuilder {
Arc::new(move |config| {
if draining.load(Ordering::SeqCst) {
Err(DataFusionError::Execution(SHUTTING_DOWN.to_owned()))
} else {
builder(config)
}
})
}
pub(crate) async fn unfinished_jobs(jobs: &dyn JobState) -> usize {
let Ok(ids) = jobs.get_all_jobs().await else {
return 0;
};
let mut unfinished = 0;
for id in ids {
let finished = matches!(
jobs.get_job_status(&id).await,
Ok(Some(ballista_core::serde::protobuf::JobStatus {
status: Some(job_status::Status::Successful(_) | job_status::Status::Failed(_)),
..
}))
);
if !finished {
unfinished += 1;
}
}
unfinished
}
pub(crate) async fn drain(jobs: &dyn JobState, timeout: Duration) -> usize {
let deadline = Instant::now() + timeout;
let mut idle_since: Option<Instant> = None;
loop {
let unfinished = unfinished_jobs(jobs).await;
let now = Instant::now();
if unfinished == 0 {
let since = *idle_since.get_or_insert(now);
if now.duration_since(since) >= LINGER {
return 0;
}
} else {
idle_since = None;
}
if now >= deadline {
return unfinished;
}
tokio::time::sleep(POLL.min(deadline - now)).await;
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
use super::*;
use ballista_scheduler::cluster::memory::InMemoryJobState;
use datafusion::prelude::SessionConfig;
fn job_state() -> InMemoryJobState {
InMemoryJobState::new(
"scheduler",
Arc::new(ballista_core::utils::default_session_builder),
Arc::new(SessionConfig::new),
)
}
#[test]
fn a_draining_scheduler_refuses_a_new_query_by_saying_so() {
let draining = Arc::new(AtomicBool::new(false));
let builder: SessionBuilder = Arc::new(ballista_core::utils::default_session_builder);
let builder = refusing_while_draining(builder, Arc::clone(&draining));
builder(SessionConfig::new()).expect("not draining yet");
draining.store(true, Ordering::SeqCst);
let err = builder(SessionConfig::new()).unwrap_err().to_string();
assert!(err.contains("shutting down"), "{err}");
}
#[tokio::test]
async fn the_drain_waits_for_an_unfinished_query_and_no_longer() {
let jobs = job_state();
jobs.accept_job(&"job-1".into(), "q", 0).unwrap();
assert_eq!(unfinished_jobs(&jobs).await, 1);
let started = Instant::now();
assert_eq!(drain(&jobs, Duration::from_millis(400)).await, 1);
let waited = started.elapsed();
assert!(waited >= Duration::from_millis(400), "{waited:?}");
assert!(waited < Duration::from_secs(5), "{waited:?}");
jobs.fail_unscheduled_job(&"job-1".into(), "planning failed".to_owned())
.await
.unwrap();
assert_eq!(
unfinished_jobs(&jobs).await,
0,
"a failed query is finished"
);
let started = Instant::now();
assert_eq!(drain(&jobs, Duration::from_secs(30)).await, 0);
let waited = started.elapsed();
assert!(waited >= LINGER, "{waited:?}");
assert!(waited < Duration::from_secs(10), "{waited:?}");
}
#[tokio::test]
async fn a_zero_drain_timeout_stops_at_once() {
let jobs = job_state();
jobs.accept_job(&"job-1".into(), "q", 0).unwrap();
let started = Instant::now();
assert_eq!(drain(&jobs, Duration::ZERO).await, 1);
assert!(started.elapsed() < Duration::from_secs(1));
}
}