use std::{
fmt,
sync::Arc,
time::{Duration, Instant, SystemTime},
};
use datum::{
Flow, KillSwitches, NotUsed, RestartSettings, RunnableGraph, SharedKillSwitch,
StreamCompletion, StreamError, StreamInstrumentationRegistry, StreamResult,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct JobId(pub u64);
pub type JobGraphFactory =
dyn Fn(JobContext) -> StreamResult<RunnableGraph<JobMat>> + Send + Sync + 'static;
#[derive(Clone)]
pub struct JobSpec {
pub name: String,
pub factory: Arc<JobGraphFactory>,
pub restart_policy: JobRestartPolicy,
pub drain_behavior: JobDrainBehavior,
}
impl JobSpec {
#[must_use]
pub fn new<F>(name: impl Into<String>, factory: F) -> Self
where
F: Fn(JobContext) -> StreamResult<RunnableGraph<JobMat>> + Send + Sync + 'static,
{
Self {
name: name.into(),
factory: Arc::new(factory),
restart_policy: JobRestartPolicy::Never,
drain_behavior: JobDrainBehavior::default(),
}
}
#[must_use]
pub fn with_restart_policy(mut self, restart_policy: JobRestartPolicy) -> Self {
self.restart_policy = restart_policy;
self
}
#[must_use]
pub fn with_drain_behavior(mut self, drain_behavior: JobDrainBehavior) -> Self {
self.drain_behavior = drain_behavior;
self
}
}
impl fmt::Debug for JobSpec {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("JobSpec")
.field("name", &self.name)
.field("restart_policy", &self.restart_policy)
.field("drain_behavior", &self.drain_behavior)
.finish_non_exhaustive()
}
}
#[derive(Clone, Debug)]
pub struct JobContext {
name: Arc<str>,
job_id: JobId,
generation: u64,
kill_switch: SharedKillSwitch,
instrumentation: StreamInstrumentationRegistry,
}
impl JobContext {
pub(crate) fn new(
name: impl Into<Arc<str>>,
job_id: JobId,
generation: u64,
kill_switch: SharedKillSwitch,
instrumentation: StreamInstrumentationRegistry,
) -> Self {
Self {
name: name.into(),
job_id,
generation,
kill_switch,
instrumentation,
}
}
#[must_use]
pub fn name(&self) -> &str {
&self.name
}
#[must_use]
pub fn job_id(&self) -> JobId {
self.job_id
}
#[must_use]
pub fn generation(&self) -> u64 {
self.generation
}
#[must_use]
pub fn kill_switch(&self) -> SharedKillSwitch {
self.kill_switch.clone()
}
#[must_use]
pub fn instrumentation_registry(&self) -> &StreamInstrumentationRegistry {
&self.instrumentation
}
#[must_use]
pub fn drain_flow<T: Send + 'static>(&self) -> Flow<T, T, SharedKillSwitch> {
self.kill_switch.flow()
}
#[must_use]
pub fn control(&self) -> JobControl {
JobControl::graceful(self.kill_switch.clone())
}
}
pub struct JobMat {
completion: StreamCompletion<NotUsed>,
control: JobControl,
}
impl JobMat {
#[must_use]
pub fn new(completion: StreamCompletion<NotUsed>, control: JobControl) -> Self {
Self {
completion,
control,
}
}
#[must_use]
pub fn graceful(completion: StreamCompletion<NotUsed>, kill_switch: SharedKillSwitch) -> Self {
Self::new(completion, JobControl::graceful(kill_switch))
}
#[must_use]
pub fn cancel_only(completion: StreamCompletion<NotUsed>) -> Self {
Self::new(completion, JobControl::cancel_only())
}
pub(crate) fn into_parts(self) -> (StreamCompletion<NotUsed>, JobControl) {
(self.completion, self.control)
}
}
impl fmt::Debug for JobMat {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("JobMat")
.field("control", &self.control)
.finish_non_exhaustive()
}
}
#[derive(Clone, Debug)]
pub struct JobControl {
kill_switch: Option<SharedKillSwitch>,
}
impl JobControl {
#[must_use]
pub fn graceful(kill_switch: SharedKillSwitch) -> Self {
Self {
kill_switch: Some(kill_switch),
}
}
#[must_use]
pub fn cancel_only() -> Self {
Self { kill_switch: None }
}
#[must_use]
pub fn drain_supported(&self) -> bool {
self.kill_switch.is_some()
}
#[must_use]
pub fn kill_switch(&self) -> Option<SharedKillSwitch> {
self.kill_switch.clone()
}
pub(crate) fn shutdown(&self) -> bool {
if let Some(kill_switch) = &self.kill_switch {
kill_switch.shutdown();
true
} else {
false
}
}
pub(crate) fn abort(&self, error: StreamError) -> bool {
if let Some(kill_switch) = &self.kill_switch {
kill_switch.abort(error);
true
} else {
false
}
}
}
#[derive(Clone, Debug)]
pub enum JobRestartPolicy {
Never,
OnFailure(RestartSettings),
Always(RestartSettings),
Manual,
}
impl JobRestartPolicy {
#[must_use]
pub fn never() -> Self {
Self::Never
}
#[must_use]
pub fn on_failure(settings: RestartSettings) -> Self {
Self::OnFailure(settings)
}
#[must_use]
pub fn always(settings: RestartSettings) -> Self {
Self::Always(settings)
}
#[must_use]
pub fn manual() -> Self {
Self::Manual
}
pub(crate) fn settings_for(&self, cause: RestartCause) -> Option<RestartSettings> {
match (self, cause) {
(Self::Always(settings), RestartCause::Failure | RestartCause::Completion)
| (Self::OnFailure(settings), RestartCause::Failure) => Some(settings.clone()),
(Self::Never | Self::Manual | Self::OnFailure(_), _) => None,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum RestartCause {
Failure,
Completion,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum JobDrainBehavior {
Graceful { timeout: Duration },
CancelOnly,
}
impl JobDrainBehavior {
#[must_use]
pub fn graceful(timeout: Duration) -> Self {
Self::Graceful { timeout }
}
#[must_use]
pub fn cancel_only() -> Self {
Self::CancelOnly
}
#[must_use]
pub fn timeout(&self) -> Option<Duration> {
match self {
Self::Graceful { timeout } => Some(*timeout),
Self::CancelOnly => None,
}
}
}
impl Default for JobDrainBehavior {
fn default() -> Self {
Self::Graceful {
timeout: Duration::from_secs(30),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DesiredJobState {
Running,
Draining,
Stopped,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum JobState {
Submitted,
Starting,
Running,
Draining,
BackingOff,
Completed,
Drained,
Stopped,
Failed,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum JobExitReason {
Completed,
Failed(StreamError),
Drained,
Stopped,
DrainTimedOut,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct JobInstrumentationSnapshot {}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct JobStatus {
pub name: String,
pub job_id: JobId,
pub state: JobState,
pub desired_state: DesiredJobState,
pub generation: u64,
pub starts_total: u64,
pub restarts_total: u64,
pub last_start_at: Option<SystemTime>,
pub last_exit_at: Option<SystemTime>,
pub last_exit_reason: Option<JobExitReason>,
pub backoff_until: Option<Instant>,
pub drain_deadline: Option<Instant>,
pub drain_supported: bool,
pub active_streams: Option<usize>,
pub instrumentation: Option<JobInstrumentationSnapshot>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct JobEvent {
pub sequence: u64,
pub timestamp: SystemTime,
pub name: String,
pub job_id: JobId,
pub generation: u64,
pub kind: JobEventKind,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum JobEventKind {
Submitted,
Started,
Failed { reason: JobExitReason },
RestartScheduled { delay: Duration },
Restarted { previous_generation: u64 },
Draining,
Drained,
Stopped { reason: JobExitReason },
Completed,
}
pub(crate) fn new_generation_kill_switch(name: &str, generation: u64) -> SharedKillSwitch {
KillSwitches::shared(format!("job:{name}:{generation}"))
}