use crate::error::{LaneError, Result};
use crate::retry::RetryPolicy;
use chrono::{DateTime, Utc};
use cron::Schedule;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::BTreeMap;
use std::str::FromStr;
use std::time::Duration;
use uuid::Uuid;
pub type JobId = String;
pub type QueueName = String;
pub type JobWorkerId = String;
pub type JobLockToken = String;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobLeaseRenewal {
pub job_id: JobId,
pub lock_token: JobLockToken,
}
impl JobLeaseRenewal {
pub fn new(job_id: impl Into<JobId>, lock_token: impl Into<JobLockToken>) -> Self {
Self {
job_id: job_id.into(),
lock_token: lock_token.into(),
}
}
}
pub type JobPriority = u32;
pub const DEFAULT_JOB_PRIORITY: JobPriority = 1000;
pub const MAX_JOB_PRIORITY: JobPriority = 2_u32.pow(21);
pub(crate) fn validate_job_priority(priority: JobPriority) -> Result<()> {
if priority > MAX_JOB_PRIORITY {
return Err(LaneError::ConfigError(format!(
"priority must be between 0 and {MAX_JOB_PRIORITY}"
)));
}
Ok(())
}
pub const DEFAULT_JOB_EVENT_RETENTION: usize = 10_000;
pub const DEFAULT_JOB_METRICS_RETENTION: usize = 10_000;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobPriorityCount {
pub priority: JobPriority,
pub count: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobMetricsMeta {
pub count: usize,
pub previous_timestamp_millis: i64,
pub previous_count: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobMetrics {
pub meta: JobMetricsMeta,
pub data: Vec<usize>,
pub count: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct JobEvent {
pub id: String,
pub event: String,
pub timestamp: DateTime<Utc>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub job_id: Option<JobId>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub prev: Option<JobState>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub fields: BTreeMap<String, Value>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobRateLimit {
pub max_claims: u64,
pub window: Duration,
}
impl JobRateLimit {
pub fn new(max_claims: u64, window: Duration) -> Self {
Self { max_claims, window }
}
pub fn per_second(max_claims: u64) -> Self {
Self::new(max_claims, Duration::from_secs(1))
}
pub fn per_minute(max_claims: u64) -> Self {
Self::new(max_claims, Duration::from_secs(60))
}
pub fn validate(&self) -> Result<()> {
if self.max_claims == 0 {
return Err(LaneError::ConfigError(
"job claim rate limit max_claims must be greater than zero".to_string(),
));
}
if self.window.is_zero() {
return Err(LaneError::ConfigError(
"job claim rate limit window must be greater than zero".to_string(),
));
}
Ok(())
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum JobState {
Waiting,
Delayed,
Active,
WaitingChildren,
Completed,
Failed,
}
impl JobState {
pub const ALL: [Self; 6] = [
Self::Waiting,
Self::Delayed,
Self::Active,
Self::WaitingChildren,
Self::Completed,
Self::Failed,
];
pub const PENDING: [Self; 3] = [Self::Waiting, Self::Delayed, Self::WaitingChildren];
pub fn is_terminal(self) -> bool {
matches!(self, Self::Completed | Self::Failed)
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobStateCount {
pub state: JobState,
pub count: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct JobLogEntry {
pub timestamp: DateTime<Utc>,
pub line: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct JobLogPage {
pub logs: Vec<JobLogEntry>,
pub count: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobListOptions {
pub state: Option<JobState>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub states: Vec<JobState>,
pub offset: usize,
pub limit: usize,
#[serde(default = "default_list_ascending")]
pub ascending: bool,
}
impl Default for JobListOptions {
fn default() -> Self {
Self {
state: None,
states: Vec::new(),
offset: 0,
limit: 100,
ascending: true,
}
}
}
impl JobListOptions {
pub fn new() -> Self {
Self::default()
}
pub fn with_state(mut self, state: JobState) -> Self {
self.state = Some(state);
self.states.clear();
self
}
pub fn with_states(mut self, states: impl IntoIterator<Item = JobState>) -> Self {
self.state = None;
self.states.clear();
for state in states {
if !self.states.contains(&state) {
self.states.push(state);
}
}
self
}
pub fn with_offset(mut self, offset: usize) -> Self {
self.offset = offset;
self
}
pub fn with_limit(mut self, limit: usize) -> Self {
self.limit = limit;
self
}
pub fn ascending(mut self) -> Self {
self.ascending = true;
self
}
pub fn descending(mut self) -> Self {
self.ascending = false;
self
}
pub fn with_ascending(mut self, ascending: bool) -> Self {
self.ascending = ascending;
self
}
pub(crate) fn selected_states(&self) -> Vec<JobState> {
let mut states = Vec::new();
if !self.states.is_empty() {
for &state in &self.states {
if !states.contains(&state) {
states.push(state);
}
}
} else if let Some(state) = self.state {
states.push(state);
} else {
states.extend_from_slice(JobState::ALL.as_slice());
}
states
}
}
fn default_list_ascending() -> bool {
true
}
fn default_repeat_list_ascending() -> bool {
false
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct JobListPage {
pub jobs: Vec<Job>,
pub total: usize,
pub offset: usize,
pub limit: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct JobSpec {
pub name: String,
pub payload: Value,
pub options: JobOptions,
}
impl JobSpec {
pub fn new(name: impl Into<String>, payload: Value) -> Self {
Self {
name: name.into(),
payload,
options: JobOptions::new(),
}
}
pub fn with_options(mut self, options: JobOptions) -> Self {
self.options = options;
self
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct JobFlow {
pub parent: Job,
pub children: Vec<Job>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct JobFlowDependencies {
pub parent: Job,
pub children: Vec<Job>,
pub pending_child_ids: Vec<JobId>,
pub missing_child_ids: Vec<JobId>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobFlowDependencyCounts {
pub processed: usize,
pub unprocessed: usize,
pub failed: usize,
pub ignored: usize,
pub missing: usize,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobFlowDependencyCountOptions {
pub processed: bool,
pub unprocessed: bool,
pub ignored: bool,
pub failed: bool,
}
impl JobFlowDependencyCountOptions {
pub fn new() -> Self {
Self::default()
}
pub fn all() -> Self {
Self {
processed: true,
unprocessed: true,
ignored: true,
failed: true,
}
}
pub fn with_processed(mut self, enabled: bool) -> Self {
self.processed = enabled;
self
}
pub fn with_unprocessed(mut self, enabled: bool) -> Self {
self.unprocessed = enabled;
self
}
pub fn with_ignored(mut self, enabled: bool) -> Self {
self.ignored = enabled;
self
}
pub fn with_failed(mut self, enabled: bool) -> Self {
self.failed = enabled;
self
}
pub fn with_kind(mut self, kind: JobFlowDependencyKind, enabled: bool) -> Self {
match kind {
JobFlowDependencyKind::Processed => self.processed = enabled,
JobFlowDependencyKind::Unprocessed => self.unprocessed = enabled,
JobFlowDependencyKind::Ignored => self.ignored = enabled,
JobFlowDependencyKind::Failed => self.failed = enabled,
}
self
}
pub(crate) fn selected(&self) -> Vec<JobFlowDependencyKind> {
let mut kinds = Vec::new();
if self.processed {
kinds.push(JobFlowDependencyKind::Processed);
}
if self.unprocessed {
kinds.push(JobFlowDependencyKind::Unprocessed);
}
if self.ignored {
kinds.push(JobFlowDependencyKind::Ignored);
}
if self.failed {
kinds.push(JobFlowDependencyKind::Failed);
}
if kinds.is_empty() {
kinds.extend_from_slice(&[
JobFlowDependencyKind::Processed,
JobFlowDependencyKind::Unprocessed,
JobFlowDependencyKind::Ignored,
JobFlowDependencyKind::Failed,
]);
}
kinds
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobFlowDependencySelectedCounts {
pub processed: Option<usize>,
pub unprocessed: Option<usize>,
pub ignored: Option<usize>,
pub failed: Option<usize>,
}
impl JobFlowDependencySelectedCounts {
pub fn get(&self, kind: JobFlowDependencyKind) -> Option<usize> {
match kind {
JobFlowDependencyKind::Processed => self.processed,
JobFlowDependencyKind::Unprocessed => self.unprocessed,
JobFlowDependencyKind::Ignored => self.ignored,
JobFlowDependencyKind::Failed => self.failed,
}
}
pub(crate) fn insert(&mut self, kind: JobFlowDependencyKind, count: usize) {
match kind {
JobFlowDependencyKind::Processed => self.processed = Some(count),
JobFlowDependencyKind::Unprocessed => self.unprocessed = Some(count),
JobFlowDependencyKind::Ignored => self.ignored = Some(count),
JobFlowDependencyKind::Failed => self.failed = Some(count),
}
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
pub struct JobFlowDependencyValues {
pub processed: BTreeMap<JobId, Value>,
pub unprocessed: Vec<JobId>,
pub failed: Vec<JobId>,
pub ignored: BTreeMap<JobId, String>,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum JobFlowDependencyKind {
Processed,
Unprocessed,
Ignored,
Failed,
}
#[cfg(feature = "redis-backend")]
impl JobFlowDependencyKind {
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::Processed => "processed",
Self::Unprocessed => "unprocessed",
Self::Ignored => "ignored",
Self::Failed => "failed",
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobFlowDependencyPageOptions {
pub kind: JobFlowDependencyKind,
pub cursor: u64,
pub count: usize,
}
impl JobFlowDependencyPageOptions {
pub fn new(kind: JobFlowDependencyKind) -> Self {
Self {
kind,
cursor: 0,
count: 20,
}
}
pub fn with_cursor(mut self, cursor: u64) -> Self {
self.cursor = cursor;
self
}
pub fn with_count(mut self, count: usize) -> Self {
self.count = count;
self
}
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobFlowDependencyPageCursor {
pub cursor: u64,
pub count: usize,
}
impl Default for JobFlowDependencyPageCursor {
fn default() -> Self {
Self {
cursor: 0,
count: 20,
}
}
}
impl JobFlowDependencyPageCursor {
pub fn new() -> Self {
Self::default()
}
pub fn with_cursor(mut self, cursor: u64) -> Self {
self.cursor = cursor;
self
}
pub fn with_count(mut self, count: usize) -> Self {
self.count = count;
self
}
}
impl From<JobFlowDependencyPageOptions> for JobFlowDependencyPageCursor {
fn from(options: JobFlowDependencyPageOptions) -> Self {
Self {
cursor: options.cursor,
count: options.count,
}
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobFlowDependencyPagesOptions {
pub processed: Option<JobFlowDependencyPageCursor>,
pub unprocessed: Option<JobFlowDependencyPageCursor>,
pub ignored: Option<JobFlowDependencyPageCursor>,
pub failed: Option<JobFlowDependencyPageCursor>,
}
impl JobFlowDependencyPagesOptions {
pub fn new() -> Self {
Self::default()
}
pub fn all() -> Self {
let cursor = JobFlowDependencyPageCursor::default();
Self {
processed: Some(cursor),
unprocessed: Some(cursor),
ignored: Some(cursor),
failed: Some(cursor),
}
}
pub fn with_processed(mut self, cursor: JobFlowDependencyPageCursor) -> Self {
self.processed = Some(cursor);
self
}
pub fn with_unprocessed(mut self, cursor: JobFlowDependencyPageCursor) -> Self {
self.unprocessed = Some(cursor);
self
}
pub fn with_ignored(mut self, cursor: JobFlowDependencyPageCursor) -> Self {
self.ignored = Some(cursor);
self
}
pub fn with_failed(mut self, cursor: JobFlowDependencyPageCursor) -> Self {
self.failed = Some(cursor);
self
}
pub fn with_kind(
mut self,
kind: JobFlowDependencyKind,
cursor: JobFlowDependencyPageCursor,
) -> Self {
match kind {
JobFlowDependencyKind::Processed => self.processed = Some(cursor),
JobFlowDependencyKind::Unprocessed => self.unprocessed = Some(cursor),
JobFlowDependencyKind::Ignored => self.ignored = Some(cursor),
JobFlowDependencyKind::Failed => self.failed = Some(cursor),
}
self
}
pub(crate) fn selected(&self) -> Vec<JobFlowDependencyPageOptions> {
[
(JobFlowDependencyKind::Processed, self.processed),
(JobFlowDependencyKind::Unprocessed, self.unprocessed),
(JobFlowDependencyKind::Ignored, self.ignored),
(JobFlowDependencyKind::Failed, self.failed),
]
.into_iter()
.filter_map(|(kind, cursor)| {
cursor.map(|cursor| JobFlowDependencyPageOptions {
kind,
cursor: cursor.cursor,
count: cursor.count,
})
})
.collect()
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum JobFlowDependencyPageItem {
Processed { child_id: JobId, value: Value },
Unprocessed { child_id: JobId },
Ignored {
child_id: JobId,
failed_reason: String,
},
Failed { child_id: JobId },
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct JobFlowDependencyPage {
pub kind: JobFlowDependencyKind,
pub items: Vec<JobFlowDependencyPageItem>,
pub next_cursor: u64,
pub count: usize,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
pub struct JobFlowDependencyPages {
pub processed: Option<JobFlowDependencyPage>,
pub unprocessed: Option<JobFlowDependencyPage>,
pub ignored: Option<JobFlowDependencyPage>,
pub failed: Option<JobFlowDependencyPage>,
}
impl JobFlowDependencyPages {
pub fn get(&self, kind: JobFlowDependencyKind) -> Option<&JobFlowDependencyPage> {
match kind {
JobFlowDependencyKind::Processed => self.processed.as_ref(),
JobFlowDependencyKind::Unprocessed => self.unprocessed.as_ref(),
JobFlowDependencyKind::Ignored => self.ignored.as_ref(),
JobFlowDependencyKind::Failed => self.failed.as_ref(),
}
}
pub(crate) fn insert(&mut self, page: JobFlowDependencyPage) {
match page.kind {
JobFlowDependencyKind::Processed => self.processed = Some(page),
JobFlowDependencyKind::Unprocessed => self.unprocessed = Some(page),
JobFlowDependencyKind::Ignored => self.ignored = Some(page),
JobFlowDependencyKind::Failed => self.failed = Some(page),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(tag = "status", rename_all = "snake_case")]
pub enum JobFinishedResult {
NotFinished,
Completed {
return_value: Option<Value>,
},
Failed {
failed_reason: Option<String>,
},
}
pub type JobFlowChildValues = BTreeMap<JobId, Value>;
pub type JobFlowIgnoredFailures = BTreeMap<JobId, String>;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(untagged)]
pub enum RepeatSchedule {
Every {
interval: Duration,
},
Cron {
#[serde(rename = "cron")]
expression: String,
},
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct RepeatOptions {
#[serde(flatten)]
pub schedule: RepeatSchedule,
pub limit: Option<u32>,
pub end_at: Option<DateTime<Utc>>,
pub key: Option<String>,
}
impl RepeatOptions {
pub fn every(interval: Duration) -> Self {
Self {
schedule: RepeatSchedule::Every { interval },
limit: None,
end_at: None,
key: None,
}
}
pub fn cron(expression: impl Into<String>) -> Self {
Self {
schedule: RepeatSchedule::Cron {
expression: expression.into(),
},
limit: None,
end_at: None,
key: None,
}
}
pub fn interval(&self) -> Option<Duration> {
match &self.schedule {
RepeatSchedule::Every { interval } => Some(*interval),
RepeatSchedule::Cron { .. } => None,
}
}
pub fn cron_expression(&self) -> Option<&str> {
match &self.schedule {
RepeatSchedule::Every { .. } => None,
RepeatSchedule::Cron { expression } => Some(expression),
}
}
pub fn with_limit(mut self, limit: u32) -> Self {
self.limit = Some(limit);
self
}
pub fn until(mut self, end_at: DateTime<Utc>) -> Self {
self.end_at = Some(end_at);
self
}
pub fn with_key(mut self, key: impl Into<String>) -> Self {
self.key = Some(key.into());
self
}
pub(crate) fn validate(&self) -> Result<()> {
match &self.schedule {
RepeatSchedule::Every { interval } => {
if interval.is_zero() {
return Err(LaneError::ConfigError(
"repeat interval must be greater than zero".to_string(),
));
}
}
RepeatSchedule::Cron { expression } => {
parse_cron_expression(expression)?;
}
}
if self.limit == Some(0) {
return Err(LaneError::ConfigError(
"repeat limit must be greater than zero".to_string(),
));
}
Ok(())
}
pub(crate) fn next_scheduled_at(&self, after: DateTime<Utc>) -> Result<Option<DateTime<Utc>>> {
let scheduled_at = match &self.schedule {
RepeatSchedule::Every { interval } => Some(add_duration(after, *interval)),
RepeatSchedule::Cron { expression } => {
let schedule = parse_cron_expression(expression)?;
schedule.after(&after).next()
}
};
Ok(scheduled_at)
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct JobRepeatEntry {
pub key: String,
pub job_id: JobId,
pub name: String,
pub state: JobState,
pub scheduled_at: DateTime<Utc>,
pub repeat_count: u32,
pub options: RepeatOptions,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobRepeatListOptions {
pub offset: usize,
pub limit: usize,
#[serde(default = "default_repeat_list_ascending")]
pub ascending: bool,
}
impl Default for JobRepeatListOptions {
fn default() -> Self {
Self {
offset: 0,
limit: 100,
ascending: false,
}
}
}
impl JobRepeatListOptions {
pub fn new() -> Self {
Self::default()
}
pub fn with_offset(mut self, offset: usize) -> Self {
self.offset = offset;
self
}
pub fn with_limit(mut self, limit: usize) -> Self {
self.limit = limit;
self
}
pub fn ascending(mut self) -> Self {
self.ascending = true;
self
}
pub fn descending(mut self) -> Self {
self.ascending = false;
self
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct JobRepeatPage {
pub repeats: Vec<JobRepeatEntry>,
pub total: usize,
pub offset: usize,
pub limit: usize,
}
pub(crate) fn page_repeat_entries(
mut repeats: Vec<JobRepeatEntry>,
options: JobRepeatListOptions,
) -> JobRepeatPage {
repeats.sort_by(|a, b| {
let order = a
.scheduled_at
.cmp(&b.scheduled_at)
.then_with(|| a.key.cmp(&b.key))
.then_with(|| a.job_id.cmp(&b.job_id));
if options.ascending {
order
} else {
order.reverse()
}
});
let total = repeats.len();
let page = repeats
.into_iter()
.skip(options.offset)
.take(options.limit)
.collect();
JobRepeatPage {
repeats: page,
total,
offset: options.offset,
limit: options.limit,
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct DeduplicationOptions {
pub id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ttl: Option<Duration>,
#[serde(default)]
pub replace: bool,
#[serde(default)]
pub keep_last_if_active: bool,
#[serde(default)]
pub extend: bool,
}
impl DeduplicationOptions {
pub fn new(id: impl Into<String>) -> Self {
Self {
id: id.into(),
ttl: None,
replace: false,
keep_last_if_active: false,
extend: false,
}
}
pub fn with_ttl(mut self, ttl: Duration) -> Self {
self.ttl = Some(ttl);
self
}
pub fn replace_delayed(mut self, replace: bool) -> Self {
self.replace = replace;
self
}
pub fn keep_last_if_active(mut self, keep: bool) -> Self {
self.keep_last_if_active = keep;
self
}
pub fn extend_ttl(mut self, extend: bool) -> Self {
self.extend = extend;
self
}
pub(crate) fn validate(&self) -> Result<()> {
if self.id.trim().is_empty() {
return Err(LaneError::ConfigError(
"deduplication id must not be empty".to_string(),
));
}
if matches!(self.ttl, Some(ttl) if ttl.is_zero()) {
return Err(LaneError::ConfigError(
"deduplication ttl must be greater than zero".to_string(),
));
}
Ok(())
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobRetention {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub age: Option<Duration>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub count: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub limit: Option<usize>,
}
impl JobRetention {
pub fn count(count: usize) -> Self {
Self {
age: None,
count: Some(count),
limit: None,
}
}
pub fn age(age: Duration) -> Self {
Self {
age: Some(age),
count: None,
limit: None,
}
}
pub fn age_and_count(age: Duration, count: usize) -> Self {
Self {
age: Some(age),
count: Some(count),
limit: None,
}
}
pub fn with_age(mut self, age: Duration) -> Self {
self.age = Some(age);
self
}
pub fn with_count(mut self, count: usize) -> Self {
self.count = Some(count);
self
}
pub fn with_limit(mut self, limit: usize) -> Self {
self.limit = Some(limit);
self
}
pub(crate) fn removes_current(&self) -> bool {
self.count == Some(0)
}
pub(crate) fn validate(&self, field: &str) -> Result<()> {
if self.age.is_none() && self.count.is_none() {
return Err(LaneError::ConfigError(format!(
"{field} must specify an age or count"
)));
}
if matches!(self.age, Some(age) if age.is_zero()) {
return Err(LaneError::ConfigError(format!(
"{field} age must be greater than zero"
)));
}
if self.limit == Some(0) {
return Err(LaneError::ConfigError(format!(
"{field} limit must be greater than zero"
)));
}
Ok(())
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct JobOptions {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub job_id: Option<JobId>,
pub priority: JobPriority,
#[serde(default, skip_serializing_if = "is_false")]
pub lifo: bool,
pub delay: Option<Duration>,
pub retry_policy: RetryPolicy,
pub timeout: Option<Duration>,
pub remove_on_complete: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub completion_retention: Option<JobRetention>,
pub remove_on_fail: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub failure_retention: Option<JobRetention>,
pub max_stalled_count: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub repeat: Option<RepeatOptions>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub deduplication: Option<DeduplicationOptions>,
#[serde(default, skip_serializing_if = "is_false")]
pub ignore_dependency_on_failure: bool,
#[serde(default, skip_serializing_if = "is_false")]
pub remove_dependency_on_failure: bool,
#[serde(default, skip_serializing_if = "is_false")]
pub continue_parent_on_failure: bool,
#[serde(default, skip_serializing_if = "is_false")]
pub fail_parent_on_failure: bool,
}
impl Default for JobOptions {
fn default() -> Self {
Self {
job_id: None,
priority: DEFAULT_JOB_PRIORITY,
lifo: false,
delay: None,
retry_policy: RetryPolicy::none(),
timeout: None,
remove_on_complete: false,
completion_retention: None,
remove_on_fail: false,
failure_retention: None,
max_stalled_count: 1,
repeat: None,
deduplication: None,
ignore_dependency_on_failure: false,
remove_dependency_on_failure: false,
continue_parent_on_failure: false,
fail_parent_on_failure: false,
}
}
}
impl JobOptions {
pub fn new() -> Self {
Self::default()
}
pub fn with_priority(mut self, priority: JobPriority) -> Self {
self.priority = priority;
self
}
pub fn with_lifo(mut self, lifo: bool) -> Self {
self.lifo = lifo;
self
}
pub fn with_job_id(mut self, job_id: impl Into<String>) -> Self {
self.job_id = Some(job_id.into());
self
}
pub fn with_delay(mut self, delay: Duration) -> Self {
self.delay = Some(delay);
self
}
pub fn with_retry_policy(mut self, retry_policy: RetryPolicy) -> Self {
self.retry_policy = retry_policy;
self
}
pub fn with_timeout(mut self, timeout: Duration) -> Self {
self.timeout = Some(timeout);
self
}
pub fn remove_on_complete(mut self, remove: bool) -> Self {
self.remove_on_complete = remove;
self.completion_retention = None;
self
}
pub fn with_completion_retention(mut self, retention: JobRetention) -> Self {
self.remove_on_complete = false;
self.completion_retention = Some(retention);
self
}
pub fn remove_on_fail(mut self, remove: bool) -> Self {
self.remove_on_fail = remove;
self.failure_retention = None;
self
}
pub fn with_failure_retention(mut self, retention: JobRetention) -> Self {
self.remove_on_fail = false;
self.failure_retention = Some(retention);
self
}
pub fn with_max_stalled_count(mut self, count: u32) -> Self {
self.max_stalled_count = count;
self
}
pub fn with_repeat(mut self, repeat: RepeatOptions) -> Self {
self.repeat = Some(repeat);
self
}
pub fn with_deduplication_id(mut self, id: impl Into<String>) -> Self {
self.deduplication = Some(DeduplicationOptions::new(id));
self
}
pub fn with_deduplication(mut self, deduplication: DeduplicationOptions) -> Self {
self.deduplication = Some(deduplication);
self
}
pub fn with_ignore_dependency_on_failure(mut self, ignore: bool) -> Self {
self.ignore_dependency_on_failure = ignore;
self
}
pub fn with_remove_dependency_on_failure(mut self, remove: bool) -> Self {
self.remove_dependency_on_failure = remove;
self
}
pub fn with_continue_parent_on_failure(mut self, continue_parent: bool) -> Self {
self.continue_parent_on_failure = continue_parent;
self
}
pub fn with_fail_parent_on_failure(mut self, fail_parent: bool) -> Self {
self.fail_parent_on_failure = fail_parent;
self
}
pub(crate) fn validate(&self) -> Result<()> {
if let Some(job_id) = self.job_id.as_deref() {
validate_job_id(job_id)?;
}
validate_job_priority(self.priority)?;
if let Some(repeat) = &self.repeat {
repeat.validate()?;
}
if let Some(deduplication) = &self.deduplication {
deduplication.validate()?;
}
if let Some(retention) = &self.completion_retention {
retention.validate("completion retention")?;
}
if let Some(retention) = &self.failure_retention {
retention.validate("failure retention")?;
}
let flow_failure_policies = [
self.ignore_dependency_on_failure,
self.remove_dependency_on_failure,
self.continue_parent_on_failure,
self.fail_parent_on_failure,
]
.into_iter()
.filter(|enabled| *enabled)
.count();
if flow_failure_policies > 1 {
return Err(LaneError::ConfigError(
"flow child failure policies are mutually exclusive".to_string(),
));
}
Ok(())
}
pub(crate) fn removes_completed_immediately(&self) -> bool {
self.remove_on_complete
|| self
.completion_retention
.as_ref()
.is_some_and(JobRetention::removes_current)
}
pub(crate) fn completed_retention(&self) -> Option<&JobRetention> {
if self.remove_on_complete {
return None;
}
self.completion_retention
.as_ref()
.filter(|retention| !retention.removes_current())
}
pub(crate) fn removes_failed_immediately(&self) -> bool {
self.remove_on_fail
|| self
.failure_retention
.as_ref()
.is_some_and(JobRetention::removes_current)
}
pub(crate) fn failed_retention(&self) -> Option<&JobRetention> {
if self.remove_on_fail {
return None;
}
self.failure_retention
.as_ref()
.filter(|retention| !retention.removes_current())
}
}
fn validate_job_id(job_id: &str) -> Result<()> {
if job_id.trim().is_empty() {
return Err(LaneError::ConfigError(
"job id must not be empty".to_string(),
));
}
if job_id == "0" || job_id.starts_with("0:") {
return Err(LaneError::ConfigError(
"job id cannot be `0` or start with `0:`".to_string(),
));
}
if is_bullmq_integer_job_id(job_id) {
return Err(LaneError::ConfigError(
"custom job id cannot be an integer".to_string(),
));
}
Ok(())
}
fn is_bullmq_integer_job_id(job_id: &str) -> bool {
let mut chars = job_id.chars();
match chars.next() {
Some('-') => matches!(chars.next(), Some('1'..='9')) && chars.all(|ch| ch.is_ascii_digit()),
Some('1'..='9') => chars.all(|ch| ch.is_ascii_digit()),
Some('0') => false,
_ => false,
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct Job {
pub id: JobId,
pub queue: QueueName,
pub name: String,
pub payload: Value,
pub options: JobOptions,
pub priority: JobPriority,
pub state: JobState,
pub attempts_made: u32,
pub stalled_count: u32,
pub created_at: DateTime<Utc>,
pub scheduled_at: DateTime<Utc>,
#[serde(default)]
pub enqueued_seq: u64,
pub processed_at: Option<DateTime<Utc>>,
pub finished_at: Option<DateTime<Utc>>,
pub worker_id: Option<JobWorkerId>,
#[serde(default, skip)]
pub lock_token: Option<JobLockToken>,
pub lease_expires_at: Option<DateTime<Utc>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub deferred_failure: Option<String>,
pub failed_reason: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub stacktrace: Vec<String>,
pub return_value: Option<Value>,
pub progress: Option<Value>,
pub logs: Vec<JobLogEntry>,
pub parent_id: Option<JobId>,
pub child_ids: Vec<JobId>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub repeat_key: Option<String>,
#[serde(default)]
pub repeat_count: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub deduplication_expires_at: Option<DateTime<Utc>>,
}
impl Job {
pub(crate) fn new(
queue: QueueName,
name: String,
payload: Value,
options: JobOptions,
now: DateTime<Utc>,
) -> Self {
let scheduled_at = options
.delay
.map(|delay| add_duration(now, delay))
.unwrap_or(now);
let state = if scheduled_at > now {
JobState::Delayed
} else {
JobState::Waiting
};
let repeat_key = options.repeat.as_ref().map(|repeat| {
repeat
.key
.clone()
.unwrap_or_else(|| format!("{queue}:{name}"))
});
let deduplication_expires_at = deduplication_expiration(&options, now);
let id = options
.job_id
.clone()
.unwrap_or_else(|| Uuid::new_v4().to_string());
Self {
id,
queue,
name,
payload,
priority: options.priority,
options,
state,
attempts_made: 0,
stalled_count: 0,
created_at: now,
scheduled_at,
enqueued_seq: 0,
processed_at: None,
finished_at: None,
worker_id: None,
lock_token: None,
lease_expires_at: None,
deferred_failure: None,
failed_reason: None,
stacktrace: Vec::new(),
return_value: None,
progress: None,
logs: Vec::new(),
parent_id: None,
child_ids: Vec::new(),
repeat_key,
repeat_count: 0,
deduplication_expires_at,
}
}
}
fn is_false(value: &bool) -> bool {
!*value
}
fn is_zero(value: &u64) -> bool {
*value == 0
}
pub(crate) fn deduplication_expiration(
options: &JobOptions,
now: DateTime<Utc>,
) -> Option<DateTime<Utc>> {
options
.deduplication
.as_ref()
.filter(|deduplication| !deduplication.keep_last_if_active)
.and_then(|deduplication| deduplication.ttl)
.map(|ttl| add_duration(now, ttl))
}
pub(crate) fn add_duration(at: DateTime<Utc>, duration: Duration) -> DateTime<Utc> {
match chrono::Duration::from_std(duration) {
Ok(delta) => at.checked_add_signed(delta).unwrap_or(at),
Err(_) => at,
}
}
fn parse_cron_expression(expression: &str) -> Result<Schedule> {
let expression = expression.trim();
if expression.is_empty() {
return Err(LaneError::ConfigError(
"repeat cron expression must not be empty".to_string(),
));
}
Schedule::from_str(expression).map_err(|error| {
LaneError::ConfigError(format!(
"invalid repeat cron expression `{expression}`: {error}"
))
})
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct JobQueueStats {
pub total: usize,
pub waiting: usize,
pub delayed: usize,
pub active: usize,
pub waiting_children: usize,
pub completed: usize,
pub failed: usize,
pub paused: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct JobQueueSnapshot {
pub queue: QueueName,
pub paused: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub events: Vec<JobEvent>,
#[serde(default, skip_serializing_if = "is_zero")]
pub event_sequence: u64,
pub jobs: Vec<Job>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub deduplication_next_jobs: Vec<Job>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub deduplication_next_flows: Vec<JobFlow>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub released_deduplication_owners: Vec<(String, JobId)>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub flow_dependency_indexes: Vec<JobFlowDependencyIndex>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)]
pub struct JobFlowDependencyIndex {
pub parent_id: JobId,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub processed: BTreeMap<JobId, Value>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub ignored: BTreeMap<JobId, String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub failed: Vec<JobId>,
}