use crate::{
Result,
error::HammerworkError,
job::{Job, JobId},
queue::DatabaseQueue,
};
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SpawnResult {
pub parent_job_id: JobId,
pub spawned_jobs: Vec<JobId>,
pub spawned_at: DateTime<Utc>,
pub spawn_operation_id: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SpawnConfig {
pub max_spawn_count: Option<usize>,
pub inherit_priority: bool,
pub inherit_retry_strategy: bool,
pub inherit_timeout: bool,
pub inherit_trace_context: bool,
pub operation_id: Option<String>,
}
impl Default for SpawnConfig {
fn default() -> Self {
Self {
max_spawn_count: Some(100), inherit_priority: true,
inherit_retry_strategy: true,
inherit_timeout: false, inherit_trace_context: true,
operation_id: None,
}
}
}
pub struct SpawnContext<DB: sqlx::Database> {
pub parent_job: Job,
pub config: SpawnConfig,
pub queue: Arc<dyn DatabaseQueue<Database = DB> + Send + Sync>,
}
#[derive(Debug, thiserror::Error)]
pub enum SpawnError {
#[error("Spawn limit exceeded: attempted to spawn {attempted} jobs, limit is {limit}")]
SpawnLimitExceeded { attempted: usize, limit: usize },
#[error("Invalid spawn configuration: {message}")]
InvalidConfig { message: String },
#[error("Parent job {parent_id} is not eligible for spawning")]
ParentNotEligible { parent_id: JobId },
#[error("Spawn operation failed: {message}")]
SpawnOperationFailed { message: String },
}
#[async_trait]
pub trait SpawnHandler<DB: sqlx::Database>: Send + Sync {
async fn spawn_jobs(&self, context: SpawnContext<DB>) -> Result<Vec<Job>>;
async fn validate_spawn(&self, _parent_job: &Job, _config: &SpawnConfig) -> Result<()> {
Ok(())
}
async fn on_spawn_complete(&self, _result: &SpawnResult) -> Result<()> {
Ok(())
}
}
pub struct ClosureSpawnHandler<F, DB: sqlx::Database> {
handler: F,
_phantom: std::marker::PhantomData<DB>,
}
impl<F, DB> ClosureSpawnHandler<F, DB>
where
F: Fn(SpawnContext<DB>) -> Result<Vec<Job>> + Send + Sync,
DB: sqlx::Database + Send + Sync,
{
pub fn new(handler: F) -> Self {
Self {
handler,
_phantom: std::marker::PhantomData,
}
}
}
#[async_trait]
impl<F, DB> SpawnHandler<DB> for ClosureSpawnHandler<F, DB>
where
F: Fn(SpawnContext<DB>) -> Result<Vec<Job>> + Send + Sync,
DB: sqlx::Database + Send + Sync,
{
async fn spawn_jobs(&self, context: SpawnContext<DB>) -> Result<Vec<Job>> {
(self.handler)(context)
}
}
pub struct SpawnManager<DB: sqlx::Database> {
handlers: std::collections::HashMap<String, Arc<dyn SpawnHandler<DB>>>,
_phantom: std::marker::PhantomData<DB>,
}
impl<DB: sqlx::Database> SpawnManager<DB> {
pub fn new() -> Self {
Self {
handlers: std::collections::HashMap::new(),
_phantom: std::marker::PhantomData,
}
}
pub fn register_handler<H>(&mut self, job_type: impl Into<String>, handler: H)
where
H: SpawnHandler<DB> + 'static,
{
self.handlers.insert(job_type.into(), Arc::new(handler));
}
pub async fn execute_spawn(
&self,
job: Job,
config: SpawnConfig,
queue: Arc<dyn DatabaseQueue<Database = DB> + Send + Sync>,
) -> Result<Option<SpawnResult>> {
let Some(child_jobs) = self.prepare_spawn(&job, &config, queue.clone()).await? else {
return Ok(None);
};
let mut spawned_job_ids = Vec::new();
for child_job in child_jobs {
let job_id = queue.enqueue(child_job).await?;
spawned_job_ids.push(job_id);
}
let spawn_result = SpawnResult {
parent_job_id: job.id,
spawned_jobs: spawned_job_ids,
spawned_at: Utc::now(),
spawn_operation_id: config.operation_id.clone(),
};
self.notify_spawn_complete(&job, &spawn_result).await?;
Ok(Some(spawn_result))
}
pub async fn prepare_spawn(
&self,
job: &Job,
config: &SpawnConfig,
queue: Arc<dyn DatabaseQueue<Database = DB> + Send + Sync>,
) -> Result<Option<Vec<Job>>> {
let Some(handler) = self.handlers.get(&job.queue_name) else {
return Ok(None);
};
handler.validate_spawn(job, config).await?;
let context = SpawnContext {
parent_job: job.clone(),
config: config.clone(),
queue,
};
let mut child_jobs = handler.spawn_jobs(context).await?;
if let Some(max_count) = config.max_spawn_count
&& child_jobs.len() > max_count
{
return Err(HammerworkError::SpawnError(
SpawnError::SpawnLimitExceeded {
attempted: child_jobs.len(),
limit: max_count,
},
));
}
fn inherit<T: Clone>(child: &mut Option<T>, parent: &Option<T>) {
if parent.is_some() {
child.clone_from(parent);
}
}
for child_job in &mut child_jobs {
if config.inherit_priority {
child_job.priority = job.priority;
}
if config.inherit_retry_strategy {
inherit(&mut child_job.retry_strategy, &job.retry_strategy);
}
if config.inherit_timeout {
inherit(&mut child_job.timeout, &job.timeout);
}
if config.inherit_trace_context {
inherit(&mut child_job.trace_id, &job.trace_id);
inherit(&mut child_job.correlation_id, &job.correlation_id);
inherit(&mut child_job.parent_span_id, &job.parent_span_id);
inherit(&mut child_job.span_context, &job.span_context);
}
if !child_job.depends_on.contains(&job.id) {
child_job.depends_on.push(job.id);
}
child_job.workflow_id = job.workflow_id;
child_job.workflow_name = job.workflow_name.clone();
}
Ok(Some(child_jobs))
}
pub async fn notify_spawn_complete(&self, parent: &Job, result: &SpawnResult) -> Result<()> {
match self.handlers.get(&parent.queue_name) {
Some(handler) => handler.on_spawn_complete(result).await,
None => Ok(()),
}
}
pub fn has_handler(&self, job_type: &str) -> bool {
self.handlers.contains_key(job_type)
}
pub fn registered_types(&self) -> Vec<String> {
self.handlers.keys().cloned().collect()
}
}
impl<DB: sqlx::Database> Default for SpawnManager<DB> {
fn default() -> Self {
Self::new()
}
}
pub trait JobSpawnExt {
fn with_spawn_config(self, config: SpawnConfig) -> Result<Self>
where
Self: Sized;
fn with_spawning(self) -> Result<Self>
where
Self: Sized;
}
impl JobSpawnExt for Job {
fn with_spawn_config(mut self, config: SpawnConfig) -> Result<Self> {
let value = serde_json::to_value(config)?;
let kind = json_kind(&self.payload);
let payload_obj =
self.payload
.as_object_mut()
.ok_or_else(|| HammerworkError::InvalidJobPayload {
message: format!(
"a job's spawn configuration is stored in its payload under \
`{SPAWN_CONFIG_KEY}`, so the payload must be a JSON object (got {kind})"
),
})?;
payload_obj.insert(SPAWN_CONFIG_KEY.to_string(), value);
Ok(self)
}
fn with_spawning(self) -> Result<Self> {
self.with_spawn_config(SpawnConfig::default())
}
}
pub const SPAWN_CONFIG_KEY: &str = "_spawn_config";
fn json_kind(value: &serde_json::Value) -> &'static str {
match value {
serde_json::Value::Null => "null",
serde_json::Value::Bool(_) => "a boolean",
serde_json::Value::Number(_) => "a number",
serde_json::Value::String(_) => "a string",
serde_json::Value::Array(_) => "an array",
serde_json::Value::Object(_) => "an object",
}
}
pub fn spawn_config_from_payload(payload: &serde_json::Value) -> Result<Option<SpawnConfig>> {
match payload.get(SPAWN_CONFIG_KEY) {
None => Ok(None),
Some(value) => serde_json::from_value::<SpawnConfig>(value.clone())
.map(Some)
.map_err(|e| HammerworkError::InvalidJobPayload {
message: format!("invalid `{SPAWN_CONFIG_KEY}` in job payload: {e}"),
}),
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
struct TestSpawnHandler;
#[async_trait]
impl<DB: sqlx::Database> SpawnHandler<DB> for TestSpawnHandler {
async fn spawn_jobs(&self, context: SpawnContext<DB>) -> Result<Vec<Job>> {
let count = context.parent_job.payload["spawn_count"]
.as_u64()
.unwrap_or(1) as usize;
let mut jobs = Vec::new();
for i in 0..count {
let job = Job::new(
"child_task".to_string(),
json!({
"index": i,
"parent_id": context.parent_job.id
}),
);
jobs.push(job);
}
Ok(jobs)
}
}
#[tokio::test]
async fn test_spawn_handler_basic() {
let _handler = TestSpawnHandler;
let _parent_job = Job::new("parent_task".to_string(), json!({"spawn_count": 3}));
}
#[test]
fn test_spawn_config_defaults() {
let config = SpawnConfig::default();
assert_eq!(config.max_spawn_count, Some(100));
assert!(config.inherit_priority);
assert!(config.inherit_retry_strategy);
assert!(!config.inherit_timeout);
assert!(config.inherit_trace_context);
}
#[cfg(feature = "postgres")]
#[test]
fn test_spawn_manager_registration() {
let mut manager: SpawnManager<sqlx::Postgres> = SpawnManager::new();
assert!(!manager.has_handler("test_job"));
manager.register_handler("test_job", TestSpawnHandler);
assert!(manager.has_handler("test_job"));
let types = manager.registered_types();
assert!(types.contains(&"test_job".to_string()));
}
#[test]
fn test_with_spawn_config_stores_config_in_object_payload() {
let job = Job::new("q".to_string(), json!({"a": 1}))
.with_spawn_config(SpawnConfig {
max_spawn_count: Some(7),
operation_id: Some("op".to_string()),
..SpawnConfig::default()
})
.unwrap();
assert_eq!(job.payload["a"], 1);
let config = spawn_config_from_payload(&job.payload).unwrap().unwrap();
assert_eq!(config.max_spawn_count, Some(7));
assert_eq!(config.operation_id.as_deref(), Some("op"));
let job = Job::new("q".to_string(), json!({}))
.with_spawning()
.unwrap();
assert!(job.payload.get(SPAWN_CONFIG_KEY).is_some());
}
#[test]
fn test_with_spawn_config_rejects_non_object_payloads() {
for payload in [
json!(null),
json!(3),
json!("text"),
json!([1, 2]),
json!(true),
] {
let job = Job::new("q".to_string(), payload.clone());
let err = job.with_spawning().unwrap_err();
assert!(
matches!(err, HammerworkError::InvalidJobPayload { .. }),
"{payload}: {err}"
);
assert!(err.to_string().contains("must be a JSON object"), "{err}");
}
}
#[test]
fn test_spawn_config_from_payload() {
assert!(spawn_config_from_payload(&json!({})).unwrap().is_none());
assert!(spawn_config_from_payload(&json!("x")).unwrap().is_none());
let err = spawn_config_from_payload(&json!({"_spawn_config": {"max_spawn_count": "many"}}))
.unwrap_err();
assert!(matches!(err, HammerworkError::InvalidJobPayload { .. }));
assert!(err.to_string().contains("_spawn_config"));
}
#[cfg(all(feature = "test", feature = "postgres"))]
struct OwnSettingsSpawner {
sibling: crate::JobId,
}
#[cfg(all(feature = "test", feature = "postgres"))]
#[async_trait]
impl<DB: sqlx::Database> SpawnHandler<DB> for OwnSettingsSpawner {
async fn spawn_jobs(&self, _context: SpawnContext<DB>) -> Result<Vec<Job>> {
Ok(vec![
Job::new("child".to_string(), json!({}))
.with_retry_strategy(crate::retry::RetryStrategy::fixed(
std::time::Duration::from_secs(7),
))
.with_timeout(std::time::Duration::from_secs(9))
.with_trace_id("child-trace")
.depends_on(&self.sibling),
])
}
}
#[cfg(all(feature = "test", feature = "postgres"))]
#[tokio::test]
async fn test_prepare_spawn_keeps_the_childs_own_settings() {
use crate::queue::test::TestQueue;
let sibling = uuid::Uuid::new_v4();
let mut manager = SpawnManager::<sqlx::Postgres>::new();
manager.register_handler("parent", OwnSettingsSpawner { sibling });
let parent = Job::new("parent".to_string(), json!({}));
assert!(parent.retry_strategy.is_none() && parent.trace_id.is_none());
let config = SpawnConfig {
inherit_timeout: true,
..Default::default()
};
let queue: Arc<dyn DatabaseQueue<Database = sqlx::Postgres> + Send + Sync> =
Arc::new(TestQueue::new());
let children = manager
.prepare_spawn(&parent, &config, queue)
.await
.unwrap()
.unwrap();
let child = &children[0];
assert_eq!(
child.retry_strategy,
Some(crate::retry::RetryStrategy::fixed(
std::time::Duration::from_secs(7)
))
);
assert_eq!(child.timeout, Some(std::time::Duration::from_secs(9)));
assert_eq!(child.trace_id.as_deref(), Some("child-trace"));
assert!(child.depends_on.contains(&parent.id));
assert!(child.depends_on.contains(&sibling));
let parent = Job::new("parent".to_string(), json!({}))
.with_retry_strategy(crate::retry::RetryStrategy::fixed(
std::time::Duration::from_secs(3),
))
.with_trace_id("parent-trace");
let mut manager = SpawnManager::<sqlx::Postgres>::new();
manager.register_handler("parent", TestSpawnHandler);
let queue: Arc<dyn DatabaseQueue<Database = sqlx::Postgres> + Send + Sync> =
Arc::new(TestQueue::new());
let children = manager
.prepare_spawn(&parent, &SpawnConfig::default(), queue)
.await
.unwrap()
.unwrap();
assert_eq!(children[0].retry_strategy, parent.retry_strategy);
assert_eq!(children[0].trace_id.as_deref(), Some("parent-trace"));
assert_eq!(children[0].depends_on, vec![parent.id]);
}
}