use std::collections::BTreeMap;
use std::fmt;
use std::future::Future;
use std::marker::PhantomData;
use std::pin::Pin;
use std::sync::Arc;
use std::time::Duration;
use serde::Serialize;
use serde::de::DeserializeOwned;
use serde_json::Value;
use hatchet_proto::root::AssignedAction;
use super::{
BatchConfig, Concurrency, ConditionSet, DesiredLabel, Idempotency, Priority, RateLimit,
StickyStrategy,
};
use crate::batch::{BatchInput, BatchMemberId, BatchOutput, BatchReport, reconcile};
use crate::{Context, DurableContext, EvictionPolicy, TaskError};
#[allow(clippy::type_complexity)]
pub(crate) type ErasedHandler = Arc<
dyn Fn(Context, Value) -> Pin<Box<dyn Future<Output = Result<Value, TaskError>> + Send>>
+ Send
+ Sync,
>;
#[allow(clippy::type_complexity)]
pub(crate) type ErasedBatchHandler = Arc<
dyn Fn(
Context,
AssignedAction,
) -> Pin<Box<dyn Future<Output = Result<BatchReport, TaskError>> + Send>>
+ Send
+ Sync,
>;
#[derive(Clone)]
pub(crate) struct BatchDef {
pub(crate) config: BatchConfig,
pub(crate) broadcast: bool,
pub(crate) handler: ErasedBatchHandler,
}
#[derive(Clone, Default)]
pub(crate) struct TaskOptions {
pub(crate) retries: u32,
pub(crate) backoff: Option<(f32, u32)>,
pub(crate) execution_timeout: Option<Duration>,
pub(crate) schedule_timeout: Option<Duration>,
pub(crate) concurrency: Vec<Concurrency>,
pub(crate) rate_limits: Vec<RateLimit>,
pub(crate) worker_labels: BTreeMap<String, DesiredLabel>,
pub(crate) wait_for: ConditionSet,
pub(crate) skip_if: ConditionSet,
pub(crate) cancel_if: ConditionSet,
pub(crate) eviction_policy: Option<EvictionPolicy>,
}
#[derive(Clone, Default)]
pub(crate) struct WorkflowOptions {
pub(crate) description: Option<String>,
pub(crate) version: Option<String>,
pub(crate) on_events: Vec<String>,
pub(crate) on_crons: Vec<String>,
pub(crate) cron_input: Option<Value>,
pub(crate) default_priority: Option<Priority>,
pub(crate) concurrency: Vec<Concurrency>,
pub(crate) sticky: Option<StickyStrategy>,
pub(crate) idempotency: Option<Idempotency>,
}
impl WorkflowOptions {
pub(crate) fn is_empty(&self) -> bool {
self.description.is_none()
&& self.version.is_none()
&& self.on_events.is_empty()
&& self.on_crons.is_empty()
&& self.cron_input.is_none()
&& self.default_priority.is_none()
&& self.concurrency.is_empty()
&& self.sticky.is_none()
&& self.idempotency.is_none()
}
}
#[derive(Clone)]
pub(crate) struct TaskDef {
pub(crate) name: String,
pub(crate) options: TaskOptions,
pub(crate) parents: Vec<String>,
pub(crate) handler: ErasedHandler,
pub(crate) durable: bool,
pub(crate) batch: Option<BatchDef>,
}
#[derive(Clone)]
pub(crate) struct RegisteredAction {
pub(crate) handler: ErasedHandler,
pub(crate) durable: bool,
pub(crate) eviction_policy: Option<EvictionPolicy>,
pub(crate) batch: Option<ErasedBatchHandler>,
}
#[must_use = "a task builder declares nothing until `handler` is called"]
pub struct TaskBuilder<I, O> {
name: String,
options: TaskOptions,
standalone: WorkflowOptions,
_types: PhantomData<fn(I) -> O>,
}
impl<I, O> fmt::Debug for TaskBuilder<I, O> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("TaskBuilder")
.field("name", &self.name)
.field("retries", &self.options.retries)
.finish_non_exhaustive()
}
}
impl<I, O> TaskBuilder<I, O> {
pub fn new(name: impl Into<String>) -> Self {
Self {
name: name.into(),
options: TaskOptions::default(),
standalone: WorkflowOptions::default(),
_types: PhantomData,
}
}
pub fn retries(mut self, retries: u32) -> Self {
self.options.retries = retries;
self
}
pub fn retry_backoff(mut self, factor: f32, max_seconds: u32) -> Self {
self.options.backoff = Some((factor, max_seconds));
self
}
pub fn execution_timeout(mut self, timeout: Duration) -> Self {
self.options.execution_timeout = Some(timeout);
self
}
pub fn schedule_timeout(mut self, timeout: Duration) -> Self {
self.options.schedule_timeout = Some(timeout);
self
}
pub fn concurrency(mut self, concurrency: Concurrency) -> Self {
self.options.concurrency.push(concurrency);
self
}
pub fn wait_for(mut self, conditions: impl Into<ConditionSet>) -> Self {
self.options.wait_for.extend(conditions.into());
self
}
pub fn skip_if(mut self, conditions: impl Into<ConditionSet>) -> Self {
self.options.skip_if.extend(conditions.into());
self
}
pub fn cancel_if(mut self, conditions: impl Into<ConditionSet>) -> Self {
self.options.cancel_if.extend(conditions.into());
self
}
pub fn workflow_concurrency(mut self, concurrency: Concurrency) -> Self {
self.standalone.concurrency.push(concurrency);
self
}
pub fn rate_limit(mut self, rate_limit: RateLimit) -> Self {
self.options.rate_limits.push(rate_limit);
self
}
pub fn worker_label(mut self, key: impl Into<String>, label: DesiredLabel) -> Self {
self.options.worker_labels.insert(key.into(), label);
self
}
pub fn description(mut self, description: impl Into<String>) -> Self {
self.standalone.description = Some(description.into());
self
}
pub fn version(mut self, version: impl Into<String>) -> Self {
self.standalone.version = Some(version.into());
self
}
pub fn on_events(mut self, keys: impl IntoIterator<Item = impl Into<String>>) -> Self {
self.standalone
.on_events
.extend(keys.into_iter().map(Into::into));
self
}
pub fn cron(mut self, expression: impl Into<String>) -> Self {
self.standalone.on_crons.push(expression.into());
self
}
pub fn cron_input(mut self, input: Value) -> Self {
self.standalone.cron_input = Some(input);
self
}
pub fn default_priority(mut self, priority: Priority) -> Self {
self.standalone.default_priority = Some(priority);
self
}
pub fn sticky(mut self, strategy: StickyStrategy) -> Self {
self.standalone.sticky = Some(strategy);
self
}
pub fn idempotency(mut self, idempotency: Idempotency) -> Self {
self.standalone.idempotency = Some(idempotency);
self
}
pub fn eviction_policy(mut self, policy: EvictionPolicy) -> Self {
self.options.eviction_policy = Some(policy);
self
}
pub fn handler<F, Fut>(self, handler: F) -> Task<I, O>
where
F: Fn(Context, I) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<O, TaskError>> + Send + 'static,
I: DeserializeOwned + Send + 'static,
O: Serialize + 'static,
{
let handler = Arc::new(handler);
let erased: ErasedHandler = Arc::new(move |context, input| {
let handler = Arc::clone(&handler);
Box::pin(async move {
let input: I = match serde_json::from_value(input) {
Ok(input) => input,
Err(e) => {
return Err(TaskError::non_retryable(format!(
"the input does not match the task's input type: {e}"
)));
}
};
let output = handler(context, input).await?;
serde_json::to_value(output).map_err(|e| {
TaskError::non_retryable(format!(
"the output could not be serialized to JSON: {e}"
))
})
})
});
Task {
def: TaskDef {
name: self.name,
options: self.options,
parents: Vec::new(),
handler: erased,
durable: false,
batch: None,
},
standalone: self.standalone,
_types: PhantomData,
}
}
pub fn durable_handler<F, Fut>(self, handler: F) -> Task<I, O>
where
F: Fn(DurableContext, I) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<O, TaskError>> + Send + 'static,
I: DeserializeOwned + Send + 'static,
O: Serialize + 'static,
{
let handler = Arc::new(handler);
let erased: ErasedHandler = Arc::new(move |context, input| {
let handler = Arc::clone(&handler);
Box::pin(async move {
let context = DurableContext::attach(context)?;
let input: I = match serde_json::from_value(input) {
Ok(input) => input,
Err(e) => {
return Err(TaskError::non_retryable(format!(
"the input does not match the task's input type: {e}"
)));
}
};
let output = handler(context, input).await?;
serde_json::to_value(output).map_err(|e| {
TaskError::non_retryable(format!(
"the output could not be serialized to JSON: {e}"
))
})
})
});
Task {
def: TaskDef {
name: self.name,
options: self.options,
parents: Vec::new(),
handler: erased,
durable: true,
batch: None,
},
standalone: self.standalone,
_types: PhantomData,
}
}
pub fn batch_handler<F, Fut>(self, config: BatchConfig, handler: F) -> Task<I, O>
where
F: Fn(Context, BatchInput<I>) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<BatchOutput<O>, TaskError>> + Send + 'static,
I: DeserializeOwned + Send + 'static,
O: Serialize + Send + 'static,
{
let handler = Arc::new(handler);
let name = self.name.clone();
let erased: ErasedBatchHandler = Arc::new(move |context, action| {
let handler = Arc::clone(&handler);
let name = name.clone();
Box::pin(async move {
let (input, mut report) = BatchInput::<I>::decode(&action)?;
let members: Vec<BatchMemberId> = input.ids().cloned().collect();
let output = handler(context, input).await?;
reconcile(&name, &members, output.into_items(), &mut report);
Ok(report)
})
});
self.into_batch_task(config, false, erased)
}
pub fn batch_broadcast_handler<F, Fut>(self, config: BatchConfig, handler: F) -> Task<I, O>
where
F: Fn(Context, BatchInput<I>) -> Fut + Send + Sync + 'static,
Fut: Future<Output = Result<O, TaskError>> + Send + 'static,
I: DeserializeOwned + Send + 'static,
O: Serialize + Send + 'static,
{
let handler = Arc::new(handler);
let name = self.name.clone();
let erased: ErasedBatchHandler = Arc::new(move |context, action| {
let handler = Arc::clone(&handler);
let name = name.clone();
Box::pin(async move {
let (input, mut report) = BatchInput::<I>::decode(&action)?;
let members: Vec<BatchMemberId> = input.ids().cloned().collect();
let output = handler(context, input).await?;
let value = serde_json::to_value(output).map_err(|e| {
TaskError::non_retryable(format!(
"the output could not be serialized to JSON: {e}"
))
})?;
let answered = members
.iter()
.map(|id| (id.clone(), Ok(value.clone())))
.collect();
reconcile(&name, &members, answered, &mut report);
Ok(report)
})
});
self.into_batch_task(config, true, erased)
}
fn into_batch_task(
self,
config: BatchConfig,
broadcast: bool,
handler: ErasedBatchHandler,
) -> Task<I, O> {
let name = self.name;
let stub_name = name.clone();
let stub: ErasedHandler = Arc::new(move |_context, _input| {
let name = stub_name.clone();
Box::pin(async move {
Err(TaskError::non_retryable(format!(
"`{name}` is a batch task, but this run was assigned as an ordinary task run; \
the engine this worker is connected to does not batch it"
)))
})
});
Task {
def: TaskDef {
name,
options: self.options,
parents: Vec::new(),
handler: stub,
durable: false,
batch: Some(BatchDef {
config,
broadcast,
handler,
}),
},
standalone: self.standalone,
_types: PhantomData,
}
}
}
#[must_use = "a declared task runs nothing until it is registered on a worker"]
pub struct Task<I, O> {
pub(crate) def: TaskDef,
pub(crate) standalone: WorkflowOptions,
_types: PhantomData<fn(I) -> O>,
}
impl<I, O> fmt::Debug for Task<I, O> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Task")
.field("name", &self.def.name)
.field("parents", &self.def.parents)
.finish_non_exhaustive()
}
}
impl<I, O> Clone for Task<I, O> {
fn clone(&self) -> Self {
Self {
def: self.def.clone(),
standalone: self.standalone.clone(),
_types: PhantomData,
}
}
}
impl<I, O> Task<I, O> {
pub fn name(&self) -> &str {
&self.def.name
}
pub fn parent<PI, PO>(mut self, parent: &Task<PI, PO>) -> Self {
self.def.parents.push(parent.def.name.clone());
self
}
}
#[cfg(test)]
mod tests {
use super::super::Condition;
use super::*;
use serde::{Deserialize, Serialize};
use serde_json::json;
use std::time::Duration;
#[derive(Deserialize, Serialize)]
struct In {
repo: String,
}
#[derive(Deserialize, Serialize, Debug, PartialEq)]
struct Out {
pr: Option<String>,
}
fn echo_task(name: &str) -> Task<In, Out> {
TaskBuilder::<In, Out>::new(name).handler(|_ctx, input: In| async move {
Ok(Out {
pr: Some(input.repo),
})
})
}
#[test]
fn builder_collects_task_options() {
let task = TaskBuilder::<In, Out>::new("agent-run")
.retries(3)
.retry_backoff(2.0, 60)
.execution_timeout(Duration::from_secs(7200))
.schedule_timeout(Duration::from_secs(300))
.concurrency(Concurrency::expr("input.device_id").max_runs(1))
.rate_limit(RateLimit::new("github-api", 1))
.worker_label("os", DesiredLabel::eq("macos").required())
.handler(|_ctx, _input: In| async move { Ok(Out { pr: None }) });
assert_eq!(task.name(), "agent-run");
let options = &task.def.options;
assert_eq!(options.retries, 3);
assert_eq!(options.backoff, Some((2.0, 60)));
assert_eq!(options.execution_timeout, Some(Duration::from_secs(7200)));
assert_eq!(options.schedule_timeout, Some(Duration::from_secs(300)));
assert_eq!(
options.concurrency,
[Concurrency::expr("input.device_id").max_runs(1)]
);
assert_eq!(options.rate_limits, [RateLimit::new("github-api", 1)]);
assert_eq!(
options.worker_labels["os"],
DesiredLabel::eq("macos").required()
);
assert!(task.standalone.is_empty());
assert!(task.def.parents.is_empty());
}
#[test]
fn builder_collects_standalone_trigger_options() {
let task = TaskBuilder::<In, Out>::new("nightly")
.description("Nightly sweep")
.version("v2")
.on_events(["approval.granted", "approval.revoked"])
.cron("0 * * * *")
.cron("30 2 * * *")
.cron_input(json!({"repo": "hatchet-rs"}))
.default_priority(Priority::High)
.handler(|_ctx, _input: In| async move { Ok(Out { pr: None }) });
let standalone = &task.standalone;
assert!(!standalone.is_empty());
assert_eq!(standalone.description.as_deref(), Some("Nightly sweep"));
assert_eq!(standalone.version.as_deref(), Some("v2"));
assert_eq!(
standalone.on_events,
["approval.granted", "approval.revoked"]
);
assert_eq!(standalone.on_crons, ["0 * * * *", "30 2 * * *"]);
assert_eq!(standalone.cron_input, Some(json!({"repo": "hatchet-rs"})));
assert_eq!(standalone.default_priority, Some(Priority::High));
}
#[test]
fn builder_collects_conditions_per_gate_and_accumulates() {
let task = TaskBuilder::<In, Out>::new("gated")
.wait_for(Condition::sleep(Duration::from_secs(5)))
.wait_for(Condition::on_event("go"))
.skip_if(Condition::on_event("skip"))
.cancel_if(Condition::on_event("stop"))
.handler(|_ctx, _input: In| async move { Ok(Out { pr: None }) });
let options = &task.def.options;
assert_eq!(
options.wait_for,
ConditionSet::from(Condition::sleep(Duration::from_secs(5)))
.and(Condition::on_event("go")),
"repeated calls accumulate rather than replace"
);
assert_eq!(options.skip_if, Condition::on_event("skip").into());
assert_eq!(options.cancel_if, Condition::on_event("stop").into());
}
#[test]
fn a_task_without_conditions_has_three_empty_gates() {
let task = echo_task("plain");
assert!(task.def.options.wait_for.is_empty());
assert!(task.def.options.skip_if.is_empty());
assert!(task.def.options.cancel_if.is_empty());
}
#[test]
fn parents_are_recorded_by_name_across_input_and_output_types() {
let a = echo_task("fetch");
let b =
TaskBuilder::<(), u32>::new("count").handler(|_ctx, _input: ()| async move { Ok(1) });
let c = echo_task("report").parent(&a).parent(&b);
assert_eq!(c.def.parents, ["fetch", "count"]);
}
#[tokio::test]
async fn erased_handler_round_trips_json() {
let task = echo_task("agent-run");
let output = (task.def.handler)(Context::placeholder(), json!({"repo": "hatchet-rs"}))
.await
.unwrap();
assert_eq!(output, json!({"pr": "hatchet-rs"}));
}
#[tokio::test]
async fn input_of_the_wrong_shape_is_a_non_retryable_error() {
let task = echo_task("agent-run");
let error = (task.def.handler)(Context::placeholder(), json!({"repo": 42}))
.await
.unwrap_err();
assert!(!error.is_retryable());
assert!(error.message().contains("input"), "{error}");
}
#[tokio::test]
async fn handler_errors_pass_through_unchanged() {
let task = TaskBuilder::<In, Out>::new("fails")
.handler(|_ctx, _input: In| async move { Err(TaskError::non_retryable("nope")) });
let error = (task.def.handler)(Context::placeholder(), json!({"repo": "x"}))
.await
.unwrap_err();
assert_eq!(error, TaskError::non_retryable("nope"));
}
#[tokio::test]
async fn output_that_cannot_be_serialized_is_a_non_retryable_error() {
use std::collections::HashMap;
let task = TaskBuilder::<(), HashMap<(i32, i32), i32>>::new("bad-output")
.handler(|_ctx, _input: ()| async move { Ok(HashMap::from([((1, 2), 3)])) });
let error = (task.def.handler)(Context::placeholder(), json!(null))
.await
.unwrap_err();
assert!(!error.is_retryable());
assert!(error.message().contains("output"), "{error}");
}
#[test]
fn debug_names_the_declaration_and_says_it_is_not_all_of_it() {
let builder = TaskBuilder::<In, Out>::new("agent-run").retries(3);
let debug = format!("{builder:?}");
assert!(debug.starts_with("TaskBuilder"), "{debug}");
assert!(debug.contains("agent-run"), "{debug}");
assert!(debug.contains(".."), "{debug}");
let task = echo_task("report").parent(&echo_task("fetch"));
let debug = format!("{task:?}");
assert!(debug.starts_with("Task"), "{debug}");
assert!(debug.contains("report"), "{debug}");
assert!(debug.contains("fetch"), "{debug}");
assert!(debug.contains(".."), "{debug}");
}
fn batch_action(items: &[(&str, Value)]) -> AssignedAction {
let payload: serde_json::Map<String, Value> = items
.iter()
.map(|(id, input)| {
(
(*id).to_string(),
json!({"workflow_run_id": format!("{id}-run"), "payload": {"input": input}}),
)
})
.collect();
AssignedAction {
action_payload: Value::Object(payload).to_string(),
batch_id: Some("b-1".to_string()),
..AssignedAction::default()
}
}
async fn run_batch(task: &Task<u32, u32>, action: &AssignedAction) -> BTreeMap<String, Value> {
let handler = task
.def
.batch
.as_ref()
.expect("a batch task")
.handler
.clone();
let report = handler(Context::placeholder(), action.clone())
.await
.expect("the batch handler answered");
report
.into_iter()
.map(|(id, result)| {
(
id,
match result {
Ok(value) => value,
Err(error) => json!({"failed": error.message()}),
},
)
})
.collect()
}
#[test]
fn a_batch_task_carries_its_config_and_says_it_is_not_a_broadcast() {
let task = TaskBuilder::<u32, u32>::new("double").batch_handler(
BatchConfig::max_size(4).group_key("input.tenant"),
|_ctx, _batch| async move { Ok(BatchOutput::new()) },
);
let batch = task.def.batch.as_ref().expect("a batch task");
assert!(!batch.broadcast);
assert_eq!(
batch.config,
BatchConfig::max_size(4).group_key("input.tenant")
);
assert!(!task.def.durable, "a batch task asks for a default slot");
}
#[tokio::test]
async fn a_batch_handler_is_called_once_and_answers_for_every_member() {
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let seen = Arc::clone(&calls);
let task = TaskBuilder::<u32, u32>::new("double").batch_handler(
BatchConfig::max_size(4),
move |_ctx, batch: BatchInput<u32>| {
let seen = Arc::clone(&seen);
async move {
seen.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Ok(batch.iter().map(|(id, n)| (id.clone(), n * 2)).collect())
}
},
);
let report = run_batch(&task, &batch_action(&[("a", json!(1)), ("b", json!(2))])).await;
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
assert_eq!(
report,
BTreeMap::from([("a".to_string(), json!(2)), ("b".to_string(), json!(4))]),
);
}
#[tokio::test]
async fn a_member_the_handler_leaves_out_fails_alone() {
let task = TaskBuilder::<u32, u32>::new("double").batch_handler(
BatchConfig::max_size(4),
|_ctx, batch: BatchInput<u32>| async move {
Ok(BatchOutput::new().ok(batch.ids().next().unwrap().clone(), 10))
},
);
let report = run_batch(&task, &batch_action(&[("a", json!(1)), ("b", json!(2))])).await;
assert_eq!(report["a"], json!(10));
assert!(
report["b"]["failed"]
.as_str()
.unwrap()
.contains("no output"),
"{report:?}"
);
}
#[tokio::test]
async fn a_member_whose_input_is_of_the_wrong_type_never_reaches_the_handler() {
let task = TaskBuilder::<u32, u32>::new("double").batch_handler(
BatchConfig::max_size(4),
|_ctx, batch: BatchInput<u32>| async move {
assert_eq!(batch.len(), 1, "only the decodable member is shown");
Ok(batch.iter().map(|(id, n)| (id.clone(), n * 2)).collect())
},
);
let report = run_batch(
&task,
&batch_action(&[("a", json!(1)), ("b", json!("not a number"))]),
)
.await;
assert_eq!(report["a"], json!(2));
assert!(
report["b"]["failed"]
.as_str()
.unwrap()
.contains("input type"),
"{report:?}"
);
}
#[tokio::test]
async fn a_broadcast_handler_gives_every_member_the_same_value() {
let task = TaskBuilder::<u32, u32>::new("warm").batch_broadcast_handler(
BatchConfig::max_size(4),
|_ctx, batch: BatchInput<u32>| async move { Ok(u32::try_from(batch.len()).unwrap()) },
);
assert!(task.def.batch.as_ref().unwrap().broadcast);
let report = run_batch(&task, &batch_action(&[("a", json!(1)), ("b", json!(2))])).await;
assert_eq!(
report,
BTreeMap::from([("a".to_string(), json!(2)), ("b".to_string(), json!(2))]),
);
}
#[tokio::test]
async fn a_batch_handler_that_fails_says_nothing_about_any_member() {
let task = TaskBuilder::<u32, u32>::new("double").batch_handler(
BatchConfig::max_size(4),
|_ctx, _batch: BatchInput<u32>| async move { Err(TaskError::new("the api is down")) },
);
let handler = task.def.batch.as_ref().unwrap().handler.clone();
let error = handler(Context::placeholder(), batch_action(&[("a", json!(1))]))
.await
.unwrap_err();
assert_eq!(error, TaskError::new("the api is down"));
}
#[tokio::test]
async fn a_batch_task_assigned_as_an_ordinary_run_fails_by_name() {
let task = TaskBuilder::<u32, u32>::new("double").batch_handler(
BatchConfig::max_size(4),
|_ctx, _batch: BatchInput<u32>| async move { Ok(BatchOutput::new()) },
);
let error = (task.def.handler)(Context::placeholder(), json!(1))
.await
.unwrap_err();
assert!(!error.is_retryable(), "{error}");
assert!(error.message().contains("double"), "{error}");
assert!(error.message().contains("batch task"), "{error}");
}
#[test]
fn tasks_are_cheap_to_clone_and_share_the_handler() {
let task = echo_task("agent-run");
let clone = task.clone();
assert!(Arc::ptr_eq(&task.def.handler, &clone.def.handler));
}
}