pub(crate) mod eviction;
pub(crate) mod stream;
pub use eviction::EvictionPolicy;
use std::collections::HashMap;
use std::marker::PhantomData;
use std::sync::{Arc, Mutex, PoisonError};
use std::time::{Duration, SystemTime};
use hatchet_proto::v1::DurableEventLogEntryRef;
use serde::Serialize;
use serde::de::DeserializeOwned;
use serde_json::{Map, Value};
use tracing::{Instrument, Span};
use crate::declare::condition::to_durable_proto;
use crate::durable::stream::DurableStream;
use crate::trigger::ParentRef;
use crate::{Condition, ConditionSet, Context, Error, RunOpts, Task, TaskError, otel};
pub(crate) fn task_error(error: Error) -> TaskError {
match error {
Error::Nondeterminism { .. }
| Error::UnsupportedFeature { .. } => TaskError::non_retryable(error.to_string()),
other => TaskError::new(other.to_string()),
}
}
pub(crate) struct DurableInvocation {
stream: DurableStream,
task_id: String,
invocation_count: i32,
sending: tokio::sync::Mutex<()>,
memos: Mutex<HashMap<String, Vec<u8>>>,
engine_version: Arc<str>,
book: Option<Arc<eviction::EvictionBook>>,
}
impl DurableInvocation {
pub(crate) fn new(
stream: DurableStream,
task_id: String,
invocation_count: i32,
engine_version: Arc<str>,
book: Option<Arc<eviction::EvictionBook>>,
) -> Self {
Self {
stream,
task_id,
invocation_count,
sending: tokio::sync::Mutex::new(()),
memos: Mutex::new(HashMap::new()),
engine_version,
book,
}
}
fn mark_waiting(&self, kind: &'static str, resource: &str) -> Option<eviction::WaitingGuard> {
self.book
.as_ref()
.map(|book| book.mark_waiting(&self.task_id, kind, resource))
}
fn cached(&self, key: &str) -> Option<Vec<u8>> {
self.memos
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get(key)
.cloned()
}
fn remember(&self, key: &str, payload: &[u8]) {
self.memos
.lock()
.unwrap_or_else(PoisonError::into_inner)
.insert(key.to_string(), payload.to_vec());
}
}
#[derive(Clone)]
pub struct DurableContext {
context: Context,
durable: Arc<DurableInvocation>,
}
impl std::fmt::Debug for DurableContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DurableContext")
.field("task_run_id", &self.context.task_run_id())
.field("invocation_count", &self.durable.invocation_count)
.finish_non_exhaustive()
}
}
impl std::ops::Deref for DurableContext {
type Target = Context;
fn deref(&self) -> &Context {
&self.context
}
}
impl DurableContext {
pub(crate) fn attach(context: Context) -> Result<Self, TaskError> {
let durable = context.durable().cloned().ok_or_else(|| {
TaskError::non_retryable(
"this task is declared durable, but the action it was given carries no durable \
invocation; the worker has no durable stream",
)
})?;
Ok(Self { context, durable })
}
#[must_use]
pub fn invocation_count(&self) -> i32 {
self.durable.invocation_count
}
pub async fn sleep_for(&self, duration: Duration) -> Result<(), TaskError> {
self.wait(Condition::sleep(duration).into(), None).await?;
Ok(())
}
pub async fn wait_for_event(&self, key: &str) -> Result<DurableEvent, TaskError> {
let matched = self.wait(Condition::on_event(key).into(), None).await?;
matched.into_iter().next().ok_or_else(|| {
TaskError::new(format!(
"the engine completed the durable wait for the event `{key}` without naming \
what matched"
))
})
}
pub async fn wait_for(
&self,
conditions: impl Into<ConditionSet>,
) -> Result<Vec<DurableEvent>, TaskError> {
self.wait(conditions.into(), None).await
}
pub async fn wait_for_labeled(
&self,
label: &str,
conditions: impl Into<ConditionSet>,
) -> Result<Vec<DurableEvent>, TaskError> {
self.wait(conditions.into(), Some(label)).await
}
pub async fn sleep_until(&self, at: SystemTime) -> Result<(), TaskError> {
let now = self.now().await?;
match at.duration_since(now) {
Ok(remaining) if !remaining.is_zero() => self.sleep_for(remaining).await,
_ => Ok(()),
}
}
pub async fn memo<T, F, Fut>(&self, key: &str, f: F) -> Result<T, TaskError>
where
T: Serialize + DeserializeOwned,
F: FnOnce() -> Fut,
Fut: Future<Output = Result<T, TaskError>>,
{
if let Some(cached) = self.durable.cached(key) {
return decode(key, &cached);
}
let ack = {
let _sending = self.durable.sending.lock().await;
self.durable
.stream
.send_memo(
self.durable.task_id.clone(),
self.durable.invocation_count,
key,
None,
)
.await
}
.map_err(task_error)?;
if let Some(recorded) = ack.existing {
self.durable.remember(key, &recorded);
return decode(key, &recorded);
}
let value = f().await?;
let payload = serde_json::to_vec(&value).map_err(|error| {
TaskError::non_retryable(format!(
"the memo `{key}` could not be serialized to JSON: {error}"
))
})?;
self.durable.remember(key, &payload);
self.durable
.stream
.send_complete_memo(ack.reference, key, payload);
Ok(value)
}
pub async fn now(&self) -> Result<SystemTime, TaskError> {
self.memo(NOW_KEY, || async { Ok(SystemTime::now()) }).await
}
pub async fn spawn<I, O>(
&self,
task: &Task<I, O>,
input: &I,
opts: RunOpts,
) -> Result<DurableRunHandle<O>, TaskError>
where
I: Serialize,
O: DeserializeOwned,
{
let span = otel::run_workflow(task.name());
let spawned = self
.spawn_all(task.name(), std::slice::from_ref(input), &opts, &span)
.instrument(span.clone())
.await;
otel::record_result(&span, &spawned);
let mut handles = spawned?;
let handle = handles.pop().expect("one input, one handle");
span.record("hatchet.child_workflow_run_id", handle.run_id());
Ok(handle)
}
pub async fn spawn_many<I, O>(
&self,
task: &Task<I, O>,
inputs: &[I],
opts: RunOpts,
) -> Result<Vec<DurableRunHandle<O>>, TaskError>
where
I: Serialize,
O: DeserializeOwned,
{
let span = otel::run_workflows(task.name(), inputs.len());
let spawned = self
.spawn_all(task.name(), inputs, &opts, &span)
.instrument(span.clone())
.await;
otel::record_result(&span, &spawned);
spawned
}
async fn spawn_all<I, O>(
&self,
name: &str,
inputs: &[I],
opts: &RunOpts,
span: &Span,
) -> Result<Vec<DurableRunHandle<O>>, TaskError>
where
I: Serialize,
O: DeserializeOwned,
{
crate::version::ensure_feature(
SPAWN_FEATURE,
crate::version::DURABLE_SPAWN_MIN_ENGINE_VERSION,
&self.durable.engine_version,
)
.map_err(task_error)?;
if inputs.is_empty() {
return Err(TaskError::non_retryable(
"`DurableContext::spawn_many` was given no inputs; a request with no triggers \
would still make a log entry that the next attempt has to reproduce",
));
}
if inputs.len() > 1 && opts.has_child_key() {
return Err(TaskError::non_retryable(format!(
"`DurableContext::spawn_many` was given {} inputs and one `child_key`; the \
engine deduplicates children on `(parent run, parent task run, child key)`, so \
they would all fold into a single run. Call `spawn` once per child instead.",
inputs.len()
)));
}
let namespace = self.namespace()?;
let mut requests = Vec::with_capacity(inputs.len());
for input in inputs {
let mut request = crate::trigger::request(
&namespace,
name,
input,
opts,
Some(ParentRef {
workflow_run_id: self.context.workflow_run_id(),
task_run_external_id: self.context.task_run_id(),
child_index: self.context.claim_child_index(),
worker_id: self.context.worker_id(),
}),
)
.map_err(task_error)?;
otel::inject(span, &mut request.additional_metadata);
requests.push(request);
}
let entries = {
let _sending = self.durable.sending.lock().await;
self.durable
.stream
.send_trigger_runs(
self.durable.task_id.clone(),
self.durable.invocation_count,
requests,
)
.await
}
.map_err(task_error)?;
if entries.len() != inputs.len() {
return Err(TaskError::non_retryable(format!(
"the engine answered a durable spawn of {} child run(s) with {} log entr(ies); \
the SDK pairs handles with entries by position and cannot do so",
inputs.len(),
entries.len(),
)));
}
entries
.into_iter()
.map(|entry| {
let reference = DurableEventLogEntryRef {
durable_task_external_id: self.durable.task_id.clone(),
invocation_count: self.durable.invocation_count,
branch_id: entry.branch_id,
node_id: entry.node_id,
};
Ok(DurableRunHandle {
run_id: entry.workflow_run_external_id,
task_name: name.to_string(),
wait: self
.durable
.stream
.claim_entry(reference)
.map_err(task_error)?,
context: self.context.clone(),
_output: PhantomData,
})
})
.collect()
}
async fn wait(
&self,
conditions: ConditionSet,
label: Option<&str>,
) -> Result<Vec<DurableEvent>, TaskError> {
let namespace = self.namespace()?;
let (kind, resource) = conditions.wait_kind();
let _waiting = self.durable.mark_waiting(kind, &resource);
let conditions =
to_durable_proto(&namespace, &conditions).map_err(TaskError::non_retryable)?;
let reference = {
let _sending = self.durable.sending.lock().await;
self.durable
.stream
.send_wait_for(
self.durable.task_id.clone(),
self.durable.invocation_count,
conditions,
label.map(str::to_string),
)
.await
}
.map_err(task_error)?;
let payload = tokio::select! {
() = self.context.cancelled() => {
return Err(TaskError::new(
"the run was cancelled while it waited on the engine",
));
}
payload = self.durable.stream.wait_for_entry(reference) => payload,
}
.map_err(task_error)?;
Ok(matches(&payload))
}
fn namespace(&self) -> Result<String, TaskError> {
Ok(self
.context
.client()
.ok_or_else(|| {
TaskError::non_retryable(
"a durable wait talks to the engine, and this context was built without a \
connection to one",
)
})?
.config()
.namespace()
.to_string())
}
}
pub(crate) const SPAWN_FEATURE: &str = "spawning a child run from a durable handler";
#[must_use = "a durable run handle waits for nothing until `result` is awaited"]
pub struct DurableRunHandle<O> {
run_id: String,
task_name: String,
wait: crate::durable::stream::EntryWait,
context: Context,
_output: PhantomData<fn() -> O>,
}
impl<O> std::fmt::Debug for DurableRunHandle<O> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DurableRunHandle")
.field("run_id", &self.run_id)
.field("task_name", &self.task_name)
.finish_non_exhaustive()
}
}
impl<O> DurableRunHandle<O> {
#[must_use]
pub fn run_id(&self) -> &str {
&self.run_id
}
}
impl<O: DeserializeOwned> DurableRunHandle<O> {
pub async fn result(self) -> Result<O, TaskError> {
let Self {
run_id,
task_name,
wait,
context,
_output,
} = self;
let _waiting = context
.durable()
.and_then(|durable| durable.mark_waiting("spawn_child", &task_name));
let payload = tokio::select! {
() = context.cancelled() => {
return Err(TaskError::new(format!(
"the run was cancelled while it waited for the child run {run_id}",
)));
}
payload = wait.completed() => payload,
}
.map_err(task_error)?;
child_output(&run_id, &task_name, &payload)
}
}
fn child_output<O: DeserializeOwned>(
run_id: &str,
task_name: &str,
payload: &[u8],
) -> Result<O, TaskError> {
if payload.is_empty() {
return Err(TaskError::non_retryable(format!(
"the durable child run {run_id} of `{task_name}` completed with an empty payload, so \
there is no output to read; on engines below {min} a child that *failed* is reported \
this way, which is why that is also the oldest engine this SDK spawns children on",
min = crate::version::DURABLE_SPAWN_MIN_ENGINE_VERSION,
)));
}
serde_json::from_slice(payload).map_err(|error| {
TaskError::non_retryable(format!(
"the output of the durable child run {run_id} of `{task_name}` does not match the \
type it was asked for: {error}"
))
})
}
const NOW_KEY: &str = "__now__";
fn decode<T: DeserializeOwned>(key: &str, payload: &[u8]) -> Result<T, TaskError> {
serde_json::from_slice(payload).map_err(|error| {
TaskError::non_retryable(format!(
"the value recorded for the memo `{key}` does not match the type it was asked for: \
{error}"
))
})
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DurableEvent {
pub key: String,
pub payload: Value,
}
fn matches(payload: &[u8]) -> Vec<DurableEvent> {
let whole = |payload: Value| {
vec![DurableEvent {
key: String::new(),
payload,
}]
};
let Ok(value) = serde_json::from_slice::<Value>(payload) else {
return whole(Value::String(String::from_utf8_lossy(payload).into_owned()));
};
let Some(inner) = unwrap_action(&value) else {
return whole(value);
};
let mut keys: Vec<&String> = inner.keys().collect();
keys.sort();
let mut events = Vec::new();
for key in keys {
match &inner[key] {
Value::Array(payloads) if !payloads.is_empty() => {
events.extend(payloads.iter().map(|payload| DurableEvent {
key: key.clone(),
payload: payload.clone(),
}));
}
Value::Array(_) => events.push(DurableEvent {
key: key.clone(),
payload: Value::Object(Map::new()),
}),
other => events.push(DurableEvent {
key: key.clone(),
payload: other.clone(),
}),
}
}
events
}
fn unwrap_action(value: &Value) -> Option<&Map<String, Value>> {
let outer = value.as_object()?;
if outer.len() != 1 {
return None;
}
outer.values().next()?.as_object()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_diverged_run_is_never_retried() {
let error = task_error(Error::Nondeterminism {
task_run_id: "run-1".to_string(),
invocation_count: 2,
node_id: 3,
message: "expected: sleep(10m0s)\n received: memo".to_string(),
});
assert!(!error.is_retryable());
assert!(
error.message().contains("received: memo"),
"{}",
error.message()
);
}
#[test]
fn a_call_the_engine_is_too_old_for_is_never_retried() {
let error = task_error(Error::UnsupportedFeature {
feature: SPAWN_FEATURE.to_string(),
found: "v0.80.0".to_string(),
minimum: crate::version::DURABLE_SPAWN_MIN_ENGINE_VERSION.to_string(),
});
assert!(!error.is_retryable());
assert!(error.message().contains("v0.80.0"), "{}", error.message());
assert!(
error
.message()
.contains(crate::version::DURABLE_SPAWN_MIN_ENGINE_VERSION),
"{}",
error.message()
);
}
#[test]
fn a_childs_output_is_read_whole_out_of_its_payload() {
let output: Value =
child_output("child-a", "child", br#"{"from":"child","n":0}"#).expect("the output");
assert_eq!(output, serde_json::json!({"from": "child", "n": 0}));
#[derive(serde::Deserialize, Debug, PartialEq)]
struct Out {
n: i32,
}
assert_eq!(
child_output::<Out>("child-a", "child", br#"{"n":7}"#).expect("the output"),
Out { n: 7 }
);
}
#[test]
fn an_output_of_the_wrong_shape_is_an_error_that_is_not_retried() {
let error = child_output::<i32>("child-a", "child", br#"{"n":7}"#)
.expect_err("an object is not an i32");
assert!(!error.is_retryable());
assert!(error.message().contains("child-a"), "{}", error.message());
}
#[test]
fn an_empty_payload_is_an_error_rather_than_a_panic_or_a_made_up_value() {
for error in [
child_output::<Value>("child-a", "child", b"").expect_err("nothing to read"),
child_output::<()>("child-a", "child", b"").expect_err("nothing to read"),
] {
assert!(
error.message().contains("empty payload"),
"{}",
error.message()
);
assert!(!error.is_retryable(), "{}", error.message());
}
child_output::<()>("child-a", "child", b"null").expect("a handler that wanted no output");
}
#[test]
fn every_other_durable_failure_may_be_retried() {
let error = task_error(Error::Grpc(Box::new(tonic::Status::unavailable(
"engine restarting",
))));
assert!(error.is_retryable());
}
fn event(key: &str, payload: Value) -> DurableEvent {
DurableEvent {
key: key.to_string(),
payload,
}
}
#[test]
fn a_completion_is_unwrapped_out_of_its_action_envelope() {
assert_eq!(
matches(br#"{"CREATE":{"sleep:6000ms":[{"sleep_duration":"6000ms"}]}}"#),
[event(
"sleep:6000ms",
serde_json::json!({"sleep_duration": "6000ms"})
)]
);
assert_eq!(
matches(br#"{"CREATE":{"approve":[{"by":"ops","amount":42}]}}"#),
[event(
"approve",
serde_json::json!({"by": "ops", "amount": 42})
)]
);
}
#[test]
fn several_matches_come_back_in_key_order() {
let matched = matches(br#"{"CREATE":{"go":[{"n":1}],"sleep:5000ms":[{}],"halt":[{}]}}"#);
let keys: Vec<&str> = matched.iter().map(|event| event.key.as_str()).collect();
assert_eq!(keys, ["go", "halt", "sleep:5000ms"]);
}
#[test]
fn a_key_with_several_payloads_keeps_every_one_of_them() {
assert_eq!(
matches(br#"{"CREATE":{"go":[{"n":1},{"n":2}]}}"#),
[
event("go", serde_json::json!({"n": 1})),
event("go", serde_json::json!({"n": 2})),
]
);
}
#[test]
fn an_empty_envelope_matches_nothing() {
assert!(matches(br#"{"CREATE":{}}"#).is_empty());
assert_eq!(
matches(br#"{"CREATE":{"go":[]}}"#),
[event("go", serde_json::json!({}))]
);
}
#[test]
fn an_answer_in_an_unknown_shape_is_handed_back_whole_rather_than_dropped() {
for raw in [
br#"{"CREATE":{"go":[{}]},"QUEUE":{"x":[{}]}}"#.as_slice(),
br#"["go"]"#.as_slice(),
br#"42"#.as_slice(),
br#"{"go":["ok"]}"#.as_slice(),
] {
let matched = matches(raw);
assert_eq!(matched.len(), 1, "{}", String::from_utf8_lossy(raw));
assert_eq!(matched[0].key, "");
assert_eq!(
matched[0].payload,
serde_json::from_slice::<Value>(raw).unwrap()
);
}
}
#[test]
fn bytes_that_are_not_json_are_handed_back_as_text() {
let matched = matches(b"not json at all");
assert_eq!(
matched,
[event("", Value::String("not json at all".into()))]
);
}
}