use std::collections::{HashMap, VecDeque};
use std::pin::Pin;
use std::sync::{Arc, Mutex, MutexGuard};
use std::time::Duration;
use hatchet_proto::v1::DurableTaskEvictInvocationRequest;
use hatchet_proto::v1::v1_dispatcher_client::V1DispatcherClient;
use hatchet_proto::v1::{
DurableEventListenerConditions, DurableEventLogEntryRef, DurableTaskAwaitedCompletedEntry,
DurableTaskCompleteMemoRequest, DurableTaskErrorResponse, DurableTaskErrorType,
DurableTaskEventLogEntryCompletedResponse, DurableTaskEventMemoAckResponse,
DurableTaskMemoRequest, DurableTaskRequest, DurableTaskRequestRegisterWorker,
DurableTaskResponse, DurableTaskRunAckEntry, DurableTaskServerEvictNotice,
DurableTaskTriggerRunsRequest, DurableTaskWaitForRequest, DurableTaskWorkerStatusRequest,
TriggerWorkflowRequest, durable_task_request, durable_task_response,
};
use tokio::sync::{Notify, mpsc, oneshot};
use tokio::time::Instant;
use tokio_stream::wrappers::ReceiverStream;
use tokio_stream::{Stream, StreamExt};
use tokio_util::sync::CancellationToken;
use tonic::Status;
use tonic::codec::CompressionEncoding;
use crate::Error;
use crate::retry::{
STREAM_MAX_CONSECUTIVE_OTHER_FAILURES, StreamErrorClass, classify_stream_end, full_jitter,
stream_backoff_cap,
};
use crate::transport::GrpcService;
const ACK_TIMEOUT: Duration = Duration::from_secs(60);
const EVICTION_ACK_TIMEOUT: Duration = Duration::from_secs(30);
const COMPLETION_TTL: Duration = Duration::from_secs(10);
const WORKER_STATUS_INTERVAL: Duration = Duration::from_secs(1);
const REQUEST_BUFFER: usize = 64;
pub(crate) type RequestStream = ReceiverStream<DurableTaskRequest>;
type ResponseStream =
Pin<Box<dyn Stream<Item = Result<DurableTaskResponse, Status>> + Send + 'static>>;
pub(crate) fn connect(
service: GrpcService,
gzip: bool,
) -> impl FnMut(RequestStream) -> Pin<Box<dyn Future<Output = Result<ResponseStream, Status>> + Send>>
{
move |requests| {
let dispatcher =
V1DispatcherClient::new(service.clone()).accept_compressed(CompressionEncoding::Gzip);
let mut dispatcher = if gzip {
dispatcher.send_compressed(CompressionEncoding::Gzip)
} else {
dispatcher
};
Box::pin(async move {
let responses = dispatcher.durable_task(requests).await?.into_inner();
Ok(Box::pin(responses) as ResponseStream)
})
}
}
type LoopFuture = Pin<Box<dyn Future<Output = ()> + Send>>;
type EvictHook = Arc<dyn Fn(&str, i32) + Send + Sync>;
#[derive(Clone)]
pub(crate) struct DurableStream {
shared: Arc<Shared>,
task: Arc<Mutex<Option<LoopFuture>>>,
}
impl DurableStream {
pub(crate) fn new<C, Fut, S>(worker_id: String, connect: C, stop: CancellationToken) -> Self
where
C: FnMut(RequestStream) -> Fut + Send + 'static,
Fut: Future<Output = Result<S, Status>> + Send + 'static,
S: Stream<Item = Result<DurableTaskResponse, Status>> + Send + Unpin + 'static,
{
Self::with_backoff(worker_id, connect, stop, |attempt| {
full_jitter(stream_backoff_cap(attempt), &mut rand::rng())
})
}
fn with_backoff<C, Fut, S, B>(
worker_id: String,
connect: C,
stop: CancellationToken,
backoff: B,
) -> Self
where
C: FnMut(RequestStream) -> Fut + Send + 'static,
Fut: Future<Output = Result<S, Status>> + Send + 'static,
S: Stream<Item = Result<DurableTaskResponse, Status>> + Send + Unpin + 'static,
B: FnMut(u32) -> Duration + Send + 'static,
{
let shared = Arc::new(Shared::new(worker_id));
let task: LoopFuture = Box::pin(run(connect, Arc::clone(&shared), stop, backoff));
Self {
shared,
task: Arc::new(Mutex::new(Some(task))),
}
}
pub(crate) fn start(&self) {
let task = lock(&self.task).take();
if let Some(task) = task {
tokio::spawn(task);
}
}
pub(crate) async fn send_wait_for(
&self,
task_id: String,
invocation_count: i32,
conditions: DurableEventListenerConditions,
label: Option<String>,
) -> crate::Result<DurableEventLogEntryRef> {
let request = request(durable_task_request::Message::WaitFor(
DurableTaskWaitForRequest {
invocation_count,
durable_task_external_id: task_id.clone(),
wait_for_conditions: Some(conditions),
label,
},
));
match self.acked(&task_id, invocation_count, request).await? {
Ack::Wait(reference) => Ok(reference),
Ack::Memo(_) | Ack::Trigger(_) => Err(mismatched_ack(&task_id, invocation_count)),
}
}
pub(crate) async fn send_memo(
&self,
task_id: String,
invocation_count: i32,
key: &str,
payload: Option<Vec<u8>>,
) -> crate::Result<MemoAck> {
let request = request(durable_task_request::Message::Memo(
DurableTaskMemoRequest {
invocation_count,
durable_task_external_id: task_id.clone(),
key: key.as_bytes().to_vec(),
payload,
},
));
match self.acked(&task_id, invocation_count, request).await? {
Ack::Memo(ack) => Ok(ack),
Ack::Wait(_) | Ack::Trigger(_) => Err(mismatched_ack(&task_id, invocation_count)),
}
}
pub(crate) async fn send_trigger_runs(
&self,
task_id: String,
invocation_count: i32,
trigger_opts: Vec<TriggerWorkflowRequest>,
) -> crate::Result<Vec<DurableTaskRunAckEntry>> {
let request = request(durable_task_request::Message::TriggerRuns(
DurableTaskTriggerRunsRequest {
invocation_count,
durable_task_external_id: task_id.clone(),
trigger_opts,
},
));
match self.acked(&task_id, invocation_count, request).await? {
Ack::Trigger(entries) => Ok(entries),
Ack::Wait(_) | Ack::Memo(_) => Err(mismatched_ack(&task_id, invocation_count)),
}
}
pub(crate) fn send_complete_memo(
&self,
reference: DurableEventLogEntryRef,
key: &str,
payload: Vec<u8>,
) {
self.queue(request(durable_task_request::Message::CompleteMemo(
DurableTaskCompleteMemoRequest {
r#ref: Some(reference),
payload,
memo_key: key.as_bytes().to_vec(),
},
)));
}
pub(crate) async fn send_evict(
&self,
task_id: &str,
invocation_count: i32,
reason: Option<String>,
) -> crate::Result<()> {
let key = (task_id.to_string(), invocation_count);
let answered = {
let mut state = self.shared.lock();
if state.stopped {
return Err(stopped());
}
if state.evictions.contains_key(&key) {
return Err(overlapping_eviction(task_id, invocation_count));
}
let (answer, answered) = oneshot::channel();
state.evictions.insert(key.clone(), answer);
state
.outgoing
.push_back(request(durable_task_request::Message::EvictInvocation(
DurableTaskEvictInvocationRequest {
invocation_count,
durable_task_external_id: task_id.to_string(),
reason,
},
)));
answered
};
self.shared.work.notify_one();
let _registered = EvictionGuard {
shared: Arc::clone(&self.shared),
key,
};
match tokio::time::timeout(EVICTION_ACK_TIMEOUT, answered).await {
Ok(Ok(())) => Ok(()),
Ok(Err(_)) => Err(stopped()),
Err(_) => Err(eviction_timed_out(task_id, invocation_count)),
}
}
pub(crate) fn claim_entry(
&self,
reference: DurableEventLogEntryRef,
) -> crate::Result<EntryWait> {
let answered = {
let mut state = self.shared.lock();
if state.stopped {
return Err(stopped());
}
state.prune_expired();
if let Some((completed, _)) = state.unclaimed.remove(&reference) {
state.consume(&reference);
return Ok(EntryWait::Completed(Box::new(
completed.into_payload(&reference),
)));
}
if state.waits.contains_key(&reference) || state.consumed.contains_key(&reference) {
return Err(duplicate_wait(&reference));
}
let (answer, answered) = oneshot::channel();
state.waits.insert(reference.clone(), answer);
state.announce = true;
answered
};
self.shared.work.notify_one();
Ok(EntryWait::Pending {
answered,
reference: reference.clone(),
_registered: WaitGuard {
shared: Arc::clone(&self.shared),
reference,
},
})
}
pub(crate) async fn wait_for_entry(
&self,
reference: DurableEventLogEntryRef,
) -> crate::Result<Vec<u8>> {
self.claim_entry(reference)?.completed().await
}
#[cfg(test)]
pub(crate) fn cleanup(&self, task_id: &str, invocation_count: i32, reason: &str) {
self.shared.cleanup(task_id, invocation_count, reason);
}
pub(crate) fn on_server_evict(&self, hook: impl Fn(&str, i32) + Send + Sync + 'static) {
*lock(&self.shared.evict) = Some(Arc::new(hook));
}
pub(crate) fn given_up(&self) -> impl Future<Output = Error> + Send + 'static {
let shared = Arc::clone(&self.shared);
async move {
shared.gave_up.cancelled().await;
let status = lock(&shared.fatal)
.clone()
.unwrap_or_else(|| Status::unknown("the durable task stream ended"));
Error::Grpc(Box::new(status))
}
}
fn queue(&self, request: DurableTaskRequest) {
{
let mut state = self.shared.lock();
if state.stopped {
return;
}
state.outgoing.push_back(request);
}
self.shared.work.notify_one();
}
async fn acked(
&self,
task_id: &str,
invocation_count: i32,
request: DurableTaskRequest,
) -> crate::Result<Ack> {
let key = (task_id.to_string(), invocation_count);
let answered = {
let mut state = self.shared.lock();
if state.stopped {
return Err(stopped());
}
if state.acks.contains_key(&key) {
return Err(overlapping_request(task_id, invocation_count));
}
let (answer, answered) = oneshot::channel();
state.acks.insert(key.clone(), answer);
state.outgoing.push_back(request);
answered
};
self.shared.work.notify_one();
let _registered = AckGuard {
shared: Arc::clone(&self.shared),
key,
};
match tokio::time::timeout(ACK_TIMEOUT, answered).await {
Ok(Ok(ack)) => ack,
Ok(Err(_)) => Err(stopped()),
Err(_) => Err(ack_timed_out(task_id, invocation_count)),
}
}
#[cfg(test)]
fn buffered_completions(&self) -> usize {
let mut state = self.shared.lock();
state.prune_expired();
state.unclaimed.len()
}
}
#[derive(Debug)]
pub(crate) struct MemoAck {
pub(crate) reference: DurableEventLogEntryRef,
pub(crate) existing: Option<Vec<u8>>,
}
#[derive(Debug)]
enum Ack {
Wait(DurableEventLogEntryRef),
Memo(MemoAck),
Trigger(Vec<DurableTaskRunAckEntry>),
}
pub(crate) enum EntryWait {
Completed(Box<crate::Result<Vec<u8>>>),
Pending {
answered: oneshot::Receiver<crate::Result<Completed>>,
reference: DurableEventLogEntryRef,
_registered: WaitGuard,
},
}
impl EntryWait {
pub(crate) async fn completed(self) -> crate::Result<Vec<u8>> {
match self {
Self::Completed(payload) => *payload,
Self::Pending {
answered,
reference,
_registered,
} => match answered.await {
Ok(completed) => completed?.into_payload(&reference),
Err(_) => Err(stopped()),
},
}
}
}
pub(crate) struct Completed {
payload: Vec<u8>,
is_failure: bool,
error_message: Option<String>,
}
impl Completed {
fn into_payload(self, reference: &DurableEventLogEntryRef) -> crate::Result<Vec<u8>> {
if self.is_failure {
return Err(Error::DurableEntryFailed {
task_run_id: reference.durable_task_external_id.clone(),
invocation_count: reference.invocation_count,
node_id: reference.node_id,
message: self
.error_message
.unwrap_or_else(|| "the engine reported no reason".to_string()),
});
}
Ok(self.payload)
}
}
struct Shared {
worker_id: String,
state: Mutex<State>,
work: Notify,
evict: Mutex<Option<EvictHook>>,
gave_up: CancellationToken,
fatal: Mutex<Option<Status>>,
}
#[derive(Default)]
struct State {
outgoing: VecDeque<DurableTaskRequest>,
announce: bool,
acks: HashMap<(String, i32), oneshot::Sender<crate::Result<Ack>>>,
evictions: HashMap<(String, i32), oneshot::Sender<()>>,
waits: HashMap<DurableEventLogEntryRef, oneshot::Sender<crate::Result<Completed>>>,
unclaimed: HashMap<DurableEventLogEntryRef, (Completed, Instant)>,
consumed: HashMap<DurableEventLogEntryRef, Instant>,
stopped: bool,
}
impl State {
fn prune_expired(&mut self) {
let now = Instant::now();
self.unclaimed.retain(|_, (_, expires)| *expires > now);
self.consumed.retain(|_, expires| *expires > now);
}
fn consume(&mut self, reference: &DurableEventLogEntryRef) {
self.consumed
.insert(reference.clone(), Instant::now() + COMPLETION_TTL);
}
fn has_pending(&self) -> bool {
!self.outgoing.is_empty() || (self.announce && !self.waits.is_empty())
}
}
impl Shared {
fn new(worker_id: String) -> Self {
Self {
worker_id,
state: Mutex::new(State::default()),
work: Notify::new(),
evict: Mutex::new(None),
gave_up: CancellationToken::new(),
fatal: Mutex::new(None),
}
}
fn lock(&self) -> MutexGuard<'_, State> {
lock(&self.state)
}
fn has_pending(&self) -> bool {
self.lock().has_pending()
}
fn take_pending(&self) -> Option<DurableTaskRequest> {
let mut state = self.lock();
if state.announce && !state.waits.is_empty() {
state.announce = false;
return Some(self.status_from(&state));
}
state.outgoing.pop_front()
}
fn announce(&self) {
let mut state = self.lock();
state.prune_expired();
state.announce = true;
}
fn worker_status(&self) -> Option<DurableTaskRequest> {
let state = self.lock();
(!state.waits.is_empty()).then(|| self.status_from(&state))
}
fn status_from(&self, state: &State) -> DurableTaskRequest {
let mut waiting_entries: Vec<DurableTaskAwaitedCompletedEntry> = state
.waits
.keys()
.map(|reference| DurableTaskAwaitedCompletedEntry {
durable_task_external_id: reference.durable_task_external_id.clone(),
branch_id: reference.branch_id,
node_id: reference.node_id,
invocation_count: reference.invocation_count,
})
.collect();
waiting_entries.sort_by(|left, right| {
(
&left.durable_task_external_id,
left.invocation_count,
left.branch_id,
left.node_id,
)
.cmp(&(
&right.durable_task_external_id,
right.invocation_count,
right.branch_id,
right.node_id,
))
});
request(durable_task_request::Message::WorkerStatus(
DurableTaskWorkerStatusRequest {
worker_id: self.worker_id.clone(),
waiting_entries,
},
))
}
fn fail_acks(&self, status: Option<&Status>) {
let acks = {
let mut state = self.lock();
state.outgoing.retain(|request| {
matches!(
request.message,
Some(
durable_task_request::Message::CompleteMemo(_)
| durable_task_request::Message::EvictInvocation(_)
)
)
});
std::mem::take(&mut state.acks)
};
if acks.is_empty() {
return;
}
let status = status.cloned().unwrap_or_else(|| {
Status::unavailable("the durable task stream ended before the engine answered")
});
for answer in acks.into_values() {
drop(answer.send(Err(Error::Grpc(Box::new(status.clone())))));
}
}
fn fail_all(&self, status: &Status) {
let (acks, waits) = {
let mut state = self.lock();
state.outgoing.clear();
state.unclaimed.clear();
state.consumed.clear();
state.evictions.clear();
(
std::mem::take(&mut state.acks),
std::mem::take(&mut state.waits),
)
};
for answer in acks.into_values() {
drop(answer.send(Err(Error::Grpc(Box::new(status.clone())))));
}
for answer in waits.into_values() {
drop(answer.send(Err(Error::Grpc(Box::new(status.clone())))));
}
}
fn stop(&self) {
{
let mut state = self.lock();
state.stopped = true;
}
self.fail_all(&Status::cancelled(GONE));
}
fn cleanup(&self, task_id: &str, invocation_count: i32, reason: &str) {
let (acks, waits) = {
let mut state = self.lock();
state
.unclaimed
.retain(|reference, _| !stale(reference, task_id, invocation_count));
state
.consumed
.retain(|reference, _| !stale(reference, task_id, invocation_count));
state
.evictions
.retain(|(id, invocation), _| id != task_id || *invocation > invocation_count);
let acks: Vec<_> = state
.acks
.keys()
.filter(|(id, invocation)| id == task_id && *invocation <= invocation_count)
.cloned()
.collect();
let acks: Vec<_> = acks
.into_iter()
.filter_map(|key| state.acks.remove(&key))
.collect();
let refs: Vec<_> = state
.waits
.keys()
.filter(|reference| stale(reference, task_id, invocation_count))
.cloned()
.collect();
let waits: Vec<_> = refs
.into_iter()
.filter_map(|reference| state.waits.remove(&reference))
.collect();
(acks, waits)
};
for answer in acks {
drop(answer.send(Err(evicted(reason))));
}
for answer in waits {
drop(answer.send(Err(evicted(reason))));
}
}
}
fn stale(reference: &DurableEventLogEntryRef, task_id: &str, invocation_count: i32) -> bool {
reference.durable_task_external_id == task_id && reference.invocation_count <= invocation_count
}
struct AckGuard {
shared: Arc<Shared>,
key: (String, i32),
}
impl Drop for AckGuard {
fn drop(&mut self) {
self.shared.lock().acks.remove(&self.key);
}
}
pub(crate) struct WaitGuard {
shared: Arc<Shared>,
reference: DurableEventLogEntryRef,
}
impl Drop for WaitGuard {
fn drop(&mut self) {
self.shared.lock().waits.remove(&self.reference);
}
}
struct EvictionGuard {
shared: Arc<Shared>,
key: (String, i32),
}
impl Drop for EvictionGuard {
fn drop(&mut self) {
self.shared.lock().evictions.remove(&self.key);
}
}
const GONE: &str = "the worker holding the durable task stream is gone";
fn stopped() -> Error {
Error::Grpc(Box::new(Status::cancelled(GONE)))
}
fn evicted(reason: &str) -> Error {
Error::Grpc(Box::new(Status::cancelled(format!(
"the engine evicted this durable invocation: {reason}"
))))
}
fn ack_timed_out(task_id: &str, invocation_count: i32) -> Error {
Error::Grpc(Box::new(Status::deadline_exceeded(format!(
"the engine did not acknowledge a durable request for task run {task_id} (invocation \
{invocation_count}) within {ACK_TIMEOUT:?}; the request may have been lost on the stream, \
so this attempt fails rather than sending it again"
))))
}
fn eviction_timed_out(task_id: &str, invocation_count: i32) -> Error {
Error::Grpc(Box::new(Status::deadline_exceeded(format!(
"the engine did not acknowledge the eviction of task run {task_id} (invocation \
{invocation_count}) within {EVICTION_ACK_TIMEOUT:?}; the run keeps its durable slot \
rather than being cancelled behind the engine's back"
))))
}
fn overlapping_eviction(task_id: &str, invocation_count: i32) -> Error {
Error::Grpc(Box::new(Status::failed_precondition(format!(
"an eviction of task run {task_id} (invocation {invocation_count}) is already waiting to \
be acknowledged"
))))
}
fn overlapping_request(task_id: &str, invocation_count: i32) -> Error {
Error::Grpc(Box::new(Status::failed_precondition(format!(
"a durable request for task run {task_id} (invocation {invocation_count}) is already \
waiting to be acknowledged; one invocation's requests must go out one at a time"
))))
}
fn mismatched_ack(task_id: &str, invocation_count: i32) -> Error {
Error::Grpc(Box::new(Status::internal(format!(
"the engine acknowledged a durable request for task run {task_id} (invocation \
{invocation_count}) with an ack of the wrong kind"
))))
}
fn duplicate_wait(reference: &DurableEventLogEntryRef) -> Error {
Error::Grpc(Box::new(Status::failed_precondition(format!(
"durable log entry {}:{} of task run {} is already being waited for",
reference.node_id, reference.branch_id, reference.durable_task_external_id
))))
}
fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
mutex
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
fn request(message: durable_task_request::Message) -> DurableTaskRequest {
DurableTaskRequest {
message: Some(message),
}
}
fn register(worker_id: &str) -> DurableTaskRequest {
request(durable_task_request::Message::RegisterWorker(
DurableTaskRequestRegisterWorker {
worker_id: worker_id.to_string(),
},
))
}
enum Outcome {
Stopped,
Ended {
status: Option<Status>,
received: bool,
},
}
async fn run<C, Fut, S, B>(
mut connect: C,
shared: Arc<Shared>,
stop: CancellationToken,
mut backoff: B,
) where
C: FnMut(RequestStream) -> Fut,
Fut: Future<Output = Result<S, Status>>,
S: Stream<Item = Result<DurableTaskResponse, Status>> + Unpin,
B: FnMut(u32) -> Duration,
{
let mut attempt: u32 = 0;
let mut consecutive_other: u32 = 0;
while !stop.is_cancelled() {
let (requests, received_by_engine) = mpsc::channel(REQUEST_BUFFER);
let _ = requests.try_send(register(&shared.worker_id));
if let Some(status) = shared.worker_status() {
let _ = requests.try_send(status);
}
let outcome = tokio::select! {
() = stop.cancelled() => Outcome::Stopped,
connected = connect(ReceiverStream::new(received_by_engine)) => match connected {
Ok(responses) => serve(responses, &requests, &shared, &stop).await,
Err(status) => Outcome::Ended { status: Some(status), received: false },
}
};
drop(requests);
let (status, received) = match outcome {
Outcome::Stopped => break,
Outcome::Ended { status, received } => (status, received),
};
shared.fail_acks(status.as_ref());
if received {
attempt = 0;
consecutive_other = 0;
}
let give_up = match classify_stream_end(status.as_ref()) {
StreamErrorClass::StopClean | StreamErrorClass::Transient => false,
StreamErrorClass::Fatal => true,
StreamErrorClass::Other => {
consecutive_other += 1;
consecutive_other >= STREAM_MAX_CONSECUTIVE_OTHER_FAILURES
}
};
if give_up {
let status = status.unwrap_or_else(|| Status::unknown("the durable task stream ended"));
tracing::warn!(
code = ?status.code(),
message = status.message(),
consecutive_other,
"giving up on the durable task stream; every durable wait fails",
);
shared.fail_all(&status);
*lock(&shared.fatal) = Some(status);
shared.gave_up.cancel();
break;
}
let delay = backoff(attempt);
attempt = attempt.saturating_add(1);
log_reconnect(attempt, consecutive_other, delay, status.as_ref());
tokio::select! {
() = stop.cancelled() => break,
() = tokio::time::sleep(delay) => {}
}
}
shared.stop();
}
async fn serve<S>(
mut responses: S,
requests: &mpsc::Sender<DurableTaskRequest>,
shared: &Shared,
stop: &CancellationToken,
) -> Outcome
where
S: Stream<Item = Result<DurableTaskResponse, Status>> + Unpin,
{
let mut received = false;
let mut ticker = tokio::time::interval(WORKER_STATUS_INTERVAL);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
ticker.tick().await;
loop {
if !send_pending(shared, requests, stop).await {
return if stop.is_cancelled() {
Outcome::Stopped
} else {
Outcome::Ended {
status: None,
received,
}
};
}
tokio::select! {
() = stop.cancelled() => return Outcome::Stopped,
() = shared.work.notified() => {}
_ = ticker.tick() => shared.announce(),
item = responses.next() => match item {
Some(Ok(response)) => {
received = true;
deliver(shared, response);
}
Some(Err(status)) => return Outcome::Ended { status: Some(status), received },
None => return Outcome::Ended { status: None, received },
}
}
}
}
async fn send_pending(
shared: &Shared,
requests: &mpsc::Sender<DurableTaskRequest>,
stop: &CancellationToken,
) -> bool {
while shared.has_pending() {
let permit = tokio::select! {
() = stop.cancelled() => return false,
reserved = requests.reserve() => match reserved {
Ok(permit) => permit,
Err(_) => return false,
}
};
let Some(request) = shared.take_pending() else {
return true;
};
permit.send(request);
}
true
}
fn deliver(shared: &Shared, response: DurableTaskResponse) {
match response.message {
Some(durable_task_response::Message::RegisterWorker(_)) => {}
Some(durable_task_response::Message::WaitForAck(ack)) => {
answer_ack(shared, ack.r#ref.clone(), |_| {
ack.r#ref.clone().map(Ack::Wait)
});
}
Some(durable_task_response::Message::MemoAck(ack)) => {
answer_ack(shared, ack.r#ref.clone(), |_| memo_ack(&ack));
}
Some(durable_task_response::Message::EntryCompleted(done)) => complete(shared, done),
Some(durable_task_response::Message::Error(error)) => fail_ref(shared, &error),
Some(durable_task_response::Message::ServerEvict(notice)) => evict(shared, ¬ice),
Some(durable_task_response::Message::TriggerRunsAck(ack)) => {
answer_by_key(
shared,
(ack.durable_task_external_id, ack.invocation_count),
Ok(Ack::Trigger(ack.run_entries)),
);
}
Some(durable_task_response::Message::EvictionAck(ack)) => {
let key = (ack.durable_task_external_id, ack.invocation_count);
let answer = shared.lock().evictions.remove(&key);
match answer {
Some(answer) => drop(answer.send(())),
None => tracing::debug!(
task_run_id = %key.0,
invocation_count = key.1,
"an eviction ack arrived for an eviction nobody is waiting for",
),
}
}
None => tracing::debug!("ignoring an empty durable response"),
}
}
fn memo_ack(ack: &DurableTaskEventMemoAckResponse) -> Option<Ack> {
if ack.memo_already_existed && ack.memo_result_payload.is_none() {
tracing::warn!(reference = ?ack.r#ref, "a memo exists on the engine but carries no payload");
}
Some(Ack::Memo(MemoAck {
reference: ack.r#ref.clone()?,
existing: ack
.memo_already_existed
.then(|| ack.memo_result_payload.clone())
.flatten(),
}))
}
fn answer_ack(
shared: &Shared,
reference: Option<DurableEventLogEntryRef>,
ack: impl FnOnce(&DurableEventLogEntryRef) -> Option<Ack>,
) {
let Some(reference) = reference else {
tracing::warn!("the engine acknowledged a durable request without a log entry ref");
return;
};
let key = (
reference.durable_task_external_id.clone(),
reference.invocation_count,
);
let answer = match ack(&reference) {
Some(ack) => Ok(ack),
None => Err(mismatched_ack(
&reference.durable_task_external_id,
reference.invocation_count,
)),
};
answer_by_key(shared, key, answer);
}
fn answer_by_key(shared: &Shared, key: (String, i32), ack: crate::Result<Ack>) {
let Some(answer) = shared.lock().acks.remove(&key) else {
tracing::debug!(
task_run_id = %key.0,
invocation_count = key.1,
"a durable ack arrived for a request nobody is waiting for",
);
return;
};
drop(answer.send(ack));
}
fn complete(shared: &Shared, done: DurableTaskEventLogEntryCompletedResponse) {
let Some(reference) = done.r#ref else {
tracing::warn!("the engine completed a durable log entry without a ref");
return;
};
let completed = Completed {
payload: done.payload,
is_failure: done.is_failure,
error_message: done.error_message,
};
let mut state = shared.lock();
state.prune_expired();
if state.consumed.contains_key(&reference) {
tracing::trace!(
task_run_id = %reference.durable_task_external_id,
invocation_count = reference.invocation_count,
node_id = reference.node_id,
"ignoring a re-delivered completion for an entry that has been consumed",
);
return;
}
if let Some(answer) = state.waits.remove(&reference) {
state.consume(&reference);
drop(state);
drop(answer.send(Ok(completed)));
} else {
state
.unclaimed
.insert(reference, (completed, Instant::now() + COMPLETION_TTL));
}
}
fn fail_ref(shared: &Shared, error: &DurableTaskErrorResponse) {
let Some(reference) = error.r#ref.clone() else {
tracing::warn!(
message = %error.error_message,
"the engine sent a durable error without a log entry ref",
);
return;
};
let nondeterminism = error.error_type == DurableTaskErrorType::Nondeterminism as i32;
let make = || {
if nondeterminism {
Error::Nondeterminism {
task_run_id: reference.durable_task_external_id.clone(),
invocation_count: reference.invocation_count,
node_id: reference.node_id,
message: error.error_message.clone(),
}
} else {
Error::Grpc(Box::new(Status::internal(error.error_message.clone())))
}
};
let (ack, wait) = {
let mut state = shared.lock();
let key = (
reference.durable_task_external_id.clone(),
reference.invocation_count,
);
state.unclaimed.remove(&reference);
(state.acks.remove(&key), state.waits.remove(&reference))
};
if let Some(answer) = ack {
drop(answer.send(Err(make())));
}
if let Some(answer) = wait {
drop(answer.send(Err(make())));
}
}
fn evict(shared: &Shared, notice: &DurableTaskServerEvictNotice) {
tracing::warn!(
task_run_id = %notice.durable_task_external_id,
invocation_count = notice.invocation_count,
reason = %notice.reason,
"the engine evicted a durable invocation",
);
shared.cleanup(
¬ice.durable_task_external_id,
notice.invocation_count,
¬ice.reason,
);
let hook = lock(&shared.evict).clone();
if let Some(hook) = hook {
hook(¬ice.durable_task_external_id, notice.invocation_count);
}
}
fn log_reconnect(attempt: u32, consecutive_other: u32, delay: Duration, status: Option<&Status>) {
let code = status.map(Status::code);
let message = status.map(Status::message).unwrap_or_default();
if attempt == 1 || attempt.is_multiple_of(5) {
tracing::warn!(
attempt,
consecutive_other,
?delay,
?code,
message,
"reconnecting to the engine's durable task stream",
);
} else {
tracing::debug!(
attempt,
consecutive_other,
?delay,
?code,
message,
"reconnecting to the engine's durable task stream",
);
}
}
#[cfg(test)]
mod tests {
use std::sync::atomic::{AtomicUsize, Ordering};
use hatchet_proto::v1::{
DurableTaskErrorResponse, DurableTaskEventMemoAckResponse,
DurableTaskEventTriggerRunsAckResponse, DurableTaskEventWaitForAckResponse,
DurableTaskEvictionAckResponse,
};
use hatchet_testkit::MockEngine;
use tokio::sync::mpsc;
use tonic::Code;
use super::*;
use crate::{Client, TlsStrategy};
const NEVER: Duration = Duration::from_secs(10 * 60);
const SETTLE_BUDGET: Duration = Duration::from_secs(5);
const WORKER: &str = "worker-1";
type ScriptedStream = ReceiverStream<Result<DurableTaskResponse, Status>>;
type ScriptedSender = mpsc::Sender<Result<DurableTaskResponse, Status>>;
type Connections = Arc<Mutex<Vec<Option<ScriptedSender>>>>;
#[derive(Clone)]
struct Script {
calls: Arc<AtomicUsize>,
responses: Connections,
requests: mpsc::UnboundedSender<(usize, DurableTaskRequest)>,
}
impl Script {
async fn connect(self, requests: RequestStream) -> Result<ScriptedStream, Status> {
let connection = self.calls.fetch_add(1, Ordering::SeqCst);
let (responses, received) = mpsc::channel(32);
lock(&self.responses).push(Some(responses));
tokio::spawn(record(connection, requests, self.requests.clone()));
Ok(ReceiverStream::new(received))
}
}
async fn record(
connection: usize,
mut requests: RequestStream,
recorder: mpsc::UnboundedSender<(usize, DurableTaskRequest)>,
) {
while let Some(request) = requests.next().await {
let _ = recorder.send((connection, request));
}
}
struct Opened {
responses: Connections,
incoming: mpsc::UnboundedReceiver<(usize, DurableTaskRequest)>,
seen: Vec<(usize, DurableTaskRequest)>,
calls: Arc<AtomicUsize>,
}
impl Opened {
async fn requests(&mut self, n: usize, count: usize) -> Vec<DurableTaskRequest> {
loop {
let sent: Vec<DurableTaskRequest> = self
.seen
.iter()
.filter(|(connection, _)| *connection == n)
.map(|(_, request)| request.clone())
.collect();
if sent.len() >= count {
return sent;
}
let item = tokio::time::timeout(NEVER, self.incoming.recv())
.await
.expect("timed out waiting for a durable request")
.expect("the durable stream dropped the recorder");
self.seen.push(item);
}
}
fn push(&self, n: usize, response: DurableTaskResponse) {
lock(&self.responses)[n]
.as_ref()
.expect("the connection is still open")
.try_send(Ok(response))
.expect("room in the connection's buffer");
}
fn end(&self, n: usize, status: Option<Status>) {
let responses = lock(&self.responses)[n]
.take()
.expect("the connection is still open");
if let Some(status) = status {
let _ = responses.try_send(Err(status));
}
}
fn calls(&self) -> usize {
self.calls.load(Ordering::SeqCst)
}
}
fn scripted() -> (DurableStream, Opened, CancellationToken) {
let calls = Arc::new(AtomicUsize::new(0));
let responses = Arc::new(Mutex::new(Vec::new()));
let (requests, incoming) = mpsc::unbounded_channel();
let script = Script {
calls: Arc::clone(&calls),
responses: Arc::clone(&responses),
requests,
};
let stop = CancellationToken::new();
let stream = DurableStream::with_backoff(
WORKER.to_string(),
move |requests| script.clone().connect(requests),
stop.clone(),
|_| Duration::ZERO,
);
(
stream,
Opened {
responses,
incoming,
seen: Vec::new(),
calls,
},
stop,
)
}
async fn until(what: &str, condition: impl Fn() -> bool) {
let started = std::time::Instant::now();
while !condition() {
assert!(
started.elapsed() < SETTLE_BUDGET,
"timed out waiting for {what}"
);
tokio::task::yield_now().await;
}
}
async fn joined<T>(handle: tokio::task::JoinHandle<T>) -> T {
tokio::time::timeout(NEVER, handle)
.await
.expect("timed out waiting for a durable caller")
.expect("the waiting task panicked")
}
fn reference(
task_id: &str,
invocation: i32,
branch: i64,
node: i64,
) -> DurableEventLogEntryRef {
DurableEventLogEntryRef {
durable_task_external_id: task_id.to_string(),
invocation_count: invocation,
branch_id: branch,
node_id: node,
}
}
fn conditions() -> DurableEventListenerConditions {
DurableEventListenerConditions::default()
}
fn response(message: durable_task_response::Message) -> DurableTaskResponse {
DurableTaskResponse {
message: Some(message),
}
}
fn wait_for_ack(reference: DurableEventLogEntryRef) -> DurableTaskResponse {
response(durable_task_response::Message::WaitForAck(
DurableTaskEventWaitForAckResponse {
r#ref: Some(reference),
},
))
}
fn memo_ack(
reference: DurableEventLogEntryRef,
existing: Option<&[u8]>,
) -> DurableTaskResponse {
response(durable_task_response::Message::MemoAck(
DurableTaskEventMemoAckResponse {
r#ref: Some(reference),
memo_already_existed: existing.is_some(),
memo_result_payload: existing.map(<[u8]>::to_vec),
},
))
}
fn entry_completed(reference: DurableEventLogEntryRef, payload: &[u8]) -> DurableTaskResponse {
response(durable_task_response::Message::EntryCompleted(
DurableTaskEventLogEntryCompletedResponse {
r#ref: Some(reference),
payload: payload.to_vec(),
is_failure: false,
error_message: None,
},
))
}
fn trigger_runs_ack(
task_id: &str,
invocation: i32,
entries: &[(i64, i64, &str)],
) -> DurableTaskResponse {
response(durable_task_response::Message::TriggerRunsAck(
DurableTaskEventTriggerRunsAckResponse {
durable_task_external_id: task_id.to_string(),
invocation_count: invocation,
run_entries: entries
.iter()
.map(|(node, branch, run_id)| DurableTaskRunAckEntry {
node_id: *node,
branch_id: *branch,
workflow_run_external_id: (*run_id).to_string(),
})
.collect(),
},
))
}
fn eviction_ack(task_id: &str, invocation: i32) -> DurableTaskResponse {
response(durable_task_response::Message::EvictionAck(
DurableTaskEvictionAckResponse {
invocation_count: invocation,
durable_task_external_id: task_id.to_string(),
},
))
}
fn trigger(name: &str) -> TriggerWorkflowRequest {
TriggerWorkflowRequest {
name: name.to_string(),
..Default::default()
}
}
fn nondeterminism(reference: DurableEventLogEntryRef) -> DurableTaskResponse {
response(durable_task_response::Message::Error(
DurableTaskErrorResponse {
r#ref: Some(reference),
error_type: DurableTaskErrorType::Nondeterminism as i32,
error_message: "expected: sleep(10m0s)\n received: memo".to_string(),
},
))
}
fn server_evict(task_id: &str, invocation: i32) -> DurableTaskResponse {
response(durable_task_response::Message::ServerEvict(
DurableTaskServerEvictNotice {
durable_task_external_id: task_id.to_string(),
invocation_count: invocation,
reason: "a newer invocation took over".to_string(),
},
))
}
fn is_register(request: &DurableTaskRequest, worker_id: &str) -> bool {
matches!(
&request.message,
Some(durable_task_request::Message::RegisterWorker(register))
if register.worker_id == worker_id
)
}
fn announced(request: &DurableTaskRequest) -> Option<Vec<String>> {
match &request.message {
Some(durable_task_request::Message::WorkerStatus(status)) => Some(
status
.waiting_entries
.iter()
.map(|entry| entry.durable_task_external_id.clone())
.collect(),
),
_ => None,
}
}
#[tokio::test(start_paused = true)]
async fn the_first_message_on_every_connection_registers_the_worker() {
let (stream, mut opened, _stop) = scripted();
stream.start();
let first = opened.requests(0, 1).await;
assert!(is_register(&first[0], WORKER), "{:?}", first[0].message);
opened.end(0, Some(Status::unavailable("gone")));
let second = opened.requests(1, 1).await;
assert!(is_register(&second[0], WORKER), "{:?}", second[0].message);
assert_eq!(opened.calls(), 2);
}
#[tokio::test(start_paused = true)]
async fn a_wait_is_answered_with_the_ref_the_engine_chose() {
let (stream, mut opened, _stop) = scripted();
stream.start();
opened.requests(0, 1).await;
let chosen = reference("run-1", 1, 7, 41);
let waiting = tokio::spawn({
let stream = stream.clone();
let chosen = chosen.clone();
async move {
let reference = stream
.send_wait_for("run-1".to_string(), 1, conditions(), None)
.await
.expect("the engine acknowledged the wait");
assert_eq!(reference, chosen, "the SDK must use the engine's ref");
stream.wait_for_entry(reference).await
}
});
let sent = opened.requests(0, 2).await;
assert!(matches!(
sent[1].message,
Some(durable_task_request::Message::WaitFor(_))
));
opened.push(0, wait_for_ack(chosen.clone()));
let announcing = opened.requests(0, 3).await;
assert_eq!(
announced(&announcing[2]).as_deref(),
Some(&["run-1".to_string()][..])
);
assert!(
!waiting.is_finished(),
"nothing is answered until it completes"
);
opened.push(0, entry_completed(chosen, br#"{"CREATE":{}}"#));
let payload = joined(waiting).await.expect("the wait is answered");
assert_eq!(payload, br#"{"CREATE":{}}"#.to_vec());
}
#[tokio::test(start_paused = true)]
async fn a_completion_that_arrives_before_its_waiter_is_kept_for_it() {
let (stream, mut opened, _stop) = scripted();
stream.start();
opened.requests(0, 1).await;
let early = reference("run-1", 1, 1, 1);
opened.push(0, entry_completed(early.clone(), b"{\"amount\":7}"));
until("the completion to be buffered", || {
stream.buffered_completions() == 1
})
.await;
let payload = stream
.wait_for_entry(early)
.await
.expect("answered from the buffer");
assert_eq!(payload, b"{\"amount\":7}".to_vec());
assert_eq!(stream.buffered_completions(), 0, "and taken out of it");
}
#[tokio::test(start_paused = true)]
async fn an_unclaimed_completion_is_forgotten_after_its_ttl() {
let (stream, mut opened, _stop) = scripted();
stream.start();
opened.requests(0, 1).await;
opened.push(0, entry_completed(reference("run-1", 1, 1, 1), b"{}"));
until("the completion to be buffered", || {
stream.buffered_completions() == 1
})
.await;
tokio::time::advance(COMPLETION_TTL + Duration::from_secs(1)).await;
assert_eq!(stream.buffered_completions(), 0);
}
#[tokio::test(start_paused = true)]
async fn a_reconnect_fails_the_acks_and_keeps_the_waits() {
let (stream, mut opened, _stop) = scripted();
stream.start();
opened.requests(0, 1).await;
let kept = reference("run-2", 1, 1, 1);
let waiting = tokio::spawn({
let stream = stream.clone();
let kept = kept.clone();
async move { stream.wait_for_entry(kept).await }
});
opened.requests(0, 2).await;
let acking = tokio::spawn({
let stream = stream.clone();
async move {
stream
.send_wait_for("run-1".to_string(), 1, conditions(), None)
.await
}
});
opened.requests(0, 3).await;
opened.end(0, Some(Status::unavailable("gone")));
let failed = joined(acking)
.await
.expect_err("the ack cannot survive the stream");
let Error::Grpc(status) = &failed else {
panic!("{failed:?}")
};
assert_eq!(status.code(), Code::Unavailable);
assert!(!waiting.is_finished(), "the wait survives the reconnect");
let announcing = opened.requests(1, 2).await;
assert!(is_register(&announcing[0], WORKER));
assert!(
announcing
.iter()
.any(|request| announced(request).is_some_and(|tasks| tasks == ["run-2"])),
"{announcing:?}",
);
assert!(
!announcing.iter().any(|request| matches!(
request.message,
Some(durable_task_request::Message::WaitFor(_))
)),
"{announcing:?}",
);
opened.push(1, entry_completed(kept, b"{}"));
assert_eq!(joined(waiting).await.expect("answered"), b"{}".to_vec());
}
#[tokio::test(start_paused = true)]
async fn a_nondeterminism_error_answers_the_ack_and_the_wait_for_that_ref() {
let (stream, mut opened, _stop) = scripted();
stream.start();
opened.requests(0, 1).await;
let acking = tokio::spawn({
let stream = stream.clone();
async move {
stream
.send_wait_for("run-1".to_string(), 2, conditions(), None)
.await
}
});
opened.requests(0, 2).await;
opened.push(0, nondeterminism(reference("run-1", 2, 1, 3)));
let failed = joined(acking).await.expect_err("the engine refused it");
let Error::Nondeterminism {
task_run_id,
invocation_count,
node_id,
..
} = &failed
else {
panic!("{failed:?}")
};
assert_eq!(
(task_run_id.as_str(), *invocation_count, *node_id),
("run-1", 2, 3)
);
let diverged = reference("run-1", 2, 1, 4);
let waiting = tokio::spawn({
let stream = stream.clone();
let diverged = diverged.clone();
async move { stream.wait_for_entry(diverged).await }
});
opened.requests(0, 3).await;
opened.push(0, nondeterminism(diverged));
let failed = joined(waiting).await.expect_err("the engine refused it");
assert!(
matches!(failed, Error::Nondeterminism { node_id: 4, .. }),
"{failed:?}"
);
}
#[tokio::test(start_paused = true)]
async fn a_server_evict_ends_the_invocations_waits_and_tells_the_worker() {
let (stream, mut opened, _stop) = scripted();
let evicted: Arc<Mutex<Vec<(String, i32)>>> = Arc::new(Mutex::new(Vec::new()));
stream.on_server_evict({
let evicted = Arc::clone(&evicted);
move |task_id, invocation| lock(&evicted).push((task_id.to_string(), invocation))
});
stream.start();
opened.requests(0, 1).await;
let stale = reference("run-1", 1, 1, 1);
let current = reference("run-1", 2, 1, 1);
let ending = tokio::spawn({
let stream = stream.clone();
let stale = stale.clone();
async move { stream.wait_for_entry(stale).await }
});
let surviving = tokio::spawn({
let stream = stream.clone();
let current = current.clone();
async move { stream.wait_for_entry(current).await }
});
opened.requests(0, 2).await;
until("both waits to be registered", || {
lock(&stream.shared.state).waits.len() == 2
})
.await;
opened.push(0, server_evict("run-1", 1));
let failed = joined(ending).await.expect_err("the invocation is stale");
let Error::Grpc(status) = &failed else {
panic!("{failed:?}")
};
assert_eq!(status.code(), Code::Cancelled);
assert!(!surviving.is_finished());
until("the worker to be told", || !lock(&evicted).is_empty()).await;
assert_eq!(*lock(&evicted), [("run-1".to_string(), 1)]);
opened.push(0, entry_completed(current, b"{}"));
assert_eq!(joined(surviving).await.expect("answered"), b"{}".to_vec());
}
#[tokio::test(start_paused = true)]
async fn a_second_request_for_one_invocation_is_refused_rather_than_stealing_the_ack() {
let (stream, mut opened, _stop) = scripted();
stream.start();
opened.requests(0, 1).await;
let first = tokio::spawn({
let stream = stream.clone();
async move {
stream
.send_wait_for("run-1".to_string(), 1, conditions(), None)
.await
}
});
opened.requests(0, 2).await;
let refused = stream
.send_memo("run-1".to_string(), 1, "key", None)
.await
.expect_err("one ack at a time");
assert!(matches!(refused, Error::Grpc(_)), "{refused:?}");
opened.push(0, wait_for_ack(reference("run-1", 1, 1, 1)));
assert_eq!(
joined(first).await.expect("acknowledged"),
reference("run-1", 1, 1, 1),
);
}
#[tokio::test(start_paused = true)]
async fn a_memo_ack_carries_the_value_an_earlier_invocation_recorded() {
let (stream, mut opened, _stop) = scripted();
stream.start();
opened.requests(0, 1).await;
let asking = tokio::spawn({
let stream = stream.clone();
async move { stream.send_memo("run-1".to_string(), 2, "now", None).await }
});
opened.requests(0, 2).await;
opened.push(
0,
memo_ack(reference("run-1", 2, 1, 1), Some(b"{\"value\":1}")),
);
let ack = joined(asking).await.expect("acknowledged");
assert_eq!(ack.reference, reference("run-1", 2, 1, 1));
assert_eq!(ack.existing.as_deref(), Some(&b"{\"value\":1}"[..]));
let asking = tokio::spawn({
let stream = stream.clone();
async move { stream.send_memo("run-2".to_string(), 1, "now", None).await }
});
opened.requests(0, 3).await;
opened.push(0, memo_ack(reference("run-2", 1, 1, 1), None));
assert!(
joined(asking)
.await
.expect("acknowledged")
.existing
.is_none()
);
}
#[tokio::test(start_paused = true)]
async fn an_unacknowledged_request_gives_up_after_the_ack_timeout() {
let (stream, mut opened, _stop) = scripted();
stream.start();
opened.requests(0, 1).await;
let failed = stream
.send_wait_for("run-1".to_string(), 1, conditions(), None)
.await
.expect_err("the request was lost on the stream");
let Error::Grpc(status) = &failed else {
panic!("{failed:?}")
};
assert_eq!(status.code(), Code::DeadlineExceeded);
}
#[tokio::test(start_paused = true)]
async fn a_fatal_status_fails_every_wait_and_ends_the_loop() {
let (stream, mut opened, _stop) = scripted();
stream.start();
opened.requests(0, 1).await;
let waiting = tokio::spawn({
let stream = stream.clone();
async move { stream.wait_for_entry(reference("run-1", 1, 1, 1)).await }
});
opened.requests(0, 2).await;
opened.end(0, Some(Status::unauthenticated("revoked")));
let failed = joined(waiting).await.expect_err("a fatal stream error");
let Error::Grpc(status) = &failed else {
panic!("{failed:?}")
};
assert_eq!(status.code(), Code::Unauthenticated);
let refused = stream
.wait_for_entry(reference("run-1", 1, 1, 2))
.await
.expect_err("the stream has ended");
assert!(matches!(refused, Error::Grpc(_)), "{refused:?}");
assert_eq!(opened.calls(), 1);
}
#[tokio::test(start_paused = true)]
async fn complete_memo_is_sent_and_not_waited_for() {
let (stream, mut opened, _stop) = scripted();
stream.start();
opened.requests(0, 1).await;
stream.send_complete_memo(reference("run-1", 1, 1, 1), "now", b"{\"at\":1}".to_vec());
let sent = opened.requests(0, 2).await;
let Some(durable_task_request::Message::CompleteMemo(completed)) = &sent[1].message else {
panic!("{:?}", sent[1].message)
};
assert_eq!(completed.r#ref, Some(reference("run-1", 1, 1, 1)));
assert_eq!(completed.memo_key, b"now".to_vec());
assert_eq!(completed.payload, b"{\"at\":1}".to_vec());
}
#[test]
fn a_reconnect_drops_the_unanswered_requests_and_keeps_the_rest() {
let shared = Shared::new(WORKER.to_string());
let (answer, mut answered) = oneshot::channel();
{
let mut state = shared.lock();
state.acks.insert(("run-1".to_string(), 1), answer);
state
.outgoing
.push_back(request(durable_task_request::Message::WaitFor(
DurableTaskWaitForRequest {
invocation_count: 1,
durable_task_external_id: "run-1".to_string(),
wait_for_conditions: Some(conditions()),
label: None,
},
)));
state
.outgoing
.push_back(request(durable_task_request::Message::CompleteMemo(
DurableTaskCompleteMemoRequest {
r#ref: Some(reference("run-1", 1, 1, 1)),
payload: b"{}".to_vec(),
memo_key: b"now".to_vec(),
},
)));
}
shared.fail_acks(Some(&Status::unavailable("gone")));
let failed = answered
.try_recv()
.expect("the ack was answered")
.expect_err("with the status the stream ended with");
assert!(matches!(failed, Error::Grpc(_)), "{failed:?}");
let state = shared.lock();
assert!(state.acks.is_empty());
assert_eq!(state.outgoing.len(), 1);
assert!(matches!(
state.outgoing[0].message,
Some(durable_task_request::Message::CompleteMemo(_))
));
}
#[tokio::test(start_paused = true)]
async fn cleanup_ends_the_waits_of_the_invocations_it_names() {
let (stream, mut opened, _stop) = scripted();
stream.start();
opened.requests(0, 1).await;
let ending = tokio::spawn({
let stream = stream.clone();
async move { stream.wait_for_entry(reference("run-1", 1, 1, 1)).await }
});
let untouched = tokio::spawn({
let stream = stream.clone();
async move { stream.wait_for_entry(reference("run-9", 1, 1, 1)).await }
});
until("both waits to be registered", || {
lock(&stream.shared.state).waits.len() == 2
})
.await;
stream.cleanup("run-1", 1, "the worker is shutting down");
let failed = joined(ending).await.expect_err("that invocation is over");
assert!(matches!(failed, Error::Grpc(_)), "{failed:?}");
assert!(!untouched.is_finished(), "another run is not touched");
}
#[tokio::test]
async fn the_engine_sees_a_register_on_every_connection() {
let engine = MockEngine::start().await;
let client = client_for(&engine);
let stop = CancellationToken::new();
let stream = DurableStream::with_backoff(
WORKER.to_string(),
connect(client.grpc_service(), client.config().gzip()),
stop.clone(),
|_| Duration::ZERO,
);
stream.start();
engine.wait_for_calls("DurableTask", 1).await;
engine.disconnect_durable_streams_with(Status::unavailable("gone"));
engine.wait_for_calls("DurableTask", 2).await;
engine
.wait_until("two register_worker requests", |engine| {
engine
.durable_requests()
.iter()
.filter(|request| is_register(request, WORKER))
.count()
>= 2
})
.await;
stop.cancel();
}
#[tokio::test(start_paused = true)]
async fn the_outstanding_waits_are_announced_every_second() {
let _clock = hatchet_testkit::pin_paused_clock();
let engine = MockEngine::start().await;
let client = client_for(&engine);
let stop = CancellationToken::new();
let stream = DurableStream::new(
WORKER.to_string(),
connect(client.grpc_service(), client.config().gzip()),
stop.clone(),
);
stream.start();
engine.wait_for_durable_streams(1).await;
let waiting = tokio::spawn({
let stream = stream.clone();
async move { stream.wait_for_entry(reference("run-1", 1, 1, 1)).await }
});
engine
.wait_until("three worker_status requests", |engine| {
engine
.durable_requests()
.iter()
.filter(|request| announced(request).is_some_and(|tasks| tasks == ["run-1"]))
.count()
>= 3
})
.await;
assert!(!waiting.is_finished(), "and it is still waiting");
waiting.abort();
stop.cancel();
}
#[tokio::test(start_paused = true)]
async fn trigger_runs_is_answered_with_the_entries_the_engine_named() {
let (stream, mut opened, _stop) = scripted();
stream.start();
let spawning = tokio::spawn({
let stream = stream.clone();
async move {
stream
.send_trigger_runs(
"run-1".to_string(),
1,
vec![trigger("dev_child"), trigger("dev_child")],
)
.await
}
});
let sent = opened.requests(0, 2).await;
let Some(durable_task_request::Message::TriggerRuns(request)) = &sent[1].message else {
panic!(
"the second request is a trigger_runs: {:?}",
sent[1].message
);
};
assert_eq!(request.durable_task_external_id, "run-1");
assert_eq!(request.invocation_count, 1);
assert_eq!(request.trigger_opts.len(), 2);
opened.push(
0,
trigger_runs_ack("run-1", 1, &[(7, 1, "child-a"), (8, 1, "child-b")]),
);
let entries = joined(spawning).await.expect("the ack answers the caller");
let ids: Vec<&str> = entries
.iter()
.map(|entry| entry.workflow_run_external_id.as_str())
.collect();
assert_eq!(ids, ["child-a", "child-b"]);
assert_eq!(entries[0].node_id, 7);
assert_eq!(entries[1].node_id, 8);
}
#[tokio::test(start_paused = true)]
async fn a_wait_ack_does_not_answer_a_trigger_runs_request() {
let (stream, mut opened, _stop) = scripted();
stream.start();
let spawning = tokio::spawn({
let stream = stream.clone();
async move {
stream
.send_trigger_runs("run-1".to_string(), 1, vec![trigger("dev_child")])
.await
}
});
let _ = opened.requests(0, 2).await;
opened.push(0, wait_for_ack(reference("run-1", 1, 1, 1)));
let error = joined(spawning).await.unwrap_err();
assert!(error.to_string().contains("wrong kind"), "{error}");
}
#[tokio::test(start_paused = true)]
async fn evict_waits_for_the_engines_eviction_ack() {
let (stream, mut opened, _stop) = scripted();
stream.start();
let evicting = tokio::spawn({
let stream = stream.clone();
async move {
stream
.send_evict("run-1", 2, Some("Worker at capacity".to_string()))
.await
}
});
let sent = opened.requests(0, 2).await;
let Some(durable_task_request::Message::EvictInvocation(request)) = &sent[1].message else {
panic!(
"the second request is an evict_invocation: {:?}",
sent[1].message
);
};
assert_eq!(request.durable_task_external_id, "run-1");
assert_eq!(request.invocation_count, 2);
assert_eq!(request.reason.as_deref(), Some("Worker at capacity"));
opened.push(0, eviction_ack("run-1", 1));
tokio::task::yield_now().await;
assert!(
!evicting.is_finished(),
"an ack for invocation 1 answered invocation 2"
);
opened.push(0, eviction_ack("run-1", 2));
joined(evicting).await.expect("the ack answers the evictor");
}
#[tokio::test(start_paused = true)]
async fn an_eviction_that_is_never_acknowledged_fails_rather_than_hangs() {
let (stream, mut opened, _stop) = scripted();
stream.start();
let evicting = tokio::spawn({
let stream = stream.clone();
async move { stream.send_evict("run-1", 1, None).await }
});
let _ = opened.requests(0, 2).await;
tokio::time::advance(EVICTION_ACK_TIMEOUT + Duration::from_secs(1)).await;
let error = joined(evicting).await.unwrap_err();
assert!(error.to_string().contains("evict"), "{error}");
}
#[tokio::test(start_paused = true)]
async fn claimed_entries_are_waited_for_from_the_moment_they_are_claimed() {
let (stream, mut opened, _stop) = scripted();
stream.start();
let first = stream
.claim_entry(reference("run-1", 1, 1, 7))
.expect("a fresh entry");
let second = stream
.claim_entry(reference("run-1", 1, 1, 8))
.expect("a fresh entry");
let sent = opened.requests(0, 2).await;
let announced: Vec<i64> = match &sent[1].message {
Some(durable_task_request::Message::WorkerStatus(status)) => {
let mut nodes: Vec<i64> = status
.waiting_entries
.iter()
.map(|entry| entry.node_id)
.collect();
nodes.sort_unstable();
nodes
}
other => panic!("the second request announces the waits: {other:?}"),
};
assert_eq!(announced, [7, 8]);
opened.push(
0,
entry_completed(reference("run-1", 1, 1, 8), br#"{"n":2}"#),
);
until("the second completion to be delivered", || {
!lock(&stream.shared.state)
.waits
.contains_key(&reference("run-1", 1, 1, 8))
})
.await;
tokio::time::advance(COMPLETION_TTL * 3).await;
opened.push(
0,
entry_completed(reference("run-1", 1, 1, 7), br#"{"n":1}"#),
);
assert_eq!(second.completed().await.unwrap(), br#"{"n":2}"#.to_vec());
assert_eq!(first.completed().await.unwrap(), br#"{"n":1}"#.to_vec());
}
#[tokio::test(start_paused = true)]
async fn a_dropped_claim_stops_waiting() {
let (stream, _opened, _stop) = scripted();
stream.start();
let claimed = stream
.claim_entry(reference("run-1", 1, 1, 7))
.expect("a fresh entry");
drop(claimed);
stream
.claim_entry(reference("run-1", 1, 1, 7))
.expect("the dropped claim gave the entry up");
}
#[tokio::test(start_paused = true)]
async fn a_re_delivered_completion_is_ignored_rather_than_buffered() {
let (stream, mut opened, _stop) = scripted();
stream.start();
opened.requests(0, 1).await;
let entry = reference("run-1", 1, 1, 1);
let claimed = stream.claim_entry(entry.clone()).expect("a fresh entry");
opened.push(
0,
entry_completed(entry.clone(), br#"{"from":"child","n":0}"#),
);
assert_eq!(
claimed.completed().await.expect("the first delivery"),
br#"{"from":"child","n":0}"#.to_vec(),
);
assert!(
!lock(&stream.shared.state).waits.contains_key(&entry),
"a consumed entry is still announced",
);
opened.push(
0,
entry_completed(entry.clone(), br#"{"n": 0, "from": "child"}"#),
);
until("the re-delivery to be read", || {
lock(&stream.shared.state).consumed.contains_key(&entry)
})
.await;
assert_eq!(
stream.buffered_completions(),
0,
"a re-delivered completion must not sit in the unclaimed buffer",
);
let Err(refused) = stream.claim_entry(entry) else {
panic!("a second claim on a spent entry would wait for ever");
};
assert!(matches!(refused, Error::Grpc(_)), "{refused:?}");
}
fn client_for(engine: &MockEngine) -> Client {
Client::builder()
.token(engine.token())
.namespace("dev")
.tls(TlsStrategy::None)
.build()
.expect("a client pointed at the mock engine")
}
}