use std::any::Any;
use std::collections::HashMap;
use std::pin::Pin;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Duration;
use hatchet_proto::root::{ActionType, AssignedAction, BatchActionEvent, StepActionEvent};
use serde_json::Value;
use tokio::sync::{Notify, Semaphore};
use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use tracing::{Instrument, Span};
use super::events;
use crate::batch::{self, BatchMemberId, BatchReport};
use crate::context::Slot;
use crate::declare::RegisteredAction;
use crate::durable::DurableInvocation;
use crate::durable::eviction::EvictionBook;
use crate::durable::stream::DurableStream;
use crate::{Client, Context, TaskError, otel};
#[allow(clippy::type_complexity)]
pub(crate) type Reporter =
Arc<dyn Fn(StepActionEvent) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
#[allow(clippy::type_complexity)]
pub(crate) type BatchReporter =
Arc<dyn Fn(BatchActionEvent) -> Pin<Box<dyn Future<Output = ()> + Send>> + Send + Sync>;
pub(crate) const CANCEL_SETTLE: Duration = Duration::from_secs(1);
pub(crate) struct Runner {
inner: Arc<Inner>,
}
struct Inner {
worker_id: String,
client: Option<Client>,
handlers: HashMap<String, RegisteredAction>,
slots: Arc<Semaphore>,
durable: Option<DurableRuntime>,
report: Reporter,
report_batch: BatchReporter,
runs: Mutex<HashMap<String, Run>>,
idle: Notify,
next_seq: AtomicU64,
}
struct Run {
seq: u64,
cancel: CancellationToken,
kill: CancellationToken,
}
pub(crate) struct DurableSetup {
pub(crate) slots: usize,
pub(crate) stream: DurableStream,
pub(crate) eviction: Option<Arc<EvictionBook>>,
pub(crate) engine_version: Arc<str>,
}
struct DurableRuntime {
slots: Arc<Semaphore>,
stream: DurableStream,
engine_version: Arc<str>,
eviction: Option<Arc<EvictionBook>>,
}
impl From<DurableSetup> for DurableRuntime {
fn from(setup: DurableSetup) -> Self {
Self {
slots: Arc::new(Semaphore::new(setup.slots)),
stream: setup.stream,
engine_version: setup.engine_version,
eviction: setup.eviction,
}
}
}
impl Runner {
pub(crate) fn new(
worker_id: String,
client: Option<Client>,
handlers: HashMap<String, RegisteredAction>,
slots: usize,
durable: Option<DurableSetup>,
report: Reporter,
report_batch: BatchReporter,
) -> Self {
Self {
inner: Arc::new(Inner {
worker_id,
client,
handlers,
slots: Arc::new(Semaphore::new(slots)),
durable: durable.map(DurableRuntime::from),
report,
report_batch,
runs: Mutex::new(HashMap::new()),
idle: Notify::new(),
next_seq: AtomicU64::new(0),
}),
}
}
pub(crate) fn dispatch(&self, action: AssignedAction) {
match ActionType::try_from(action.action_type) {
Ok(ActionType::StartStepRun) => self.start(action),
Ok(ActionType::StartBatch) => self.start_batch(action),
Ok(ActionType::CancelStepRun) => {
let span = otel::cancel_step_run(&action.task_run_external_id);
span.in_scope(|| self.cancel(&action.task_run_external_id));
}
Ok(other) => tracing::warn!(
action_type = other.as_str_name(),
action_id = %action.action_id,
task_run_external_id = %action.task_run_external_id,
"ignoring an action type this SDK does not run",
),
Err(_) => tracing::warn!(
action_type = action.action_type,
action_id = %action.action_id,
"ignoring an action type this SDK does not know",
),
}
}
pub(crate) fn in_flight(&self) -> usize {
self.inner.in_flight()
}
pub(crate) async fn drain(&self, grace: Duration) {
if tokio::time::timeout(grace, self.inner.wait_until_idle())
.await
.is_ok()
{
return;
}
let asked = self.inner.cancel_all();
tracing::info!(
runs = asked,
"the grace period is over; asking the runs still in flight to stop",
);
if tokio::time::timeout(CANCEL_SETTLE, self.inner.wait_until_idle())
.await
.is_ok()
{
return;
}
self.drop_everything("dropping runs that did not stop when asked to")
.await;
}
pub(crate) async fn abort_all(&self) {
self.inner.cancel_all();
self.drop_everything("dropping the runs still in flight")
.await;
}
fn start(&self, action: AssignedAction) {
let id = action.task_run_external_id.clone();
let cancel = CancellationToken::new();
let kill = CancellationToken::new();
let seq = self.inner.register(&id, cancel.clone(), kill.clone());
let registration = Registration {
inner: Arc::clone(&self.inner),
id,
seq,
};
let inner = Arc::clone(&self.inner);
tokio::spawn(async move {
let _registration = registration;
tokio::select! {
() = kill.cancelled() => {}
() = run(&inner, action, cancel) => {}
}
});
}
fn start_batch(&self, action: AssignedAction) {
let id = batch::run_key(&action);
let cancel = CancellationToken::new();
let kill = CancellationToken::new();
let seq = self.inner.register(&id, cancel.clone(), kill.clone());
let registration = Registration {
inner: Arc::clone(&self.inner),
id,
seq,
};
let inner = Arc::clone(&self.inner);
tokio::spawn(async move {
let _registration = registration;
tokio::select! {
() = kill.cancelled() => {}
() = run_batch(&inner, action, cancel) => {}
}
});
}
pub(crate) fn cancel(&self, id: &str) {
if self.inner.cancel_run(id) {
tracing::debug!(task_run_external_id = %id, "a run was asked to stop");
} else {
tracing::debug!(
task_run_external_id = %id,
"ignoring a cancellation for a run this worker is not holding",
);
}
}
async fn drop_everything(&self, reason: &str) {
let dropped = self.inner.kill_all();
if dropped > 0 {
tracing::warn!(runs = dropped, "{reason}");
tokio::task::yield_now().await;
}
}
}
impl Inner {
fn register(&self, id: &str, cancel: CancellationToken, kill: CancellationToken) -> u64 {
let seq = self.next_seq.fetch_add(1, Ordering::Relaxed);
let previous = self
.runs
.lock()
.unwrap_or_else(PoisonError::into_inner)
.insert(id.to_string(), Run { seq, cancel, kill });
if let Some(previous) = previous {
tracing::warn!(
task_run_external_id = %id,
"the engine assigned a run that is already in flight; dropping the earlier one",
);
previous.cancel.cancel();
previous.kill.cancel();
}
seq
}
fn finish(&self, id: &str, seq: u64) {
let idle = {
let mut runs = self.runs.lock().unwrap_or_else(PoisonError::into_inner);
if runs.get(id).is_some_and(|run| run.seq == seq) {
runs.remove(id);
}
runs.is_empty()
};
if idle {
self.idle.notify_waiters();
}
}
fn in_flight(&self) -> usize {
self.runs
.lock()
.unwrap_or_else(PoisonError::into_inner)
.len()
}
fn cancel_run(&self, id: &str) -> bool {
let token = self
.runs
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get(id)
.map(|run| run.cancel.clone());
match token {
Some(token) => {
token.cancel();
true
}
None => false,
}
}
fn cancel_all(&self) -> usize {
let tokens: Vec<CancellationToken> = self
.runs
.lock()
.unwrap_or_else(PoisonError::into_inner)
.values()
.map(|run| run.cancel.clone())
.collect();
for token in &tokens {
token.cancel();
}
tokens.len()
}
fn kill_all(&self) -> usize {
let runs = {
let mut guard = self.runs.lock().unwrap_or_else(PoisonError::into_inner);
std::mem::take(&mut *guard)
};
for run in runs.values() {
run.cancel.cancel();
run.kill.cancel();
}
if !runs.is_empty() {
self.idle.notify_waiters();
}
runs.len()
}
fn slots_for(&self, action_id: &str) -> Arc<Semaphore> {
let durable = self
.handlers
.get(action_id)
.is_some_and(|registered| registered.durable);
match (durable, &self.durable) {
(true, Some(runtime)) => Arc::clone(&runtime.slots),
_ => Arc::clone(&self.slots),
}
}
async fn wait_until_idle(&self) {
loop {
let notified = self.idle.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if self.in_flight() == 0 {
return;
}
notified.await;
}
}
}
async fn run(inner: &Inner, action: AssignedAction, cancel: CancellationToken) {
let acquired = tokio::select! {
biased;
() = cancel.cancelled() => {
tracing::debug!(
task_run_external_id = %action.task_run_external_id,
"a run was cancelled while it waited for a slot; not starting it",
);
return;
}
permit = inner.slots_for(&action.action_id).acquire_owned() => permit,
};
let Ok(permit) = acquired else {
tracing::error!("the worker's slots were closed; dropping the run");
return;
};
let slot = Slot::holding(permit);
let _slot_guard = SlotGuard(slot.clone());
if cancel.is_cancelled() {
tracing::debug!(
task_run_external_id = %action.task_run_external_id,
"a run was cancelled before it started; not starting it",
);
return;
}
let span = otel::start_step_run(&action, &inner.worker_id);
let started = tokio::spawn({
let report = Arc::clone(&inner.report);
let event = events::started(&action, &inner.worker_id);
async move { report(event).await }
});
let outcome = execute(inner, &action, cancel.clone(), slot, &span)
.instrument(span.clone())
.await;
if let Err(error) = started.await {
tracing::warn!(
%error,
task_run_external_id = %action.task_run_external_id,
"reporting that a run started did not finish",
);
}
if cancel.is_cancelled() {
tracing::debug!(
task_run_external_id = %action.task_run_external_id,
"a cancelled run finished; not reporting an outcome",
);
return;
}
let event = match outcome {
Outcome::Completed(output) => {
otel::record_ok(&span);
events::completed(&action, &inner.worker_id, &output)
}
Outcome::Failed(error) => {
otel::record_error(&span, error.message());
span.in_scope(|| {
tracing::warn!(
task_run_external_id = %action.task_run_external_id,
action_id = %action.action_id,
retryable = error.is_retryable(),
message = error.message(),
"a task run failed",
);
});
events::failed(&action, &inner.worker_id, &error)
}
Outcome::Stopped => return,
};
(inner.report)(event).await;
}
async fn execute(
inner: &Inner,
action: &AssignedAction,
cancel: CancellationToken,
slot: Slot,
span: &Span,
) -> Outcome {
let Some(registered) = inner.handlers.get(&action.action_id).cloned() else {
return Outcome::Failed(TaskError::non_retryable(format!(
"this worker has no task registered for the action `{}`",
action.action_id
)));
};
let (durable, _registration) = if registered.durable {
let Some(runtime) = inner.durable.as_ref() else {
return Outcome::Failed(TaskError::non_retryable(format!(
"the action `{}` is durable but this worker has no durable stream",
action.action_id
)));
};
let invocation_count = action.durable_task_invocation_count.unwrap_or(1).max(1);
let registration = runtime.eviction.as_ref().map(|book| {
book.take_on(
&action.task_run_external_id,
invocation_count,
registered.eviction_policy.clone(),
)
});
(
Some(Arc::new(DurableInvocation::new(
runtime.stream.clone(),
action.task_run_external_id.clone(),
invocation_count,
Arc::clone(&runtime.engine_version),
runtime.eviction.clone(),
))),
registration,
)
} else {
(None, None)
};
let handler = registered.handler;
let context = match Context::new(
action.clone(),
inner.worker_id.clone(),
inner.client.clone(),
cancel,
slot,
durable,
) {
Ok(context) => context,
Err(reason) => return Outcome::Failed(TaskError::non_retryable(reason)),
};
let input = context.input().clone();
let mut handler_task = AbortOnDrop(tokio::spawn(
async move { handler(context, input).await }.instrument(span.clone()),
));
match (&mut handler_task.0).await {
Ok(Ok(output)) => Outcome::Completed(output),
Ok(Err(error)) => Outcome::Failed(error),
Err(error) if error.is_panic() => {
let message = panic_message(error.into_panic());
tracing::error!(
task_run_external_id = %action.task_run_external_id,
action_id = %action.action_id,
message,
"a task handler panicked",
);
Outcome::Failed(TaskError::new(message))
}
Err(_) => Outcome::Stopped,
}
}
async fn run_batch(inner: &Inner, action: AssignedAction, cancel: CancellationToken) {
let members = batch::member_ids(&action);
if members.is_empty() {
tracing::error!(
action_id = %action.action_id,
batch_id = ?action.batch_id,
"a batch arrived naming no members; dropping it",
);
return;
}
let acquired = tokio::select! {
biased;
() = cancel.cancelled() => {
tracing::debug!(
batch_id = ?action.batch_id,
"a batch was cancelled while it waited for a slot; not starting it",
);
return;
}
permit = inner.slots_for(&action.action_id).acquire_owned() => permit,
};
let Ok(permit) = acquired else {
tracing::error!("the worker's slots were closed; dropping the batch");
return;
};
let slot = Slot::holding(permit);
let _slot_guard = SlotGuard(slot.clone());
if cancel.is_cancelled() {
tracing::debug!(
batch_id = ?action.batch_id,
"a batch was cancelled before it started; not starting it",
);
return;
}
let span = otel::start_batch_run(&action, &inner.worker_id, members.len());
let started = tokio::spawn({
let report = Arc::clone(&inner.report_batch);
let event = events::batch_started(&action, &inner.worker_id, &members);
async move { report(event).await }
});
let outcome = execute_batch(inner, &action, cancel.clone(), slot, &span)
.instrument(span.clone())
.await;
if let Err(error) = started.await {
tracing::warn!(
%error,
batch_id = ?action.batch_id,
"reporting that a batch started did not finish",
);
}
if cancel.is_cancelled() {
tracing::debug!(
batch_id = ?action.batch_id,
"a cancelled batch finished; not reporting an outcome",
);
return;
}
let report = match outcome {
BatchOutcome::Reported(report) => {
otel::record_ok(&span);
report
}
BatchOutcome::FailedAll(error) => {
otel::record_error(&span, error.message());
span.in_scope(|| {
tracing::warn!(
batch_id = ?action.batch_id,
action_id = %action.action_id,
members = members.len(),
retryable = error.is_retryable(),
message = error.message(),
"a batch failed as a whole",
);
});
batch::fail_all(&members, &error)
}
BatchOutcome::Stopped => return,
};
let mut completed: Vec<(BatchMemberId, Value)> = Vec::new();
let mut failed: Vec<(BatchMemberId, TaskError)> = Vec::new();
for (id, result) in report {
match result {
Ok(output) => completed.push((id, output)),
Err(error) => failed.push((id, error)),
}
}
if !completed.is_empty() {
(inner.report_batch)(events::batch_completed(
&action,
&inner.worker_id,
&completed,
))
.await;
}
if !failed.is_empty() {
tracing::warn!(
batch_id = ?action.batch_id,
action_id = %action.action_id,
members = failed.len(),
"some members of a batch failed",
);
(inner.report_batch)(events::batch_failed(&action, &inner.worker_id, &failed)).await;
}
}
async fn execute_batch(
inner: &Inner,
action: &AssignedAction,
cancel: CancellationToken,
slot: Slot,
span: &Span,
) -> BatchOutcome {
let Some(handler) = inner
.handlers
.get(&action.action_id)
.and_then(|registered| registered.batch.clone())
else {
return BatchOutcome::FailedAll(TaskError::non_retryable(format!(
"this worker has no batch task registered for the action `{}`",
action.action_id
)));
};
let context = match Context::new(
action.clone(),
inner.worker_id.clone(),
inner.client.clone(),
cancel,
slot,
None,
) {
Ok(context) => context,
Err(reason) => return BatchOutcome::FailedAll(TaskError::non_retryable(reason)),
};
let assignment = action.clone();
let mut handler_task = AbortOnDrop(tokio::spawn(
async move { handler(context, assignment).await }.instrument(span.clone()),
));
match (&mut handler_task.0).await {
Ok(Ok(report)) => BatchOutcome::Reported(report),
Ok(Err(error)) => BatchOutcome::FailedAll(error),
Err(error) if error.is_panic() => {
let message = panic_message(error.into_panic());
tracing::error!(
batch_id = ?action.batch_id,
action_id = %action.action_id,
message,
"a batch handler panicked",
);
BatchOutcome::FailedAll(TaskError::new(message))
}
Err(_) => BatchOutcome::Stopped,
}
}
enum BatchOutcome {
Reported(BatchReport),
FailedAll(TaskError),
Stopped,
}
enum Outcome {
Completed(Value),
Failed(TaskError),
Stopped,
}
struct Registration {
inner: Arc<Inner>,
id: String,
seq: u64,
}
impl Drop for Registration {
fn drop(&mut self) {
self.inner.finish(&self.id, self.seq);
}
}
struct SlotGuard(Slot);
impl Drop for SlotGuard {
fn drop(&mut self) {
self.0.release();
}
}
struct AbortOnDrop<T>(JoinHandle<T>);
impl<T> Drop for AbortOnDrop<T> {
fn drop(&mut self) {
self.0.abort();
}
}
fn panic_message(payload: Box<dyn Any + Send>) -> String {
let detail = payload
.downcast_ref::<&'static str>()
.map(|message| (*message).to_string())
.or_else(|| payload.downcast_ref::<String>().cloned());
match detail {
Some(detail) => format!("task panicked: {detail}"),
None => "task panicked".to_string(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{Register, TaskBuilder, TaskError};
use hatchet_proto::root::{ActionType, StepActionEventType};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use tokio::sync::Semaphore;
use tokio::sync::mpsc;
use tokio::sync::mpsc::error::TryRecvError;
use tokio::time::Instant;
#[derive(Deserialize, Serialize)]
struct In {
repo: String,
}
#[derive(Deserialize, Serialize)]
struct Out {
pr: String,
}
const NEVER: Duration = Duration::from_secs(10);
fn handler_of<R: Register>(task: &R) -> (String, RegisteredAction) {
let mut actions = task
.definition()
.actions("")
.expect("the task declaration is valid");
actions.remove(0)
}
fn runner_with(
handlers: Vec<(String, RegisteredAction)>,
slots: usize,
) -> (Runner, mpsc::UnboundedReceiver<StepActionEvent>) {
let (tx, rx) = mpsc::unbounded_channel();
let report: Reporter = Arc::new(move |event: StepActionEvent| {
let tx = tx.clone();
Box::pin(async move {
let _ = tx.send(event);
})
});
let runner = Runner::new(
"worker-1".to_string(),
None,
handlers.into_iter().collect(),
slots,
None,
report,
discarded_batch_reports(),
);
(runner, rx)
}
fn runner_with_batch(
handlers: Vec<(String, RegisteredAction)>,
slots: usize,
) -> (Runner, mpsc::UnboundedReceiver<BatchActionEvent>) {
let (tx, rx) = mpsc::unbounded_channel();
let report_batch: BatchReporter = Arc::new(move |event: BatchActionEvent| {
let tx = tx.clone();
Box::pin(async move {
let _ = tx.send(event);
})
});
let runner = Runner::new(
"worker-1".to_string(),
None,
handlers.into_iter().collect(),
slots,
None,
Arc::new(|_event| Box::pin(async {})),
report_batch,
);
(runner, rx)
}
fn discarded_batch_reports() -> BatchReporter {
Arc::new(|_event| Box::pin(async {}))
}
fn start_action(action_id: &str, run_id: &str, input: Value) -> AssignedAction {
start_action_with_payload(action_id, run_id, &json!({ "input": input }).to_string())
}
fn start_action_with_payload(action_id: &str, run_id: &str, payload: &str) -> AssignedAction {
AssignedAction {
action_type: ActionType::StartStepRun as i32,
action_id: action_id.to_string(),
task_run_external_id: run_id.to_string(),
action_payload: payload.to_string(),
..AssignedAction::default()
}
}
fn cancel_action(run_id: &str) -> AssignedAction {
AssignedAction {
action_type: ActionType::CancelStepRun as i32,
task_run_external_id: run_id.to_string(),
..AssignedAction::default()
}
}
fn event_type(event: &StepActionEvent) -> StepActionEventType {
StepActionEventType::try_from(event.event_type).expect("a known event type")
}
async fn next<T>(rx: &mut mpsc::UnboundedReceiver<T>) -> T {
tokio::time::timeout(NEVER, rx.recv())
.await
.expect("timed out waiting for the next item")
.expect("the channel is still open")
}
async fn within<F: Future>(future: F) -> F::Output {
tokio::time::timeout(NEVER, future)
.await
.expect("timed out waiting for the worker to let go")
}
async fn wait_for_in_flight(runner: &Runner, count: usize) {
for _ in 0..1000 {
if runner.in_flight() == count {
return;
}
tokio::task::yield_now().await;
}
panic!("{} runs stayed in flight, not {count}", runner.in_flight());
}
async fn wait_until_idle(runner: &Runner) {
wait_for_in_flight(runner, 0).await;
}
fn gated_task(
name: &str,
gate: Arc<Semaphore>,
) -> (crate::Task<Value, Value>, mpsc::UnboundedReceiver<()>) {
let (entered_tx, entered_rx) = mpsc::unbounded_channel();
let task = TaskBuilder::<Value, Value>::new(name).handler(move |_ctx, input: Value| {
let gate = Arc::clone(&gate);
let entered = entered_tx.clone();
async move {
let _ = entered.send(());
gate.acquire().await.expect("the gate stays open").forget();
Ok(input)
}
});
(task, entered_rx)
}
#[tokio::test]
async fn success_reports_started_then_completed() {
let task = TaskBuilder::<In, Out>::new("agent-run")
.handler(|_ctx, input: In| async move { Ok(Out { pr: input.repo }) });
let (action_id, handler) = handler_of(&task);
let (runner, mut events) = runner_with(vec![(action_id.clone(), handler)], 4);
runner.dispatch(start_action(&action_id, "run-1", json!({"repo": "x"})));
let started = next(&mut events).await;
assert_eq!(
event_type(&started),
StepActionEventType::StepEventTypeStarted
);
assert_eq!(started.task_run_external_id, "run-1");
assert_eq!(started.worker_id, "worker-1");
assert_eq!(started.event_payload, "");
let completed = next(&mut events).await;
assert_eq!(
event_type(&completed),
StepActionEventType::StepEventTypeCompleted
);
assert_eq!(completed.event_payload, json!({"pr": "x"}).to_string());
assert_eq!(completed.should_not_retry, None);
wait_until_idle(&runner).await;
}
#[tokio::test]
async fn a_retryable_error_reports_failed_without_should_not_retry() {
let task = TaskBuilder::<Value, Value>::new("flaky")
.handler(|_ctx, _input: Value| async move { Err(TaskError::new("the api was down")) });
let (action_id, handler) = handler_of(&task);
let (runner, mut events) = runner_with(vec![(action_id.clone(), handler)], 4);
runner.dispatch(start_action(&action_id, "run-1", json!(null)));
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
let failed = next(&mut events).await;
assert_eq!(
event_type(&failed),
StepActionEventType::StepEventTypeFailed
);
assert_eq!(failed.event_payload, "\"the api was down\"");
assert_eq!(failed.should_not_retry, None);
}
#[tokio::test]
async fn a_non_retryable_error_reports_failed_with_should_not_retry() {
let task =
TaskBuilder::<Value, Value>::new("doomed").handler(|_ctx, _input: Value| async move {
Err(TaskError::non_retryable("the repo does not exist"))
});
let (action_id, handler) = handler_of(&task);
let (runner, mut events) = runner_with(vec![(action_id.clone(), handler)], 4);
runner.dispatch(start_action(&action_id, "run-1", json!(null)));
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
let failed = next(&mut events).await;
assert_eq!(
event_type(&failed),
StepActionEventType::StepEventTypeFailed
);
assert_eq!(failed.event_payload, "\"the repo does not exist\"");
assert_eq!(failed.should_not_retry, Some(true));
}
#[tokio::test]
async fn a_panic_reports_failed_and_the_runner_keeps_working() {
let exploding = TaskBuilder::<Value, Value>::new("exploding")
.handler(|_ctx, _input: Value| async move { panic!("boom") });
let fine = TaskBuilder::<Value, Value>::new("fine")
.handler(|_ctx, input: Value| async move { Ok(input) });
let (bad_id, bad_handler) = handler_of(&exploding);
let (good_id, good_handler) = handler_of(&fine);
let (runner, mut events) = runner_with(
vec![
(bad_id.clone(), bad_handler),
(good_id.clone(), good_handler),
],
4,
);
runner.dispatch(start_action(&bad_id, "run-1", json!(null)));
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
let failed = next(&mut events).await;
assert_eq!(
event_type(&failed),
StepActionEventType::StepEventTypeFailed
);
assert!(failed.event_payload.contains("boom"), "{failed:?}");
assert_eq!(
failed.should_not_retry, None,
"a panic might not repeat, so it stays retryable"
);
runner.dispatch(start_action(&good_id, "run-2", json!("still here")));
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
let completed = next(&mut events).await;
assert_eq!(
event_type(&completed),
StepActionEventType::StepEventTypeCompleted
);
assert_eq!(completed.task_run_external_id, "run-2");
}
#[tokio::test]
async fn input_of_the_wrong_shape_reports_a_non_retryable_failure_and_the_runner_keeps_working()
{
let task = TaskBuilder::<In, Out>::new("agent-run")
.handler(|_ctx, input: In| async move { Ok(Out { pr: input.repo }) });
let (action_id, handler) = handler_of(&task);
let (runner, mut events) = runner_with(vec![(action_id.clone(), handler)], 4);
runner.dispatch(start_action(&action_id, "run-1", json!({"repo": 42})));
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
let failed = next(&mut events).await;
assert_eq!(
event_type(&failed),
StepActionEventType::StepEventTypeFailed
);
assert_eq!(failed.should_not_retry, Some(true));
assert!(failed.event_payload.contains("input"), "{failed:?}");
runner.dispatch(start_action(&action_id, "run-2", json!({"repo": "x"})));
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
let completed = next(&mut events).await;
assert_eq!(
event_type(&completed),
StepActionEventType::StepEventTypeCompleted
);
assert_eq!(completed.event_payload, json!({"pr": "x"}).to_string());
}
#[tokio::test]
async fn a_payload_that_is_not_json_reports_a_non_retryable_failure() {
let task = TaskBuilder::<Value, Value>::new("fine")
.handler(|_ctx, input: Value| async move { Ok(input) });
let (action_id, handler) = handler_of(&task);
let (runner, mut events) = runner_with(vec![(action_id.clone(), handler)], 4);
runner.dispatch(start_action_with_payload(
&action_id,
"run-1",
"not json at all",
));
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
let failed = next(&mut events).await;
assert_eq!(
event_type(&failed),
StepActionEventType::StepEventTypeFailed
);
assert_eq!(failed.should_not_retry, Some(true));
assert!(failed.event_payload.contains("payload"), "{failed:?}");
wait_until_idle(&runner).await;
}
#[tokio::test]
async fn an_unknown_action_reports_a_non_retryable_failure_naming_it() {
let (runner, mut events) = runner_with(Vec::new(), 4);
runner.dispatch(start_action("ghost:step", "run-1", json!(null)));
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
let failed = next(&mut events).await;
assert_eq!(
event_type(&failed),
StepActionEventType::StepEventTypeFailed
);
assert_eq!(failed.should_not_retry, Some(true));
assert!(failed.event_payload.contains("ghost:step"), "{failed:?}");
}
#[tokio::test]
async fn cancel_fires_the_token_and_suppresses_the_terminal_event() {
let observed = Arc::new(AtomicBool::new(false));
let waiting = TaskBuilder::<Value, Value>::new("waiting").handler({
let observed = Arc::clone(&observed);
move |ctx: crate::Context, _input: Value| {
let observed = Arc::clone(&observed);
async move {
ctx.cancelled().await;
observed.store(true, Ordering::SeqCst);
Ok(json!("finished anyway"))
}
}
});
let fine = TaskBuilder::<Value, Value>::new("fine")
.handler(|_ctx, input: Value| async move { Ok(input) });
let (waiting_id, waiting_handler) = handler_of(&waiting);
let (fine_id, fine_handler) = handler_of(&fine);
let (runner, mut events) = runner_with(
vec![
(waiting_id.clone(), waiting_handler),
(fine_id.clone(), fine_handler),
],
4,
);
runner.dispatch(start_action(&waiting_id, "run-1", json!(null)));
let started = next(&mut events).await;
assert_eq!(
event_type(&started),
StepActionEventType::StepEventTypeStarted
);
assert_eq!(started.task_run_external_id, "run-1");
runner.dispatch(cancel_action("run-1"));
wait_until_idle(&runner).await;
assert!(
observed.load(Ordering::SeqCst),
"the handler never saw the cancellation"
);
runner.dispatch(start_action(&fine_id, "run-2", json!("ok")));
let started = next(&mut events).await;
assert_eq!(
event_type(&started),
StepActionEventType::StepEventTypeStarted
);
assert_eq!(started.task_run_external_id, "run-2");
let completed = next(&mut events).await;
assert_eq!(
event_type(&completed),
StepActionEventType::StepEventTypeCompleted
);
assert_eq!(completed.task_run_external_id, "run-2");
assert!(matches!(events.try_recv(), Err(TryRecvError::Empty)));
}
#[tokio::test]
async fn a_report_that_panics_still_lets_go_of_the_run() {
let task = TaskBuilder::<Value, Value>::new("fine")
.handler(|_ctx, input: Value| async move { Ok(input) });
let (action_id, handler) = handler_of(&task);
let (tx, mut events) = mpsc::unbounded_channel();
let report: Reporter = Arc::new(move |event: StepActionEvent| {
let tx = tx.clone();
Box::pin(async move {
let terminal = event.event_type != StepActionEventType::StepEventTypeStarted as i32;
let _ = tx.send(event);
assert!(!terminal, "this reporter refuses terminal events");
})
});
let runner = Runner::new(
"worker-1".to_string(),
None,
HashMap::from([(action_id.clone(), handler)]),
4,
None,
report,
discarded_batch_reports(),
);
runner.dispatch(start_action(&action_id, "run-1", json!("ok")));
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeCompleted
);
wait_until_idle(&runner).await;
}
#[tokio::test]
async fn cancel_for_an_unknown_run_is_ignored() {
let task = TaskBuilder::<Value, Value>::new("fine")
.handler(|_ctx, input: Value| async move { Ok(input) });
let (action_id, handler) = handler_of(&task);
let (runner, mut events) = runner_with(vec![(action_id.clone(), handler)], 4);
runner.dispatch(cancel_action("a-run-from-another-worker"));
assert_eq!(runner.in_flight(), 0);
runner.dispatch(start_action(&action_id, "run-1", json!("ok")));
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeCompleted
);
}
fn batch_action(action_id: &str, payload: &str) -> AssignedAction {
AssignedAction {
action_type: ActionType::StartBatch as i32,
action_id: action_id.to_string(),
action_payload: payload.to_string(),
batch_id: Some("b-1".to_string()),
..AssignedAction::default()
}
}
#[tokio::test]
async fn a_batch_naming_no_members_is_dropped_without_a_report() {
let task = TaskBuilder::<Value, Value>::new("t")
.batch_handler(crate::BatchConfig::max_size(2), |_ctx, _batch| async move {
Ok(crate::BatchOutput::new())
});
let (action_id, handler) = handler_of(&task);
let (runner, mut events) = runner_with_batch(vec![(action_id.clone(), handler)], 4);
for nobody in ["{}", "[1, 2]", "not json at all", ""] {
runner.dispatch(batch_action(&action_id, nobody));
}
wait_until_idle(&runner).await;
assert!(
matches!(events.try_recv(), Err(TryRecvError::Empty)),
"a batch with no members reported something",
);
}
#[tokio::test]
async fn a_batch_for_an_unknown_action_fails_every_member() {
let (runner, mut events) = runner_with_batch(Vec::new(), 4);
runner.dispatch(batch_action(
"ghost:step",
&json!({"run-a": {}, "run-b": {}}).to_string(),
));
let started = next(&mut events).await;
assert_eq!(
started.event_type,
StepActionEventType::StepEventTypeStarted as i32
);
assert_eq!(started.items.len(), 2);
let failed = next(&mut events).await;
assert_eq!(
failed.event_type,
StepActionEventType::StepEventTypeFailed as i32
);
assert_eq!(failed.items.len(), 2);
assert!(
failed.items[0].event_payload.contains("ghost:step"),
"{:?}",
failed.items[0],
);
assert!(
failed
.items
.iter()
.all(|item| item.should_not_retry == Some(true))
);
wait_until_idle(&runner).await;
}
#[tokio::test]
async fn unsupported_action_types_are_ignored() {
let task = TaskBuilder::<Value, Value>::new("fine")
.handler(|_ctx, input: Value| async move { Ok(input) });
let (action_id, handler) = handler_of(&task);
let (runner, mut events) = runner_with(vec![(action_id.clone(), handler)], 4);
runner.dispatch(AssignedAction {
action_type: ActionType::StartGetGroupKey as i32,
..start_action(&action_id, "run-1", json!(null))
});
for _ in 0..10 {
tokio::task::yield_now().await;
}
assert_eq!(runner.in_flight(), 0);
assert!(matches!(events.try_recv(), Err(TryRecvError::Empty)));
}
#[tokio::test]
async fn slots_bound_concurrency() {
let gate = Arc::new(Semaphore::new(0));
let running = Arc::new(AtomicUsize::new(0));
let peak = Arc::new(AtomicUsize::new(0));
let (entered_tx, mut entered) = mpsc::unbounded_channel();
let task = TaskBuilder::<Value, Value>::new("gated").handler({
let gate = Arc::clone(&gate);
let running = Arc::clone(&running);
let peak = Arc::clone(&peak);
move |_ctx, input: Value| {
let gate = Arc::clone(&gate);
let running = Arc::clone(&running);
let peak = Arc::clone(&peak);
let entered = entered_tx.clone();
async move {
let now = running.fetch_add(1, Ordering::SeqCst) + 1;
peak.fetch_max(now, Ordering::SeqCst);
let _ = entered.send(());
gate.acquire().await.expect("the gate stays open").forget();
running.fetch_sub(1, Ordering::SeqCst);
Ok(input)
}
}
});
let (action_id, handler) = handler_of(&task);
let (runner, mut events) = runner_with(vec![(action_id.clone(), handler)], 2);
for n in 1..=3 {
runner.dispatch(start_action(&action_id, &format!("run-{n}"), json!(n)));
}
for _ in 0..2 {
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
}
assert!(
matches!(events.try_recv(), Err(TryRecvError::Empty)),
"only two runs hold a slot, so nothing else can be reported yet"
);
for _ in 0..2 {
next(&mut entered).await;
}
assert!(
matches!(entered.try_recv(), Err(TryRecvError::Empty)),
"a third run entered its handler without a slot"
);
assert_eq!(runner.in_flight(), 3);
gate.add_permits(3);
let mut completed = 0;
while completed < 3 {
if event_type(&next(&mut events).await) == StepActionEventType::StepEventTypeCompleted {
completed += 1;
}
}
assert_eq!(
peak.load(Ordering::SeqCst),
2,
"more than two handlers ran at once"
);
wait_until_idle(&runner).await;
}
async fn drain_until(
events: &mut mpsc::UnboundedReceiver<StepActionEvent>,
run_id: &str,
kind: StepActionEventType,
) -> Vec<StepActionEvent> {
let mut seen = Vec::new();
loop {
let event = next(events).await;
let found = event.task_run_external_id == run_id && event_type(&event) == kind;
seen.push(event);
if found {
return seen;
}
}
}
#[tokio::test]
async fn release_slot_frees_the_local_slot_for_the_next_run() {
let waiting_gate = Arc::new(Semaphore::new(0));
let (releasing_tx, mut released) = mpsc::unbounded_channel();
let releasing = TaskBuilder::<Value, Value>::new("releasing").handler({
let gate = Arc::clone(&waiting_gate);
move |ctx: crate::Context, input: Value| {
let gate = Arc::clone(&gate);
let entered = releasing_tx.clone();
async move {
ctx.release_local_slot();
let _ = entered.send(());
gate.acquire().await.expect("the gate stays open").forget();
Ok(input)
}
}
});
let fine = TaskBuilder::<Value, Value>::new("fine")
.handler(|_ctx, input: Value| async move { Ok(input) });
let second_gate = Arc::new(Semaphore::new(0));
let (gated, mut gated_entered) = gated_task("gated", Arc::clone(&second_gate));
let (releasing_id, releasing_handler) = handler_of(&releasing);
let (fine_id, fine_handler) = handler_of(&fine);
let (gated_id, gated_handler) = handler_of(&gated);
let (runner, mut events) = runner_with(
vec![
(releasing_id.clone(), releasing_handler),
(fine_id.clone(), fine_handler),
(gated_id.clone(), gated_handler),
],
1,
);
runner.dispatch(start_action(&releasing_id, "run-1", json!(null)));
next(&mut released).await;
runner.dispatch(start_action(&fine_id, "run-2", json!("second")));
let seen = drain_until(
&mut events,
"run-2",
StepActionEventType::StepEventTypeCompleted,
)
.await;
assert!(
!seen.iter().any(|event| {
event.task_run_external_id == "run-1"
&& event_type(event) == StepActionEventType::StepEventTypeCompleted
}),
"the first run finished before its gate opened: {seen:?}",
);
waiting_gate.add_permits(1);
drain_until(
&mut events,
"run-1",
StepActionEventType::StepEventTypeCompleted,
)
.await;
wait_until_idle(&runner).await;
runner.dispatch(start_action(&gated_id, "run-3", json!(null)));
runner.dispatch(start_action(&gated_id, "run-4", json!(null)));
next(&mut gated_entered).await;
wait_for_in_flight(&runner, 2).await;
for _ in 0..100 {
tokio::task::yield_now().await;
}
assert!(
matches!(gated_entered.try_recv(), Err(TryRecvError::Empty)),
"two runs held the worker's single slot: the permit was released twice",
);
second_gate.add_permits(2);
wait_until_idle(&runner).await;
}
#[tokio::test]
async fn a_leaked_context_clone_does_not_pin_the_slot() {
let leaky = TaskBuilder::<Value, Value>::new("leaky").handler(
|ctx: crate::Context, input: Value| async move {
tokio::spawn(async move {
let _leaked = ctx;
std::future::pending::<()>().await;
});
Ok(input)
},
);
let fine = TaskBuilder::<Value, Value>::new("fine")
.handler(|_ctx, input: Value| async move { Ok(input) });
let (leaky_id, leaky_handler) = handler_of(&leaky);
let (fine_id, fine_handler) = handler_of(&fine);
let (runner, mut events) = runner_with(
vec![
(leaky_id.clone(), leaky_handler),
(fine_id.clone(), fine_handler),
],
1,
);
runner.dispatch(start_action(&leaky_id, "run-1", json!(null)));
drain_until(
&mut events,
"run-1",
StepActionEventType::StepEventTypeCompleted,
)
.await;
wait_until_idle(&runner).await;
runner.dispatch(start_action(&fine_id, "run-2", json!("ok")));
let completed = drain_until(
&mut events,
"run-2",
StepActionEventType::StepEventTypeCompleted,
)
.await;
assert_eq!(
completed.last().expect("the run completed").event_payload,
"\"ok\""
);
}
#[tokio::test]
async fn cancelling_a_queued_run_skips_it_entirely() {
let gate = Arc::new(Semaphore::new(0));
let (task, mut entered) = gated_task("gated", Arc::clone(&gate));
let (action_id, handler) = handler_of(&task);
let (runner, mut events) = runner_with(vec![(action_id.clone(), handler)], 1);
runner.dispatch(start_action(&action_id, "run-1", json!("first")));
let started = next(&mut events).await;
assert_eq!(started.task_run_external_id, "run-1");
next(&mut entered).await;
runner.dispatch(start_action(&action_id, "run-2", json!("second")));
assert_eq!(runner.in_flight(), 2);
runner.dispatch(cancel_action("run-2"));
wait_for_in_flight(&runner, 1).await;
gate.add_permits(1);
let completed = next(&mut events).await;
assert_eq!(
event_type(&completed),
StepActionEventType::StepEventTypeCompleted
);
assert_eq!(completed.task_run_external_id, "run-1");
wait_until_idle(&runner).await;
assert!(
matches!(entered.try_recv(), Err(TryRecvError::Empty)),
"the cancelled run ran its handler"
);
assert!(
matches!(events.try_recv(), Err(TryRecvError::Empty)),
"the cancelled run reported something"
);
}
#[tokio::test(start_paused = true)]
async fn drain_waits_for_in_flight_runs() {
let task =
TaskBuilder::<Value, Value>::new("slow").handler(|_ctx, input: Value| async move {
tokio::time::sleep(Duration::from_secs(3)).await;
Ok(input)
});
let (action_id, handler) = handler_of(&task);
let (runner, mut events) = runner_with(vec![(action_id.clone(), handler)], 4);
let started_at = Instant::now();
runner.dispatch(start_action(&action_id, "run-1", json!("slow")));
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
within(runner.drain(Duration::from_secs(30))).await;
let elapsed = started_at.elapsed();
assert!(
(Duration::from_secs(3)..Duration::from_secs(4)).contains(&elapsed),
"drain returned after {elapsed:?}"
);
assert_eq!(runner.in_flight(), 0);
let completed = next(&mut events).await;
assert_eq!(
event_type(&completed),
StepActionEventType::StepEventTypeCompleted
);
}
#[tokio::test(start_paused = true)]
async fn drain_cancels_what_outlives_the_grace_period() {
let observed = Arc::new(AtomicBool::new(false));
let task = TaskBuilder::<Value, Value>::new("waiting").handler({
let observed = Arc::clone(&observed);
move |ctx: crate::Context, input: Value| {
let observed = Arc::clone(&observed);
async move {
ctx.cancelled().await;
observed.store(true, Ordering::SeqCst);
Ok(input)
}
}
});
let (action_id, handler) = handler_of(&task);
let (runner, mut events) = runner_with(vec![(action_id.clone(), handler)], 4);
let started_at = Instant::now();
runner.dispatch(start_action(&action_id, "run-1", json!("waiting")));
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
within(runner.drain(Duration::from_secs(5))).await;
let elapsed = started_at.elapsed();
assert!(
(Duration::from_secs(5)..Duration::from_millis(6_500)).contains(&elapsed),
"drain returned after {elapsed:?}"
);
assert!(
observed.load(Ordering::SeqCst),
"the handler never saw the cancellation"
);
assert_eq!(runner.in_flight(), 0);
assert!(
matches!(events.try_recv(), Err(TryRecvError::Empty)),
"a cancelled run reported a terminal event"
);
}
#[tokio::test(start_paused = true)]
async fn drain_aborts_handlers_that_ignore_cancellation() {
let task = TaskBuilder::<Value, Value>::new("stuck").handler(|_ctx, _input: Value| async {
std::future::pending::<()>().await;
Ok(json!(null))
});
let (action_id, handler) = handler_of(&task);
let (runner, mut events) = runner_with(vec![(action_id.clone(), handler)], 4);
let started_at = Instant::now();
runner.dispatch(start_action(&action_id, "run-1", json!(null)));
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
within(runner.drain(Duration::from_secs(5))).await;
let elapsed = started_at.elapsed();
assert!(
elapsed < Duration::from_secs(7),
"drain returned after {elapsed:?}"
);
assert_eq!(runner.in_flight(), 0);
assert!(matches!(events.try_recv(), Err(TryRecvError::Empty)));
}
#[tokio::test(start_paused = true)]
async fn abort_all_drops_running_runs_and_frees_their_slots() {
let (entered_tx, mut entered) = mpsc::unbounded_channel();
let stuck =
TaskBuilder::<Value, Value>::new("stuck").handler(move |_ctx, _input: Value| {
let entered = entered_tx.clone();
async move {
let _ = entered.send(());
std::future::pending::<()>().await;
Ok(json!(null))
}
});
let fine = TaskBuilder::<Value, Value>::new("fine")
.handler(|_ctx, input: Value| async move { Ok(input) });
let (stuck_id, stuck_handler) = handler_of(&stuck);
let (fine_id, fine_handler) = handler_of(&fine);
let (runner, mut events) = runner_with(
vec![
(stuck_id.clone(), stuck_handler),
(fine_id.clone(), fine_handler),
],
1,
);
runner.dispatch(start_action(&stuck_id, "run-1", json!(null)));
assert_eq!(
event_type(&next(&mut events).await),
StepActionEventType::StepEventTypeStarted
);
next(&mut entered).await;
within(runner.abort_all()).await;
assert_eq!(runner.in_flight(), 0);
runner.dispatch(start_action(&fine_id, "run-2", json!("ok")));
let started = next(&mut events).await;
assert_eq!(
event_type(&started),
StepActionEventType::StepEventTypeStarted
);
assert_eq!(started.task_run_external_id, "run-2");
let completed = next(&mut events).await;
assert_eq!(
event_type(&completed),
StepActionEventType::StepEventTypeCompleted
);
assert_eq!(
completed.task_run_external_id, "run-2",
"the dropped run must not report anything"
);
}
}