use std::fmt;
use std::marker::PhantomData;
use hatchet_proto::root::{StepRunResult, WorkflowRunEvent};
use serde::Serialize;
use serde::de::DeserializeOwned;
use tracing::{Instrument, Span};
use crate::error::{Error, Result, TaskFailure};
use crate::{Client, RunOpts, Task, Workflow, otel, retry, trigger};
pub(crate) mod bulk;
pub(crate) mod client;
pub(crate) mod details;
pub(crate) mod filter;
pub(crate) mod listener;
pub(crate) mod stream;
pub use client::RunsClient;
pub use details::{RunDetails, TaskRunDetail};
pub use filter::{RunStatus, TasksFilter};
pub use stream::{OutputStream, StreamChunk};
impl<I, O> Task<I, O> {
pub async fn run_no_wait(
&self,
client: &Client,
input: &I,
opts: RunOpts,
) -> Result<RunHandle<O>>
where
I: Serialize,
{
let span = otel::run_workflow(self.name());
let run_id = trigger(client, self.name(), input, &opts, &span)
.instrument(span.clone())
.await;
otel::record_result(&span, &run_id);
Ok(self.handle(client, run_id?))
}
pub async fn run(&self, client: &Client, input: &I, opts: RunOpts) -> Result<O>
where
I: Serialize,
O: DeserializeOwned,
{
let span = otel::run_workflow(self.name());
let output = async {
let run_id = trigger(client, self.name(), input, &opts, &span).await?;
self.handle(client, run_id).result().await
}
.instrument(span.clone())
.await;
otel::record_result(&span, &output);
output
}
pub async fn run_many(
&self,
client: &Client,
inputs: impl IntoIterator<Item = (I, RunOpts)>,
) -> Result<Vec<RunHandle<O>>>
where
I: Serialize,
{
let inputs: Vec<(I, RunOpts)> = inputs.into_iter().collect();
let span = otel::run_workflows(self.name(), inputs.len());
let run_ids = trigger_all(client, self.name(), &inputs, &span)
.instrument(span.clone())
.await;
otel::record_result(&span, &run_ids);
Ok(run_ids?
.into_iter()
.map(|run_id| self.handle(client, run_id))
.collect())
}
fn handle(&self, client: &Client, run_id: String) -> RunHandle<O> {
RunHandle {
run_id,
task_name: self.name().to_string(),
client: client.clone(),
_output: PhantomData,
}
}
}
impl Workflow {
pub async fn run_no_wait<I: Serialize>(
&self,
client: &Client,
input: &I,
opts: RunOpts,
) -> Result<WorkflowRunHandle> {
let span = otel::run_workflow(self.name());
let run_id = trigger(client, self.name(), input, &opts, &span)
.instrument(span.clone())
.await;
otel::record_result(&span, &run_id);
Ok(self.handle(client, run_id?))
}
pub async fn run<I: Serialize>(
&self,
client: &Client,
input: &I,
opts: RunOpts,
) -> Result<WorkflowResult> {
let span = otel::run_workflow(self.name());
let result = async {
let run_id = trigger(client, self.name(), input, &opts, &span).await?;
self.handle(client, run_id).result().await
}
.instrument(span.clone())
.await;
otel::record_result(&span, &result);
result
}
pub async fn run_many<I: Serialize>(
&self,
client: &Client,
inputs: impl IntoIterator<Item = (I, RunOpts)>,
) -> Result<Vec<WorkflowRunHandle>> {
let inputs: Vec<(I, RunOpts)> = inputs.into_iter().collect();
let span = otel::run_workflows(self.name(), inputs.len());
let run_ids = trigger_all(client, self.name(), &inputs, &span)
.instrument(span.clone())
.await;
otel::record_result(&span, &run_ids);
Ok(run_ids?
.into_iter()
.map(|run_id| self.handle(client, run_id))
.collect())
}
fn handle(&self, client: &Client, run_id: String) -> WorkflowRunHandle {
WorkflowRunHandle {
run_id,
client: client.clone(),
}
}
}
#[must_use = "a run handle waits for nothing until `result` is awaited"]
pub struct RunHandle<O> {
run_id: String,
task_name: String,
client: Client,
_output: PhantomData<fn() -> O>,
}
impl<O> Clone for RunHandle<O> {
fn clone(&self) -> Self {
Self {
run_id: self.run_id.clone(),
task_name: self.task_name.clone(),
client: self.client.clone(),
_output: PhantomData,
}
}
}
impl<O> fmt::Debug for RunHandle<O> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("RunHandle")
.field("run_id", &self.run_id)
.field("task", &self.task_name)
.finish_non_exhaustive()
}
}
impl<O> RunHandle<O> {
pub fn id(&self) -> &str {
&self.run_id
}
pub async fn cancel(&self) -> Result<()> {
cancel_run(&self.client, &self.run_id).await
}
pub async fn stream(&self) -> Result<OutputStream> {
self.client.runs().stream(&self.run_id).await
}
}
impl<O: DeserializeOwned> RunHandle<O> {
pub async fn result(&self) -> Result<O> {
let event = self.client.run_listener().wait(self.run_id.clone()).await?;
output_of(&event, &self.task_name)
}
}
#[must_use = "a run handle waits for nothing until `result` is awaited"]
#[derive(Clone)]
pub struct WorkflowRunHandle {
run_id: String,
client: Client,
}
impl fmt::Debug for WorkflowRunHandle {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("WorkflowRunHandle")
.field("run_id", &self.run_id)
.finish_non_exhaustive()
}
}
impl WorkflowRunHandle {
pub fn id(&self) -> &str {
&self.run_id
}
pub async fn cancel(&self) -> Result<()> {
cancel_run(&self.client, &self.run_id).await
}
pub async fn stream(&self) -> Result<OutputStream> {
self.client.runs().stream(&self.run_id).await
}
pub async fn result(&self) -> Result<WorkflowResult> {
let event = self.client.run_listener().wait(self.run_id.clone()).await?;
succeeded(&event)?;
Ok(WorkflowResult {
run_id: event.workflow_run_id,
results: event.results,
})
}
}
#[derive(Clone, Debug)]
pub struct WorkflowResult {
run_id: String,
results: Vec<StepRunResult>,
}
impl WorkflowResult {
pub fn run_id(&self) -> &str {
&self.run_id
}
pub fn output<PI, O: DeserializeOwned>(&self, task: &Task<PI, O>) -> Result<O> {
self.output_by_name(task.name())
}
pub fn output_by_name<T: DeserializeOwned>(&self, name: &str) -> Result<T> {
let result = named(&self.results, name).ok_or_else(|| Error::RunOutput {
run_id: self.run_id.clone(),
reason: format!(
"the run produced no output for task {name:?}; it ran {}",
listed(&self.results)
),
})?;
decode(&self.run_id, result)
}
pub fn was_skipped<PI, O>(&self, task: &Task<PI, O>) -> bool {
self.was_skipped_by_name(task.name())
}
pub fn was_skipped_by_name(&self, name: &str) -> bool {
named(&self.results, name).is_some_and(was_skipped)
}
}
fn named<'a>(results: &'a [StepRunResult], name: &str) -> Option<&'a StepRunResult> {
results
.iter()
.find(|result| result.task_name == name)
.or_else(|| {
results
.iter()
.find(|result| result.task_name.eq_ignore_ascii_case(name))
})
}
fn listed(results: &[StepRunResult]) -> String {
if results.is_empty() {
return "nothing".to_string();
}
results
.iter()
.map(|result| format!("{:?}", result.task_name))
.collect::<Vec<_>>()
.join(", ")
}
async fn cancel_run(client: &Client, run_id: &str) -> Result<()> {
client.runs().cancel([run_id]).await?;
Ok(())
}
fn succeeded(event: &WorkflowRunEvent) -> Result<()> {
let mut errors: Vec<TaskFailure> = Vec::new();
let mut cancelled: Vec<String> = Vec::new();
for result in &event.results {
match result.error.as_deref() {
Some(message) if !message.trim().is_empty() => errors.push(TaskFailure {
task_name: result.task_name.clone(),
message: message.to_string(),
}),
_ if cancelled_shape(result) => cancelled.push(result.task_name.clone()),
_ => {}
}
}
if !errors.is_empty() {
return Err(Error::RunFailed {
run_id: event.workflow_run_id.clone(),
errors,
});
}
if !cancelled.is_empty() {
return Err(Error::RunCancelled {
run_id: event.workflow_run_id.clone(),
tasks: cancelled,
});
}
Ok(())
}
fn cancelled_shape(result: &StepRunResult) -> bool {
result.output.is_none()
&& result
.error
.as_deref()
.is_some_and(|message| message.trim().is_empty())
}
pub(crate) fn output_of<O: DeserializeOwned>(
event: &WorkflowRunEvent,
task_name: &str,
) -> Result<O> {
succeeded(event)?;
let lone = match event.results.as_slice() {
[only] => Some(only),
_ => None,
};
let result = named(&event.results, task_name)
.or(lone)
.ok_or_else(|| Error::RunOutput {
run_id: event.workflow_run_id.clone(),
reason: format!(
"the run produced no output for task {task_name:?}; it ran {}",
listed(&event.results)
),
})?;
decode(&event.workflow_run_id, result)
}
fn was_skipped(result: &StepRunResult) -> bool {
result.output.as_deref().is_some_and(|output| {
serde_json::from_str::<serde_json::Value>(output)
.ok()
.and_then(|value| value.get("skipped").cloned())
== Some(serde_json::Value::Bool(true))
})
}
fn decode<T: DeserializeOwned>(run_id: &str, result: &StepRunResult) -> Result<T> {
if was_skipped(result) {
return Err(Error::RunOutput {
run_id: run_id.to_string(),
reason: format!(
"task {:?} was skipped, so it produced no output",
result.task_name
),
});
}
let output = result.output.as_deref().ok_or_else(|| Error::RunOutput {
run_id: run_id.to_string(),
reason: format!("task {:?} recorded no output", result.task_name),
})?;
serde_json::from_str(output).map_err(|error| Error::RunOutput {
run_id: run_id.to_string(),
reason: format!("task {:?}: {error}", result.task_name),
})
}
async fn trigger<I: Serialize>(
client: &Client,
name: &str,
input: &I,
opts: &RunOpts,
span: &Span,
) -> Result<String> {
let mut request = trigger::request(client.config().namespace(), name, input, opts, None)?;
otel::inject(span, &mut request.additional_metadata);
let response = retry::unary(|| {
let mut workflows = client.workflows_grpc();
let request = retry::timed(request.clone());
async move { workflows.trigger_workflow(request).await }
})
.await
.map_err(|status| crate::collision::from_trigger(Error::from(status)))?;
let run_id = response.into_inner().workflow_run_id;
span.record("hatchet.child_workflow_run_id", run_id.as_str());
Ok(run_id)
}
async fn trigger_all<I: Serialize>(
client: &Client,
name: &str,
inputs: &[(I, RunOpts)],
span: &Span,
) -> Result<Vec<String>> {
let requests = inputs
.iter()
.map(|(input, opts)| {
let mut request =
trigger::request(client.config().namespace(), name, input, opts, None)?;
otel::inject(span, &mut request.additional_metadata);
Ok(request)
})
.collect::<Result<Vec<_>>>()?;
bulk::trigger_many(client, requests).await
}
#[cfg(test)]
mod tests {
use super::*;
use hatchet_proto::root::WorkflowRunEventType;
use serde::Deserialize;
use serde_json::json;
#[derive(Debug, Deserialize, PartialEq)]
struct Out {
pr: Option<String>,
}
fn result(task_name: &str, output: Option<&str>, error: Option<&str>) -> StepRunResult {
StepRunResult {
task_name: task_name.to_string(),
output: output.map(str::to_string),
error: error.map(str::to_string),
..StepRunResult::default()
}
}
fn event(results: Vec<StepRunResult>) -> WorkflowRunEvent {
WorkflowRunEvent {
workflow_run_id: "run-1".to_string(),
event_type: WorkflowRunEventType::Finished as i32,
results,
..WorkflowRunEvent::default()
}
}
#[test]
fn the_output_of_the_named_task_is_the_one_that_is_read() {
let event = event(vec![
result("fetch", Some(r#"{"pr":"other"}"#), None),
result("agent-run", Some(r#"{"pr":null}"#), None),
]);
let out: Out = output_of(&event, "agent-run").unwrap();
assert_eq!(out, Out { pr: None });
}
#[test]
fn a_name_is_matched_without_regard_to_case() {
let event = event(vec![result("agent-run", Some(r#"{"pr":null}"#), None)]);
let out: Out = output_of(&event, "Agent-Run").unwrap();
assert_eq!(out, Out { pr: None });
}
#[test]
fn a_skipped_task_is_reported_as_skipped_rather_than_as_a_missing_output() {
let event = event(vec![
result("fetch", Some(r#"{"pr":"ok"}"#), None),
result("maybe", Some(r#"{"skipped":true}"#), None),
]);
let result = WorkflowResult {
run_id: event.workflow_run_id.clone(),
results: event.results.clone(),
};
assert!(result.was_skipped_by_name("maybe"));
assert!(!result.was_skipped_by_name("fetch"));
assert!(!result.was_skipped_by_name("never-ran"));
let error = result.output_by_name::<Out>("maybe").unwrap_err();
let Error::RunOutput { reason, .. } = &error else {
panic!("expected RunOutput: {error:?}");
};
assert!(reason.contains("skipped"), "{reason}");
assert!(succeeded(&event).is_ok());
}
#[test]
fn a_lone_result_is_the_output_whatever_it_is_called() {
let event = event(vec![result("step-one", Some(r#"{"pr":"x"}"#), None)]);
let out: Out = output_of(&event, "agent-run").unwrap();
assert_eq!(
out,
Out {
pr: Some("x".to_string())
}
);
}
#[test]
fn a_result_that_is_neither_named_nor_alone_is_a_run_output_error() {
let event = event(vec![
result("fetch", Some("1"), None),
result("report", Some("2"), None),
]);
let error = output_of::<Out>(&event, "agent-run").unwrap_err();
let Error::RunOutput { run_id, reason } = &error else {
panic!("{error:?}")
};
assert_eq!(run_id, "run-1");
assert!(reason.contains("\"fetch\", \"report\""), "{reason}");
}
#[test]
fn an_output_of_the_wrong_shape_is_a_run_output_error() {
let event = event(vec![result("agent-run", Some(r#"{"pr":42}"#), None)]);
let error = output_of::<Out>(&event, "agent-run").unwrap_err();
assert!(matches!(error, Error::RunOutput { .. }), "{error:?}");
}
#[test]
fn a_missing_output_is_a_run_output_error() {
let event = event(vec![result("agent-run", None, None)]);
let error = output_of::<Out>(&event, "agent-run").unwrap_err();
let Error::RunOutput { reason, .. } = &error else {
panic!("{error:?}")
};
assert!(reason.contains("recorded no output"), "{reason}");
}
#[test]
fn a_reported_error_fails_the_run_before_any_output_is_read() {
let event = event(vec![
result("fetch", Some("1"), None),
result("agent-run", None, Some("boom")),
]);
let error = output_of::<Out>(&event, "agent-run").unwrap_err();
let Error::RunFailed { run_id, errors } = &error else {
panic!("{error:?}")
};
assert_eq!(run_id, "run-1");
assert_eq!(
errors,
&[TaskFailure {
task_name: "agent-run".to_string(),
message: "boom".to_string(),
}]
);
}
#[test]
fn a_blank_error_with_no_output_is_a_cancellation() {
let event = event(vec![
result("lead", Some(r#"{"ok":true}"#), None),
result("hold", None, Some("")),
]);
let error = succeeded(&event).unwrap_err();
let Error::RunCancelled { run_id, tasks } = &error else {
panic!("expected a cancellation: {error:?}")
};
assert_eq!(run_id, "run-1");
assert_eq!(tasks, &["hold".to_string()]);
}
#[test]
fn a_success_row_and_a_cancel_row_for_one_task_is_still_a_cancellation() {
let event = event(vec![
result("lead", Some(r#"{"ok":true}"#), None),
result("lead", None, Some("")),
result("hold", None, Some("")),
]);
let error = succeeded(&event).unwrap_err();
let Error::RunCancelled { tasks, .. } = &error else {
panic!("expected a cancellation: {error:?}")
};
assert_eq!(tasks, &["lead".to_string(), "hold".to_string()]);
}
#[test]
fn a_real_failure_next_to_a_cancelled_task_is_a_failure() {
let event = event(vec![
result("fetch", None, Some("boom")),
result("report", None, Some("")),
]);
let error = succeeded(&event).unwrap_err();
let Error::RunFailed { errors, .. } = &error else {
panic!("expected a failure: {error:?}")
};
assert_eq!(
errors,
&[TaskFailure {
task_name: "fetch".to_string(),
message: "boom".to_string(),
}]
);
}
#[test]
fn a_json_encoded_empty_failure_message_is_a_failure_not_a_cancellation() {
let event = event(vec![result("agent-run", None, Some("\"\""))]);
let error = succeeded(&event).unwrap_err();
let Error::RunFailed { errors, .. } = &error else {
panic!("expected a failure: {error:?}")
};
assert_eq!(errors[0].message, "\"\"");
}
#[test]
fn a_blank_error_next_to_an_output_is_neither_a_failure_nor_a_cancellation() {
let event = event(vec![result("agent-run", Some(r#"{"pr":null}"#), Some(""))]);
succeeded(&event).expect("a row with an output is not a cancellation");
let out: Out = output_of(&event, "agent-run").unwrap();
assert_eq!(out, Out { pr: None });
}
#[test]
fn a_cancelled_run_is_reported_before_any_output_is_read() {
let event = event(vec![result("agent-run", None, Some(""))]);
let error = output_of::<Out>(&event, "agent-run").unwrap_err();
assert!(matches!(error, Error::RunCancelled { .. }), "{error:?}");
}
#[test]
fn a_workflow_result_reads_each_task_by_name() {
let event = event(vec![
result("fetch", Some(r#"{"n":1}"#), None),
result("report", Some(r#""done""#), None),
]);
succeeded(&event).unwrap();
let workflow_result = WorkflowResult {
run_id: event.workflow_run_id.clone(),
results: event.results.clone(),
};
assert_eq!(workflow_result.run_id(), "run-1");
assert_eq!(
workflow_result
.output_by_name::<serde_json::Value>("fetch")
.unwrap(),
json!({"n": 1})
);
assert_eq!(
workflow_result.output_by_name::<String>("report").unwrap(),
"done"
);
let error = workflow_result
.output_by_name::<serde_json::Value>("missing")
.unwrap_err();
assert!(matches!(error, Error::RunOutput { .. }), "{error:?}");
}
#[test]
fn a_workflow_result_does_not_fall_back_to_a_lone_result() {
let workflow_result = WorkflowResult {
run_id: "run-1".to_string(),
results: vec![result("fetch", Some("1"), None)],
};
let error = workflow_result.output_by_name::<u32>("report").unwrap_err();
assert!(matches!(error, Error::RunOutput { .. }), "{error:?}");
}
}