Skip to main content

hatchet_client/
error.rs

1//! The SDK's error type.
2
3/// Errors returned by `hatchet-client`.
4#[derive(Debug, thiserror::Error)]
5#[non_exhaustive]
6pub enum Error {
7    /// The API token is not a JWT carrying the claims the SDK needs.
8    /// The message never contains the token itself.
9    #[error("invalid token: {0}")]
10    InvalidToken(String),
11
12    /// The configuration is incomplete or contradictory.
13    #[error("invalid configuration: {0}")]
14    Config(String),
15
16    /// The gRPC channel could not be set up.
17    #[error("could not set up the connection to {address}: {}", source_chain(.source))]
18    Transport {
19        /// The `host:port` being connected to.
20        address: String,
21        /// The underlying transport error.
22        #[source]
23        source: tonic::transport::Error,
24    },
25
26    /// The engine answered a gRPC call with an error status.
27    #[error("gRPC call failed: {0}")]
28    Grpc(Box<tonic::Status>),
29
30    /// The engine reported a version string that is not `vMAJOR.MINOR.PATCH`.
31    #[error("the engine reported an unparseable version {0:?}")]
32    InvalidEngineVersion(String),
33
34    /// The engine is older than the SDK supports.
35    #[error("Hatchet engine {found} is older than the minimum supported version {minimum}")]
36    UnsupportedEngine {
37        /// The version the engine reported.
38        found: String,
39        /// The oldest version this SDK works with.
40        minimum: String,
41    },
42
43    /// The engine is new enough for this SDK, but not for one thing a
44    /// declaration asked for. Reported when the worker registers, rather
45    /// than letting the engine quietly ignore the field.
46    #[error("this engine ({found}) does not support {feature}; it needs at least {minimum}")]
47    UnsupportedFeature {
48        /// What was asked for, e.g. `"idempotency"`.
49        feature: String,
50        /// The version the engine reported.
51        found: String,
52        /// The oldest version that supports it.
53        minimum: String,
54    },
55
56    /// A task or workflow declaration is invalid (duplicate task names, an
57    /// unknown parent, …). Reported when the declaration is registered.
58    #[error("invalid declaration: {0}")]
59    InvalidDeclaration(String),
60
61    /// A parent task's output was asked for but is not available: either the
62    /// named task is not actually a recorded parent, or its output does not
63    /// deserialize into the type asked for.
64    #[error("parent output for task {task:?}: {reason}")]
65    ParentOutput {
66        /// The parent task name that was asked for.
67        task: String,
68        /// Why the lookup failed.
69        reason: String,
70    },
71
72    /// The call needs a [`Context`](crate::Context) attached to a client
73    /// (e.g. a `refresh_timeout` or `spawn` call), but this one is not — it
74    /// was built for tests, without a worker behind it.
75    #[error("this task context is not attached to an engine: {0}")]
76    Detached(String),
77
78    /// An argument given to the SDK is not valid (a zero-duration timeout
79    /// increment, a priority out of range, …).
80    #[error("invalid argument: {0}")]
81    InvalidArgument(String),
82
83    /// A value the SDK was asked to send could not be serialized to JSON: a
84    /// run's input, or a metadata map.
85    #[error("could not serialize to JSON: {0}")]
86    Json(String),
87
88    /// A run that was waited for finished with at least one failed task.
89    ///
90    /// A run that was *cancelled* is [`Error::RunCancelled`] instead, except
91    /// when something in it really failed as well: a run with one of each is
92    /// reported here, because the failure is the part that needs fixing.
93    #[error("workflow run {run_id} failed: {}", failures(.errors))]
94    RunFailed {
95        /// The run that failed.
96        run_id: String,
97        /// Every task of that run that reported an error.
98        errors: Vec<TaskFailure>,
99    },
100
101    /// A run that was waited for was cancelled before it finished: by
102    /// [`RunsClient::cancel`](crate::RunsClient::cancel), by
103    /// [`RunHandle::cancel`](crate::RunHandle::cancel), by a
104    /// [`Concurrency`](crate::Concurrency) limit that cancels rather than
105    /// queues, or from the dashboard.
106    ///
107    /// Told apart from [`Error::RunFailed`] by what the engine records for a
108    /// cancelled task: an `error` it never filled in (the empty string) and
109    /// no output at all. That is the only signal there is — the result stream
110    /// has one event type (`FINISHED`) and its rows carry no status field —
111    /// and it was measured on both supported engine versions, for an explicit
112    /// `CancelTasks` and for a concurrency limit. A run whose tasks both
113    /// failed and were cancelled is reported as [`Error::RunFailed`] instead.
114    ///
115    /// ```
116    /// use hatchet_client::Error;
117    ///
118    /// fn report(result: Result<u32, Error>) {
119    ///     match result {
120    ///         Ok(count) => println!("counted {count}"),
121    ///         // Not a failure: someone stopped it on purpose. Retrying
122    ///         // would undo their decision.
123    ///         Err(Error::RunCancelled { run_id, .. }) => println!("{run_id} was stopped"),
124    ///         Err(other) => println!("failed: {other}"),
125    ///     }
126    /// }
127    /// ```
128    #[error("workflow run {run_id} was cancelled{}", cancelled(.tasks))]
129    RunCancelled {
130        /// The run that was cancelled.
131        run_id: String,
132        /// The tasks of that run the engine recorded as cancelled, as
133        /// declared. Empty when the engine named none.
134        tasks: Vec<String>,
135    },
136
137    /// A finished run's output is not there, or does not deserialize into the
138    /// type it was asked for. Deserializing is *reading* rather than sending,
139    /// which is why this is not [`Error::Json`].
140    #[error("the output of workflow run {run_id} could not be read: {reason}")]
141    RunOutput {
142        /// The run whose output was asked for.
143        run_id: String,
144        /// Why the output could not be handed back.
145        reason: String,
146    },
147
148    /// The engine reported a run status this SDK has no name for. Only an
149    /// engine newer than this SDK can produce it: the statuses of
150    /// `api-contracts/v1/workflows.proto` are all covered. Reported rather
151    /// than rounded to a status we do know, which is what the official Go
152    /// SDK does (`pkg/client/admin.go:136-137`, unknown → `QUEUED`).
153    #[error(
154        "the engine reported run status {status} for run {run_id}, which this SDK does not know"
155    )]
156    UnknownRunStatus {
157        /// The run being asked about.
158        run_id: String,
159        /// The value on the wire.
160        status: i32,
161    },
162
163    /// A run's output stream skipped a chunk: the engine sent an
164    /// `event_index` further along than the one expected next, and the chunks
165    /// in between did not turn up.
166    ///
167    /// Yielded **as one item of the stream** — one item per missing stretch,
168    /// in the place that stretch would have taken — and the stream carries on
169    /// from where the engine is. Losing part of the output is not a reason to
170    /// stop delivering the rest, but it is not something to hide either: a
171    /// consumer that must have everything should treat this as fatal itself.
172    ///
173    /// Chunks are numbered per *task run*, by
174    /// [`Context::put_stream`](crate::Context::put_stream), so `task_run_id`
175    /// names the sequence with the gap in it.
176    ///
177    /// ```
178    /// use hatchet_client::Error;
179    ///
180    /// fn report(item: Result<hatchet_client::StreamChunk, Error>) {
181    ///     match item {
182    ///         Ok(chunk) => println!("{}", chunk.payload),
183    ///         // The stream goes on; this is one hole in it, not the end.
184    ///         Err(Error::StreamGap { expected, got, .. }) => {
185    ///             println!("missed chunks {expected}..{got}");
186    ///         }
187    ///         Err(other) => println!("the stream failed: {other}"),
188    ///     }
189    /// }
190    /// ```
191    #[error("the output stream of task run {task_run_id} skipped chunks {expected}..{got}")]
192    StreamGap {
193        /// The task run whose chunk sequence has the gap.
194        task_run_id: String,
195        /// The index that was expected next.
196        expected: i64,
197        /// The index that arrived instead.
198        got: i64,
199    },
200
201    /// The stream a run's result would have arrived on was given up on: the
202    /// engine ended it in a way reconnecting cannot fix, or kept ending it.
203    /// The run itself is unaffected — it is still running, and waiting for it
204    /// again opens a new subscription.
205    #[error("the run result stream failed: {0}")]
206    ResultStream(Box<tonic::Status>),
207
208    /// A [`Task::run_many`](crate::Task::run_many) or
209    /// [`Workflow::run_many`](crate::Workflow::run_many) call started some
210    /// runs before failing: the ids in `run_ids` were really triggered and
211    /// are not lost, even though the call as a whole failed. They come from
212    /// the chunks that succeeded and, when the failure is a
213    /// [`Error::BulkTriggerMismatch`], from the too-short answer itself —
214    /// every id the engine returned names a run it started. Only returned
215    /// once at least one run id was obtained; when nothing was started,
216    /// `source` is returned on its own instead.
217    #[error(
218        "a bulk trigger started {} run(s) before failing: {source}",
219        .run_ids.len()
220    )]
221    PartialBulkTrigger {
222        /// The ids of the runs that were started, in input order.
223        run_ids: Vec<String>,
224        /// Why the failing chunk failed.
225        source: Box<Error>,
226    },
227
228    /// The engine answered a `BulkTriggerWorkflow` call with a different
229    /// number of run ids than the chunk sent had requests. Treated as that
230    /// chunk's failure.
231    #[error("a bulk trigger chunk of {expected} request(s) got {got} id(s) back")]
232    BulkTriggerMismatch {
233        /// How many requests the chunk sent.
234        expected: usize,
235        /// How many ids the engine answered with.
236        got: usize,
237    },
238
239    /// A trigger was refused because its idempotency key is already held by
240    /// another run. The workflow declared the key with
241    /// [`Workflow::idempotency`](crate::Workflow::idempotency) or
242    /// [`TaskBuilder::idempotency`](crate::TaskBuilder::idempotency); no run
243    /// was started by the call that got this. Only enforced on engines at or
244    /// above [`IDEMPOTENCY_MIN_ENGINE_VERSION`](crate::IDEMPOTENCY_MIN_ENGINE_VERSION);
245    /// older engines silently ignore the key instead of returning this.
246    ///
247    /// ```
248    /// use hatchet_client::Error;
249    ///
250    /// fn handle(result: Result<String, Error>) {
251    ///     match result {
252    ///         Ok(run_id) => println!("started {run_id}"),
253    ///         Err(Error::IdempotencyCollision(collision)) => {
254    ///             println!("already running as {}", collision.existing_run_id);
255    ///         }
256    ///         Err(other) => println!("failed: {other}"),
257    ///     }
258    /// }
259    /// ```
260    #[error(
261        "a run with the same idempotency key already exists: {}",
262        .0.existing_run_id
263    )]
264    IdempotencyCollision(IdempotencyCollision),
265
266    /// A [`Task::run_many`](crate::Task::run_many) or
267    /// [`Workflow::run_many`](crate::Workflow::run_many) call in which some
268    /// requests were refused on their idempotency key. The ones in `started`
269    /// really were triggered.
270    #[error(
271        "a bulk trigger started {} run(s) and hit {} idempotency collision(s)",
272        .started.len(),
273        .collisions.len()
274    )]
275    BulkIdempotencyCollision {
276        /// The runs the same call did start, in the engine's order.
277        started: Vec<String>,
278        /// One entry per refused request.
279        collisions: Vec<IdempotencyCollision>,
280    },
281
282    /// The engine rejected a durable request because this attempt did not
283    /// repeat what the last one did.
284    ///
285    /// A durable handler is re-executed from the top on every attempt, and the
286    /// engine matches each durable operation against its event log by
287    /// position. Reaching a different operation — or reaching them in a
288    /// different order — is this error, and **retrying cannot help**: the run
289    /// is reported failed with `should_not_retry`. Common causes: racing
290    /// durable waits against `tokio::time::timeout` or `select!`, branching on
291    /// something that is not memoised (the clock, a random number, a live API
292    /// call), or changing the handler's durable steps while runs are in
293    /// flight.
294    #[error(
295        "durable task run {task_run_id} (invocation {invocation_count}, node {node_id}) diverged \
296         from its recorded event log: {message}"
297    )]
298    Nondeterminism {
299        /// The durable task run: `Context::task_run_id`.
300        task_run_id: String,
301        /// Which attempt this was.
302        invocation_count: i32,
303        /// The log entry the engine was expecting something else at.
304        node_id: i64,
305        /// What the engine said. The wording differs between engine versions,
306        /// so match on this variant rather than on the text.
307        message: String,
308    },
309
310    /// A durable log entry the engine completed as a failure.
311    ///
312    /// The engine says a durable wait is over *and* that what it was waiting
313    /// for went wrong; turning that into a success would hide it, so the
314    /// handler sees the failure instead. Retrying is allowed: the attempt is
315    /// reported as an ordinary task failure.
316    #[error(
317        "durable log entry for task run {task_run_id} (invocation {invocation_count}, node \
318         {node_id}) failed: {message}"
319    )]
320    DurableEntryFailed {
321        /// The durable task run: `Context::task_run_id`.
322        task_run_id: String,
323        /// Which attempt this was.
324        invocation_count: i32,
325        /// The log entry that failed.
326        node_id: i64,
327        /// What the engine said, or a stand-in when it said nothing.
328        message: String,
329    },
330}
331
332/// One task's failure within a run, as the engine reported it.
333#[derive(Clone, Debug, PartialEq, Eq)]
334pub struct TaskFailure {
335    /// The task that failed, as declared.
336    pub task_name: String,
337    /// What the engine said went wrong.
338    pub message: String,
339}
340
341/// One idempotency-key collision, as the engine reported it.
342///
343/// The engine's `v1.IdempotencyCollisionError` also carries
344/// `colliding_run_external_id` (the run the refused trigger would have
345/// created), but it was observed to always arrive empty, so it is not
346/// exposed here.
347#[derive(Clone, Debug, PartialEq, Eq)]
348pub struct IdempotencyCollision {
349    /// The run that already holds the key.
350    pub existing_run_id: String,
351}
352
353/// Joins the failed tasks of a run into one line: `task: why`, separated by
354/// `"; "`.
355fn failures(errors: &[TaskFailure]) -> String {
356    errors
357        .iter()
358        .map(|failure| format!("{}: {}", failure.task_name, failure.message))
359        .collect::<Vec<_>>()
360        .join("; ")
361}
362
363/// `" (cancelled: a, b)"`, or nothing when the engine named no tasks.
364fn cancelled(tasks: &[String]) -> String {
365    if tasks.is_empty() {
366        return String::new();
367    }
368    format!(" (cancelled: {})", tasks.join(", "))
369}
370
371/// Joins an error's message with the messages of its whole `source()` chain,
372/// separated by `": "`, so the underlying cause is not lost behind a generic
373/// top-level message (e.g. tonic's "transport error").
374fn source_chain(mut error: &dyn std::error::Error) -> String {
375    let mut message = error.to_string();
376    while let Some(source) = error.source() {
377        message.push_str(": ");
378        message.push_str(&source.to_string());
379        error = source;
380    }
381    message
382}
383
384impl From<tonic::Status> for Error {
385    fn from(status: tonic::Status) -> Self {
386        Self::Grpc(Box::new(status))
387    }
388}
389
390/// `Result` with the SDK's [`Error`].
391pub type Result<T> = std::result::Result<T, Error>;
392
393#[cfg(test)]
394mod tests {
395    use super::*;
396
397    #[test]
398    fn transport_error_display_includes_the_cause_chain() {
399        let source = tonic::transport::Endpoint::from_shared("not a uri").unwrap_err();
400        let error = Error::Transport {
401            address: "example.com:7077".to_string(),
402            source,
403        };
404        let message = error.to_string();
405        assert!(
406            message.starts_with("could not set up the connection to example.com:7077: "),
407            "{message}"
408        );
409        assert!(message != "could not set up the connection to example.com:7077: transport error");
410        assert!(
411            message.len()
412                > "could not set up the connection to example.com:7077: transport error".len(),
413            "{message}: expected more than just the generic transport error message"
414        );
415    }
416
417    /// serde's own message ("key must be a string") says nothing about where
418    /// it came from, so the variant says it.
419    #[test]
420    fn a_json_error_names_what_failed_before_the_serde_message() {
421        let error = Error::Json("key must be a string".to_string());
422        assert_eq!(
423            error.to_string(),
424            "could not serialize to JSON: key must be a string"
425        );
426    }
427
428    #[test]
429    fn a_failed_run_names_every_task_that_failed_and_why() {
430        let error = Error::RunFailed {
431            run_id: "run-1".to_string(),
432            errors: vec![
433                TaskFailure {
434                    task_name: "fetch".to_string(),
435                    message: "boom".to_string(),
436                },
437                TaskFailure {
438                    task_name: "report".to_string(),
439                    message: "no input".to_string(),
440                },
441            ],
442        };
443        assert_eq!(
444            error.to_string(),
445            "workflow run run-1 failed: fetch: boom; report: no input"
446        );
447    }
448
449    #[test]
450    fn a_cancelled_run_names_the_tasks_the_engine_recorded() {
451        let error = Error::RunCancelled {
452            run_id: "run-1".to_string(),
453            tasks: vec!["fetch".to_string(), "report".to_string()],
454        };
455        assert_eq!(
456            error.to_string(),
457            "workflow run run-1 was cancelled (cancelled: fetch, report)"
458        );
459    }
460
461    /// The engine can cancel a run before it has any task to name, and the
462    /// message must still read as a sentence.
463    #[test]
464    fn a_cancelled_run_with_no_named_tasks_still_reads() {
465        let error = Error::RunCancelled {
466            run_id: "run-1".to_string(),
467            tasks: Vec::new(),
468        };
469        assert_eq!(error.to_string(), "workflow run run-1 was cancelled");
470    }
471
472    #[test]
473    fn source_chain_joins_nested_causes() {
474        #[derive(Debug)]
475        struct Inner;
476        impl std::fmt::Display for Inner {
477            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
478                write!(f, "inner cause")
479            }
480        }
481        impl std::error::Error for Inner {}
482
483        #[derive(Debug)]
484        struct Outer(Inner);
485        impl std::fmt::Display for Outer {
486            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
487                write!(f, "outer cause")
488            }
489        }
490        impl std::error::Error for Outer {
491            fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
492                Some(&self.0)
493            }
494        }
495
496        let outer = Outer(Inner);
497        assert_eq!(source_chain(&outer), "outer cause: inner cause");
498    }
499}