hatchet_client/error.rs
1//! The SDK's error type.
2
3/// Errors returned by `hatchet-client`.
4#[derive(Debug, thiserror::Error)]
5#[non_exhaustive]
6pub enum Error {
7 /// The API token is not a JWT carrying the claims the SDK needs.
8 /// The message never contains the token itself.
9 #[error("invalid token: {0}")]
10 InvalidToken(String),
11
12 /// The configuration is incomplete or contradictory.
13 #[error("invalid configuration: {0}")]
14 Config(String),
15
16 /// The gRPC channel could not be set up.
17 #[error("could not set up the connection to {address}: {}", source_chain(.source))]
18 Transport {
19 /// The `host:port` being connected to.
20 address: String,
21 /// The underlying transport error.
22 #[source]
23 source: tonic::transport::Error,
24 },
25
26 /// The engine answered a gRPC call with an error status.
27 #[error("gRPC call failed: {0}")]
28 Grpc(Box<tonic::Status>),
29
30 /// The engine reported a version string that is not `vMAJOR.MINOR.PATCH`.
31 #[error("the engine reported an unparseable version {0:?}")]
32 InvalidEngineVersion(String),
33
34 /// The engine is older than the SDK supports.
35 #[error("Hatchet engine {found} is older than the minimum supported version {minimum}")]
36 UnsupportedEngine {
37 /// The version the engine reported.
38 found: String,
39 /// The oldest version this SDK works with.
40 minimum: String,
41 },
42
43 /// The engine is new enough for this SDK, but not for one thing a
44 /// declaration asked for. Reported when the worker registers, rather
45 /// than letting the engine quietly ignore the field.
46 #[error("this engine ({found}) does not support {feature}; it needs at least {minimum}")]
47 UnsupportedFeature {
48 /// What was asked for, e.g. `"idempotency"`.
49 feature: String,
50 /// The version the engine reported.
51 found: String,
52 /// The oldest version that supports it.
53 minimum: String,
54 },
55
56 /// A task or workflow declaration is invalid (duplicate task names, an
57 /// unknown parent, …). Reported when the declaration is registered.
58 #[error("invalid declaration: {0}")]
59 InvalidDeclaration(String),
60
61 /// A parent task's output was asked for but is not available: either the
62 /// named task is not actually a recorded parent, or its output does not
63 /// deserialize into the type asked for.
64 #[error("parent output for task {task:?}: {reason}")]
65 ParentOutput {
66 /// The parent task name that was asked for.
67 task: String,
68 /// Why the lookup failed.
69 reason: String,
70 },
71
72 /// The call needs a [`Context`](crate::Context) attached to a client
73 /// (e.g. a `refresh_timeout` or `spawn` call), but this one is not — it
74 /// was built for tests, without a worker behind it.
75 #[error("this task context is not attached to an engine: {0}")]
76 Detached(String),
77
78 /// An argument given to the SDK is not valid (a zero-duration timeout
79 /// increment, a priority out of range, …).
80 #[error("invalid argument: {0}")]
81 InvalidArgument(String),
82
83 /// A value the SDK was asked to send could not be serialized to JSON: a
84 /// run's input, or a metadata map.
85 #[error("could not serialize to JSON: {0}")]
86 Json(String),
87
88 /// A run that was waited for finished with at least one failed task.
89 ///
90 /// A run that was *cancelled* is [`Error::RunCancelled`] instead, except
91 /// when something in it really failed as well: a run with one of each is
92 /// reported here, because the failure is the part that needs fixing.
93 #[error("workflow run {run_id} failed: {}", failures(.errors))]
94 RunFailed {
95 /// The run that failed.
96 run_id: String,
97 /// Every task of that run that reported an error.
98 errors: Vec<TaskFailure>,
99 },
100
101 /// A run that was waited for was cancelled before it finished: by
102 /// [`RunsClient::cancel`](crate::RunsClient::cancel), by
103 /// [`RunHandle::cancel`](crate::RunHandle::cancel), by a
104 /// [`Concurrency`](crate::Concurrency) limit that cancels rather than
105 /// queues, or from the dashboard.
106 ///
107 /// Told apart from [`Error::RunFailed`] by what the engine records for a
108 /// cancelled task: an `error` it never filled in (the empty string) and
109 /// no output at all. That is the only signal there is — the result stream
110 /// has one event type (`FINISHED`) and its rows carry no status field —
111 /// and it was measured on both supported engine versions, for an explicit
112 /// `CancelTasks` and for a concurrency limit. A run whose tasks both
113 /// failed and were cancelled is reported as [`Error::RunFailed`] instead.
114 ///
115 /// ```
116 /// use hatchet_client::Error;
117 ///
118 /// fn report(result: Result<u32, Error>) {
119 /// match result {
120 /// Ok(count) => println!("counted {count}"),
121 /// // Not a failure: someone stopped it on purpose. Retrying
122 /// // would undo their decision.
123 /// Err(Error::RunCancelled { run_id, .. }) => println!("{run_id} was stopped"),
124 /// Err(other) => println!("failed: {other}"),
125 /// }
126 /// }
127 /// ```
128 #[error("workflow run {run_id} was cancelled{}", cancelled(.tasks))]
129 RunCancelled {
130 /// The run that was cancelled.
131 run_id: String,
132 /// The tasks of that run the engine recorded as cancelled, as
133 /// declared. Empty when the engine named none.
134 tasks: Vec<String>,
135 },
136
137 /// A finished run's output is not there, or does not deserialize into the
138 /// type it was asked for. Deserializing is *reading* rather than sending,
139 /// which is why this is not [`Error::Json`].
140 #[error("the output of workflow run {run_id} could not be read: {reason}")]
141 RunOutput {
142 /// The run whose output was asked for.
143 run_id: String,
144 /// Why the output could not be handed back.
145 reason: String,
146 },
147
148 /// The engine reported a run status this SDK has no name for. Only an
149 /// engine newer than this SDK can produce it: the statuses of
150 /// `api-contracts/v1/workflows.proto` are all covered. Reported rather
151 /// than rounded to a status we do know, which is what the official Go
152 /// SDK does (`pkg/client/admin.go:136-137`, unknown → `QUEUED`).
153 #[error(
154 "the engine reported run status {status} for run {run_id}, which this SDK does not know"
155 )]
156 UnknownRunStatus {
157 /// The run being asked about.
158 run_id: String,
159 /// The value on the wire.
160 status: i32,
161 },
162
163 /// A run's output stream skipped a chunk: the engine sent an
164 /// `event_index` further along than the one expected next, and the chunks
165 /// in between did not turn up.
166 ///
167 /// Yielded **as one item of the stream** — one item per missing stretch,
168 /// in the place that stretch would have taken — and the stream carries on
169 /// from where the engine is. Losing part of the output is not a reason to
170 /// stop delivering the rest, but it is not something to hide either: a
171 /// consumer that must have everything should treat this as fatal itself.
172 ///
173 /// Chunks are numbered per *task run*, by
174 /// [`Context::put_stream`](crate::Context::put_stream), so `task_run_id`
175 /// names the sequence with the gap in it.
176 ///
177 /// ```
178 /// use hatchet_client::Error;
179 ///
180 /// fn report(item: Result<hatchet_client::StreamChunk, Error>) {
181 /// match item {
182 /// Ok(chunk) => println!("{}", chunk.payload),
183 /// // The stream goes on; this is one hole in it, not the end.
184 /// Err(Error::StreamGap { expected, got, .. }) => {
185 /// println!("missed chunks {expected}..{got}");
186 /// }
187 /// Err(other) => println!("the stream failed: {other}"),
188 /// }
189 /// }
190 /// ```
191 #[error("the output stream of task run {task_run_id} skipped chunks {expected}..{got}")]
192 StreamGap {
193 /// The task run whose chunk sequence has the gap.
194 task_run_id: String,
195 /// The index that was expected next.
196 expected: i64,
197 /// The index that arrived instead.
198 got: i64,
199 },
200
201 /// The stream a run's result would have arrived on was given up on: the
202 /// engine ended it in a way reconnecting cannot fix, or kept ending it.
203 /// The run itself is unaffected — it is still running, and waiting for it
204 /// again opens a new subscription.
205 #[error("the run result stream failed: {0}")]
206 ResultStream(Box<tonic::Status>),
207
208 /// A [`Task::run_many`](crate::Task::run_many) or
209 /// [`Workflow::run_many`](crate::Workflow::run_many) call started some
210 /// runs before failing: the ids in `run_ids` were really triggered and
211 /// are not lost, even though the call as a whole failed. They come from
212 /// the chunks that succeeded and, when the failure is a
213 /// [`Error::BulkTriggerMismatch`], from the too-short answer itself —
214 /// every id the engine returned names a run it started. Only returned
215 /// once at least one run id was obtained; when nothing was started,
216 /// `source` is returned on its own instead.
217 #[error(
218 "a bulk trigger started {} run(s) before failing: {source}",
219 .run_ids.len()
220 )]
221 PartialBulkTrigger {
222 /// The ids of the runs that were started, in input order.
223 run_ids: Vec<String>,
224 /// Why the failing chunk failed.
225 source: Box<Error>,
226 },
227
228 /// The engine answered a `BulkTriggerWorkflow` call with a different
229 /// number of run ids than the chunk sent had requests. Treated as that
230 /// chunk's failure.
231 #[error("a bulk trigger chunk of {expected} request(s) got {got} id(s) back")]
232 BulkTriggerMismatch {
233 /// How many requests the chunk sent.
234 expected: usize,
235 /// How many ids the engine answered with.
236 got: usize,
237 },
238
239 /// A trigger was refused because its idempotency key is already held by
240 /// another run. The workflow declared the key with
241 /// [`Workflow::idempotency`](crate::Workflow::idempotency) or
242 /// [`TaskBuilder::idempotency`](crate::TaskBuilder::idempotency); no run
243 /// was started by the call that got this. Only enforced on engines at or
244 /// above [`IDEMPOTENCY_MIN_ENGINE_VERSION`](crate::IDEMPOTENCY_MIN_ENGINE_VERSION);
245 /// older engines silently ignore the key instead of returning this.
246 ///
247 /// ```
248 /// use hatchet_client::Error;
249 ///
250 /// fn handle(result: Result<String, Error>) {
251 /// match result {
252 /// Ok(run_id) => println!("started {run_id}"),
253 /// Err(Error::IdempotencyCollision(collision)) => {
254 /// println!("already running as {}", collision.existing_run_id);
255 /// }
256 /// Err(other) => println!("failed: {other}"),
257 /// }
258 /// }
259 /// ```
260 #[error(
261 "a run with the same idempotency key already exists: {}",
262 .0.existing_run_id
263 )]
264 IdempotencyCollision(IdempotencyCollision),
265
266 /// A [`Task::run_many`](crate::Task::run_many) or
267 /// [`Workflow::run_many`](crate::Workflow::run_many) call in which some
268 /// requests were refused on their idempotency key. The ones in `started`
269 /// really were triggered.
270 #[error(
271 "a bulk trigger started {} run(s) and hit {} idempotency collision(s)",
272 .started.len(),
273 .collisions.len()
274 )]
275 BulkIdempotencyCollision {
276 /// The runs the same call did start, in the engine's order.
277 started: Vec<String>,
278 /// One entry per refused request.
279 collisions: Vec<IdempotencyCollision>,
280 },
281
282 /// The engine rejected a durable request because this attempt did not
283 /// repeat what the last one did.
284 ///
285 /// A durable handler is re-executed from the top on every attempt, and the
286 /// engine matches each durable operation against its event log by
287 /// position. Reaching a different operation — or reaching them in a
288 /// different order — is this error, and **retrying cannot help**: the run
289 /// is reported failed with `should_not_retry`. Common causes: racing
290 /// durable waits against `tokio::time::timeout` or `select!`, branching on
291 /// something that is not memoised (the clock, a random number, a live API
292 /// call), or changing the handler's durable steps while runs are in
293 /// flight.
294 #[error(
295 "durable task run {task_run_id} (invocation {invocation_count}, node {node_id}) diverged \
296 from its recorded event log: {message}"
297 )]
298 Nondeterminism {
299 /// The durable task run: `Context::task_run_id`.
300 task_run_id: String,
301 /// Which attempt this was.
302 invocation_count: i32,
303 /// The log entry the engine was expecting something else at.
304 node_id: i64,
305 /// What the engine said. The wording differs between engine versions,
306 /// so match on this variant rather than on the text.
307 message: String,
308 },
309
310 /// A durable log entry the engine completed as a failure.
311 ///
312 /// The engine says a durable wait is over *and* that what it was waiting
313 /// for went wrong; turning that into a success would hide it, so the
314 /// handler sees the failure instead. Retrying is allowed: the attempt is
315 /// reported as an ordinary task failure.
316 #[error(
317 "durable log entry for task run {task_run_id} (invocation {invocation_count}, node \
318 {node_id}) failed: {message}"
319 )]
320 DurableEntryFailed {
321 /// The durable task run: `Context::task_run_id`.
322 task_run_id: String,
323 /// Which attempt this was.
324 invocation_count: i32,
325 /// The log entry that failed.
326 node_id: i64,
327 /// What the engine said, or a stand-in when it said nothing.
328 message: String,
329 },
330}
331
332/// One task's failure within a run, as the engine reported it.
333#[derive(Clone, Debug, PartialEq, Eq)]
334pub struct TaskFailure {
335 /// The task that failed, as declared.
336 pub task_name: String,
337 /// What the engine said went wrong.
338 pub message: String,
339}
340
341/// One idempotency-key collision, as the engine reported it.
342///
343/// The engine's `v1.IdempotencyCollisionError` also carries
344/// `colliding_run_external_id` (the run the refused trigger would have
345/// created), but it was observed to always arrive empty, so it is not
346/// exposed here.
347#[derive(Clone, Debug, PartialEq, Eq)]
348pub struct IdempotencyCollision {
349 /// The run that already holds the key.
350 pub existing_run_id: String,
351}
352
353/// Joins the failed tasks of a run into one line: `task: why`, separated by
354/// `"; "`.
355fn failures(errors: &[TaskFailure]) -> String {
356 errors
357 .iter()
358 .map(|failure| format!("{}: {}", failure.task_name, failure.message))
359 .collect::<Vec<_>>()
360 .join("; ")
361}
362
363/// `" (cancelled: a, b)"`, or nothing when the engine named no tasks.
364fn cancelled(tasks: &[String]) -> String {
365 if tasks.is_empty() {
366 return String::new();
367 }
368 format!(" (cancelled: {})", tasks.join(", "))
369}
370
371/// Joins an error's message with the messages of its whole `source()` chain,
372/// separated by `": "`, so the underlying cause is not lost behind a generic
373/// top-level message (e.g. tonic's "transport error").
374fn source_chain(mut error: &dyn std::error::Error) -> String {
375 let mut message = error.to_string();
376 while let Some(source) = error.source() {
377 message.push_str(": ");
378 message.push_str(&source.to_string());
379 error = source;
380 }
381 message
382}
383
384impl From<tonic::Status> for Error {
385 fn from(status: tonic::Status) -> Self {
386 Self::Grpc(Box::new(status))
387 }
388}
389
390/// `Result` with the SDK's [`Error`].
391pub type Result<T> = std::result::Result<T, Error>;
392
393#[cfg(test)]
394mod tests {
395 use super::*;
396
397 #[test]
398 fn transport_error_display_includes_the_cause_chain() {
399 let source = tonic::transport::Endpoint::from_shared("not a uri").unwrap_err();
400 let error = Error::Transport {
401 address: "example.com:7077".to_string(),
402 source,
403 };
404 let message = error.to_string();
405 assert!(
406 message.starts_with("could not set up the connection to example.com:7077: "),
407 "{message}"
408 );
409 assert!(message != "could not set up the connection to example.com:7077: transport error");
410 assert!(
411 message.len()
412 > "could not set up the connection to example.com:7077: transport error".len(),
413 "{message}: expected more than just the generic transport error message"
414 );
415 }
416
417 /// serde's own message ("key must be a string") says nothing about where
418 /// it came from, so the variant says it.
419 #[test]
420 fn a_json_error_names_what_failed_before_the_serde_message() {
421 let error = Error::Json("key must be a string".to_string());
422 assert_eq!(
423 error.to_string(),
424 "could not serialize to JSON: key must be a string"
425 );
426 }
427
428 #[test]
429 fn a_failed_run_names_every_task_that_failed_and_why() {
430 let error = Error::RunFailed {
431 run_id: "run-1".to_string(),
432 errors: vec![
433 TaskFailure {
434 task_name: "fetch".to_string(),
435 message: "boom".to_string(),
436 },
437 TaskFailure {
438 task_name: "report".to_string(),
439 message: "no input".to_string(),
440 },
441 ],
442 };
443 assert_eq!(
444 error.to_string(),
445 "workflow run run-1 failed: fetch: boom; report: no input"
446 );
447 }
448
449 #[test]
450 fn a_cancelled_run_names_the_tasks_the_engine_recorded() {
451 let error = Error::RunCancelled {
452 run_id: "run-1".to_string(),
453 tasks: vec!["fetch".to_string(), "report".to_string()],
454 };
455 assert_eq!(
456 error.to_string(),
457 "workflow run run-1 was cancelled (cancelled: fetch, report)"
458 );
459 }
460
461 /// The engine can cancel a run before it has any task to name, and the
462 /// message must still read as a sentence.
463 #[test]
464 fn a_cancelled_run_with_no_named_tasks_still_reads() {
465 let error = Error::RunCancelled {
466 run_id: "run-1".to_string(),
467 tasks: Vec::new(),
468 };
469 assert_eq!(error.to_string(), "workflow run run-1 was cancelled");
470 }
471
472 #[test]
473 fn source_chain_joins_nested_causes() {
474 #[derive(Debug)]
475 struct Inner;
476 impl std::fmt::Display for Inner {
477 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
478 write!(f, "inner cause")
479 }
480 }
481 impl std::error::Error for Inner {}
482
483 #[derive(Debug)]
484 struct Outer(Inner);
485 impl std::fmt::Display for Outer {
486 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
487 write!(f, "outer cause")
488 }
489 }
490 impl std::error::Error for Outer {
491 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
492 Some(&self.0)
493 }
494 }
495
496 let outer = Outer(Inner);
497 assert_eq!(source_chain(&outer), "outer cause: inner cause");
498 }
499}