1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
//! The SDK's error type.
/// Errors returned by `hatchet-client`.
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum Error {
/// The API token is not a JWT carrying the claims the SDK needs.
/// The message never contains the token itself.
#[error("invalid token: {0}")]
InvalidToken(String),
/// The configuration is incomplete or contradictory.
#[error("invalid configuration: {0}")]
Config(String),
/// The gRPC channel could not be set up.
#[error("could not set up the connection to {address}: {}", source_chain(.source))]
Transport {
/// The `host:port` being connected to.
address: String,
/// The underlying transport error.
#[source]
source: tonic::transport::Error,
},
/// The engine answered a gRPC call with an error status.
#[error("gRPC call failed: {0}")]
Grpc(Box<tonic::Status>),
/// The engine reported a version string that is not `vMAJOR.MINOR.PATCH`.
#[error("the engine reported an unparseable version {0:?}")]
InvalidEngineVersion(String),
/// The engine is older than the SDK supports.
#[error("Hatchet engine {found} is older than the minimum supported version {minimum}")]
UnsupportedEngine {
/// The version the engine reported.
found: String,
/// The oldest version this SDK works with.
minimum: String,
},
/// The engine is new enough for this SDK, but not for one thing a
/// declaration asked for. Reported when the worker registers, rather
/// than letting the engine quietly ignore the field.
#[error("this engine ({found}) does not support {feature}; it needs at least {minimum}")]
UnsupportedFeature {
/// What was asked for, e.g. `"idempotency"`.
feature: String,
/// The version the engine reported.
found: String,
/// The oldest version that supports it.
minimum: String,
},
/// A task or workflow declaration is invalid (duplicate task names, an
/// unknown parent, …). Reported when the declaration is registered.
#[error("invalid declaration: {0}")]
InvalidDeclaration(String),
/// A parent task's output was asked for but is not available: either the
/// named task is not actually a recorded parent, or its output does not
/// deserialize into the type asked for.
#[error("parent output for task {task:?}: {reason}")]
ParentOutput {
/// The parent task name that was asked for.
task: String,
/// Why the lookup failed.
reason: String,
},
/// The call needs a [`Context`](crate::Context) attached to a client
/// (e.g. a `refresh_timeout` or `spawn` call), but this one is not — it
/// was built for tests, without a worker behind it.
#[error("this task context is not attached to an engine: {0}")]
Detached(String),
/// An argument given to the SDK is not valid (a zero-duration timeout
/// increment, a priority out of range, …).
#[error("invalid argument: {0}")]
InvalidArgument(String),
/// A value the SDK was asked to send could not be serialized to JSON: a
/// run's input, or a metadata map.
#[error("could not serialize to JSON: {0}")]
Json(String),
/// A run that was waited for finished with at least one failed task.
///
/// A run that was *cancelled* is [`Error::RunCancelled`] instead, except
/// when something in it really failed as well: a run with one of each is
/// reported here, because the failure is the part that needs fixing.
#[error("workflow run {run_id} failed: {}", failures(.errors))]
RunFailed {
/// The run that failed.
run_id: String,
/// Every task of that run that reported an error.
errors: Vec<TaskFailure>,
},
/// A run that was waited for was cancelled before it finished: by
/// [`RunsClient::cancel`](crate::RunsClient::cancel), by
/// [`RunHandle::cancel`](crate::RunHandle::cancel), by a
/// [`Concurrency`](crate::Concurrency) limit that cancels rather than
/// queues, or from the dashboard.
///
/// Told apart from [`Error::RunFailed`] by what the engine records for a
/// cancelled task: an `error` it never filled in (the empty string) and
/// no output at all. That is the only signal there is — the result stream
/// has one event type (`FINISHED`) and its rows carry no status field —
/// and it was measured on both supported engine versions, for an explicit
/// `CancelTasks` and for a concurrency limit. A run whose tasks both
/// failed and were cancelled is reported as [`Error::RunFailed`] instead.
///
/// ```
/// use hatchet_client::Error;
///
/// fn report(result: Result<u32, Error>) {
/// match result {
/// Ok(count) => println!("counted {count}"),
/// // Not a failure: someone stopped it on purpose. Retrying
/// // would undo their decision.
/// Err(Error::RunCancelled { run_id, .. }) => println!("{run_id} was stopped"),
/// Err(other) => println!("failed: {other}"),
/// }
/// }
/// ```
#[error("workflow run {run_id} was cancelled{}", cancelled(.tasks))]
RunCancelled {
/// The run that was cancelled.
run_id: String,
/// The tasks of that run the engine recorded as cancelled, as
/// declared. Empty when the engine named none.
tasks: Vec<String>,
},
/// A finished run's output is not there, or does not deserialize into the
/// type it was asked for. Deserializing is *reading* rather than sending,
/// which is why this is not [`Error::Json`].
#[error("the output of workflow run {run_id} could not be read: {reason}")]
RunOutput {
/// The run whose output was asked for.
run_id: String,
/// Why the output could not be handed back.
reason: String,
},
/// The engine reported a run status this SDK has no name for. Only an
/// engine newer than this SDK can produce it: the statuses of
/// `api-contracts/v1/workflows.proto` are all covered. Reported rather
/// than rounded to a status we do know, which is what the official Go
/// SDK does (`pkg/client/admin.go:136-137`, unknown → `QUEUED`).
#[error(
"the engine reported run status {status} for run {run_id}, which this SDK does not know"
)]
UnknownRunStatus {
/// The run being asked about.
run_id: String,
/// The value on the wire.
status: i32,
},
/// A run's output stream skipped a chunk: the engine sent an
/// `event_index` further along than the one expected next, and the chunks
/// in between did not turn up.
///
/// Yielded **as one item of the stream** — one item per missing stretch,
/// in the place that stretch would have taken — and the stream carries on
/// from where the engine is. Losing part of the output is not a reason to
/// stop delivering the rest, but it is not something to hide either: a
/// consumer that must have everything should treat this as fatal itself.
///
/// Chunks are numbered per *task run*, by
/// [`Context::put_stream`](crate::Context::put_stream), so `task_run_id`
/// names the sequence with the gap in it.
///
/// ```
/// use hatchet_client::Error;
///
/// fn report(item: Result<hatchet_client::StreamChunk, Error>) {
/// match item {
/// Ok(chunk) => println!("{}", chunk.payload),
/// // The stream goes on; this is one hole in it, not the end.
/// Err(Error::StreamGap { expected, got, .. }) => {
/// println!("missed chunks {expected}..{got}");
/// }
/// Err(other) => println!("the stream failed: {other}"),
/// }
/// }
/// ```
#[error("the output stream of task run {task_run_id} skipped chunks {expected}..{got}")]
StreamGap {
/// The task run whose chunk sequence has the gap.
task_run_id: String,
/// The index that was expected next.
expected: i64,
/// The index that arrived instead.
got: i64,
},
/// The stream a run's result would have arrived on was given up on: the
/// engine ended it in a way reconnecting cannot fix, or kept ending it.
/// The run itself is unaffected — it is still running, and waiting for it
/// again opens a new subscription.
#[error("the run result stream failed: {0}")]
ResultStream(Box<tonic::Status>),
/// A [`Task::run_many`](crate::Task::run_many) or
/// [`Workflow::run_many`](crate::Workflow::run_many) call started some
/// runs before failing: the ids in `run_ids` were really triggered and
/// are not lost, even though the call as a whole failed. They come from
/// the chunks that succeeded and, when the failure is a
/// [`Error::BulkTriggerMismatch`], from the too-short answer itself —
/// every id the engine returned names a run it started. Only returned
/// once at least one run id was obtained; when nothing was started,
/// `source` is returned on its own instead.
#[error(
"a bulk trigger started {} run(s) before failing: {source}",
.run_ids.len()
)]
PartialBulkTrigger {
/// The ids of the runs that were started, in input order.
run_ids: Vec<String>,
/// Why the failing chunk failed.
source: Box<Error>,
},
/// The engine answered a `BulkTriggerWorkflow` call with a different
/// number of run ids than the chunk sent had requests. Treated as that
/// chunk's failure.
#[error("a bulk trigger chunk of {expected} request(s) got {got} id(s) back")]
BulkTriggerMismatch {
/// How many requests the chunk sent.
expected: usize,
/// How many ids the engine answered with.
got: usize,
},
/// A trigger was refused because its idempotency key is already held by
/// another run. The workflow declared the key with
/// [`Workflow::idempotency`](crate::Workflow::idempotency) or
/// [`TaskBuilder::idempotency`](crate::TaskBuilder::idempotency); no run
/// was started by the call that got this. Only enforced on engines at or
/// above [`IDEMPOTENCY_MIN_ENGINE_VERSION`](crate::IDEMPOTENCY_MIN_ENGINE_VERSION);
/// older engines silently ignore the key instead of returning this.
///
/// ```
/// use hatchet_client::Error;
///
/// fn handle(result: Result<String, Error>) {
/// match result {
/// Ok(run_id) => println!("started {run_id}"),
/// Err(Error::IdempotencyCollision(collision)) => {
/// println!("already running as {}", collision.existing_run_id);
/// }
/// Err(other) => println!("failed: {other}"),
/// }
/// }
/// ```
#[error(
"a run with the same idempotency key already exists: {}",
.0.existing_run_id
)]
IdempotencyCollision(IdempotencyCollision),
/// A [`Task::run_many`](crate::Task::run_many) or
/// [`Workflow::run_many`](crate::Workflow::run_many) call in which some
/// requests were refused on their idempotency key. The ones in `started`
/// really were triggered.
#[error(
"a bulk trigger started {} run(s) and hit {} idempotency collision(s)",
.started.len(),
.collisions.len()
)]
BulkIdempotencyCollision {
/// The runs the same call did start, in the engine's order.
started: Vec<String>,
/// One entry per refused request.
collisions: Vec<IdempotencyCollision>,
},
/// The engine rejected a durable request because this attempt did not
/// repeat what the last one did.
///
/// A durable handler is re-executed from the top on every attempt, and the
/// engine matches each durable operation against its event log by
/// position. Reaching a different operation — or reaching them in a
/// different order — is this error, and **retrying cannot help**: the run
/// is reported failed with `should_not_retry`. Common causes: racing
/// durable waits against `tokio::time::timeout` or `select!`, branching on
/// something that is not memoised (the clock, a random number, a live API
/// call), or changing the handler's durable steps while runs are in
/// flight.
#[error(
"durable task run {task_run_id} (invocation {invocation_count}, node {node_id}) diverged \
from its recorded event log: {message}"
)]
Nondeterminism {
/// The durable task run: `Context::task_run_id`.
task_run_id: String,
/// Which attempt this was.
invocation_count: i32,
/// The log entry the engine was expecting something else at.
node_id: i64,
/// What the engine said. The wording differs between engine versions,
/// so match on this variant rather than on the text.
message: String,
},
/// A durable log entry the engine completed as a failure.
///
/// The engine says a durable wait is over *and* that what it was waiting
/// for went wrong; turning that into a success would hide it, so the
/// handler sees the failure instead. Retrying is allowed: the attempt is
/// reported as an ordinary task failure.
#[error(
"durable log entry for task run {task_run_id} (invocation {invocation_count}, node \
{node_id}) failed: {message}"
)]
DurableEntryFailed {
/// The durable task run: `Context::task_run_id`.
task_run_id: String,
/// Which attempt this was.
invocation_count: i32,
/// The log entry that failed.
node_id: i64,
/// What the engine said, or a stand-in when it said nothing.
message: String,
},
}
/// One task's failure within a run, as the engine reported it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TaskFailure {
/// The task that failed, as declared.
pub task_name: String,
/// What the engine said went wrong.
pub message: String,
}
/// One idempotency-key collision, as the engine reported it.
///
/// The engine's `v1.IdempotencyCollisionError` also carries
/// `colliding_run_external_id` (the run the refused trigger would have
/// created), but it was observed to always arrive empty, so it is not
/// exposed here.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct IdempotencyCollision {
/// The run that already holds the key.
pub existing_run_id: String,
}
/// Joins the failed tasks of a run into one line: `task: why`, separated by
/// `"; "`.
fn failures(errors: &[TaskFailure]) -> String {
errors
.iter()
.map(|failure| format!("{}: {}", failure.task_name, failure.message))
.collect::<Vec<_>>()
.join("; ")
}
/// `" (cancelled: a, b)"`, or nothing when the engine named no tasks.
fn cancelled(tasks: &[String]) -> String {
if tasks.is_empty() {
return String::new();
}
format!(" (cancelled: {})", tasks.join(", "))
}
/// Joins an error's message with the messages of its whole `source()` chain,
/// separated by `": "`, so the underlying cause is not lost behind a generic
/// top-level message (e.g. tonic's "transport error").
fn source_chain(mut error: &dyn std::error::Error) -> String {
let mut message = error.to_string();
while let Some(source) = error.source() {
message.push_str(": ");
message.push_str(&source.to_string());
error = source;
}
message
}
impl From<tonic::Status> for Error {
fn from(status: tonic::Status) -> Self {
Self::Grpc(Box::new(status))
}
}
/// `Result` with the SDK's [`Error`].
pub type Result<T> = std::result::Result<T, Error>;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn transport_error_display_includes_the_cause_chain() {
let source = tonic::transport::Endpoint::from_shared("not a uri").unwrap_err();
let error = Error::Transport {
address: "example.com:7077".to_string(),
source,
};
let message = error.to_string();
assert!(
message.starts_with("could not set up the connection to example.com:7077: "),
"{message}"
);
assert!(message != "could not set up the connection to example.com:7077: transport error");
assert!(
message.len()
> "could not set up the connection to example.com:7077: transport error".len(),
"{message}: expected more than just the generic transport error message"
);
}
/// serde's own message ("key must be a string") says nothing about where
/// it came from, so the variant says it.
#[test]
fn a_json_error_names_what_failed_before_the_serde_message() {
let error = Error::Json("key must be a string".to_string());
assert_eq!(
error.to_string(),
"could not serialize to JSON: key must be a string"
);
}
#[test]
fn a_failed_run_names_every_task_that_failed_and_why() {
let error = Error::RunFailed {
run_id: "run-1".to_string(),
errors: vec![
TaskFailure {
task_name: "fetch".to_string(),
message: "boom".to_string(),
},
TaskFailure {
task_name: "report".to_string(),
message: "no input".to_string(),
},
],
};
assert_eq!(
error.to_string(),
"workflow run run-1 failed: fetch: boom; report: no input"
);
}
#[test]
fn a_cancelled_run_names_the_tasks_the_engine_recorded() {
let error = Error::RunCancelled {
run_id: "run-1".to_string(),
tasks: vec!["fetch".to_string(), "report".to_string()],
};
assert_eq!(
error.to_string(),
"workflow run run-1 was cancelled (cancelled: fetch, report)"
);
}
/// The engine can cancel a run before it has any task to name, and the
/// message must still read as a sentence.
#[test]
fn a_cancelled_run_with_no_named_tasks_still_reads() {
let error = Error::RunCancelled {
run_id: "run-1".to_string(),
tasks: Vec::new(),
};
assert_eq!(error.to_string(), "workflow run run-1 was cancelled");
}
#[test]
fn source_chain_joins_nested_causes() {
#[derive(Debug)]
struct Inner;
impl std::fmt::Display for Inner {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "inner cause")
}
}
impl std::error::Error for Inner {}
#[derive(Debug)]
struct Outer(Inner);
impl std::fmt::Display for Outer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "outer cause")
}
}
impl std::error::Error for Outer {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&self.0)
}
}
let outer = Outer(Inner);
assert_eq!(source_chain(&outer), "outer cause: inner cause");
}
}