hatchet-client 0.1.0

Rust SDK for Hatchet: run workers and trigger tasks
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
//! Controlling runs that already exist: cancelling, replaying, and asking
//! what happened.

use hatchet_proto::root::{RestoreEvictedTaskRequest, SubscribeToWorkflowEventsRequest};
use hatchet_proto::v1::{CancelTasksRequest, GetRunDetailsRequest, ReplayTasksRequest};
use tonic::Code;

use crate::error::{Error, Result};
use crate::runs::details::RunDetails;
use crate::runs::filter::TasksFilter;
use crate::runs::stream::{self, OutputStream};
use crate::{Client, retry};

/// Cancels, replays and inspects runs the engine already knows about.
///
/// Get one from [`Client::runs`]. Everything here is `v1.AdminService` over
/// the same gRPC connection the rest of the SDK uses — no REST client and no
/// extra credentials. The official Go and Python SDKs cancel and replay over
/// the REST API instead (`sdks/go/features/runs.go:130-147`), which is why
/// their argument shapes differ from these.
///
/// ```no_run
/// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
/// # let task = hatchet_client::TaskBuilder::<(), ()>::new("t")
/// #     .handler(|_ctx, _input| async { Ok(()) });
/// let run = task
///     .run_no_wait(&client, &(), hatchet_client::RunOpts::default())
///     .await?;
///
/// // Somewhere else, later: stop it.
/// let cancelled = client.runs().cancel([run.id()]).await?;
/// println!("cancelled {} run(s)", cancelled.len());
/// # Ok(())
/// # }
/// ```
#[derive(Clone, Debug)]
pub struct RunsClient {
    client: Client,
}

impl RunsClient {
    pub(crate) fn new(client: Client) -> Self {
        Self { client }
    }

    /// Cancels the runs named by `ids`, and hands back the ids the engine
    /// says it cancelled.
    ///
    /// `ids` are workflow run ids — the same ids a [`RunHandle::id`](crate::RunHandle::id)
    /// or [`WorkflowRunHandle::id`](crate::WorkflowRunHandle::id) hands back,
    /// or a `run_id` you got from [`RunsClient::details`].
    ///
    /// The answer is an echo of the request, not a report of effect: both
    /// supported engine versions echoed back an id they did not know, an
    /// already-cancelled run and an already-finished run alike, and refused
    /// only a malformed id (`InvalidArgument`). To learn whether a run was
    /// actually stopped, wait on its handle (it ends as
    /// [`Error::RunCancelled`]) or ask [`RunsClient::details`].
    ///
    /// # Errors
    ///
    /// [`Error::InvalidArgument`] when `ids` is empty or holds a blank id —
    /// checked before any call, because an empty request is not a no-op to
    /// the engine. Otherwise [`Error::Grpc`].
    pub async fn cancel(
        &self,
        ids: impl IntoIterator<Item = impl Into<String>>,
    ) -> Result<Vec<String>> {
        let external_ids = checked_ids(ids, "cancel")?;
        let request = CancelTasksRequest {
            external_ids,
            filter: None,
        };
        let response = retry::unary(|| {
            let mut admin = self.client.admin_grpc();
            let request = retry::timed(request.clone());
            async move { admin.cancel_tasks(request).await }
        })
        .await?;
        Ok(response.into_inner().cancelled_tasks)
    }

    /// Replays the runs named by `ids` — the engine runs them again from the
    /// start — and hands back the ids it says it replayed.
    ///
    /// A replayed run keeps the id it was given here; it is not handed a new
    /// one.
    ///
    /// # Retries
    ///
    /// Like every unary call, a replay whose answer is lost to a transport
    /// error is retried, and the engine may then have replayed the run
    /// twice. Go's REST client refuses to retry anything but a GET for the
    /// same reason; if a double replay would hurt, call
    /// [`details`](RunsClient::details) before retrying yourself.
    ///
    /// # Errors
    ///
    /// Those of [`cancel`](RunsClient::cancel).
    pub async fn replay(
        &self,
        ids: impl IntoIterator<Item = impl Into<String>>,
    ) -> Result<Vec<String>> {
        let external_ids = checked_ids(ids, "replay")?;
        let request = ReplayTasksRequest {
            external_ids,
            filter: None,
        };
        let response = retry::unary(|| {
            let mut admin = self.client.admin_grpc();
            let request = retry::timed(request.clone());
            async move { admin.replay_tasks(request).await }
        })
        .await?;
        Ok(response.into_inner().replayed_tasks)
    }

    /// Cancels every run that matches `filter`, and hands back the ids the
    /// engine says it cancelled.
    ///
    /// The engine decides what "matches" means and the answer is the only
    /// record of what was hit, so **read it**: this is the one call in the
    /// SDK whose blast radius is not bounded by its arguments.
    ///
    /// # Errors
    ///
    /// [`Error::InvalidArgument`] when the filter contradicts itself (an
    /// `until` before its `since`) or holds a blank id or metadata key, and
    /// [`Error::Grpc`] when the call fails.
    pub async fn cancel_matching(&self, filter: TasksFilter) -> Result<Vec<String>> {
        let filter = filter.to_proto()?;
        let request = CancelTasksRequest {
            external_ids: Vec::new(),
            filter: Some(filter),
        };
        let response = retry::unary(|| {
            let mut admin = self.client.admin_grpc();
            let request = retry::timed(request.clone());
            async move { admin.cancel_tasks(request).await }
        })
        .await?;
        Ok(response.into_inner().cancelled_tasks)
    }

    /// Replays every run that matches `filter`, and hands back the ids the
    /// engine says it replayed.
    ///
    /// # Errors
    ///
    /// Those of [`cancel_matching`](RunsClient::cancel_matching).
    pub async fn replay_matching(&self, filter: TasksFilter) -> Result<Vec<String>> {
        let filter = filter.to_proto()?;
        let request = ReplayTasksRequest {
            external_ids: Vec::new(),
            filter: Some(filter),
        };
        let response = retry::unary(|| {
            let mut admin = self.client.admin_grpc();
            let request = retry::timed(request.clone());
            async move { admin.replay_tasks(request).await }
        })
        .await?;
        Ok(response.into_inner().replayed_tasks)
    }

    /// Asks the engine to queue an **evicted** task run again, now.
    ///
    /// A task run is evicted when the worker holding it hands it back while it
    /// waits, because of the
    /// [`EvictionPolicy`](crate::EvictionPolicy) its task declared. The engine
    /// queues such a run again by itself as soon as what it was waiting for
    /// happens — measured at 0.7 to 1.5 seconds after the event, on both
    /// supported engines — so this is for the other case: an operator
    /// unsticking a run whose wait may never end.
    ///
    /// **`task_run_id` is a task run id, not a workflow run id** — a
    /// [`TaskRunDetail::id`](crate::TaskRunDetail::id), which
    /// [`details`](RunsClient::details) hands back through
    /// [`RunDetails::tasks`](crate::RunDetails::tasks). The same detail's
    /// [`is_evicted`](crate::TaskRunDetail::is_evicted) is how to tell an
    /// evicted task run from a running one: an evicted one's status stays
    /// [`RunStatus::Running`](crate::RunStatus::Running).
    ///
    /// # The answer is not proof
    ///
    /// It is the engine's `requeued` flag, and both supported versions were
    /// measured answering `true` **unconditionally** — including for a random
    /// UUID naming no run at all. So a `true` says the call was accepted, not
    /// that anything was queued; read
    /// [`details`](RunsClient::details) to learn what actually happened. It is
    /// returned rather than swallowed because the engine may one day mean
    /// something by it.
    ///
    /// # Retries
    ///
    /// Like every unary call, one whose answer is lost to a transport error is
    /// retried. Queueing a run that is already queued is what the engine was
    /// measured doing anyway, so a repeat costs nothing.
    ///
    /// # Errors
    ///
    /// [`Error::InvalidArgument`] when `task_run_id` is blank — checked before
    /// any call — and [`Error::Grpc`] when the call fails.
    ///
    /// ```no_run
    /// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
    /// # let run_id = "d627fab0-385f-4eb4-98f9-59a783bc8631";
    /// let details = client.runs().details(run_id).await?;
    /// for (_name, task) in details.tasks() {
    ///     if task.is_evicted() {
    ///         client.runs().restore_evicted(task.id()).await?;
    ///     }
    /// }
    /// # Ok(())
    /// # }
    /// ```
    pub async fn restore_evicted(&self, task_run_id: &str) -> Result<bool> {
        if task_run_id.trim().is_empty() {
            return Err(Error::InvalidArgument(
                "a task run id must not be blank".to_string(),
            ));
        }
        let request = RestoreEvictedTaskRequest {
            task_run_external_id: task_run_id.to_string(),
        };
        let response = retry::unary(|| {
            let mut dispatcher = self.client.dispatcher_grpc();
            let request = retry::timed(request.clone());
            async move { dispatcher.restore_evicted_task(request).await }
        })
        .await?;
        Ok(response.into_inner().requeued)
    }

    /// What the engine remembers about the run called `run_id`.
    ///
    /// Works for a run this process never triggered and for one that is
    /// still going — [`RunDetails::is_done`] says which.
    ///
    /// # Errors
    ///
    /// [`Error::InvalidArgument`] when `run_id` is blank;
    /// [`Error::UnsupportedFeature`] when the engine does not have the
    /// `GetRunDetails` RPC (answered `Unimplemented`);
    /// [`Error::UnknownRunStatus`] when it reports a status this SDK has no
    /// name for; and [`Error::Grpc`] otherwise.
    pub async fn details(&self, run_id: &str) -> Result<RunDetails> {
        if run_id.trim().is_empty() {
            return Err(Error::InvalidArgument(
                "a run id must not be blank".to_string(),
            ));
        }
        let request = GetRunDetailsRequest {
            external_id: run_id.to_string(),
        };
        let response = retry::unary(|| {
            let mut admin = self.client.admin_grpc();
            let request = retry::timed(request.clone());
            async move { admin.get_run_details(request).await }
        })
        .await;
        let response = match response {
            Ok(response) => response,
            // Translated here rather than gated by a version call before
            // every lookup: one round trip instead of two on the happy
            // path. `GetRunDetails` exists on both supported engine
            // versions; this only matters against an engine that removed
            // it, or one older than either we support.
            Err(status) if status.code() == Code::Unimplemented => {
                return Err(self.no_run_details().await);
            }
            Err(status) => return Err(status.into()),
        };
        RunDetails::from_proto(run_id, response.into_inner())
    }

    /// The output stream of the run called `run_id`: the chunks its handlers
    /// publish with [`Context::put_stream`](crate::Context::put_stream), in
    /// order.
    ///
    /// **Chunks are not replayed**, so a subscription only ever sees what is
    /// published after the engine registers it. Opening one needs a run id,
    /// and a run id only exists once the run has been triggered — so this call
    /// cannot be made before the run starts, and a handler that publishes
    /// immediately may be a chunk or two ahead by the time the subscription is
    /// in place. That is not theoretical: against a v0.80.0 engine, a
    /// subscription opened four seconds after the trigger began at chunk 1 and
    /// chunk 0 was never delivered. When that matters, subscribe by metadata
    /// *before* triggering — see
    /// [`stream_by_metadata`](RunsClient::stream_by_metadata), the only
    /// race-free way to read a run's output from the start.
    ///
    /// **This call waits up to five seconds for the engine to answer**, and
    /// then hands back a stream that may still be being opened. That is the
    /// ordinary case rather than a failure: neither supported engine sends
    /// anything at all on a fresh subscription — not even response headers —
    /// until it has an event for it, which has been measured at nineteen
    /// seconds and more. The subscription is registered engine-side long
    /// before that, and it is that same one the stream goes on with, so
    /// nothing published in the meantime is lost.
    ///
    /// The stream ends when the engine hangs up, when it fails in a way
    /// reconnecting cannot fix (the failure is the last item), or when it is
    /// dropped. Anything else — a dropped connection, a restarting engine — is
    /// resubscribed under the SDK's stream backoff, and chunks already
    /// delivered are not delivered again. A hang-up means the engine closed
    /// this subscription, not that every chunk was delivered: a multi-task run
    /// on v0.80.0 hung up before its chunks arrived, and subscribing again
    /// then delivered them.
    ///
    /// ```no_run
    /// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
    /// # let run_id = "d627fab0-385f-4eb4-98f9-59a783bc8631";
    /// let mut chunks = client.runs().stream(run_id).await?;
    /// while let Some(chunk) = chunks.next().await {
    ///     print!("{}", chunk?.payload);
    /// }
    /// # Ok(())
    /// # }
    /// ```
    ///
    /// # Errors
    ///
    /// [`Error::InvalidArgument`] when `run_id` is blank — checked before any
    /// call — and [`Error::Grpc`] when the engine refuses the subscription in
    /// a way reconnecting cannot fix: a rejected token, or a run id it does
    /// not know (`NotFound`). A failure that is worth retrying is not reported
    /// here: the stream is handed back and retries in the background, and
    /// gives up as [`Error::ResultStream`] on the stream itself.
    pub async fn stream(&self, run_id: &str) -> Result<OutputStream> {
        if run_id.trim().is_empty() {
            return Err(Error::InvalidArgument(
                "a run id must not be blank".to_string(),
            ));
        }
        stream::open(
            &self.client,
            SubscribeToWorkflowEventsRequest {
                workflow_run_id: Some(run_id.to_string()),
                additional_meta_key: None,
                additional_meta_value: None,
            },
        )
        .await
    }

    /// The output streams of every run whose additional metadata has `key` set
    /// to `value`, on one subscription.
    ///
    /// This is the race-free way to read a run's output, because it does not
    /// need a run id: open the subscription, *then* trigger, and nothing can
    /// be published before the engine knows you are listening.
    /// [`StreamChunk::run_id`](crate::StreamChunk::run_id) is what tells the
    /// runs apart.
    ///
    /// ```no_run
    /// # use hatchet_client::{Client, RunOpts, TaskBuilder};
    /// # async fn example(client: Client) -> hatchet_client::Result<()> {
    /// # let summarise = TaskBuilder::<String, ()>::new("summarise")
    /// #     .handler(|_ctx, _input| async { Ok(()) });
    /// // A tag only this call will use, so the subscription matches this run
    /// // and no other.
    /// let tag = uuid_of_your_choosing();
    ///
    /// // Listening first is the whole point: chunks are not replayed.
    /// let mut chunks = client.runs().stream_by_metadata("job", &tag).await?;
    /// let run = summarise
    ///     .run_no_wait(
    ///         &client,
    ///         &"the document".to_string(),
    ///         RunOpts::default().metadata("job", tag.as_str()),
    ///     )
    ///     .await?;
    ///
    /// while let Some(chunk) = chunks.next().await {
    ///     let chunk = chunk?;
    ///     if chunk.run_id == run.id() {
    ///         print!("{}", chunk.payload);
    ///     }
    /// }
    /// # Ok(())
    /// # }
    /// # fn uuid_of_your_choosing() -> String { "3f2a…".to_string() }
    /// ```
    ///
    /// `value` is matched as text against the run's metadata value, so a
    /// metadata value that is not a string (a number, an object) has no text
    /// form to give here.
    ///
    /// **This call waits up to five seconds for the engine to answer**, as
    /// [`stream`](RunsClient::stream) does — and here it nearly always uses
    /// the whole five, because a subscription opened before anything is
    /// triggered has nothing to match yet and the engine answers it only when
    /// it does. The wait costs latency, not chunks: the subscription is
    /// registered before the answer comes, and the stream carries on with that
    /// same one.
    ///
    /// Nothing ends this stream on its own *by design* — it covers runs that
    /// have not been triggered yet, so there is no point at which the engine
    /// can say there will be no more — but the engine does hang up on it once
    /// the runs it matched are over, and a hang-up ends the stream. Open a new
    /// one to keep watching, drop it when you are done, and keep a time limit
    /// of your own around a wait for a chunk that may never come.
    ///
    /// # Errors
    ///
    /// [`Error::InvalidArgument`] when `key` or `value` is blank, and
    /// [`Error::Grpc`] as for [`stream`](RunsClient::stream).
    pub async fn stream_by_metadata(&self, key: &str, value: &str) -> Result<OutputStream> {
        if key.trim().is_empty() || value.trim().is_empty() {
            return Err(Error::InvalidArgument(format!(
                "a metadata subscription needs both halves of the pair; got key {key:?} and \
                 value {value:?}"
            )));
        }
        stream::open(
            &self.client,
            SubscribeToWorkflowEventsRequest {
                workflow_run_id: None,
                additional_meta_key: Some(key.to_string()),
                additional_meta_value: Some(value.to_string()),
            },
        )
        .await
    }

    /// The error for an engine without `GetRunDetails`: the version it
    /// reports, best-effort, and the version that has the RPC.
    async fn no_run_details(&self) -> Error {
        let found = self
            .client
            .engine_version()
            .await
            .unwrap_or_else(|_| "unknown".to_string());
        Error::UnsupportedFeature {
            feature: "run details (GetRunDetails)".to_string(),
            found,
            minimum: crate::version::MIN_ENGINE_VERSION.to_string(),
        }
    }
}

/// The ids of a `cancel`/`replay` call, refused before the wire when they
/// could not mean anything.
///
/// An empty list is refused because `external_ids: []` with no filter is not
/// a no-op on the wire — the engine decides what it means, and the official
/// Python SDK refuses the same shape (`features/runs.py:67-73`).
fn checked_ids(
    ids: impl IntoIterator<Item = impl Into<String>>,
    what: &str,
) -> Result<Vec<String>> {
    let ids: Vec<String> = ids.into_iter().map(Into::into).collect();
    if ids.is_empty() {
        return Err(Error::InvalidArgument(format!(
            "{what} was given no run ids; naming none would leave it to the engine to decide \
             what that means"
        )));
    }
    if let Some(position) = ids.iter().position(|id| id.trim().is_empty()) {
        return Err(Error::InvalidArgument(format!(
            "{what} was given a blank run id at position {position}"
        )));
    }
    Ok(ids)
}