Skip to main content

hatchet_client/runs/
client.rs

1//! Controlling runs that already exist: cancelling, replaying, and asking
2//! what happened.
3
4use hatchet_proto::root::{RestoreEvictedTaskRequest, SubscribeToWorkflowEventsRequest};
5use hatchet_proto::v1::{CancelTasksRequest, GetRunDetailsRequest, ReplayTasksRequest};
6use tonic::Code;
7
8use crate::error::{Error, Result};
9use crate::runs::details::RunDetails;
10use crate::runs::filter::TasksFilter;
11use crate::runs::stream::{self, OutputStream};
12use crate::{Client, retry};
13
14/// Cancels, replays and inspects runs the engine already knows about.
15///
16/// Get one from [`Client::runs`]. Everything here is `v1.AdminService` over
17/// the same gRPC connection the rest of the SDK uses — no REST client and no
18/// extra credentials. The official Go and Python SDKs cancel and replay over
19/// the REST API instead (`sdks/go/features/runs.go:130-147`), which is why
20/// their argument shapes differ from these.
21///
22/// ```no_run
23/// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
24/// # let task = hatchet_client::TaskBuilder::<(), ()>::new("t")
25/// #     .handler(|_ctx, _input| async { Ok(()) });
26/// let run = task
27///     .run_no_wait(&client, &(), hatchet_client::RunOpts::default())
28///     .await?;
29///
30/// // Somewhere else, later: stop it.
31/// let cancelled = client.runs().cancel([run.id()]).await?;
32/// println!("cancelled {} run(s)", cancelled.len());
33/// # Ok(())
34/// # }
35/// ```
36#[derive(Clone, Debug)]
37pub struct RunsClient {
38    client: Client,
39}
40
41impl RunsClient {
42    pub(crate) fn new(client: Client) -> Self {
43        Self { client }
44    }
45
46    /// Cancels the runs named by `ids`, and hands back the ids the engine
47    /// says it cancelled.
48    ///
49    /// `ids` are workflow run ids — the same ids a [`RunHandle::id`](crate::RunHandle::id)
50    /// or [`WorkflowRunHandle::id`](crate::WorkflowRunHandle::id) hands back,
51    /// or a `run_id` you got from [`RunsClient::details`].
52    ///
53    /// The answer is an echo of the request, not a report of effect: both
54    /// supported engine versions echoed back an id they did not know, an
55    /// already-cancelled run and an already-finished run alike, and refused
56    /// only a malformed id (`InvalidArgument`). To learn whether a run was
57    /// actually stopped, wait on its handle (it ends as
58    /// [`Error::RunCancelled`]) or ask [`RunsClient::details`].
59    ///
60    /// # Errors
61    ///
62    /// [`Error::InvalidArgument`] when `ids` is empty or holds a blank id —
63    /// checked before any call, because an empty request is not a no-op to
64    /// the engine. Otherwise [`Error::Grpc`].
65    pub async fn cancel(
66        &self,
67        ids: impl IntoIterator<Item = impl Into<String>>,
68    ) -> Result<Vec<String>> {
69        let external_ids = checked_ids(ids, "cancel")?;
70        let request = CancelTasksRequest {
71            external_ids,
72            filter: None,
73        };
74        let response = retry::unary(|| {
75            let mut admin = self.client.admin_grpc();
76            let request = retry::timed(request.clone());
77            async move { admin.cancel_tasks(request).await }
78        })
79        .await?;
80        Ok(response.into_inner().cancelled_tasks)
81    }
82
83    /// Replays the runs named by `ids` — the engine runs them again from the
84    /// start — and hands back the ids it says it replayed.
85    ///
86    /// A replayed run keeps the id it was given here; it is not handed a new
87    /// one.
88    ///
89    /// # Retries
90    ///
91    /// Like every unary call, a replay whose answer is lost to a transport
92    /// error is retried, and the engine may then have replayed the run
93    /// twice. Go's REST client refuses to retry anything but a GET for the
94    /// same reason; if a double replay would hurt, call
95    /// [`details`](RunsClient::details) before retrying yourself.
96    ///
97    /// # Errors
98    ///
99    /// Those of [`cancel`](RunsClient::cancel).
100    pub async fn replay(
101        &self,
102        ids: impl IntoIterator<Item = impl Into<String>>,
103    ) -> Result<Vec<String>> {
104        let external_ids = checked_ids(ids, "replay")?;
105        let request = ReplayTasksRequest {
106            external_ids,
107            filter: None,
108        };
109        let response = retry::unary(|| {
110            let mut admin = self.client.admin_grpc();
111            let request = retry::timed(request.clone());
112            async move { admin.replay_tasks(request).await }
113        })
114        .await?;
115        Ok(response.into_inner().replayed_tasks)
116    }
117
118    /// Cancels every run that matches `filter`, and hands back the ids the
119    /// engine says it cancelled.
120    ///
121    /// The engine decides what "matches" means and the answer is the only
122    /// record of what was hit, so **read it**: this is the one call in the
123    /// SDK whose blast radius is not bounded by its arguments.
124    ///
125    /// # Errors
126    ///
127    /// [`Error::InvalidArgument`] when the filter contradicts itself (an
128    /// `until` before its `since`) or holds a blank id or metadata key, and
129    /// [`Error::Grpc`] when the call fails.
130    pub async fn cancel_matching(&self, filter: TasksFilter) -> Result<Vec<String>> {
131        let filter = filter.to_proto()?;
132        let request = CancelTasksRequest {
133            external_ids: Vec::new(),
134            filter: Some(filter),
135        };
136        let response = retry::unary(|| {
137            let mut admin = self.client.admin_grpc();
138            let request = retry::timed(request.clone());
139            async move { admin.cancel_tasks(request).await }
140        })
141        .await?;
142        Ok(response.into_inner().cancelled_tasks)
143    }
144
145    /// Replays every run that matches `filter`, and hands back the ids the
146    /// engine says it replayed.
147    ///
148    /// # Errors
149    ///
150    /// Those of [`cancel_matching`](RunsClient::cancel_matching).
151    pub async fn replay_matching(&self, filter: TasksFilter) -> Result<Vec<String>> {
152        let filter = filter.to_proto()?;
153        let request = ReplayTasksRequest {
154            external_ids: Vec::new(),
155            filter: Some(filter),
156        };
157        let response = retry::unary(|| {
158            let mut admin = self.client.admin_grpc();
159            let request = retry::timed(request.clone());
160            async move { admin.replay_tasks(request).await }
161        })
162        .await?;
163        Ok(response.into_inner().replayed_tasks)
164    }
165
166    /// Asks the engine to queue an **evicted** task run again, now.
167    ///
168    /// A task run is evicted when the worker holding it hands it back while it
169    /// waits, because of the
170    /// [`EvictionPolicy`](crate::EvictionPolicy) its task declared. The engine
171    /// queues such a run again by itself as soon as what it was waiting for
172    /// happens — measured at 0.7 to 1.5 seconds after the event, on both
173    /// supported engines — so this is for the other case: an operator
174    /// unsticking a run whose wait may never end.
175    ///
176    /// **`task_run_id` is a task run id, not a workflow run id** — a
177    /// [`TaskRunDetail::id`](crate::TaskRunDetail::id), which
178    /// [`details`](RunsClient::details) hands back through
179    /// [`RunDetails::tasks`](crate::RunDetails::tasks). The same detail's
180    /// [`is_evicted`](crate::TaskRunDetail::is_evicted) is how to tell an
181    /// evicted task run from a running one: an evicted one's status stays
182    /// [`RunStatus::Running`](crate::RunStatus::Running).
183    ///
184    /// # The answer is not proof
185    ///
186    /// It is the engine's `requeued` flag, and both supported versions were
187    /// measured answering `true` **unconditionally** — including for a random
188    /// UUID naming no run at all. So a `true` says the call was accepted, not
189    /// that anything was queued; read
190    /// [`details`](RunsClient::details) to learn what actually happened. It is
191    /// returned rather than swallowed because the engine may one day mean
192    /// something by it.
193    ///
194    /// # Retries
195    ///
196    /// Like every unary call, one whose answer is lost to a transport error is
197    /// retried. Queueing a run that is already queued is what the engine was
198    /// measured doing anyway, so a repeat costs nothing.
199    ///
200    /// # Errors
201    ///
202    /// [`Error::InvalidArgument`] when `task_run_id` is blank — checked before
203    /// any call — and [`Error::Grpc`] when the call fails.
204    ///
205    /// ```no_run
206    /// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
207    /// # let run_id = "d627fab0-385f-4eb4-98f9-59a783bc8631";
208    /// let details = client.runs().details(run_id).await?;
209    /// for (_name, task) in details.tasks() {
210    ///     if task.is_evicted() {
211    ///         client.runs().restore_evicted(task.id()).await?;
212    ///     }
213    /// }
214    /// # Ok(())
215    /// # }
216    /// ```
217    pub async fn restore_evicted(&self, task_run_id: &str) -> Result<bool> {
218        if task_run_id.trim().is_empty() {
219            return Err(Error::InvalidArgument(
220                "a task run id must not be blank".to_string(),
221            ));
222        }
223        let request = RestoreEvictedTaskRequest {
224            task_run_external_id: task_run_id.to_string(),
225        };
226        let response = retry::unary(|| {
227            let mut dispatcher = self.client.dispatcher_grpc();
228            let request = retry::timed(request.clone());
229            async move { dispatcher.restore_evicted_task(request).await }
230        })
231        .await?;
232        Ok(response.into_inner().requeued)
233    }
234
235    /// What the engine remembers about the run called `run_id`.
236    ///
237    /// Works for a run this process never triggered and for one that is
238    /// still going — [`RunDetails::is_done`] says which.
239    ///
240    /// # Errors
241    ///
242    /// [`Error::InvalidArgument`] when `run_id` is blank;
243    /// [`Error::UnsupportedFeature`] when the engine does not have the
244    /// `GetRunDetails` RPC (answered `Unimplemented`);
245    /// [`Error::UnknownRunStatus`] when it reports a status this SDK has no
246    /// name for; and [`Error::Grpc`] otherwise.
247    pub async fn details(&self, run_id: &str) -> Result<RunDetails> {
248        if run_id.trim().is_empty() {
249            return Err(Error::InvalidArgument(
250                "a run id must not be blank".to_string(),
251            ));
252        }
253        let request = GetRunDetailsRequest {
254            external_id: run_id.to_string(),
255        };
256        let response = retry::unary(|| {
257            let mut admin = self.client.admin_grpc();
258            let request = retry::timed(request.clone());
259            async move { admin.get_run_details(request).await }
260        })
261        .await;
262        let response = match response {
263            Ok(response) => response,
264            // Translated here rather than gated by a version call before
265            // every lookup: one round trip instead of two on the happy
266            // path. `GetRunDetails` exists on both supported engine
267            // versions; this only matters against an engine that removed
268            // it, or one older than either we support.
269            Err(status) if status.code() == Code::Unimplemented => {
270                return Err(self.no_run_details().await);
271            }
272            Err(status) => return Err(status.into()),
273        };
274        RunDetails::from_proto(run_id, response.into_inner())
275    }
276
277    /// The output stream of the run called `run_id`: the chunks its handlers
278    /// publish with [`Context::put_stream`](crate::Context::put_stream), in
279    /// order.
280    ///
281    /// **Chunks are not replayed**, so a subscription only ever sees what is
282    /// published after the engine registers it. Opening one needs a run id,
283    /// and a run id only exists once the run has been triggered — so this call
284    /// cannot be made before the run starts, and a handler that publishes
285    /// immediately may be a chunk or two ahead by the time the subscription is
286    /// in place. That is not theoretical: against a v0.80.0 engine, a
287    /// subscription opened four seconds after the trigger began at chunk 1 and
288    /// chunk 0 was never delivered. When that matters, subscribe by metadata
289    /// *before* triggering — see
290    /// [`stream_by_metadata`](RunsClient::stream_by_metadata), the only
291    /// race-free way to read a run's output from the start.
292    ///
293    /// **This call waits up to five seconds for the engine to answer**, and
294    /// then hands back a stream that may still be being opened. That is the
295    /// ordinary case rather than a failure: neither supported engine sends
296    /// anything at all on a fresh subscription — not even response headers —
297    /// until it has an event for it, which has been measured at nineteen
298    /// seconds and more. The subscription is registered engine-side long
299    /// before that, and it is that same one the stream goes on with, so
300    /// nothing published in the meantime is lost.
301    ///
302    /// The stream ends when the engine hangs up, when it fails in a way
303    /// reconnecting cannot fix (the failure is the last item), or when it is
304    /// dropped. Anything else — a dropped connection, a restarting engine — is
305    /// resubscribed under the SDK's stream backoff, and chunks already
306    /// delivered are not delivered again. A hang-up means the engine closed
307    /// this subscription, not that every chunk was delivered: a multi-task run
308    /// on v0.80.0 hung up before its chunks arrived, and subscribing again
309    /// then delivered them.
310    ///
311    /// ```no_run
312    /// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
313    /// # let run_id = "d627fab0-385f-4eb4-98f9-59a783bc8631";
314    /// let mut chunks = client.runs().stream(run_id).await?;
315    /// while let Some(chunk) = chunks.next().await {
316    ///     print!("{}", chunk?.payload);
317    /// }
318    /// # Ok(())
319    /// # }
320    /// ```
321    ///
322    /// # Errors
323    ///
324    /// [`Error::InvalidArgument`] when `run_id` is blank — checked before any
325    /// call — and [`Error::Grpc`] when the engine refuses the subscription in
326    /// a way reconnecting cannot fix: a rejected token, or a run id it does
327    /// not know (`NotFound`). A failure that is worth retrying is not reported
328    /// here: the stream is handed back and retries in the background, and
329    /// gives up as [`Error::ResultStream`] on the stream itself.
330    pub async fn stream(&self, run_id: &str) -> Result<OutputStream> {
331        if run_id.trim().is_empty() {
332            return Err(Error::InvalidArgument(
333                "a run id must not be blank".to_string(),
334            ));
335        }
336        stream::open(
337            &self.client,
338            SubscribeToWorkflowEventsRequest {
339                workflow_run_id: Some(run_id.to_string()),
340                additional_meta_key: None,
341                additional_meta_value: None,
342            },
343        )
344        .await
345    }
346
347    /// The output streams of every run whose additional metadata has `key` set
348    /// to `value`, on one subscription.
349    ///
350    /// This is the race-free way to read a run's output, because it does not
351    /// need a run id: open the subscription, *then* trigger, and nothing can
352    /// be published before the engine knows you are listening.
353    /// [`StreamChunk::run_id`](crate::StreamChunk::run_id) is what tells the
354    /// runs apart.
355    ///
356    /// ```no_run
357    /// # use hatchet_client::{Client, RunOpts, TaskBuilder};
358    /// # async fn example(client: Client) -> hatchet_client::Result<()> {
359    /// # let summarise = TaskBuilder::<String, ()>::new("summarise")
360    /// #     .handler(|_ctx, _input| async { Ok(()) });
361    /// // A tag only this call will use, so the subscription matches this run
362    /// // and no other.
363    /// let tag = uuid_of_your_choosing();
364    ///
365    /// // Listening first is the whole point: chunks are not replayed.
366    /// let mut chunks = client.runs().stream_by_metadata("job", &tag).await?;
367    /// let run = summarise
368    ///     .run_no_wait(
369    ///         &client,
370    ///         &"the document".to_string(),
371    ///         RunOpts::default().metadata("job", tag.as_str()),
372    ///     )
373    ///     .await?;
374    ///
375    /// while let Some(chunk) = chunks.next().await {
376    ///     let chunk = chunk?;
377    ///     if chunk.run_id == run.id() {
378    ///         print!("{}", chunk.payload);
379    ///     }
380    /// }
381    /// # Ok(())
382    /// # }
383    /// # fn uuid_of_your_choosing() -> String { "3f2a…".to_string() }
384    /// ```
385    ///
386    /// `value` is matched as text against the run's metadata value, so a
387    /// metadata value that is not a string (a number, an object) has no text
388    /// form to give here.
389    ///
390    /// **This call waits up to five seconds for the engine to answer**, as
391    /// [`stream`](RunsClient::stream) does — and here it nearly always uses
392    /// the whole five, because a subscription opened before anything is
393    /// triggered has nothing to match yet and the engine answers it only when
394    /// it does. The wait costs latency, not chunks: the subscription is
395    /// registered before the answer comes, and the stream carries on with that
396    /// same one.
397    ///
398    /// Nothing ends this stream on its own *by design* — it covers runs that
399    /// have not been triggered yet, so there is no point at which the engine
400    /// can say there will be no more — but the engine does hang up on it once
401    /// the runs it matched are over, and a hang-up ends the stream. Open a new
402    /// one to keep watching, drop it when you are done, and keep a time limit
403    /// of your own around a wait for a chunk that may never come.
404    ///
405    /// # Errors
406    ///
407    /// [`Error::InvalidArgument`] when `key` or `value` is blank, and
408    /// [`Error::Grpc`] as for [`stream`](RunsClient::stream).
409    pub async fn stream_by_metadata(&self, key: &str, value: &str) -> Result<OutputStream> {
410        if key.trim().is_empty() || value.trim().is_empty() {
411            return Err(Error::InvalidArgument(format!(
412                "a metadata subscription needs both halves of the pair; got key {key:?} and \
413                 value {value:?}"
414            )));
415        }
416        stream::open(
417            &self.client,
418            SubscribeToWorkflowEventsRequest {
419                workflow_run_id: None,
420                additional_meta_key: Some(key.to_string()),
421                additional_meta_value: Some(value.to_string()),
422            },
423        )
424        .await
425    }
426
427    /// The error for an engine without `GetRunDetails`: the version it
428    /// reports, best-effort, and the version that has the RPC.
429    async fn no_run_details(&self) -> Error {
430        let found = self
431            .client
432            .engine_version()
433            .await
434            .unwrap_or_else(|_| "unknown".to_string());
435        Error::UnsupportedFeature {
436            feature: "run details (GetRunDetails)".to_string(),
437            found,
438            minimum: crate::version::MIN_ENGINE_VERSION.to_string(),
439        }
440    }
441}
442
443/// The ids of a `cancel`/`replay` call, refused before the wire when they
444/// could not mean anything.
445///
446/// An empty list is refused because `external_ids: []` with no filter is not
447/// a no-op on the wire — the engine decides what it means, and the official
448/// Python SDK refuses the same shape (`features/runs.py:67-73`).
449fn checked_ids(
450    ids: impl IntoIterator<Item = impl Into<String>>,
451    what: &str,
452) -> Result<Vec<String>> {
453    let ids: Vec<String> = ids.into_iter().map(Into::into).collect();
454    if ids.is_empty() {
455        return Err(Error::InvalidArgument(format!(
456            "{what} was given no run ids; naming none would leave it to the engine to decide \
457             what that means"
458        )));
459    }
460    if let Some(position) = ids.iter().position(|id| id.trim().is_empty()) {
461        return Err(Error::InvalidArgument(format!(
462            "{what} was given a blank run id at position {position}"
463        )));
464    }
465    Ok(ids)
466}