hatchet_client/runs/client.rs
1//! Controlling runs that already exist: cancelling, replaying, and asking
2//! what happened.
3
4use hatchet_proto::root::{RestoreEvictedTaskRequest, SubscribeToWorkflowEventsRequest};
5use hatchet_proto::v1::{CancelTasksRequest, GetRunDetailsRequest, ReplayTasksRequest};
6use tonic::Code;
7
8use crate::error::{Error, Result};
9use crate::runs::details::RunDetails;
10use crate::runs::filter::TasksFilter;
11use crate::runs::stream::{self, OutputStream};
12use crate::{Client, retry};
13
14/// Cancels, replays and inspects runs the engine already knows about.
15///
16/// Get one from [`Client::runs`]. Everything here is `v1.AdminService` over
17/// the same gRPC connection the rest of the SDK uses — no REST client and no
18/// extra credentials. The official Go and Python SDKs cancel and replay over
19/// the REST API instead (`sdks/go/features/runs.go:130-147`), which is why
20/// their argument shapes differ from these.
21///
22/// ```no_run
23/// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
24/// # let task = hatchet_client::TaskBuilder::<(), ()>::new("t")
25/// # .handler(|_ctx, _input| async { Ok(()) });
26/// let run = task
27/// .run_no_wait(&client, &(), hatchet_client::RunOpts::default())
28/// .await?;
29///
30/// // Somewhere else, later: stop it.
31/// let cancelled = client.runs().cancel([run.id()]).await?;
32/// println!("cancelled {} run(s)", cancelled.len());
33/// # Ok(())
34/// # }
35/// ```
36#[derive(Clone, Debug)]
37pub struct RunsClient {
38 client: Client,
39}
40
41impl RunsClient {
42 pub(crate) fn new(client: Client) -> Self {
43 Self { client }
44 }
45
46 /// Cancels the runs named by `ids`, and hands back the ids the engine
47 /// says it cancelled.
48 ///
49 /// `ids` are workflow run ids — the same ids a [`RunHandle::id`](crate::RunHandle::id)
50 /// or [`WorkflowRunHandle::id`](crate::WorkflowRunHandle::id) hands back,
51 /// or a `run_id` you got from [`RunsClient::details`].
52 ///
53 /// The answer is an echo of the request, not a report of effect: both
54 /// supported engine versions echoed back an id they did not know, an
55 /// already-cancelled run and an already-finished run alike, and refused
56 /// only a malformed id (`InvalidArgument`). To learn whether a run was
57 /// actually stopped, wait on its handle (it ends as
58 /// [`Error::RunCancelled`]) or ask [`RunsClient::details`].
59 ///
60 /// # Errors
61 ///
62 /// [`Error::InvalidArgument`] when `ids` is empty or holds a blank id —
63 /// checked before any call, because an empty request is not a no-op to
64 /// the engine. Otherwise [`Error::Grpc`].
65 pub async fn cancel(
66 &self,
67 ids: impl IntoIterator<Item = impl Into<String>>,
68 ) -> Result<Vec<String>> {
69 let external_ids = checked_ids(ids, "cancel")?;
70 let request = CancelTasksRequest {
71 external_ids,
72 filter: None,
73 };
74 let response = retry::unary(|| {
75 let mut admin = self.client.admin_grpc();
76 let request = retry::timed(request.clone());
77 async move { admin.cancel_tasks(request).await }
78 })
79 .await?;
80 Ok(response.into_inner().cancelled_tasks)
81 }
82
83 /// Replays the runs named by `ids` — the engine runs them again from the
84 /// start — and hands back the ids it says it replayed.
85 ///
86 /// A replayed run keeps the id it was given here; it is not handed a new
87 /// one.
88 ///
89 /// # Retries
90 ///
91 /// Like every unary call, a replay whose answer is lost to a transport
92 /// error is retried, and the engine may then have replayed the run
93 /// twice. Go's REST client refuses to retry anything but a GET for the
94 /// same reason; if a double replay would hurt, call
95 /// [`details`](RunsClient::details) before retrying yourself.
96 ///
97 /// # Errors
98 ///
99 /// Those of [`cancel`](RunsClient::cancel).
100 pub async fn replay(
101 &self,
102 ids: impl IntoIterator<Item = impl Into<String>>,
103 ) -> Result<Vec<String>> {
104 let external_ids = checked_ids(ids, "replay")?;
105 let request = ReplayTasksRequest {
106 external_ids,
107 filter: None,
108 };
109 let response = retry::unary(|| {
110 let mut admin = self.client.admin_grpc();
111 let request = retry::timed(request.clone());
112 async move { admin.replay_tasks(request).await }
113 })
114 .await?;
115 Ok(response.into_inner().replayed_tasks)
116 }
117
118 /// Cancels every run that matches `filter`, and hands back the ids the
119 /// engine says it cancelled.
120 ///
121 /// The engine decides what "matches" means and the answer is the only
122 /// record of what was hit, so **read it**: this is the one call in the
123 /// SDK whose blast radius is not bounded by its arguments.
124 ///
125 /// # Errors
126 ///
127 /// [`Error::InvalidArgument`] when the filter contradicts itself (an
128 /// `until` before its `since`) or holds a blank id or metadata key, and
129 /// [`Error::Grpc`] when the call fails.
130 pub async fn cancel_matching(&self, filter: TasksFilter) -> Result<Vec<String>> {
131 let filter = filter.to_proto()?;
132 let request = CancelTasksRequest {
133 external_ids: Vec::new(),
134 filter: Some(filter),
135 };
136 let response = retry::unary(|| {
137 let mut admin = self.client.admin_grpc();
138 let request = retry::timed(request.clone());
139 async move { admin.cancel_tasks(request).await }
140 })
141 .await?;
142 Ok(response.into_inner().cancelled_tasks)
143 }
144
145 /// Replays every run that matches `filter`, and hands back the ids the
146 /// engine says it replayed.
147 ///
148 /// # Errors
149 ///
150 /// Those of [`cancel_matching`](RunsClient::cancel_matching).
151 pub async fn replay_matching(&self, filter: TasksFilter) -> Result<Vec<String>> {
152 let filter = filter.to_proto()?;
153 let request = ReplayTasksRequest {
154 external_ids: Vec::new(),
155 filter: Some(filter),
156 };
157 let response = retry::unary(|| {
158 let mut admin = self.client.admin_grpc();
159 let request = retry::timed(request.clone());
160 async move { admin.replay_tasks(request).await }
161 })
162 .await?;
163 Ok(response.into_inner().replayed_tasks)
164 }
165
166 /// Asks the engine to queue an **evicted** task run again, now.
167 ///
168 /// A task run is evicted when the worker holding it hands it back while it
169 /// waits, because of the
170 /// [`EvictionPolicy`](crate::EvictionPolicy) its task declared. The engine
171 /// queues such a run again by itself as soon as what it was waiting for
172 /// happens — measured at 0.7 to 1.5 seconds after the event, on both
173 /// supported engines — so this is for the other case: an operator
174 /// unsticking a run whose wait may never end.
175 ///
176 /// **`task_run_id` is a task run id, not a workflow run id** — a
177 /// [`TaskRunDetail::id`](crate::TaskRunDetail::id), which
178 /// [`details`](RunsClient::details) hands back through
179 /// [`RunDetails::tasks`](crate::RunDetails::tasks). The same detail's
180 /// [`is_evicted`](crate::TaskRunDetail::is_evicted) is how to tell an
181 /// evicted task run from a running one: an evicted one's status stays
182 /// [`RunStatus::Running`](crate::RunStatus::Running).
183 ///
184 /// # The answer is not proof
185 ///
186 /// It is the engine's `requeued` flag, and both supported versions were
187 /// measured answering `true` **unconditionally** — including for a random
188 /// UUID naming no run at all. So a `true` says the call was accepted, not
189 /// that anything was queued; read
190 /// [`details`](RunsClient::details) to learn what actually happened. It is
191 /// returned rather than swallowed because the engine may one day mean
192 /// something by it.
193 ///
194 /// # Retries
195 ///
196 /// Like every unary call, one whose answer is lost to a transport error is
197 /// retried. Queueing a run that is already queued is what the engine was
198 /// measured doing anyway, so a repeat costs nothing.
199 ///
200 /// # Errors
201 ///
202 /// [`Error::InvalidArgument`] when `task_run_id` is blank — checked before
203 /// any call — and [`Error::Grpc`] when the call fails.
204 ///
205 /// ```no_run
206 /// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
207 /// # let run_id = "d627fab0-385f-4eb4-98f9-59a783bc8631";
208 /// let details = client.runs().details(run_id).await?;
209 /// for (_name, task) in details.tasks() {
210 /// if task.is_evicted() {
211 /// client.runs().restore_evicted(task.id()).await?;
212 /// }
213 /// }
214 /// # Ok(())
215 /// # }
216 /// ```
217 pub async fn restore_evicted(&self, task_run_id: &str) -> Result<bool> {
218 if task_run_id.trim().is_empty() {
219 return Err(Error::InvalidArgument(
220 "a task run id must not be blank".to_string(),
221 ));
222 }
223 let request = RestoreEvictedTaskRequest {
224 task_run_external_id: task_run_id.to_string(),
225 };
226 let response = retry::unary(|| {
227 let mut dispatcher = self.client.dispatcher_grpc();
228 let request = retry::timed(request.clone());
229 async move { dispatcher.restore_evicted_task(request).await }
230 })
231 .await?;
232 Ok(response.into_inner().requeued)
233 }
234
235 /// What the engine remembers about the run called `run_id`.
236 ///
237 /// Works for a run this process never triggered and for one that is
238 /// still going — [`RunDetails::is_done`] says which.
239 ///
240 /// # Errors
241 ///
242 /// [`Error::InvalidArgument`] when `run_id` is blank;
243 /// [`Error::UnsupportedFeature`] when the engine does not have the
244 /// `GetRunDetails` RPC (answered `Unimplemented`);
245 /// [`Error::UnknownRunStatus`] when it reports a status this SDK has no
246 /// name for; and [`Error::Grpc`] otherwise.
247 pub async fn details(&self, run_id: &str) -> Result<RunDetails> {
248 if run_id.trim().is_empty() {
249 return Err(Error::InvalidArgument(
250 "a run id must not be blank".to_string(),
251 ));
252 }
253 let request = GetRunDetailsRequest {
254 external_id: run_id.to_string(),
255 };
256 let response = retry::unary(|| {
257 let mut admin = self.client.admin_grpc();
258 let request = retry::timed(request.clone());
259 async move { admin.get_run_details(request).await }
260 })
261 .await;
262 let response = match response {
263 Ok(response) => response,
264 // Translated here rather than gated by a version call before
265 // every lookup: one round trip instead of two on the happy
266 // path. `GetRunDetails` exists on both supported engine
267 // versions; this only matters against an engine that removed
268 // it, or one older than either we support.
269 Err(status) if status.code() == Code::Unimplemented => {
270 return Err(self.no_run_details().await);
271 }
272 Err(status) => return Err(status.into()),
273 };
274 RunDetails::from_proto(run_id, response.into_inner())
275 }
276
277 /// The output stream of the run called `run_id`: the chunks its handlers
278 /// publish with [`Context::put_stream`](crate::Context::put_stream), in
279 /// order.
280 ///
281 /// **Chunks are not replayed**, so a subscription only ever sees what is
282 /// published after the engine registers it. Opening one needs a run id,
283 /// and a run id only exists once the run has been triggered — so this call
284 /// cannot be made before the run starts, and a handler that publishes
285 /// immediately may be a chunk or two ahead by the time the subscription is
286 /// in place. That is not theoretical: against a v0.80.0 engine, a
287 /// subscription opened four seconds after the trigger began at chunk 1 and
288 /// chunk 0 was never delivered. When that matters, subscribe by metadata
289 /// *before* triggering — see
290 /// [`stream_by_metadata`](RunsClient::stream_by_metadata), the only
291 /// race-free way to read a run's output from the start.
292 ///
293 /// **This call waits up to five seconds for the engine to answer**, and
294 /// then hands back a stream that may still be being opened. That is the
295 /// ordinary case rather than a failure: neither supported engine sends
296 /// anything at all on a fresh subscription — not even response headers —
297 /// until it has an event for it, which has been measured at nineteen
298 /// seconds and more. The subscription is registered engine-side long
299 /// before that, and it is that same one the stream goes on with, so
300 /// nothing published in the meantime is lost.
301 ///
302 /// The stream ends when the engine hangs up, when it fails in a way
303 /// reconnecting cannot fix (the failure is the last item), or when it is
304 /// dropped. Anything else — a dropped connection, a restarting engine — is
305 /// resubscribed under the SDK's stream backoff, and chunks already
306 /// delivered are not delivered again. A hang-up means the engine closed
307 /// this subscription, not that every chunk was delivered: a multi-task run
308 /// on v0.80.0 hung up before its chunks arrived, and subscribing again
309 /// then delivered them.
310 ///
311 /// ```no_run
312 /// # async fn example(client: hatchet_client::Client) -> hatchet_client::Result<()> {
313 /// # let run_id = "d627fab0-385f-4eb4-98f9-59a783bc8631";
314 /// let mut chunks = client.runs().stream(run_id).await?;
315 /// while let Some(chunk) = chunks.next().await {
316 /// print!("{}", chunk?.payload);
317 /// }
318 /// # Ok(())
319 /// # }
320 /// ```
321 ///
322 /// # Errors
323 ///
324 /// [`Error::InvalidArgument`] when `run_id` is blank — checked before any
325 /// call — and [`Error::Grpc`] when the engine refuses the subscription in
326 /// a way reconnecting cannot fix: a rejected token, or a run id it does
327 /// not know (`NotFound`). A failure that is worth retrying is not reported
328 /// here: the stream is handed back and retries in the background, and
329 /// gives up as [`Error::ResultStream`] on the stream itself.
330 pub async fn stream(&self, run_id: &str) -> Result<OutputStream> {
331 if run_id.trim().is_empty() {
332 return Err(Error::InvalidArgument(
333 "a run id must not be blank".to_string(),
334 ));
335 }
336 stream::open(
337 &self.client,
338 SubscribeToWorkflowEventsRequest {
339 workflow_run_id: Some(run_id.to_string()),
340 additional_meta_key: None,
341 additional_meta_value: None,
342 },
343 )
344 .await
345 }
346
347 /// The output streams of every run whose additional metadata has `key` set
348 /// to `value`, on one subscription.
349 ///
350 /// This is the race-free way to read a run's output, because it does not
351 /// need a run id: open the subscription, *then* trigger, and nothing can
352 /// be published before the engine knows you are listening.
353 /// [`StreamChunk::run_id`](crate::StreamChunk::run_id) is what tells the
354 /// runs apart.
355 ///
356 /// ```no_run
357 /// # use hatchet_client::{Client, RunOpts, TaskBuilder};
358 /// # async fn example(client: Client) -> hatchet_client::Result<()> {
359 /// # let summarise = TaskBuilder::<String, ()>::new("summarise")
360 /// # .handler(|_ctx, _input| async { Ok(()) });
361 /// // A tag only this call will use, so the subscription matches this run
362 /// // and no other.
363 /// let tag = uuid_of_your_choosing();
364 ///
365 /// // Listening first is the whole point: chunks are not replayed.
366 /// let mut chunks = client.runs().stream_by_metadata("job", &tag).await?;
367 /// let run = summarise
368 /// .run_no_wait(
369 /// &client,
370 /// &"the document".to_string(),
371 /// RunOpts::default().metadata("job", tag.as_str()),
372 /// )
373 /// .await?;
374 ///
375 /// while let Some(chunk) = chunks.next().await {
376 /// let chunk = chunk?;
377 /// if chunk.run_id == run.id() {
378 /// print!("{}", chunk.payload);
379 /// }
380 /// }
381 /// # Ok(())
382 /// # }
383 /// # fn uuid_of_your_choosing() -> String { "3f2a…".to_string() }
384 /// ```
385 ///
386 /// `value` is matched as text against the run's metadata value, so a
387 /// metadata value that is not a string (a number, an object) has no text
388 /// form to give here.
389 ///
390 /// **This call waits up to five seconds for the engine to answer**, as
391 /// [`stream`](RunsClient::stream) does — and here it nearly always uses
392 /// the whole five, because a subscription opened before anything is
393 /// triggered has nothing to match yet and the engine answers it only when
394 /// it does. The wait costs latency, not chunks: the subscription is
395 /// registered before the answer comes, and the stream carries on with that
396 /// same one.
397 ///
398 /// Nothing ends this stream on its own *by design* — it covers runs that
399 /// have not been triggered yet, so there is no point at which the engine
400 /// can say there will be no more — but the engine does hang up on it once
401 /// the runs it matched are over, and a hang-up ends the stream. Open a new
402 /// one to keep watching, drop it when you are done, and keep a time limit
403 /// of your own around a wait for a chunk that may never come.
404 ///
405 /// # Errors
406 ///
407 /// [`Error::InvalidArgument`] when `key` or `value` is blank, and
408 /// [`Error::Grpc`] as for [`stream`](RunsClient::stream).
409 pub async fn stream_by_metadata(&self, key: &str, value: &str) -> Result<OutputStream> {
410 if key.trim().is_empty() || value.trim().is_empty() {
411 return Err(Error::InvalidArgument(format!(
412 "a metadata subscription needs both halves of the pair; got key {key:?} and \
413 value {value:?}"
414 )));
415 }
416 stream::open(
417 &self.client,
418 SubscribeToWorkflowEventsRequest {
419 workflow_run_id: None,
420 additional_meta_key: Some(key.to_string()),
421 additional_meta_value: Some(value.to_string()),
422 },
423 )
424 .await
425 }
426
427 /// The error for an engine without `GetRunDetails`: the version it
428 /// reports, best-effort, and the version that has the RPC.
429 async fn no_run_details(&self) -> Error {
430 let found = self
431 .client
432 .engine_version()
433 .await
434 .unwrap_or_else(|_| "unknown".to_string());
435 Error::UnsupportedFeature {
436 feature: "run details (GetRunDetails)".to_string(),
437 found,
438 minimum: crate::version::MIN_ENGINE_VERSION.to_string(),
439 }
440 }
441}
442
443/// The ids of a `cancel`/`replay` call, refused before the wire when they
444/// could not mean anything.
445///
446/// An empty list is refused because `external_ids: []` with no filter is not
447/// a no-op on the wire — the engine decides what it means, and the official
448/// Python SDK refuses the same shape (`features/runs.py:67-73`).
449fn checked_ids(
450 ids: impl IntoIterator<Item = impl Into<String>>,
451 what: &str,
452) -> Result<Vec<String>> {
453 let ids: Vec<String> = ids.into_iter().map(Into::into).collect();
454 if ids.is_empty() {
455 return Err(Error::InvalidArgument(format!(
456 "{what} was given no run ids; naming none would leave it to the engine to decide \
457 what that means"
458 )));
459 }
460 if let Some(position) = ids.iter().position(|id| id.trim().is_empty()) {
461 return Err(Error::InvalidArgument(format!(
462 "{what} was given a blank run id at position {position}"
463 )));
464 }
465 Ok(ids)
466}