Skip to main content

runnerd_core/
actions.rs

1//! The owner's job actions (ADR 0134): cancelling a job's workflow run and re-running a job, or its run's failed jobs,
2//! on GitHub, for `POST /v1/jobs/{id}/cancel` and `POST /v1/jobs/{id}/rerun`. A request without a confirmation gets a
3//! preview: the job and its run's other jobs from the history, the run's current attempt read from GitHub, and a
4//! confirmation issued for exactly that (`flights`). A confirmed request is authorized against one snapshot of the
5//! config (`policy`) and sent once through the engine's own GitHub client, off the engine's loop. Once the token is
6//! in hand, and before each send (a 401's second included), the snapshot is checked, the run read again for its
7//! attempt, and the snapshot checked once more, with nothing in between that and the send.
8//!
9//! It must not write the job history (GitHub's listings bring what the action did), kill a runner, keep a deadline,
10//! send an ambiguous request again, or put GitHub's text or the token anywhere: a refusal is its class and status.
11
12mod flights;
13pub mod policy;
14
15use std::sync::Arc;
16use std::time::{Duration, Instant};
17
18use futures_util::future::BoxFuture;
19use tokio::sync::watch;
20use tracing::info;
21
22use self::flights::{Boarding, Flights, Issued, Landed, Redeemed};
23use self::policy::{Denied, Policy};
24use crate::api::{
25    self, ActionAnswer, ActionPreview, ActionRefusal, ActionResult, ActionTarget, JobAction,
26    JobSummary, PreviewFacts, RunSeen, SIBLINGS_MAX, Superseded, UnknownCause,
27};
28use crate::github::{ActionFailure, GithubApi, JobActionRequest, Permit, RunNow};
29use crate::scheduler::engine::{Control, Shutdown};
30use crate::store::{Store, StoreError};
31use crate::time;
32
33/// How long one action may take to send, its token read, its read of the run and a 401's second try included: below
34/// the 30 s a client waits for an answer, so the client hears the outcome, if only that it is unknown. A preview's
35/// read of the run has as long.
36pub const DEADLINE: Duration = Duration::from_secs(20);
37
38/// How long the job actions wait for GitHub, and how long a confirmation lasts.
39#[derive(Debug, Clone, Copy)]
40pub struct Limits {
41    /// For one action, or one preview's read of the run.
42    pub deadline: Duration,
43    /// For a confirmation to be sent back.
44    pub confirmation: Duration,
45}
46
47/// The daemon's limits: [`DEADLINE`] and [`api::CONFIRM_SECS`].
48pub const LIMITS: Limits = Limits {
49    deadline: DEADLINE,
50    confirmation: Duration::from_secs(api::CONFIRM_SECS),
51};
52
53/// The GitHub calls the job actions make. Every [`GithubApi`] is one, so the engine's own client sends them; a test
54/// gives a fake.
55pub trait ActionsGithub: Send + Sync + 'static {
56    /// Reads the run's current attempt and whether it completed.
57    fn run<'call>(
58        &'call self,
59        repository: &'call str,
60        run: u64,
61    ) -> BoxFuture<'call, Result<RunNow, ActionFailure>>;
62
63    /// Sends `request` once, asking `permit` once the token is in hand and before each send; GitHub's success
64    /// status, or why not.
65    fn send<'call>(
66        &'call self,
67        repository: &'call str,
68        request: JobActionRequest,
69        permit: &'call Permit<'call>,
70    ) -> BoxFuture<'call, Result<u16, ActionFailure>>;
71}
72
73impl<G: GithubApi + 'static> ActionsGithub for G {
74    fn run<'call>(
75        &'call self,
76        repository: &'call str,
77        run: u64,
78    ) -> BoxFuture<'call, Result<RunNow, ActionFailure>> {
79        Box::pin(GithubApi::action_run(self, repository, run))
80    }
81
82    fn send<'call>(
83        &'call self,
84        repository: &'call str,
85        request: JobActionRequest,
86        permit: &'call Permit<'call>,
87    ) -> BoxFuture<'call, Result<u16, ActionFailure>> {
88        Box::pin(GithubApi::job_action(self, repository, request, permit))
89    }
90}
91
92/// What a request got.
93#[derive(Debug)]
94pub enum Asked {
95    /// It was not confirmed, or its confirmation no longer holds: nothing was sent (409).
96    Preview(Box<ActionPreview>),
97    /// GitHub was asked, by this request or by the same one in flight.
98    Done(ActionResult),
99}
100
101/// Why a request was answered before GitHub was asked.
102#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
103pub enum ActionError {
104    /// No such job in the history (404).
105    #[error("{0}")]
106    NotFound(String),
107    /// Job actions are off, or the repository is no longer configured (403).
108    #[error("{0}")]
109    Forbidden(String),
110    /// The request cannot apply to this job (422).
111    #[error("{0}")]
112    Invalid(String),
113    /// Another action is in flight for the run (409).
114    #[error("{0}")]
115    Busy(String),
116    /// The history could not be read, or no confirmation could be made (500).
117    #[error("{0}")]
118    Failed(String),
119    /// runnerd is stopping (503).
120    #[error("runnerd is stopping")]
121    Stopped,
122}
123
124/// Sends the owner's job actions. Cloned into each request; every clone shares the confirmations and flights.
125#[derive(Clone)]
126pub struct JobActions {
127    github: Arc<dyn ActionsGithub>,
128    store: Store,
129    control: Arc<Control>,
130    flights: Arc<Flights>,
131    deadline: Duration,
132}
133
134impl std::fmt::Debug for JobActions {
135    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
136        f.debug_struct("JobActions").finish_non_exhaustive()
137    }
138}
139
140impl JobActions {
141    /// Actions sent through `github`, for the jobs `store` has, under the config `control` holds, within `limits`
142    /// ([`LIMITS`] but in tests).
143    pub fn new(
144        github: Arc<dyn ActionsGithub>,
145        store: Store,
146        control: Arc<Control>,
147        limits: Limits,
148    ) -> JobActions {
149        JobActions {
150            github,
151            store,
152            control,
153            flights: Arc::new(Flights::new(limits.confirmation)),
154            deadline: limits.deadline,
155        }
156    }
157
158    /// Asks `action` of job `job`. Unless `confirm` is a confirmation this daemon issued with a preview of this very
159    /// action and job, still current, the answer is a new preview and nothing is sent. A confirmed one is sent once,
160    /// or joins the same one in flight for the run; another action in flight for the run refuses it.
161    pub async fn ask(
162        &self,
163        job: u64,
164        action: JobAction,
165        confirm: Option<&str>,
166    ) -> Result<Asked, ActionError> {
167        if action == JobAction::Unsupported {
168            return Err(ActionError::Invalid("no such job action".to_owned()));
169        }
170        if self.control.mode() != Shutdown::No {
171            return Err(ActionError::Stopped);
172        }
173        let summary = self
174            .store
175            .job(job)
176            .await
177            .map_err(|error| read_failed(&error))?
178            .ok_or_else(|| ActionError::NotFound(format!("no job {job} in the history")))?
179            .summary;
180        if summary.run_id == 0 {
181            return Err(ActionError::Invalid(api::actions_no_run(job)));
182        }
183        // The config of this moment, one snapshot: a reload since the preview may have turned actions off or removed
184        // the repository.
185        let policy = self.control.action_policy();
186        allowed(&policy, &summary.repository)?;
187        let superseded = match confirm {
188            None => None,
189            Some(token) => {
190                match self
191                    .flights
192                    .redeem(token, action, job, policy.revision(), Instant::now())
193                {
194                    Redeemed::Board { boarding, issued } => {
195                        return self.board(boarding, issued, &summary).await;
196                    }
197                    Redeemed::Stale(why) => Some(why),
198                }
199            }
200        };
201        let run = self.read_run(&summary.repository, summary.run_id).await;
202        self.preview(&summary, action, run, superseded, &policy)
203            .await
204    }
205
206    /// The preview of `action` on `summary`'s job with the run as `run` says, and a confirmation for it if the run was
207    /// read.
208    async fn preview(
209        &self,
210        summary: &JobSummary,
211        action: JobAction,
212        run: RunSeen,
213        superseded: Option<Superseded>,
214        policy: &Policy,
215    ) -> Result<Asked, ActionError> {
216        let others = self
217            .store
218            .run_jobs(
219                &summary.repository,
220                summary.run_id,
221                summary.id,
222                SIBLINGS_MAX,
223            )
224            .await
225            .map_err(|error| read_failed(&error))?;
226        let mut target = ActionTarget {
227            action,
228            job: summary.id,
229            name: summary.name.clone(),
230            repository: summary.repository.clone(),
231            run: summary.run_id,
232            attempt: None,
233        };
234        let confirm = match run {
235            RunSeen::Now { attempt, .. } => {
236                target.attempt = Some(attempt);
237                let issued = Issued {
238                    target: target.clone(),
239                    revision: policy.revision(),
240                };
241                Some(
242                    self.flights
243                        .issue(issued, Instant::now())
244                        .map_err(|error| {
245                            ActionError::Failed(format!("cannot make a confirmation: {error}"))
246                        })?,
247                )
248            }
249            RunSeen::Unread { .. } => None,
250        };
251        Ok(Asked::Preview(Box::new(ActionPreview::new(PreviewFacts {
252            paused: self.control.is_paused(&target.repository),
253            target,
254            job_attempt: summary.run_attempt,
255            run,
256            superseded,
257            confirm,
258            siblings: others.jobs,
259            more: others.more,
260        }))))
261    }
262
263    /// Sends what `issued` names, or joins it in flight, and answers as it lands.
264    async fn board(
265        &self,
266        boarding: Boarding,
267        issued: Issued,
268        summary: &JobSummary,
269    ) -> Result<Asked, ActionError> {
270        let answer = match boarding {
271            Boarding::Busy(other) => {
272                return Err(ActionError::Busy(api::actions_in_flight(
273                    other,
274                    issued.target.run,
275                )));
276            }
277            Boarding::Follow(answer) => answer,
278            Boarding::Lead { landing, answer } => {
279                // A task of its own: a client that goes away neither stops the request midway nor leaves the flight.
280                let actions = self.clone();
281                let sent = issued.clone();
282                tokio::spawn(async move {
283                    let landed = actions.dispatch(&sent).await;
284                    landing.land(landed);
285                });
286                answer
287            }
288        };
289        match wait(answer, &issued.target).await {
290            Landed::Done(result) => Ok(Asked::Done(result)),
291            Landed::Repreview { run, why } => {
292                let policy = self.control.action_policy();
293                allowed(&policy, &summary.repository)?;
294                self.preview(summary, issued.target.action, run, Some(why), &policy)
295                    .await
296            }
297        }
298    }
299
300    /// Reads the run for a preview, within the deadline, unless a stop comes first.
301    async fn read_run(&self, repository: &str, run: u64) -> RunSeen {
302        tokio::select! {
303            biased;
304            () = self.control.stop_asked() => RunSeen::Unread { status: None },
305            read = tokio::time::timeout(self.deadline, self.github.run(repository, run)) => match read {
306                Ok(Ok(now)) => seen(now),
307                Ok(Err(failure)) => RunSeen::Unread { status: status_of(failure) },
308                Err(_) => RunSeen::Unread { status: None },
309            },
310        }
311    }
312
313    /// Sends `issued`'s action once, within the deadline, unless a stop comes first.
314    async fn dispatch(&self, issued: &Issued) -> Landed {
315        let target = &issued.target;
316        let unknown = |cause| {
317            Landed::Done(ActionResult::new(
318                target,
319                ActionAnswer::Unknown {
320                    cause,
321                    status: None,
322                },
323                time::now_ms() / 1000,
324            ))
325        };
326        tokio::select! {
327            biased;
328            () = self.control.stop_asked() => unknown(UnknownCause::Stopping),
329            landed = tokio::time::timeout(self.deadline, self.send(issued)) => {
330                landed.unwrap_or_else(|_| unknown(UnknownCause::TimedOut))
331            },
332        }
333    }
334
335    /// Sends the action once, if, once the token is in hand, the config is the confirmation's and the run is at the
336    /// attempt it showed.
337    async fn send(&self, issued: &Issued) -> Landed {
338        let target = &issued.target;
339        let repository = &target.repository;
340        let request = match target.action {
341            JobAction::Cancel => JobActionRequest::CancelRun { run: target.run },
342            JobAction::Rerun => JobActionRequest::RerunJob { job: target.job },
343            JobAction::RerunFailed | JobAction::Unsupported => {
344                JobActionRequest::RerunFailed { run: target.run }
345            }
346        };
347        // Why the permit said no, when it did.
348        let withheld: std::sync::Mutex<Option<Landed>> = std::sync::Mutex::new(None);
349        let permit = || -> BoxFuture<'_, bool> {
350            Box::pin(async {
351                let why = self.verify(issued).await;
352                let go = why.is_none();
353                *crate::sync::lock(&withheld) = why;
354                go
355            })
356        };
357        let (slug, run, job) = (target.action.slug(), target.run, target.job);
358        info!("{repository}: sending the {slug} of run {run} (job {job}) to GitHub");
359        let answer = match self.github.send(repository, request, &permit).await {
360            Err(ActionFailure::Withdrawn) => match crate::sync::lock(&withheld).take() {
361                Some(landed) => return landed,
362                None => classify(Err(ActionFailure::Withdrawn)),
363            },
364            // A 409 says the run cannot be cancelled now, not that it completed: only a read of it says that.
365            Err(ActionFailure::Status { status: 409 }) if target.action == JobAction::Cancel => {
366                match self.github.run(repository, target.run).await {
367                    Ok(now) if now.completed => ActionAnswer::AlreadyFinished { status: 409 },
368                    _ => ActionAnswer::Refused {
369                        refusal: ActionRefusal::Conflict,
370                        status: Some(409),
371                        until: None,
372                    },
373                }
374            }
375            sent => classify(sent),
376        };
377        let result = ActionResult::new(target, answer, time::now_ms() / 1000);
378        // The class and status only: GitHub's text never reaches a log.
379        info!(
380            "{repository}: GitHub's answer to the {slug} of run {run}: {:?}, status {:?}, {:?}",
381            result.outcome, result.status, result.refusal
382        );
383        Landed::Done(result)
384    }
385
386    /// Why `issued` may not go now, if it may not: the run is not at the attempt it showed, or the config changed
387    /// since its confirmation. The config is read after the run, so nothing but the send follows the check.
388    async fn verify(&self, issued: &Issued) -> Option<Landed> {
389        let target = &issued.target;
390        let withdrawn = || {
391            Landed::Done(ActionResult::new(
392                target,
393                classify(Err(ActionFailure::Withdrawn)),
394                time::now_ms() / 1000,
395            ))
396        };
397        match self.github.run(&target.repository, target.run).await {
398            Ok(now) if Some(now.attempt) == target.attempt => {}
399            Ok(now) => {
400                return Some(Landed::Repreview {
401                    run: seen(now),
402                    why: Superseded::AttemptChanged,
403                });
404            }
405            Err(failure) => {
406                return Some(Landed::Repreview {
407                    run: RunSeen::Unread {
408                        status: status_of(failure),
409                    },
410                    why: Superseded::Unverified,
411                });
412            }
413        }
414        (self.control.action_policy().revision() != issued.revision).then(withdrawn)
415    }
416
417    /// How many runs have an action in flight.
418    #[cfg(test)]
419    fn in_flight(&self) -> usize {
420        self.flights.len()
421    }
422
423    /// How many requests wait for an answer from GitHub, the lead of each flight included.
424    #[cfg(test)]
425    fn waiting(&self) -> usize {
426        self.flights.waiting()
427    }
428}
429
430/// Whether `policy` lets the owner act on `repository`'s jobs.
431fn allowed(policy: &Policy, repository: &str) -> Result<(), ActionError> {
432    policy.allows(repository).map_err(|denied| {
433        ActionError::Forbidden(match denied {
434            Denied::Off => api::actions_off().to_owned(),
435            Denied::NotConfigured => api::actions_not_configured(repository),
436        })
437    })
438}
439
440/// What the flight lands; if it ends without an answer, as when the runtime stops, the outcome is unknown.
441async fn wait(mut answer: watch::Receiver<Option<Landed>>, target: &ActionTarget) -> Landed {
442    let stopped = || {
443        Landed::Done(ActionResult::new(
444            target,
445            ActionAnswer::Unknown {
446                cause: UnknownCause::Stopping,
447                status: None,
448            },
449            time::now_ms() / 1000,
450        ))
451    };
452    match answer.wait_for(Option::is_some).await {
453        Ok(landed) => landed.clone().unwrap_or_else(stopped),
454        Err(_) => stopped(),
455    }
456}
457
458fn seen(now: RunNow) -> RunSeen {
459    RunSeen::Now {
460        attempt: now.attempt,
461        completed: now.completed,
462    }
463}
464
465/// The HTTP status GitHub answered with, if it did.
466fn status_of(failure: ActionFailure) -> Option<u16> {
467    match failure {
468        ActionFailure::Status { status } => Some(status),
469        ActionFailure::RateLimited { status, .. } => status,
470        _ => None,
471    }
472}
473
474fn read_failed(error: &StoreError) -> ActionError {
475    ActionError::Failed(format!("cannot read the job history: {error}"))
476}
477
478/// What GitHub's answer says, as a class and a status: never its text.
479fn classify(sent: Result<u16, ActionFailure>) -> ActionAnswer {
480    let refused = |refusal, status| ActionAnswer::Refused {
481        refusal,
482        status,
483        until: None,
484    };
485    match sent {
486        Ok(status) => ActionAnswer::Accepted { status },
487        Err(ActionFailure::NoToken) => refused(ActionRefusal::NoToken, None),
488        Err(ActionFailure::Withdrawn) => refused(ActionRefusal::Withdrawn, None),
489        // Nothing left this machine.
490        Err(ActionFailure::Unreachable) => refused(ActionRefusal::Unreachable, None),
491        Err(ActionFailure::RateLimited { status, until }) => ActionAnswer::Refused {
492            refusal: ActionRefusal::RateLimited,
493            status,
494            until: Some(
495                until
496                    .duration_since(std::time::SystemTime::UNIX_EPOCH)
497                    .unwrap_or_default()
498                    .as_secs(),
499            ),
500        },
501        Err(ActionFailure::Status { status }) => {
502            let code = Some(status);
503            match status {
504                401 => refused(ActionRefusal::Unauthorized, code),
505                403 => refused(ActionRefusal::Forbidden, code),
506                404 | 410 => refused(ActionRefusal::NotFound, code),
507                409 => refused(ActionRefusal::Conflict, code),
508                422 => refused(ActionRefusal::Unprocessable, code),
509                // A server error does not say whether GitHub acted before it failed.
510                500..=599 => ActionAnswer::Unknown {
511                    cause: UnknownCause::ServerError,
512                    status: code,
513                },
514                _ => refused(ActionRefusal::Failed, code),
515            }
516        }
517        Err(ActionFailure::TimedOut) => ActionAnswer::Unknown {
518            cause: UnknownCause::TimedOut,
519            status: None,
520        },
521        Err(ActionFailure::Broken) => ActionAnswer::Unknown {
522            cause: UnknownCause::Broken,
523            status: None,
524        },
525    }
526}
527
528#[cfg(test)]
529mod tests;