Skip to main content

runnerd_core/
actions.rs

1//! The owner's job actions (ADR 0134): cancelling a job's workflow run and re-running a job, or its run's failed jobs,
2//! on GitHub, for `POST /v1/jobs/{id}/cancel` and `POST /v1/jobs/{id}/rerun`. A request without a confirmation gets a
3//! preview: the job and its run's other jobs from the history, the run's current attempt read from GitHub, and a
4//! confirmation issued for exactly that (`flights`). A confirmed request is authorized against one snapshot of the
5//! config (`policy`) and sent once through the engine's own GitHub client, off the engine's loop. Once the token is
6//! in hand, and before each send (a 401's second included), the snapshot is checked, the run read again for its
7//! attempt, and the snapshot checked once more, with nothing in between that and the send.
8//!
9//! `rerun` sends one action itself: the re-run of a run whose runner failed one of its jobs (ADR 0153), through the
10//! same flights and the same check before the send, without a preview.
11//!
12//! It must not write the job history (GitHub's listings bring what the action did) beyond noting an automatic re-run
13//! GitHub accepted, kill a runner, keep a deadline, send an owner's ambiguous request again, or put GitHub's text or
14//! the token anywhere: a refusal is its class and status.
15
16mod faults;
17mod flights;
18pub mod policy;
19mod rerun;
20
21use std::sync::Arc;
22use std::time::{Duration, Instant};
23
24use futures_util::future::BoxFuture;
25use tokio::sync::watch;
26use tracing::info;
27
28use self::faults::Faults;
29use self::flights::{Boarding, Flights, Issued, Landed, Origin, Redeemed, Withheld};
30use self::policy::{Denied, Policy};
31use crate::api::{
32    self, ActionAnswer, ActionPreview, ActionRefusal, ActionResult, ActionTarget, JobAction,
33    JobSummary, PreviewFacts, RunSeen, SIBLINGS_MAX, Superseded, UnknownCause,
34};
35use crate::github::{ActionFailure, AttemptJob, GithubApi, JobActionRequest, Permit, RunNow};
36use crate::scheduler::engine::{Control, Shutdown};
37use crate::store::{Store, StoreError};
38use crate::time;
39
40/// How long one action may take to send, its token read, its read of the run and a 401's second try included: below
41/// the 30 s a client waits for an answer, so the client hears the outcome, if only that it is unknown. A preview's
42/// read of the run has as long.
43pub const DEADLINE: Duration = Duration::from_secs(20);
44
45/// How long the job actions wait for GitHub, and how long a confirmation lasts.
46#[derive(Debug, Clone, Copy)]
47pub struct Limits {
48    /// For one action, or one preview's read of the run.
49    pub deadline: Duration,
50    /// For a confirmation to be sent back.
51    pub confirmation: Duration,
52}
53
54/// The daemon's limits: [`DEADLINE`] and [`api::CONFIRM_SECS`].
55pub const LIMITS: Limits = Limits {
56    deadline: DEADLINE,
57    confirmation: Duration::from_secs(api::CONFIRM_SECS),
58};
59
60/// The GitHub calls the job actions make. Every [`GithubApi`] is one, so the engine's own client sends them; a test
61/// gives a fake.
62pub trait ActionsGithub: Send + Sync + 'static {
63    /// Reads the run's current attempt and whether it completed.
64    fn run<'call>(
65        &'call self,
66        repository: &'call str,
67        run: u64,
68    ) -> BoxFuture<'call, Result<RunNow, ActionFailure>>;
69
70    /// Reads the jobs of attempt `attempt` of the run, every runner's, with their conclusions.
71    fn attempt_jobs<'call>(
72        &'call self,
73        repository: &'call str,
74        run: u64,
75        attempt: u32,
76    ) -> BoxFuture<'call, Result<Vec<AttemptJob>, ActionFailure>>;
77
78    /// Sends `request` once, asking `permit` once the token is in hand and before each send; GitHub's success
79    /// status, or why not.
80    fn send<'call>(
81        &'call self,
82        repository: &'call str,
83        request: JobActionRequest,
84        permit: &'call Permit<'call>,
85    ) -> BoxFuture<'call, Result<u16, ActionFailure>>;
86}
87
88impl<G: GithubApi + 'static> ActionsGithub for G {
89    fn run<'call>(
90        &'call self,
91        repository: &'call str,
92        run: u64,
93    ) -> BoxFuture<'call, Result<RunNow, ActionFailure>> {
94        Box::pin(GithubApi::action_run(self, repository, run))
95    }
96
97    fn attempt_jobs<'call>(
98        &'call self,
99        repository: &'call str,
100        run: u64,
101        attempt: u32,
102    ) -> BoxFuture<'call, Result<Vec<AttemptJob>, ActionFailure>> {
103        Box::pin(GithubApi::attempt_jobs(self, repository, run, attempt))
104    }
105
106    fn send<'call>(
107        &'call self,
108        repository: &'call str,
109        request: JobActionRequest,
110        permit: &'call Permit<'call>,
111    ) -> BoxFuture<'call, Result<u16, ActionFailure>> {
112        Box::pin(GithubApi::job_action(self, repository, request, permit))
113    }
114}
115
116/// What a request got.
117#[derive(Debug)]
118pub enum Asked {
119    /// It was not confirmed, or its confirmation no longer holds: nothing was sent (409).
120    Preview(Box<ActionPreview>),
121    /// GitHub was asked, by this request or by the same one in flight.
122    Done(ActionResult),
123}
124
125/// Why a request was answered before GitHub was asked.
126#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
127pub enum ActionError {
128    /// No such job in the history (404).
129    #[error("{0}")]
130    NotFound(String),
131    /// Job actions are off, or the repository is no longer configured (403).
132    #[error("{0}")]
133    Forbidden(String),
134    /// The request cannot apply to this job (422).
135    #[error("{0}")]
136    Invalid(String),
137    /// Another action is in flight for the run (409).
138    #[error("{0}")]
139    Busy(String),
140    /// The history could not be read, or no confirmation could be made (500).
141    #[error("{0}")]
142    Failed(String),
143    /// runnerd is stopping (503).
144    #[error("runnerd is stopping")]
145    Stopped,
146}
147
148/// Sends the owner's job actions. Cloned into each request; every clone shares the confirmations and flights.
149#[derive(Clone)]
150pub struct JobActions {
151    github: Arc<dyn ActionsGithub>,
152    store: Store,
153    control: Arc<Control>,
154    flights: Arc<Flights>,
155    faults: Arc<Faults>,
156    deadline: Duration,
157}
158
159impl std::fmt::Debug for JobActions {
160    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
161        f.debug_struct("JobActions").finish_non_exhaustive()
162    }
163}
164
165impl JobActions {
166    /// Actions sent through `github`, for the jobs `store` has, under the config `control` holds, within `limits`
167    /// ([`LIMITS`] but in tests).
168    pub fn new(
169        github: Arc<dyn ActionsGithub>,
170        store: Store,
171        control: Arc<Control>,
172        limits: Limits,
173    ) -> JobActions {
174        JobActions {
175            github,
176            store,
177            control,
178            flights: Arc::new(Flights::new(limits.confirmation)),
179            faults: Arc::new(Faults::default()),
180            deadline: limits.deadline,
181        }
182    }
183
184    /// Asks `action` of job `job`. Unless `confirm` is a confirmation this daemon issued with a preview of this very
185    /// action and job, still current, the answer is a new preview and nothing is sent. A confirmed one is sent once,
186    /// or joins the same one in flight for the run; another action in flight for the run refuses it.
187    pub async fn ask(
188        &self,
189        job: u64,
190        action: JobAction,
191        confirm: Option<&str>,
192    ) -> Result<Asked, ActionError> {
193        if action == JobAction::Unsupported {
194            return Err(ActionError::Invalid("no such job action".to_owned()));
195        }
196        if self.control.mode() != Shutdown::No {
197            return Err(ActionError::Stopped);
198        }
199        let summary = self
200            .store
201            .job(job)
202            .await
203            .map_err(|error| read_failed(&error))?
204            .ok_or_else(|| ActionError::NotFound(format!("no job {job} in the history")))?
205            .summary;
206        if summary.run_id == 0 {
207            return Err(ActionError::Invalid(api::actions_no_run(job)));
208        }
209        // The config of this moment, one snapshot: a reload since the preview may have turned actions off or removed
210        // the repository.
211        let policy = self.control.action_policy();
212        allowed(&policy, &summary.repository)?;
213        let superseded = match confirm {
214            None => None,
215            Some(token) => {
216                match self
217                    .flights
218                    .redeem(token, action, job, policy.revision(), Instant::now())
219                {
220                    Redeemed::Board { boarding, issued } => {
221                        return self.board(boarding, issued, &summary).await;
222                    }
223                    Redeemed::Stale(why) => Some(why),
224                }
225            }
226        };
227        let run = self.read_run(&summary.repository, summary.run_id).await;
228        self.preview(&summary, action, run, superseded, &policy)
229            .await
230    }
231
232    /// The preview of `action` on `summary`'s job with the run as `run` says, and a confirmation for it if the run was
233    /// read.
234    async fn preview(
235        &self,
236        summary: &JobSummary,
237        action: JobAction,
238        run: RunSeen,
239        superseded: Option<Superseded>,
240        policy: &Policy,
241    ) -> Result<Asked, ActionError> {
242        let others = self
243            .store
244            .run_jobs(
245                &summary.repository,
246                summary.run_id,
247                summary.id,
248                SIBLINGS_MAX,
249            )
250            .await
251            .map_err(|error| read_failed(&error))?;
252        let mut target = ActionTarget {
253            action,
254            job: summary.id,
255            name: summary.name.clone(),
256            repository: summary.repository.clone(),
257            run: summary.run_id,
258            attempt: None,
259        };
260        let confirm = match run {
261            RunSeen::Now { attempt, .. } => {
262                target.attempt = Some(attempt);
263                let issued = Issued {
264                    target: target.clone(),
265                    revision: policy.revision(),
266                    origin: Origin::Owner,
267                };
268                Some(
269                    self.flights
270                        .issue(issued, Instant::now())
271                        .map_err(|error| {
272                            ActionError::Failed(format!("cannot make a confirmation: {error}"))
273                        })?,
274                )
275            }
276            RunSeen::Unread { .. } => None,
277        };
278        Ok(Asked::Preview(Box::new(ActionPreview::new(PreviewFacts {
279            paused: self.control.is_paused(&target.repository),
280            target,
281            job_attempt: summary.run_attempt,
282            run,
283            superseded,
284            confirm,
285            siblings: others.jobs,
286            more: others.more,
287        }))))
288    }
289
290    /// Sends what `issued` names, or joins it in flight, and answers as it lands.
291    async fn board(
292        &self,
293        boarding: Boarding,
294        issued: Issued,
295        summary: &JobSummary,
296    ) -> Result<Asked, ActionError> {
297        let answer = match boarding {
298            Boarding::Busy(other) => {
299                return Err(ActionError::Busy(api::actions_in_flight(
300                    other,
301                    issued.target.run,
302                )));
303            }
304            Boarding::Follow(answer) => answer,
305            Boarding::Lead { landing, answer } => {
306                // A task of its own: a client that goes away neither stops the request midway nor leaves the flight.
307                let actions = self.clone();
308                let sent = issued.clone();
309                tokio::spawn(async move {
310                    let landed = actions.dispatch(&sent).await;
311                    landing.land(landed);
312                });
313                answer
314            }
315        };
316        match wait(answer, &issued.target).await {
317            Landed::Done(result) => Ok(Asked::Done(result)),
318            Landed::Repreview { run, why } => {
319                let policy = self.control.action_policy();
320                allowed(&policy, &summary.repository)?;
321                self.preview(summary, issued.target.action, run, Some(why), &policy)
322                    .await
323            }
324            // The owner joined runnerd's own re-run, whose permit said no: nothing was sent.
325            Landed::Withheld(_) => {
326                let policy = self.control.action_policy();
327                allowed(&policy, &summary.repository)?;
328                let run = RunSeen::Unread { status: None };
329                let why = Some(Superseded::Unverified);
330                self.preview(summary, issued.target.action, run, why, &policy)
331                    .await
332            }
333        }
334    }
335
336    /// Reads the run for a preview, within the deadline, unless a stop comes first.
337    async fn read_run(&self, repository: &str, run: u64) -> RunSeen {
338        tokio::select! {
339            biased;
340            () = self.control.stop_asked() => RunSeen::Unread { status: None },
341            read = tokio::time::timeout(self.deadline, self.github.run(repository, run)) => match read {
342                Ok(Ok(now)) => seen(now),
343                Ok(Err(failure)) => RunSeen::Unread { status: status_of(failure) },
344                Err(_) => RunSeen::Unread { status: None },
345            },
346        }
347    }
348
349    /// Sends `issued`'s action once, within the deadline, unless a stop comes first.
350    async fn dispatch(&self, issued: &Issued) -> Landed {
351        let target = &issued.target;
352        let unknown = |cause| {
353            Landed::Done(ActionResult::new(
354                target,
355                ActionAnswer::Unknown {
356                    cause,
357                    status: None,
358                },
359                time::now_ms() / 1000,
360            ))
361        };
362        tokio::select! {
363            biased;
364            () = self.control.stop_asked() => unknown(UnknownCause::Stopping),
365            landed = tokio::time::timeout(self.deadline, self.send(issued)) => {
366                landed.unwrap_or_else(|_| unknown(UnknownCause::TimedOut))
367            },
368        }
369    }
370
371    /// Sends the action once, if, once the token is in hand, the config is the confirmation's and the run is at the
372    /// attempt it showed.
373    async fn send(&self, issued: &Issued) -> Landed {
374        let target = &issued.target;
375        let repository = &target.repository;
376        let request = match target.action {
377            JobAction::Cancel => JobActionRequest::CancelRun { run: target.run },
378            JobAction::Rerun => JobActionRequest::RerunJob { job: target.job },
379            JobAction::RerunFailed | JobAction::Unsupported => {
380                JobActionRequest::RerunFailed { run: target.run }
381            }
382        };
383        // Why the permit said no, when it did.
384        let withheld: std::sync::Mutex<Option<Landed>> = std::sync::Mutex::new(None);
385        // Whether the permit let a send go: from then on the request may be on its way.
386        let opened = std::sync::atomic::AtomicBool::new(false);
387        let permit = || -> BoxFuture<'_, bool> {
388            Box::pin(async {
389                let why = self.verify(issued).await;
390                let go = why.is_none();
391                if go {
392                    opened.store(true, std::sync::atomic::Ordering::SeqCst);
393                }
394                *crate::sync::lock(&withheld) = why;
395                go
396            })
397        };
398        let (slug, run, job) = (target.action.slug(), target.run, target.job);
399        info!("{repository}: sending the {slug} of run {run} (job {job}) to GitHub");
400        let sent = async {
401            match issued.origin {
402                Origin::Owner => self.github.send(repository, request, &permit).await,
403                // A reload stops the request where it is. Before the permit let it go, nothing was sent: it is
404                // withdrawn. After, it may have reached GitHub, which may act on it yet: its outcome is unknown, as
405                // for a broken connection, and it is never sent again (ADR 0153).
406                Origin::Automatic => tokio::select! {
407                    biased;
408                    () = self.control.policy_moved(issued.revision) => {
409                        if opened.load(std::sync::atomic::Ordering::SeqCst) {
410                            Err(ActionFailure::Broken)
411                        } else {
412                            Err(ActionFailure::Withdrawn)
413                        }
414                    }
415                    sent = self.github.send(repository, request, &permit) => sent,
416                },
417            }
418        };
419        let answer = match sent.await {
420            Err(ActionFailure::Withdrawn) => match crate::sync::lock(&withheld).take() {
421                Some(landed) => return landed,
422                None => classify(Err(ActionFailure::Withdrawn)),
423            },
424            // A 409 says the run cannot be cancelled now, not that it completed: only a read of it says that.
425            Err(ActionFailure::Status { status: 409 }) if target.action == JobAction::Cancel => {
426                match self.github.run(repository, target.run).await {
427                    Ok(now) if now.completed => ActionAnswer::AlreadyFinished { status: 409 },
428                    _ => ActionAnswer::Refused {
429                        refusal: ActionRefusal::Conflict,
430                        status: Some(409),
431                        until: None,
432                    },
433                }
434            }
435            sent => classify(sent),
436        };
437        let result = ActionResult::new(target, answer, time::now_ms() / 1000);
438        // The class and status only: GitHub's text never reaches a log.
439        info!(
440            "{repository}: GitHub's answer to the {slug} of run {run}: {:?}, status {:?}, {:?}",
441            result.outcome, result.status, result.refusal
442        );
443        Landed::Done(result)
444    }
445
446    /// Why `issued` may not go now, if it may not: the run is not at the attempt it showed, or the config changed
447    /// since its confirmation. The config is read after the run, so nothing but the send follows the check.
448    async fn verify(&self, issued: &Issued) -> Option<Landed> {
449        let target = &issued.target;
450        let withdrawn = || {
451            Landed::Done(ActionResult::new(
452                target,
453                classify(Err(ActionFailure::Withdrawn)),
454                time::now_ms() / 1000,
455            ))
456        };
457        match self.github.run(&target.repository, target.run).await {
458            // runnerd's own re-run goes only to a run completed as `failure`, whatever its jobs said before.
459            Ok(now)
460                if Some(now.attempt) == target.attempt
461                    && issued.origin == Origin::Automatic
462                    && !(now.completed && now.failed) =>
463            {
464                return Some(Landed::Withheld(Withheld::NotFailed));
465            }
466            Ok(now) if Some(now.attempt) == target.attempt => {}
467            Ok(now) => {
468                return Some(Landed::Repreview {
469                    run: seen(now),
470                    why: Superseded::AttemptChanged,
471                });
472            }
473            Err(failure) if issued.origin == Origin::Automatic => {
474                return Some(Landed::Withheld(Withheld::Unread(failure)));
475            }
476            Err(failure) => {
477                return Some(Landed::Repreview {
478                    run: RunSeen::Unread {
479                        status: status_of(failure),
480                    },
481                    why: Superseded::Unverified,
482                });
483            }
484        }
485        (self.control.action_policy().revision() != issued.revision).then(withdrawn)
486    }
487
488    /// Why `repository`'s runner faults are not re-run, if GitHub refused one under the config of this moment
489    /// (ADR 0153): what its status shows.
490    pub fn rerun_refused(&self, repository: &str) -> Option<api::RerunRefused> {
491        self.faults
492            .held(
493                repository,
494                self.control.action_policy().revision(),
495                tokio::time::Instant::now(),
496            )
497            .map(|held| held.refused)
498    }
499
500    /// How many runs have an action in flight.
501    #[cfg(test)]
502    fn in_flight(&self) -> usize {
503        self.flights.len()
504    }
505
506    /// How many requests wait for an answer from GitHub, the lead of each flight included.
507    #[cfg(test)]
508    fn waiting(&self) -> usize {
509        self.flights.waiting()
510    }
511}
512
513/// Whether `policy` lets the owner act on `repository`'s jobs.
514fn allowed(policy: &Policy, repository: &str) -> Result<(), ActionError> {
515    policy.allows(repository).map_err(|denied| {
516        ActionError::Forbidden(match denied {
517            Denied::Off => api::actions_off().to_owned(),
518            // `allows` never says `RerunsOff`.
519            Denied::NotConfigured | Denied::RerunsOff => api::actions_not_configured(repository),
520        })
521    })
522}
523
524/// What the flight lands; if it ends without an answer, as when the runtime stops, the outcome is unknown.
525async fn wait(mut answer: watch::Receiver<Option<Landed>>, target: &ActionTarget) -> Landed {
526    let stopped = || {
527        Landed::Done(ActionResult::new(
528            target,
529            ActionAnswer::Unknown {
530                cause: UnknownCause::Stopping,
531                status: None,
532            },
533            time::now_ms() / 1000,
534        ))
535    };
536    match answer.wait_for(Option::is_some).await {
537        Ok(landed) => landed.clone().unwrap_or_else(stopped),
538        Err(_) => stopped(),
539    }
540}
541
542fn seen(now: RunNow) -> RunSeen {
543    RunSeen::Now {
544        attempt: now.attempt,
545        completed: now.completed,
546    }
547}
548
549/// The HTTP status GitHub answered with, if it did.
550fn status_of(failure: ActionFailure) -> Option<u16> {
551    match failure {
552        ActionFailure::Status { status } => Some(status),
553        ActionFailure::RateLimited { status, .. } => status,
554        _ => None,
555    }
556}
557
558fn read_failed(error: &StoreError) -> ActionError {
559    ActionError::Failed(format!("cannot read the job history: {error}"))
560}
561
562/// What GitHub's answer says, as a class and a status: never its text.
563fn classify(sent: Result<u16, ActionFailure>) -> ActionAnswer {
564    let refused = |refusal, status| ActionAnswer::Refused {
565        refusal,
566        status,
567        until: None,
568    };
569    match sent {
570        Ok(status) => ActionAnswer::Accepted { status },
571        Err(ActionFailure::NoToken) => refused(ActionRefusal::NoToken, None),
572        Err(ActionFailure::Withdrawn) => refused(ActionRefusal::Withdrawn, None),
573        // Nothing left this machine.
574        Err(ActionFailure::Unreachable) => refused(ActionRefusal::Unreachable, None),
575        Err(ActionFailure::RateLimited { status, until }) => ActionAnswer::Refused {
576            refusal: ActionRefusal::RateLimited,
577            status,
578            until: Some(
579                until
580                    .duration_since(std::time::SystemTime::UNIX_EPOCH)
581                    .unwrap_or_default()
582                    .as_secs(),
583            ),
584        },
585        Err(ActionFailure::Status { status }) => {
586            let code = Some(status);
587            match status {
588                401 => refused(ActionRefusal::Unauthorized, code),
589                403 => refused(ActionRefusal::Forbidden, code),
590                404 | 410 => refused(ActionRefusal::NotFound, code),
591                409 => refused(ActionRefusal::Conflict, code),
592                422 => refused(ActionRefusal::Unprocessable, code),
593                // A server error does not say whether GitHub acted before it failed.
594                500..=599 => ActionAnswer::Unknown {
595                    cause: UnknownCause::ServerError,
596                    status: code,
597                },
598                _ => refused(ActionRefusal::Failed, code),
599            }
600        }
601        Err(ActionFailure::TimedOut) => ActionAnswer::Unknown {
602            cause: UnknownCause::TimedOut,
603            status: None,
604        },
605        Err(ActionFailure::Broken) => ActionAnswer::Unknown {
606            cause: UnknownCause::Broken,
607            status: None,
608        },
609    }
610}
611
612#[cfg(test)]
613mod tests;