Skip to main content

runnerd_core/actions/
rerun.rs

1//! Re-runs a run whose runner failed one of its jobs, once per attempt, through the job actions' own path (ADR 0153):
2//! GitHub's re-run of the run's failed jobs, without a preview, on the engine's own GitHub client. Off unless a
3//! repository's `rerun_runner_faults` asks: a runner fault is read from the job's own output.
4//!
5//! runnerd decides to re-run a run only while GitHub says its current attempt is the one the fault happened in and it
6//! completed as `failure`, every job of that attempt that failed or was cancelled is one the runner failed, and the
7//! attempt is the first or the second; the permit reads the attempt and the run's conclusion once more right before
8//! the send. The rule binds runnerd's decision, not the request: GitHub's re-run names no attempt, so one that a
9//! person's re-run overtakes re-runs whatever attempt is current when GitHub handles it.
10//!
11//! Before the send, the attempt is claimed in the history, synced to disk; neither this daemon nor a restarted one
12//! sends a claimed attempt again. The claim goes only when nothing reached GitHub: the permit stopped it, a reload
13//! withdrew it before the permit, or GitHub refused it. A send GitHub did not answer, or that a reload stopped after
14//! the permit let it go, keeps its claim; a read of the run says whether it moved, but not who moved it, so only an
15//! accepted re-run is noted as runnerd's. Faults of a repository whose `rerun_runner_faults` or `[github] job_actions`
16//! is off, or that the config no longer has, wait unread until a reload turns them on or the run is no longer
17//! followed. A read GitHub refuses in a way asking again soon cannot change, such as a token without Actions: write,
18//! holds the repository's re-runs for an hour or until the config changes, with one attention item; a rate limit
19//! stops every run's reads until it resets. A read after an unanswered send does the same.
20//!
21//! It must not send an attempt it claimed before, decide on a run from its third attempt on, send a held repository's
22//! re-run, send while the rate limit answered, read GitHub for a run more than once a minute, or write the history
23//! beyond its claims and the note of a re-run GitHub accepted.
24
25use std::time::Duration;
26
27use tokio::time::Instant;
28
29use tokio::sync::broadcast::error::RecvError;
30use tracing::{info, warn};
31
32use super::faults::{Due, RunKey};
33use super::flights::{Boarding, Issued, Landed, Origin, Withheld};
34use super::{JobActions, classify, wait};
35use crate::api::{
36    ActionAnswer, ActionOutcome, ActionRefusal, ActionResult, ActionTarget, Event, JobAction,
37    OutcomeKind, RerunRefused, RunSeen, Superseded,
38};
39use crate::github::{ActionFailure, AttemptJob};
40use crate::store::Claim;
41use crate::time;
42
43/// How far back a start reads the history's faults: as long as a run is followed.
44const LOOKBACK: Duration = super::faults::FOLLOWED;
45
46/// How long until a run that has not completed, or a run busy with another action, is read again.
47const POLL: Duration = Duration::from_secs(60);
48
49/// How long until a check that failed is made again.
50const BACKOFF: Duration = Duration::from_secs(120);
51
52/// How long every read waits after a rate limit GitHub answered without saying when it resets.
53const COOLDOWN: Duration = Duration::from_secs(60);
54
55/// The attempts a fault is re-run from: the first and the second, so a run reaches at most its third by runnerd.
56const ATTEMPTS_MAX: u32 = 2;
57
58/// What one check of a run came to.
59enum Checked {
60    /// Nothing more to do for the attempt.
61    Settled,
62    /// Check again this long after the check ended, or sooner if another fault of the attempt comes; no failure.
63    Wait(Duration),
64    /// Check again at this time and no sooner: a hold or a rate limit.
65    Hold(Instant),
66    /// The check failed; check again later, a few times.
67    Failed,
68}
69
70impl JobActions {
71    /// Follows the history's runner faults and re-runs their runs, until a stop. Spawned once by the engine.
72    pub async fn rerun_faults(self) {
73        let mut events = self.store.subscribe();
74        self.read_faults(Instant::now()).await;
75        loop {
76            let next = self.faults.next();
77            let due = async {
78                match next {
79                    Some(at) => tokio::time::sleep_until(at).await,
80                    None => std::future::pending().await,
81                }
82            };
83            tokio::select! {
84                biased;
85                () = self.control.stop_asked() => return,
86                event = events.recv() => match event {
87                    Ok(envelope) => {
88                        if let Event::JobFinished { job } = &envelope.event
89                            && job.outcome().kind == OutcomeKind::RunnerFault
90                        {
91                            self.faults.seen(
92                                RunKey { repository: job.repository.clone(), run: job.run_id },
93                                job.run_attempt,
94                                job.id,
95                                Instant::now(),
96                            );
97                        }
98                    }
99                    // Events were missed: the history has the faults among them.
100                    Err(RecvError::Lagged(_)) => self.read_faults(Instant::now()).await,
101                    Err(RecvError::Closed) => return,
102                },
103                () = due => self.pass(Instant::now()).await,
104            }
105        }
106    }
107
108    /// Checks every run whose check is due at `now`, one after the other. Each next check is timed from when its own
109    /// ended, and a rate limit GitHub answered stops the rest of the pass.
110    pub(super) async fn pass(&self, now: Instant) {
111        // The pass's `now`, or later once its checks took time.
112        let observed = || now.max(Instant::now());
113        for due in self.faults.due(now) {
114            if let Some(until) = self.faults.cooling(observed()) {
115                self.faults.hold(&due, until);
116                continue;
117            }
118            match self.check(&due, now).await {
119                Checked::Settled => self.faults.settle(&due),
120                Checked::Wait(after) => self.faults.wait(&due, observed() + after),
121                Checked::Hold(at) => self.faults.hold(&due, at),
122                Checked::Failed => {
123                    if !self.faults.failed(&due, observed() + BACKOFF) {
124                        warn!(
125                            "{}: gave up re-running run {} after {} failed checks",
126                            due.key.repository,
127                            due.key.run,
128                            super::faults::TRIES_MAX
129                        );
130                    }
131                }
132            }
133        }
134    }
135
136    /// Follows the faults the history has from the last [`LOOKBACK`] that runnerd has not re-run.
137    pub(super) async fn read_faults(&self, now: Instant) {
138        let since =
139            time::now_ms().saturating_sub(LOOKBACK.as_millis().try_into().unwrap_or(u64::MAX));
140        match self.store.runner_faults(since).await {
141            Ok(faults) => {
142                for fault in faults {
143                    self.faults.seen(
144                        RunKey {
145                            repository: fault.repository,
146                            run: fault.run,
147                        },
148                        fault.attempt,
149                        fault.job,
150                        now,
151                    );
152                }
153            }
154            Err(error) => warn!("cannot read the runner faults to re-run: {error}"),
155        }
156    }
157
158    /// Checks `due`'s run at `now` and re-runs its failed jobs if the rule says so.
159    async fn check(&self, due: &Due, now: Instant) -> Checked {
160        let RunKey { repository, run } = &due.key;
161        let (repository, run) = (repository.as_str(), *run);
162        if due.attempt > ATTEMPTS_MAX {
163            info!(
164                "{repository}: not re-running run {run}: its runner failed attempt {}, and runnerd re-runs only the \
165                 first {ATTEMPTS_MAX}",
166                due.attempt
167            );
168            return Checked::Settled;
169        }
170        let policy = self.control.action_policy();
171        // Off, or not configured: a reload may turn it on again while the run is followed. Nothing is read meanwhile.
172        if policy.reruns(repository).is_err() {
173            return Checked::Wait(POLL);
174        }
175        if let Some(held) = self.faults.held(repository, policy.revision(), now) {
176            return Checked::Hold(held.until);
177        }
178        match self.read(self.github.run(repository, run)).await {
179            Ok(read) if read.attempt != due.attempt => {
180                info!(
181                    "{repository}: not re-running run {run}: it is at attempt {} already",
182                    read.attempt
183                );
184                return Checked::Settled;
185            }
186            Ok(read) if !read.completed => return Checked::Wait(POLL),
187            Ok(read) if !read.failed => {
188                info!("{repository}: not re-running run {run}: it did not complete as a failure");
189                return Checked::Settled;
190            }
191            Ok(_) => {}
192            Err(failure) => return self.refused(due, failure, policy.revision()),
193        }
194        let jobs = match self
195            .read(self.github.attempt_jobs(repository, run, due.attempt))
196            .await
197        {
198            Ok(jobs) => jobs,
199            Err(failure) => return self.refused(due, failure, policy.revision()),
200        };
201        let faulted = match self.faulted(&jobs).await {
202            Ok(faulted) => faulted,
203            Err(Unfit::Unfinished) => return Checked::Wait(POLL),
204            Err(Unfit::Other(job)) => {
205                info!(
206                    "{repository}: not re-running run {run}: job {job} of attempt {} failed or was cancelled, and \
207                     not by its runner",
208                    due.attempt
209                );
210                return Checked::Settled;
211            }
212            Err(Unfit::Unread) => return Checked::Failed,
213        };
214        if faulted.is_empty() {
215            return Checked::Settled;
216        }
217        self.send_rerun(due, &faulted, policy.revision()).await
218    }
219
220    /// Sends the re-run of `due`'s run, whose runner failed `faulted`, under config revision `revision`, and what
221    /// GitHub's answer comes to.
222    async fn send_rerun(&self, due: &Due, faulted: &[u64], revision: u64) -> Checked {
223        let RunKey { repository, run } = &due.key;
224        match self
225            .store
226            .claim_rerun(repository, *run, due.attempt, time::now_ms())
227            .await
228        {
229            Claim::Claimed => {}
230            Claim::Taken => {
231                info!(
232                    "{repository}: not re-running run {run}: runnerd sent the re-run of attempt {} already",
233                    due.attempt
234                );
235                return Checked::Settled;
236            }
237            Claim::Unstored => {
238                warn!(
239                    "{repository}: cannot claim the re-run of run {run} in the history; not sent"
240                );
241                return Checked::Failed;
242            }
243        }
244        self.send_claimed(due, faulted, revision).await
245    }
246
247    /// Sends the re-run `send_rerun` claimed, and releases the claim if nothing of it reached GitHub.
248    async fn send_claimed(&self, due: &Due, faulted: &[u64], revision: u64) -> Checked {
249        let RunKey { repository, run } = &due.key;
250        let release = || self.store.release_rerun(repository, *run, due.attempt);
251        let issued = Issued {
252            target: ActionTarget {
253                action: JobAction::RerunFailed,
254                job: due.job,
255                name: String::new(),
256                repository: repository.clone(),
257                run: *run,
258                attempt: Some(due.attempt),
259            },
260            revision,
261            origin: Origin::Automatic,
262        };
263        let answer = match self.flights.board(&issued) {
264            Boarding::Busy(_) => {
265                release();
266                return Checked::Wait(POLL);
267            }
268            Boarding::Follow(answer) => answer,
269            Boarding::Lead { landing, answer } => {
270                let landed = self.dispatch(&issued).await;
271                landing.land(landed);
272                answer
273            }
274        };
275        let landed = wait(answer, &issued.target).await;
276        // Only an answer that may mean GitHub acted keeps the claim.
277        if !matches!(&landed, Landed::Done(result) if matches!(result.outcome, ActionOutcome::Accepted | ActionOutcome::Unknown))
278        {
279            release();
280        }
281        let result = match landed {
282            Landed::Done(result) => result,
283            Landed::Repreview {
284                why: Superseded::AttemptChanged,
285                run: seen,
286            } => {
287                let attempt = match seen {
288                    RunSeen::Now { attempt, .. } => attempt.to_string(),
289                    RunSeen::Unread { .. } => "a later one".to_owned(),
290                };
291                info!(
292                    "{repository}: not re-running run {run}: it moved to attempt {attempt} before the send"
293                );
294                return Checked::Settled;
295            }
296            Landed::Withheld(Withheld::NotFailed) => {
297                info!("{repository}: not re-running run {run}: it is no longer a failed run");
298                return Checked::Settled;
299            }
300            Landed::Withheld(Withheld::Unread(failure)) => {
301                return self.refused(due, failure, revision);
302            }
303            Landed::Repreview { .. } => return Checked::Failed,
304        };
305        self.landed(due, faulted, &result, revision).await
306    }
307
308    /// What a read GitHub failed or refused comes to: a refusal asking again soon cannot change holds the repository,
309    /// a rate limit stops every read until it resets, anything else is a failed check.
310    fn refused(&self, due: &Due, failure: ActionFailure, revision: u64) -> Checked {
311        let RunKey { repository, run } = &due.key;
312        let answer = classify(Err(failure));
313        match answer {
314            ActionAnswer::Refused {
315                refusal:
316                    ActionRefusal::NoToken
317                    | ActionRefusal::Unauthorized
318                    | ActionRefusal::Forbidden
319                    | ActionRefusal::NotFound,
320                ..
321            } => {
322                let refused = RerunRefused::unread(repository, *run, answer, time::now_ms() / 1000);
323                self.hold_repository(repository, *run, refused, revision)
324            }
325            ActionAnswer::Refused {
326                refusal: ActionRefusal::RateLimited,
327                until,
328                ..
329            } => self.cool(until),
330            ActionAnswer::Refused {
331                status: Some(429), ..
332            } => self.cool(None),
333            _ => Checked::Failed,
334        }
335    }
336
337    /// Holds `repository`'s re-runs for the hour, as `refused` says.
338    fn hold_repository(
339        &self,
340        repository: &str,
341        run: u64,
342        refused: RerunRefused,
343        revision: u64,
344    ) -> Checked {
345        // The class and status only: GitHub's text never reaches a log.
346        warn!(
347            "{repository}: GitHub refused runnerd's automatic re-run of run {run} ({:?}, status {:?}); its runner \
348             faults wait an hour",
349            refused.refusal, refused.status
350        );
351        Checked::Hold(
352            self.faults
353                .refuse(repository, revision, refused, Instant::now()),
354        )
355    }
356
357    /// Stops every read until the rate limit that resets at `until`, seconds since the epoch, does; [`COOLDOWN`] when
358    /// it does not say.
359    fn cool(&self, until: Option<u64>) -> Checked {
360        let epoch = time::now_ms() / 1000;
361        let wait = until.map_or(COOLDOWN, |until| {
362            Duration::from_secs(until.saturating_sub(epoch)).max(Duration::from_secs(1))
363        });
364        let until = Instant::now() + wait;
365        self.faults.cool(until);
366        Checked::Hold(until)
367    }
368
369    /// What `result`, GitHub's answer to the re-run of `due`'s run, comes to.
370    async fn landed(
371        &self,
372        due: &Due,
373        faulted: &[u64],
374        result: &ActionResult,
375        revision: u64,
376    ) -> Checked {
377        let RunKey { repository, run } = &due.key;
378        match (result.outcome, result.refusal) {
379            (ActionOutcome::Accepted, _) => {
380                self.noted(due, faulted);
381                Checked::Settled
382            }
383            // GitHub did not say: the claim stays, so it is never sent again, as it may land yet. A read says whether
384            // the run moved, not who moved it, so nothing is noted as runnerd's; a refused read holds or cools as
385            // any other.
386            (ActionOutcome::Unknown, _) => {
387                match self.read(self.github.run(repository, *run)).await {
388                    Ok(read) if read.attempt > due.attempt => info!(
389                        "{repository}: run {run} moved to attempt {} after a re-run GitHub did not answer; it may be \
390                         runnerd's or a person's",
391                        read.attempt
392                    ),
393                    Ok(_) => warn!(
394                        "{repository}: GitHub did not say whether it re-ran run {run} (status {:?}); runnerd does not \
395                         ask again",
396                        result.status
397                    ),
398                    Err(failure) => {
399                        warn!(
400                            "{repository}: GitHub did not say whether it re-ran run {run}, nor answered a read of it; \
401                             runnerd does not ask again"
402                        );
403                        let _ = self.refused(due, failure, revision);
404                    }
405                }
406                Checked::Settled
407            }
408            (ActionOutcome::Refused, Some(ActionRefusal::RateLimited)) => self.cool(result.until),
409            (ActionOutcome::Refused, _) if result.status == Some(429) => self.cool(None),
410            (
411                ActionOutcome::Refused,
412                Some(
413                    ActionRefusal::NoToken
414                    | ActionRefusal::Unauthorized
415                    | ActionRefusal::Forbidden
416                    | ActionRefusal::NotFound,
417                ),
418            ) => {
419                let refused = RerunRefused::new(result, time::now_ms() / 1000);
420                self.hold_repository(repository, *run, refused, revision)
421            }
422            (ActionOutcome::Refused, Some(ActionRefusal::Unprocessable)) => {
423                warn!("{repository}: GitHub cannot re-run run {run} (422); left to its owner");
424                Checked::Settled
425            }
426            // A reload withdrew it: the next check reads the config again.
427            (ActionOutcome::Refused, Some(ActionRefusal::Withdrawn)) => Checked::Wait(POLL),
428            // A conflict, or a connection that failed before anything was sent: the next check reads the config and
429            // the run's attempt again before any send.
430            _ => Checked::Failed,
431        }
432    }
433
434    /// Notes that GitHub re-ran `due`'s run, whose runner failed `faulted`.
435    fn noted(&self, due: &Due, faulted: &[u64]) {
436        let RunKey { repository, run } = &due.key;
437        info!(
438            "{repository}: re-ran the failed jobs of run {run} automatically, as attempt {}: its runner failed job {}",
439            due.attempt + 1,
440            due.job
441        );
442        self.store.reran(faulted.to_vec(), time::now_ms());
443    }
444
445    /// The jobs of `jobs` the runner failed, if every one that failed or was cancelled is one.
446    async fn faulted(&self, jobs: &[AttemptJob]) -> Result<Vec<u64>, Unfit> {
447        let mut faulted = Vec::new();
448        for job in jobs {
449            if job.status != "completed" {
450                return Err(Unfit::Unfinished);
451            }
452            if matches!(
453                job.conclusion.as_deref(),
454                Some("success" | "skipped" | "neutral")
455            ) {
456                continue;
457            }
458            let summary = match self.store.job(job.id).await {
459                Ok(Some(found)) => found.summary,
460                Ok(None) => return Err(Unfit::Other(job.id)),
461                Err(error) => {
462                    warn!("cannot read job {} from the history: {error}", job.id);
463                    return Err(Unfit::Unread);
464                }
465            };
466            // The history may not have closed a job GitHub completed yet: its outcome comes once it has.
467            if !summary.state.is_final() {
468                return Err(Unfit::Unfinished);
469            }
470            if summary.outcome().kind != OutcomeKind::RunnerFault {
471                return Err(Unfit::Other(job.id));
472            }
473            faulted.push(job.id);
474        }
475        Ok(faulted)
476    }
477
478    /// `read`, within the deadline and unless a stop comes first.
479    async fn read<T>(
480        &self,
481        read: impl Future<Output = Result<T, ActionFailure>>,
482    ) -> Result<T, ActionFailure> {
483        tokio::select! {
484            biased;
485            () = self.control.stop_asked() => Err(ActionFailure::TimedOut),
486            read = tokio::time::timeout(self.deadline, read) => read.unwrap_or(Err(ActionFailure::TimedOut)),
487        }
488    }
489}
490
491/// Why an attempt's failed jobs are not all runner faults now.
492enum Unfit {
493    /// A job has not completed, on GitHub or in the history.
494    Unfinished,
495    /// This job failed or was cancelled, and not by its runner, or not on this machine.
496    Other(u64),
497    /// The history could not be read.
498    Unread,
499}