1use std::time::Duration;
26
27use tokio::time::Instant;
28
29use tokio::sync::broadcast::error::RecvError;
30use tracing::{info, warn};
31
32use super::faults::{Due, RunKey};
33use super::flights::{Boarding, Issued, Landed, Origin, Withheld};
34use super::{JobActions, classify, wait};
35use crate::api::{
36 ActionAnswer, ActionOutcome, ActionRefusal, ActionResult, ActionTarget, Event, JobAction,
37 OutcomeKind, RerunRefused, RunSeen, Superseded,
38};
39use crate::github::{ActionFailure, AttemptJob};
40use crate::store::Claim;
41use crate::time;
42
43const LOOKBACK: Duration = super::faults::FOLLOWED;
45
46const POLL: Duration = Duration::from_secs(60);
48
49const BACKOFF: Duration = Duration::from_secs(120);
51
52const COOLDOWN: Duration = Duration::from_secs(60);
54
55const ATTEMPTS_MAX: u32 = 2;
57
58enum Checked {
60 Settled,
62 Wait(Duration),
64 Hold(Instant),
66 Failed,
68}
69
70impl JobActions {
71 pub async fn rerun_faults(self) {
73 let mut events = self.store.subscribe();
74 self.read_faults(Instant::now()).await;
75 loop {
76 let next = self.faults.next();
77 let due = async {
78 match next {
79 Some(at) => tokio::time::sleep_until(at).await,
80 None => std::future::pending().await,
81 }
82 };
83 tokio::select! {
84 biased;
85 () = self.control.stop_asked() => return,
86 event = events.recv() => match event {
87 Ok(envelope) => {
88 if let Event::JobFinished { job } = &envelope.event
89 && job.outcome().kind == OutcomeKind::RunnerFault
90 {
91 self.faults.seen(
92 RunKey { repository: job.repository.clone(), run: job.run_id },
93 job.run_attempt,
94 job.id,
95 Instant::now(),
96 );
97 }
98 }
99 Err(RecvError::Lagged(_)) => self.read_faults(Instant::now()).await,
101 Err(RecvError::Closed) => return,
102 },
103 () = due => self.pass(Instant::now()).await,
104 }
105 }
106 }
107
108 pub(super) async fn pass(&self, now: Instant) {
111 let observed = || now.max(Instant::now());
113 for due in self.faults.due(now) {
114 if let Some(until) = self.faults.cooling(observed()) {
115 self.faults.hold(&due, until);
116 continue;
117 }
118 match self.check(&due, now).await {
119 Checked::Settled => self.faults.settle(&due),
120 Checked::Wait(after) => self.faults.wait(&due, observed() + after),
121 Checked::Hold(at) => self.faults.hold(&due, at),
122 Checked::Failed => {
123 if !self.faults.failed(&due, observed() + BACKOFF) {
124 warn!(
125 "{}: gave up re-running run {} after {} failed checks",
126 due.key.repository,
127 due.key.run,
128 super::faults::TRIES_MAX
129 );
130 }
131 }
132 }
133 }
134 }
135
136 pub(super) async fn read_faults(&self, now: Instant) {
138 let since =
139 time::now_ms().saturating_sub(LOOKBACK.as_millis().try_into().unwrap_or(u64::MAX));
140 match self.store.runner_faults(since).await {
141 Ok(faults) => {
142 for fault in faults {
143 self.faults.seen(
144 RunKey {
145 repository: fault.repository,
146 run: fault.run,
147 },
148 fault.attempt,
149 fault.job,
150 now,
151 );
152 }
153 }
154 Err(error) => warn!("cannot read the runner faults to re-run: {error}"),
155 }
156 }
157
158 async fn check(&self, due: &Due, now: Instant) -> Checked {
160 let RunKey { repository, run } = &due.key;
161 let (repository, run) = (repository.as_str(), *run);
162 if due.attempt > ATTEMPTS_MAX {
163 info!(
164 "{repository}: not re-running run {run}: its runner failed attempt {}, and runnerd re-runs only the \
165 first {ATTEMPTS_MAX}",
166 due.attempt
167 );
168 return Checked::Settled;
169 }
170 let policy = self.control.action_policy();
171 if policy.reruns(repository).is_err() {
173 return Checked::Wait(POLL);
174 }
175 if let Some(held) = self.faults.held(repository, policy.revision(), now) {
176 return Checked::Hold(held.until);
177 }
178 match self.read(self.github.run(repository, run)).await {
179 Ok(read) if read.attempt != due.attempt => {
180 info!(
181 "{repository}: not re-running run {run}: it is at attempt {} already",
182 read.attempt
183 );
184 return Checked::Settled;
185 }
186 Ok(read) if !read.completed => return Checked::Wait(POLL),
187 Ok(read) if !read.failed => {
188 info!("{repository}: not re-running run {run}: it did not complete as a failure");
189 return Checked::Settled;
190 }
191 Ok(_) => {}
192 Err(failure) => return self.refused(due, failure, policy.revision()),
193 }
194 let jobs = match self
195 .read(self.github.attempt_jobs(repository, run, due.attempt))
196 .await
197 {
198 Ok(jobs) => jobs,
199 Err(failure) => return self.refused(due, failure, policy.revision()),
200 };
201 let faulted = match self.faulted(&jobs).await {
202 Ok(faulted) => faulted,
203 Err(Unfit::Unfinished) => return Checked::Wait(POLL),
204 Err(Unfit::Other(job)) => {
205 info!(
206 "{repository}: not re-running run {run}: job {job} of attempt {} failed or was cancelled, and \
207 not by its runner",
208 due.attempt
209 );
210 return Checked::Settled;
211 }
212 Err(Unfit::Unread) => return Checked::Failed,
213 };
214 if faulted.is_empty() {
215 return Checked::Settled;
216 }
217 self.send_rerun(due, &faulted, policy.revision()).await
218 }
219
220 async fn send_rerun(&self, due: &Due, faulted: &[u64], revision: u64) -> Checked {
223 let RunKey { repository, run } = &due.key;
224 match self
225 .store
226 .claim_rerun(repository, *run, due.attempt, time::now_ms())
227 .await
228 {
229 Claim::Claimed => {}
230 Claim::Taken => {
231 info!(
232 "{repository}: not re-running run {run}: runnerd sent the re-run of attempt {} already",
233 due.attempt
234 );
235 return Checked::Settled;
236 }
237 Claim::Unstored => {
238 warn!(
239 "{repository}: cannot claim the re-run of run {run} in the history; not sent"
240 );
241 return Checked::Failed;
242 }
243 }
244 self.send_claimed(due, faulted, revision).await
245 }
246
247 async fn send_claimed(&self, due: &Due, faulted: &[u64], revision: u64) -> Checked {
249 let RunKey { repository, run } = &due.key;
250 let release = || self.store.release_rerun(repository, *run, due.attempt);
251 let issued = Issued {
252 target: ActionTarget {
253 action: JobAction::RerunFailed,
254 job: due.job,
255 name: String::new(),
256 repository: repository.clone(),
257 run: *run,
258 attempt: Some(due.attempt),
259 },
260 revision,
261 origin: Origin::Automatic,
262 };
263 let answer = match self.flights.board(&issued) {
264 Boarding::Busy(_) => {
265 release();
266 return Checked::Wait(POLL);
267 }
268 Boarding::Follow(answer) => answer,
269 Boarding::Lead { landing, answer } => {
270 let landed = self.dispatch(&issued).await;
271 landing.land(landed);
272 answer
273 }
274 };
275 let landed = wait(answer, &issued.target).await;
276 if !matches!(&landed, Landed::Done(result) if matches!(result.outcome, ActionOutcome::Accepted | ActionOutcome::Unknown))
278 {
279 release();
280 }
281 let result = match landed {
282 Landed::Done(result) => result,
283 Landed::Repreview {
284 why: Superseded::AttemptChanged,
285 run: seen,
286 } => {
287 let attempt = match seen {
288 RunSeen::Now { attempt, .. } => attempt.to_string(),
289 RunSeen::Unread { .. } => "a later one".to_owned(),
290 };
291 info!(
292 "{repository}: not re-running run {run}: it moved to attempt {attempt} before the send"
293 );
294 return Checked::Settled;
295 }
296 Landed::Withheld(Withheld::NotFailed) => {
297 info!("{repository}: not re-running run {run}: it is no longer a failed run");
298 return Checked::Settled;
299 }
300 Landed::Withheld(Withheld::Unread(failure)) => {
301 return self.refused(due, failure, revision);
302 }
303 Landed::Repreview { .. } => return Checked::Failed,
304 };
305 self.landed(due, faulted, &result, revision).await
306 }
307
308 fn refused(&self, due: &Due, failure: ActionFailure, revision: u64) -> Checked {
311 let RunKey { repository, run } = &due.key;
312 let answer = classify(Err(failure));
313 match answer {
314 ActionAnswer::Refused {
315 refusal:
316 ActionRefusal::NoToken
317 | ActionRefusal::Unauthorized
318 | ActionRefusal::Forbidden
319 | ActionRefusal::NotFound,
320 ..
321 } => {
322 let refused = RerunRefused::unread(repository, *run, answer, time::now_ms() / 1000);
323 self.hold_repository(repository, *run, refused, revision)
324 }
325 ActionAnswer::Refused {
326 refusal: ActionRefusal::RateLimited,
327 until,
328 ..
329 } => self.cool(until),
330 ActionAnswer::Refused {
331 status: Some(429), ..
332 } => self.cool(None),
333 _ => Checked::Failed,
334 }
335 }
336
337 fn hold_repository(
339 &self,
340 repository: &str,
341 run: u64,
342 refused: RerunRefused,
343 revision: u64,
344 ) -> Checked {
345 warn!(
347 "{repository}: GitHub refused runnerd's automatic re-run of run {run} ({:?}, status {:?}); its runner \
348 faults wait an hour",
349 refused.refusal, refused.status
350 );
351 Checked::Hold(
352 self.faults
353 .refuse(repository, revision, refused, Instant::now()),
354 )
355 }
356
357 fn cool(&self, until: Option<u64>) -> Checked {
360 let epoch = time::now_ms() / 1000;
361 let wait = until.map_or(COOLDOWN, |until| {
362 Duration::from_secs(until.saturating_sub(epoch)).max(Duration::from_secs(1))
363 });
364 let until = Instant::now() + wait;
365 self.faults.cool(until);
366 Checked::Hold(until)
367 }
368
369 async fn landed(
371 &self,
372 due: &Due,
373 faulted: &[u64],
374 result: &ActionResult,
375 revision: u64,
376 ) -> Checked {
377 let RunKey { repository, run } = &due.key;
378 match (result.outcome, result.refusal) {
379 (ActionOutcome::Accepted, _) => {
380 self.noted(due, faulted);
381 Checked::Settled
382 }
383 (ActionOutcome::Unknown, _) => {
387 match self.read(self.github.run(repository, *run)).await {
388 Ok(read) if read.attempt > due.attempt => info!(
389 "{repository}: run {run} moved to attempt {} after a re-run GitHub did not answer; it may be \
390 runnerd's or a person's",
391 read.attempt
392 ),
393 Ok(_) => warn!(
394 "{repository}: GitHub did not say whether it re-ran run {run} (status {:?}); runnerd does not \
395 ask again",
396 result.status
397 ),
398 Err(failure) => {
399 warn!(
400 "{repository}: GitHub did not say whether it re-ran run {run}, nor answered a read of it; \
401 runnerd does not ask again"
402 );
403 let _ = self.refused(due, failure, revision);
404 }
405 }
406 Checked::Settled
407 }
408 (ActionOutcome::Refused, Some(ActionRefusal::RateLimited)) => self.cool(result.until),
409 (ActionOutcome::Refused, _) if result.status == Some(429) => self.cool(None),
410 (
411 ActionOutcome::Refused,
412 Some(
413 ActionRefusal::NoToken
414 | ActionRefusal::Unauthorized
415 | ActionRefusal::Forbidden
416 | ActionRefusal::NotFound,
417 ),
418 ) => {
419 let refused = RerunRefused::new(result, time::now_ms() / 1000);
420 self.hold_repository(repository, *run, refused, revision)
421 }
422 (ActionOutcome::Refused, Some(ActionRefusal::Unprocessable)) => {
423 warn!("{repository}: GitHub cannot re-run run {run} (422); left to its owner");
424 Checked::Settled
425 }
426 (ActionOutcome::Refused, Some(ActionRefusal::Withdrawn)) => Checked::Wait(POLL),
428 _ => Checked::Failed,
431 }
432 }
433
434 fn noted(&self, due: &Due, faulted: &[u64]) {
436 let RunKey { repository, run } = &due.key;
437 info!(
438 "{repository}: re-ran the failed jobs of run {run} automatically, as attempt {}: its runner failed job {}",
439 due.attempt + 1,
440 due.job
441 );
442 self.store.reran(faulted.to_vec(), time::now_ms());
443 }
444
445 async fn faulted(&self, jobs: &[AttemptJob]) -> Result<Vec<u64>, Unfit> {
447 let mut faulted = Vec::new();
448 for job in jobs {
449 if job.status != "completed" {
450 return Err(Unfit::Unfinished);
451 }
452 if matches!(
453 job.conclusion.as_deref(),
454 Some("success" | "skipped" | "neutral")
455 ) {
456 continue;
457 }
458 let summary = match self.store.job(job.id).await {
459 Ok(Some(found)) => found.summary,
460 Ok(None) => return Err(Unfit::Other(job.id)),
461 Err(error) => {
462 warn!("cannot read job {} from the history: {error}", job.id);
463 return Err(Unfit::Unread);
464 }
465 };
466 if !summary.state.is_final() {
468 return Err(Unfit::Unfinished);
469 }
470 if summary.outcome().kind != OutcomeKind::RunnerFault {
471 return Err(Unfit::Other(job.id));
472 }
473 faulted.push(job.id);
474 }
475 Ok(faulted)
476 }
477
478 async fn read<T>(
480 &self,
481 read: impl Future<Output = Result<T, ActionFailure>>,
482 ) -> Result<T, ActionFailure> {
483 tokio::select! {
484 biased;
485 () = self.control.stop_asked() => Err(ActionFailure::TimedOut),
486 read = tokio::time::timeout(self.deadline, read) => read.unwrap_or(Err(ActionFailure::TimedOut)),
487 }
488 }
489}
490
491enum Unfit {
493 Unfinished,
495 Other(u64),
497 Unread,
499}