1mod faults;
17mod flights;
18pub mod policy;
19mod rerun;
20
21use std::sync::Arc;
22use std::time::{Duration, Instant};
23
24use futures_util::future::BoxFuture;
25use tokio::sync::watch;
26use tracing::info;
27
28use self::faults::Faults;
29use self::flights::{Boarding, Flights, Issued, Landed, Origin, Redeemed, Withheld};
30use self::policy::{Denied, Policy};
31use crate::api::{
32 self, ActionAnswer, ActionPreview, ActionRefusal, ActionResult, ActionTarget, JobAction,
33 JobSummary, PreviewFacts, RunSeen, SIBLINGS_MAX, Superseded, UnknownCause,
34};
35use crate::github::{ActionFailure, AttemptJob, GithubApi, JobActionRequest, Permit, RunNow};
36use crate::scheduler::engine::{Control, Shutdown};
37use crate::store::{Store, StoreError};
38use crate::time;
39
40pub const DEADLINE: Duration = Duration::from_secs(20);
44
45#[derive(Debug, Clone, Copy)]
47pub struct Limits {
48 pub deadline: Duration,
50 pub confirmation: Duration,
52}
53
54pub const LIMITS: Limits = Limits {
56 deadline: DEADLINE,
57 confirmation: Duration::from_secs(api::CONFIRM_SECS),
58};
59
60pub trait ActionsGithub: Send + Sync + 'static {
63 fn run<'call>(
65 &'call self,
66 repository: &'call str,
67 run: u64,
68 ) -> BoxFuture<'call, Result<RunNow, ActionFailure>>;
69
70 fn attempt_jobs<'call>(
72 &'call self,
73 repository: &'call str,
74 run: u64,
75 attempt: u32,
76 ) -> BoxFuture<'call, Result<Vec<AttemptJob>, ActionFailure>>;
77
78 fn send<'call>(
81 &'call self,
82 repository: &'call str,
83 request: JobActionRequest,
84 permit: &'call Permit<'call>,
85 ) -> BoxFuture<'call, Result<u16, ActionFailure>>;
86}
87
88impl<G: GithubApi + 'static> ActionsGithub for G {
89 fn run<'call>(
90 &'call self,
91 repository: &'call str,
92 run: u64,
93 ) -> BoxFuture<'call, Result<RunNow, ActionFailure>> {
94 Box::pin(GithubApi::action_run(self, repository, run))
95 }
96
97 fn attempt_jobs<'call>(
98 &'call self,
99 repository: &'call str,
100 run: u64,
101 attempt: u32,
102 ) -> BoxFuture<'call, Result<Vec<AttemptJob>, ActionFailure>> {
103 Box::pin(GithubApi::attempt_jobs(self, repository, run, attempt))
104 }
105
106 fn send<'call>(
107 &'call self,
108 repository: &'call str,
109 request: JobActionRequest,
110 permit: &'call Permit<'call>,
111 ) -> BoxFuture<'call, Result<u16, ActionFailure>> {
112 Box::pin(GithubApi::job_action(self, repository, request, permit))
113 }
114}
115
116#[derive(Debug)]
118pub enum Asked {
119 Preview(Box<ActionPreview>),
121 Done(ActionResult),
123}
124
125#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
127pub enum ActionError {
128 #[error("{0}")]
130 NotFound(String),
131 #[error("{0}")]
133 Forbidden(String),
134 #[error("{0}")]
136 Invalid(String),
137 #[error("{0}")]
139 Busy(String),
140 #[error("{0}")]
142 Failed(String),
143 #[error("runnerd is stopping")]
145 Stopped,
146}
147
148#[derive(Clone)]
150pub struct JobActions {
151 github: Arc<dyn ActionsGithub>,
152 store: Store,
153 control: Arc<Control>,
154 flights: Arc<Flights>,
155 faults: Arc<Faults>,
156 deadline: Duration,
157}
158
159impl std::fmt::Debug for JobActions {
160 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
161 f.debug_struct("JobActions").finish_non_exhaustive()
162 }
163}
164
165impl JobActions {
166 pub fn new(
169 github: Arc<dyn ActionsGithub>,
170 store: Store,
171 control: Arc<Control>,
172 limits: Limits,
173 ) -> JobActions {
174 JobActions {
175 github,
176 store,
177 control,
178 flights: Arc::new(Flights::new(limits.confirmation)),
179 faults: Arc::new(Faults::default()),
180 deadline: limits.deadline,
181 }
182 }
183
184 pub async fn ask(
188 &self,
189 job: u64,
190 action: JobAction,
191 confirm: Option<&str>,
192 ) -> Result<Asked, ActionError> {
193 if action == JobAction::Unsupported {
194 return Err(ActionError::Invalid("no such job action".to_owned()));
195 }
196 if self.control.mode() != Shutdown::No {
197 return Err(ActionError::Stopped);
198 }
199 let summary = self
200 .store
201 .job(job)
202 .await
203 .map_err(|error| read_failed(&error))?
204 .ok_or_else(|| ActionError::NotFound(format!("no job {job} in the history")))?
205 .summary;
206 if summary.run_id == 0 {
207 return Err(ActionError::Invalid(api::actions_no_run(job)));
208 }
209 let policy = self.control.action_policy();
212 allowed(&policy, &summary.repository)?;
213 let superseded = match confirm {
214 None => None,
215 Some(token) => {
216 match self
217 .flights
218 .redeem(token, action, job, policy.revision(), Instant::now())
219 {
220 Redeemed::Board { boarding, issued } => {
221 return self.board(boarding, issued, &summary).await;
222 }
223 Redeemed::Stale(why) => Some(why),
224 }
225 }
226 };
227 let run = self.read_run(&summary.repository, summary.run_id).await;
228 self.preview(&summary, action, run, superseded, &policy)
229 .await
230 }
231
232 async fn preview(
235 &self,
236 summary: &JobSummary,
237 action: JobAction,
238 run: RunSeen,
239 superseded: Option<Superseded>,
240 policy: &Policy,
241 ) -> Result<Asked, ActionError> {
242 let others = self
243 .store
244 .run_jobs(
245 &summary.repository,
246 summary.run_id,
247 summary.id,
248 SIBLINGS_MAX,
249 )
250 .await
251 .map_err(|error| read_failed(&error))?;
252 let mut target = ActionTarget {
253 action,
254 job: summary.id,
255 name: summary.name.clone(),
256 repository: summary.repository.clone(),
257 run: summary.run_id,
258 attempt: None,
259 };
260 let confirm = match run {
261 RunSeen::Now { attempt, .. } => {
262 target.attempt = Some(attempt);
263 let issued = Issued {
264 target: target.clone(),
265 revision: policy.revision(),
266 origin: Origin::Owner,
267 };
268 Some(
269 self.flights
270 .issue(issued, Instant::now())
271 .map_err(|error| {
272 ActionError::Failed(format!("cannot make a confirmation: {error}"))
273 })?,
274 )
275 }
276 RunSeen::Unread { .. } => None,
277 };
278 Ok(Asked::Preview(Box::new(ActionPreview::new(PreviewFacts {
279 paused: self.control.is_paused(&target.repository),
280 target,
281 job_attempt: summary.run_attempt,
282 run,
283 superseded,
284 confirm,
285 siblings: others.jobs,
286 more: others.more,
287 }))))
288 }
289
290 async fn board(
292 &self,
293 boarding: Boarding,
294 issued: Issued,
295 summary: &JobSummary,
296 ) -> Result<Asked, ActionError> {
297 let answer = match boarding {
298 Boarding::Busy(other) => {
299 return Err(ActionError::Busy(api::actions_in_flight(
300 other,
301 issued.target.run,
302 )));
303 }
304 Boarding::Follow(answer) => answer,
305 Boarding::Lead { landing, answer } => {
306 let actions = self.clone();
308 let sent = issued.clone();
309 tokio::spawn(async move {
310 let landed = actions.dispatch(&sent).await;
311 landing.land(landed);
312 });
313 answer
314 }
315 };
316 match wait(answer, &issued.target).await {
317 Landed::Done(result) => Ok(Asked::Done(result)),
318 Landed::Repreview { run, why } => {
319 let policy = self.control.action_policy();
320 allowed(&policy, &summary.repository)?;
321 self.preview(summary, issued.target.action, run, Some(why), &policy)
322 .await
323 }
324 Landed::Withheld(_) => {
326 let policy = self.control.action_policy();
327 allowed(&policy, &summary.repository)?;
328 let run = RunSeen::Unread { status: None };
329 let why = Some(Superseded::Unverified);
330 self.preview(summary, issued.target.action, run, why, &policy)
331 .await
332 }
333 }
334 }
335
336 async fn read_run(&self, repository: &str, run: u64) -> RunSeen {
338 tokio::select! {
339 biased;
340 () = self.control.stop_asked() => RunSeen::Unread { status: None },
341 read = tokio::time::timeout(self.deadline, self.github.run(repository, run)) => match read {
342 Ok(Ok(now)) => seen(now),
343 Ok(Err(failure)) => RunSeen::Unread { status: status_of(failure) },
344 Err(_) => RunSeen::Unread { status: None },
345 },
346 }
347 }
348
349 async fn dispatch(&self, issued: &Issued) -> Landed {
351 let target = &issued.target;
352 let unknown = |cause| {
353 Landed::Done(ActionResult::new(
354 target,
355 ActionAnswer::Unknown {
356 cause,
357 status: None,
358 },
359 time::now_ms() / 1000,
360 ))
361 };
362 tokio::select! {
363 biased;
364 () = self.control.stop_asked() => unknown(UnknownCause::Stopping),
365 landed = tokio::time::timeout(self.deadline, self.send(issued)) => {
366 landed.unwrap_or_else(|_| unknown(UnknownCause::TimedOut))
367 },
368 }
369 }
370
371 async fn send(&self, issued: &Issued) -> Landed {
374 let target = &issued.target;
375 let repository = &target.repository;
376 let request = match target.action {
377 JobAction::Cancel => JobActionRequest::CancelRun { run: target.run },
378 JobAction::Rerun => JobActionRequest::RerunJob { job: target.job },
379 JobAction::RerunFailed | JobAction::Unsupported => {
380 JobActionRequest::RerunFailed { run: target.run }
381 }
382 };
383 let withheld: std::sync::Mutex<Option<Landed>> = std::sync::Mutex::new(None);
385 let opened = std::sync::atomic::AtomicBool::new(false);
387 let permit = || -> BoxFuture<'_, bool> {
388 Box::pin(async {
389 let why = self.verify(issued).await;
390 let go = why.is_none();
391 if go {
392 opened.store(true, std::sync::atomic::Ordering::SeqCst);
393 }
394 *crate::sync::lock(&withheld) = why;
395 go
396 })
397 };
398 let (slug, run, job) = (target.action.slug(), target.run, target.job);
399 info!("{repository}: sending the {slug} of run {run} (job {job}) to GitHub");
400 let sent = async {
401 match issued.origin {
402 Origin::Owner => self.github.send(repository, request, &permit).await,
403 Origin::Automatic => tokio::select! {
407 biased;
408 () = self.control.policy_moved(issued.revision) => {
409 if opened.load(std::sync::atomic::Ordering::SeqCst) {
410 Err(ActionFailure::Broken)
411 } else {
412 Err(ActionFailure::Withdrawn)
413 }
414 }
415 sent = self.github.send(repository, request, &permit) => sent,
416 },
417 }
418 };
419 let answer = match sent.await {
420 Err(ActionFailure::Withdrawn) => match crate::sync::lock(&withheld).take() {
421 Some(landed) => return landed,
422 None => classify(Err(ActionFailure::Withdrawn)),
423 },
424 Err(ActionFailure::Status { status: 409 }) if target.action == JobAction::Cancel => {
426 match self.github.run(repository, target.run).await {
427 Ok(now) if now.completed => ActionAnswer::AlreadyFinished { status: 409 },
428 _ => ActionAnswer::Refused {
429 refusal: ActionRefusal::Conflict,
430 status: Some(409),
431 until: None,
432 },
433 }
434 }
435 sent => classify(sent),
436 };
437 let result = ActionResult::new(target, answer, time::now_ms() / 1000);
438 info!(
440 "{repository}: GitHub's answer to the {slug} of run {run}: {:?}, status {:?}, {:?}",
441 result.outcome, result.status, result.refusal
442 );
443 Landed::Done(result)
444 }
445
446 async fn verify(&self, issued: &Issued) -> Option<Landed> {
449 let target = &issued.target;
450 let withdrawn = || {
451 Landed::Done(ActionResult::new(
452 target,
453 classify(Err(ActionFailure::Withdrawn)),
454 time::now_ms() / 1000,
455 ))
456 };
457 match self.github.run(&target.repository, target.run).await {
458 Ok(now)
460 if Some(now.attempt) == target.attempt
461 && issued.origin == Origin::Automatic
462 && !(now.completed && now.failed) =>
463 {
464 return Some(Landed::Withheld(Withheld::NotFailed));
465 }
466 Ok(now) if Some(now.attempt) == target.attempt => {}
467 Ok(now) => {
468 return Some(Landed::Repreview {
469 run: seen(now),
470 why: Superseded::AttemptChanged,
471 });
472 }
473 Err(failure) if issued.origin == Origin::Automatic => {
474 return Some(Landed::Withheld(Withheld::Unread(failure)));
475 }
476 Err(failure) => {
477 return Some(Landed::Repreview {
478 run: RunSeen::Unread {
479 status: status_of(failure),
480 },
481 why: Superseded::Unverified,
482 });
483 }
484 }
485 (self.control.action_policy().revision() != issued.revision).then(withdrawn)
486 }
487
488 pub fn rerun_refused(&self, repository: &str) -> Option<api::RerunRefused> {
491 self.faults
492 .held(
493 repository,
494 self.control.action_policy().revision(),
495 tokio::time::Instant::now(),
496 )
497 .map(|held| held.refused)
498 }
499
500 #[cfg(test)]
502 fn in_flight(&self) -> usize {
503 self.flights.len()
504 }
505
506 #[cfg(test)]
508 fn waiting(&self) -> usize {
509 self.flights.waiting()
510 }
511}
512
513fn allowed(policy: &Policy, repository: &str) -> Result<(), ActionError> {
515 policy.allows(repository).map_err(|denied| {
516 ActionError::Forbidden(match denied {
517 Denied::Off => api::actions_off().to_owned(),
518 Denied::NotConfigured | Denied::RerunsOff => api::actions_not_configured(repository),
520 })
521 })
522}
523
524async fn wait(mut answer: watch::Receiver<Option<Landed>>, target: &ActionTarget) -> Landed {
526 let stopped = || {
527 Landed::Done(ActionResult::new(
528 target,
529 ActionAnswer::Unknown {
530 cause: UnknownCause::Stopping,
531 status: None,
532 },
533 time::now_ms() / 1000,
534 ))
535 };
536 match answer.wait_for(Option::is_some).await {
537 Ok(landed) => landed.clone().unwrap_or_else(stopped),
538 Err(_) => stopped(),
539 }
540}
541
542fn seen(now: RunNow) -> RunSeen {
543 RunSeen::Now {
544 attempt: now.attempt,
545 completed: now.completed,
546 }
547}
548
549fn status_of(failure: ActionFailure) -> Option<u16> {
551 match failure {
552 ActionFailure::Status { status } => Some(status),
553 ActionFailure::RateLimited { status, .. } => status,
554 _ => None,
555 }
556}
557
558fn read_failed(error: &StoreError) -> ActionError {
559 ActionError::Failed(format!("cannot read the job history: {error}"))
560}
561
562fn classify(sent: Result<u16, ActionFailure>) -> ActionAnswer {
564 let refused = |refusal, status| ActionAnswer::Refused {
565 refusal,
566 status,
567 until: None,
568 };
569 match sent {
570 Ok(status) => ActionAnswer::Accepted { status },
571 Err(ActionFailure::NoToken) => refused(ActionRefusal::NoToken, None),
572 Err(ActionFailure::Withdrawn) => refused(ActionRefusal::Withdrawn, None),
573 Err(ActionFailure::Unreachable) => refused(ActionRefusal::Unreachable, None),
575 Err(ActionFailure::RateLimited { status, until }) => ActionAnswer::Refused {
576 refusal: ActionRefusal::RateLimited,
577 status,
578 until: Some(
579 until
580 .duration_since(std::time::SystemTime::UNIX_EPOCH)
581 .unwrap_or_default()
582 .as_secs(),
583 ),
584 },
585 Err(ActionFailure::Status { status }) => {
586 let code = Some(status);
587 match status {
588 401 => refused(ActionRefusal::Unauthorized, code),
589 403 => refused(ActionRefusal::Forbidden, code),
590 404 | 410 => refused(ActionRefusal::NotFound, code),
591 409 => refused(ActionRefusal::Conflict, code),
592 422 => refused(ActionRefusal::Unprocessable, code),
593 500..=599 => ActionAnswer::Unknown {
595 cause: UnknownCause::ServerError,
596 status: code,
597 },
598 _ => refused(ActionRefusal::Failed, code),
599 }
600 }
601 Err(ActionFailure::TimedOut) => ActionAnswer::Unknown {
602 cause: UnknownCause::TimedOut,
603 status: None,
604 },
605 Err(ActionFailure::Broken) => ActionAnswer::Unknown {
606 cause: UnknownCause::Broken,
607 status: None,
608 },
609 }
610}
611
612#[cfg(test)]
613mod tests;