1mod flights;
13pub mod policy;
14
15use std::sync::Arc;
16use std::time::{Duration, Instant};
17
18use futures_util::future::BoxFuture;
19use tokio::sync::watch;
20use tracing::info;
21
22use self::flights::{Boarding, Flights, Issued, Landed, Redeemed};
23use self::policy::{Denied, Policy};
24use crate::api::{
25 self, ActionAnswer, ActionPreview, ActionRefusal, ActionResult, ActionTarget, JobAction,
26 JobSummary, PreviewFacts, RunSeen, SIBLINGS_MAX, Superseded, UnknownCause,
27};
28use crate::github::{ActionFailure, GithubApi, JobActionRequest, Permit, RunNow};
29use crate::scheduler::engine::{Control, Shutdown};
30use crate::store::{Store, StoreError};
31use crate::time;
32
33pub const DEADLINE: Duration = Duration::from_secs(20);
37
38#[derive(Debug, Clone, Copy)]
40pub struct Limits {
41 pub deadline: Duration,
43 pub confirmation: Duration,
45}
46
47pub const LIMITS: Limits = Limits {
49 deadline: DEADLINE,
50 confirmation: Duration::from_secs(api::CONFIRM_SECS),
51};
52
53pub trait ActionsGithub: Send + Sync + 'static {
56 fn run<'call>(
58 &'call self,
59 repository: &'call str,
60 run: u64,
61 ) -> BoxFuture<'call, Result<RunNow, ActionFailure>>;
62
63 fn send<'call>(
66 &'call self,
67 repository: &'call str,
68 request: JobActionRequest,
69 permit: &'call Permit<'call>,
70 ) -> BoxFuture<'call, Result<u16, ActionFailure>>;
71}
72
73impl<G: GithubApi + 'static> ActionsGithub for G {
74 fn run<'call>(
75 &'call self,
76 repository: &'call str,
77 run: u64,
78 ) -> BoxFuture<'call, Result<RunNow, ActionFailure>> {
79 Box::pin(GithubApi::action_run(self, repository, run))
80 }
81
82 fn send<'call>(
83 &'call self,
84 repository: &'call str,
85 request: JobActionRequest,
86 permit: &'call Permit<'call>,
87 ) -> BoxFuture<'call, Result<u16, ActionFailure>> {
88 Box::pin(GithubApi::job_action(self, repository, request, permit))
89 }
90}
91
92#[derive(Debug)]
94pub enum Asked {
95 Preview(Box<ActionPreview>),
97 Done(ActionResult),
99}
100
101#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
103pub enum ActionError {
104 #[error("{0}")]
106 NotFound(String),
107 #[error("{0}")]
109 Forbidden(String),
110 #[error("{0}")]
112 Invalid(String),
113 #[error("{0}")]
115 Busy(String),
116 #[error("{0}")]
118 Failed(String),
119 #[error("runnerd is stopping")]
121 Stopped,
122}
123
124#[derive(Clone)]
126pub struct JobActions {
127 github: Arc<dyn ActionsGithub>,
128 store: Store,
129 control: Arc<Control>,
130 flights: Arc<Flights>,
131 deadline: Duration,
132}
133
134impl std::fmt::Debug for JobActions {
135 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
136 f.debug_struct("JobActions").finish_non_exhaustive()
137 }
138}
139
140impl JobActions {
141 pub fn new(
144 github: Arc<dyn ActionsGithub>,
145 store: Store,
146 control: Arc<Control>,
147 limits: Limits,
148 ) -> JobActions {
149 JobActions {
150 github,
151 store,
152 control,
153 flights: Arc::new(Flights::new(limits.confirmation)),
154 deadline: limits.deadline,
155 }
156 }
157
158 pub async fn ask(
162 &self,
163 job: u64,
164 action: JobAction,
165 confirm: Option<&str>,
166 ) -> Result<Asked, ActionError> {
167 if action == JobAction::Unsupported {
168 return Err(ActionError::Invalid("no such job action".to_owned()));
169 }
170 if self.control.mode() != Shutdown::No {
171 return Err(ActionError::Stopped);
172 }
173 let summary = self
174 .store
175 .job(job)
176 .await
177 .map_err(|error| read_failed(&error))?
178 .ok_or_else(|| ActionError::NotFound(format!("no job {job} in the history")))?
179 .summary;
180 if summary.run_id == 0 {
181 return Err(ActionError::Invalid(api::actions_no_run(job)));
182 }
183 let policy = self.control.action_policy();
186 allowed(&policy, &summary.repository)?;
187 let superseded = match confirm {
188 None => None,
189 Some(token) => {
190 match self
191 .flights
192 .redeem(token, action, job, policy.revision(), Instant::now())
193 {
194 Redeemed::Board { boarding, issued } => {
195 return self.board(boarding, issued, &summary).await;
196 }
197 Redeemed::Stale(why) => Some(why),
198 }
199 }
200 };
201 let run = self.read_run(&summary.repository, summary.run_id).await;
202 self.preview(&summary, action, run, superseded, &policy)
203 .await
204 }
205
206 async fn preview(
209 &self,
210 summary: &JobSummary,
211 action: JobAction,
212 run: RunSeen,
213 superseded: Option<Superseded>,
214 policy: &Policy,
215 ) -> Result<Asked, ActionError> {
216 let others = self
217 .store
218 .run_jobs(
219 &summary.repository,
220 summary.run_id,
221 summary.id,
222 SIBLINGS_MAX,
223 )
224 .await
225 .map_err(|error| read_failed(&error))?;
226 let mut target = ActionTarget {
227 action,
228 job: summary.id,
229 name: summary.name.clone(),
230 repository: summary.repository.clone(),
231 run: summary.run_id,
232 attempt: None,
233 };
234 let confirm = match run {
235 RunSeen::Now { attempt, .. } => {
236 target.attempt = Some(attempt);
237 let issued = Issued {
238 target: target.clone(),
239 revision: policy.revision(),
240 };
241 Some(
242 self.flights
243 .issue(issued, Instant::now())
244 .map_err(|error| {
245 ActionError::Failed(format!("cannot make a confirmation: {error}"))
246 })?,
247 )
248 }
249 RunSeen::Unread { .. } => None,
250 };
251 Ok(Asked::Preview(Box::new(ActionPreview::new(PreviewFacts {
252 paused: self.control.is_paused(&target.repository),
253 target,
254 job_attempt: summary.run_attempt,
255 run,
256 superseded,
257 confirm,
258 siblings: others.jobs,
259 more: others.more,
260 }))))
261 }
262
263 async fn board(
265 &self,
266 boarding: Boarding,
267 issued: Issued,
268 summary: &JobSummary,
269 ) -> Result<Asked, ActionError> {
270 let answer = match boarding {
271 Boarding::Busy(other) => {
272 return Err(ActionError::Busy(api::actions_in_flight(
273 other,
274 issued.target.run,
275 )));
276 }
277 Boarding::Follow(answer) => answer,
278 Boarding::Lead { landing, answer } => {
279 let actions = self.clone();
281 let sent = issued.clone();
282 tokio::spawn(async move {
283 let landed = actions.dispatch(&sent).await;
284 landing.land(landed);
285 });
286 answer
287 }
288 };
289 match wait(answer, &issued.target).await {
290 Landed::Done(result) => Ok(Asked::Done(result)),
291 Landed::Repreview { run, why } => {
292 let policy = self.control.action_policy();
293 allowed(&policy, &summary.repository)?;
294 self.preview(summary, issued.target.action, run, Some(why), &policy)
295 .await
296 }
297 }
298 }
299
300 async fn read_run(&self, repository: &str, run: u64) -> RunSeen {
302 tokio::select! {
303 biased;
304 () = self.control.stop_asked() => RunSeen::Unread { status: None },
305 read = tokio::time::timeout(self.deadline, self.github.run(repository, run)) => match read {
306 Ok(Ok(now)) => seen(now),
307 Ok(Err(failure)) => RunSeen::Unread { status: status_of(failure) },
308 Err(_) => RunSeen::Unread { status: None },
309 },
310 }
311 }
312
313 async fn dispatch(&self, issued: &Issued) -> Landed {
315 let target = &issued.target;
316 let unknown = |cause| {
317 Landed::Done(ActionResult::new(
318 target,
319 ActionAnswer::Unknown {
320 cause,
321 status: None,
322 },
323 time::now_ms() / 1000,
324 ))
325 };
326 tokio::select! {
327 biased;
328 () = self.control.stop_asked() => unknown(UnknownCause::Stopping),
329 landed = tokio::time::timeout(self.deadline, self.send(issued)) => {
330 landed.unwrap_or_else(|_| unknown(UnknownCause::TimedOut))
331 },
332 }
333 }
334
335 async fn send(&self, issued: &Issued) -> Landed {
338 let target = &issued.target;
339 let repository = &target.repository;
340 let request = match target.action {
341 JobAction::Cancel => JobActionRequest::CancelRun { run: target.run },
342 JobAction::Rerun => JobActionRequest::RerunJob { job: target.job },
343 JobAction::RerunFailed | JobAction::Unsupported => {
344 JobActionRequest::RerunFailed { run: target.run }
345 }
346 };
347 let withheld: std::sync::Mutex<Option<Landed>> = std::sync::Mutex::new(None);
349 let permit = || -> BoxFuture<'_, bool> {
350 Box::pin(async {
351 let why = self.verify(issued).await;
352 let go = why.is_none();
353 *crate::sync::lock(&withheld) = why;
354 go
355 })
356 };
357 let (slug, run, job) = (target.action.slug(), target.run, target.job);
358 info!("{repository}: sending the {slug} of run {run} (job {job}) to GitHub");
359 let answer = match self.github.send(repository, request, &permit).await {
360 Err(ActionFailure::Withdrawn) => match crate::sync::lock(&withheld).take() {
361 Some(landed) => return landed,
362 None => classify(Err(ActionFailure::Withdrawn)),
363 },
364 Err(ActionFailure::Status { status: 409 }) if target.action == JobAction::Cancel => {
366 match self.github.run(repository, target.run).await {
367 Ok(now) if now.completed => ActionAnswer::AlreadyFinished { status: 409 },
368 _ => ActionAnswer::Refused {
369 refusal: ActionRefusal::Conflict,
370 status: Some(409),
371 until: None,
372 },
373 }
374 }
375 sent => classify(sent),
376 };
377 let result = ActionResult::new(target, answer, time::now_ms() / 1000);
378 info!(
380 "{repository}: GitHub's answer to the {slug} of run {run}: {:?}, status {:?}, {:?}",
381 result.outcome, result.status, result.refusal
382 );
383 Landed::Done(result)
384 }
385
386 async fn verify(&self, issued: &Issued) -> Option<Landed> {
389 let target = &issued.target;
390 let withdrawn = || {
391 Landed::Done(ActionResult::new(
392 target,
393 classify(Err(ActionFailure::Withdrawn)),
394 time::now_ms() / 1000,
395 ))
396 };
397 match self.github.run(&target.repository, target.run).await {
398 Ok(now) if Some(now.attempt) == target.attempt => {}
399 Ok(now) => {
400 return Some(Landed::Repreview {
401 run: seen(now),
402 why: Superseded::AttemptChanged,
403 });
404 }
405 Err(failure) => {
406 return Some(Landed::Repreview {
407 run: RunSeen::Unread {
408 status: status_of(failure),
409 },
410 why: Superseded::Unverified,
411 });
412 }
413 }
414 (self.control.action_policy().revision() != issued.revision).then(withdrawn)
415 }
416
417 #[cfg(test)]
419 fn in_flight(&self) -> usize {
420 self.flights.len()
421 }
422
423 #[cfg(test)]
425 fn waiting(&self) -> usize {
426 self.flights.waiting()
427 }
428}
429
430fn allowed(policy: &Policy, repository: &str) -> Result<(), ActionError> {
432 policy.allows(repository).map_err(|denied| {
433 ActionError::Forbidden(match denied {
434 Denied::Off => api::actions_off().to_owned(),
435 Denied::NotConfigured => api::actions_not_configured(repository),
436 })
437 })
438}
439
440async fn wait(mut answer: watch::Receiver<Option<Landed>>, target: &ActionTarget) -> Landed {
442 let stopped = || {
443 Landed::Done(ActionResult::new(
444 target,
445 ActionAnswer::Unknown {
446 cause: UnknownCause::Stopping,
447 status: None,
448 },
449 time::now_ms() / 1000,
450 ))
451 };
452 match answer.wait_for(Option::is_some).await {
453 Ok(landed) => landed.clone().unwrap_or_else(stopped),
454 Err(_) => stopped(),
455 }
456}
457
458fn seen(now: RunNow) -> RunSeen {
459 RunSeen::Now {
460 attempt: now.attempt,
461 completed: now.completed,
462 }
463}
464
465fn status_of(failure: ActionFailure) -> Option<u16> {
467 match failure {
468 ActionFailure::Status { status } => Some(status),
469 ActionFailure::RateLimited { status, .. } => status,
470 _ => None,
471 }
472}
473
474fn read_failed(error: &StoreError) -> ActionError {
475 ActionError::Failed(format!("cannot read the job history: {error}"))
476}
477
478fn classify(sent: Result<u16, ActionFailure>) -> ActionAnswer {
480 let refused = |refusal, status| ActionAnswer::Refused {
481 refusal,
482 status,
483 until: None,
484 };
485 match sent {
486 Ok(status) => ActionAnswer::Accepted { status },
487 Err(ActionFailure::NoToken) => refused(ActionRefusal::NoToken, None),
488 Err(ActionFailure::Withdrawn) => refused(ActionRefusal::Withdrawn, None),
489 Err(ActionFailure::Unreachable) => refused(ActionRefusal::Unreachable, None),
491 Err(ActionFailure::RateLimited { status, until }) => ActionAnswer::Refused {
492 refusal: ActionRefusal::RateLimited,
493 status,
494 until: Some(
495 until
496 .duration_since(std::time::SystemTime::UNIX_EPOCH)
497 .unwrap_or_default()
498 .as_secs(),
499 ),
500 },
501 Err(ActionFailure::Status { status }) => {
502 let code = Some(status);
503 match status {
504 401 => refused(ActionRefusal::Unauthorized, code),
505 403 => refused(ActionRefusal::Forbidden, code),
506 404 | 410 => refused(ActionRefusal::NotFound, code),
507 409 => refused(ActionRefusal::Conflict, code),
508 422 => refused(ActionRefusal::Unprocessable, code),
509 500..=599 => ActionAnswer::Unknown {
511 cause: UnknownCause::ServerError,
512 status: code,
513 },
514 _ => refused(ActionRefusal::Failed, code),
515 }
516 }
517 Err(ActionFailure::TimedOut) => ActionAnswer::Unknown {
518 cause: UnknownCause::TimedOut,
519 status: None,
520 },
521 Err(ActionFailure::Broken) => ActionAnswer::Unknown {
522 cause: UnknownCause::Broken,
523 status: None,
524 },
525 }
526}
527
528#[cfg(test)]
529mod tests;