mod faults;
mod flights;
pub mod policy;
mod rerun;
use std::sync::Arc;
use std::time::{Duration, Instant};
use futures_util::future::BoxFuture;
use tokio::sync::watch;
use tracing::info;
use self::faults::Faults;
use self::flights::{Boarding, Flights, Issued, Landed, Origin, Redeemed, Withheld};
use self::policy::{Denied, Policy};
use crate::api::{
self, ActionAnswer, ActionPreview, ActionRefusal, ActionResult, ActionTarget, JobAction,
JobSummary, PreviewFacts, RunSeen, SIBLINGS_MAX, Superseded, UnknownCause,
};
use crate::github::{ActionFailure, AttemptJob, GithubApi, JobActionRequest, Permit, RunNow};
use crate::scheduler::engine::{Control, Shutdown};
use crate::store::{Store, StoreError};
use crate::time;
pub const DEADLINE: Duration = Duration::from_secs(20);
#[derive(Debug, Clone, Copy)]
pub struct Limits {
pub deadline: Duration,
pub confirmation: Duration,
}
pub const LIMITS: Limits = Limits {
deadline: DEADLINE,
confirmation: Duration::from_secs(api::CONFIRM_SECS),
};
pub trait ActionsGithub: Send + Sync + 'static {
fn run<'call>(
&'call self,
repository: &'call str,
run: u64,
) -> BoxFuture<'call, Result<RunNow, ActionFailure>>;
fn attempt_jobs<'call>(
&'call self,
repository: &'call str,
run: u64,
attempt: u32,
) -> BoxFuture<'call, Result<Vec<AttemptJob>, ActionFailure>>;
fn send<'call>(
&'call self,
repository: &'call str,
request: JobActionRequest,
permit: &'call Permit<'call>,
) -> BoxFuture<'call, Result<u16, ActionFailure>>;
}
impl<G: GithubApi + 'static> ActionsGithub for G {
fn run<'call>(
&'call self,
repository: &'call str,
run: u64,
) -> BoxFuture<'call, Result<RunNow, ActionFailure>> {
Box::pin(GithubApi::action_run(self, repository, run))
}
fn attempt_jobs<'call>(
&'call self,
repository: &'call str,
run: u64,
attempt: u32,
) -> BoxFuture<'call, Result<Vec<AttemptJob>, ActionFailure>> {
Box::pin(GithubApi::attempt_jobs(self, repository, run, attempt))
}
fn send<'call>(
&'call self,
repository: &'call str,
request: JobActionRequest,
permit: &'call Permit<'call>,
) -> BoxFuture<'call, Result<u16, ActionFailure>> {
Box::pin(GithubApi::job_action(self, repository, request, permit))
}
}
#[derive(Debug)]
pub enum Asked {
Preview(Box<ActionPreview>),
Done(ActionResult),
}
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum ActionError {
#[error("{0}")]
NotFound(String),
#[error("{0}")]
Forbidden(String),
#[error("{0}")]
Invalid(String),
#[error("{0}")]
Busy(String),
#[error("{0}")]
Failed(String),
#[error("runnerd is stopping")]
Stopped,
}
#[derive(Clone)]
pub struct JobActions {
github: Arc<dyn ActionsGithub>,
store: Store,
control: Arc<Control>,
flights: Arc<Flights>,
faults: Arc<Faults>,
deadline: Duration,
}
impl std::fmt::Debug for JobActions {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("JobActions").finish_non_exhaustive()
}
}
impl JobActions {
pub fn new(
github: Arc<dyn ActionsGithub>,
store: Store,
control: Arc<Control>,
limits: Limits,
) -> JobActions {
JobActions {
github,
store,
control,
flights: Arc::new(Flights::new(limits.confirmation)),
faults: Arc::new(Faults::default()),
deadline: limits.deadline,
}
}
pub async fn ask(
&self,
job: u64,
action: JobAction,
confirm: Option<&str>,
) -> Result<Asked, ActionError> {
if action == JobAction::Unsupported {
return Err(ActionError::Invalid("no such job action".to_owned()));
}
if self.control.mode() != Shutdown::No {
return Err(ActionError::Stopped);
}
let summary = self
.store
.job(job)
.await
.map_err(|error| read_failed(&error))?
.ok_or_else(|| ActionError::NotFound(format!("no job {job} in the history")))?
.summary;
if summary.run_id == 0 {
return Err(ActionError::Invalid(api::actions_no_run(job)));
}
let policy = self.control.action_policy();
allowed(&policy, &summary.repository)?;
let superseded = match confirm {
None => None,
Some(token) => {
match self
.flights
.redeem(token, action, job, policy.revision(), Instant::now())
{
Redeemed::Board { boarding, issued } => {
return self.board(boarding, issued, &summary).await;
}
Redeemed::Stale(why) => Some(why),
}
}
};
let run = self.read_run(&summary.repository, summary.run_id).await;
self.preview(&summary, action, run, superseded, &policy)
.await
}
async fn preview(
&self,
summary: &JobSummary,
action: JobAction,
run: RunSeen,
superseded: Option<Superseded>,
policy: &Policy,
) -> Result<Asked, ActionError> {
let others = self
.store
.run_jobs(
&summary.repository,
summary.run_id,
summary.id,
SIBLINGS_MAX,
)
.await
.map_err(|error| read_failed(&error))?;
let mut target = ActionTarget {
action,
job: summary.id,
name: summary.name.clone(),
repository: summary.repository.clone(),
run: summary.run_id,
attempt: None,
};
let confirm = match run {
RunSeen::Now { attempt, .. } => {
target.attempt = Some(attempt);
let issued = Issued {
target: target.clone(),
revision: policy.revision(),
origin: Origin::Owner,
};
Some(
self.flights
.issue(issued, Instant::now())
.map_err(|error| {
ActionError::Failed(format!("cannot make a confirmation: {error}"))
})?,
)
}
RunSeen::Unread { .. } => None,
};
Ok(Asked::Preview(Box::new(ActionPreview::new(PreviewFacts {
paused: self.control.is_paused(&target.repository),
target,
job_attempt: summary.run_attempt,
run,
superseded,
confirm,
siblings: others.jobs,
more: others.more,
}))))
}
async fn board(
&self,
boarding: Boarding,
issued: Issued,
summary: &JobSummary,
) -> Result<Asked, ActionError> {
let answer = match boarding {
Boarding::Busy(other) => {
return Err(ActionError::Busy(api::actions_in_flight(
other,
issued.target.run,
)));
}
Boarding::Follow(answer) => answer,
Boarding::Lead { landing, answer } => {
let actions = self.clone();
let sent = issued.clone();
tokio::spawn(async move {
let landed = actions.dispatch(&sent).await;
landing.land(landed);
});
answer
}
};
match wait(answer, &issued.target).await {
Landed::Done(result) => Ok(Asked::Done(result)),
Landed::Repreview { run, why } => {
let policy = self.control.action_policy();
allowed(&policy, &summary.repository)?;
self.preview(summary, issued.target.action, run, Some(why), &policy)
.await
}
Landed::Withheld(_) => {
let policy = self.control.action_policy();
allowed(&policy, &summary.repository)?;
let run = RunSeen::Unread { status: None };
let why = Some(Superseded::Unverified);
self.preview(summary, issued.target.action, run, why, &policy)
.await
}
}
}
async fn read_run(&self, repository: &str, run: u64) -> RunSeen {
tokio::select! {
biased;
() = self.control.stop_asked() => RunSeen::Unread { status: None },
read = tokio::time::timeout(self.deadline, self.github.run(repository, run)) => match read {
Ok(Ok(now)) => seen(now),
Ok(Err(failure)) => RunSeen::Unread { status: status_of(failure) },
Err(_) => RunSeen::Unread { status: None },
},
}
}
async fn dispatch(&self, issued: &Issued) -> Landed {
let target = &issued.target;
let unknown = |cause| {
Landed::Done(ActionResult::new(
target,
ActionAnswer::Unknown {
cause,
status: None,
},
time::now_ms() / 1000,
))
};
tokio::select! {
biased;
() = self.control.stop_asked() => unknown(UnknownCause::Stopping),
landed = tokio::time::timeout(self.deadline, self.send(issued)) => {
landed.unwrap_or_else(|_| unknown(UnknownCause::TimedOut))
},
}
}
async fn send(&self, issued: &Issued) -> Landed {
let target = &issued.target;
let repository = &target.repository;
let request = match target.action {
JobAction::Cancel => JobActionRequest::CancelRun { run: target.run },
JobAction::Rerun => JobActionRequest::RerunJob { job: target.job },
JobAction::RerunFailed | JobAction::Unsupported => {
JobActionRequest::RerunFailed { run: target.run }
}
};
let withheld: std::sync::Mutex<Option<Landed>> = std::sync::Mutex::new(None);
let opened = std::sync::atomic::AtomicBool::new(false);
let permit = || -> BoxFuture<'_, bool> {
Box::pin(async {
let why = self.verify(issued).await;
let go = why.is_none();
if go {
opened.store(true, std::sync::atomic::Ordering::SeqCst);
}
*crate::sync::lock(&withheld) = why;
go
})
};
let (slug, run, job) = (target.action.slug(), target.run, target.job);
info!("{repository}: sending the {slug} of run {run} (job {job}) to GitHub");
let sent = async {
match issued.origin {
Origin::Owner => self.github.send(repository, request, &permit).await,
Origin::Automatic => tokio::select! {
biased;
() = self.control.policy_moved(issued.revision) => {
if opened.load(std::sync::atomic::Ordering::SeqCst) {
Err(ActionFailure::Broken)
} else {
Err(ActionFailure::Withdrawn)
}
}
sent = self.github.send(repository, request, &permit) => sent,
},
}
};
let answer = match sent.await {
Err(ActionFailure::Withdrawn) => match crate::sync::lock(&withheld).take() {
Some(landed) => return landed,
None => classify(Err(ActionFailure::Withdrawn)),
},
Err(ActionFailure::Status { status: 409 }) if target.action == JobAction::Cancel => {
match self.github.run(repository, target.run).await {
Ok(now) if now.completed => ActionAnswer::AlreadyFinished { status: 409 },
_ => ActionAnswer::Refused {
refusal: ActionRefusal::Conflict,
status: Some(409),
until: None,
},
}
}
sent => classify(sent),
};
let result = ActionResult::new(target, answer, time::now_ms() / 1000);
info!(
"{repository}: GitHub's answer to the {slug} of run {run}: {:?}, status {:?}, {:?}",
result.outcome, result.status, result.refusal
);
Landed::Done(result)
}
async fn verify(&self, issued: &Issued) -> Option<Landed> {
let target = &issued.target;
let withdrawn = || {
Landed::Done(ActionResult::new(
target,
classify(Err(ActionFailure::Withdrawn)),
time::now_ms() / 1000,
))
};
match self.github.run(&target.repository, target.run).await {
Ok(now)
if Some(now.attempt) == target.attempt
&& issued.origin == Origin::Automatic
&& !(now.completed && now.failed) =>
{
return Some(Landed::Withheld(Withheld::NotFailed));
}
Ok(now) if Some(now.attempt) == target.attempt => {}
Ok(now) => {
return Some(Landed::Repreview {
run: seen(now),
why: Superseded::AttemptChanged,
});
}
Err(failure) if issued.origin == Origin::Automatic => {
return Some(Landed::Withheld(Withheld::Unread(failure)));
}
Err(failure) => {
return Some(Landed::Repreview {
run: RunSeen::Unread {
status: status_of(failure),
},
why: Superseded::Unverified,
});
}
}
(self.control.action_policy().revision() != issued.revision).then(withdrawn)
}
pub fn rerun_refused(&self, repository: &str) -> Option<api::RerunRefused> {
self.faults
.held(
repository,
self.control.action_policy().revision(),
tokio::time::Instant::now(),
)
.map(|held| held.refused)
}
#[cfg(test)]
fn in_flight(&self) -> usize {
self.flights.len()
}
#[cfg(test)]
fn waiting(&self) -> usize {
self.flights.waiting()
}
}
fn allowed(policy: &Policy, repository: &str) -> Result<(), ActionError> {
policy.allows(repository).map_err(|denied| {
ActionError::Forbidden(match denied {
Denied::Off => api::actions_off().to_owned(),
Denied::NotConfigured | Denied::RerunsOff => api::actions_not_configured(repository),
})
})
}
async fn wait(mut answer: watch::Receiver<Option<Landed>>, target: &ActionTarget) -> Landed {
let stopped = || {
Landed::Done(ActionResult::new(
target,
ActionAnswer::Unknown {
cause: UnknownCause::Stopping,
status: None,
},
time::now_ms() / 1000,
))
};
match answer.wait_for(Option::is_some).await {
Ok(landed) => landed.clone().unwrap_or_else(stopped),
Err(_) => stopped(),
}
}
fn seen(now: RunNow) -> RunSeen {
RunSeen::Now {
attempt: now.attempt,
completed: now.completed,
}
}
fn status_of(failure: ActionFailure) -> Option<u16> {
match failure {
ActionFailure::Status { status } => Some(status),
ActionFailure::RateLimited { status, .. } => status,
_ => None,
}
}
fn read_failed(error: &StoreError) -> ActionError {
ActionError::Failed(format!("cannot read the job history: {error}"))
}
fn classify(sent: Result<u16, ActionFailure>) -> ActionAnswer {
let refused = |refusal, status| ActionAnswer::Refused {
refusal,
status,
until: None,
};
match sent {
Ok(status) => ActionAnswer::Accepted { status },
Err(ActionFailure::NoToken) => refused(ActionRefusal::NoToken, None),
Err(ActionFailure::Withdrawn) => refused(ActionRefusal::Withdrawn, None),
Err(ActionFailure::Unreachable) => refused(ActionRefusal::Unreachable, None),
Err(ActionFailure::RateLimited { status, until }) => ActionAnswer::Refused {
refusal: ActionRefusal::RateLimited,
status,
until: Some(
until
.duration_since(std::time::SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_secs(),
),
},
Err(ActionFailure::Status { status }) => {
let code = Some(status);
match status {
401 => refused(ActionRefusal::Unauthorized, code),
403 => refused(ActionRefusal::Forbidden, code),
404 | 410 => refused(ActionRefusal::NotFound, code),
409 => refused(ActionRefusal::Conflict, code),
422 => refused(ActionRefusal::Unprocessable, code),
500..=599 => ActionAnswer::Unknown {
cause: UnknownCause::ServerError,
status: code,
},
_ => refused(ActionRefusal::Failed, code),
}
}
Err(ActionFailure::TimedOut) => ActionAnswer::Unknown {
cause: UnknownCause::TimedOut,
status: None,
},
Err(ActionFailure::Broken) => ActionAnswer::Unknown {
cause: UnknownCause::Broken,
status: None,
},
}
}
#[cfg(test)]
mod tests;