use crate::bus::{Bus, Operation, payload_of};
use crate::channels::{
BrowseAnnotationRequested, Channel, JobCancel, JobCancelRequested, JobCheckpoint, JobClaim,
JobComplete, JobFail, JobQueued, JobReportProgress, JobStart, MarkCommit, Request,
};
use crate::errors::{BusRequestErrorCode, SemiontError, TransportError, TransportErrorCode};
use crate::job_filter::job_matches_filter;
use crate::locked;
use crate::timing::{HELD_JOB_STALL, HELD_JOB_STALL_CHECK, JOB_CLAIM_TIMEOUT, MARK_COMMIT_TIMEOUT};
use crate::transport::{ConnectionState, Envelope, TraceCarrier, Transport};
use crate::types::{
Annotation, AnnotationId, BrowseAnnotationRequest, DurabilityEvidence, FailureClass,
JobCancelCommand, JobCheckpointCommand, JobClaimCommand, JobCompleteCommand, JobFailCommand,
JobFilter, JobId, JobMetadata, JobParams, JobProgress, JobReportProgressCommand, JobRunning,
JobStartCommand, JobType, MarkCommitCommand, MarkJobCompleteCommand, MarkJobResult, ResourceId,
UnitCursor, YieldJobCompleteCommand, YieldJobResult,
};
use serde_json::Value;
use std::collections::BTreeMap;
use std::future::Future;
use std::marker::PhantomData;
use std::pin::Pin;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, SystemTime};
use tokio::sync::{Notify, mpsc, watch};
use tokio::time::{Instant, MissedTickBehavior};
pub const JOB_CLAIM_CHANNELS: [&str; 4] = [
<<JobClaim as Request>::Result as Channel>::NAME,
<<JobClaim as Request>::Failure as Channel>::NAME,
JobQueued::NAME,
JobCancelRequested::NAME,
];
pub const JOB_COMMIT_CHANNELS: [&str; 4] = [
<<MarkCommit as Request>::Result as Channel>::NAME,
<<MarkCommit as Request>::Failure as Channel>::NAME,
<<BrowseAnnotationRequested as Request>::Result as Channel>::NAME,
<<BrowseAnnotationRequested as Request>::Failure as Channel>::NAME,
];
const fn weakness(evidence: DurabilityEvidence) -> u8 {
match evidence {
DurabilityEvidence::Acknowledged => 0,
DurabilityEvidence::ProbeConfirmed => 1,
DurabilityEvidence::ProbeRefused | DurabilityEvidence::ProbeUnreachable => 2,
}
}
pub fn will_retry_after(metadata: &JobMetadata, failure_class: Option<FailureClass>) -> bool {
let retried = match failure_class {
None | Some(FailureClass::Transient) => true,
Some(FailureClass::Deterministic | FailureClass::Withheld) => false,
};
retried && metadata.retry_count < metadata.max_retries
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ClaimTiming {
pub job_claim: Duration,
pub held_job_stall: Duration,
pub held_job_stall_check: Duration,
pub mark_commit: Duration,
}
impl Default for ClaimTiming {
fn default() -> ClaimTiming {
ClaimTiming {
job_claim: JOB_CLAIM_TIMEOUT,
held_job_stall: HELD_JOB_STALL,
held_job_stall_check: HELD_JOB_STALL_CHECK,
mark_commit: MARK_COMMIT_TIMEOUT,
}
}
}
#[derive(Debug, Clone)]
pub struct ClaimOptions {
pub accepts: Vec<JobFilter>,
pub timing: ClaimTiming,
}
impl ClaimOptions {
pub fn new(accepts: Vec<JobFilter>) -> ClaimOptions {
ClaimOptions {
accepts,
timing: ClaimTiming::default(),
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct ClaimRefusal {
pub code: Option<BusRequestErrorCode>,
pub message: String,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ActiveJob {
pub job_id: JobId,
pub job_type: JobType,
pub since: SystemTime,
}
#[derive(Debug, Clone, PartialEq)]
pub struct WorkerVitals {
pub last_queued_event_at: Option<SystemTime>,
pub last_claim_at: Option<SystemTime>,
pub last_finished_at: Option<SystemTime>,
pub last_activity_at: Option<SystemTime>,
pub active_job: Option<ActiveJob>,
pub jobs_completed: u64,
}
#[derive(Debug, Clone, PartialEq)]
pub struct HeldJobStall {
pub job_id: JobId,
pub job_type: JobType,
pub held_since: SystemTime,
pub last_activity_at: SystemTime,
pub silent_for: Duration,
pub threshold: Duration,
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct JobFailure {
pub failure_class: Option<FailureClass>,
pub completed_units: Option<Vec<String>>,
pub unit_cursors: Option<BTreeMap<String, UnitCursor>>,
}
const STOPPED_WHILE_HELD: &str = "The worker stopped while it held the job";
const DROPPED_UNSETTLED: &str = "The worker let go of the job without settling it";
fn here(message: String) -> SemiontError {
TransportError::without_response(message, TransportErrorCode::Error).into()
}
struct Holding {
resource_id: ResourceId,
job_id: JobId,
job_type: JobType,
attempt: i64,
annotation_id: Option<AnnotationId>,
budget: JobMetadata,
trace: Option<TraceCarrier>,
settled: AtomicBool,
cancelled: watch::Sender<bool>,
durability: Mutex<Option<DurabilityEvidence>>,
}
impl Holding {
fn observe(&self, evidence: DurabilityEvidence) {
let mut remembered = locked(&self.durability);
if remembered.is_none_or(|kept| weakness(evidence) > weakness(kept)) {
*remembered = Some(evidence);
}
}
fn observed(&self) -> Option<DurabilityEvidence> {
*locked(&self.durability)
}
fn fail(&self, error: &str, failure: JobFailure) -> JobFailCommand {
let will_retry = will_retry_after(&self.budget, failure.failure_class);
JobFailCommand {
attempt: Some(self.attempt),
annotation_id: self.annotation_id.clone(),
completed_units: failure.completed_units,
unit_cursors: failure.unit_cursors,
failure_class: failure.failure_class,
will_retry: Some(will_retry),
durability: self.observed().filter(|observed| {
weakness(*observed) > weakness(DurabilityEvidence::ProbeConfirmed)
}),
..JobFailCommand::new(
self.resource_id.clone(),
self.job_id.clone(),
self.job_type,
error,
)
}
}
}
#[derive(Default)]
struct State {
held: Option<Arc<Holding>>,
stall_reported: bool,
last_queued_event_at: Option<SystemTime>,
last_claim_at: Option<SystemTime>,
last_finished_at: Option<SystemTime>,
last_activity: Option<(Instant, SystemTime)>,
held_since: Option<SystemTime>,
jobs_completed: u64,
}
struct Inner {
transport: Arc<dyn Transport>,
wire: Bus,
accepts: Vec<JobFilter>,
timing: ClaimTiming,
state: Mutex<State>,
settled: Notify,
stall: watch::Sender<Option<HeldJobStall>>,
}
impl Inner {
fn holding(&self) -> bool {
locked(&self.state).held.is_some()
}
fn active(&self) {
locked(&self.state).last_activity = Some((Instant::now(), SystemTime::now()));
}
async fn ask<R: Request>(&self, payload: &R::Payload) -> Result<(), SemiontError> {
self.wire
.request_of(
&Operation::of::<R>(),
payload_of(payload)?,
self.timing.mark_commit,
)
.await?;
Ok(())
}
async fn ask_whether_recorded(
&self,
resource_id: &ResourceId,
annotation_id: AnnotationId,
) -> DurabilityEvidence {
let question = BrowseAnnotationRequest {
resource_id: resource_id.clone(),
annotation_id,
};
match self.ask::<BrowseAnnotationRequested>(&question).await {
Ok(()) => DurabilityEvidence::ProbeConfirmed,
Err(SemiontError::Bus(answered)) if answered.code == BusRequestErrorCode::Rejected => {
DurabilityEvidence::ProbeRefused
}
Err(_) => DurabilityEvidence::ProbeUnreachable,
}
}
fn hold(&self, holding: &Arc<Holding>) {
let now = SystemTime::now();
let mut state = locked(&self.state);
state.held = Some(holding.clone());
state.stall_reported = false;
state.last_claim_at = Some(now);
state.last_activity = Some((Instant::now(), now));
state.held_since = Some(now);
}
fn released(&self, holding: &Arc<Holding>, completed: bool) {
{
let mut state = locked(&self.state);
if !state
.held
.as_ref()
.is_some_and(|held| Arc::ptr_eq(held, holding))
{
return;
}
let now = SystemTime::now();
state.held = None;
state.held_since = None;
state.last_finished_at = Some(now);
state.last_activity = Some((Instant::now(), now));
if completed {
state.jobs_completed += 1;
}
}
self.settled.notify_one();
}
async fn fail_held(&self, holding: &Arc<Holding>, error: &str) {
if holding.settled.swap(true, Ordering::SeqCst) {
return;
}
let failed = holding.fail(error, JobFailure::default());
let _ = self
.wire
.emit::<JobFail>(&failed, Envelope::default())
.await;
self.released(holding, false);
}
fn signal_cancellation(&self, job_id: &str) {
let held = locked(&self.state).held.clone();
if let Some(holding) = held
&& holding.job_id.as_str() == job_id
&& !holding.settled.load(Ordering::SeqCst)
{
holding.cancelled.send_replace(true);
}
}
fn look_for_stall(&self) {
let stall = {
let mut state = locked(&self.state);
let (Some(holding), Some((active, told)), Some(since)) =
(state.held.clone(), state.last_activity, state.held_since)
else {
return;
};
let silent_for = active.elapsed();
if state.stall_reported || silent_for <= self.timing.held_job_stall {
return;
}
state.stall_reported = true;
HeldJobStall {
job_id: holding.job_id.clone(),
job_type: holding.job_type,
held_since: since,
last_activity_at: told,
silent_for,
threshold: self.timing.held_job_stall,
}
};
self.stall.send_replace(Some(stall));
}
fn vitals(&self) -> WorkerVitals {
let state = locked(&self.state);
WorkerVitals {
last_queued_event_at: state.last_queued_event_at,
last_claim_at: state.last_claim_at,
last_finished_at: state.last_finished_at,
last_activity_at: state.last_activity.map(|(_, told)| told),
active_job: match (&state.held, state.held_since) {
(Some(holding), Some(since)) => Some(ActiveJob {
job_id: holding.job_id.clone(),
job_type: holding.job_type,
since,
}),
_ => None,
},
jobs_completed: state.jobs_completed,
}
}
}
struct Core {
inner: Arc<Inner>,
holding: Arc<Holding>,
params: JobParams,
completed_units: Vec<String>,
unit_cursors: BTreeMap<String, UnitCursor>,
begun: AtomicBool,
}
impl Core {
fn unsettled(&self, saying: &str) -> Result<(), SemiontError> {
if self.holding.settled.load(Ordering::SeqCst) {
return Err(here(format!(
"Job {} is already settled: it cannot say {saying}",
self.holding.job_id
)));
}
Ok(())
}
async fn settle<C: Channel>(
self,
payload: C::Payload,
completed: bool,
) -> Result<(), SemiontError> {
if self.holding.settled.swap(true, Ordering::SeqCst) {
return Err(here(format!(
"Job {} is already settled: it cannot say {}",
self.holding.job_id,
C::NAME
)));
}
let sent = self
.inner
.wire
.emit::<C>(&payload, Envelope::default())
.await;
self.inner.released(&self.holding, completed);
sent.map(|_| ()).map_err(SemiontError::from)
}
}
impl Drop for Core {
fn drop(&mut self) {
if self.holding.settled.load(Ordering::SeqCst) {
return;
}
let (inner, holding) = (self.inner.clone(), self.holding.clone());
match tokio::runtime::Handle::try_current() {
Ok(runtime) => {
runtime.spawn(async move {
inner.fail_held(&holding, DROPPED_UNSETTLED).await;
});
}
Err(_) => {
if !holding.settled.swap(true, Ordering::SeqCst) {
inner.released(&holding, false);
}
}
}
}
}
pub struct Mark;
pub struct Yield;
pub struct Held<V> {
core: Core,
verb: PhantomData<fn() -> V>,
}
pub type HeldMarkJob = Held<Mark>;
pub type HeldYieldJob = Held<Yield>;
impl<V> Held<V> {
pub fn job_id(&self) -> &JobId {
&self.core.holding.job_id
}
pub fn job_type(&self) -> JobType {
self.core.holding.job_type
}
pub fn resource_id(&self) -> &ResourceId {
&self.core.holding.resource_id
}
pub fn params(&self) -> &JobParams {
&self.core.params
}
pub fn completed_units(&self) -> &[String] {
&self.core.completed_units
}
pub fn unit_cursors(&self) -> &BTreeMap<String, UnitCursor> {
&self.core.unit_cursors
}
pub fn retry_count(&self) -> u64 {
self.core.holding.budget.retry_count
}
pub fn max_retries(&self) -> u64 {
self.core.holding.budget.max_retries
}
pub fn attempt(&self) -> i64 {
self.core.holding.attempt
}
pub fn annotation_id(&self) -> Option<&AnnotationId> {
self.core.holding.annotation_id.as_ref()
}
pub fn trace(&self) -> Option<&TraceCarrier> {
self.core.holding.trace.as_ref()
}
pub fn cancelled(&self) -> watch::Receiver<bool> {
self.core.holding.cancelled.subscribe()
}
pub async fn start(&self) -> Result<(), SemiontError> {
self.core.unsettled(JobStart::NAME)?;
if self.core.begun.swap(true, Ordering::SeqCst) {
return Err(here(format!(
"job:start is a held job's first message, said once: job {} has already said more",
self.job_id()
)));
}
let holding = &self.core.holding;
let started = JobStartCommand {
annotation_id: holding.annotation_id.clone(),
attempt: Some(holding.attempt),
..JobStartCommand::new(
holding.resource_id.clone(),
holding.job_id.clone(),
holding.job_type,
)
};
self.core
.inner
.wire
.emit::<JobStart>(&started, Envelope::default())
.await?;
Ok(())
}
pub async fn progress(&self, progress: JobProgress) -> Result<(), SemiontError> {
self.core.unsettled(JobReportProgress::NAME)?;
self.core.begun.store(true, Ordering::SeqCst);
self.core.inner.active();
let holding = &self.core.holding;
let percentage = progress.percentage;
let reported = JobReportProgressCommand {
attempt: Some(holding.attempt),
annotation_id: holding.annotation_id.clone(),
progress: Some(JobProgress {
annotation_id: holding
.annotation_id
.clone()
.or(progress.annotation_id.clone()),
..progress
}),
..JobReportProgressCommand::new(
holding.resource_id.clone(),
holding.job_id.clone(),
holding.job_type,
percentage,
)
};
self.core
.inner
.wire
.emit::<JobReportProgress>(&reported, Envelope::default())
.await?;
Ok(())
}
pub async fn checkpoint(
&self,
completed_units: Vec<String>,
unit_cursors: Option<BTreeMap<String, UnitCursor>>,
) -> Result<(), SemiontError> {
self.core.unsettled(JobCheckpoint::NAME)?;
self.core.begun.store(true, Ordering::SeqCst);
self.core.inner.active();
let reached = JobCheckpointCommand {
unit_cursors,
..JobCheckpointCommand::new(self.job_id().clone(), completed_units)
};
self.core
.inner
.wire
.emit::<JobCheckpoint>(&reached, Envelope::default())
.await?;
Ok(())
}
pub async fn commit(
&self,
resource_id: &ResourceId,
annotations: Vec<Annotation>,
) -> Result<(), SemiontError> {
self.core.unsettled(MarkCommit::NAME)?;
let Some(last) = annotations.last().map(|last| last.id.clone()) else {
return Ok(());
};
let (inner, holding) = (&self.core.inner, &self.core.holding);
let batch = MarkCommitCommand {
job_id: Some(holding.job_id.clone()),
..MarkCommitCommand::new(resource_id.clone(), annotations)
};
let unanswered = match inner.ask::<MarkCommit>(&batch).await {
Ok(()) => {
holding.observe(DurabilityEvidence::Acknowledged);
return Ok(());
}
Err(SemiontError::Bus(unanswered))
if unanswered.code == BusRequestErrorCode::Timeout =>
{
unanswered
}
Err(failed) => return Err(failed),
};
let observed = inner.ask_whether_recorded(resource_id, last).await;
holding.observe(observed);
if observed == DurabilityEvidence::ProbeConfirmed {
Ok(())
} else {
Err(unanswered.into())
}
}
pub async fn fail(
self,
error: impl Into<String>,
failure: JobFailure,
) -> Result<(), SemiontError> {
let failed = self.core.holding.fail(&error.into(), failure);
self.core.settle::<JobFail>(failed, false).await
}
pub async fn cancel(
self,
completed_units: Option<Vec<String>>,
unit_cursors: Option<BTreeMap<String, UnitCursor>>,
) -> Result<(), SemiontError> {
let holding = &self.core.holding;
let cancelled = JobCancelCommand {
annotation_id: holding.annotation_id.clone(),
completed_units,
unit_cursors,
..JobCancelCommand::new(
holding.resource_id.clone(),
holding.job_id.clone(),
holding.job_type,
)
};
self.core.settle::<JobCancel>(cancelled, false).await
}
}
impl Held<Mark> {
pub async fn complete(self, result: MarkJobResult) -> Result<(), SemiontError> {
let holding = &self.core.holding;
let completed = MarkJobCompleteCommand {
attempt: Some(holding.attempt),
result: Some(result),
durability: holding.observed(),
..MarkJobCompleteCommand::new(holding.resource_id.clone(), holding.job_id.clone())
};
self.core
.settle::<JobComplete>(JobCompleteCommand::MarkJobCompleteCommand(completed), true)
.await
}
}
impl Held<Yield> {
pub async fn complete(self, result: YieldJobResult) -> Result<(), SemiontError> {
let holding = &self.core.holding;
let completed = YieldJobCompleteCommand {
attempt: Some(holding.attempt),
annotation_id: holding.annotation_id.clone(),
result: Some(result),
durability: holding.observed(),
..YieldJobCompleteCommand::new(holding.resource_id.clone(), holding.job_id.clone())
};
self.core
.settle::<JobComplete>(JobCompleteCommand::YieldJobCompleteCommand(completed), true)
.await
}
}
pub enum HeldJob {
Mark(HeldMarkJob),
Yield(HeldYieldJob),
}
macro_rules! either {
($held:expr, $job:ident => $call:expr) => {
match $held {
HeldJob::Mark($job) => $call,
HeldJob::Yield($job) => $call,
}
};
}
impl HeldJob {
fn settled(&self) -> bool {
either!(self, job => job.core.holding.settled.load(Ordering::SeqCst))
}
pub fn job_id(&self) -> &JobId {
either!(self, job => job.job_id())
}
pub fn job_type(&self) -> JobType {
either!(self, job => job.job_type())
}
pub fn resource_id(&self) -> &ResourceId {
either!(self, job => job.resource_id())
}
pub fn params(&self) -> &JobParams {
either!(self, job => job.params())
}
pub fn completed_units(&self) -> &[String] {
either!(self, job => job.completed_units())
}
pub fn unit_cursors(&self) -> &BTreeMap<String, UnitCursor> {
either!(self, job => job.unit_cursors())
}
pub fn retry_count(&self) -> u64 {
either!(self, job => job.retry_count())
}
pub fn max_retries(&self) -> u64 {
either!(self, job => job.max_retries())
}
pub fn attempt(&self) -> i64 {
either!(self, job => job.attempt())
}
pub fn annotation_id(&self) -> Option<&AnnotationId> {
either!(self, job => job.annotation_id())
}
pub fn trace(&self) -> Option<&TraceCarrier> {
either!(self, job => job.trace())
}
pub fn cancelled(&self) -> watch::Receiver<bool> {
either!(self, job => job.cancelled())
}
pub async fn start(&self) -> Result<(), SemiontError> {
either!(self, job => job.start().await)
}
pub async fn progress(&self, progress: JobProgress) -> Result<(), SemiontError> {
either!(self, job => job.progress(progress).await)
}
pub async fn checkpoint(
&self,
completed_units: Vec<String>,
unit_cursors: Option<BTreeMap<String, UnitCursor>>,
) -> Result<(), SemiontError> {
either!(self, job => job.checkpoint(completed_units, unit_cursors).await)
}
pub async fn commit(
&self,
resource_id: &ResourceId,
annotations: Vec<Annotation>,
) -> Result<(), SemiontError> {
either!(self, job => job.commit(resource_id, annotations).await)
}
pub async fn fail(
self,
error: impl Into<String>,
failure: JobFailure,
) -> Result<(), SemiontError> {
either!(self, job => job.fail(error, failure).await)
}
pub async fn cancel(
self,
completed_units: Option<Vec<String>>,
unit_cursors: Option<BTreeMap<String, UnitCursor>>,
) -> Result<(), SemiontError> {
either!(self, job => job.cancel(completed_units, unit_cursors).await)
}
}
fn anchor_of(job_type: JobType, params: &JobParams) -> Option<AnnotationId> {
if job_type != JobType::Yield {
return None;
}
let focus = params.rest.get("context")?.get("focus")?;
if focus.get("kind")?.as_str()? != "annotation" {
return None;
}
focus.get("annotation")?.get("id")?.as_str()?.parse().ok()
}
fn held(
inner: &Arc<Inner>,
claimed: JobRunning,
trace: Option<TraceCarrier>,
) -> (Arc<Holding>, HeldJob) {
let JobRunning {
metadata, params, ..
} = claimed;
let holding = Arc::new(Holding {
resource_id: params.resource_id.clone(),
job_id: metadata.id.clone(),
job_type: metadata.r#type,
attempt: i64::try_from(metadata.retry_count)
.unwrap_or(i64::MAX)
.saturating_add(1),
annotation_id: anchor_of(metadata.r#type, ¶ms),
trace,
settled: AtomicBool::new(false),
cancelled: watch::channel(false).0,
durability: Mutex::new(None),
budget: metadata.clone(),
});
let core = Core {
inner: inner.clone(),
holding: holding.clone(),
params,
completed_units: metadata.completed_units.unwrap_or_default(),
unit_cursors: metadata.unit_cursors.unwrap_or_default(),
begun: AtomicBool::new(false),
};
let job = match holding.job_type {
JobType::Mark => HeldJob::Mark(Held {
core,
verb: PhantomData,
}),
JobType::Yield => HeldJob::Yield(Held {
core,
verb: PhantomData,
}),
};
(holding, job)
}
enum ClaimOutcome {
Job(Arc<Holding>, Box<HeldJob>),
Declined,
Refused(ClaimRefusal),
}
async fn claim_next(inner: Arc<Inner>) -> ClaimOutcome {
let claim = JobClaimCommand::new(inner.accepts.clone());
match inner
.wire
.result::<JobClaim>(&claim, inner.timing.job_claim)
.await
{
Ok(claimed) => {
let (holding, job) = held(&inner, claimed.payload.response, claimed.trace);
ClaimOutcome::Job(holding, Box::new(job))
}
Err(SemiontError::Bus(refused)) if refused.code == BusRequestErrorCode::NonePending => {
ClaimOutcome::Declined
}
Err(SemiontError::Bus(refused)) => ClaimOutcome::Refused(ClaimRefusal {
code: Some(refused.code),
message: refused.message,
}),
Err(other) => ClaimOutcome::Refused(ClaimRefusal {
code: None,
message: other.to_string(),
}),
}
}
type Claiming = Pin<Box<dyn Future<Output = ClaimOutcome> + Send>>;
async fn answered(claim: &mut Option<Claiming>) -> ClaimOutcome {
match claim {
Some(claim) => claim.as_mut().await,
None => std::future::pending().await,
}
}
type Handed = Result<HeldJob, ClaimRefusal>;
async fn run(inner: Arc<Inner>, out: mpsc::Sender<Handed>, mut stop: watch::Receiver<bool>) {
let transport = inner.transport.clone();
let unnamed: Vec<&str> = JOB_CLAIM_CHANNELS
.iter()
.copied()
.filter(|channel| !transport.is_subscribed(channel))
.collect();
let streams = (
inner.wire.frames_on(JobQueued::NAME),
inner.wire.frames_on(JobCancelRequested::NAME),
);
let (mut queued, mut cancels) = match streams {
(Ok(queued), Ok(cancels)) if unnamed.is_empty() => (queued, cancels),
_ => {
let _ = out
.send(Err(ClaimRefusal {
code: Some(BusRequestErrorCode::Unsubscribed),
message: format!(
"This transport's stream does not name {}: a worker on it would never be answered, woken or told of a cancellation. Give the transport JOB_CLAIM_CHANNELS.",
if unnamed.is_empty() { JOB_CLAIM_CHANNELS.join(", ") } else { unnamed.join(", ") }
),
}))
.await;
return;
}
};
let mut state = transport.state();
let mut was_open = *state.borrow_and_update() == ConnectionState::Open;
let mut want = was_open;
let mut wake_pending = false;
let mut claim: Option<Claiming> = None;
let check = inner.timing.held_job_stall_check;
let mut stall = tokio::time::interval_at(Instant::now() + check, check);
stall.set_missed_tick_behavior(MissedTickBehavior::Delay);
loop {
if want {
want = false;
if inner.holding() {
} else if claim.is_some() {
wake_pending = true;
} else {
wake_pending = false;
claim = Some(Box::pin(claim_next(inner.clone())));
}
}
tokio::select! {
biased;
_ = stop.changed() => break,
_ = out.closed() => break,
outcome = answered(&mut claim) => {
claim = None;
let again = match outcome {
ClaimOutcome::Job(holding, job) => {
inner.hold(&holding);
if out.send(Ok(*job)).await.is_err() {
break;
}
false
}
ClaimOutcome::Declined => wake_pending,
ClaimOutcome::Refused(refusal) => {
if out.send(Err(refusal)).await.is_err() {
break;
}
wake_pending
}
};
wake_pending = false;
want = again;
}
announced = queued.next() => match announced {
Some(Ok(Ok(announced))) => {
locked(&inner.state).last_queued_event_at = Some(SystemTime::now());
let announced = Value::Object(announced.payload);
want = inner
.accepts
.iter()
.any(|filter| job_matches_filter(filter, &announced));
}
Some(Ok(Err(_))) => {
locked(&inner.state).last_queued_event_at = Some(SystemTime::now());
want = true;
}
Some(Err(_)) => want = true,
None => break,
},
request = cancels.next() => match request {
Some(Ok(Ok(request))) => {
if let Some(job_id) = request.payload.get("jobId").and_then(Value::as_str) {
inner.signal_cancellation(job_id);
}
}
Some(Ok(Err(_)) | Err(_)) => {}
None => break,
},
changed = state.changed() => {
if changed.is_err() {
break;
}
let open = *state.borrow_and_update() == ConnectionState::Open;
want = open && !was_open;
was_open = open;
}
_ = inner.settled.notified() => want = true,
_ = stall.tick() => inner.look_for_stall(),
}
}
if let Some(claim) = claim
&& let ClaimOutcome::Job(holding, job) = claim.await
{
inner.fail_held(&holding, STOPPED_WHILE_HELD).await;
drop(job);
}
}
pub struct Claims {
inner: Arc<Inner>,
out: Mutex<Option<mpsc::Sender<Handed>>>,
handed: tokio::sync::Mutex<mpsc::Receiver<Handed>>,
stop: watch::Sender<bool>,
}
impl Claims {
pub(crate) fn new(transport: Arc<dyn Transport>, options: ClaimOptions) -> Claims {
let (out, handed) = mpsc::channel(1);
Claims {
inner: Arc::new(Inner {
wire: Bus::new(transport.clone()),
transport,
accepts: options.accepts,
timing: options.timing,
state: Mutex::new(State::default()),
settled: Notify::new(),
stall: watch::channel(None).0,
}),
out: Mutex::new(Some(out)),
handed: tokio::sync::Mutex::new(handed),
stop: watch::channel(false).0,
}
}
pub async fn next(&self) -> Option<Handed> {
let out = locked(&self.out).take();
if let Some(out) = out {
tokio::spawn(run(self.inner.clone(), out, self.stop.subscribe()));
}
let mut handed = self.handed.lock().await;
loop {
match handed.recv().await {
Some(Ok(job)) if job.settled() => {}
next => return next,
}
}
}
pub fn vitals(&self) -> WorkerVitals {
self.inner.vitals()
}
pub fn stalled(&self) -> watch::Receiver<Option<HeldJobStall>> {
self.inner.stall.subscribe()
}
pub async fn stop(&self) {
drop(locked(&self.out).take());
self.stop.send_replace(true);
let held = locked(&self.inner.state).held.clone();
if let Some(holding) = held {
self.inner.fail_held(&holding, STOPPED_WHILE_HELD).await;
}
}
}