use std::sync::Arc;
use std::time::Duration;
use serde::Serialize;
use serde::de::DeserializeOwned;
use tracing::Instrument;
use crate::checkpoint::{PendingStep, StepPlacement};
use crate::client::Client;
use crate::enqueue::EnqueueOptions;
use crate::error::{DurableError, EngineOnly, Error, Result};
use crate::handle::WorkflowHandle;
use crate::identity::refuse_app_name;
use crate::instance::Executor;
use crate::registry::{QualifiedWorkflowName, WorkflowRef};
use crate::serialization::{PayloadEncoding, SerializationFormat, encode_input};
use crate::sysdb::INTERNAL_QUEUE;
use crate::sysdb::types::{
Debounce, DebounceRequest, Timestamp, WorkflowIdReusePolicy, step_names,
validate_positive_duration,
};
use crate::workflow::{
Debounced, OwnedQueueRequest, OwnedStart, PendingStart, QueueRequest, Role, TargetVersion,
Timeout, create, encode_attributes, placed_workflow_id, refused_start, spawn_tracked,
validate_start,
};
const DEBOUNCE_OPERATION: &str = "debounce a workflow";
const RETRY_PAUSE: Duration = Duration::from_millis(100);
pub(crate) fn deduplication_id(workflow: &QualifiedWorkflowName, key: &str) -> String {
let name = workflow.name();
match (workflow.class_name(), workflow.config_name()) {
(None, None) => format!("{name}-{key}"),
(Some(class), None) => format!("{name}/{class}-{key}"),
(Some(class), Some(config)) => format!("{name}/{class}/{config}-{key}"),
(None, Some(config)) => format!("{name}//{config}-{key}"),
}
}
#[derive(Debug, Clone, Copy, Default)]
pub struct DebounceOptions<'a> {
pub workflow_id: Option<&'a str>,
}
#[derive(Debug, PartialEq, Eq)]
enum BounceAction {
Coalesce(String),
Enqueue,
Refuse(String),
Retry,
}
fn classify(
outcome: Debounce,
workflow: &QualifiedWorkflowName,
target_app: Option<&str>,
) -> BounceAction {
match outcome {
Debounce::Bounced { workflow_id } => BounceAction::Coalesce(workflow_id),
Debounce::Unheld => BounceAction::Enqueue,
Debounce::Held(holder) => {
fn present(part: Option<&str>) -> Option<&str> {
part.filter(|part| !part.is_empty())
}
let same_workflow = holder.workflow_name.as_deref() == Some(workflow.name())
&& present(holder.class_name.as_deref()) == workflow.class_name()
&& present(holder.config_name.as_deref()) == workflow.config_name();
let another_app = matches!(
(target_app, holder.application_name.as_deref()),
(Some(target), Some(owner)) if target != owner
);
if !holder.is_debounced || !same_workflow || another_app {
BounceAction::Refuse(holder.workflow_id)
} else {
BounceAction::Retry
}
}
}
}
struct Bounce<'a> {
workflow: &'a QualifiedWorkflowName,
queue_name: &'a str,
deduplication_id: String,
period: Duration,
input: String,
encoding: PayloadEncoding,
target_app: Option<&'a str>,
}
impl Bounce<'_> {
fn request(&self) -> DebounceRequest<'_> {
DebounceRequest {
workflow: self.workflow,
queue_name: self.queue_name,
deduplication_id: &self.deduplication_id,
period: self.period,
inputs: Some(&self.input),
serialization: Some(self.encoding.name()),
application_name: self.target_app,
}
}
fn action(&self, outcome: Debounce) -> BounceAction {
classify(outcome, self.workflow, self.target_app)
}
fn deduplicated(self, holder: String) -> Error {
Error::SystemDatabase(crate::sysdb::Error::QueueDeduplicated {
workflow_id: holder,
queue_name: self.queue_name.to_owned(),
deduplication_id: self.deduplication_id,
})
}
}
fn lost_the_key(result: &Result<impl Sized>) -> bool {
matches!(
result,
Err(Error::SystemDatabase(
crate::sysdb::Error::QueueDeduplicated { .. }
))
)
}
fn refuse_durations(period: Duration, debounce_timeout: Option<Duration>) -> Result<()> {
validate_positive_duration("period", period)
.map_err(|error| Error::refused_argument(DEBOUNCE_OPERATION, error))?;
if let Some(timeout) = debounce_timeout {
validate_positive_duration("debounce_timeout", timeout)
.map_err(|error| Error::refused_argument(DEBOUNCE_OPERATION, error))?;
}
Ok(())
}
pub struct Debouncer<'a, P, R, E = EngineOnly> {
pub workflow: WorkflowRef<P, R, E>,
pub queue: Option<&'a str>,
pub serialization: SerializationFormat,
pub timeout: Option<Duration>,
pub attributes: Option<&'a serde_json::Map<String, serde_json::Value>>,
pub debounce_timeout: Option<Duration>,
pub app_name: Option<&'a str>,
}
impl<'a, P, R, E> Debouncer<'a, P, R, E> {
pub fn new(workflow: &WorkflowRef<P, R, E>) -> Self {
Self {
workflow: workflow.clone(),
queue: None,
serialization: SerializationFormat::Default,
timeout: None,
attributes: None,
debounce_timeout: None,
app_name: None,
}
}
fn queue_request(&self) -> QueueRequest<'a> {
QueueRequest {
serialization: self.serialization,
..QueueRequest::new(self.queue.unwrap_or(INTERNAL_QUEUE))
}
}
}
impl<P, R, E> Clone for Debouncer<'_, P, R, E> {
fn clone(&self) -> Self {
Self {
workflow: self.workflow.clone(),
queue: self.queue,
serialization: self.serialization,
timeout: self.timeout,
attributes: self.attributes,
debounce_timeout: self.debounce_timeout,
app_name: self.app_name,
}
}
}
impl<P, R, E> std::fmt::Debug for Debouncer<'_, P, R, E> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Debouncer")
.field("workflow", &self.workflow)
.field("queue", &self.queue)
.field("serialization", &self.serialization)
.field("timeout", &self.timeout)
.field("attributes", &self.attributes)
.field("debounce_timeout", &self.debounce_timeout)
.field("app_name", &self.app_name)
.finish()
}
}
impl<P, R, E> Debouncer<'_, P, R, E>
where
P: Serialize + DeserializeOwned + Send + 'static,
R: Serialize + DeserializeOwned + Send + 'static,
E: DurableError,
{
pub fn debounce<'b>(&'b self, key: &str, period: Duration, input: P) -> PendingStart<'b, R, E> {
self.debounce_with(key, period, input, DebounceOptions::default())
}
pub fn debounce_with<'b>(
&'b self,
key: &str,
period: Duration,
input: P,
options: DebounceOptions<'b>,
) -> PendingStart<'b, R, E> {
let deduplication_id = deduplication_id(self.workflow.qualified_name(), key);
let built = self
.place(period)
.map(|(executor, bounce, start)| ((executor, start), bounce));
PendingStart::from_step(PendingStep::placed(
step_names::DEBOUNCE,
built,
move |(executor, start), bounce| async move {
self.debounced(
executor,
bounce,
start,
deduplication_id,
period,
input,
options,
)
.await
},
))
}
fn place(&self, period: Duration) -> Result<(Arc<Executor>, StepPlacement, StepPlacement)> {
let executor = self.workflow.executor(DEBOUNCE_OPERATION)?;
validate_start(
Some(&self.queue_request()),
self.timeout,
None,
WorkflowIdReusePolicy::ReturnExisting,
)?;
refuse_durations(period, self.debounce_timeout)?;
refuse_app_name(DEBOUNCE_OPERATION, self.app_name)?;
let bounce = StepPlacement::of(executor.connection(), DEBOUNCE_OPERATION)?;
if matches!(bounce, StepPlacement::InsideStep { .. }) {
return Err(Error::InsideStep {
operation: "debouncing a workflow".into(),
});
}
let start = StepPlacement::of(executor.connection(), DEBOUNCE_OPERATION)?;
Ok((executor, bounce, start))
}
#[allow(clippy::too_many_arguments)]
async fn debounced(
&self,
executor: Arc<Executor>,
mut bounce_at: StepPlacement,
mut start: StepPlacement,
deduplication_id: String,
period: Duration,
input: P,
call: DebounceOptions<'_>,
) -> Result<WorkflowHandle<R, E>> {
let conn = executor.connection();
let name: &QualifiedWorkflowName = self.workflow.qualified_name();
let queue = self.queue_request();
let encoding = PayloadEncoding::resolve(
queue.serialization,
PayloadEncoding::resolve(
self.workflow.serialization(),
executor.serializer().encoding(),
),
);
let bounce = Bounce {
workflow: name,
queue_name: queue.name,
deduplication_id,
period,
input: encode_input(&input, &encoding)?,
encoding,
target_app: Some(self.app_name.unwrap_or(executor.app_name())),
};
let attributes = encode_attributes(self.attributes)?;
let timeout = self.timeout.map_or(Timeout::None, Timeout::Explicit);
loop {
let outcome = conn
.sysdb()
.debounce_delayed_workflow(
&bounce.request(),
conn.step_caller(bounce_at.step_writer()),
)
.await
.map_err(|error| Error::refused_argument(DEBOUNCE_OPERATION, error))?;
match bounce.action(outcome) {
BounceAction::Coalesce(workflow_id) => {
return Ok(WorkflowHandle::polling(Arc::clone(conn), workflow_id, true));
}
BounceAction::Refuse(holder) => return Err(bounce.deduplicated(holder)),
BounceAction::Retry => crate::rt::sleep(RETRY_PAUSE).await,
BounceAction::Enqueue => {
let fresh = QueueRequest {
deduplication_id: Some(&bounce.deduplication_id),
delay: Some(period),
..queue.clone()
};
let request = OwnedStart {
workflow_id: placed_workflow_id(call.workflow_id, &start),
workflow_id_reuse_policy: WorkflowIdReusePolicy::ReturnExisting,
timeout,
queue: Some(OwnedQueueRequest::from(&fresh)),
input: Some(bounce.input.clone()),
encoding: bounce.encoding.clone(),
attributes: attributes.clone(),
started_at: Timestamp::now(),
debounce: Some(Debounced {
timeout: self.debounce_timeout,
}),
app_name: self.app_name.map(str::to_owned),
app_version: TargetVersion::Executor,
};
let created = self
.created(&executor, start, request)
.await
.map_err(|error| refused_start(Some(queue.name), error));
if !lost_the_key(&created) {
return created;
}
}
}
bounce_at = StepPlacement::of(conn, DEBOUNCE_OPERATION)?;
start = StepPlacement::of(conn, DEBOUNCE_OPERATION)?;
}
}
async fn created(
&self,
executor: &Arc<Executor>,
start: StepPlacement,
request: OwnedStart,
) -> Result<WorkflowHandle<R, E>> {
let workflow_id = request.workflow_id.clone();
let tracked = match &start {
StepPlacement::Recorded { ctx, .. } => Some(ctx.starts().token()),
_ => None,
};
let creating = create::<R, E>(
Arc::clone(executor),
start,
self.workflow.qualified_name().clone(),
request,
)
.instrument(tracing::Span::current());
let creating = async move {
let _tracked = tracked;
creating.await
};
let label = format!("debounce {workflow_id}");
match spawn_tracked(executor, Role::Workflow, label, creating).await {
Ok(created) => created,
Err(join) if join.is_cancelled() => Err(Error::Interrupted { workflow_id }),
Err(join) => std::panic::resume_unwind(join.into_panic()),
}
}
}
#[derive(Debug, Clone)]
pub struct DebouncerClient<'a> {
pub client: Client,
pub workflow: QualifiedWorkflowName,
pub queue: &'a str,
pub serialization: SerializationFormat,
pub timeout: Option<Duration>,
pub attributes: Option<&'a serde_json::Map<String, serde_json::Value>>,
pub app_version: Option<&'a str>,
pub app_name: Option<&'a str>,
pub debounce_timeout: Option<Duration>,
}
impl<'a> DebouncerClient<'a> {
pub fn new(
client: &Client,
workflow: impl Into<QualifiedWorkflowName>,
queue: &'a str,
) -> Self {
Self {
client: client.clone(),
workflow: workflow.into(),
queue,
serialization: SerializationFormat::Default,
timeout: None,
attributes: None,
app_version: None,
app_name: None,
debounce_timeout: None,
}
}
fn enqueue_options(&self) -> EnqueueOptions<'a> {
EnqueueOptions {
app_name: self.app_name,
app_version: self.app_version,
timeout: self.timeout.map_or(Timeout::None, Timeout::Explicit),
attributes: self.attributes,
serialization: self.serialization,
..EnqueueOptions::new(self.queue)
}
}
pub async fn debounce<P, R, E>(
&self,
key: &str,
period: Duration,
input: P,
) -> Result<WorkflowHandle<R, E>>
where
P: Serialize,
{
self.debounce_with(key, period, input, DebounceOptions::default())
.await
}
pub async fn debounce_with<P, R, E>(
&self,
key: &str,
period: Duration,
input: P,
call: DebounceOptions<'_>,
) -> Result<WorkflowHandle<R, E>>
where
P: Serialize,
{
refuse_app_name(DEBOUNCE_OPERATION, self.app_name)?;
let options = self.enqueue_options();
options.validate(&self.workflow)?;
refuse_durations(period, self.debounce_timeout)?;
let conn = self.client.connection();
let encoding = PayloadEncoding::resolve(self.serialization, conn.serializer().encoding());
let bounce = Bounce {
workflow: &self.workflow,
queue_name: self.queue,
deduplication_id: deduplication_id(&self.workflow, key),
period,
input: encode_input(&input, &encoding)?,
encoding,
target_app: self.app_name.or(conn.app_name()),
};
loop {
let outcome = conn
.sysdb()
.debounce_delayed_workflow(&bounce.request(), None)
.await
.map_err(|error| Error::refused_argument(DEBOUNCE_OPERATION, error))?;
match bounce.action(outcome) {
BounceAction::Coalesce(workflow_id) => {
return Ok(WorkflowHandle::polling(Arc::clone(conn), workflow_id, true));
}
BounceAction::Refuse(holder) => return Err(bounce.deduplicated(holder)),
BounceAction::Retry => crate::rt::sleep(RETRY_PAUSE).await,
BounceAction::Enqueue => {
let options = EnqueueOptions {
deduplication_id: Some(&bounce.deduplication_id),
delay: Some(period),
workflow_id: call.workflow_id,
app_name: bounce.target_app,
..options.clone()
};
let created = self
.client
.enqueue_encoded(
&self.workflow,
&bounce.input,
bounce.encoding.clone(),
options,
Some(Debounced {
timeout: self.debounce_timeout,
}),
)
.await;
if !lost_the_key(&created) {
return created;
}
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::sysdb::types::DebounceHolder;
#[test]
fn each_key_spells_its_deduplication_id() {
let id = |workflow: QualifiedWorkflowName| deduplication_id(&workflow, "k");
assert_eq!(id(QualifiedWorkflowName::new("f")), "f-k");
assert_eq!(id(QualifiedWorkflowName::method("f", "C")), "f/C-k");
assert_eq!(
id(QualifiedWorkflowName::instance("f", "C", "eu")),
"f/C/eu-k"
);
assert_eq!(
id(QualifiedWorkflowName::instance("f", "", "eu")),
"f//eu-k"
);
}
fn holder(name: &str, is_debounced: bool, app: Option<&str>) -> Debounce {
Debounce::Held(DebounceHolder {
workflow_id: "holder".to_owned(),
is_debounced,
workflow_name: Some(name.to_owned()),
class_name: None,
config_name: None,
application_name: app.map(str::to_owned),
})
}
#[test]
fn the_deduplication_id_is_the_workflow_then_the_key() {
let free = |name: &str| QualifiedWorkflowName::new(name);
assert_eq!(deduplication_id(&free("wf"), "key"), "wf-key");
assert_eq!(
deduplication_id(&free("a"), "b-c"),
deduplication_id(&free("a-b"), "c")
);
}
#[test]
fn the_deduplication_id_spells_the_whole_name() {
let eu = QualifiedWorkflowName::instance("wf", "Billing", "eu");
let us = QualifiedWorkflowName::instance("wf", "Billing", "us");
let method = QualifiedWorkflowName::method("wf", "Billing");
assert_eq!(deduplication_id(&eu, "key"), "wf/Billing/eu-key");
assert_eq!(deduplication_id(&method, "key"), "wf/Billing-key");
let ids = [
deduplication_id(&eu, "key"),
deduplication_id(&us, "key"),
deduplication_id(&method, "key"),
deduplication_id(&QualifiedWorkflowName::new("wf"), "key"),
];
let distinct: std::collections::HashSet<_> = ids.iter().collect();
assert_eq!(distinct.len(), ids.len(), "{ids:?}");
}
#[test]
fn a_bounce_is_classified_by_its_holder() {
let wf = QualifiedWorkflowName::new("wf");
assert_eq!(
classify(
Debounce::Bounced {
workflow_id: "w".to_owned()
},
&wf,
Some("app")
),
BounceAction::Coalesce("w".to_owned())
);
assert_eq!(
classify(Debounce::Unheld, &wf, Some("app")),
BounceAction::Enqueue
);
assert_eq!(
classify(holder("wf", false, Some("app")), &wf, Some("app")),
BounceAction::Refuse("holder".to_owned())
);
assert_eq!(
classify(holder("wf-b", true, Some("app")), &wf, Some("app")),
BounceAction::Refuse("holder".to_owned())
);
let east = QualifiedWorkflowName::instance("wf", "Billing", "east");
let west = Debounce::Held(DebounceHolder {
workflow_id: "holder".to_owned(),
is_debounced: true,
workflow_name: Some("wf".to_owned()),
class_name: Some("Billing".to_owned()),
config_name: Some("west".to_owned()),
application_name: None,
});
assert_eq!(
classify(west, &east, Some("app")),
BounceAction::Refuse("holder".to_owned())
);
let blank = Debounce::Held(DebounceHolder {
workflow_id: "holder".to_owned(),
is_debounced: true,
workflow_name: Some("wf".to_owned()),
class_name: Some(String::new()),
config_name: Some(String::new()),
application_name: Some("app".to_owned()),
});
assert_eq!(classify(blank, &wf, Some("app")), BounceAction::Retry);
assert_eq!(
classify(holder("wf", true, Some("other")), &wf, Some("app")),
BounceAction::Refuse("holder".to_owned())
);
assert_eq!(
classify(holder("wf", true, Some("app")), &wf, Some("app")),
BounceAction::Retry
);
assert_eq!(
classify(holder("wf", true, Some("other")), &wf, None),
BounceAction::Retry
);
assert_eq!(
classify(holder("wf", true, None), &wf, Some("app")),
BounceAction::Retry
);
}
#[test]
fn a_zero_period_or_timeout_is_refused() {
assert!(matches!(
refuse_durations(Duration::ZERO, None),
Err(Error::InvalidArgument { .. })
));
assert!(matches!(
refuse_durations(Duration::from_secs(1), Some(Duration::ZERO)),
Err(Error::InvalidArgument { .. })
));
refuse_durations(Duration::from_secs(1), Some(Duration::from_secs(1))).expect("valid");
}
#[test]
fn a_bad_app_name_is_refused() {
for bad in ["", "No"] {
let error = refuse_app_name(DEBOUNCE_OPERATION, Some(bad)).expect_err(bad);
assert!(
matches!(&error, Error::InvalidArgument { operation, detail }
if operation == DEBOUNCE_OPERATION
&& detail.contains("`app_name`")
&& !detail.contains("application_name")),
"{error}"
);
}
refuse_app_name(DEBOUNCE_OPERATION, Some("some-app")).expect("a valid name");
refuse_app_name(DEBOUNCE_OPERATION, None).expect("no name");
}
}