use std::{
collections::HashMap,
fmt,
sync::{Arc, Mutex, PoisonError},
time::Duration,
};
use futures_util::future::BoxFuture;
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use sqlx::{PgPool, Row};
use time::{OffsetDateTime, SignedDuration};
use tokio::sync::Mutex as AsyncMutex;
use crate::{
db::await_task_result_snapshot,
error::{Error, Result, map_sqlx_error},
task::{TaskRef, decode_result},
types::{ClaimedTask, Json, JsonObject, RunId, TaskId},
workflow::{Sleep, Step},
};
const MAX_WORKFLOW_STORAGE_NAME_BYTES: usize = 1024;
const STEP_PREFIX: &str = "$step:";
const SLEEP_PREFIX: &str = "$sleep:";
const TASK_WAIT_PREFIX: &str = "$await-task:";
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
struct TaskWaitCheckpoint<Output> {
task_name: String,
output: Output,
}
#[derive(Debug, Clone)]
pub struct TaskContext {
inner: Arc<TaskContextInner>,
}
struct TaskContextInner {
pool: PgPool,
queue_name: String,
task: ClaimedTask,
headers: JsonObject,
checkpoint_cache: Mutex<HashMap<String, Json>>,
checkpoint_locks: Mutex<HashMap<String, Arc<AsyncMutex<()>>>>,
}
impl fmt::Debug for TaskContextInner {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("TaskContextInner")
.field("task_id", &self.task.task_id)
.field("run_id", &self.task.run_id)
.field("task_name", &self.task.task_name)
.field("attempt", &self.task.attempt)
.field("pool", &self.pool)
.field("queue_name", &self.queue_name)
.field("task", &self.task)
.field("headers", &self.headers)
.field("checkpoint_cache", &self.checkpoint_cache)
.field("checkpoint_locks", &self.checkpoint_locks)
.finish()
}
}
impl TaskContext {
pub(crate) async fn new(pool: PgPool, queue_name: String, task: ClaimedTask) -> Result<Self> {
let rows = sqlx::query(
r#"
SELECT checkpoint_name, checkpoint_state
FROM steda.get_task_checkpoint_states($1, $2, $3)
"#,
)
.bind(&queue_name)
.bind(task.task_id)
.bind(task.run_id)
.fetch_all(&pool)
.await
.map_err(map_sqlx_error)?;
let checkpoint_cache: HashMap<String, Json> = rows
.into_iter()
.map(|row| {
let name: String = row.get("checkpoint_name");
let state: Json = row.get("checkpoint_state");
(name, state)
})
.collect();
let headers = task.headers.clone().unwrap_or_default();
Ok(Self {
inner: Arc::new(TaskContextInner {
pool,
queue_name,
task,
headers,
checkpoint_cache: Mutex::new(checkpoint_cache),
checkpoint_locks: Mutex::new(HashMap::new()),
}),
})
}
pub fn task_id(&self) -> TaskId {
self.inner.task.task_id
}
pub fn run_id(&self) -> RunId {
self.inner.task.run_id
}
pub fn task_name(&self) -> &str {
&self.inner.task.task_name
}
pub fn queue_name(&self) -> &str {
&self.inner.queue_name
}
pub fn attempt(&self) -> u32 {
self.inner.task.attempt
}
pub fn headers(&self) -> &JsonObject {
&self.inner.headers
}
pub async fn step<Output, F, Fut>(&self, step: Step<Output>, f: F) -> Result<Output>
where
Output: Serialize + DeserializeOwned + Send + 'static,
F: FnOnce() -> Fut,
Fut: Future<Output = Result<Output>> + Send,
{
let name = workflow_storage_name(STEP_PREFIX, step.name())?;
self.checkpoint(&name, f).await
}
pub async fn sleep_for(&self, sleep: Sleep, duration: Duration) -> Result<()> {
let duration = SignedDuration::try_from(duration)
.map_err(|err| Error::InvalidOptions(err.to_string()))?;
let now = self.database_time().await?;
let wake_at = now
.checked_add(duration)
.ok_or_else(|| Error::InvalidOptions("sleep wake time is out of range".to_owned()))?;
self.sleep_until_inner(sleep, wake_at).await
}
pub async fn sleep_until(&self, sleep: Sleep, wake_at: OffsetDateTime) -> Result<()> {
self.sleep_until_inner(sleep, wake_at).await
}
async fn sleep_until_inner(&self, sleep: Sleep, wake_at: OffsetDateTime) -> Result<()> {
let name = workflow_storage_name(SLEEP_PREFIX, sleep.name())?;
let checkpoint_lock = self.checkpoint_lock(&name);
let _guard = checkpoint_lock.lock().await;
let actual_wake_at = if let Some(cached) = self.cached_checkpoint(&name) {
serde_json::from_value(cached)?
} else {
let serialized = serde_json::to_value(wake_at)?;
let (checkpoint_state, _) = self.persist_checkpoint(&name, serialized).await?;
serde_json::from_value(checkpoint_state)?
};
match self.schedule_run(actual_wake_at).await? {
ScheduleOutcome::Ready => Ok(()),
ScheduleOutcome::Suspended => Err(Error::Suspended),
ScheduleOutcome::Cancelled => Err(Error::Cancelled),
}
}
pub fn await_task<'a, Input, Output>(
&'a self,
task: &TaskRef<Input, Output>,
) -> TaskWait<'a, Input, Output>
where
Input: Serialize + DeserializeOwned + Send + 'static,
Output: Serialize + DeserializeOwned + Send + 'static,
{
TaskWait::new(self, task.clone())
}
async fn await_task_ref<Input, Output>(
&self,
task: TaskRef<Input, Output>,
timeout: Option<Duration>,
) -> Result<Output>
where
Input: Serialize + DeserializeOwned + Send + 'static,
Output: Serialize + DeserializeOwned + Send + 'static,
{
if task.queue_name() == self.inner.queue_name {
return Err(Error::InvalidOptions(
"TaskContext::await_task cannot wait on tasks in the same queue because this can deadlock workers. Spawn the child in a different queue.".to_owned(),
));
}
let checkpoint_name = format!("{TASK_WAIT_PREFIX}{}:{}", task.queue_name(), task.task_id());
let pool = self.inner.pool.clone();
let queue_name = task.queue_name().to_owned();
let expected_task_name = task.task_name().to_owned();
let task_name = expected_task_name.clone();
let task_id = task.task_id();
let checkpoint: TaskWaitCheckpoint<Output> = self
.checkpoint(&checkpoint_name, async move || {
let snapshot =
await_task_result_snapshot(&pool, &queue_name, &task_name, task_id, timeout)
.await?;
let output = decode_result(snapshot)?;
Ok(TaskWaitCheckpoint { task_name, output })
})
.await?;
if checkpoint.task_name != expected_task_name {
return Err(Error::TaskNameMismatch {
task_id,
expected: expected_task_name,
actual: checkpoint.task_name,
});
}
Ok(checkpoint.output)
}
async fn checkpoint<T, F, Fut>(&self, name: &str, f: F) -> Result<T>
where
T: Serialize + DeserializeOwned + Send + 'static,
F: FnOnce() -> Fut,
Fut: Future<Output = Result<T>> + Send,
{
let checkpoint_lock = self.checkpoint_lock(name);
let _guard = checkpoint_lock.lock().await;
if let Some(state) = self.cached_checkpoint(name) {
return Ok(serde_json::from_value(state)?);
}
let value = f().await?;
let serialized = serde_json::to_value(&value)?;
let (checkpoint_state, written) = self.persist_checkpoint(name, serialized).await?;
if written { Ok(value) } else { Ok(serde_json::from_value(checkpoint_state)?) }
}
fn checkpoint_lock(&self, name: &str) -> Arc<AsyncMutex<()>> {
let mut locks = self.inner.checkpoint_locks.lock().unwrap_or_else(PoisonError::into_inner);
Arc::clone(locks.entry(name.to_owned()).or_insert_with(|| Arc::new(AsyncMutex::new(()))))
}
fn cached_checkpoint(&self, name: &str) -> Option<Json> {
self.inner
.checkpoint_cache
.lock()
.unwrap_or_else(PoisonError::into_inner)
.get(name)
.cloned()
}
async fn persist_checkpoint(&self, name: &str, value: Json) -> Result<(Json, bool)> {
let row = sqlx::query(
r#"
SELECT checkpoint_state, written
FROM steda.set_task_checkpoint_state($1, $2, $3, $4, $5)
"#,
)
.bind(&self.inner.queue_name)
.bind(self.inner.task.task_id)
.bind(name)
.bind(value)
.bind(self.inner.task.run_id)
.fetch_one(&self.inner.pool)
.await
.map_err(map_sqlx_error)?;
let checkpoint_state: Json = row.get("checkpoint_state");
let written: bool = row.get("written");
self.cache_checkpoint(name, checkpoint_state.clone());
Ok((checkpoint_state, written))
}
async fn database_time(&self) -> Result<OffsetDateTime> {
let now =
sqlx::query_scalar("SELECT steda.current_time()").fetch_one(&self.inner.pool).await?;
Ok(now)
}
async fn schedule_run(&self, wake_at: OffsetDateTime) -> Result<ScheduleOutcome> {
let outcome: String = sqlx::query_scalar("SELECT steda.schedule_run($1, $2, $3)")
.bind(&self.inner.queue_name)
.bind(self.inner.task.run_id)
.bind(wake_at)
.fetch_one(&self.inner.pool)
.await
.map_err(map_sqlx_error)?;
match outcome.as_str() {
"ready" => Ok(ScheduleOutcome::Ready),
"suspended" => Ok(ScheduleOutcome::Suspended),
"cancelled" => Ok(ScheduleOutcome::Cancelled),
other => {
Err(Error::Other(format!("PostgreSQL returned unknown schedule outcome {other:?}")))
}
}
}
fn cache_checkpoint(&self, name: &str, value: Json) {
self.inner
.checkpoint_cache
.lock()
.unwrap_or_else(PoisonError::into_inner)
.insert(name.to_owned(), value);
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum ScheduleOutcome {
Ready,
Suspended,
Cancelled,
}
fn workflow_storage_name(prefix: &str, name: &str) -> Result<String> {
if name.trim().is_empty() {
return Err(Error::InvalidOptions("workflow identity must not be empty".to_owned()));
}
let maximum_name_bytes = MAX_WORKFLOW_STORAGE_NAME_BYTES
.checked_sub(prefix.len())
.expect("workflow namespace prefix must fit the PostgreSQL checkpoint name limit");
if name.len() > maximum_name_bytes {
return Err(Error::InvalidOptions(format!(
"workflow identity must be at most {maximum_name_bytes} bytes"
)));
}
Ok(format!("{prefix}{name}"))
}
#[must_use = "task waits do nothing until awaited"]
pub struct TaskWait<'a, Input, Output> {
context: &'a TaskContext,
task: TaskRef<Input, Output>,
timeout: Option<Duration>,
}
impl<Input, Output> fmt::Debug for TaskWait<'_, Input, Output> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("TaskWait")
.field("task", &self.task)
.field("timeout", &self.timeout)
.finish_non_exhaustive()
}
}
impl<'a, Input, Output> TaskWait<'a, Input, Output> {
const fn new(context: &'a TaskContext, task: TaskRef<Input, Output>) -> Self {
Self { context, task, timeout: None }
}
pub const fn timeout(mut self, timeout: Duration) -> Self {
self.timeout = Some(timeout);
self
}
}
impl<'a, Input, Output> IntoFuture for TaskWait<'a, Input, Output>
where
Input: Serialize + DeserializeOwned + Send + 'static,
Output: Serialize + DeserializeOwned + Send + 'static,
{
type Output = Result<Output>;
type IntoFuture = BoxFuture<'a, Self::Output>;
fn into_future(self) -> Self::IntoFuture {
Box::pin(async move { self.context.await_task_ref(self.task, self.timeout).await })
}
}
#[cfg(test)]
mod tests {
use super::{SLEEP_PREFIX, STEP_PREFIX, workflow_storage_name};
#[test]
fn workflow_identity_limit_includes_internal_namespace() {
let maximum_step_name = "x".repeat(1024 - STEP_PREFIX.len());
assert_eq!(
workflow_storage_name(STEP_PREFIX, &maximum_step_name)
.expect("maximum step name should fit")
.len(),
1024
);
assert!(workflow_storage_name(STEP_PREFIX, &(maximum_step_name + "x")).is_err());
let maximum_sleep_name = "x".repeat(1024 - SLEEP_PREFIX.len());
assert_eq!(
workflow_storage_name(SLEEP_PREFIX, &maximum_sleep_name)
.expect("maximum sleep name should fit")
.len(),
1024
);
assert!(workflow_storage_name(SLEEP_PREFIX, &(maximum_sleep_name + "x")).is_err());
}
}