use crate::error::{Error, Result};
use crate::workflow::WorkflowContext;
use metrics::{counter, histogram};
use serde::{de::DeserializeOwned, Serialize};
use std::future::Future;
use std::marker::PhantomData;
use std::time::Instant;
pub trait Executable<O> {
fn execute_in_context(
self,
ctx: &WorkflowContext,
step_name: &str,
) -> impl Future<Output = Result<O>> + Send;
}
pub struct TaskExecution<I, O> {
input: I,
_output: PhantomData<O>,
}
impl<I, O> TaskExecution<I, O>
where
I: Serialize + Send + Sync + 'static,
O: DeserializeOwned + Send + Sync + 'static,
{
pub fn new(input: I) -> Self {
Self {
input,
_output: PhantomData,
}
}
}
impl<I, O> Executable<O> for TaskExecution<I, O>
where
I: Serialize + Send + Sync + 'static,
O: DeserializeOwned + Send + Sync + 'static,
{
async fn execute_in_context(self, ctx: &WorkflowContext, step_name: &str) -> Result<O> {
use crate::workflow::command::{ScheduleTaskCommand, TaskRetryPolicy};
use std::time::Duration;
let start = Instant::now();
counter!(
"orcher.workflow.execute.task.calls",
"step_name" => step_name.to_string()
)
.increment(1);
let sequence = ctx.next_sequence();
let task_id = format!("{}_{}", step_name, sequence);
let state = ctx.state.lock().unwrap();
let replaying = state.replaying;
let cached_result = state.pending_task_results.get(&task_id).cloned();
if replaying && cached_result.is_some() {
state.observe(&task_id);
}
tracing::warn!(
task_id = %task_id,
step_name = %step_name,
replaying = replaying,
has_cached = cached_result.is_some(),
cached_len = cached_result.as_ref().map(|v| v.len()).unwrap_or(0),
total_pending = state.pending_task_results.len(),
"🔍 TASK_EXEC: Checking for cached result"
);
drop(state);
if replaying {
if let Some(result_bytes) = cached_result {
if result_bytes.is_empty() {
tracing::error!(
task_id = %task_id,
step_name = %step_name,
"🔴 TASK_EXEC: Found EMPTY cached result during replay - this should never happen!"
);
return Err(Error::Workflow(crate::error::WorkflowError::StateError(
format!(
"Invalid replay state: Task '{}' has empty result in cache. \
This indicates a bug in result injection or journal conversion.",
step_name
),
)));
}
if let Ok(v) = serde_json::from_slice::<serde_json::Value>(&result_bytes) {
if v.get(crate::error::TASK_FAILED_SENTINEL_KEY)
.and_then(|b| b.as_bool())
.unwrap_or(false)
{
let reason = v
.get("message")
.and_then(|m| m.as_str())
.unwrap_or("task failed")
.to_string();
tracing::info!(
task_id = %task_id,
step_name = %step_name,
reason = %reason,
"Replaying terminal task failure from journal"
);
return Err(Error::Workflow(crate::error::WorkflowError::TaskFailed {
task_type: step_name.to_string(),
attempts: v.get("attempts").and_then(|a| a.as_u64()).unwrap_or(1)
as u32,
reason,
}));
}
}
counter!(
"orcher.workflow.execute.cache_hits",
"step_name" => step_name.to_string(),
"step_type" => "task"
)
.increment(1);
tracing::debug!(
task_id = %task_id,
step_name = %step_name,
result_len = result_bytes.len(),
"Replaying task execution from journal"
);
let output: O = serde_json::from_slice(&result_bytes).map_err(|e| {
tracing::error!(
task_id = %task_id,
step_name = %step_name,
result_len = result_bytes.len(),
result_preview = ?&result_bytes[..result_bytes.len().min(100)],
error = %e,
"🔴 TASK_EXEC: Failed to deserialize cached result"
);
Error::Serialization(format!(
"Failed to deserialize task result for '{}': {} (result length: {})",
step_name,
e,
result_bytes.len()
))
})?;
histogram!(
"orcher.workflow.step.duration_ms",
"step_type" => "task",
"step_name" => step_name.to_string(),
"cached" => "true"
)
.record(start.elapsed().as_millis() as f64);
return Ok(output);
}
tracing::info!(
task_id = %task_id,
step_name = %step_name,
"Task result not in cache during replay - this is a new step, will execute fresh"
);
}
counter!(
"orcher.workflow.execute.cache_misses",
"step_name" => step_name.to_string(),
"step_type" => "task"
)
.increment(1);
let input_bytes = serde_json::to_vec(&self.input).map_err(|e| {
Error::Serialization(format!(
"Failed to serialize task input for '{}': {}",
step_name, e
))
})?;
tracing::debug!(
task_id = %task_id,
step_name = %step_name,
"Scheduling task execution (will be sent to server)"
);
let task_queue = {
let mut state = ctx.state.lock().unwrap();
state
.task_queue_override
.take()
.unwrap_or_else(|| ctx.execution().task_queue.clone())
};
let retry_policy = crate::worker::registration::task_retry_policy(step_name)
.map(|p| TaskRetryPolicy {
max_attempts: p.max_attempts,
initial_interval: p.initial_interval,
max_interval: p.max_interval,
backoff_coefficient: p.backoff_coefficient,
non_retryable_errors: p.non_retryable_errors,
})
.unwrap_or(TaskRetryPolicy {
max_attempts: 3,
initial_interval: Duration::from_secs(1),
max_interval: Duration::from_secs(60),
backoff_coefficient: 2.0,
non_retryable_errors: vec![],
});
let declared = crate::worker::registration::task_limits(step_name);
let command = ScheduleTaskCommand {
sequence,
task_id: task_id.clone(),
task_type: step_name.to_string(),
task_queue,
input: input_bytes,
timeout: declared
.and_then(|d| d.timeout_secs)
.map(Duration::from_secs)
.unwrap_or(Duration::from_secs(300)),
heartbeat_timeout: declared
.and_then(|d| d.heartbeat_timeout_secs)
.map(Duration::from_secs),
queue_timeout: None,
retry_policy: Some(retry_policy),
headers: vec![],
};
{
let mut state = ctx.state.lock().unwrap();
state.step_type_map.insert(task_id.clone(), 0); }
ctx.add_command(crate::workflow::command::WorkflowCommand::ScheduleTask(
command,
));
histogram!(
"orcher.workflow.step.duration_ms",
"step_type" => "task",
"step_name" => step_name.to_string(),
"cached" => "false"
)
.record(start.elapsed().as_millis() as f64);
Err(Error::Workflow(crate::error::WorkflowError::Suspended {
reason: format!("Task '{}' scheduled for execution", step_name),
pending_operations: vec![task_id],
}))
}
}
pub struct ClosureExecution<F, Fut, O> {
closure: F,
_marker: PhantomData<(Fut, O)>,
}
impl<F, Fut, O> ClosureExecution<F, Fut, O>
where
F: FnOnce() -> Fut + Send + 'static,
Fut: Future<Output = Result<O>> + Send,
O: Serialize + DeserializeOwned + Send + Sync + 'static,
{
pub fn new(closure: F) -> Self {
Self {
closure,
_marker: PhantomData,
}
}
}
impl<F, Fut, O> Executable<O> for ClosureExecution<F, Fut, O>
where
F: FnOnce() -> Fut + Send + 'static,
Fut: Future<Output = Result<O>> + Send,
O: Serialize + DeserializeOwned + Send + Sync + 'static,
{
async fn execute_in_context(self, ctx: &WorkflowContext, step_name: &str) -> Result<O> {
let start = Instant::now();
counter!(
"orcher.workflow.execute.closure.calls",
"step_name" => step_name.to_string()
)
.increment(1);
let step_id = format!("{}_{}", step_name, ctx.next_sequence());
let (replaying, cached_result) = {
let state = ctx.state.lock().unwrap();
let replaying = state.replaying;
let cached_result = state.pending_task_results.get(&step_id).cloned();
(replaying, cached_result)
};
if replaying {
if let Some(result_bytes) = cached_result {
counter!(
"orcher.workflow.execute.cache_hits",
"step_name" => step_name.to_string(),
"step_type" => "closure"
)
.increment(1);
tracing::debug!(
step_id = %step_id,
step_name = %step_name,
"Replaying closure execution from journal"
);
let output: O = serde_json::from_slice(&result_bytes).map_err(|e| {
Error::Serialization(format!(
"Failed to deserialize closure result for '{}': {}",
step_name, e
))
})?;
histogram!(
"orcher.workflow.step.duration_ms",
"step_type" => "closure",
"step_name" => step_name.to_string(),
"cached" => "true"
)
.record(start.elapsed().as_millis() as f64);
return Ok(output);
}
tracing::info!(
step_id = %step_id,
step_name = %step_name,
"Closure result not in cache during replay - this is a new step, will execute fresh"
);
}
counter!(
"orcher.workflow.execute.cache_misses",
"step_name" => step_name.to_string(),
"step_type" => "closure"
)
.increment(1);
tracing::debug!(
step_id = %step_id,
step_name = %step_name,
"Executing closure (fresh execution)"
);
let future = (self.closure)();
let exec_result = future.await;
{
let mut state = ctx.state.lock().unwrap();
let count = state
.step_attempt_counts
.entry(step_id.clone())
.or_insert(0);
*count += 1;
}
let result = match exec_result {
Ok(r) => r,
Err(e) => {
let failure = orcher_proto::orcher::v1::Failure {
message: e.to_string(),
source: "ClosureExecution".to_string(),
failure_type: "ClosureError".to_string(),
..Default::default()
};
{
let mut state = ctx.state.lock().unwrap();
state.pending_task_failures.insert(step_id.clone(), failure);
state.step_type_map.insert(step_id.clone(), 1);
}
return Err(e);
}
};
let result_bytes = serde_json::to_vec(&result).map_err(|e| {
Error::Serialization(format!(
"Failed to serialize closure result for '{}': {}",
step_name, e
))
})?;
{
let mut state = ctx.state.lock().unwrap();
state
.pending_task_results
.insert(step_id.clone(), result_bytes);
state.step_type_map.insert(step_id.clone(), 1); }
tracing::info!(
step_id = %step_id,
step_name = %step_name,
duration_ms = start.elapsed().as_millis(),
"Closure executed successfully"
);
histogram!(
"orcher.workflow.step.duration_ms",
"step_type" => "closure",
"step_name" => step_name.to_string(),
"cached" => "false"
)
.record(start.elapsed().as_millis() as f64);
Ok(result)
}
}
#[doc(hidden)]
pub fn task_execution<I, O>(input: I) -> TaskExecution<I, O>
where
I: Serialize + Send + Sync + 'static,
O: DeserializeOwned + Send + Sync + 'static,
{
TaskExecution::new(input)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_task_execution_construction() {
let _task_exec = TaskExecution::<String, String>::new("input".to_string());
}
}