use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Instant;
use tokio::sync::{RwLock, Semaphore};
use crate::error::Error;
use crate::payload::Payload;
use crate::worker::registration::{TaskHandlerFn, WorkflowHandlerFn};
use crate::workflow::closure_commands::extract_closure_commands;
use crate::workflow::ReachedStep;
use crate::workflow::{WorkflowCommand, WorkflowContext};
use crate::TaskContext;
use crate::worker::activation::{data, is_terminal, kernel_command, Produced};
use orcher_proto::host::v1::{task_outcome, TaskAssignment, TaskOutcome};
use orcher_proto::kernel::v1::{
command, query_answer, update_answer, Command, EndCancelled, Failure, FailureCode, QueryAnswer,
UpdateAnswer,
};
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct RuntimeConfig {
pub max_concurrent_workflows: usize,
pub max_concurrent_tasks: usize,
pub default_task_queue: String,
pub namespace: String,
pub codec_chain: Option<Arc<orcher_sdk_core::payload::codec::CodecChain>>,
}
impl Default for RuntimeConfig {
fn default() -> Self {
Self {
max_concurrent_workflows: 100,
max_concurrent_tasks: 200,
default_task_queue: "default".to_string(),
namespace: "default".to_string(),
codec_chain: None,
}
}
}
#[derive(Debug, Clone)]
pub struct RuntimeStats {
pub workflows_executed: u64,
pub workflows_succeeded: u64,
pub workflows_failed: u64,
pub tasks_executed: u64,
pub tasks_succeeded: u64,
pub tasks_failed: u64,
}
pub struct ExecutionRuntime {
config: RuntimeConfig,
workflow_handlers: Arc<RwLock<HashMap<String, WorkflowHandlerFn>>>,
task_handlers: Arc<RwLock<HashMap<String, TaskHandlerFn>>>,
workflow_semaphore: Arc<Semaphore>,
task_semaphore: Arc<Semaphore>,
workflows_executed: AtomicU64,
workflows_succeeded: AtomicU64,
workflows_failed: AtomicU64,
tasks_executed: AtomicU64,
tasks_succeeded: AtomicU64,
tasks_failed: AtomicU64,
last_success_time: std::sync::Mutex<Option<Instant>>,
}
impl ExecutionRuntime {
pub fn new(config: RuntimeConfig) -> Self {
let workflow_semaphore = Arc::new(Semaphore::new(config.max_concurrent_workflows));
let task_semaphore = Arc::new(Semaphore::new(config.max_concurrent_tasks));
Self {
config,
workflow_handlers: Arc::new(RwLock::new(HashMap::new())),
task_handlers: Arc::new(RwLock::new(HashMap::new())),
workflow_semaphore,
task_semaphore,
workflows_executed: AtomicU64::new(0),
workflows_succeeded: AtomicU64::new(0),
workflows_failed: AtomicU64::new(0),
tasks_executed: AtomicU64::new(0),
tasks_succeeded: AtomicU64::new(0),
tasks_failed: AtomicU64::new(0),
last_success_time: std::sync::Mutex::new(None),
}
}
pub fn stats(&self) -> RuntimeStats {
RuntimeStats {
workflows_executed: self.workflows_executed.load(Ordering::Relaxed),
workflows_succeeded: self.workflows_succeeded.load(Ordering::Relaxed),
workflows_failed: self.workflows_failed.load(Ordering::Relaxed),
tasks_executed: self.tasks_executed.load(Ordering::Relaxed),
tasks_succeeded: self.tasks_succeeded.load(Ordering::Relaxed),
tasks_failed: self.tasks_failed.load(Ordering::Relaxed),
}
}
pub fn last_success_time(&self) -> Option<Instant> {
*self.last_success_time.lock().unwrap()
}
pub async fn register_workflow_handler(
&self,
workflow_type: impl Into<String>,
handler: WorkflowHandlerFn,
) {
let workflow_type = workflow_type.into();
tracing::info!(workflow_type = %workflow_type, "Registering workflow handler in runtime");
self.workflow_handlers
.write()
.await
.insert(workflow_type, handler);
}
pub async fn register_task_handler(
&self,
task_type: impl Into<String>,
handler: TaskHandlerFn,
) {
let task_type = task_type.into();
tracing::info!(task_type = %task_type, "Registering task handler in runtime");
self.task_handlers.write().await.insert(task_type, handler);
}
pub(crate) async fn execute_activation(
&self,
activation: orcher_proto::kernel::v1::RunActivation,
) -> std::result::Result<(Produced, Vec<ReachedStep>), String> {
let loaded = crate::worker::activation::load(activation, &self.config.namespace)?;
let execution = loaded.ctx.execution().clone();
self.workflows_executed.fetch_add(1, Ordering::Relaxed);
let _permit = self
.workflow_semaphore
.acquire()
.await
.map_err(|e| format!("Failed to acquire execution permit: {e}"))?;
let handler = {
let handlers = self.workflow_handlers.read().await;
handlers.get(&execution.workflow_type).cloned()
};
let Some(handler) = handler else {
tracing::error!(
workflow_type = %execution.workflow_type,
"No handler registered for workflow type"
);
return Ok((
Produced::failed(code_failure(format!(
"No handler registered for workflow type: {}",
execution.workflow_type
))),
Vec::new(),
));
};
let input = self.decode_input(loaded.input.unwrap_or(serde_json::Value::Null));
Ok(self
.run_handler(loaded.ctx, input, handler, &loaded.asked)
.await)
}
fn decode_input(&self, input: serde_json::Value) -> Payload {
let mut payload =
Payload::from_json(&input).unwrap_or_else(|_| Payload::new_data(Vec::new()));
if let Some(ref codec) = self.config.codec_chain {
use orcher_sdk_core::payload::codec::PayloadCodec;
match codec.decode(&payload) {
Ok(decoded) => payload = decoded,
Err(e) => {
tracing::warn!(error = %e, "Failed to decode workflow input with codec chain");
}
}
}
payload
}
pub(crate) async fn run_handler(
&self,
ctx: WorkflowContext,
input: Payload,
handler: WorkflowHandlerFn,
asked: &[Asked],
) -> (Produced, Vec<ReachedStep>) {
let execution = ctx.execution().clone();
let (workflow_id, task_queue) = (
execution.workflow_id.as_str(),
execution.task_queue.as_str(),
);
let handler_result = handler(ctx.clone(), input).await;
let (queries, updates) = self.answer_asked(&ctx, asked).await;
let reached = ctx.take_reached_steps();
let produced = match handler_result {
Ok(output) => {
let r = self.handle_workflow_completion(
Ending::Completed(output),
ctx,
task_queue,
queries,
updates,
);
self.workflows_succeeded.fetch_add(1, Ordering::Relaxed);
*self.last_success_time.lock().unwrap() = Some(Instant::now());
r
}
Err(Error::Workflow(crate::error::WorkflowError::Canceled))
if ctx.is_cancel_requested() =>
{
tracing::info!(workflow_id = %workflow_id, "Workflow ended as cancelled");
self.handle_workflow_completion(
Ending::Cancelled,
ctx,
task_queue,
queries,
updates,
)
}
Err(Error::Workflow(crate::error::WorkflowError::Suspended {
reason,
pending_operations,
})) => {
tracing::info!(
workflow_id = %workflow_id,
reason = %reason,
pending_count = pending_operations.len(),
"Workflow suspended"
);
self.handle_workflow_suspension(ctx, task_queue, queries, updates)
}
Err(e) => {
tracing::error!(error = %e, "Workflow handler failed");
self.workflows_failed.fetch_add(1, Ordering::Relaxed);
return (
Produced {
issued: self.issued_commands(&ctx, task_queue),
calls_known: true,
..Produced::failed(workflow_failure(&e))
},
reached,
);
}
};
(produced, reached)
}
fn issued_commands(&self, ctx: &WorkflowContext, task_queue: &str) -> Vec<Command> {
let mut issued = self.closure_result_commands(ctx, task_queue);
let kernel_ids = ctx.kernel_assigns_ids();
issued.extend(
ctx.take_commands()
.into_iter()
.map(|cmd| kernel_command(cmd, task_queue, kernel_ids)),
);
issued
}
fn handle_workflow_completion(
&self,
ending: Ending,
ctx: WorkflowContext,
task_queue: &str,
queries: Vec<QueryAnswer>,
updates: Vec<UpdateAnswer>,
) -> Produced {
let mut issued = self.issued_commands(&ctx, task_queue);
if !issued.iter().any(is_terminal) {
issued.push(match ending {
Ending::Completed(output) => kernel_command(
crate::workflow::WorkflowCommand::CompleteWorkflow(
crate::workflow::CompleteWorkflowCommand {
result: output.data,
},
),
task_queue,
ctx.kernel_assigns_ids(),
),
Ending::Cancelled => Command {
what: Some(command::What::EndCancelled(EndCancelled::default())),
..Default::default()
},
});
}
Produced {
issued,
queries,
updates,
..Default::default()
}
}
fn closure_result_commands(&self, ctx: &WorkflowContext, task_queue: &str) -> Vec<Command> {
let mut closure_commands = match extract_closure_commands(ctx) {
Ok(cmds) => cmds,
Err(e) => {
tracing::warn!(error = %e, "Failed to extract closure commands");
return Vec::new();
}
};
closure_commands.sort_by_key(|cmd| {
cmd.step_name
.rsplit(['_', '-'])
.next()
.and_then(|n| n.parse::<u64>().ok())
.unwrap_or(u64::MAX)
});
closure_commands
.into_iter()
.map(|cmd| {
kernel_command(
WorkflowCommand::RecordStepResult(cmd),
task_queue,
ctx.kernel_assigns_ids(),
)
})
.collect()
}
fn handle_workflow_suspension(
&self,
ctx: WorkflowContext,
task_queue: &str,
queries: Vec<QueryAnswer>,
updates: Vec<UpdateAnswer>,
) -> Produced {
let sdk_commands = ctx.take_commands();
if sdk_commands.is_empty() {
return Produced::failed(code_failure(
"Workflow suspended without generating commands".to_string(),
));
}
let mut issued = self.closure_result_commands(&ctx, task_queue);
let kernel_ids = ctx.kernel_assigns_ids();
issued.extend(
sdk_commands
.into_iter()
.map(|cmd| kernel_command(cmd, task_queue, kernel_ids)),
);
Produced {
issued,
queries,
updates,
..Default::default()
}
}
pub(crate) async fn workflow_types(&self) -> Vec<String> {
self.workflow_handlers
.read()
.await
.keys()
.cloned()
.collect()
}
pub(crate) async fn task_types(&self) -> Vec<String> {
self.task_handlers.read().await.keys().cloned().collect()
}
pub(crate) async fn execute_assignment(
&self,
assignment: TaskAssignment,
host: &orcher_sdk_core::EncodedHost,
) -> TaskOutcome {
let token = assignment.token.clone();
let run = assignment.run.clone().unwrap_or_default();
let ctx = TaskContext::with_host(
run.workflow_id,
run.run_id,
assignment.task_id.clone(),
assignment.attempt as i32,
assignment
.heartbeat_timeout
.as_ref()
.map(|d| std::time::Duration::new(d.seconds.max(0) as u64, d.nanos.max(0) as u32)),
host.clone(),
token.clone(),
);
let input = assignment
.input
.into_iter()
.next()
.map(|payload| payload.data)
.unwrap_or_default();
let result = match self.run_task(ctx, &assignment.task_type, input).await {
Ok(output) => task_outcome::Result::Value(data(output)),
Err(error) => task_outcome::Result::Failure(reported_failure(&error)),
};
TaskOutcome {
token,
result: Some(result),
..Default::default()
}
}
#[allow(clippy::result_large_err)]
async fn run_task(
&self,
ctx: TaskContext,
task_type: &str,
input: Vec<u8>,
) -> orcher_sdk_core::Result<Vec<u8>> {
self.tasks_executed.fetch_add(1, Ordering::Relaxed);
tracing::info!(
task_id = %ctx.task_id(),
task_type = %task_type,
"Executing task in sdk-rust runtime"
);
let _permit = self.task_semaphore.acquire().await.map_err(|e| {
self.tasks_failed.fetch_add(1, Ordering::Relaxed);
orcher_sdk_core::Error::internal(format!("Failed to acquire task permit: {}", e))
})?;
let handler = {
let handlers = self.task_handlers.read().await;
handlers.get(task_type).cloned()
};
let Some(handler) = handler else {
self.tasks_failed.fetch_add(1, Ordering::Relaxed);
return Err(orcher_sdk_core::Error::internal(format!(
"No handler registered for task type: {}",
task_type
)));
};
let mut input = Payload::new_data(input);
if let Some(ref codec) = self.config.codec_chain {
use orcher_sdk_core::payload::codec::PayloadCodec;
match codec.decode(&input) {
Ok(decoded) => {
input = decoded;
}
Err(e) => {
tracing::warn!(error = %e, "Failed to decode task input with codec chain");
}
}
}
match handler(ctx, input).await {
Ok(output) => {
self.tasks_succeeded.fetch_add(1, Ordering::Relaxed);
*self.last_success_time.lock().unwrap() = Some(Instant::now());
Ok(output.data)
}
Err(e) => {
self.tasks_failed.fetch_add(1, Ordering::Relaxed);
Err(task_failure(e))
}
}
}
async fn answer_asked(
&self,
ctx: &WorkflowContext,
asked: &[Asked],
) -> (Vec<QueryAnswer>, Vec<UpdateAnswer>) {
let mut queries = Vec::new();
let mut updates = Vec::new();
for asked in asked {
match asked {
Asked::Query { id, name, args } => queries.push(QueryAnswer {
query_id: id.clone(),
answer: Some(match ctx.handle_query_request(name, args) {
Ok(output) => query_answer::Answer::Value(data(output)),
Err(e) => query_answer::Answer::Failure(Failure {
message: e.to_string(),
..Default::default()
}),
}),
..Default::default()
}),
Asked::Update { id, name, args } => updates.push(UpdateAnswer {
update_id: id.clone(),
answer: Some(match ctx.handle_update_request(name, args.clone()).await {
Ok(output) => update_answer::Answer::Value(data(output)),
Err(e) => update_answer::Answer::Rejected(Failure {
message: e.to_string(),
..Default::default()
}),
}),
..Default::default()
}),
}
}
(queries, updates)
}
}
pub(crate) fn inject_child_failure(ctx: &WorkflowContext, workflow_id: &str, message: &str) {
ctx.inject_step_result(
format!("child:{workflow_id}"),
crate::workflow::handle::encode_child_failure(message),
);
}
pub(crate) enum Asked {
Query {
id: String,
name: String,
args: Vec<u8>,
},
Update {
id: String,
name: String,
args: Vec<u8>,
},
}
enum Ending {
Completed(Payload),
Cancelled,
}
fn task_failure(error: crate::Error) -> orcher_sdk_core::Error {
match error {
crate::Error::Task(crate::error::TaskError::Application {
error_type,
message,
non_retryable,
}) => orcher_sdk_core::TaskFailure::new(message)
.with_type(error_type)
.with_non_retryable(non_retryable)
.into(),
other => orcher_sdk_core::Error::internal(other.to_string()),
}
}
#[cfg(feature = "auto-register")]
pub(crate) fn failure_of(error: crate::Error) -> Failure {
reported_failure(&task_failure(error))
}
fn reported_failure(error: &orcher_sdk_core::Error) -> Failure {
match error.task_failure() {
Some(failure) => Failure {
message: failure.message.clone(),
error_type: failure.error_type.clone().unwrap_or_default(),
non_retryable: failure.non_retryable,
..Default::default()
},
None => Failure {
message: error.to_string(),
..Default::default()
},
}
}
fn workflow_failure(error: &Error) -> Failure {
match error {
Error::Workflow(crate::error::WorkflowError::NonDeterministic(_)) => Failure {
message: error.to_string(),
non_retryable: true,
code: FailureCode::NonDeterminism as i32,
..Default::default()
},
_ => Failure {
message: error.to_string(),
non_retryable: matches!(
error,
Error::Task(crate::error::TaskError::Application {
non_retryable: true,
..
})
),
code: FailureCode::Application as i32,
..Default::default()
},
}
}
fn code_failure(message: String) -> Failure {
Failure {
message,
non_retryable: false,
code: FailureCode::Application as i32,
..Default::default()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error::TaskError;
#[test]
fn a_named_task_failure_keeps_its_type_and_mark() {
let reported = task_failure(TaskError::non_retryable("CardDeclined", "declined").into());
let failure = reported.task_failure().expect("a typed failure");
assert_eq!(failure.error_type.as_deref(), Some("CardDeclined"));
assert!(failure.non_retryable);
assert_eq!(reported.to_string(), "declined");
let reported = task_failure(TaskError::application("Throttled", "slow down").into());
let failure = reported.task_failure().expect("a typed failure");
assert_eq!(failure.error_type.as_deref(), Some("Throttled"));
assert!(!failure.non_retryable);
let reported = task_failure(TaskError::execution_failed("boom").into());
assert!(
reported.task_failure().is_none(),
"an unnamed error has no type"
);
}
#[test]
fn a_non_deterministic_workflow_is_reported_as_non_determinism() {
let reported = workflow_failure(&Error::Workflow(
crate::error::WorkflowError::non_deterministic("step 3 changed"),
));
assert_eq!(reported.code, FailureCode::NonDeterminism as i32);
assert!(reported.non_retryable);
assert!(reported.message.contains("step 3 changed"));
let reported = workflow_failure(&Error::Workflow(crate::error::WorkflowError::Panic(
"boom".into(),
)));
assert_eq!(reported.code, FailureCode::Application as i32);
}
#[test]
fn a_workflow_failure_is_non_retryable_only_when_raised_so() {
let raised = workflow_failure(&TaskError::non_retryable("OutOfStock", "gone").into());
assert!(raised.non_retryable);
assert!(raised.message.contains("gone"));
for error in [
TaskError::application("Throttled", "slow down").into(),
TaskError::execution_failed("boom").into(),
Error::Workflow(crate::error::WorkflowError::Panic("boom".into())),
Error::Workflow(crate::error::WorkflowError::task_failed(
"charge", 3, "declined",
)),
] {
let reported = workflow_failure(&error);
assert!(
!reported.non_retryable,
"{error:?} was reported non-retryable"
);
}
}
#[test]
fn test_runtime_config_default() {
let config = RuntimeConfig::default();
assert_eq!(config.max_concurrent_workflows, 100);
assert_eq!(config.max_concurrent_tasks, 200);
assert_eq!(config.default_task_queue, "default");
}
#[tokio::test]
async fn test_runtime_handler_registration() {
let runtime = ExecutionRuntime::new(RuntimeConfig::default());
let handler: WorkflowHandlerFn =
Arc::new(|_ctx, _input| Box::pin(async move { Ok(Payload::new_data(Vec::new())) }));
runtime
.register_workflow_handler("TestWorkflow", handler)
.await;
let handlers = runtime.workflow_handlers.read().await;
assert!(handlers.contains_key("TestWorkflow"));
}
#[tokio::test]
async fn test_runtime_task_handler_registration() {
let runtime = ExecutionRuntime::new(RuntimeConfig::default());
let handler: TaskHandlerFn =
Arc::new(|_ctx, _input| Box::pin(async move { Ok(Payload::new_data(Vec::new())) }));
runtime.register_task_handler("TestTask", handler).await;
let handlers = runtime.task_handlers.read().await;
assert!(handlers.contains_key("TestTask"));
}
}