mod activation;
mod builder;
pub mod runtime;
#[cfg(test)]
mod replay_tests;
#[cfg(feature = "auto-register")]
pub mod registry;
#[cfg(not(feature = "auto-register"))]
mod registry;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::RwLock;
pub mod registration;
pub use builder::WorkerBuilder;
pub use registry::Registry;
pub use runtime::{ExecutionRuntime, RuntimeConfig, RuntimeStats};
use crate::interceptor::InterceptorChain;
use crate::error::{Error, Result};
#[cfg(feature = "auto-register")]
use orcher_proto::host::v1::{actor_result, ActorMode, ActorOperation, ActorResult};
use orcher_proto::host::v1::{task_outcome, TaskAssignment};
use orcher_proto::kernel::v1::run_activation_result;
use orcher_proto::prost::Message;
use orcher_sdk_core::observability::WireMetricsSnapshot;
use orcher_sdk_core::{EncodedHost, Host, HostConfig, Registrations};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum HealthStatus {
Healthy,
Degraded,
Unhealthy,
}
impl std::fmt::Display for HealthStatus {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Healthy => write!(f, "HEALTHY"),
Self::Degraded => write!(f, "DEGRADED"),
Self::Unhealthy => write!(f, "UNHEALTHY"),
}
}
}
#[derive(Debug, Clone)]
pub struct HealthReport {
pub status: HealthStatus,
pub message: String,
pub stats: RuntimeStats,
}
pub struct Worker {
config: WorkerConfig,
runtime: Arc<ExecutionRuntime>,
registry: Arc<RwLock<Registry>>,
#[cfg(feature = "auto-register")]
actor_executor: Option<Arc<crate::actor::ActorExecutor>>,
#[allow(dead_code)]
service_id: String,
shutdown_tx: tokio::sync::broadcast::Sender<()>,
started_at: std::time::Instant,
interceptor_chain: Arc<InterceptorChain>,
host: std::sync::Mutex<Option<Host>>,
}
#[derive(Clone)]
#[non_exhaustive]
pub struct WorkerConfig {
pub server_url: String,
pub namespace: String,
pub task_queue: String,
pub identity: String,
pub max_concurrent_workflows: usize,
pub max_concurrent_tasks: usize,
pub workflow_poller_count: usize,
pub task_poller_count: usize,
pub actor_poller_count: usize,
pub max_concurrent_actor_operations: usize,
pub poll_interval: Duration,
pub heartbeat_interval: Duration,
pub organization_id: Option<String>,
pub api_key: Option<String>,
pub codec_chain: Option<Arc<orcher_sdk_core::payload::codec::CodecChain>>,
pub version_id: Option<String>,
pub tls: Option<crate::client::ClientTlsConfig>,
pub max_sessions: usize,
}
impl Default for WorkerConfig {
fn default() -> Self {
let identity = format!("orcher-rust-{}", uuid::Uuid::new_v4());
Self {
server_url: "http://localhost:50051".to_string(),
namespace: "default".to_string(),
task_queue: "default".to_string(),
identity,
max_concurrent_workflows: 100,
max_concurrent_tasks: 200,
workflow_poller_count: 4,
task_poller_count: 4,
actor_poller_count: 4,
max_concurrent_actor_operations: 100,
poll_interval: Duration::from_millis(100),
heartbeat_interval: Duration::from_secs(10),
organization_id: None,
api_key: None,
codec_chain: None,
version_id: std::env::var("ORCHER_VERSION_ID")
.ok()
.filter(|v| !v.trim().is_empty()),
max_sessions: 10,
tls: None,
}
}
}
impl WorkerConfig {
pub fn from_env() -> Self {
Self::default()
}
}
impl std::fmt::Debug for WorkerConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("WorkerConfig")
.field("server_url", &self.server_url)
.field("namespace", &self.namespace)
.field("task_queue", &self.task_queue)
.field("identity", &self.identity)
.field("max_concurrent_workflows", &self.max_concurrent_workflows)
.field("max_concurrent_tasks", &self.max_concurrent_tasks)
.field("workflow_poller_count", &self.workflow_poller_count)
.field("task_poller_count", &self.task_poller_count)
.field("actor_poller_count", &self.actor_poller_count)
.field(
"max_concurrent_actor_operations",
&self.max_concurrent_actor_operations,
)
.field("poll_interval", &self.poll_interval)
.field("heartbeat_interval", &self.heartbeat_interval)
.field("organization_id", &self.organization_id)
.field("tls", &self.tls)
.field("api_key", &self.api_key.as_ref().map(|_| "***"))
.field(
"codec_chain",
&self
.codec_chain
.as_ref()
.map(|c| format!("<CodecChain len={}>", c.len())),
)
.field("version_id", &self.version_id)
.finish()
}
}
impl Worker {
pub fn builder() -> WorkerBuilder {
WorkerBuilder::new()
}
pub fn runtime(&self) -> &Arc<ExecutionRuntime> {
&self.runtime
}
pub fn registry(&self) -> &Arc<RwLock<Registry>> {
&self.registry
}
#[cfg(feature = "auto-register")]
pub fn actor_executor(&self) -> Option<&Arc<crate::actor::ActorExecutor>> {
self.actor_executor.as_ref()
}
pub fn config(&self) -> &WorkerConfig {
&self.config
}
pub fn get_stats(&self) -> RuntimeStats {
self.runtime.stats()
}
pub fn get_health(&self) -> HealthReport {
let stats = self.runtime.stats();
let total = stats.workflows_executed + stats.tasks_executed;
let failures = stats.workflows_failed + stats.tasks_failed;
let last_success = self.runtime.last_success_time();
let secs_since_success = last_success
.map(|t| t.elapsed().as_secs())
.unwrap_or(self.started_at.elapsed().as_secs());
let (status, message) = if total == 0 {
(HealthStatus::Healthy, "No executions yet".to_string())
} else {
let error_rate = failures as f64 / total as f64;
if error_rate > 0.5 || secs_since_success > 60 {
(
HealthStatus::Unhealthy,
format!(
"Error rate: {:.1}%, last success: {}s ago",
error_rate * 100.0,
secs_since_success
),
)
} else if error_rate > 0.1 || secs_since_success > 30 {
(
HealthStatus::Degraded,
format!(
"Error rate: {:.1}%, last success: {}s ago",
error_rate * 100.0,
secs_since_success
),
)
} else {
(HealthStatus::Healthy, "Operating normally".to_string())
}
};
HealthReport {
status,
message,
stats,
}
}
pub fn shutdown(&self) {
tracing::info!("Shutdown requested");
let _ = self.shutdown_tx.send(());
}
pub fn wire_metrics(&self) -> Option<WireMetricsSnapshot> {
self.host
.lock()
.ok()
.and_then(|host| host.as_ref().map(Host::wire_metrics))
}
fn core_tls(&self) -> Option<orcher_sdk_core::TlsConfig> {
self.config.tls.as_ref().map(crate::tls::to_core)
}
fn host_config(&self) -> HostConfig {
let c = &self.config;
let mut config = HostConfig::new(&c.server_url, &c.task_queue)
.with_namespace(&c.namespace)
.with_identity(&c.identity)
.with_concurrency(
c.max_concurrent_workflows as u32,
c.max_concurrent_tasks as u32,
c.max_concurrent_actor_operations as u32,
)
.with_pollers(
c.workflow_poller_count,
c.task_poller_count,
c.actor_poller_count,
)
.with_heartbeat_interval(c.heartbeat_interval)
.with_max_sessions(c.max_sessions)
.with_sdk("rust", env!("CARGO_PKG_VERSION"))
.with_protocol_version(orcher_sdk_core::worker_protocol::WORKFLOW_FAILURE_RETRYABILITY)
.with_step_ids([
orcher_sdk_core::StepIdScheme::Kernel,
orcher_sdk_core::StepIdScheme::RustV0,
]);
if let Some(api_key) = &c.api_key {
config = config.with_api_key(api_key);
}
if let Some(organization_id) = &c.organization_id {
config = config.with_organization_id(organization_id);
}
if let Some(version_id) = &c.version_id {
config = config.with_version_id(version_id);
}
if let Some(tls) = self.core_tls() {
config = config.with_tls(tls);
}
config
}
async fn registrations(&self) -> Registrations {
#[allow(unused_mut)]
let mut registrations = Registrations::new(
self.runtime.workflow_types().await,
self.runtime.task_types().await,
[] as [&str; 0],
);
#[cfg(feature = "auto-register")]
{
let registry = self.registry.read().await;
let mut handlers = Vec::new();
for (actor, operations) in registry.actor_handlers() {
for (operation, registration) in operations {
handlers.push(match registration.mode {
crate::actor::types::OperationMode::Shared => {
orcher_sdk_core::ActorHandler::shared(actor, operation)
}
crate::actor::types::OperationMode::Exclusive => {
orcher_sdk_core::ActorHandler::exclusive(actor, operation)
}
});
}
}
registrations = registrations.with_actor_handlers(handlers);
}
registrations
}
pub async fn run(&self) -> Result<()> {
let mut shutdown_rx = self.shutdown_tx.subscribe();
tracing::info!(
identity = %self.config.identity,
namespace = %self.config.namespace,
task_queue = %self.config.task_queue,
"Starting ORCHER Worker"
);
self.register_handlers_from_registry().await?;
let host = tokio::select! {
started = Host::start(self.host_config(), self.registrations().await) => {
started.map_err(|e| {
Error::Worker(crate::error::WorkerError::StartupFailed(e.to_string()))
})?
}
_ = shutdown_rx.recv() => {
tracing::info!("Shutdown requested while the worker was starting");
return Ok(());
}
};
if let Ok(mut held) = self.host.lock() {
*held = Some(host.clone());
}
let workflows = tokio::spawn(run_workflows(
host.encoded(),
Arc::clone(&self.runtime),
Arc::clone(&self.interceptor_chain),
self.config.max_concurrent_workflows,
self.config.namespace.clone(),
));
let tasks = tokio::spawn(run_tasks(
host.encoded(),
Arc::clone(&self.runtime),
Arc::clone(&self.interceptor_chain),
self.config.max_concurrent_tasks,
self.config.namespace.clone(),
));
#[cfg(feature = "auto-register")]
let actors = self.actor_executor.clone().map(|executor| {
tokio::spawn(run_actors(
host.encoded(),
executor,
self.config.max_concurrent_actor_operations,
))
});
tracing::info!(identity = %self.config.identity, "Worker started");
let _ = shutdown_rx.recv().await;
tracing::info!("Shutdown signal received");
host.shutdown().await;
let _ = workflows.await;
let _ = tasks.await;
#[cfg(feature = "auto-register")]
if let Some(actors) = actors {
let _ = actors.await;
}
tracing::info!("Worker stopped");
Ok(())
}
async fn register_handlers_from_registry(&self) -> Result<()> {
#[cfg(feature = "auto-register")]
let registry = self.registry.read().await;
#[cfg(feature = "auto-register")]
{
for name in registry.workflow_names() {
if let Some(registration) = registry.get_workflow(name) {
if let Some(handler) = ®istration.handler {
tracing::debug!(workflow_type = %name, "Registering workflow handler");
self.runtime
.register_workflow_handler(name.to_string(), handler.clone())
.await;
}
}
}
for name in registry.task_names() {
if let Some(registration) = registry.get_task(name) {
if let Some(handler) = ®istration.handler {
tracing::debug!(task_type = %name, "Registering task handler");
self.runtime
.register_task_handler(name.to_string(), handler.clone())
.await;
}
}
}
tracing::info!(
workflows = registry.workflow_count(),
tasks = registry.task_count(),
"Handlers registered in execution runtime"
);
}
#[cfg(not(feature = "auto-register"))]
{
tracing::debug!("auto-register feature not enabled, no handlers to register");
}
Ok(())
}
}
async fn run_workflows(
host: EncodedHost,
runtime: Arc<ExecutionRuntime>,
interceptors: Arc<InterceptorChain>,
limit: usize,
namespace: String,
) {
let room = Arc::new(tokio::sync::Semaphore::new(limit.max(1)));
loop {
let Ok(permit) = Arc::clone(&room).acquire_owned().await else {
return;
};
let Some(encoded) = host.next_activation().await else {
return;
};
let (host, runtime, interceptors, namespace) = (
host.clone(),
Arc::clone(&runtime),
Arc::clone(&interceptors),
namespace.clone(),
);
tokio::spawn(async move {
let activation = match activation::decode_activation(&encoded) {
Ok(activation) => activation,
Err(message) => {
let refused = activation::unreadable(message).encode_to_vec();
if let Err(e) = host.complete_activation(&refused).await {
tracing::error!(error = %e, "Failed to refuse the unreadable activation");
}
drop(permit);
return;
}
};
let run = activation.run.clone().unwrap_or_default();
let ictx = orcher_sdk_core::interceptor::InterceptorContext::workflow(
&run.workflow_id,
&run.workflow_type,
1,
&namespace,
&run.task_queue,
);
interceptors.before_execution(&ictx);
let start = std::time::Instant::now();
let answer = runtime.run_activation(activation).await;
let duration_ms = start.elapsed().as_millis() as u64;
match &answer.outcome {
Some(run_activation_result::Outcome::Commands(_)) => {
interceptors.after_execution(&ictx, duration_ms)
}
Some(run_activation_result::Outcome::WorkflowFailed(failure)) => {
interceptors.on_error(&ictx, &failure.message, duration_ms)
}
Some(run_activation_result::Outcome::ProtocolError(error)) => {
interceptors.on_error(&ictx, &error.message, duration_ms)
}
None => interceptors.on_error(&ictx, "no outcome", duration_ms),
}
if let Err(e) = host.complete_activation(&answer.encode_to_vec()).await {
tracing::error!(error = %e, "Failed to hand the workflow activation's answer back");
}
drop(permit);
});
}
}
async fn run_tasks(
host: EncodedHost,
runtime: Arc<ExecutionRuntime>,
interceptors: Arc<InterceptorChain>,
limit: usize,
namespace: String,
) {
let room = Arc::new(tokio::sync::Semaphore::new(limit.max(1)));
loop {
let Ok(permit) = Arc::clone(&room).acquire_owned().await else {
return;
};
let Some(assignment) = host.next_task().await else {
return;
};
let assignment = match TaskAssignment::decode(assignment.as_slice()) {
Ok(assignment) => assignment,
Err(e) => {
tracing::error!(
error = %e,
"The worker host handed over bytes that are not a TaskAssignment"
);
continue;
}
};
let (host, runtime, interceptors, namespace) = (
host.clone(),
Arc::clone(&runtime),
Arc::clone(&interceptors),
namespace.clone(),
);
tokio::spawn(async move {
let task_queue = assignment
.run
.as_ref()
.map(|run| run.task_queue.clone())
.unwrap_or_default();
let ictx = orcher_sdk_core::interceptor::InterceptorContext::task(
&assignment.task_id,
&assignment.task_type,
assignment.attempt.max(1),
&namespace,
&task_queue,
);
interceptors.before_execution(&ictx);
let start = std::time::Instant::now();
let outcome = runtime.execute_assignment(assignment, &host).await;
let duration_ms = start.elapsed().as_millis() as u64;
match &outcome.result {
Some(task_outcome::Result::Failure(failure)) => {
interceptors.on_error(&ictx, &failure.message, duration_ms)
}
_ => interceptors.after_execution(&ictx, duration_ms),
}
if let Err(e) = host.complete_task(&outcome.encode_to_vec()).await {
tracing::error!(error = %e, "Failed to report the task's outcome");
}
drop(permit);
});
}
}
#[cfg(feature = "auto-register")]
async fn run_actors(host: EncodedHost, executor: Arc<crate::actor::ActorExecutor>, limit: usize) {
let room = Arc::new(tokio::sync::Semaphore::new(limit.max(1)));
loop {
let Ok(permit) = Arc::clone(&room).acquire_owned().await else {
return;
};
let Some(operation) = host.next_actor_operation().await else {
return;
};
let operation = match ActorOperation::decode(operation.as_slice()) {
Ok(operation) => operation,
Err(e) => {
tracing::error!(
error = %e,
"The worker host handed over bytes that are not an ActorOperation"
);
continue;
}
};
let (host, executor) = (host.clone(), Arc::clone(&executor));
tokio::spawn(async move {
let result = run_actor_operation(host.host(), &executor, &operation).await;
let result = ActorResult {
operation_id: operation.operation_id,
execution_id: operation.execution_id,
result: Some(result),
..Default::default()
};
if let Err(e) = host.complete_actor_operation(&result.encode_to_vec()).await {
tracing::error!(error = %e, "Failed to report the actor operation's result");
}
drop(permit);
});
}
}
#[cfg(feature = "auto-register")]
async fn run_actor_operation(
host: &Host,
executor: &crate::actor::ActorExecutor,
operation: &ActorOperation,
) -> actor_result::Result {
tracing::info!(
actor = %operation.actor_name,
key = %operation.key,
operation = %operation.operation,
operation_id = %operation.operation_id,
shared = operation.mode == ActorMode::Shared as i32,
"Executing actor operation"
);
let state = Arc::new(crate::actor::ActorStateClient::from_host(host.clone()));
let context = crate::actor::ActorContextBuilder::new(
crate::actor::ActorKey::new(operation.actor_name.clone(), operation.key.clone()),
state,
)
.execution_id(&operation.execution_id)
.build();
let payload = operation
.payload
.as_ref()
.map(|payload| payload.data.clone())
.unwrap_or_default();
match executor
.execute(
&operation.actor_name,
&operation.key,
&operation.operation,
payload,
context,
)
.await
{
Ok(output) => actor_result::Result::Value(activation::data(output)),
Err(error) => actor_result::Result::Failure(runtime::failure_of(error)),
}
}
impl std::fmt::Debug for Worker {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Worker")
.field("server_url", &self.config.server_url)
.field("namespace", &self.config.namespace)
.field("task_queue", &self.config.task_queue)
.field("identity", &self.config.identity)
.field(
"max_concurrent_workflows",
&self.config.max_concurrent_workflows,
)
.field("max_concurrent_tasks", &self.config.max_concurrent_tasks)
.finish()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_service_config_defaults() {
let config = WorkerConfig::default();
assert_eq!(config.server_url, "http://localhost:50051");
assert_eq!(config.namespace, "default");
assert_eq!(config.task_queue, "default");
assert!(config.identity.starts_with("orcher-rust-"));
assert_eq!(config.max_concurrent_workflows, 100);
assert_eq!(config.max_concurrent_tasks, 200);
assert_eq!(config.workflow_poller_count, 4);
assert_eq!(config.task_poller_count, 4);
assert_eq!(config.actor_poller_count, 4);
assert_eq!(config.max_concurrent_actor_operations, 100);
}
}