pub(crate) mod events;
pub(crate) mod heartbeat;
pub(crate) mod listener;
pub(crate) mod runner;
use std::collections::{BTreeMap, HashMap};
use std::fmt;
use std::pin::Pin;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use hatchet_proto::prost_types::Timestamp;
use hatchet_proto::root::{
AssignedAction, BatchActionEvent, HeartbeatRequest, RuntimeInfo, StepActionEvent,
WorkerListenRequest, WorkerRegisterRequest, WorkerUnsubscribeRequest,
};
use hatchet_proto::v1;
use tokio::sync::mpsc;
use tokio_stream::Stream;
use tokio_util::sync::CancellationToken;
use tonic::{Code, Request, Status};
use crate::declare::{Register, RegisteredAction, WorkflowDef};
use crate::durable::eviction::{self, EvictionBook};
use crate::durable::stream::DurableStream;
use crate::worker::runner::{BatchReporter, DurableSetup, Reporter, Runner};
use crate::{Client, Error, LabelValue, Result, retry, version};
const DEFAULT_SLOTS: u32 = 100;
const DEFAULT_DURABLE_SLOTS: u32 = 1000;
const DEFAULT_SHUTDOWN_GRACE: Duration = Duration::from_secs(30);
const DEFAULT_STREAM_ERROR_DRAIN: Duration = Duration::from_secs(30);
const UNSUBSCRIBE_TIMEOUT: Duration = Duration::from_secs(5);
type ActionStream = Pin<Box<dyn Stream<Item = std::result::Result<AssignedAction, Status>> + Send>>;
#[must_use = "a worker builder configures nothing until `build` is awaited"]
pub struct WorkerBuilder {
client: Client,
name: String,
slots: u32,
durable_slots: u32,
labels: BTreeMap<String, LabelValue>,
declarations: Vec<WorkflowDef>,
grace: Duration,
stream_error_drain: Duration,
}
impl fmt::Debug for WorkerBuilder {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("WorkerBuilder")
.field("name", &self.name)
.field("slots", &self.slots)
.field("durable_slots", &self.durable_slots)
.field("labels", &self.labels)
.field("declarations", &self.declarations.len())
.field("shutdown_grace", &self.grace)
.field("stream_error_drain", &self.stream_error_drain)
.finish_non_exhaustive()
}
}
impl WorkerBuilder {
pub(crate) fn new(client: Client, name: impl Into<String>) -> Self {
Self {
client,
name: name.into(),
slots: DEFAULT_SLOTS,
durable_slots: DEFAULT_DURABLE_SLOTS,
labels: BTreeMap::new(),
declarations: Vec::new(),
grace: DEFAULT_SHUTDOWN_GRACE,
stream_error_drain: DEFAULT_STREAM_ERROR_DRAIN,
}
}
#[must_use = "this returns the builder to carry on from"]
pub fn slots(mut self, slots: u32) -> Self {
self.slots = slots;
self
}
#[must_use = "this returns the builder to carry on from"]
pub fn durable_slots(mut self, slots: u32) -> Self {
self.durable_slots = slots;
self
}
#[must_use = "this returns the builder to carry on from"]
pub fn label(mut self, key: impl Into<String>, value: impl Into<LabelValue>) -> Self {
self.labels.insert(key.into(), value.into());
self
}
#[must_use = "this returns the builder to carry on from"]
pub fn register(mut self, declaration: &impl Register) -> Self {
self.declarations.push(declaration.definition());
self
}
#[must_use = "this returns the builder to carry on from"]
pub fn shutdown_grace(mut self, grace: Duration) -> Self {
self.grace = grace;
self
}
#[must_use = "this returns the builder to carry on from"]
pub fn stream_error_drain(mut self, timeout: Duration) -> Self {
self.stream_error_drain = timeout;
self
}
pub async fn build(self) -> Result<Worker> {
if self.slots == 0 {
return Err(Error::Config(format!(
"worker `{}` was given no slots; it could never run anything",
self.name
)));
}
if self.declarations.is_empty() {
return Err(Error::InvalidDeclaration(format!(
"worker `{}` has nothing to run; register a task or a workflow",
self.name
)));
}
let serves_durable = self.declarations.iter().any(WorkflowDef::has_durable);
if serves_durable && self.durable_slots == 0 {
return Err(Error::Config(format!(
"worker `{}` registers a durable task but was given no durable slots",
self.name
)));
}
let namespace = self.client.config().namespace().to_string();
let mut requests = Vec::with_capacity(self.declarations.len());
let mut handlers: HashMap<String, RegisteredAction> = HashMap::new();
for declaration in &self.declarations {
requests.push(declaration.to_request(&namespace)?);
for (action, handler) in declaration.actions(&namespace)? {
if handlers.insert(action.clone(), handler).is_some() {
return Err(Error::InvalidDeclaration(format!(
"more than one registered declaration serves the action `{action}`"
)));
}
}
}
let registration = self.registration(handlers.keys().cloned().collect());
let engine_version = self.client.engine_version().await?;
version::ensure_supported(&engine_version)?;
for declaration in &self.declarations {
declaration.check_engine_features(&engine_version)?;
}
if serves_durable
&& version::ensure_feature(
crate::durable::SPAWN_FEATURE,
version::DURABLE_SPAWN_MIN_ENGINE_VERSION,
&engine_version,
)
.is_err()
{
tracing::warn!(
engine_version,
required = version::DURABLE_SPAWN_MIN_ENGINE_VERSION,
"this engine is too old for durable child spawns; \
`DurableContext::spawn` will be refused on this worker",
);
}
let engine_version: Arc<str> = Arc::from(engine_version.as_str());
for request in requests {
put_workflow(&self.client, &request).await?;
}
let worker_id = register(&self.client, ®istration).await?;
tracing::info!(
worker_id,
worker_name = %self.name,
slots = self.slots,
actions = handlers.len(),
batch_actions = handlers
.values()
.filter(|action| action.batch.is_some())
.count(),
"registered a worker with the engine",
);
let report = reporter(self.client.clone());
let report_batch = batch_reporter(self.client.clone());
let durable_stop = CancellationToken::new();
let durable = serves_durable.then(|| {
DurableStream::new(
worker_id.clone(),
crate::durable::stream::connect(
self.client.grpc_service(),
self.client.config().gzip(),
),
durable_stop.clone(),
)
});
let evictable: Vec<&str> = handlers
.iter()
.filter(|(_, action)| action.durable && action.eviction_policy.is_some())
.map(|(action_id, _)| action_id.as_str())
.collect();
let eviction = if evictable.is_empty() {
None
} else if eviction::supported(&engine_version) {
Some(Arc::new(EvictionBook::default()))
} else {
tracing::warn!(
%engine_version,
required = version::EVICTION_MIN_ENGINE_VERSION,
tasks = ?evictable,
"this engine does not support durable eviction; the eviction policies on these \
tasks are ignored",
);
None
};
let durable_setup = durable.clone().map(|stream| DurableSetup {
slots: self.durable_slots as usize,
stream,
engine_version,
eviction: eviction.clone(),
});
let runner = Arc::new(Runner::new(
worker_id.clone(),
Some(self.client.clone()),
handlers,
self.slots as usize,
durable_setup,
report,
report_batch,
));
if let Some(stream) = &durable {
let runner = Arc::downgrade(&runner);
stream.on_server_evict(move |task_id, invocation_count| {
tracing::info!(
task_run_external_id = %task_id,
invocation_count,
"the engine evicted a durable invocation",
);
if let Some(runner) = runner.upgrade() {
runner.cancel(task_id);
}
});
}
Ok(Worker {
client: self.client,
id: worker_id,
slots: self.slots,
durable_slots: self.durable_slots,
grace: self.grace,
stream_error_drain: self.stream_error_drain,
runner,
durable,
durable_stop,
eviction,
shutdown: CancellationToken::new(),
})
}
fn registration(&self, mut actions: Vec<String>) -> WorkerRegisterRequest {
actions.sort();
WorkerRegisterRequest {
worker_name: self.name.clone(),
actions,
labels: self
.labels
.iter()
.map(|(key, value)| (key.clone(), value.to_proto()))
.collect(),
runtime_info: Some(RuntimeInfo {
sdk_version: Some(env!("CARGO_PKG_VERSION").to_string()),
language: None,
language_version: None,
os: Some(std::env::consts::OS.to_string()),
extra: Some("rust".to_string()),
}),
slot_config: self.slot_config(),
slots: None,
..WorkerRegisterRequest::default()
}
}
fn slot_config(&self) -> HashMap<String, i32> {
let mut slot_config = HashMap::new();
if self.declarations.iter().any(WorkflowDef::has_default) {
slot_config.insert("default".to_string(), saturating(self.slots));
}
if self.declarations.iter().any(WorkflowDef::has_durable) {
slot_config.insert("durable".to_string(), saturating(self.durable_slots));
}
if slot_config.is_empty() {
slot_config.insert("default".to_string(), saturating(self.slots));
}
slot_config
}
}
fn saturating(slots: u32) -> i32 {
i32::try_from(slots).unwrap_or(i32::MAX)
}
#[must_use = "a worker does nothing until `run` is awaited"]
pub struct Worker {
client: Client,
id: String,
slots: u32,
durable_slots: u32,
grace: Duration,
stream_error_drain: Duration,
runner: Arc<Runner>,
durable: Option<DurableStream>,
durable_stop: CancellationToken,
eviction: Option<Arc<EvictionBook>>,
shutdown: CancellationToken,
}
impl fmt::Debug for Worker {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Worker")
.field("id", &self.id)
.field("slots", &self.slots)
.field("in_flight", &self.runner.in_flight())
.field("shutdown_grace", &self.grace)
.field("stream_error_drain", &self.stream_error_drain)
.field("shutting_down", &self.shutdown.is_cancelled())
.finish_non_exhaustive()
}
}
impl Worker {
pub fn id(&self) -> &str {
&self.id
}
pub fn shutdown_handle(&self) -> ShutdownHandle {
ShutdownHandle {
token: self.shutdown.clone(),
}
}
pub async fn run(self) -> Result<()> {
let heartbeat_stop = CancellationToken::new();
let _stop_the_heartbeat = heartbeat_stop.clone().drop_guard();
let _stop_the_durable_stream = self.durable_stop.clone().drop_guard();
let heartbeat = tokio::spawn(heartbeat::run(
beating(self.client.clone(), self.id.clone()),
heartbeat_stop.clone(),
));
if let Some(durable) = &self.durable {
durable.start();
}
if let (Some(stream), Some(book)) = (&self.durable, &self.eviction) {
let runner = Arc::downgrade(&self.runner);
tokio::spawn(eviction::run(
Arc::clone(book),
stream.clone(),
move |task_id| {
if let Some(runner) = runner.upgrade() {
runner.cancel(task_id);
}
},
self.durable_slots as usize,
eviction::CONFIG,
self.durable_stop.clone(),
));
}
let runner = Arc::clone(&self.runner);
let listening = listener::run(
connecting(self.client.clone(), self.id.clone()),
move |action| runner.dispatch(action),
self.shutdown.clone(),
);
let outcome = match &self.durable {
Some(durable) => {
let given_up = durable.given_up();
tokio::pin!(listening);
tokio::select! {
outcome = &mut listening => outcome,
error = given_up => {
tracing::error!(
worker_id = %self.id,
%error,
"the worker's durable task stream ended for good",
);
Err(error)
}
}
}
None => listening.await,
};
let result = match outcome {
Ok(()) => {
tracing::info!(worker_id = %self.id, "the worker is shutting down");
self.unsubscribe().await;
self.runner.drain(self.grace).await;
Ok(())
}
Err(error) => {
if refuses_this_worker(&error) {
tracing::error!(
worker_id = %self.id,
%error,
runs = self.runner.in_flight(),
"the engine refused this worker's credentials; dropping the runs in flight",
);
self.runner.abort_all().await;
} else {
tracing::error!(
worker_id = %self.id,
%error,
runs = self.runner.in_flight(),
drain = ?self.stream_error_drain,
"the worker's action stream ended for good; draining the runs in flight",
);
self.unsubscribe().await;
self.runner.drain(self.stream_error_drain).await;
}
Err(error)
}
};
self.durable_stop.cancel();
heartbeat_stop.cancel();
if let Err(error) = heartbeat.await {
tracing::warn!(%error, "the heartbeat task did not end cleanly");
}
result
}
pub async fn run_until_shutdown(self) -> Result<()> {
let (sender, receiver) = mpsc::channel(1);
let forward = shutdown_signals(sender)?;
let watcher = tokio::spawn(forward);
let result = self.run_until(receiver).await;
watcher.abort();
result
}
async fn run_until(self, mut signals: mpsc::Receiver<()>) -> Result<()> {
let handle = self.shutdown_handle();
let runner = Arc::clone(&self.runner);
let worker_id = self.id.clone();
let running = self.run();
tokio::pin!(running);
let mut asked_to_stop = false;
loop {
let signal = tokio::select! {
result = &mut running => return result,
signal = signals.recv() => signal,
};
if signal.is_none() {
return running.await;
}
if asked_to_stop {
tracing::warn!(
worker_id = %worker_id,
runs = runner.in_flight(),
"a second shutdown signal arrived; dropping the runs still in flight",
);
runner.abort_all().await;
return Ok(());
}
asked_to_stop = true;
tracing::info!(worker_id = %worker_id, "received a shutdown signal");
handle.shutdown();
}
}
async fn unsubscribe(&self) {
let mut request = Request::new(WorkerUnsubscribeRequest {
worker_id: self.id.clone(),
});
request.set_timeout(UNSUBSCRIBE_TIMEOUT);
let mut dispatcher = self.client.dispatcher_grpc();
let sent = tokio::time::timeout(UNSUBSCRIBE_TIMEOUT, dispatcher.unsubscribe(request)).await;
match sent {
Ok(Ok(_)) => {}
Ok(Err(status)) => tracing::warn!(
worker_id = %self.id,
code = ?status.code(),
message = status.message(),
"could not unsubscribe from the engine",
),
Err(_) => tracing::warn!(
worker_id = %self.id,
timeout = ?UNSUBSCRIBE_TIMEOUT,
"unsubscribing from the engine timed out",
),
}
}
}
#[derive(Clone, Debug)]
pub struct ShutdownHandle {
token: CancellationToken,
}
impl ShutdownHandle {
pub fn shutdown(&self) {
self.token.cancel();
}
}
fn refuses_this_worker(error: &Error) -> bool {
matches!(
error,
Error::Grpc(status)
if matches!(
status.code(),
Code::Unauthenticated | Code::PermissionDenied
)
)
}
fn connecting(
client: Client,
worker_id: String,
) -> impl FnMut() -> Pin<Box<dyn Future<Output = std::result::Result<ActionStream, Status>> + Send>>
{
move || {
let mut dispatcher = client.dispatcher_grpc();
let worker_id = worker_id.clone();
Box::pin(async move {
let actions = dispatcher
.listen_v2(WorkerListenRequest { worker_id })
.await?
.into_inner();
Ok(Box::pin(actions) as ActionStream)
})
}
}
fn beating(
client: Client,
worker_id: String,
) -> impl FnMut() -> Pin<Box<dyn Future<Output = std::result::Result<(), Status>> + Send>> {
move || {
let mut dispatcher = client.dispatcher_grpc();
let worker_id = worker_id.clone();
Box::pin(async move {
let mut request = Request::new(HeartbeatRequest {
worker_id,
heartbeat_at: Some(Timestamp::from(SystemTime::now())),
});
request.set_timeout(heartbeat::TIMEOUT);
match tokio::time::timeout(heartbeat::TIMEOUT, dispatcher.heartbeat(request)).await {
Ok(result) => result.map(|_| ()),
Err(_) => Err(Status::deadline_exceeded("the heartbeat attempt timed out")),
}
})
}
}
fn batch_reporter(client: Client) -> BatchReporter {
Arc::new(move |event: BatchActionEvent| {
let client = client.clone();
Box::pin(async move {
let sent = retry::unary(|| {
let mut dispatcher = client.dispatcher_grpc();
let request = retry::timed(event.clone());
async move { dispatcher.send_batch_action_event(request).await }
})
.await;
if let Err(status) = sent {
tracing::error!(
batch_id = ?event.batch_id,
action_id = %event.action_id,
event_type = event.event_type,
items = event.items.len(),
code = ?status.code(),
message = status.message(),
"could not report a batch event to the engine",
);
}
})
})
}
fn reporter(client: Client) -> Reporter {
Arc::new(move |event: StepActionEvent| {
let client = client.clone();
Box::pin(async move {
let sent = retry::unary(|| {
let mut dispatcher = client.dispatcher_grpc();
let request = retry::timed(event.clone());
async move { dispatcher.send_step_action_event(request).await }
})
.await;
if let Err(status) = sent {
tracing::error!(
task_run_external_id = %event.task_run_external_id,
event_type = event.event_type,
code = ?status.code(),
message = status.message(),
"could not report a task run event to the engine",
);
}
})
})
}
async fn put_workflow(client: &Client, request: &v1::CreateWorkflowVersionRequest) -> Result<()> {
retry::unary(|| {
let mut admin = client.admin_grpc();
let request = retry::timed(request.clone());
async move { admin.put_workflow(request).await }
})
.await?;
Ok(())
}
async fn register(client: &Client, request: &WorkerRegisterRequest) -> Result<String> {
let response = retry::unary(|| {
let mut dispatcher = client.dispatcher_grpc();
let request = retry::timed(request.clone());
async move { dispatcher.register(request).await }
})
.await?;
Ok(response.into_inner().worker_id)
}
#[cfg(unix)]
fn shutdown_signals(
signals: mpsc::Sender<()>,
) -> Result<impl Future<Output = ()> + Send + 'static> {
use tokio::signal::unix::{SignalKind, signal};
let install = |kind: SignalKind, name: &str| {
signal(kind).map_err(|error| Error::Config(format!("could not listen for {name}: {error}")))
};
let mut interrupt = install(SignalKind::interrupt(), "SIGINT")?;
let mut terminate = install(SignalKind::terminate(), "SIGTERM")?;
Ok(async move {
loop {
tokio::select! {
_ = interrupt.recv() => {}
_ = terminate.recv() => {}
}
if signals.send(()).await.is_err() {
return;
}
}
})
}
#[cfg(not(unix))]
fn shutdown_signals(
signals: mpsc::Sender<()>,
) -> Result<impl Future<Output = ()> + Send + 'static> {
Ok(async move {
loop {
if let Err(error) = tokio::signal::ctrl_c().await {
tracing::error!(
%error,
"could not listen for Ctrl-C; the worker will not stop on it",
);
return;
}
if signals.send(()).await.is_err() {
return;
}
}
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{TaskBuilder, TlsStrategy, Workflow};
use hatchet_proto::root::{ActionType, StepActionEventType};
use hatchet_testkit::MockEngine;
use serde_json::{Value, json};
use tokio::sync::Semaphore;
const TOKEN: &str = "e30.eyJzdWIiOiJ0Iiwic2VydmVyX3VybCI6Imh0dHA6Ly9sb2NhbGhvc3Q6ODg4OCIsImdycGNfYnJvYWRjYXN0X2FkZHJlc3MiOiJsb2NhbGhvc3Q6NzA3NyJ9.sig";
const NEVER: Duration = Duration::from_secs(30);
fn client() -> Client {
Client::builder()
.token(TOKEN)
.namespace("dev")
.tls(TlsStrategy::None)
.build()
.expect("the test token is well formed")
}
fn client_for(engine: &MockEngine) -> Client {
Client::builder()
.token(engine.token())
.namespace("dev")
.tls(TlsStrategy::None)
.build()
.expect("the mock engine's token is well formed")
}
fn echo(name: &str) -> crate::Task<Value, Value> {
TaskBuilder::new(name).handler(|_ctx, input| async move { Ok(input) })
}
fn durable(name: &str) -> crate::Task<Value, Value> {
TaskBuilder::new(name).durable_handler(|_ctx, input| async move { Ok(input) })
}
fn start_action(run_id: &str, action_id: &str) -> AssignedAction {
AssignedAction {
task_run_external_id: run_id.to_string(),
action_id: action_id.to_string(),
action_type: ActionType::StartStepRun as i32,
action_payload: json!({ "input": null }).to_string(),
..AssignedAction::default()
}
}
fn reported(engine: &MockEngine, run_id: &str, kind: StepActionEventType) -> bool {
engine
.requests::<StepActionEvent>()
.iter()
.any(|event| event.task_run_external_id == run_id && event.event_type == kind as i32)
}
async fn wait_for_event(engine: &MockEngine, run_id: &str, kind: StepActionEventType) {
let what = format!("{kind:?} for {run_id}");
engine
.wait_until(&what, |engine| reported(engine, run_id, kind))
.await;
}
async fn stopped(running: tokio::task::JoinHandle<Result<()>>) -> Result<()> {
tokio::time::timeout(NEVER, running)
.await
.expect("the worker never stopped")
.expect("the worker's task panicked")
}
#[test]
fn only_the_credential_codes_mean_nothing_can_be_reported() {
for code in [Code::Unauthenticated, Code::PermissionDenied] {
let error = Error::from(Status::new(code, "no"));
assert!(refuses_this_worker(&error), "{code:?}");
}
for code in [
Code::InvalidArgument,
Code::FailedPrecondition,
Code::NotFound,
Code::Unimplemented,
Code::Unknown,
Code::Internal,
] {
let error = Error::from(Status::new(code, "no"));
assert!(!refuses_this_worker(&error), "{code:?}");
}
assert!(!refuses_this_worker(&Error::Config("no slots".to_string())));
}
#[tokio::test]
async fn one_shutdown_signal_waits_for_the_runs_in_flight() {
let engine = MockEngine::start().await;
let gate = Arc::new(Semaphore::new(0));
let gated = TaskBuilder::<Value, Value>::new("gated").handler({
let gate = Arc::clone(&gate);
move |_ctx, _input| {
let gate = Arc::clone(&gate);
async move {
gate.acquire().await.expect("the gate stays open").forget();
Ok(json!("done"))
}
}
});
let worker = client_for(&engine)
.worker("w")
.register(&gated)
.build()
.await
.expect("the mock engine registers the worker");
let (signals, receiver) = mpsc::channel(4);
let running = tokio::spawn(worker.run_until(receiver));
engine.wait_for_listeners(1).await;
engine.push_action(start_action("in-flight", "dev_gated:gated"));
wait_for_event(
&engine,
"in-flight",
StepActionEventType::StepEventTypeStarted,
)
.await;
signals.send(()).await.expect("the loop is listening");
engine.wait_for_calls("Unsubscribe", 1).await;
assert!(
!running.is_finished(),
"the first signal must not abandon the run in flight",
);
gate.add_permits(1);
stopped(running)
.await
.expect("a graceful stop is not an error");
assert!(reported(
&engine,
"in-flight",
StepActionEventType::StepEventTypeCompleted
));
}
#[tokio::test]
async fn a_second_shutdown_signal_stops_without_waiting() {
let engine = MockEngine::start().await;
let (entered, mut has_entered) = mpsc::unbounded_channel();
let (finished, mut has_finished) = mpsc::unbounded_channel();
let stubborn = TaskBuilder::<Value, Value>::new("stubborn").handler(move |_ctx, _input| {
let (entered, finished) = (entered.clone(), finished.clone());
async move {
let _ = entered.send(());
std::future::pending::<()>().await;
let _ = finished.send(());
Ok(json!("never"))
}
});
let worker = client_for(&engine)
.worker("w")
.shutdown_grace(Duration::from_secs(300))
.register(&stubborn)
.build()
.await
.expect("the mock engine registers the worker");
let (signals, receiver) = mpsc::channel(4);
let running = tokio::spawn(worker.run_until(receiver));
engine.wait_for_listeners(1).await;
engine.push_action(start_action("stuck", "dev_stubborn:stubborn"));
wait_for_event(&engine, "stuck", StepActionEventType::StepEventTypeStarted).await;
has_entered.recv().await.expect("the handler ran");
signals.send(()).await.expect("the loop is listening");
engine.wait_for_calls("Unsubscribe", 1).await;
assert!(!running.is_finished());
signals.send(()).await.expect("the loop is still listening");
stopped(running)
.await
.expect("a forced stop is not an error");
assert!(
has_finished.try_recv().is_err(),
"the handler was not still blocked, so nothing was proven",
);
for kind in [
StepActionEventType::StepEventTypeCompleted,
StepActionEventType::StepEventTypeFailed,
] {
assert!(
!reported(&engine, "stuck", kind),
"a dropped run reported {kind:?}",
);
}
}
#[tokio::test]
async fn the_registration_request_describes_the_worker() {
let builder = client()
.worker("mac-studio-1")
.slots(4)
.label("os", "macos")
.label("cores", 12)
.register(&echo("agent-run"));
let request = builder.registration(vec!["dev_agent-run:agent-run".to_string()]);
assert_eq!(request.worker_name, "mac-studio-1");
assert_eq!(request.actions, ["dev_agent-run:agent-run"]);
assert_eq!(request.slot_config["default"], 4);
assert_eq!(request.slots, None);
assert_eq!(request.labels["os"].str_value.as_deref(), Some("macos"));
assert_eq!(request.labels["cores"].int_value, Some(12));
let runtime = request.runtime_info.expect("runtime info is always sent");
assert_eq!(
runtime.language, None,
"the engine rejects SDKS_UNKNOWN, so no language is sent"
);
assert_eq!(runtime.language_version, None);
assert_eq!(runtime.extra.as_deref(), Some("rust"));
}
#[tokio::test]
async fn the_slot_config_names_only_the_slot_types_the_worker_needs() {
let only_durable = client()
.worker("durable-only")
.durable_slots(9)
.register(&durable("approve"))
.registration(Vec::new());
assert_eq!(
only_durable.slot_config,
HashMap::from([("durable".to_string(), 9)])
);
let only_default = client()
.worker("plain")
.slots(7)
.register(&echo("fetch"))
.registration(Vec::new());
assert_eq!(
only_default.slot_config,
HashMap::from([("default".to_string(), 7)])
);
let mixed = client()
.worker("mixed")
.slots(7)
.durable_slots(9)
.register(&durable("approve"))
.register(&echo("fetch"))
.registration(Vec::new());
assert_eq!(
mixed.slot_config,
HashMap::from([("default".to_string(), 7), ("durable".to_string(), 9)])
);
let defaults = client()
.worker("defaults")
.register(&durable("approve"))
.registration(Vec::new());
assert_eq!(defaults.slot_config["durable"], 1000);
}
#[tokio::test]
async fn a_worker_without_durable_slots_is_rejected_before_it_talks_to_the_engine() {
let error = client()
.worker("no-durable-slots")
.durable_slots(0)
.register(&durable("approve"))
.build()
.await
.expect_err("no durable slots is a configuration mistake");
assert!(matches!(error, Error::Config(_)), "{error:?}");
client()
.worker("plain")
.durable_slots(0)
.register(&echo("fetch"))
.registration(Vec::new());
}
#[tokio::test]
async fn a_label_set_twice_keeps_the_last_value() {
let builder = client()
.worker("w")
.label("os", "linux")
.label("os", "macos")
.register(&echo("a"));
let request = builder.registration(Vec::new());
assert_eq!(request.labels.len(), 1);
assert_eq!(request.labels["os"].str_value.as_deref(), Some("macos"));
}
#[tokio::test]
async fn a_worker_without_slots_is_rejected_before_it_talks_to_the_engine() {
let error = client()
.worker("w")
.slots(0)
.register(&echo("a"))
.build()
.await
.expect_err("no slots is a configuration mistake");
assert!(matches!(error, Error::Config(_)), "{error:?}");
}
#[tokio::test]
async fn a_worker_with_nothing_to_run_is_rejected() {
let error = client()
.worker("w")
.build()
.await
.expect_err("a worker that runs nothing is a mistake");
let Error::InvalidDeclaration(message) = &error else {
panic!("{error:?}")
};
assert!(message.contains('w'), "{message}");
}
#[tokio::test]
async fn two_declarations_serving_the_same_action_are_rejected_by_name() {
let error = client()
.worker("w")
.register(&Workflow::new("pipeline").task(&echo("build")))
.register(&Workflow::new("Pipeline").task(&echo("Build")))
.build()
.await
.expect_err("two declarations cannot serve one action");
let Error::InvalidDeclaration(message) = &error else {
panic!("{error:?}")
};
assert!(message.contains("dev_pipeline:build"), "{message}");
}
#[tokio::test]
async fn an_invalid_declaration_is_rejected() {
let error = client()
.worker("w")
.register(&Workflow::new("empty"))
.build()
.await
.expect_err("a workflow without tasks is invalid");
assert!(matches!(error, Error::InvalidDeclaration(_)), "{error:?}");
}
#[tokio::test]
async fn debug_output_names_the_worker_without_its_token() {
let builder = client().worker("mac-studio-1").label("os", "macos");
let debug = format!("{builder:?}");
assert!(debug.starts_with("WorkerBuilder"), "{debug}");
assert!(debug.contains("mac-studio-1"), "{debug}");
assert!(debug.contains(".."), "{debug}");
assert!(!debug.contains(TOKEN), "{debug}");
let client = client();
let worker = Worker {
id: "worker-1".to_string(),
slots: 4,
grace: DEFAULT_SHUTDOWN_GRACE,
stream_error_drain: DEFAULT_STREAM_ERROR_DRAIN,
runner: Arc::new(Runner::new(
"worker-1".to_string(),
Some(client.clone()),
HashMap::new(),
4,
None,
reporter(client.clone()),
batch_reporter(client.clone()),
)),
durable: None,
durable_slots: DEFAULT_DURABLE_SLOTS,
durable_stop: CancellationToken::new(),
eviction: None,
shutdown: CancellationToken::new(),
client,
};
let debug = format!("{worker:?}");
assert!(debug.starts_with("Worker"), "{debug}");
assert!(debug.contains("worker-1"), "{debug}");
assert!(debug.contains("in_flight: 0"), "{debug}");
assert!(!debug.contains(TOKEN), "{debug}");
assert!(!format!("{:?}", worker.shutdown_handle()).contains(TOKEN));
}
}