rusty_worker/lib.rs
1//! # rusty-worker
2//!
3//! The worker-side SDK for `rusty-agent-runtime` remote node execution: *one
4//! `Node` trait, remote impls behind the same trait*. A worker is just an HTTP
5//! service that hosts [`Node`] handlers by name; a graph node registered as
6//! [`rusty_agent_runtime::remote::RemoteNode`] calls into it transparently.
7//!
8//! ## Endpoints
9//!
10//! - `POST /execute` — accepts a JSON [`NodeTask`], dispatches to the handler
11//! registered under [`NodeTask::node`], and replies with a JSON
12//! [`NodeTaskResponse`]:
13//! - `Ok(output)` → `{ "output": ... }`
14//! - `Err(interrupt)` → `{ "interrupt": <value> }` (HITL across the wire)
15//! - `Err(e)` → `{ "error": "<message>" }`
16//! - `GET /ok` — liveness + capability probe: protocol version and the
17//! registered handler names.
18//!
19//! ## Registering handlers
20//!
21//! [`WorkerRegistry::register`] accepts **anything that implements
22//! [`Node`]** — which, thanks to the blanket impl in the core crate, includes
23//! ordinary async closures `Fn(NodeContext) -> impl Future<Output =
24//! Result<NodeOutput>>`, named `Node` impls, and `Arc<dyn Node>`.
25//!
26//! ```no_run
27//! use rusty_agent_runtime::prelude::*;
28//! use rusty_worker::{serve, WorkerRegistry};
29//!
30//! # async fn demo() -> std::io::Result<()> {
31//! let mut registry = WorkerRegistry::new();
32//! registry.register("greeter", |ctx: NodeContext| async move {
33//! let name = ctx
34//! .state()
35//! .get("name")
36//! .and_then(|v| v.as_str())
37//! .unwrap_or("world")
38//! .to_string();
39//! Ok(NodeOutput::update("greeting", serde_json::json!(format!("hello, {name}!"))))
40//! });
41//!
42//! serve(registry, "127.0.0.1:8200").await
43//! # }
44//! ```
45//!
46//! On the graph side, point a `RemoteNode` at the same handler name:
47//!
48//! ```ignore
49//! builder.add_node("greeter", RemoteNode::new("greeter", "http://127.0.0.1:8200"));
50//! ```
51//!
52//! ## Error semantics across the wire
53//!
54//! Handler errors are flattened to a message string in
55//! [`NodeTaskResponse::error`] and arrive client-side as
56//! `RustyError::Node`, which the executor treats as a **hard failure** —
57//! the retryable classes (`Llm`, `Tool`) do not survive the wire. A remote
58//! node whose transient failures should be retried must therefore rely on
59//! transport-level retry (connection/timeout/5xx on the client) or surface
60//! retryable outcomes through its own protocol on top of `extra`.
61//!
62//! ## Durable activities (R0.6)
63//!
64//! [`ActivityWorker`] is the pull-based counterpart to [`serve`]: it claims
65//! leased tasks (`kind` + JSON `payload`) from the rusty-agent-server task queue
66//! (`POST /tasks/claim`), executes the [`activity::Activity`] registered for
67//! the task's kind while a background heartbeat renews the lease every
68//! `lease / 3`, and settles with `complete` or a classified `fail`. Lease
69//! loss (`409`) aborts the activity via a `CancellationToken`, and
70//! cancelling the shutdown token drains the worker: claiming stops, the
71//! in-flight activity settles within a bounded grace (default
72//! [`activity::DEFAULT_DRAIN_GRACE`]), and an attempt that outlives the
73//! grace is aborted and left for the server to reassign at lease expiry.
74//! See the [`activity`] module for the protocol and semantics.
75
76use std::collections::HashMap;
77use std::sync::Arc;
78
79use axum::extract::State as AxumState;
80use axum::http::StatusCode;
81use axum::response::Json;
82use axum::routing::{get, post};
83use axum::Router;
84use rusty_agent_runtime::error::RustyError;
85use rusty_agent_runtime::node::{Node, NodeContext};
86use rusty_agent_runtime::remote::{NodeTask, NodeTaskResponse, PROTOCOL_VERSION};
87use serde::Serialize;
88use serde_json::{json, Value};
89use tracing::Instrument;
90use uuid::Uuid;
91
92pub mod activity;
93
94pub use activity::{Activity, ActivityCompletion, ActivityContext, ActivityWorker};
95
96/// The registry of named node handlers a worker serves.
97///
98/// Cheap to clone (handlers are `Arc`'d); build one up front, then hand it to
99/// [`router`] or [`serve`].
100#[derive(Clone, Default)]
101pub struct WorkerRegistry {
102 handlers: HashMap<String, Arc<dyn Node>>,
103}
104
105impl WorkerRegistry {
106 /// An empty registry.
107 pub fn new() -> Self {
108 Self::default()
109 }
110
111 /// Register a handler under `name`.
112 ///
113 /// Accepts any [`Node`] implementation — including plain async closures
114 /// via the core blanket impl, so the ergonomics match
115 /// `GraphBuilder::add_node` exactly.
116 ///
117 /// Registering the same name twice replaces the previous handler.
118 pub fn register<N>(&mut self, name: impl Into<String>, node: N) -> &mut Self
119 where
120 N: Node + 'static,
121 {
122 self.handlers.insert(name.into(), Arc::new(node));
123 self
124 }
125
126 /// Builder-style variant of [`WorkerRegistry::register`].
127 pub fn with<N>(mut self, name: impl Into<String>, node: N) -> Self
128 where
129 N: Node + 'static,
130 {
131 self.register(name, node);
132 self
133 }
134
135 /// `true` if a handler is registered under `name`.
136 pub fn contains(&self, name: &str) -> bool {
137 self.handlers.contains_key(name)
138 }
139
140 /// Number of registered handlers.
141 pub fn len(&self) -> usize {
142 self.handlers.len()
143 }
144
145 /// `true` if no handlers are registered.
146 pub fn is_empty(&self) -> bool {
147 self.handlers.is_empty()
148 }
149
150 /// All registered handler names.
151 pub fn names(&self) -> impl Iterator<Item = &str> {
152 self.handlers.keys().map(String::as_str)
153 }
154
155 /// Look up a handler by name.
156 pub fn handler(&self, name: &str) -> Option<Arc<dyn Node>> {
157 self.handlers.get(name).cloned()
158 }
159}
160
161/// The shared state handed to axum handlers.
162type SharedRegistry = Arc<WorkerRegistry>;
163
164/// Liveness response for `GET /ok`.
165#[derive(Debug, Serialize)]
166struct OkResponse {
167 status: &'static str,
168 protocol_version: u32,
169 nodes: Vec<String>,
170}
171
172/// `GET /ok`: liveness + capability probe.
173async fn ok_handler(AxumState(registry): AxumState<SharedRegistry>) -> Json<OkResponse> {
174 let mut nodes: Vec<String> = registry.names().map(str::to_owned).collect();
175 nodes.sort();
176 Json(OkResponse {
177 status: "ok",
178 protocol_version: PROTOCOL_VERSION,
179 nodes,
180 })
181}
182
183/// `POST /execute`: dispatch a [`NodeTask`] to its handler and shape the
184/// outcome as a [`NodeTaskResponse`].
185///
186/// Status codes:
187///
188/// - `200 OK` for all handler-level outcomes (success, handler error,
189/// interrupt, unknown handler, handler panic) — outcome lives in the
190/// response body, so `RemoteNode` never mistakes a worker-side application
191/// error for a transport failure. A panicking handler is caught and
192/// returned as an error body rather than dropped connection, because the
193/// client retries transport failures and node logic must not be replayed
194/// silently.
195/// - `400 Bad Request` when the protocol version is unsupported (a
196/// client/worker mismatch the client should not retry blindly).
197async fn execute_handler(
198 AxumState(registry): AxumState<SharedRegistry>,
199 Json(task): Json<NodeTask>,
200) -> (StatusCode, Json<NodeTaskResponse>) {
201 let request_id = Uuid::new_v4();
202 let span = tracing::info_span!(
203 "rusty.execute",
204 %request_id,
205 node = %task.node,
206 thread_id = %task.config.thread_id,
207 step = task.config.step,
208 protocol_version = task.protocol_version,
209 );
210 // Attached via `.instrument()` (never `.enter()`) so no span guard is
211 // held across `.await` points — the same discipline as the core
212 // executor, which otherwise misattributes spans on multi-threaded
213 // runtimes.
214 execute_task(registry, task).instrument(span).await
215}
216
217/// The body of `execute_handler`, run inside the `rusty.execute` span.
218async fn execute_task(
219 registry: SharedRegistry,
220 task: NodeTask,
221) -> (StatusCode, Json<NodeTaskResponse>) {
222 if task.protocol_version != PROTOCOL_VERSION {
223 tracing::warn!("unsupported protocol version");
224 return (
225 StatusCode::BAD_REQUEST,
226 Json(NodeTaskResponse::error(format!(
227 "unsupported protocol_version {} (this worker speaks {})",
228 task.protocol_version, PROTOCOL_VERSION
229 ))),
230 );
231 }
232
233 let Some(handler) = registry.handler(&task.node) else {
234 tracing::warn!("no handler registered for node");
235 let mut registered: Vec<&str> = registry.names().collect();
236 registered.sort_unstable();
237 return (
238 StatusCode::OK,
239 Json(NodeTaskResponse::error(format!(
240 "no handler registered for node `{}` on this worker (registered: {registered:?})",
241 task.node,
242 ))),
243 );
244 };
245
246 let node_name = task.node.clone();
247 let resuming = task.config.resume.is_some();
248 let ctx = NodeContext::new(task.state, task.config);
249 // The handler runs on its own task so a panic surfaces as a `JoinError`
250 // instead of tearing down the connection: a dropped connection reaches
251 // the client as a *transport* failure, which `RemoteNode` retries —
252 // silently replaying node logic the protocol says must never be
253 // replayed. Mapping the panic to a 200 + error body keeps it a hard,
254 // non-retried node failure.
255 let outcome = tokio::spawn(async move { handler.run(ctx).await }).await;
256 let result = match outcome {
257 Ok(result) => result,
258 Err(join_err) => {
259 let detail = if join_err.is_panic() {
260 let payload = join_err.into_panic();
261 let message = payload
262 .downcast_ref::<&str>()
263 .map(|s| (*s).to_owned())
264 .or_else(|| payload.downcast_ref::<String>().cloned())
265 .unwrap_or_else(|| "non-string panic payload".to_owned());
266 format!("handler panicked: {message}")
267 } else {
268 // Only reachable if the runtime is shutting down mid-request.
269 "handler task cancelled".to_owned()
270 };
271 Err(RustyError::Node(format!("node `{node_name}` {detail}")))
272 }
273 };
274 match result {
275 Ok(output) => {
276 tracing::info!(resuming, "node executed");
277 (StatusCode::OK, Json(NodeTaskResponse::ok(output)))
278 }
279 Err(e) if e.is_interrupt() => {
280 let value = e.interrupt_value().cloned().unwrap_or(Value::Null);
281 tracing::info!(payload = %value, "node interrupted");
282 (StatusCode::OK, Json(NodeTaskResponse::interrupt(value)))
283 }
284 Err(e) => {
285 tracing::warn!(error = %e, "node failed");
286 (StatusCode::OK, Json(NodeTaskResponse::error(e.to_string())))
287 }
288 }
289}
290
291/// Build the axum [`Router`] for a registry (`POST /execute` + `GET /ok`).
292///
293/// Exposed separately from [`serve`] so tests and embedders can bind their
294/// own listener (e.g. an ephemeral port) and drive the app with
295/// `axum::serve`.
296pub fn router(registry: WorkerRegistry) -> Router {
297 Router::new()
298 .route("/execute", post(execute_handler))
299 .route("/ok", get(ok_handler))
300 .with_state(Arc::new(registry))
301}
302
303/// Serve a registry on `addr` until the process is stopped.
304///
305/// ```no_run
306/// # use rusty_worker::{serve, WorkerRegistry};
307/// # async fn demo() -> std::io::Result<()> {
308/// serve(WorkerRegistry::new(), "127.0.0.1:8200").await
309/// # }
310/// ```
311pub async fn serve(registry: WorkerRegistry, addr: impl AsRef<str>) -> std::io::Result<()> {
312 let listener = tokio::net::TcpListener::bind(addr.as_ref()).await?;
313 let local_addr = listener.local_addr()?;
314 tracing::info!(
315 addr = %local_addr,
316 nodes = ?registry.names().collect::<Vec<_>>(),
317 "rusty worker listening"
318 );
319 axum::serve(listener, router(registry)).await
320}
321
322/// Convenience JSON body for quick manual probes (`curl` examples in docs).
323pub fn probe_body() -> Value {
324 json!({
325 "protocol_version": PROTOCOL_VERSION,
326 "node": "<handler-name>",
327 "state": {},
328 "config": { "thread_id": "t-1", "step": 0, "resume": null, "extra": {} }
329 })
330}
331
332#[cfg(test)]
333mod tests {
334 use super::*;
335 use rusty_agent_runtime::node::NodeOutput;
336 use rusty_agent_runtime::state::State;
337 use serde_json::json;
338
339 #[test]
340 fn registry_register_and_lookup() {
341 let mut registry = WorkerRegistry::new();
342 assert!(registry.is_empty());
343
344 registry.register("a", |_ctx: NodeContext| async { Ok(NodeOutput::empty()) });
345 registry.register("b", |_ctx: NodeContext| async { Ok(NodeOutput::empty()) });
346
347 assert_eq!(registry.len(), 2);
348 assert!(registry.contains("a"));
349 assert!(registry.contains("b"));
350 assert!(!registry.contains("c"));
351 assert!(registry.handler("a").is_some());
352
353 let mut names: Vec<&str> = registry.names().collect();
354 names.sort();
355 assert_eq!(names, vec!["a", "b"]);
356 }
357
358 #[test]
359 fn registry_builder_style_and_replace() {
360 let registry = WorkerRegistry::new()
361 .with("x", |_ctx: NodeContext| async {
362 Ok(NodeOutput::update("v", json!(1)))
363 })
364 .with("x", |_ctx: NodeContext| async {
365 Ok(NodeOutput::update("v", json!(2)))
366 });
367 assert_eq!(registry.len(), 1);
368 }
369
370 #[test]
371 fn probe_body_is_a_valid_node_task_shape() {
372 let body = probe_body();
373 let task: std::result::Result<NodeTask, _> = serde_json::from_value(body);
374 let task = task.unwrap();
375 assert_eq!(task.protocol_version, PROTOCOL_VERSION);
376 assert_eq!(task.state, State::new());
377 }
378}