use std::collections::HashMap;
use std::time::Duration;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::error::{Result, RustyError};
use crate::llm::{backoff_delay, truncate_body};
use crate::node::{Command, Node, NodeContext, NodeOutput};
use crate::state::State;
pub const PROTOCOL_VERSION: u32 = 1;
pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
pub const DEFAULT_MAX_RETRIES: u32 = 2;
pub const DEFAULT_BASE_BACKOFF: Duration = Duration::from_millis(100);
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NodeTask {
pub protocol_version: u32,
pub node: String,
pub state: State,
pub config: crate::node::NodeConfig,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub task_id: Option<String>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct WireNodeOutput {
#[serde(default)]
pub updates: HashMap<String, Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub command: Option<Command>,
}
impl From<NodeOutput> for WireNodeOutput {
fn from(output: NodeOutput) -> Self {
Self {
updates: output.updates,
command: output.command,
}
}
}
impl From<WireNodeOutput> for NodeOutput {
fn from(wire: WireNodeOutput) -> Self {
Self {
updates: wire.updates,
command: wire.command,
}
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct NodeTaskResponse {
#[serde(default)]
pub protocol_version: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output: Option<WireNodeOutput>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub interrupt: Option<Value>,
}
impl NodeTaskResponse {
pub fn ok(output: impl Into<WireNodeOutput>) -> Self {
Self {
protocol_version: PROTOCOL_VERSION,
output: Some(output.into()),
error: None,
interrupt: None,
}
}
pub fn error(message: impl Into<String>) -> Self {
Self {
protocol_version: PROTOCOL_VERSION,
output: None,
error: Some(message.into()),
interrupt: None,
}
}
pub fn interrupt(value: Value) -> Self {
Self {
protocol_version: PROTOCOL_VERSION,
output: None,
error: None,
interrupt: Some(value),
}
}
pub fn into_result(self) -> Result<NodeOutput> {
let set = self.output.is_some() as u8
+ self.error.is_some() as u8
+ self.interrupt.is_some() as u8;
if set != 1 {
return Err(RustyError::Node(format!(
"malformed worker response: expected exactly one of \
output/error/interrupt, found {set} set"
)));
}
if let Some(value) = self.interrupt {
return Err(RustyError::Interrupt { value });
}
if let Some(message) = self.error {
return Err(RustyError::Node(message));
}
Ok(self.output.expect("checked exactly-one above").into())
}
}
#[derive(Debug)]
enum AttemptError {
Retryable {
error: RustyError,
retry_after: Option<Duration>,
},
Fatal(RustyError),
}
#[derive(Clone)]
pub struct RemoteNode {
node_name: String,
execute_url: String,
client: reqwest::Client,
timeout: Duration,
max_retries: u32,
base_backoff: Duration,
}
impl RemoteNode {
pub fn new(node_name: impl Into<String>, base_url: impl Into<String>) -> Self {
let base = base_url.into();
let base = base.trim_end_matches('/');
let execute_url = if base.ends_with("/execute") {
base.to_owned()
} else {
format!("{base}/execute")
};
Self {
node_name: node_name.into(),
execute_url,
client: Self::build_client(DEFAULT_TIMEOUT),
timeout: DEFAULT_TIMEOUT,
max_retries: DEFAULT_MAX_RETRIES,
base_backoff: DEFAULT_BASE_BACKOFF,
}
}
pub fn with_timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout;
self.client = Self::build_client(timeout);
self
}
pub fn with_retries(mut self, max_retries: u32) -> Self {
self.max_retries = max_retries;
self
}
pub fn with_backoff(mut self, base_backoff: Duration) -> Self {
self.base_backoff = base_backoff;
self
}
pub fn node_name(&self) -> &str {
&self.node_name
}
pub fn execute_url(&self) -> &str {
&self.execute_url
}
fn build_client(timeout: Duration) -> reqwest::Client {
reqwest::Client::builder()
.timeout(timeout)
.build()
.expect("reqwest client builder with rustls must succeed")
}
async fn try_once(
&self,
task: &NodeTask,
) -> std::result::Result<NodeTaskResponse, AttemptError> {
let response = self
.client
.post(&self.execute_url)
.json(task)
.send()
.await
.map_err(|e| {
let err = RustyError::Node(format!(
"remote node `{}`: POST {} failed: {e}",
self.node_name, self.execute_url
));
if e.is_timeout() || e.is_connect() {
AttemptError::Retryable {
error: err,
retry_after: None,
}
} else {
AttemptError::Fatal(err)
}
})?;
let status = response.status();
if !status.is_success() {
let retry_after = response
.headers()
.get(reqwest::header::RETRY_AFTER)
.and_then(|v| v.to_str().ok())
.and_then(|s| s.trim().parse::<u64>().ok())
.map(Duration::from_secs);
let body = response.text().await.unwrap_or_default();
let err = RustyError::Node(format!(
"remote node `{}`: worker at {} returned {status}: {}",
self.node_name,
self.execute_url,
truncate_body(&body)
));
let retryable = status.is_server_error()
|| status == reqwest::StatusCode::REQUEST_TIMEOUT
|| status == reqwest::StatusCode::TOO_MANY_REQUESTS;
return Err(if retryable {
AttemptError::Retryable {
error: err,
retry_after,
}
} else {
AttemptError::Fatal(err)
});
}
response.json::<NodeTaskResponse>().await.map_err(|e| {
AttemptError::Fatal(RustyError::Node(format!(
"remote node `{}`: could not decode worker response: {e}",
self.node_name
)))
})
}
}
#[async_trait]
impl Node for RemoteNode {
fn name(&self) -> &str {
&self.node_name
}
fn effect(&self) -> crate::record::Effect {
crate::record::Effect::NonIdempotent
}
async fn run(&self, ctx: NodeContext) -> Result<NodeOutput> {
let task = NodeTask {
protocol_version: PROTOCOL_VERSION,
node: self.node_name.clone(),
state: ctx.state().clone(),
config: ctx.config().clone(),
task_id: None,
};
let mut attempt: u32 = 0;
loop {
match self.try_once(&task).await {
Ok(response) => return response.into_result(),
Err(AttemptError::Fatal(e)) => return Err(e),
Err(AttemptError::Retryable { error, retry_after })
if attempt < self.max_retries =>
{
let mut delay = backoff_delay(self.base_backoff, attempt);
if let Some(floor) = retry_after {
delay = delay.max(floor);
}
tracing::warn!(
node = %self.node_name,
url = %self.execute_url,
attempt = attempt + 1,
max_retries = self.max_retries,
backoff_ms = delay.as_millis() as u64,
error = %error,
"remote node attempt failed; retrying"
);
tokio::time::sleep(delay).await;
attempt += 1;
}
Err(AttemptError::Retryable { error, .. }) => return Err(error),
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::node::NodeConfig;
use serde_json::json;
use std::net::SocketAddr;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use tokio::task::JoinHandle;
#[test]
fn node_task_serde_roundtrip() {
let task = NodeTask {
protocol_version: PROTOCOL_VERSION,
node: "doubler".into(),
state: State::from_value(json!({"n": 21, "xs": [1, 2]})).unwrap(),
config: NodeConfig {
thread_id: "t-1".into(),
step: 3,
resume: Some(json!({"approved": true})),
extra: HashMap::from([("tag".to_string(), json!("demo"))]),
},
task_id: Some("task-7".into()),
};
let json_str = serde_json::to_string(&task).unwrap();
let back: NodeTask = serde_json::from_str(&json_str).unwrap();
assert_eq!(back.protocol_version, PROTOCOL_VERSION);
assert_eq!(back.node, "doubler");
assert_eq!(back.state.get("n"), Some(&json!(21)));
assert_eq!(back.config.thread_id, "t-1");
assert_eq!(back.config.step, 3);
assert_eq!(back.config.resume, Some(json!({"approved": true})));
assert_eq!(back.config.extra.get("tag"), Some(&json!("demo")));
assert_eq!(back.task_id.as_deref(), Some("task-7"));
}
#[test]
fn node_task_task_id_is_additive_within_v1() {
let legacy = r#"{
"protocol_version": 1,
"node": "doubler",
"state": {"n": 21},
"config": {"thread_id": "t-1", "step": 3, "resume": null, "extra": {}}
}"#;
let task: NodeTask = serde_json::from_str(legacy).unwrap();
assert_eq!(task.task_id, None);
let task = NodeTask {
protocol_version: PROTOCOL_VERSION,
node: "doubler".into(),
state: State::new(),
config: NodeConfig::default(),
task_id: None,
};
let value = serde_json::to_value(&task).unwrap();
assert!(value.get("task_id").is_none());
}
#[test]
fn node_task_response_serde_roundtrips_all_variants() {
let ok = NodeTaskResponse::ok(
NodeOutput::update("x", json!(1)).with_command(Command::goto("next")),
);
let back: NodeTaskResponse =
serde_json::from_str(&serde_json::to_string(&ok).unwrap()).unwrap();
assert_eq!(back.protocol_version, ok.protocol_version);
assert_eq!(
back.output.as_ref().unwrap().updates,
ok.output.as_ref().unwrap().updates
);
assert_eq!(back.error, ok.error);
assert_eq!(back.interrupt, ok.interrupt);
assert_eq!(
back.output.as_ref().unwrap().command.as_ref().unwrap().goto,
vec!["next".to_string()]
);
let err = NodeTaskResponse::error("boom");
let back: NodeTaskResponse =
serde_json::from_str(&serde_json::to_string(&err).unwrap()).unwrap();
assert_eq!(back.error.as_deref(), Some("boom"));
assert!(back.output.is_none() && back.interrupt.is_none());
let interrupt = NodeTaskResponse::interrupt(json!({"question": "approve?"}));
let back: NodeTaskResponse =
serde_json::from_str(&serde_json::to_string(&interrupt).unwrap()).unwrap();
assert_eq!(back.interrupt, Some(json!({"question": "approve?"})));
assert!(back.output.is_none() && back.error.is_none());
}
#[test]
fn loose_interrupt_shape_parses_and_maps_to_interrupt() {
let response: NodeTaskResponse =
serde_json::from_str(r#"{"interrupt": {"question": "approve?"}}"#).unwrap();
let err = response.into_result().unwrap_err();
assert!(err.is_interrupt());
assert_eq!(
err.interrupt_value(),
Some(&json!({"question": "approve?"}))
);
}
#[test]
fn response_requires_exactly_one_payload() {
let none = NodeTaskResponse::default();
let err = none.into_result().unwrap_err();
assert!(matches!(err, RustyError::Node(_)));
assert!(err.to_string().contains("malformed"));
let mut both = NodeTaskResponse::ok(NodeOutput::empty());
both.error = Some("also an error".into());
let err = both.into_result().unwrap_err();
assert!(matches!(err, RustyError::Node(_)));
assert!(err.to_string().contains("malformed"));
}
#[test]
fn wire_node_output_converts_losslessly_both_ways() {
let output = NodeOutput::update("a", json!(1))
.with_update("b", json!([2, 3]))
.with_command(Command::goto_many(["n1", "n2"]));
let wire = WireNodeOutput::from(output.clone());
let back: NodeOutput = wire.clone().into();
assert_eq!(back.updates, output.updates);
assert_eq!(
back.command.as_ref().unwrap().goto,
output.command.as_ref().unwrap().goto
);
let wire2: WireNodeOutput =
serde_json::from_str(&serde_json::to_string(&wire).unwrap()).unwrap();
assert_eq!(wire2.updates, wire.updates);
assert_eq!(
wire2.command.as_ref().unwrap().goto,
wire.command.as_ref().unwrap().goto
);
}
enum MockBehavior {
Respond { status: u16, body: String },
Hang,
}
struct MockServer {
addr: SocketAddr,
attempts: Arc<AtomicUsize>,
bodies: Arc<std::sync::Mutex<Vec<String>>>,
_handle: JoinHandle<()>,
}
fn start_mock<F>(handler: F) -> MockServer
where
F: Fn(usize, String) -> MockBehavior + Send + Sync + 'static,
{
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let addr = listener.local_addr().unwrap();
let attempts = Arc::new(AtomicUsize::new(0));
let bodies: Arc<std::sync::Mutex<Vec<String>>> =
Arc::new(std::sync::Mutex::new(Vec::new()));
let attempts2 = attempts.clone();
let bodies2 = bodies.clone();
let handler = Arc::new(handler);
let handle = tokio::spawn(async move {
let listener = TcpListener::from_std(listener).unwrap();
loop {
let Ok((mut stream, _)) = listener.accept().await else {
continue;
};
let attempts = attempts2.clone();
let bodies = bodies2.clone();
let handler = handler.clone();
tokio::spawn(async move {
let Some(body) = read_http_request(&mut stream).await else {
return; };
let n = attempts.fetch_add(1, Ordering::SeqCst) + 1;
bodies.lock().unwrap().push(body.clone());
match handler(n, body) {
MockBehavior::Hang => {
std::future::pending::<()>().await;
}
MockBehavior::Respond { status, body } => {
let reason = match status {
200 => "OK",
400 => "Bad Request",
500 => "Internal Server Error",
_ => "Status",
};
let response = format!(
"HTTP/1.1 {status} {reason}\r\n\
content-type: application/json\r\n\
content-length: {}\r\n\
connection: close\r\n\
\r\n\
{body}",
body.len()
);
let _ = stream.write_all(response.as_bytes()).await;
let _ = stream.flush().await;
}
}
});
}
});
MockServer {
addr,
attempts,
bodies,
_handle: handle,
}
}
async fn read_http_request(stream: &mut tokio::net::TcpStream) -> Option<String> {
let mut buf: Vec<u8> = Vec::new();
let mut chunk = [0u8; 4096];
let header_end = loop {
if let Some(pos) = find_subslice(&buf, b"\r\n\r\n") {
break pos + 4;
}
let n = stream.read(&mut chunk).await.ok()?;
if n == 0 {
return None;
}
buf.extend_from_slice(&chunk[..n]);
};
let headers = String::from_utf8_lossy(&buf[..header_end]).to_lowercase();
let content_length: usize = headers
.lines()
.find_map(|line| line.strip_prefix("content-length:"))
.and_then(|v| v.trim().parse().ok())
.unwrap_or(0);
while buf.len() < header_end + content_length {
let n = stream.read(&mut chunk).await.ok()?;
if n == 0 {
break;
}
buf.extend_from_slice(&chunk[..n]);
}
String::from_utf8(buf[header_end..header_end + content_length].to_vec()).ok()
}
fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option<usize> {
haystack
.windows(needle.len())
.position(|window| window == needle)
}
fn test_ctx() -> NodeContext {
NodeContext::new(
State::from_value(json!({"n": 21})).unwrap(),
NodeConfig {
thread_id: "t-test".into(),
step: 2,
resume: None,
extra: HashMap::new(),
},
)
}
#[tokio::test]
async fn remote_node_success_over_http() {
let server = start_mock(|_n, _body| MockBehavior::Respond {
status: 200,
body: serde_json::to_string(&NodeTaskResponse::ok(
NodeOutput::update("doubled", json!(42)).with_command(Command::goto("final")),
))
.unwrap(),
});
let node = RemoteNode::new("doubler", format!("http://{}", server.addr)).with_retries(0);
let out = Node::run(&node, test_ctx()).await.unwrap();
assert_eq!(out.updates.get("doubled"), Some(&json!(42)));
assert_eq!(
out.command.as_ref().unwrap().goto,
vec!["final".to_string()]
);
assert_eq!(node.name(), "doubler");
assert!(node.execute_url().ends_with("/execute"));
assert_eq!(server.attempts.load(Ordering::SeqCst), 1);
let sent = server.bodies.lock().unwrap()[0].clone();
let task: NodeTask = serde_json::from_str(&sent).unwrap();
assert_eq!(task.protocol_version, PROTOCOL_VERSION);
assert_eq!(task.node, "doubler");
assert_eq!(task.state.get("n"), Some(&json!(21)));
assert_eq!(task.config.thread_id, "t-test");
assert_eq!(task.config.step, 2);
}
#[tokio::test]
async fn remote_node_retries_on_5xx_then_succeeds() {
let server = start_mock(|n, _body| {
if n < 3 {
MockBehavior::Respond {
status: 500,
body: r#"{"error":"worker overloaded"}"#.to_string(),
}
} else {
MockBehavior::Respond {
status: 200,
body: serde_json::to_string(&NodeTaskResponse::ok(NodeOutput::update(
"x",
json!("third time lucky"),
)))
.unwrap(),
}
}
});
let node = RemoteNode::new("flaky", format!("http://{}", server.addr))
.with_retries(5)
.with_backoff(Duration::from_millis(1));
let out = Node::run(&node, test_ctx()).await.unwrap();
assert_eq!(out.updates.get("x"), Some(&json!("third time lucky")));
assert_eq!(server.attempts.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn remote_node_gives_up_after_retries_exhausted() {
let server = start_mock(|_n, _body| MockBehavior::Respond {
status: 500,
body: r#"{"error":"always down"}"#.to_string(),
});
let node = RemoteNode::new("down", format!("http://{}", server.addr))
.with_retries(2)
.with_backoff(Duration::from_millis(1));
let err = Node::run(&node, test_ctx()).await.unwrap_err();
assert!(matches!(err, RustyError::Node(_)));
assert!(err.to_string().contains("500"));
assert!(!err.is_interrupt());
assert_eq!(server.attempts.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn remote_node_timeout_is_retryable_and_bounded() {
let server = start_mock(|_n, _body| MockBehavior::Hang);
let node = RemoteNode::new("slow", format!("http://{}", server.addr))
.with_timeout(Duration::from_millis(100))
.with_retries(1)
.with_backoff(Duration::from_millis(1));
let started = std::time::Instant::now();
let err = Node::run(&node, test_ctx()).await.unwrap_err();
let elapsed = started.elapsed();
assert!(matches!(err, RustyError::Node(_)));
assert_eq!(server.attempts.load(Ordering::SeqCst), 2);
assert!(elapsed < Duration::from_secs(5), "took {elapsed:?}");
}
#[tokio::test]
async fn remote_node_worker_error_maps_to_node_error_without_retry() {
let server = start_mock(|_n, _body| MockBehavior::Respond {
status: 200,
body: serde_json::to_string(&NodeTaskResponse::error(
"handler `doubler` panicked: divide by zero",
))
.unwrap(),
});
let node = RemoteNode::new("doubler", format!("http://{}", server.addr))
.with_retries(5)
.with_backoff(Duration::from_millis(1));
let err = Node::run(&node, test_ctx()).await.unwrap_err();
match err {
RustyError::Node(message) => {
assert!(message.contains("divide by zero"), "got: {message}")
}
other => panic!("expected Node error, got {other:?}"),
}
assert_eq!(server.attempts.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn remote_node_interrupt_passes_through_without_retry() {
let server = start_mock(|_n, _body| MockBehavior::Respond {
status: 200,
body: r#"{"interrupt": {"question": "approve deployment?"}}"#.to_string(),
});
let node = RemoteNode::new("gate", format!("http://{}", server.addr))
.with_retries(5)
.with_backoff(Duration::from_millis(1));
let err = Node::run(&node, test_ctx()).await.unwrap_err();
assert!(err.is_interrupt());
assert_eq!(
err.interrupt_value(),
Some(&json!({"question": "approve deployment?"}))
);
assert_eq!(server.attempts.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn remote_node_resume_value_crosses_the_wire() {
let server = start_mock(|_n, body| {
let task: NodeTask = serde_json::from_str(&body).unwrap();
let response = match task.config.resume {
Some(v) => NodeTaskResponse::ok(NodeOutput::update("approved", v)),
None => NodeTaskResponse::interrupt(json!({"question": "approve?"})),
};
MockBehavior::Respond {
status: 200,
body: serde_json::to_string(&response).unwrap(),
}
});
let node = RemoteNode::new("gate", format!("http://{}", server.addr)).with_retries(0);
let err = Node::run(&node, test_ctx()).await.unwrap_err();
assert!(err.is_interrupt());
let resumed = NodeContext::new(
test_ctx().state().clone(),
NodeConfig {
resume: Some(json!(true)),
..NodeConfig::default()
},
);
let out = Node::run(&node, resumed).await.unwrap();
assert_eq!(out.updates.get("approved"), Some(&json!(true)));
assert_eq!(server.attempts.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn remote_node_retries_on_429_then_succeeds() {
let server = start_mock(|n, _body| {
if n < 2 {
MockBehavior::Respond {
status: 429,
body: r#"{"error":"rate limited"}"#.to_string(),
}
} else {
MockBehavior::Respond {
status: 200,
body: serde_json::to_string(&NodeTaskResponse::ok(NodeOutput::update(
"x",
json!("through"),
)))
.unwrap(),
}
}
});
let node = RemoteNode::new("limited", format!("http://{}", server.addr))
.with_retries(2)
.with_backoff(Duration::from_millis(1));
let out = Node::run(&node, test_ctx()).await.unwrap();
assert_eq!(out.updates.get("x"), Some(&json!("through")));
assert_eq!(server.attempts.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn remote_node_does_not_retry_fatal_4xx() {
let server = start_mock(|_n, _body| MockBehavior::Respond {
status: 400,
body: r#"{"error":"malformed task"}"#.to_string(),
});
let node = RemoteNode::new("bad", format!("http://{}", server.addr))
.with_retries(5)
.with_backoff(Duration::from_millis(1));
let err = Node::run(&node, test_ctx()).await.unwrap_err();
assert!(matches!(err, RustyError::Node(_)));
assert!(err.to_string().contains("400"));
assert_eq!(server.attempts.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn execute_url_is_used_verbatim_when_already_ending_in_execute() {
let node = RemoteNode::new("n", "http://127.0.0.1:1/execute");
assert_eq!(node.execute_url(), "http://127.0.0.1:1/execute");
let node = RemoteNode::new("n", "http://127.0.0.1:1/");
assert_eq!(node.execute_url(), "http://127.0.0.1:1/execute");
}
}