Skip to main content

rusty_agent_runtime/
remote.rs

1//! Remote node execution: the worker wire protocol + [`RemoteNode`].
2//!
3//! Phase B of the distribution story: **one [`Node`] trait, remote impls
4//! behind the same trait**. A [`RemoteNode`] is registered in a graph exactly
5//! like any local node; when the executor runs it, the invocation is
6//! serialized into a [`NodeTask`] and POSTed to a worker's `/execute`
7//! endpoint. The worker replies with a [`NodeTaskResponse`] which is turned
8//! back into a [`NodeOutput`] (or an error / interrupt) locally.
9//!
10//! ## Wire protocol (version 1)
11//!
12//! `POST {base_url}/execute` with a JSON [`NodeTask`]:
13//!
14//! ```json
15//! {
16//!   "protocol_version": 1,
17//!   "node": "doubler",
18//!   "state": { "n": 21 },
19//!   "config": { "thread_id": "t-1", "step": 3, "resume": null, "extra": {} }
20//! }
21//! ```
22//!
23//! The response is a JSON [`NodeTaskResponse`] carrying **exactly one** of:
24//!
25//! - `output`   — the node's partial updates + optional routing command,
26//! - `error`    — a worker-side failure message, surfaced locally as
27//!   [`RustyError::Node`],
28//! - `interrupt`— an interrupt payload, surfaced locally as
29//!   [`RustyError::Interrupt`] so HITL works across the wire
30//!   (a bare `{ "interrupt": <value> }` body is also accepted).
31//!
32//! ## Reliability
33//!
34//! [`RemoteNode`] applies a per-attempt timeout plus configurable retries
35//! with capped, jittered exponential backoff. Only *transport-class*
36//! failures are retried (connect errors, timeouts, HTTP 5xx / 408 / 429; a
37//! `Retry-After` header floors the delay). Worker-reported errors and
38//! interrupts are never retried — the worker already made a definitive
39//! decision, and node logic is only contractually idempotent across
40//! *executor-level* re-execution, not silent client replays.
41
42use std::collections::HashMap;
43use std::time::Duration;
44
45use async_trait::async_trait;
46use serde::{Deserialize, Serialize};
47use serde_json::Value;
48
49use crate::error::{Result, RustyError};
50use crate::llm::{backoff_delay, truncate_body};
51use crate::node::{Command, Node, NodeContext, NodeOutput};
52use crate::state::State;
53
54/// The wire protocol version spoken by this client.
55///
56/// Workers must reject tasks whose `protocol_version` they do not support.
57/// Responses are accepted regardless of their version field so newer workers
58/// can serve older clients (additive-only evolution within v1).
59pub const PROTOCOL_VERSION: u32 = 1;
60
61/// Default per-attempt HTTP timeout.
62pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
63
64/// Default number of retries *after* the initial attempt.
65pub const DEFAULT_MAX_RETRIES: u32 = 2;
66
67/// Default base delay for exponential backoff (attempt *n* waits roughly
68/// `base * 2^n`, exponent capped and jittered, before retry *n+1*).
69pub const DEFAULT_BASE_BACKOFF: Duration = Duration::from_millis(100);
70
71/// A node invocation sent to a worker (`POST {base_url}/execute`).
72///
73/// The same envelope can be embedded as the payload of a durable task
74/// (Durable Work R0.6, `rusty_agent_runtime::durable::TaskEnvelope`) — in
75/// that flow [`NodeTask::task_id`] carries the leased task's identity.
76/// Direct `/execute` invocations built by [`RemoteNode`] always leave it
77/// `None`, so previously written tasks keep deserializing unchanged
78/// (additive-only evolution within v1).
79#[derive(Debug, Clone, Serialize, Deserialize)]
80pub struct NodeTask {
81    /// Protocol version of the sender. See [`PROTOCOL_VERSION`].
82    pub protocol_version: u32,
83
84    /// The handler name the worker should dispatch to. This is the *remote*
85    /// identity of the node (the name registered in the worker's registry),
86    /// which by [`RemoteNode`] convention equals the graph node name.
87    pub node: String,
88
89    /// The immutable super-step state snapshot (same visibility rules as a
90    /// local node: this is the state as of the start of the super-step).
91    pub state: State,
92
93    /// The per-run, per-node configuration (thread id, step, resume value,
94    /// extensions). Carried verbatim so `resume` round-trips across the wire.
95    pub config: crate::node::NodeConfig,
96
97    /// The durable task identity assigned by the server's task queue,
98    /// present only when this invocation rides the durable-task flow (R0.6)
99    /// rather than a direct `/execute` call. It lets a remote node handler
100    /// correlate external side effects with the durable task (idempotency
101    /// keys, receipts) when a run dispatches node work through the queue.
102    #[serde(default, skip_serializing_if = "Option::is_none")]
103    pub task_id: Option<String>,
104}
105
106/// Serializable form of [`NodeOutput`].
107///
108/// [`NodeOutput`] itself is deliberately not `serde`-enabled in the core API;
109/// this wire type converts in both directions losslessly.
110#[derive(Debug, Clone, Default, Serialize, Deserialize)]
111pub struct WireNodeOutput {
112    /// Partial state updates, keyed by channel name.
113    #[serde(default)]
114    pub updates: HashMap<String, Value>,
115
116    /// Optional dynamic routing decision.
117    #[serde(default, skip_serializing_if = "Option::is_none")]
118    pub command: Option<Command>,
119}
120
121impl From<NodeOutput> for WireNodeOutput {
122    fn from(output: NodeOutput) -> Self {
123        Self {
124            updates: output.updates,
125            command: output.command,
126        }
127    }
128}
129
130impl From<WireNodeOutput> for NodeOutput {
131    fn from(wire: WireNodeOutput) -> Self {
132        Self {
133            updates: wire.updates,
134            command: wire.command,
135        }
136    }
137}
138
139/// The worker's reply to a [`NodeTask`].
140///
141/// Exactly one of `output` / `error` / `interrupt` must be set; anything else
142/// is a malformed response. All fields are optional on the wire so a minimal
143/// worker can answer with e.g. `{ "interrupt": <value> }`.
144#[derive(Debug, Clone, Default, Serialize, Deserialize)]
145pub struct NodeTaskResponse {
146    /// Protocol version of the responder. Informational on the client side
147    /// (v1 evolution is additive-only); workers should always set it.
148    #[serde(default)]
149    pub protocol_version: u32,
150
151    /// Success payload: the node's output.
152    #[serde(default, skip_serializing_if = "Option::is_none")]
153    pub output: Option<WireNodeOutput>,
154
155    /// Failure payload: a human-readable worker-side error message.
156    #[serde(default, skip_serializing_if = "Option::is_none")]
157    pub error: Option<String>,
158
159    /// Suspend payload: the node called `interrupt(value)`.
160    #[serde(default, skip_serializing_if = "Option::is_none")]
161    pub interrupt: Option<Value>,
162}
163
164impl NodeTaskResponse {
165    /// A successful response carrying the node's output.
166    pub fn ok(output: impl Into<WireNodeOutput>) -> Self {
167        Self {
168            protocol_version: PROTOCOL_VERSION,
169            output: Some(output.into()),
170            error: None,
171            interrupt: None,
172        }
173    }
174
175    /// A failure response. Surfaced client-side as [`RustyError::Node`].
176    pub fn error(message: impl Into<String>) -> Self {
177        Self {
178            protocol_version: PROTOCOL_VERSION,
179            output: None,
180            error: Some(message.into()),
181            interrupt: None,
182        }
183    }
184
185    /// An interrupt (suspend) response. Surfaced client-side as
186    /// [`RustyError::Interrupt`], so the executor's HITL machinery works
187    /// unchanged for remote nodes.
188    pub fn interrupt(value: Value) -> Self {
189        Self {
190            protocol_version: PROTOCOL_VERSION,
191            output: None,
192            error: None,
193            interrupt: Some(value),
194        }
195    }
196
197    /// Convert into a local node result.
198    ///
199    /// Mapping:
200    ///
201    /// - `output`   → `Ok(NodeOutput)`
202    /// - `error`    → `Err(RustyError::Node(message))`
203    /// - `interrupt`→ `Err(RustyError::Interrupt { value })`
204    ///
205    /// Zero or multiple set payloads are a malformed-response node error.
206    pub fn into_result(self) -> Result<NodeOutput> {
207        let set = self.output.is_some() as u8
208            + self.error.is_some() as u8
209            + self.interrupt.is_some() as u8;
210        if set != 1 {
211            return Err(RustyError::Node(format!(
212                "malformed worker response: expected exactly one of \
213                 output/error/interrupt, found {set} set"
214            )));
215        }
216        if let Some(value) = self.interrupt {
217            return Err(RustyError::Interrupt { value });
218        }
219        if let Some(message) = self.error {
220            return Err(RustyError::Node(message));
221        }
222        // `set == 1` guarantees `output` is the remaining possibility.
223        Ok(self.output.expect("checked exactly-one above").into())
224    }
225}
226
227/// Internal classification of a failed HTTP attempt.
228#[derive(Debug)]
229enum AttemptError {
230    /// Transport-class failure eligible for retry (connect, timeout, 5xx,
231    /// 408, 429), with an optional server-provided `Retry-After` floor.
232    Retryable {
233        error: RustyError,
234        retry_after: Option<Duration>,
235    },
236    /// Definitive failure; never retried (other 4xx, decode errors, ...).
237    Fatal(RustyError),
238}
239
240/// A [`Node`] that executes its work on a remote worker over HTTP.
241///
242/// Registered in a graph exactly like a local node — this is the whole point
243/// of the design: *one `Node` trait, remote impls behind the same trait*.
244///
245/// **Error semantics across the wire.** A worker-side failure arrives as a
246/// plain message string and flattens to [`RustyError::Node`] — a hard,
247/// non-retryable failure. Even when the remote failure originated in the
248/// worker's LLM or tool layer, that retryability does not survive the wire:
249/// the executor's retry classification ([`RustyError::Llm`] /
250/// [`RustyError::Tool`] are the transient classes) only applies to
251/// errors raised by *local* nodes, and [`RustyError::Node`] is never in
252/// it. This client's own retries cover transport-class failures only (see
253/// the module-level reliability notes).
254///
255/// ```no_run
256/// use std::time::Duration;
257/// use rusty_agent_runtime::graph::GraphBuilder;
258/// use rusty_agent_runtime::remote::RemoteNode;
259///
260/// let node = RemoteNode::new("doubler", "http://127.0.0.1:8200")
261///     .with_timeout(Duration::from_secs(5))
262///     .with_retries(2);
263/// let mut builder = GraphBuilder::new();
264/// builder.add_node("double", node);
265/// ```
266#[derive(Clone)]
267pub struct RemoteNode {
268    /// The handler name sent in [`NodeTask::node`].
269    node_name: String,
270    /// Full URL of the worker's execute endpoint (`{base}/execute`).
271    execute_url: String,
272    /// HTTP client (carries the per-attempt timeout).
273    client: reqwest::Client,
274    /// Per-attempt timeout (kept so `with_*` builders can rebuild `client`).
275    timeout: Duration,
276    /// Retries after the initial attempt.
277    max_retries: u32,
278    /// Base delay for exponential backoff.
279    base_backoff: Duration,
280}
281
282impl RemoteNode {
283    /// A remote node dispatched to the worker handler `node_name`, served by
284    /// the worker at `base_url` (e.g. `"http://127.0.0.1:8200"`).
285    ///
286    /// A trailing `/` is trimmed and `/execute` appended. If `base_url`
287    /// already ends in `/execute` it is used verbatim.
288    pub fn new(node_name: impl Into<String>, base_url: impl Into<String>) -> Self {
289        let base = base_url.into();
290        let base = base.trim_end_matches('/');
291        let execute_url = if base.ends_with("/execute") {
292            base.to_owned()
293        } else {
294            format!("{base}/execute")
295        };
296        Self {
297            node_name: node_name.into(),
298            execute_url,
299            client: Self::build_client(DEFAULT_TIMEOUT),
300            timeout: DEFAULT_TIMEOUT,
301            max_retries: DEFAULT_MAX_RETRIES,
302            base_backoff: DEFAULT_BASE_BACKOFF,
303        }
304    }
305
306    /// Override the per-attempt timeout.
307    pub fn with_timeout(mut self, timeout: Duration) -> Self {
308        self.timeout = timeout;
309        self.client = Self::build_client(timeout);
310        self
311    }
312
313    /// Override the number of retries after the initial attempt
314    /// (`0` = single attempt, no retries).
315    pub fn with_retries(mut self, max_retries: u32) -> Self {
316        self.max_retries = max_retries;
317        self
318    }
319
320    /// Override the base backoff delay (attempt *n* waits roughly
321    /// `base * 2^n`, exponent capped and jittered).
322    pub fn with_backoff(mut self, base_backoff: Duration) -> Self {
323        self.base_backoff = base_backoff;
324        self
325    }
326
327    /// The handler name sent to the worker.
328    pub fn node_name(&self) -> &str {
329        &self.node_name
330    }
331
332    /// The full execute-endpoint URL.
333    pub fn execute_url(&self) -> &str {
334        &self.execute_url
335    }
336
337    fn build_client(timeout: Duration) -> reqwest::Client {
338        // Invariant: the builder only sets a timeout — no TLS roots, proxy,
339        // or redirect config — and the rustls backend needs no platform
340        // initialization, so construction cannot realistically fail. Kept
341        // infallible to preserve `RemoteNode::new`'s non-`Result` signature.
342        reqwest::Client::builder()
343            .timeout(timeout)
344            .build()
345            .expect("reqwest client builder with rustls must succeed")
346    }
347
348    /// One HTTP attempt. `Ok` means the worker replied with a well-formed
349    /// [`NodeTaskResponse`] (which may still carry `error`/`interrupt`);
350    /// `Err` is classified for retry. Error bodies are truncated
351    /// ([`crate::llm::truncate_body`]) so a verbose worker cannot bloat logs.
352    async fn try_once(
353        &self,
354        task: &NodeTask,
355    ) -> std::result::Result<NodeTaskResponse, AttemptError> {
356        let response = self
357            .client
358            .post(&self.execute_url)
359            .json(task)
360            .send()
361            .await
362            .map_err(|e| {
363                let err = RustyError::Node(format!(
364                    "remote node `{}`: POST {} failed: {e}",
365                    self.node_name, self.execute_url
366                ));
367                if e.is_timeout() || e.is_connect() {
368                    AttemptError::Retryable {
369                        error: err,
370                        retry_after: None,
371                    }
372                } else {
373                    AttemptError::Fatal(err)
374                }
375            })?;
376
377        let status = response.status();
378        if !status.is_success() {
379            let retry_after = response
380                .headers()
381                .get(reqwest::header::RETRY_AFTER)
382                .and_then(|v| v.to_str().ok())
383                .and_then(|s| s.trim().parse::<u64>().ok())
384                .map(Duration::from_secs);
385            let body = response.text().await.unwrap_or_default();
386            let err = RustyError::Node(format!(
387                "remote node `{}`: worker at {} returned {status}: {}",
388                self.node_name,
389                self.execute_url,
390                truncate_body(&body)
391            ));
392            // 5xx and 408/429 are transient by convention (same policy as
393            // the LLM client); other 4xx are definitive.
394            let retryable = status.is_server_error()
395                || status == reqwest::StatusCode::REQUEST_TIMEOUT
396                || status == reqwest::StatusCode::TOO_MANY_REQUESTS;
397            return Err(if retryable {
398                AttemptError::Retryable {
399                    error: err,
400                    retry_after,
401                }
402            } else {
403                AttemptError::Fatal(err)
404            });
405        }
406
407        response.json::<NodeTaskResponse>().await.map_err(|e| {
408            AttemptError::Fatal(RustyError::Node(format!(
409                "remote node `{}`: could not decode worker response: {e}",
410                self.node_name
411            )))
412        })
413    }
414}
415
416#[async_trait]
417impl Node for RemoteNode {
418    fn name(&self) -> &str {
419        &self.node_name
420    }
421
422    fn effect(&self) -> crate::record::Effect {
423        // A remote invocation crosses a process boundary and performs work
424        // the runtime cannot inspect; the restrictive class applies until
425        // workers declare narrower effects (a worker manifest field is the
426        // R0.6+ mechanism for that).
427        crate::record::Effect::NonIdempotent
428    }
429
430    async fn run(&self, ctx: NodeContext) -> Result<NodeOutput> {
431        let task = NodeTask {
432            protocol_version: PROTOCOL_VERSION,
433            node: self.node_name.clone(),
434            state: ctx.state().clone(),
435            config: ctx.config().clone(),
436            // Direct `/execute` calls are not leased; only the server's task
437            // queue assigns `task_id` in the durable-task flow.
438            task_id: None,
439        };
440
441        let mut attempt: u32 = 0;
442        loop {
443            match self.try_once(&task).await {
444                Ok(response) => return response.into_result(),
445                Err(AttemptError::Fatal(e)) => return Err(e),
446                Err(AttemptError::Retryable { error, retry_after })
447                    if attempt < self.max_retries =>
448                {
449                    // Capped exponent + jitter (crate::llm::backoff_delay):
450                    // uncapped growth turns with_retries(20) into a ~14-hour
451                    // sleep, and lockstep retries stampede a recovering
452                    // worker.
453                    let mut delay = backoff_delay(self.base_backoff, attempt);
454                    if let Some(floor) = retry_after {
455                        // A worker that says Retry-After knows better than
456                        // our backoff guess.
457                        delay = delay.max(floor);
458                    }
459                    tracing::warn!(
460                        node = %self.node_name,
461                        url = %self.execute_url,
462                        attempt = attempt + 1,
463                        max_retries = self.max_retries,
464                        backoff_ms = delay.as_millis() as u64,
465                        error = %error,
466                        "remote node attempt failed; retrying"
467                    );
468                    tokio::time::sleep(delay).await;
469                    attempt += 1;
470                }
471                Err(AttemptError::Retryable { error, .. }) => return Err(error),
472            }
473        }
474    }
475}
476
477#[cfg(test)]
478mod tests {
479    use super::*;
480    use crate::node::NodeConfig;
481    use serde_json::json;
482    use std::net::SocketAddr;
483    use std::sync::atomic::{AtomicUsize, Ordering};
484    use std::sync::Arc;
485    use tokio::io::{AsyncReadExt, AsyncWriteExt};
486    use tokio::net::TcpListener;
487    use tokio::task::JoinHandle;
488
489    // ---------- serde roundtrips ----------
490
491    #[test]
492    fn node_task_serde_roundtrip() {
493        let task = NodeTask {
494            protocol_version: PROTOCOL_VERSION,
495            node: "doubler".into(),
496            state: State::from_value(json!({"n": 21, "xs": [1, 2]})).unwrap(),
497            config: NodeConfig {
498                thread_id: "t-1".into(),
499                step: 3,
500                resume: Some(json!({"approved": true})),
501                extra: HashMap::from([("tag".to_string(), json!("demo"))]),
502            },
503            task_id: Some("task-7".into()),
504        };
505        let json_str = serde_json::to_string(&task).unwrap();
506        let back: NodeTask = serde_json::from_str(&json_str).unwrap();
507        assert_eq!(back.protocol_version, PROTOCOL_VERSION);
508        assert_eq!(back.node, "doubler");
509        assert_eq!(back.state.get("n"), Some(&json!(21)));
510        assert_eq!(back.config.thread_id, "t-1");
511        assert_eq!(back.config.step, 3);
512        assert_eq!(back.config.resume, Some(json!({"approved": true})));
513        assert_eq!(back.config.extra.get("tag"), Some(&json!("demo")));
514        assert_eq!(back.task_id.as_deref(), Some("task-7"));
515    }
516
517    #[test]
518    fn node_task_task_id_is_additive_within_v1() {
519        // Tasks written before `task_id` existed (the direct `/execute`
520        // shape) must keep deserializing...
521        let legacy = r#"{
522            "protocol_version": 1,
523            "node": "doubler",
524            "state": {"n": 21},
525            "config": {"thread_id": "t-1", "step": 3, "resume": null, "extra": {}}
526        }"#;
527        let task: NodeTask = serde_json::from_str(legacy).unwrap();
528        assert_eq!(task.task_id, None);
529
530        // ...and a task with no lease identity must serialize byte-for-byte
531        // into the old shape (no `task_id` key) so old workers accept it.
532        let task = NodeTask {
533            protocol_version: PROTOCOL_VERSION,
534            node: "doubler".into(),
535            state: State::new(),
536            config: NodeConfig::default(),
537            task_id: None,
538        };
539        let value = serde_json::to_value(&task).unwrap();
540        assert!(value.get("task_id").is_none());
541    }
542
543    #[test]
544    fn node_task_response_serde_roundtrips_all_variants() {
545        let ok = NodeTaskResponse::ok(
546            NodeOutput::update("x", json!(1)).with_command(Command::goto("next")),
547        );
548        let back: NodeTaskResponse =
549            serde_json::from_str(&serde_json::to_string(&ok).unwrap()).unwrap();
550        assert_eq!(back.protocol_version, ok.protocol_version);
551        assert_eq!(
552            back.output.as_ref().unwrap().updates,
553            ok.output.as_ref().unwrap().updates
554        );
555        assert_eq!(back.error, ok.error);
556        assert_eq!(back.interrupt, ok.interrupt);
557        assert_eq!(
558            back.output.as_ref().unwrap().command.as_ref().unwrap().goto,
559            vec!["next".to_string()]
560        );
561
562        let err = NodeTaskResponse::error("boom");
563        let back: NodeTaskResponse =
564            serde_json::from_str(&serde_json::to_string(&err).unwrap()).unwrap();
565        assert_eq!(back.error.as_deref(), Some("boom"));
566        assert!(back.output.is_none() && back.interrupt.is_none());
567
568        let interrupt = NodeTaskResponse::interrupt(json!({"question": "approve?"}));
569        let back: NodeTaskResponse =
570            serde_json::from_str(&serde_json::to_string(&interrupt).unwrap()).unwrap();
571        assert_eq!(back.interrupt, Some(json!({"question": "approve?"})));
572        assert!(back.output.is_none() && back.error.is_none());
573    }
574
575    #[test]
576    fn loose_interrupt_shape_parses_and_maps_to_interrupt() {
577        // A minimal worker may reply with just `{ "interrupt": <value> }`.
578        let response: NodeTaskResponse =
579            serde_json::from_str(r#"{"interrupt": {"question": "approve?"}}"#).unwrap();
580        let err = response.into_result().unwrap_err();
581        assert!(err.is_interrupt());
582        assert_eq!(
583            err.interrupt_value(),
584            Some(&json!({"question": "approve?"}))
585        );
586    }
587
588    #[test]
589    fn response_requires_exactly_one_payload() {
590        let none = NodeTaskResponse::default();
591        let err = none.into_result().unwrap_err();
592        assert!(matches!(err, RustyError::Node(_)));
593        assert!(err.to_string().contains("malformed"));
594
595        let mut both = NodeTaskResponse::ok(NodeOutput::empty());
596        both.error = Some("also an error".into());
597        let err = both.into_result().unwrap_err();
598        assert!(matches!(err, RustyError::Node(_)));
599        assert!(err.to_string().contains("malformed"));
600    }
601
602    #[test]
603    fn wire_node_output_converts_losslessly_both_ways() {
604        let output = NodeOutput::update("a", json!(1))
605            .with_update("b", json!([2, 3]))
606            .with_command(Command::goto_many(["n1", "n2"]));
607        let wire = WireNodeOutput::from(output.clone());
608        let back: NodeOutput = wire.clone().into();
609        assert_eq!(back.updates, output.updates);
610        assert_eq!(
611            back.command.as_ref().unwrap().goto,
612            output.command.as_ref().unwrap().goto
613        );
614        // And through JSON.
615        let wire2: WireNodeOutput =
616            serde_json::from_str(&serde_json::to_string(&wire).unwrap()).unwrap();
617        assert_eq!(wire2.updates, wire.updates);
618        assert_eq!(
619            wire2.command.as_ref().unwrap().goto,
620            wire.command.as_ref().unwrap().goto
621        );
622    }
623
624    // ---------- hand-rolled mock HTTP server (no extra deps) ----------
625
626    /// What the mock should do with a request.
627    enum MockBehavior {
628        /// Reply with a status + JSON body.
629        Respond { status: u16, body: String },
630        /// Never reply (the connection just hangs) — exercises timeouts.
631        Hang,
632    }
633
634    struct MockServer {
635        addr: SocketAddr,
636        attempts: Arc<AtomicUsize>,
637        bodies: Arc<std::sync::Mutex<Vec<String>>>,
638        _handle: JoinHandle<()>,
639    }
640
641    /// Start a mock HTTP/1.1 server on an ephemeral port. `handler` receives
642    /// the 1-based attempt number and the request body.
643    fn start_mock<F>(handler: F) -> MockServer
644    where
645        F: Fn(usize, String) -> MockBehavior + Send + Sync + 'static,
646    {
647        let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
648        listener.set_nonblocking(true).unwrap();
649        let addr = listener.local_addr().unwrap();
650        let attempts = Arc::new(AtomicUsize::new(0));
651        let bodies: Arc<std::sync::Mutex<Vec<String>>> =
652            Arc::new(std::sync::Mutex::new(Vec::new()));
653
654        let attempts2 = attempts.clone();
655        let bodies2 = bodies.clone();
656        let handler = Arc::new(handler);
657        let handle = tokio::spawn(async move {
658            let listener = TcpListener::from_std(listener).unwrap();
659            loop {
660                let Ok((mut stream, _)) = listener.accept().await else {
661                    continue;
662                };
663                let attempts = attempts2.clone();
664                let bodies = bodies2.clone();
665                let handler = handler.clone();
666                tokio::spawn(async move {
667                    let Some(body) = read_http_request(&mut stream).await else {
668                        return; // malformed request: just drop the connection
669                    };
670                    let n = attempts.fetch_add(1, Ordering::SeqCst) + 1;
671                    bodies.lock().unwrap().push(body.clone());
672                    match handler(n, body) {
673                        MockBehavior::Hang => {
674                            // Hold the connection open forever (until dropped).
675                            std::future::pending::<()>().await;
676                        }
677                        MockBehavior::Respond { status, body } => {
678                            let reason = match status {
679                                200 => "OK",
680                                400 => "Bad Request",
681                                500 => "Internal Server Error",
682                                _ => "Status",
683                            };
684                            let response = format!(
685                                "HTTP/1.1 {status} {reason}\r\n\
686                                 content-type: application/json\r\n\
687                                 content-length: {}\r\n\
688                                 connection: close\r\n\
689                                 \r\n\
690                                 {body}",
691                                body.len()
692                            );
693                            let _ = stream.write_all(response.as_bytes()).await;
694                            let _ = stream.flush().await;
695                        }
696                    }
697                });
698            }
699        });
700        MockServer {
701            addr,
702            attempts,
703            bodies,
704            _handle: handle,
705        }
706    }
707
708    /// Minimal HTTP/1.1 request reader: headers up to `\r\n\r\n`, then
709    /// `content-length` bytes of body. Returns the body as a string.
710    async fn read_http_request(stream: &mut tokio::net::TcpStream) -> Option<String> {
711        let mut buf: Vec<u8> = Vec::new();
712        let mut chunk = [0u8; 4096];
713        // Read until end of headers.
714        let header_end = loop {
715            if let Some(pos) = find_subslice(&buf, b"\r\n\r\n") {
716                break pos + 4;
717            }
718            let n = stream.read(&mut chunk).await.ok()?;
719            if n == 0 {
720                return None;
721            }
722            buf.extend_from_slice(&chunk[..n]);
723        };
724        let headers = String::from_utf8_lossy(&buf[..header_end]).to_lowercase();
725        let content_length: usize = headers
726            .lines()
727            .find_map(|line| line.strip_prefix("content-length:"))
728            .and_then(|v| v.trim().parse().ok())
729            .unwrap_or(0);
730        while buf.len() < header_end + content_length {
731            let n = stream.read(&mut chunk).await.ok()?;
732            if n == 0 {
733                break;
734            }
735            buf.extend_from_slice(&chunk[..n]);
736        }
737        String::from_utf8(buf[header_end..header_end + content_length].to_vec()).ok()
738    }
739
740    fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option<usize> {
741        haystack
742            .windows(needle.len())
743            .position(|window| window == needle)
744    }
745
746    fn test_ctx() -> NodeContext {
747        NodeContext::new(
748            State::from_value(json!({"n": 21})).unwrap(),
749            NodeConfig {
750                thread_id: "t-test".into(),
751                step: 2,
752                resume: None,
753                extra: HashMap::new(),
754            },
755        )
756    }
757
758    // ---------- RemoteNode over HTTP ----------
759
760    #[tokio::test]
761    async fn remote_node_success_over_http() {
762        let server = start_mock(|_n, _body| MockBehavior::Respond {
763            status: 200,
764            body: serde_json::to_string(&NodeTaskResponse::ok(
765                NodeOutput::update("doubled", json!(42)).with_command(Command::goto("final")),
766            ))
767            .unwrap(),
768        });
769
770        let node = RemoteNode::new("doubler", format!("http://{}", server.addr)).with_retries(0);
771        let out = Node::run(&node, test_ctx()).await.unwrap();
772        assert_eq!(out.updates.get("doubled"), Some(&json!(42)));
773        assert_eq!(
774            out.command.as_ref().unwrap().goto,
775            vec!["final".to_string()]
776        );
777        assert_eq!(node.name(), "doubler");
778        assert!(node.execute_url().ends_with("/execute"));
779        assert_eq!(server.attempts.load(Ordering::SeqCst), 1);
780
781        // The request carried a well-formed NodeTask.
782        let sent = server.bodies.lock().unwrap()[0].clone();
783        let task: NodeTask = serde_json::from_str(&sent).unwrap();
784        assert_eq!(task.protocol_version, PROTOCOL_VERSION);
785        assert_eq!(task.node, "doubler");
786        assert_eq!(task.state.get("n"), Some(&json!(21)));
787        assert_eq!(task.config.thread_id, "t-test");
788        assert_eq!(task.config.step, 2);
789    }
790
791    #[tokio::test]
792    async fn remote_node_retries_on_5xx_then_succeeds() {
793        let server = start_mock(|n, _body| {
794            if n < 3 {
795                MockBehavior::Respond {
796                    status: 500,
797                    body: r#"{"error":"worker overloaded"}"#.to_string(),
798                }
799            } else {
800                MockBehavior::Respond {
801                    status: 200,
802                    body: serde_json::to_string(&NodeTaskResponse::ok(NodeOutput::update(
803                        "x",
804                        json!("third time lucky"),
805                    )))
806                    .unwrap(),
807                }
808            }
809        });
810
811        let node = RemoteNode::new("flaky", format!("http://{}", server.addr))
812            .with_retries(5)
813            .with_backoff(Duration::from_millis(1));
814        let out = Node::run(&node, test_ctx()).await.unwrap();
815        assert_eq!(out.updates.get("x"), Some(&json!("third time lucky")));
816        assert_eq!(server.attempts.load(Ordering::SeqCst), 3);
817    }
818
819    #[tokio::test]
820    async fn remote_node_gives_up_after_retries_exhausted() {
821        let server = start_mock(|_n, _body| MockBehavior::Respond {
822            status: 500,
823            body: r#"{"error":"always down"}"#.to_string(),
824        });
825
826        let node = RemoteNode::new("down", format!("http://{}", server.addr))
827            .with_retries(2)
828            .with_backoff(Duration::from_millis(1));
829        let err = Node::run(&node, test_ctx()).await.unwrap_err();
830        assert!(matches!(err, RustyError::Node(_)));
831        assert!(err.to_string().contains("500"));
832        assert!(!err.is_interrupt());
833        // 1 initial + 2 retries = 3 attempts.
834        assert_eq!(server.attempts.load(Ordering::SeqCst), 3);
835    }
836
837    #[tokio::test]
838    async fn remote_node_timeout_is_retryable_and_bounded() {
839        let server = start_mock(|_n, _body| MockBehavior::Hang);
840
841        let node = RemoteNode::new("slow", format!("http://{}", server.addr))
842            .with_timeout(Duration::from_millis(100))
843            .with_retries(1)
844            .with_backoff(Duration::from_millis(1));
845        let started = std::time::Instant::now();
846        let err = Node::run(&node, test_ctx()).await.unwrap_err();
847        let elapsed = started.elapsed();
848
849        assert!(matches!(err, RustyError::Node(_)));
850        // 1 initial + 1 retry = 2 attempts.
851        assert_eq!(server.attempts.load(Ordering::SeqCst), 2);
852        // Two 100ms timeouts + backoff: must be far below the default 30s
853        // timeout, and comfortably below 5s of wall clock.
854        assert!(elapsed < Duration::from_secs(5), "took {elapsed:?}");
855    }
856
857    #[tokio::test]
858    async fn remote_node_worker_error_maps_to_node_error_without_retry() {
859        let server = start_mock(|_n, _body| MockBehavior::Respond {
860            status: 200,
861            body: serde_json::to_string(&NodeTaskResponse::error(
862                "handler `doubler` panicked: divide by zero",
863            ))
864            .unwrap(),
865        });
866
867        let node = RemoteNode::new("doubler", format!("http://{}", server.addr))
868            .with_retries(5)
869            .with_backoff(Duration::from_millis(1));
870        let err = Node::run(&node, test_ctx()).await.unwrap_err();
871        match err {
872            RustyError::Node(message) => {
873                assert!(message.contains("divide by zero"), "got: {message}")
874            }
875            other => panic!("expected Node error, got {other:?}"),
876        }
877        // Worker errors are definitive: no retries despite with_retries(5).
878        assert_eq!(server.attempts.load(Ordering::SeqCst), 1);
879    }
880
881    #[tokio::test]
882    async fn remote_node_interrupt_passes_through_without_retry() {
883        let server = start_mock(|_n, _body| MockBehavior::Respond {
884            status: 200,
885            body: r#"{"interrupt": {"question": "approve deployment?"}}"#.to_string(),
886        });
887
888        let node = RemoteNode::new("gate", format!("http://{}", server.addr))
889            .with_retries(5)
890            .with_backoff(Duration::from_millis(1));
891        let err = Node::run(&node, test_ctx()).await.unwrap_err();
892        assert!(err.is_interrupt());
893        assert_eq!(
894            err.interrupt_value(),
895            Some(&json!({"question": "approve deployment?"}))
896        );
897        // Interrupts are control flow, not failures: never retried.
898        assert_eq!(server.attempts.load(Ordering::SeqCst), 1);
899    }
900
901    #[tokio::test]
902    async fn remote_node_resume_value_crosses_the_wire() {
903        let server = start_mock(|_n, body| {
904            let task: NodeTask = serde_json::from_str(&body).unwrap();
905            // Echo the resume value back like a resumable worker handler would.
906            let response = match task.config.resume {
907                Some(v) => NodeTaskResponse::ok(NodeOutput::update("approved", v)),
908                None => NodeTaskResponse::interrupt(json!({"question": "approve?"})),
909            };
910            MockBehavior::Respond {
911                status: 200,
912                body: serde_json::to_string(&response).unwrap(),
913            }
914        });
915
916        let node = RemoteNode::new("gate", format!("http://{}", server.addr)).with_retries(0);
917
918        // First invocation: no resume value → interrupt.
919        let err = Node::run(&node, test_ctx()).await.unwrap_err();
920        assert!(err.is_interrupt());
921
922        // Second invocation: resume value in NodeConfig → success echoing it.
923        let resumed = NodeContext::new(
924            test_ctx().state().clone(),
925            NodeConfig {
926                resume: Some(json!(true)),
927                ..NodeConfig::default()
928            },
929        );
930        let out = Node::run(&node, resumed).await.unwrap();
931        assert_eq!(out.updates.get("approved"), Some(&json!(true)));
932        assert_eq!(server.attempts.load(Ordering::SeqCst), 2);
933    }
934
935    #[tokio::test]
936    async fn remote_node_retries_on_429_then_succeeds() {
937        let server = start_mock(|n, _body| {
938            if n < 2 {
939                MockBehavior::Respond {
940                    status: 429,
941                    body: r#"{"error":"rate limited"}"#.to_string(),
942                }
943            } else {
944                MockBehavior::Respond {
945                    status: 200,
946                    body: serde_json::to_string(&NodeTaskResponse::ok(NodeOutput::update(
947                        "x",
948                        json!("through"),
949                    )))
950                    .unwrap(),
951                }
952            }
953        });
954
955        let node = RemoteNode::new("limited", format!("http://{}", server.addr))
956            .with_retries(2)
957            .with_backoff(Duration::from_millis(1));
958        let out = Node::run(&node, test_ctx()).await.unwrap();
959        assert_eq!(out.updates.get("x"), Some(&json!("through")));
960        assert_eq!(server.attempts.load(Ordering::SeqCst), 2);
961    }
962
963    #[tokio::test]
964    async fn remote_node_does_not_retry_fatal_4xx() {
965        let server = start_mock(|_n, _body| MockBehavior::Respond {
966            status: 400,
967            body: r#"{"error":"malformed task"}"#.to_string(),
968        });
969
970        let node = RemoteNode::new("bad", format!("http://{}", server.addr))
971            .with_retries(5)
972            .with_backoff(Duration::from_millis(1));
973        let err = Node::run(&node, test_ctx()).await.unwrap_err();
974        assert!(matches!(err, RustyError::Node(_)));
975        assert!(err.to_string().contains("400"));
976        // Definitive failure: no retries despite with_retries(5).
977        assert_eq!(server.attempts.load(Ordering::SeqCst), 1);
978    }
979
980    #[tokio::test]
981    async fn execute_url_is_used_verbatim_when_already_ending_in_execute() {
982        let node = RemoteNode::new("n", "http://127.0.0.1:1/execute");
983        assert_eq!(node.execute_url(), "http://127.0.0.1:1/execute");
984        let node = RemoteNode::new("n", "http://127.0.0.1:1/");
985        assert_eq!(node.execute_url(), "http://127.0.0.1:1/execute");
986    }
987}