1use std::collections::HashMap;
43use std::time::Duration;
44
45use async_trait::async_trait;
46use serde::{Deserialize, Serialize};
47use serde_json::Value;
48
49use crate::error::{Result, RustyError};
50use crate::llm::{backoff_delay, truncate_body};
51use crate::node::{Command, Node, NodeContext, NodeOutput};
52use crate::state::State;
53
54pub const PROTOCOL_VERSION: u32 = 1;
60
61pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
63
64pub const DEFAULT_MAX_RETRIES: u32 = 2;
66
67pub const DEFAULT_BASE_BACKOFF: Duration = Duration::from_millis(100);
70
71#[derive(Debug, Clone, Serialize, Deserialize)]
80pub struct NodeTask {
81 pub protocol_version: u32,
83
84 pub node: String,
88
89 pub state: State,
92
93 pub config: crate::node::NodeConfig,
96
97 #[serde(default, skip_serializing_if = "Option::is_none")]
103 pub task_id: Option<String>,
104}
105
106#[derive(Debug, Clone, Default, Serialize, Deserialize)]
111pub struct WireNodeOutput {
112 #[serde(default)]
114 pub updates: HashMap<String, Value>,
115
116 #[serde(default, skip_serializing_if = "Option::is_none")]
118 pub command: Option<Command>,
119}
120
121impl From<NodeOutput> for WireNodeOutput {
122 fn from(output: NodeOutput) -> Self {
123 Self {
124 updates: output.updates,
125 command: output.command,
126 }
127 }
128}
129
130impl From<WireNodeOutput> for NodeOutput {
131 fn from(wire: WireNodeOutput) -> Self {
132 Self {
133 updates: wire.updates,
134 command: wire.command,
135 }
136 }
137}
138
139#[derive(Debug, Clone, Default, Serialize, Deserialize)]
145pub struct NodeTaskResponse {
146 #[serde(default)]
149 pub protocol_version: u32,
150
151 #[serde(default, skip_serializing_if = "Option::is_none")]
153 pub output: Option<WireNodeOutput>,
154
155 #[serde(default, skip_serializing_if = "Option::is_none")]
157 pub error: Option<String>,
158
159 #[serde(default, skip_serializing_if = "Option::is_none")]
161 pub interrupt: Option<Value>,
162}
163
164impl NodeTaskResponse {
165 pub fn ok(output: impl Into<WireNodeOutput>) -> Self {
167 Self {
168 protocol_version: PROTOCOL_VERSION,
169 output: Some(output.into()),
170 error: None,
171 interrupt: None,
172 }
173 }
174
175 pub fn error(message: impl Into<String>) -> Self {
177 Self {
178 protocol_version: PROTOCOL_VERSION,
179 output: None,
180 error: Some(message.into()),
181 interrupt: None,
182 }
183 }
184
185 pub fn interrupt(value: Value) -> Self {
189 Self {
190 protocol_version: PROTOCOL_VERSION,
191 output: None,
192 error: None,
193 interrupt: Some(value),
194 }
195 }
196
197 pub fn into_result(self) -> Result<NodeOutput> {
207 let set = self.output.is_some() as u8
208 + self.error.is_some() as u8
209 + self.interrupt.is_some() as u8;
210 if set != 1 {
211 return Err(RustyError::Node(format!(
212 "malformed worker response: expected exactly one of \
213 output/error/interrupt, found {set} set"
214 )));
215 }
216 if let Some(value) = self.interrupt {
217 return Err(RustyError::Interrupt { value });
218 }
219 if let Some(message) = self.error {
220 return Err(RustyError::Node(message));
221 }
222 Ok(self.output.expect("checked exactly-one above").into())
224 }
225}
226
227#[derive(Debug)]
229enum AttemptError {
230 Retryable {
233 error: RustyError,
234 retry_after: Option<Duration>,
235 },
236 Fatal(RustyError),
238}
239
240#[derive(Clone)]
267pub struct RemoteNode {
268 node_name: String,
270 execute_url: String,
272 client: reqwest::Client,
274 timeout: Duration,
276 max_retries: u32,
278 base_backoff: Duration,
280}
281
282impl RemoteNode {
283 pub fn new(node_name: impl Into<String>, base_url: impl Into<String>) -> Self {
289 let base = base_url.into();
290 let base = base.trim_end_matches('/');
291 let execute_url = if base.ends_with("/execute") {
292 base.to_owned()
293 } else {
294 format!("{base}/execute")
295 };
296 Self {
297 node_name: node_name.into(),
298 execute_url,
299 client: Self::build_client(DEFAULT_TIMEOUT),
300 timeout: DEFAULT_TIMEOUT,
301 max_retries: DEFAULT_MAX_RETRIES,
302 base_backoff: DEFAULT_BASE_BACKOFF,
303 }
304 }
305
306 pub fn with_timeout(mut self, timeout: Duration) -> Self {
308 self.timeout = timeout;
309 self.client = Self::build_client(timeout);
310 self
311 }
312
313 pub fn with_retries(mut self, max_retries: u32) -> Self {
316 self.max_retries = max_retries;
317 self
318 }
319
320 pub fn with_backoff(mut self, base_backoff: Duration) -> Self {
323 self.base_backoff = base_backoff;
324 self
325 }
326
327 pub fn node_name(&self) -> &str {
329 &self.node_name
330 }
331
332 pub fn execute_url(&self) -> &str {
334 &self.execute_url
335 }
336
337 fn build_client(timeout: Duration) -> reqwest::Client {
338 reqwest::Client::builder()
343 .timeout(timeout)
344 .build()
345 .expect("reqwest client builder with rustls must succeed")
346 }
347
348 async fn try_once(
353 &self,
354 task: &NodeTask,
355 ) -> std::result::Result<NodeTaskResponse, AttemptError> {
356 let response = self
357 .client
358 .post(&self.execute_url)
359 .json(task)
360 .send()
361 .await
362 .map_err(|e| {
363 let err = RustyError::Node(format!(
364 "remote node `{}`: POST {} failed: {e}",
365 self.node_name, self.execute_url
366 ));
367 if e.is_timeout() || e.is_connect() {
368 AttemptError::Retryable {
369 error: err,
370 retry_after: None,
371 }
372 } else {
373 AttemptError::Fatal(err)
374 }
375 })?;
376
377 let status = response.status();
378 if !status.is_success() {
379 let retry_after = response
380 .headers()
381 .get(reqwest::header::RETRY_AFTER)
382 .and_then(|v| v.to_str().ok())
383 .and_then(|s| s.trim().parse::<u64>().ok())
384 .map(Duration::from_secs);
385 let body = response.text().await.unwrap_or_default();
386 let err = RustyError::Node(format!(
387 "remote node `{}`: worker at {} returned {status}: {}",
388 self.node_name,
389 self.execute_url,
390 truncate_body(&body)
391 ));
392 let retryable = status.is_server_error()
395 || status == reqwest::StatusCode::REQUEST_TIMEOUT
396 || status == reqwest::StatusCode::TOO_MANY_REQUESTS;
397 return Err(if retryable {
398 AttemptError::Retryable {
399 error: err,
400 retry_after,
401 }
402 } else {
403 AttemptError::Fatal(err)
404 });
405 }
406
407 response.json::<NodeTaskResponse>().await.map_err(|e| {
408 AttemptError::Fatal(RustyError::Node(format!(
409 "remote node `{}`: could not decode worker response: {e}",
410 self.node_name
411 )))
412 })
413 }
414}
415
416#[async_trait]
417impl Node for RemoteNode {
418 fn name(&self) -> &str {
419 &self.node_name
420 }
421
422 fn effect(&self) -> crate::record::Effect {
423 crate::record::Effect::NonIdempotent
428 }
429
430 async fn run(&self, ctx: NodeContext) -> Result<NodeOutput> {
431 let task = NodeTask {
432 protocol_version: PROTOCOL_VERSION,
433 node: self.node_name.clone(),
434 state: ctx.state().clone(),
435 config: ctx.config().clone(),
436 task_id: None,
439 };
440
441 let mut attempt: u32 = 0;
442 loop {
443 match self.try_once(&task).await {
444 Ok(response) => return response.into_result(),
445 Err(AttemptError::Fatal(e)) => return Err(e),
446 Err(AttemptError::Retryable { error, retry_after })
447 if attempt < self.max_retries =>
448 {
449 let mut delay = backoff_delay(self.base_backoff, attempt);
454 if let Some(floor) = retry_after {
455 delay = delay.max(floor);
458 }
459 tracing::warn!(
460 node = %self.node_name,
461 url = %self.execute_url,
462 attempt = attempt + 1,
463 max_retries = self.max_retries,
464 backoff_ms = delay.as_millis() as u64,
465 error = %error,
466 "remote node attempt failed; retrying"
467 );
468 tokio::time::sleep(delay).await;
469 attempt += 1;
470 }
471 Err(AttemptError::Retryable { error, .. }) => return Err(error),
472 }
473 }
474 }
475}
476
477#[cfg(test)]
478mod tests {
479 use super::*;
480 use crate::node::NodeConfig;
481 use serde_json::json;
482 use std::net::SocketAddr;
483 use std::sync::atomic::{AtomicUsize, Ordering};
484 use std::sync::Arc;
485 use tokio::io::{AsyncReadExt, AsyncWriteExt};
486 use tokio::net::TcpListener;
487 use tokio::task::JoinHandle;
488
489 #[test]
492 fn node_task_serde_roundtrip() {
493 let task = NodeTask {
494 protocol_version: PROTOCOL_VERSION,
495 node: "doubler".into(),
496 state: State::from_value(json!({"n": 21, "xs": [1, 2]})).unwrap(),
497 config: NodeConfig {
498 thread_id: "t-1".into(),
499 step: 3,
500 resume: Some(json!({"approved": true})),
501 extra: HashMap::from([("tag".to_string(), json!("demo"))]),
502 },
503 task_id: Some("task-7".into()),
504 };
505 let json_str = serde_json::to_string(&task).unwrap();
506 let back: NodeTask = serde_json::from_str(&json_str).unwrap();
507 assert_eq!(back.protocol_version, PROTOCOL_VERSION);
508 assert_eq!(back.node, "doubler");
509 assert_eq!(back.state.get("n"), Some(&json!(21)));
510 assert_eq!(back.config.thread_id, "t-1");
511 assert_eq!(back.config.step, 3);
512 assert_eq!(back.config.resume, Some(json!({"approved": true})));
513 assert_eq!(back.config.extra.get("tag"), Some(&json!("demo")));
514 assert_eq!(back.task_id.as_deref(), Some("task-7"));
515 }
516
517 #[test]
518 fn node_task_task_id_is_additive_within_v1() {
519 let legacy = r#"{
522 "protocol_version": 1,
523 "node": "doubler",
524 "state": {"n": 21},
525 "config": {"thread_id": "t-1", "step": 3, "resume": null, "extra": {}}
526 }"#;
527 let task: NodeTask = serde_json::from_str(legacy).unwrap();
528 assert_eq!(task.task_id, None);
529
530 let task = NodeTask {
533 protocol_version: PROTOCOL_VERSION,
534 node: "doubler".into(),
535 state: State::new(),
536 config: NodeConfig::default(),
537 task_id: None,
538 };
539 let value = serde_json::to_value(&task).unwrap();
540 assert!(value.get("task_id").is_none());
541 }
542
543 #[test]
544 fn node_task_response_serde_roundtrips_all_variants() {
545 let ok = NodeTaskResponse::ok(
546 NodeOutput::update("x", json!(1)).with_command(Command::goto("next")),
547 );
548 let back: NodeTaskResponse =
549 serde_json::from_str(&serde_json::to_string(&ok).unwrap()).unwrap();
550 assert_eq!(back.protocol_version, ok.protocol_version);
551 assert_eq!(
552 back.output.as_ref().unwrap().updates,
553 ok.output.as_ref().unwrap().updates
554 );
555 assert_eq!(back.error, ok.error);
556 assert_eq!(back.interrupt, ok.interrupt);
557 assert_eq!(
558 back.output.as_ref().unwrap().command.as_ref().unwrap().goto,
559 vec!["next".to_string()]
560 );
561
562 let err = NodeTaskResponse::error("boom");
563 let back: NodeTaskResponse =
564 serde_json::from_str(&serde_json::to_string(&err).unwrap()).unwrap();
565 assert_eq!(back.error.as_deref(), Some("boom"));
566 assert!(back.output.is_none() && back.interrupt.is_none());
567
568 let interrupt = NodeTaskResponse::interrupt(json!({"question": "approve?"}));
569 let back: NodeTaskResponse =
570 serde_json::from_str(&serde_json::to_string(&interrupt).unwrap()).unwrap();
571 assert_eq!(back.interrupt, Some(json!({"question": "approve?"})));
572 assert!(back.output.is_none() && back.error.is_none());
573 }
574
575 #[test]
576 fn loose_interrupt_shape_parses_and_maps_to_interrupt() {
577 let response: NodeTaskResponse =
579 serde_json::from_str(r#"{"interrupt": {"question": "approve?"}}"#).unwrap();
580 let err = response.into_result().unwrap_err();
581 assert!(err.is_interrupt());
582 assert_eq!(
583 err.interrupt_value(),
584 Some(&json!({"question": "approve?"}))
585 );
586 }
587
588 #[test]
589 fn response_requires_exactly_one_payload() {
590 let none = NodeTaskResponse::default();
591 let err = none.into_result().unwrap_err();
592 assert!(matches!(err, RustyError::Node(_)));
593 assert!(err.to_string().contains("malformed"));
594
595 let mut both = NodeTaskResponse::ok(NodeOutput::empty());
596 both.error = Some("also an error".into());
597 let err = both.into_result().unwrap_err();
598 assert!(matches!(err, RustyError::Node(_)));
599 assert!(err.to_string().contains("malformed"));
600 }
601
602 #[test]
603 fn wire_node_output_converts_losslessly_both_ways() {
604 let output = NodeOutput::update("a", json!(1))
605 .with_update("b", json!([2, 3]))
606 .with_command(Command::goto_many(["n1", "n2"]));
607 let wire = WireNodeOutput::from(output.clone());
608 let back: NodeOutput = wire.clone().into();
609 assert_eq!(back.updates, output.updates);
610 assert_eq!(
611 back.command.as_ref().unwrap().goto,
612 output.command.as_ref().unwrap().goto
613 );
614 let wire2: WireNodeOutput =
616 serde_json::from_str(&serde_json::to_string(&wire).unwrap()).unwrap();
617 assert_eq!(wire2.updates, wire.updates);
618 assert_eq!(
619 wire2.command.as_ref().unwrap().goto,
620 wire.command.as_ref().unwrap().goto
621 );
622 }
623
624 enum MockBehavior {
628 Respond { status: u16, body: String },
630 Hang,
632 }
633
634 struct MockServer {
635 addr: SocketAddr,
636 attempts: Arc<AtomicUsize>,
637 bodies: Arc<std::sync::Mutex<Vec<String>>>,
638 _handle: JoinHandle<()>,
639 }
640
641 fn start_mock<F>(handler: F) -> MockServer
644 where
645 F: Fn(usize, String) -> MockBehavior + Send + Sync + 'static,
646 {
647 let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
648 listener.set_nonblocking(true).unwrap();
649 let addr = listener.local_addr().unwrap();
650 let attempts = Arc::new(AtomicUsize::new(0));
651 let bodies: Arc<std::sync::Mutex<Vec<String>>> =
652 Arc::new(std::sync::Mutex::new(Vec::new()));
653
654 let attempts2 = attempts.clone();
655 let bodies2 = bodies.clone();
656 let handler = Arc::new(handler);
657 let handle = tokio::spawn(async move {
658 let listener = TcpListener::from_std(listener).unwrap();
659 loop {
660 let Ok((mut stream, _)) = listener.accept().await else {
661 continue;
662 };
663 let attempts = attempts2.clone();
664 let bodies = bodies2.clone();
665 let handler = handler.clone();
666 tokio::spawn(async move {
667 let Some(body) = read_http_request(&mut stream).await else {
668 return; };
670 let n = attempts.fetch_add(1, Ordering::SeqCst) + 1;
671 bodies.lock().unwrap().push(body.clone());
672 match handler(n, body) {
673 MockBehavior::Hang => {
674 std::future::pending::<()>().await;
676 }
677 MockBehavior::Respond { status, body } => {
678 let reason = match status {
679 200 => "OK",
680 400 => "Bad Request",
681 500 => "Internal Server Error",
682 _ => "Status",
683 };
684 let response = format!(
685 "HTTP/1.1 {status} {reason}\r\n\
686 content-type: application/json\r\n\
687 content-length: {}\r\n\
688 connection: close\r\n\
689 \r\n\
690 {body}",
691 body.len()
692 );
693 let _ = stream.write_all(response.as_bytes()).await;
694 let _ = stream.flush().await;
695 }
696 }
697 });
698 }
699 });
700 MockServer {
701 addr,
702 attempts,
703 bodies,
704 _handle: handle,
705 }
706 }
707
708 async fn read_http_request(stream: &mut tokio::net::TcpStream) -> Option<String> {
711 let mut buf: Vec<u8> = Vec::new();
712 let mut chunk = [0u8; 4096];
713 let header_end = loop {
715 if let Some(pos) = find_subslice(&buf, b"\r\n\r\n") {
716 break pos + 4;
717 }
718 let n = stream.read(&mut chunk).await.ok()?;
719 if n == 0 {
720 return None;
721 }
722 buf.extend_from_slice(&chunk[..n]);
723 };
724 let headers = String::from_utf8_lossy(&buf[..header_end]).to_lowercase();
725 let content_length: usize = headers
726 .lines()
727 .find_map(|line| line.strip_prefix("content-length:"))
728 .and_then(|v| v.trim().parse().ok())
729 .unwrap_or(0);
730 while buf.len() < header_end + content_length {
731 let n = stream.read(&mut chunk).await.ok()?;
732 if n == 0 {
733 break;
734 }
735 buf.extend_from_slice(&chunk[..n]);
736 }
737 String::from_utf8(buf[header_end..header_end + content_length].to_vec()).ok()
738 }
739
740 fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option<usize> {
741 haystack
742 .windows(needle.len())
743 .position(|window| window == needle)
744 }
745
746 fn test_ctx() -> NodeContext {
747 NodeContext::new(
748 State::from_value(json!({"n": 21})).unwrap(),
749 NodeConfig {
750 thread_id: "t-test".into(),
751 step: 2,
752 resume: None,
753 extra: HashMap::new(),
754 },
755 )
756 }
757
758 #[tokio::test]
761 async fn remote_node_success_over_http() {
762 let server = start_mock(|_n, _body| MockBehavior::Respond {
763 status: 200,
764 body: serde_json::to_string(&NodeTaskResponse::ok(
765 NodeOutput::update("doubled", json!(42)).with_command(Command::goto("final")),
766 ))
767 .unwrap(),
768 });
769
770 let node = RemoteNode::new("doubler", format!("http://{}", server.addr)).with_retries(0);
771 let out = Node::run(&node, test_ctx()).await.unwrap();
772 assert_eq!(out.updates.get("doubled"), Some(&json!(42)));
773 assert_eq!(
774 out.command.as_ref().unwrap().goto,
775 vec!["final".to_string()]
776 );
777 assert_eq!(node.name(), "doubler");
778 assert!(node.execute_url().ends_with("/execute"));
779 assert_eq!(server.attempts.load(Ordering::SeqCst), 1);
780
781 let sent = server.bodies.lock().unwrap()[0].clone();
783 let task: NodeTask = serde_json::from_str(&sent).unwrap();
784 assert_eq!(task.protocol_version, PROTOCOL_VERSION);
785 assert_eq!(task.node, "doubler");
786 assert_eq!(task.state.get("n"), Some(&json!(21)));
787 assert_eq!(task.config.thread_id, "t-test");
788 assert_eq!(task.config.step, 2);
789 }
790
791 #[tokio::test]
792 async fn remote_node_retries_on_5xx_then_succeeds() {
793 let server = start_mock(|n, _body| {
794 if n < 3 {
795 MockBehavior::Respond {
796 status: 500,
797 body: r#"{"error":"worker overloaded"}"#.to_string(),
798 }
799 } else {
800 MockBehavior::Respond {
801 status: 200,
802 body: serde_json::to_string(&NodeTaskResponse::ok(NodeOutput::update(
803 "x",
804 json!("third time lucky"),
805 )))
806 .unwrap(),
807 }
808 }
809 });
810
811 let node = RemoteNode::new("flaky", format!("http://{}", server.addr))
812 .with_retries(5)
813 .with_backoff(Duration::from_millis(1));
814 let out = Node::run(&node, test_ctx()).await.unwrap();
815 assert_eq!(out.updates.get("x"), Some(&json!("third time lucky")));
816 assert_eq!(server.attempts.load(Ordering::SeqCst), 3);
817 }
818
819 #[tokio::test]
820 async fn remote_node_gives_up_after_retries_exhausted() {
821 let server = start_mock(|_n, _body| MockBehavior::Respond {
822 status: 500,
823 body: r#"{"error":"always down"}"#.to_string(),
824 });
825
826 let node = RemoteNode::new("down", format!("http://{}", server.addr))
827 .with_retries(2)
828 .with_backoff(Duration::from_millis(1));
829 let err = Node::run(&node, test_ctx()).await.unwrap_err();
830 assert!(matches!(err, RustyError::Node(_)));
831 assert!(err.to_string().contains("500"));
832 assert!(!err.is_interrupt());
833 assert_eq!(server.attempts.load(Ordering::SeqCst), 3);
835 }
836
837 #[tokio::test]
838 async fn remote_node_timeout_is_retryable_and_bounded() {
839 let server = start_mock(|_n, _body| MockBehavior::Hang);
840
841 let node = RemoteNode::new("slow", format!("http://{}", server.addr))
842 .with_timeout(Duration::from_millis(100))
843 .with_retries(1)
844 .with_backoff(Duration::from_millis(1));
845 let started = std::time::Instant::now();
846 let err = Node::run(&node, test_ctx()).await.unwrap_err();
847 let elapsed = started.elapsed();
848
849 assert!(matches!(err, RustyError::Node(_)));
850 assert_eq!(server.attempts.load(Ordering::SeqCst), 2);
852 assert!(elapsed < Duration::from_secs(5), "took {elapsed:?}");
855 }
856
857 #[tokio::test]
858 async fn remote_node_worker_error_maps_to_node_error_without_retry() {
859 let server = start_mock(|_n, _body| MockBehavior::Respond {
860 status: 200,
861 body: serde_json::to_string(&NodeTaskResponse::error(
862 "handler `doubler` panicked: divide by zero",
863 ))
864 .unwrap(),
865 });
866
867 let node = RemoteNode::new("doubler", format!("http://{}", server.addr))
868 .with_retries(5)
869 .with_backoff(Duration::from_millis(1));
870 let err = Node::run(&node, test_ctx()).await.unwrap_err();
871 match err {
872 RustyError::Node(message) => {
873 assert!(message.contains("divide by zero"), "got: {message}")
874 }
875 other => panic!("expected Node error, got {other:?}"),
876 }
877 assert_eq!(server.attempts.load(Ordering::SeqCst), 1);
879 }
880
881 #[tokio::test]
882 async fn remote_node_interrupt_passes_through_without_retry() {
883 let server = start_mock(|_n, _body| MockBehavior::Respond {
884 status: 200,
885 body: r#"{"interrupt": {"question": "approve deployment?"}}"#.to_string(),
886 });
887
888 let node = RemoteNode::new("gate", format!("http://{}", server.addr))
889 .with_retries(5)
890 .with_backoff(Duration::from_millis(1));
891 let err = Node::run(&node, test_ctx()).await.unwrap_err();
892 assert!(err.is_interrupt());
893 assert_eq!(
894 err.interrupt_value(),
895 Some(&json!({"question": "approve deployment?"}))
896 );
897 assert_eq!(server.attempts.load(Ordering::SeqCst), 1);
899 }
900
901 #[tokio::test]
902 async fn remote_node_resume_value_crosses_the_wire() {
903 let server = start_mock(|_n, body| {
904 let task: NodeTask = serde_json::from_str(&body).unwrap();
905 let response = match task.config.resume {
907 Some(v) => NodeTaskResponse::ok(NodeOutput::update("approved", v)),
908 None => NodeTaskResponse::interrupt(json!({"question": "approve?"})),
909 };
910 MockBehavior::Respond {
911 status: 200,
912 body: serde_json::to_string(&response).unwrap(),
913 }
914 });
915
916 let node = RemoteNode::new("gate", format!("http://{}", server.addr)).with_retries(0);
917
918 let err = Node::run(&node, test_ctx()).await.unwrap_err();
920 assert!(err.is_interrupt());
921
922 let resumed = NodeContext::new(
924 test_ctx().state().clone(),
925 NodeConfig {
926 resume: Some(json!(true)),
927 ..NodeConfig::default()
928 },
929 );
930 let out = Node::run(&node, resumed).await.unwrap();
931 assert_eq!(out.updates.get("approved"), Some(&json!(true)));
932 assert_eq!(server.attempts.load(Ordering::SeqCst), 2);
933 }
934
935 #[tokio::test]
936 async fn remote_node_retries_on_429_then_succeeds() {
937 let server = start_mock(|n, _body| {
938 if n < 2 {
939 MockBehavior::Respond {
940 status: 429,
941 body: r#"{"error":"rate limited"}"#.to_string(),
942 }
943 } else {
944 MockBehavior::Respond {
945 status: 200,
946 body: serde_json::to_string(&NodeTaskResponse::ok(NodeOutput::update(
947 "x",
948 json!("through"),
949 )))
950 .unwrap(),
951 }
952 }
953 });
954
955 let node = RemoteNode::new("limited", format!("http://{}", server.addr))
956 .with_retries(2)
957 .with_backoff(Duration::from_millis(1));
958 let out = Node::run(&node, test_ctx()).await.unwrap();
959 assert_eq!(out.updates.get("x"), Some(&json!("through")));
960 assert_eq!(server.attempts.load(Ordering::SeqCst), 2);
961 }
962
963 #[tokio::test]
964 async fn remote_node_does_not_retry_fatal_4xx() {
965 let server = start_mock(|_n, _body| MockBehavior::Respond {
966 status: 400,
967 body: r#"{"error":"malformed task"}"#.to_string(),
968 });
969
970 let node = RemoteNode::new("bad", format!("http://{}", server.addr))
971 .with_retries(5)
972 .with_backoff(Duration::from_millis(1));
973 let err = Node::run(&node, test_ctx()).await.unwrap_err();
974 assert!(matches!(err, RustyError::Node(_)));
975 assert!(err.to_string().contains("400"));
976 assert_eq!(server.attempts.load(Ordering::SeqCst), 1);
978 }
979
980 #[tokio::test]
981 async fn execute_url_is_used_verbatim_when_already_ending_in_execute() {
982 let node = RemoteNode::new("n", "http://127.0.0.1:1/execute");
983 assert_eq!(node.execute_url(), "http://127.0.0.1:1/execute");
984 let node = RemoteNode::new("n", "http://127.0.0.1:1/");
985 assert_eq!(node.execute_url(), "http://127.0.0.1:1/execute");
986 }
987}