1use std::sync::{Arc, OnceLock};
4use std::time::Duration;
5
6use base64::Engine as _;
7use base64::engine::general_purpose::STANDARD as BASE64;
8use futures::future::BoxFuture;
9use indexmap::IndexMap;
10use reqwest::Method;
11use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
12use serde::Deserialize;
13
14use graphrt_core::dsl::NodeDoc;
15use graphrt_core::node::{ConfigError, Node, NodeContext, NodeError, NodeOutput, NodeResult, parse_config};
16use graphrt_core::template::Template;
17use graphrt_core::value::{File, FileKind, FileTransfer, SECRET_MASK};
18use graphrt_core::{Selector, Value, VariablePool};
19
20use crate::body::{MultipartField, data_url, encode_multipart, inline_file_bytes, render_json};
21use crate::client::HttpClient;
22use crate::policy::{EgressError, EgressPolicy};
23
24pub const KIND: &str = "http-request";
26
27pub const INLINE_FILE_LIMIT: usize = 1024 * 1024;
29
30#[derive(Clone, Debug, Deserialize)]
32#[serde(untagged)]
33enum KvSpec {
34 Text(String),
35 List(Vec<KvItem>),
36 Map(IndexMap<String, Template>),
37}
38
39impl Default for KvSpec {
40 fn default() -> Self {
41 KvSpec::List(Vec::new())
42 }
43}
44
45#[derive(Clone, Debug, Deserialize)]
46struct KvItem {
47 key: Template,
48 #[serde(default)]
49 value: Template,
50}
51
52impl KvSpec {
53 fn pairs(&self) -> Vec<(Template, Template)> {
55 match self {
56 KvSpec::Text(text) => text
57 .lines()
58 .filter(|line| !line.trim().is_empty())
59 .map(|line| match line.split_once(':') {
60 Some((k, v)) => (Template::parse(k.trim()), Template::parse(v.trim())),
61 None => (Template::parse(line.trim()), Template::parse("")),
62 })
63 .collect(),
64 KvSpec::List(items) => items.iter().map(|i| (i.key.clone(), i.value.clone())).collect(),
65 KvSpec::Map(map) => map.iter().map(|(k, v)| (Template::parse(k), v.clone())).collect(),
66 }
67 }
68}
69
70#[derive(Clone, Debug, Default, Deserialize)]
71struct Authorization {
72 #[serde(rename = "type", default)]
73 kind: AuthKind,
74 #[serde(default)]
75 config: Option<AuthConfig>,
76}
77
78#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
79#[serde(rename_all = "kebab-case")]
80enum AuthKind {
81 #[default]
82 NoAuth,
83 ApiKey,
84}
85
86#[derive(Clone, Debug, Deserialize)]
87struct AuthConfig {
88 #[serde(rename = "type", default)]
89 scheme: AuthScheme,
90 #[serde(default)]
91 api_key: Template,
92 #[serde(default)]
93 header: Option<String>,
94}
95
96#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
97#[serde(rename_all = "lowercase")]
98enum AuthScheme {
99 #[default]
100 Bearer,
101 Basic,
102 Custom,
103}
104
105#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
106#[serde(rename_all = "kebab-case")]
107enum BodyKind {
108 #[default]
109 None,
110 Json,
111 RawText,
112 FormData,
113 XWwwFormUrlencoded,
114 Binary,
115}
116
117#[derive(Clone, Debug, Default, Deserialize)]
118struct BodyConfig {
119 #[serde(rename = "type", default)]
120 kind: BodyKind,
121 #[serde(default)]
122 data: BodyData,
123}
124
125#[derive(Clone, Debug, Deserialize)]
126#[serde(untagged)]
127enum BodyData {
128 Text(String),
129 Items(Vec<BodyItem>),
130}
131
132impl Default for BodyData {
133 fn default() -> Self {
134 BodyData::Items(Vec::new())
135 }
136}
137
138#[derive(Clone, Debug, Deserialize)]
139struct BodyItem {
140 #[serde(default)]
141 key: Template,
142 #[serde(rename = "type", default)]
143 kind: Option<String>,
144 #[serde(default)]
145 value: Template,
146 #[serde(default)]
147 file: Option<Selector>,
148}
149
150impl BodyData {
151 fn raw(&self) -> Template {
153 match self {
154 BodyData::Text(t) => Template::parse(t),
155 BodyData::Items(items) => items.first().map(|i| i.value.clone()).unwrap_or_default(),
156 }
157 }
158
159 fn items(&self) -> &[BodyItem] {
160 match self {
161 BodyData::Text(_) => &[],
162 BodyData::Items(items) => items,
163 }
164 }
165}
166
167#[derive(Clone, Copy, Debug, Default, Deserialize)]
168struct TimeoutConfig {
169 #[serde(default, alias = "max_connect_timeout")]
170 connect: Option<f64>,
171 #[serde(default, alias = "max_read_timeout")]
172 read: Option<f64>,
173 #[serde(default, alias = "max_write_timeout")]
174 write: Option<f64>,
175}
176
177#[derive(Clone, Debug, Deserialize)]
178struct HttpConfig {
179 #[serde(default = "default_method")]
180 method: String,
181 url: Template,
182 #[serde(default)]
183 headers: KvSpec,
184 #[serde(default)]
185 params: KvSpec,
186 #[serde(default)]
187 authorization: Authorization,
188 #[serde(default)]
189 body: Option<BodyConfig>,
190 #[serde(default)]
191 timeout: Option<TimeoutConfig>,
192 #[serde(default = "yes")]
193 ssl_verify: bool,
194 #[serde(default = "yes")]
195 fail_on_error_status: bool,
196 #[serde(default)]
197 max_response_bytes: Option<usize>,
198 #[serde(default)]
199 retry_non_idempotent: bool,
200 #[serde(default)]
201 parse_json: ParseJson,
202}
203
204#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
206#[serde(rename_all = "lowercase")]
207enum ParseJson {
208 #[default]
210 Lenient,
211 Strict,
213}
214
215fn default_method() -> String {
216 "get".to_owned()
217}
218
219fn yes() -> bool {
220 true
221}
222
223#[derive(Debug)]
243pub struct HttpRequestNode {
244 method: Method,
245 url: Template,
246 headers: Vec<(Template, Template)>,
247 params: Vec<(Template, Template)>,
248 auth: Option<AuthConfig>,
249 body: BodyConfig,
250 timeout: Option<Duration>,
251 ssl_verify: bool,
252 fail_on_error_status: bool,
253 max_response_bytes: Option<usize>,
254 retry_non_idempotent: bool,
255 parse_json: ParseJson,
256 default_client: OnceLock<Result<HttpClient, String>>,
257}
258
259const CREDENTIAL_HEADERS: &[&str] = &["authorization", "cookie", "proxy-authorization"];
261
262struct PreparedRequest {
264 method: Method,
265 url: reqwest::Url,
266 headers: HeaderMap,
267 sensitive: Vec<String>,
269 body: Option<Vec<u8>>,
270}
271
272impl HttpRequestNode {
273 pub fn from_doc(doc: &NodeDoc) -> Result<Self, ConfigError> {
275 let cfg: HttpConfig = parse_config(doc)?;
276 let method = parse_method(&cfg.method)
277 .ok_or_else(|| ConfigError::new(doc, format!("unsupported HTTP method `{}`", cfg.method)))?;
278 if cfg.url.source().trim().is_empty() {
279 return Err(ConfigError::new(doc, "`url` must not be empty"));
280 }
281 let auth = match cfg.authorization.kind {
282 AuthKind::NoAuth => None,
283 AuthKind::ApiKey => {
284 let config = cfg
285 .authorization
286 .config
287 .ok_or_else(|| ConfigError::new(doc, "api-key authorization requires `config`"))?;
288 if config.scheme == AuthScheme::Custom && config.header.as_deref().is_none_or(|h| h.trim().is_empty()) {
289 return Err(ConfigError::new(doc, "custom authorization requires a `header` name"));
290 }
291 Some(config)
292 }
293 };
294 let timeout = cfg.timeout.and_then(|t| {
295 let total: f64 = [t.connect, t.read, t.write].iter().flatten().filter(|s| s.is_finite() && **s > 0.0).sum();
296 (total > 0.0).then(|| Duration::from_secs_f64(total.min(86_400.0)))
297 });
298 Ok(Self {
299 method,
300 url: cfg.url,
301 headers: cfg.headers.pairs(),
302 params: cfg.params.pairs(),
303 auth,
304 body: cfg.body.unwrap_or_default(),
305 timeout,
306 ssl_verify: cfg.ssl_verify,
307 fail_on_error_status: cfg.fail_on_error_status,
308 max_response_bytes: cfg.max_response_bytes,
309 retry_non_idempotent: cfg.retry_non_idempotent,
310 parse_json: cfg.parse_json,
311 default_client: OnceLock::new(),
312 })
313 }
314
315 fn client<'a>(&'a self, ctx: &'a NodeContext) -> Result<&'a HttpClient, NodeError> {
316 if let Some(client) = ctx.service::<HttpClient>() {
317 return Ok(client);
318 }
319 self.default_client
320 .get_or_init(|| HttpClient::new(EgressPolicy::default()).map_err(|e| e.to_string()))
321 .as_ref()
322 .map_err(|e| NodeError::fatal("http_client", e.clone()))
323 }
324
325 fn may_retry(&self) -> bool {
326 self.retry_non_idempotent || is_idempotent(&self.method)
327 }
328
329 async fn execute(&self, ctx: &NodeContext) -> Result<NodeOutput, NodeError> {
330 let pool = &ctx.pool;
331 let client = self.client(ctx)?;
332 let policy = client.policy();
333
334 let raw_url = self.url.render_exposed(pool);
336 let mut url = reqwest::Url::parse(raw_url.trim())
337 .map_err(|e| NodeError::fatal("invalid_input", format!("invalid URL `{}`: {e}", self.url.render(pool))))?;
338 if !self.params.is_empty() {
339 let mut query = url.query_pairs_mut();
340 for (k, v) in &self.params {
341 if let Some(key) = render_key(k, pool, "query parameter")? {
342 query.append_pair(&key, &v.render_exposed(pool));
343 }
344 }
345 }
346 client.check_url(&url).map_err(egress_error)?;
347
348 let mut headers = HeaderMap::new();
350 let mut sensitive: Vec<String> = CREDENTIAL_HEADERS.iter().map(|h| (*h).to_owned()).collect();
351 let mut shown_headers = IndexMap::new();
352 for (k, v) in &self.headers {
353 let Some(name) = render_key(k, pool, "header")? else { continue };
354 let name = name.trim().to_ascii_lowercase();
355 let exposed = v.render_exposed(pool);
356 let shown = v.render(pool);
357 if exposed != shown {
358 sensitive.push(name.clone());
360 }
361 insert_header(&mut headers, &name, &exposed)?;
362 shown_headers.insert(name, Value::from(shown));
363 }
364 if let Some(auth) = &self.auth {
365 let key = auth.api_key.render_exposed(pool);
366 let (name, value) = match auth.scheme {
367 AuthScheme::Bearer => ("authorization".to_owned(), format!("Bearer {key}")),
368 AuthScheme::Basic => {
369 let encoded = if key.contains(':') { BASE64.encode(key.as_bytes()) } else { key };
370 ("authorization".to_owned(), format!("Basic {encoded}"))
371 }
372 AuthScheme::Custom => (auth.header.clone().unwrap_or_default().trim().to_ascii_lowercase(), key),
373 };
374 insert_header(&mut headers, &name, &value)?;
375 sensitive.push(name.clone());
376 shown_headers.insert(name, Value::from(SECRET_MASK));
377 }
378
379 let body = self.build_body(ctx)?;
381 let body_len = body.as_ref().map_or(0, |(_, b)| b.len());
382 let body = match body {
383 Some((content_type, bytes)) => {
384 if !headers.contains_key(reqwest::header::CONTENT_TYPE) {
385 insert_header(&mut headers, "content-type", &content_type)?;
386 }
387 Some(bytes)
388 }
389 None => None,
390 };
391
392 let mut shown_url = self.url.render(pool);
393 if url.query().is_some() && !self.params.is_empty() {
394 shown_url = format!("{shown_url}?{}", masked_query(&self.params, pool));
395 }
396
397 let total_timeout = self.timeout.map_or(policy.total_timeout, |t| t.min(policy.total_timeout));
399 let total_timeout = match ctx.remaining() {
400 Some(remaining) => total_timeout.min(remaining),
401 None => total_timeout,
402 };
403 let cap = self.max_response_bytes.map_or(policy.max_response_bytes, |c| c.min(policy.max_response_bytes));
404
405 let prepared = PreparedRequest { method: self.method.clone(), url, headers, sensitive, body };
406 let (response, final_url, redirects) = self.send_following_redirects(client, prepared, total_timeout).await?;
407
408 let request_meta = Value::object([
409 ("method", Value::from(self.method.as_str())),
410 ("url", Value::from(shown_url)),
411 ("headers", Value::Object(Arc::new(shown_headers.into_iter().collect()))),
412 ("body_bytes", Value::Int(i64::try_from(body_len).unwrap_or(i64::MAX))),
413 ("redirects", Value::Int(i64::try_from(redirects).unwrap_or(i64::MAX))),
414 ]);
415 let output = self.read_response(response, cap, &final_url).await?;
416 Ok(output.meta("request", request_meta))
417 }
418
419 async fn send_following_redirects(
422 &self,
423 client: &HttpClient,
424 mut request: PreparedRequest,
425 total_timeout: Duration,
426 ) -> Result<(reqwest::Response, reqwest::Url, usize), NodeError> {
427 let started = std::time::Instant::now();
428 let max_redirects = client.policy().max_redirects;
429 let http = client.manual_redirect_client(self.ssl_verify);
430 let mut redirects = 0usize;
431 loop {
432 let remaining = total_timeout.saturating_sub(started.elapsed());
433 if remaining.is_zero() {
434 let mut err = NodeError::new("timeout", "HTTP request timed out while following redirects");
435 err.retryable = self.may_retry();
436 return Err(err);
437 }
438 let mut builder = http
439 .request(request.method.clone(), request.url.clone())
440 .headers(request.headers.clone())
441 .timeout(remaining);
442 if let Some(bytes) = &request.body {
443 builder = builder.body(bytes.clone());
444 }
445 let response = builder.send().await.map_err(|e| self.transport_error(e))?;
446
447 let status = response.status();
448 let location = response.headers().get(reqwest::header::LOCATION).and_then(|v| v.to_str().ok());
449 let Some(location) = location.filter(|_| status.is_redirection() && max_redirects > 0) else {
450 return Ok((response, request.url, redirects));
451 };
452 if redirects >= max_redirects {
453 return Err(NodeError::fatal(
454 "too_many_redirects",
455 format!("stopped after {max_redirects} redirects (policy limit)"),
456 ));
457 }
458 let next = request.url.join(location).map_err(|_| {
459 NodeError::fatal("invalid_response", format!("HTTP {status} with an invalid Location header"))
460 })?;
461 client.check_url(&next).map_err(egress_error)?;
462
463 if next.origin() != request.url.origin() {
464 for name in &request.sensitive {
465 request.headers.remove(name.as_str());
466 }
467 }
468 let code = status.as_u16();
469 let becomes_get = code == 303 || (matches!(code, 301 | 302) && request.method == Method::POST);
470 if becomes_get {
471 if request.method != Method::HEAD {
472 request.method = Method::GET;
473 }
474 request.body = None;
475 request.headers.remove(reqwest::header::CONTENT_TYPE);
476 request.headers.remove(reqwest::header::CONTENT_LENGTH);
477 }
478 request.url = next;
479 redirects += 1;
480 }
481 }
482
483 fn build_body(&self, ctx: &NodeContext) -> Result<Option<(String, Vec<u8>)>, NodeError> {
484 let pool = &ctx.pool;
485 Ok(match self.body.kind {
486 BodyKind::None => None,
487 BodyKind::Json => {
488 let value = render_json(&self.body.data.raw(), pool)?;
489 if value.is_null() && self.body.data.raw().source().trim().is_empty() {
490 None
491 } else {
492 let bytes = serde_json::to_vec(&value)
493 .map_err(|e| NodeError::fatal("invalid_input", format!("cannot encode JSON body: {e}")))?;
494 Some(("application/json".to_owned(), bytes))
495 }
496 }
497 BodyKind::RawText => {
498 Some(("text/plain; charset=utf-8".to_owned(), self.body.data.raw().render_exposed(pool).into_bytes()))
499 }
500 BodyKind::XWwwFormUrlencoded => {
501 let mut form = url::form_urlencoded::Serializer::new(String::new());
502 for item in self.body.data.items() {
503 if let Some(key) = render_key(&item.key, pool, "form field")? {
504 form.append_pair(&key, &item.value.render_exposed(pool));
505 }
506 }
507 Some(("application/x-www-form-urlencoded".to_owned(), form.finish().into_bytes()))
508 }
509 BodyKind::FormData => {
510 let mut fields = Vec::new();
511 for item in self.body.data.items() {
512 let Some(name) = render_key(&item.key, pool, "form field")? else { continue };
513 if item.kind.as_deref() == Some("file") {
514 for file in files_at(pool, item.file.as_ref())? {
515 fields.push(MultipartField::File {
516 name: name.clone(),
517 filename: file.name.clone(),
518 mime: file.mime_type.clone().unwrap_or_else(|| "application/octet-stream".to_owned()),
519 bytes: inline_file_bytes(&file)?,
520 });
521 }
522 } else {
523 fields.push(MultipartField::Text { name, value: item.value.render_exposed(pool) });
524 }
525 }
526 let boundary = format!("graphrt-{}", ctx.exec_id.replace(|c: char| !c.is_ascii_alphanumeric(), ""));
527 let (content_type, body) = encode_multipart(&fields, &boundary);
528 Some((content_type, body))
529 }
530 BodyKind::Binary => {
531 let selector = self.body.data.items().first().and_then(|i| i.file.as_ref());
532 let file = files_at(pool, selector)?
533 .into_iter()
534 .next()
535 .ok_or_else(|| NodeError::fatal("invalid_input", "binary body requires a file"))?;
536 let mime = file.mime_type.clone().unwrap_or_else(|| "application/octet-stream".to_owned());
537 Some((mime, inline_file_bytes(&file)?))
538 }
539 })
540 }
541
542 async fn read_response(
543 &self,
544 mut response: reqwest::Response,
545 cap: usize,
546 url: &reqwest::Url,
547 ) -> Result<NodeOutput, NodeError> {
548 let status = response.status();
549 let error_status = self.fail_on_error_status && !status.is_success();
552 if let Some(len) = response.content_length() {
553 if !error_status && usize::try_from(len).map_or(true, |l| l > cap) {
554 return Err(too_large(cap));
555 }
556 }
557 let mut header_map: IndexMap<String, Value> = IndexMap::new();
558 for (name, value) in response.headers() {
559 let text = String::from_utf8_lossy(value.as_bytes()).into_owned();
560 let key = name.as_str().to_owned();
561 let merged = match header_map.get(&key).and_then(Value::as_str) {
562 Some(prev) => format!("{prev}, {text}"),
563 None => text,
564 };
565 header_map.insert(key, Value::from(merged));
566 }
567 let content_type =
568 header_map.get("content-type").and_then(Value::as_str).unwrap_or_default().to_ascii_lowercase();
569 let disposition = header_map.get("content-disposition").and_then(Value::as_str).unwrap_or_default().to_owned();
570 let retry_after_ms = header_map.get("retry-after").and_then(Value::as_str).and_then(parse_retry_after_ms);
571
572 let mut bytes: Vec<u8> = Vec::new();
573 loop {
574 match response.chunk().await {
575 Ok(Some(chunk)) => {
576 if bytes.len().saturating_add(chunk.len()) > cap {
577 if error_status {
578 break;
579 }
580 return Err(too_large(cap));
581 }
582 bytes.extend_from_slice(&chunk);
583 }
584 Ok(None) => break,
585 Err(e) => return Err(self.transport_error(e)),
586 }
587 }
588
589 if error_status {
590 let snippet: String = String::from_utf8_lossy(&bytes).chars().take(500).collect();
591 let message = format!("HTTP {status} from {}: {snippet}", url.host_str().unwrap_or_default());
592 let retryable_status = status.is_server_error() || matches!(status.as_u16(), 408 | 429);
593 let mut err = NodeError::new("http_status", message);
594 err.retryable = retryable_status && self.may_retry();
595 if matches!(status.as_u16(), 429 | 503) {
596 err.retry_after_ms = retry_after_ms;
597 }
598 return Err(err);
599 }
600
601 let status_code = Value::Int(i64::from(status.as_u16()));
602 let headers = Value::Object(Arc::new(header_map.into_iter().collect()));
603 let mut output = NodeOutput::new().with("status_code", status_code).with("headers", headers);
604
605 let decoded = if is_binary(&content_type, &disposition) { None } else { decode_text(&bytes, &content_type) };
606 match decoded {
607 Some(DecodedText { text, lossy }) => {
608 if is_json_type(&content_type) {
609 match serde_json::from_str::<serde_json::Value>(&text) {
610 Ok(json) => output = output.with("json", Value::from_json(json)),
611 Err(e) if self.parse_json == ParseJson::Strict => {
612 return Err(NodeError::fatal(
613 "invalid_response",
614 format!("response declared as JSON is not valid JSON: {e}"),
615 ));
616 }
617 Err(e) => {
618 output = output.with("json", Value::Null).meta("json_error", e.to_string());
619 }
620 }
621 }
622 if lossy {
623 output = output.meta("body_lossy", true);
624 }
625 output = output.with("body", text).with("files", Value::array([]));
626 }
627 None => {
628 if bytes.len() > INLINE_FILE_LIMIT {
629 return Err(NodeError::fatal(
630 "response_too_large",
631 format!(
632 "binary response of {} bytes exceeds the inline file limit of {INLINE_FILE_LIMIT} bytes",
633 bytes.len()
634 ),
635 ));
636 }
637 let mime = content_type.split(';').next().unwrap_or_default().trim();
638 let mime = if mime.is_empty() { "application/octet-stream" } else { mime };
639 let name = filename_from_disposition(&disposition)
640 .or_else(|| {
641 url.path_segments().and_then(|mut s| s.next_back()).filter(|s| !s.is_empty()).map(str::to_owned)
642 })
643 .unwrap_or_else(|| "download".to_owned());
644 let file = File {
645 marker: true,
646 id: None,
647 extension: name.rsplit_once('.').map(|(_, ext)| ext.to_ascii_lowercase()),
648 name,
649 kind: file_kind(mime),
650 transfer: FileTransfer::Inline,
651 url: Some(data_url(mime, &bytes)),
652 reference: None,
653 mime_type: Some(mime.to_owned()),
654 size: u64::try_from(bytes.len()).ok(),
655 };
656 output = output.with("body", "").with("files", Value::array([Value::from(file)]));
657 }
658 }
659 Ok(output)
660 }
661
662 fn transport_error(&self, error: reqwest::Error) -> NodeError {
664 let error = error.without_url();
665 if let Some(egress) = find_egress_error(&error) {
666 return egress_error(egress.clone());
667 }
668 if error.is_redirect() {
669 return NodeError::fatal("egress_blocked", error_chain(&error));
670 }
671 let mut err = if error.is_timeout() {
672 NodeError::new("timeout", error_chain(&error))
673 } else if error.is_connect() {
674 return NodeError::new("connection", error_chain(&error));
676 } else {
677 NodeError::new("http", error_chain(&error))
678 };
679 err.retryable = self.may_retry();
680 err
681 }
682}
683
684impl Node for HttpRequestNode {
685 fn run(&self, ctx: NodeContext) -> BoxFuture<'_, NodeResult> {
686 Box::pin(async move { Ok(self.execute(&ctx).await?.into()) })
687 }
688
689 fn inputs(&self) -> Vec<Selector> {
690 let mut out: Vec<Selector> = self.url.selectors().cloned().collect();
691 for (k, v) in self.headers.iter().chain(self.params.iter()) {
692 out.extend(k.selectors().cloned());
693 out.extend(v.selectors().cloned());
694 }
695 if let Some(auth) = &self.auth {
696 out.extend(auth.api_key.selectors().cloned());
697 }
698 match &self.body.data {
699 BodyData::Text(text) => out.extend(Template::parse(text).selectors().cloned()),
700 BodyData::Items(items) => {
701 for item in items {
702 out.extend(item.key.selectors().cloned());
703 out.extend(item.value.selectors().cloned());
704 out.extend(item.file.iter().cloned());
705 }
706 }
707 }
708 out
709 }
710}
711
712fn parse_method(method: &str) -> Option<Method> {
713 Some(match method.trim().to_ascii_uppercase().as_str() {
714 "GET" => Method::GET,
715 "POST" => Method::POST,
716 "PUT" => Method::PUT,
717 "PATCH" => Method::PATCH,
718 "DELETE" => Method::DELETE,
719 "HEAD" => Method::HEAD,
720 "OPTIONS" => Method::OPTIONS,
721 _ => return None,
722 })
723}
724
725fn is_idempotent(method: &Method) -> bool {
726 matches!(*method, Method::GET | Method::HEAD | Method::OPTIONS | Method::PUT | Method::DELETE)
727}
728
729fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(), NodeError> {
730 let name = HeaderName::from_bytes(name.as_bytes())
731 .map_err(|_| NodeError::fatal("invalid_input", format!("invalid header name `{name}`")))?;
732 let value = HeaderValue::from_str(value.trim())
733 .map_err(|_| NodeError::fatal("invalid_input", format!("invalid value for header `{name}`")))?;
734 headers.insert(name, value);
735 Ok(())
736}
737
738fn files_at(pool: &VariablePool, selector: Option<&Selector>) -> Result<Vec<File>, NodeError> {
744 let selector = selector.ok_or_else(|| NodeError::fatal("invalid_input", "file field has no `file` selector"))?;
745 match pool.get(selector) {
746 Some(Value::File(f)) => Ok(vec![(*f).clone()]),
747 Some(Value::Array(items)) => items
748 .iter()
749 .map(|v| match v {
750 Value::File(f) => Ok((**f).clone()),
751 other => Err(NodeError::fatal(
752 "invalid_input",
753 format!("`{selector}` contains a {}, expected only files", other.type_name()),
754 )),
755 })
756 .collect(),
757 Some(Value::Null) => Ok(Vec::new()),
758 None => Err(NodeError::fatal("invalid_input", format!("file variable `{selector}` not found"))),
759 Some(other) => {
760 Err(NodeError::fatal("invalid_input", format!("`{selector}` is a {}, not a file", other.type_name())))
761 }
762 }
763}
764
765fn masked_query(params: &[(Template, Template)], pool: &VariablePool) -> String {
767 let mut q = url::form_urlencoded::Serializer::new(String::new());
768 for (k, v) in params {
769 let key = k.render(pool);
770 if !key.is_empty() {
771 q.append_pair(&key, &v.render(pool));
772 }
773 }
774 q.finish()
775}
776
777fn render_key(template: &Template, pool: &VariablePool, what: &str) -> Result<Option<String>, NodeError> {
783 if template.source().trim().is_empty() {
784 return Ok(None);
785 }
786 let key = template.render_exposed(pool);
787 if key.trim().is_empty() {
788 return Err(NodeError::fatal(
789 "invalid_input",
790 format!("{what} name `{}` rendered empty (missing variable?)", template.source()),
791 ));
792 }
793 Ok(Some(key))
794}
795
796struct DecodedText {
798 text: String,
799 lossy: bool,
801}
802
803fn decode_text(bytes: &[u8], content_type: &str) -> Option<DecodedText> {
810 let charset = charset_of(content_type);
811 match charset.as_deref() {
812 Some("iso-8859-1" | "iso8859-1" | "latin1" | "latin-1" | "l1" | "us-ascii" | "ascii") => {
813 return Some(DecodedText { text: bytes.iter().map(|&b| char::from(b)).collect(), lossy: false });
814 }
815 Some("windows-1252" | "cp1252" | "x-cp1252") => {
816 return Some(DecodedText { text: bytes.iter().map(|&b| windows_1252_char(b)).collect(), lossy: false });
817 }
818 _ => {}
819 }
820 match std::str::from_utf8(bytes) {
821 Ok(text) => Some(DecodedText { text: text.to_owned(), lossy: false }),
822 Err(_) if is_textual(content_type) || charset.is_some() => {
823 Some(DecodedText { text: String::from_utf8_lossy(bytes).into_owned(), lossy: true })
824 }
825 Err(_) => None,
826 }
827}
828
829fn charset_of(content_type: &str) -> Option<String> {
831 content_type.split(';').skip(1).find_map(|param| {
832 let (name, value) = param.split_once('=')?;
833 name.trim().eq_ignore_ascii_case("charset").then(|| value.trim().trim_matches('"').to_ascii_lowercase())
834 })
835}
836
837fn windows_1252_char(b: u8) -> char {
839 const HIGH: [char; 32] = [
840 '\u{20AC}', '\u{FFFD}', '\u{201A}', '\u{0192}', '\u{201E}', '\u{2026}', '\u{2020}', '\u{2021}', '\u{02C6}',
841 '\u{2030}', '\u{0160}', '\u{2039}', '\u{0152}', '\u{FFFD}', '\u{017D}', '\u{FFFD}', '\u{FFFD}', '\u{2018}',
842 '\u{2019}', '\u{201C}', '\u{201D}', '\u{2022}', '\u{2013}', '\u{2014}', '\u{02DC}', '\u{2122}', '\u{0161}',
843 '\u{203A}', '\u{0153}', '\u{FFFD}', '\u{017E}', '\u{0178}',
844 ];
845 match b {
846 0x80..=0x9F => HIGH.get(usize::from(b - 0x80)).copied().unwrap_or('\u{FFFD}'),
847 _ => char::from(b),
848 }
849}
850
851fn is_textual(content_type: &str) -> bool {
853 let ct = content_type.split(';').next().unwrap_or_default().trim();
854 ct.starts_with("text/")
855 || ct.contains("json")
856 || ct.contains("xml")
857 || ct.contains("javascript")
858 || ct == "application/x-www-form-urlencoded"
859}
860
861fn is_json_type(content_type: &str) -> bool {
862 content_type.split(';').next().unwrap_or_default().trim().contains("json")
863}
864
865fn is_binary(content_type: &str, disposition: &str) -> bool {
866 if disposition.to_ascii_lowercase().contains("attachment") {
867 return true;
868 }
869 let ct = content_type.split(';').next().unwrap_or_default().trim();
870 if ct.is_empty()
871 || ct.starts_with("text/")
872 || ct.contains("json")
873 || ct.contains("xml")
874 || ct.contains("javascript")
875 {
876 return false;
877 }
878 ct.starts_with("image/")
879 || ct.starts_with("audio/")
880 || ct.starts_with("video/")
881 || ct.starts_with("font/")
882 || matches!(
883 ct,
884 "application/octet-stream"
885 | "application/pdf"
886 | "application/zip"
887 | "application/gzip"
888 | "application/x-tar"
889 | "application/msword"
890 )
891 || ct.starts_with("application/vnd.")
892}
893
894fn file_kind(mime: &str) -> FileKind {
895 if mime.starts_with("image/") {
896 FileKind::Image
897 } else if mime.starts_with("audio/") {
898 FileKind::Audio
899 } else if mime.starts_with("video/") {
900 FileKind::Video
901 } else if mime == "application/pdf" || mime.starts_with("application/vnd.") || mime == "application/msword" {
902 FileKind::Document
903 } else {
904 FileKind::Custom
905 }
906}
907
908fn filename_from_disposition(disposition: &str) -> Option<String> {
909 disposition.split(';').find_map(|part| {
910 let part = part.trim();
911 let value = part.strip_prefix("filename=")?;
912 let name = value.trim_matches('"').rsplit(['/', '\\']).next().unwrap_or_default().trim();
913 (!name.is_empty()).then(|| name.to_owned())
914 })
915}
916
917fn parse_retry_after_ms(value: &str) -> Option<u64> {
920 let seconds: u64 = value.trim().parse().ok()?;
921 Some(seconds.min(3600).saturating_mul(1000))
922}
923
924fn too_large(cap: usize) -> NodeError {
925 NodeError::fatal("response_too_large", format!("response body exceeds the limit of {cap} bytes"))
926}
927
928fn egress_error(e: EgressError) -> NodeError {
929 NodeError::fatal("egress_blocked", e.to_string())
930}
931
932fn find_egress_error<'a>(error: &'a (dyn std::error::Error + 'static)) -> Option<&'a EgressError> {
933 let mut current: Option<&(dyn std::error::Error + 'static)> = Some(error);
934 while let Some(e) = current {
935 if let Some(egress) = e.downcast_ref::<EgressError>() {
936 return Some(egress);
937 }
938 current = e.source();
939 }
940 None
941}
942
943fn error_chain(error: &(dyn std::error::Error + 'static)) -> String {
944 let mut parts = vec![error.to_string()];
945 let mut current = error.source();
946 while let Some(e) = current {
947 let text = e.to_string();
948 if parts.last() != Some(&text) {
949 parts.push(text);
950 }
951 current = e.source();
952 }
953 parts.join(": ")
954}
955
956#[cfg(test)]
957mod tests {
958 #[test]
959 fn retry_after_parsing() {
960 assert_eq!(super::parse_retry_after_ms(" 2 "), Some(2000));
961 assert_eq!(super::parse_retry_after_ms("999999"), Some(3_600_000));
962 assert_eq!(super::parse_retry_after_ms("Wed, 21 Oct 2015 07:28:00 GMT"), None);
963 }
964
965 use super::*;
966
967 fn doc(config: serde_json::Value) -> NodeDoc {
968 NodeDoc::new("h", KIND, config)
969 }
970
971 #[test]
972 fn kv_text_form() {
973 let spec = KvSpec::Text("a: 1\n\nb:two:three\nflag".into());
974 let pool = VariablePool::new();
975 let pairs: Vec<(String, String)> =
976 spec.pairs().iter().map(|(k, v)| (k.render(&pool), v.render(&pool))).collect();
977 assert_eq!(pairs, vec![("a".into(), "1".into()), ("b".into(), "two:three".into()), ("flag".into(), "".into())]);
978 }
979
980 #[test]
981 fn kv_list_and_map_forms() {
982 let list: KvSpec = serde_json::from_value(serde_json::json!([{"key": "x", "value": "{{#n.v#}}"}])).unwrap();
983 let map: KvSpec = serde_json::from_value(serde_json::json!({"x": "{{#n.v#}}"})).unwrap();
984 let mut pool = VariablePool::new();
985 pool.insert("n", "v", Value::from("val"));
986 for spec in [list, map] {
987 let pairs = spec.pairs();
988 assert_eq!(pairs.len(), 1);
989 assert_eq!(pairs[0].1.render(&pool), "val");
990 }
991 }
992
993 #[test]
994 fn config_validation() {
995 assert!(HttpRequestNode::from_doc(&doc(serde_json::json!({"url": "https://x", "method": "TRACE"}))).is_err());
996 assert!(HttpRequestNode::from_doc(&doc(serde_json::json!({"url": " "}))).is_err());
997 assert!(
998 HttpRequestNode::from_doc(&doc(serde_json::json!({
999 "url": "https://x",
1000 "authorization": {"type": "api-key", "config": {"type": "custom", "api_key": "k"}}
1001 })))
1002 .is_err()
1003 );
1004 let n = HttpRequestNode::from_doc(&doc(serde_json::json!({
1005 "url": "https://x", "method": "options", "timeout": {"connect": 1, "read": 2.5, "write": 0}
1006 })))
1007 .unwrap();
1008 assert_eq!(n.method, Method::OPTIONS);
1009 assert_eq!(n.timeout, Some(Duration::from_millis(3500)));
1010 }
1011
1012 #[test]
1013 fn binary_classification() {
1014 assert!(is_binary("image/png", ""));
1015 assert!(is_binary("application/pdf", ""));
1016 assert!(is_binary("text/plain", "attachment; filename=a.txt"));
1017 assert!(!is_binary("application/json; charset=utf-8", ""));
1018 assert!(!is_binary("text/html", ""));
1019 assert!(!is_binary("", ""));
1020 assert_eq!(filename_from_disposition("attachment; filename=\"../r.pdf\""), Some("r.pdf".into()));
1021 }
1022}