1use std::sync::{Arc, OnceLock};
4use std::time::Duration;
5
6use base64::Engine as _;
7use base64::engine::general_purpose::STANDARD as BASE64;
8use futures::future::BoxFuture;
9use indexmap::IndexMap;
10use reqwest::Method;
11use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
12use serde::Deserialize;
13
14use graphrt_core::dsl::NodeDoc;
15use graphrt_core::node::{ConfigError, Node, NodeContext, NodeError, NodeOutput, NodeResult, parse_config};
16use graphrt_core::template::Template;
17use graphrt_core::value::{File, FileKind, FileTransfer, SECRET_MASK};
18use graphrt_core::{Selector, Value, VariablePool};
19
20use crate::body::{MultipartField, data_url, encode_multipart, inline_file_bytes, render_json};
21use crate::client::HttpClient;
22use crate::policy::{EgressError, EgressPolicy};
23
24pub const KIND: &str = "http-request";
26
27pub const INLINE_FILE_LIMIT: usize = 1024 * 1024;
29
30#[derive(Clone, Debug, Deserialize)]
32#[serde(untagged)]
33enum KvSpec {
34 Text(String),
35 List(Vec<KvItem>),
36 Map(IndexMap<String, Template>),
37}
38
39impl Default for KvSpec {
40 fn default() -> Self {
41 KvSpec::List(Vec::new())
42 }
43}
44
45#[derive(Clone, Debug, Deserialize)]
46struct KvItem {
47 key: Template,
48 #[serde(default)]
49 value: Template,
50}
51
52impl KvSpec {
53 fn pairs(&self) -> Vec<(Template, Template)> {
55 match self {
56 KvSpec::Text(text) => text
57 .lines()
58 .filter(|line| !line.trim().is_empty())
59 .map(|line| match line.split_once(':') {
60 Some((k, v)) => (Template::parse(k.trim()), Template::parse(v.trim())),
61 None => (Template::parse(line.trim()), Template::parse("")),
62 })
63 .collect(),
64 KvSpec::List(items) => items.iter().map(|i| (i.key.clone(), i.value.clone())).collect(),
65 KvSpec::Map(map) => map.iter().map(|(k, v)| (Template::parse(k), v.clone())).collect(),
66 }
67 }
68}
69
70#[derive(Clone, Debug, Default, Deserialize)]
71struct Authorization {
72 #[serde(rename = "type", default)]
73 kind: AuthKind,
74 #[serde(default)]
75 config: Option<AuthConfig>,
76}
77
78#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
79#[serde(rename_all = "kebab-case")]
80enum AuthKind {
81 #[default]
82 NoAuth,
83 ApiKey,
84}
85
86#[derive(Clone, Debug, Deserialize)]
87struct AuthConfig {
88 #[serde(rename = "type", default)]
89 scheme: AuthScheme,
90 #[serde(default)]
91 api_key: Template,
92 #[serde(default)]
93 header: Option<String>,
94}
95
96#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
97#[serde(rename_all = "lowercase")]
98enum AuthScheme {
99 #[default]
100 Bearer,
101 Basic,
102 Custom,
103}
104
105#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
106#[serde(rename_all = "kebab-case")]
107enum BodyKind {
108 #[default]
109 None,
110 Json,
111 RawText,
112 FormData,
113 XWwwFormUrlencoded,
114 Binary,
115}
116
117#[derive(Clone, Debug, Default, Deserialize)]
118struct BodyConfig {
119 #[serde(rename = "type", default)]
120 kind: BodyKind,
121 #[serde(default)]
122 data: BodyData,
123}
124
125#[derive(Clone, Debug, Deserialize)]
126#[serde(untagged)]
127enum BodyData {
128 Text(String),
129 Items(Vec<BodyItem>),
130}
131
132impl Default for BodyData {
133 fn default() -> Self {
134 BodyData::Items(Vec::new())
135 }
136}
137
138#[derive(Clone, Debug, Deserialize)]
139struct BodyItem {
140 #[serde(default)]
141 key: Template,
142 #[serde(rename = "type", default)]
143 kind: Option<String>,
144 #[serde(default)]
145 value: Template,
146 #[serde(default)]
147 file: Option<Selector>,
148}
149
150impl BodyData {
151 fn raw(&self) -> Template {
153 match self {
154 BodyData::Text(t) => Template::parse(t),
155 BodyData::Items(items) => items.first().map(|i| i.value.clone()).unwrap_or_default(),
156 }
157 }
158
159 fn items(&self) -> &[BodyItem] {
160 match self {
161 BodyData::Text(_) => &[],
162 BodyData::Items(items) => items,
163 }
164 }
165}
166
167#[derive(Clone, Copy, Debug, Default, Deserialize)]
168struct TimeoutConfig {
169 #[serde(default, alias = "max_connect_timeout")]
170 connect: Option<f64>,
171 #[serde(default, alias = "max_read_timeout")]
172 read: Option<f64>,
173 #[serde(default, alias = "max_write_timeout")]
174 write: Option<f64>,
175}
176
177#[derive(Clone, Debug, Deserialize)]
178struct HttpConfig {
179 #[serde(default = "default_method")]
180 method: String,
181 url: Template,
182 #[serde(default)]
183 headers: KvSpec,
184 #[serde(default)]
185 params: KvSpec,
186 #[serde(default)]
187 authorization: Authorization,
188 #[serde(default)]
189 body: Option<BodyConfig>,
190 #[serde(default)]
191 timeout: Option<TimeoutConfig>,
192 #[serde(default = "yes")]
193 ssl_verify: bool,
194 #[serde(default = "yes")]
195 fail_on_error_status: bool,
196 #[serde(default)]
197 max_response_bytes: Option<usize>,
198 #[serde(default)]
199 retry_non_idempotent: bool,
200 #[serde(default)]
201 parse_json: ParseJson,
202}
203
204#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
206#[serde(rename_all = "lowercase")]
207enum ParseJson {
208 #[default]
210 Lenient,
211 Strict,
213}
214
215fn default_method() -> String {
216 "get".to_owned()
217}
218
219fn yes() -> bool {
220 true
221}
222
223#[derive(Debug)]
243pub struct HttpRequestNode {
244 method: Method,
245 url: Template,
246 headers: Vec<(Template, Template)>,
247 params: Vec<(Template, Template)>,
248 auth: Option<AuthConfig>,
249 body: BodyConfig,
250 timeout: Option<Duration>,
251 ssl_verify: bool,
252 fail_on_error_status: bool,
253 max_response_bytes: Option<usize>,
254 retry_non_idempotent: bool,
255 parse_json: ParseJson,
256 default_client: OnceLock<Result<HttpClient, String>>,
257}
258
259const CREDENTIAL_HEADERS: &[&str] = &["authorization", "cookie", "proxy-authorization"];
261
262struct PreparedRequest {
264 method: Method,
265 url: reqwest::Url,
266 headers: HeaderMap,
267 sensitive: Vec<String>,
269 body: Option<Vec<u8>>,
270}
271
272impl HttpRequestNode {
273 pub fn from_doc(doc: &NodeDoc) -> Result<Self, ConfigError> {
275 let cfg: HttpConfig = parse_config(doc)?;
276 let method = parse_method(&cfg.method)
277 .ok_or_else(|| ConfigError::new(doc, format!("unsupported HTTP method `{}`", cfg.method)))?;
278 if cfg.url.source().trim().is_empty() {
279 return Err(ConfigError::new(doc, "`url` must not be empty"));
280 }
281 let auth = match cfg.authorization.kind {
282 AuthKind::NoAuth => None,
283 AuthKind::ApiKey => {
284 let config = cfg
285 .authorization
286 .config
287 .ok_or_else(|| ConfigError::new(doc, "api-key authorization requires `config`"))?;
288 if config.scheme == AuthScheme::Custom && config.header.as_deref().is_none_or(|h| h.trim().is_empty()) {
289 return Err(ConfigError::new(doc, "custom authorization requires a `header` name"));
290 }
291 Some(config)
292 }
293 };
294 let timeout = cfg.timeout.and_then(|t| {
295 let total: f64 = [t.connect, t.read, t.write].iter().flatten().filter(|s| s.is_finite() && **s > 0.0).sum();
296 (total > 0.0).then(|| Duration::from_secs_f64(total.min(86_400.0)))
297 });
298 Ok(Self {
299 method,
300 url: cfg.url,
301 headers: cfg.headers.pairs(),
302 params: cfg.params.pairs(),
303 auth,
304 body: cfg.body.unwrap_or_default(),
305 timeout,
306 ssl_verify: cfg.ssl_verify,
307 fail_on_error_status: cfg.fail_on_error_status,
308 max_response_bytes: cfg.max_response_bytes,
309 retry_non_idempotent: cfg.retry_non_idempotent,
310 parse_json: cfg.parse_json,
311 default_client: OnceLock::new(),
312 })
313 }
314
315 fn client<'a>(&'a self, ctx: &'a NodeContext) -> Result<&'a HttpClient, NodeError> {
316 if let Some(client) = ctx.service::<HttpClient>() {
317 return Ok(client);
318 }
319 self.default_client
320 .get_or_init(|| HttpClient::new(EgressPolicy::default()).map_err(|e| e.to_string()))
321 .as_ref()
322 .map_err(|e| NodeError::fatal("http_client", e.clone()))
323 }
324
325 fn may_retry(&self) -> bool {
326 self.retry_non_idempotent || is_idempotent(&self.method)
327 }
328
329 async fn execute(&self, ctx: &NodeContext) -> Result<NodeOutput, NodeError> {
330 let pool = &ctx.pool;
331 let client = self.client(ctx)?;
332 let policy = client.policy();
333
334 let raw_url = self.url.render_exposed(pool);
336 let mut url = reqwest::Url::parse(raw_url.trim())
337 .map_err(|e| NodeError::fatal("invalid_input", format!("invalid URL `{}`: {e}", self.url.render(pool))))?;
338 if !self.params.is_empty() {
339 let mut query = url.query_pairs_mut();
340 for (k, v) in &self.params {
341 if let Some(key) = render_key(k, pool, "query parameter")? {
342 query.append_pair(&key, &v.render_exposed(pool));
343 }
344 }
345 }
346 client.check_url(&url).map_err(egress_error)?;
347
348 let mut headers = HeaderMap::new();
350 let mut sensitive: Vec<String> = CREDENTIAL_HEADERS.iter().map(|h| (*h).to_owned()).collect();
351 let mut shown_headers = IndexMap::new();
352 for (k, v) in &self.headers {
353 let Some(name) = render_key(k, pool, "header")? else { continue };
354 let name = name.trim().to_ascii_lowercase();
355 let exposed = v.render_exposed(pool);
356 let shown = v.render(pool);
357 if exposed != shown {
358 sensitive.push(name.clone());
360 }
361 insert_header(&mut headers, &name, &exposed)?;
362 shown_headers.insert(name, Value::from(shown));
363 }
364 if let Some(auth) = &self.auth {
365 let key = auth.api_key.render_exposed(pool);
366 let (name, value) = match auth.scheme {
367 AuthScheme::Bearer => ("authorization".to_owned(), format!("Bearer {key}")),
368 AuthScheme::Basic => {
369 let encoded = if key.contains(':') { BASE64.encode(key.as_bytes()) } else { key };
370 ("authorization".to_owned(), format!("Basic {encoded}"))
371 }
372 AuthScheme::Custom => (auth.header.clone().unwrap_or_default().trim().to_ascii_lowercase(), key),
373 };
374 insert_header(&mut headers, &name, &value)?;
375 sensitive.push(name.clone());
376 shown_headers.insert(name, Value::from(SECRET_MASK));
377 }
378
379 let body = self.build_body(ctx)?;
381 let body_len = body.as_ref().map_or(0, |(_, b)| b.len());
382 let body = match body {
383 Some((content_type, bytes)) => {
384 if !headers.contains_key(reqwest::header::CONTENT_TYPE) {
385 insert_header(&mut headers, "content-type", &content_type)?;
386 }
387 Some(bytes)
388 }
389 None => None,
390 };
391
392 let mut shown_url = self.url.render(pool);
393 if url.query().is_some() && !self.params.is_empty() {
394 shown_url = format!("{shown_url}?{}", masked_query(&self.params, pool));
395 }
396
397 let total_timeout = self.timeout.map_or(policy.total_timeout, |t| t.min(policy.total_timeout));
399 let total_timeout = match ctx.remaining() {
400 Some(remaining) => total_timeout.min(remaining),
401 None => total_timeout,
402 };
403 let cap = self.max_response_bytes.map_or(policy.max_response_bytes, |c| c.min(policy.max_response_bytes));
404
405 let prepared = PreparedRequest { method: self.method.clone(), url, headers, sensitive, body };
406 let (response, final_url, redirects) = self.send_following_redirects(client, prepared, total_timeout).await?;
407
408 let request_meta = Value::object([
409 ("method", Value::from(self.method.as_str())),
410 ("url", Value::from(shown_url)),
411 ("headers", Value::Object(Arc::new(shown_headers.into_iter().collect()))),
412 ("body_bytes", Value::Int(i64::try_from(body_len).unwrap_or(i64::MAX))),
413 ("redirects", Value::Int(i64::try_from(redirects).unwrap_or(i64::MAX))),
414 ]);
415 let output = self.read_response(response, cap, &final_url).await?;
416 Ok(output.meta("request", request_meta))
417 }
418
419 async fn send_following_redirects(
422 &self,
423 client: &HttpClient,
424 mut request: PreparedRequest,
425 total_timeout: Duration,
426 ) -> Result<(reqwest::Response, reqwest::Url, usize), NodeError> {
427 let started = std::time::Instant::now();
428 let max_redirects = client.policy().max_redirects;
429 let http = client
430 .manual_redirect_client(self.ssl_verify)
431 .map_err(|e| NodeError::fatal("http_client", e.to_string()))?;
432 let mut redirects = 0usize;
433 loop {
434 let remaining = total_timeout.saturating_sub(started.elapsed());
435 if remaining.is_zero() {
436 let mut err = NodeError::new("timeout", "HTTP request timed out while following redirects");
437 err.retryable = self.may_retry();
438 return Err(err);
439 }
440 let mut builder = http
441 .request(request.method.clone(), request.url.clone())
442 .headers(request.headers.clone())
443 .timeout(remaining);
444 if let Some(bytes) = &request.body {
445 builder = builder.body(bytes.clone());
446 }
447 let response = builder.send().await.map_err(|e| self.transport_error(e))?;
448
449 let status = response.status();
450 let location = response.headers().get(reqwest::header::LOCATION).and_then(|v| v.to_str().ok());
451 let Some(location) = location.filter(|_| status.is_redirection() && max_redirects > 0) else {
452 return Ok((response, request.url, redirects));
453 };
454 if redirects >= max_redirects {
455 return Err(NodeError::fatal(
456 "too_many_redirects",
457 format!("stopped after {max_redirects} redirects (policy limit)"),
458 ));
459 }
460 let next = request.url.join(location).map_err(|_| {
461 NodeError::fatal("invalid_response", format!("HTTP {status} with an invalid Location header"))
462 })?;
463 client.check_url(&next).map_err(egress_error)?;
464
465 if next.origin() != request.url.origin() {
466 for name in &request.sensitive {
467 request.headers.remove(name.as_str());
468 }
469 }
470 let code = status.as_u16();
471 let becomes_get = code == 303 || (matches!(code, 301 | 302) && request.method == Method::POST);
472 if becomes_get {
473 if request.method != Method::HEAD {
474 request.method = Method::GET;
475 }
476 request.body = None;
477 request.headers.remove(reqwest::header::CONTENT_TYPE);
478 request.headers.remove(reqwest::header::CONTENT_LENGTH);
479 }
480 request.url = next;
481 redirects += 1;
482 }
483 }
484
485 fn build_body(&self, ctx: &NodeContext) -> Result<Option<(String, Vec<u8>)>, NodeError> {
486 let pool = &ctx.pool;
487 Ok(match self.body.kind {
488 BodyKind::None => None,
489 BodyKind::Json => {
490 let value = render_json(&self.body.data.raw(), pool)?;
491 if value.is_null() && self.body.data.raw().source().trim().is_empty() {
492 None
493 } else {
494 let bytes = serde_json::to_vec(&value)
495 .map_err(|e| NodeError::fatal("invalid_input", format!("cannot encode JSON body: {e}")))?;
496 Some(("application/json".to_owned(), bytes))
497 }
498 }
499 BodyKind::RawText => {
500 Some(("text/plain; charset=utf-8".to_owned(), self.body.data.raw().render_exposed(pool).into_bytes()))
501 }
502 BodyKind::XWwwFormUrlencoded => {
503 let mut form = url::form_urlencoded::Serializer::new(String::new());
504 for item in self.body.data.items() {
505 if let Some(key) = render_key(&item.key, pool, "form field")? {
506 form.append_pair(&key, &item.value.render_exposed(pool));
507 }
508 }
509 Some(("application/x-www-form-urlencoded".to_owned(), form.finish().into_bytes()))
510 }
511 BodyKind::FormData => {
512 let mut fields = Vec::new();
513 for item in self.body.data.items() {
514 let Some(name) = render_key(&item.key, pool, "form field")? else { continue };
515 if item.kind.as_deref() == Some("file") {
516 for file in files_at(pool, item.file.as_ref())? {
517 fields.push(MultipartField::File {
518 name: name.clone(),
519 filename: file.name.clone(),
520 mime: file.mime_type.clone().unwrap_or_else(|| "application/octet-stream".to_owned()),
521 bytes: inline_file_bytes(&file)?,
522 });
523 }
524 } else {
525 fields.push(MultipartField::Text { name, value: item.value.render_exposed(pool) });
526 }
527 }
528 let boundary = format!("graphrt-{}", ctx.exec_id.replace(|c: char| !c.is_ascii_alphanumeric(), ""));
529 let (content_type, body) = encode_multipart(&fields, &boundary);
530 Some((content_type, body))
531 }
532 BodyKind::Binary => {
533 let selector = self.body.data.items().first().and_then(|i| i.file.as_ref());
534 let file = files_at(pool, selector)?
535 .into_iter()
536 .next()
537 .ok_or_else(|| NodeError::fatal("invalid_input", "binary body requires a file"))?;
538 let mime = file.mime_type.clone().unwrap_or_else(|| "application/octet-stream".to_owned());
539 Some((mime, inline_file_bytes(&file)?))
540 }
541 })
542 }
543
544 async fn read_response(
545 &self,
546 mut response: reqwest::Response,
547 cap: usize,
548 url: &reqwest::Url,
549 ) -> Result<NodeOutput, NodeError> {
550 let status = response.status();
551 let error_status = self.fail_on_error_status && !status.is_success();
554 if let Some(len) = response.content_length() {
555 if !error_status && usize::try_from(len).map_or(true, |l| l > cap) {
556 return Err(too_large(cap));
557 }
558 }
559 let mut header_map: IndexMap<String, Value> = IndexMap::new();
560 for (name, value) in response.headers() {
561 let text = String::from_utf8_lossy(value.as_bytes()).into_owned();
562 let key = name.as_str().to_owned();
563 let merged = match header_map.get(&key).and_then(Value::as_str) {
564 Some(prev) => format!("{prev}, {text}"),
565 None => text,
566 };
567 header_map.insert(key, Value::from(merged));
568 }
569 let content_type =
570 header_map.get("content-type").and_then(Value::as_str).unwrap_or_default().to_ascii_lowercase();
571 let disposition = header_map.get("content-disposition").and_then(Value::as_str).unwrap_or_default().to_owned();
572 let retry_after_ms = header_map.get("retry-after").and_then(Value::as_str).and_then(parse_retry_after_ms);
573
574 let mut bytes: Vec<u8> = Vec::new();
575 loop {
576 match response.chunk().await {
577 Ok(Some(chunk)) => {
578 if bytes.len().saturating_add(chunk.len()) > cap {
579 if error_status {
580 break;
581 }
582 return Err(too_large(cap));
583 }
584 bytes.extend_from_slice(&chunk);
585 }
586 Ok(None) => break,
587 Err(e) => return Err(self.transport_error(e)),
588 }
589 }
590
591 if error_status {
592 let snippet: String = String::from_utf8_lossy(&bytes).chars().take(500).collect();
593 let message = format!("HTTP {status} from {}: {snippet}", url.host_str().unwrap_or_default());
594 let retryable_status = status.is_server_error() || matches!(status.as_u16(), 408 | 429);
595 let mut err = NodeError::new("http_status", message);
596 err.retryable = retryable_status && self.may_retry();
597 if matches!(status.as_u16(), 429 | 503) {
598 err.retry_after_ms = retry_after_ms;
599 }
600 return Err(err);
601 }
602
603 let status_code = Value::Int(i64::from(status.as_u16()));
604 let headers = Value::Object(Arc::new(header_map.into_iter().collect()));
605 let mut output = NodeOutput::new().with("status_code", status_code).with("headers", headers);
606
607 let decoded = if is_binary(&content_type, &disposition) { None } else { decode_text(&bytes, &content_type) };
608 match decoded {
609 Some(DecodedText { text, lossy }) => {
610 if is_json_type(&content_type) {
611 match serde_json::from_str::<serde_json::Value>(&text) {
612 Ok(json) => output = output.with("json", Value::from_json(json)),
613 Err(e) if self.parse_json == ParseJson::Strict => {
614 return Err(NodeError::fatal(
615 "invalid_response",
616 format!("response declared as JSON is not valid JSON: {e}"),
617 ));
618 }
619 Err(e) => {
620 output = output.with("json", Value::Null).meta("json_error", e.to_string());
621 }
622 }
623 }
624 if lossy {
625 output = output.meta("body_lossy", true);
626 }
627 output = output.with("body", text).with("files", Value::array([]));
628 }
629 None => {
630 if bytes.len() > INLINE_FILE_LIMIT {
631 return Err(NodeError::fatal(
632 "response_too_large",
633 format!(
634 "binary response of {} bytes exceeds the inline file limit of {INLINE_FILE_LIMIT} bytes",
635 bytes.len()
636 ),
637 ));
638 }
639 let mime = content_type.split(';').next().unwrap_or_default().trim();
640 let mime = if mime.is_empty() { "application/octet-stream" } else { mime };
641 let name = filename_from_disposition(&disposition)
642 .or_else(|| {
643 url.path_segments().and_then(|mut s| s.next_back()).filter(|s| !s.is_empty()).map(str::to_owned)
644 })
645 .unwrap_or_else(|| "download".to_owned());
646 let file = File {
647 marker: true,
648 id: None,
649 extension: name.rsplit_once('.').map(|(_, ext)| ext.to_ascii_lowercase()),
650 name,
651 kind: file_kind(mime),
652 transfer: FileTransfer::Inline,
653 url: Some(data_url(mime, &bytes)),
654 reference: None,
655 mime_type: Some(mime.to_owned()),
656 size: u64::try_from(bytes.len()).ok(),
657 };
658 output = output.with("body", "").with("files", Value::array([Value::from(file)]));
659 }
660 }
661 Ok(output)
662 }
663
664 fn transport_error(&self, error: reqwest::Error) -> NodeError {
666 let error = error.without_url();
667 if let Some(egress) = find_egress_error(&error) {
668 return egress_error(egress.clone());
669 }
670 if error.is_redirect() {
671 return NodeError::fatal("egress_blocked", error_chain(&error));
672 }
673 let mut err = if error.is_timeout() {
674 NodeError::new("timeout", error_chain(&error))
675 } else if error.is_connect() {
676 return NodeError::new("connection", error_chain(&error));
678 } else {
679 NodeError::new("http", error_chain(&error))
680 };
681 err.retryable = self.may_retry();
682 err
683 }
684}
685
686impl Node for HttpRequestNode {
687 fn run(&self, ctx: NodeContext) -> BoxFuture<'_, NodeResult> {
688 Box::pin(async move { Ok(self.execute(&ctx).await?.into()) })
689 }
690
691 fn inputs(&self) -> Vec<Selector> {
692 let mut out: Vec<Selector> = self.url.selectors().cloned().collect();
693 for (k, v) in self.headers.iter().chain(self.params.iter()) {
694 out.extend(k.selectors().cloned());
695 out.extend(v.selectors().cloned());
696 }
697 if let Some(auth) = &self.auth {
698 out.extend(auth.api_key.selectors().cloned());
699 }
700 match &self.body.data {
701 BodyData::Text(text) => out.extend(Template::parse(text).selectors().cloned()),
702 BodyData::Items(items) => {
703 for item in items {
704 out.extend(item.key.selectors().cloned());
705 out.extend(item.value.selectors().cloned());
706 out.extend(item.file.iter().cloned());
707 }
708 }
709 }
710 out
711 }
712}
713
714fn parse_method(method: &str) -> Option<Method> {
715 Some(match method.trim().to_ascii_uppercase().as_str() {
716 "GET" => Method::GET,
717 "POST" => Method::POST,
718 "PUT" => Method::PUT,
719 "PATCH" => Method::PATCH,
720 "DELETE" => Method::DELETE,
721 "HEAD" => Method::HEAD,
722 "OPTIONS" => Method::OPTIONS,
723 _ => return None,
724 })
725}
726
727fn is_idempotent(method: &Method) -> bool {
728 matches!(*method, Method::GET | Method::HEAD | Method::OPTIONS | Method::PUT | Method::DELETE)
729}
730
731fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(), NodeError> {
732 let name = HeaderName::from_bytes(name.as_bytes())
733 .map_err(|_| NodeError::fatal("invalid_input", format!("invalid header name `{name}`")))?;
734 let value = HeaderValue::from_str(value.trim())
735 .map_err(|_| NodeError::fatal("invalid_input", format!("invalid value for header `{name}`")))?;
736 headers.insert(name, value);
737 Ok(())
738}
739
740fn files_at(pool: &VariablePool, selector: Option<&Selector>) -> Result<Vec<File>, NodeError> {
746 let selector = selector.ok_or_else(|| NodeError::fatal("invalid_input", "file field has no `file` selector"))?;
747 match pool.get(selector) {
748 Some(Value::File(f)) => Ok(vec![(*f).clone()]),
749 Some(Value::Array(items)) => items
750 .iter()
751 .map(|v| match v {
752 Value::File(f) => Ok((**f).clone()),
753 other => Err(NodeError::fatal(
754 "invalid_input",
755 format!("`{selector}` contains a {}, expected only files", other.type_name()),
756 )),
757 })
758 .collect(),
759 Some(Value::Null) => Ok(Vec::new()),
760 None => Err(NodeError::fatal("invalid_input", format!("file variable `{selector}` not found"))),
761 Some(other) => {
762 Err(NodeError::fatal("invalid_input", format!("`{selector}` is a {}, not a file", other.type_name())))
763 }
764 }
765}
766
767fn masked_query(params: &[(Template, Template)], pool: &VariablePool) -> String {
769 let mut q = url::form_urlencoded::Serializer::new(String::new());
770 for (k, v) in params {
771 let key = k.render(pool);
772 if !key.is_empty() {
773 q.append_pair(&key, &v.render(pool));
774 }
775 }
776 q.finish()
777}
778
779fn render_key(template: &Template, pool: &VariablePool, what: &str) -> Result<Option<String>, NodeError> {
785 if template.source().trim().is_empty() {
786 return Ok(None);
787 }
788 let key = template.render_exposed(pool);
789 if key.trim().is_empty() {
790 return Err(NodeError::fatal(
791 "invalid_input",
792 format!("{what} name `{}` rendered empty (missing variable?)", template.source()),
793 ));
794 }
795 Ok(Some(key))
796}
797
798struct DecodedText {
800 text: String,
801 lossy: bool,
803}
804
805fn decode_text(bytes: &[u8], content_type: &str) -> Option<DecodedText> {
812 let charset = charset_of(content_type);
813 match charset.as_deref() {
814 Some("iso-8859-1" | "iso8859-1" | "latin1" | "latin-1" | "l1" | "us-ascii" | "ascii") => {
815 return Some(DecodedText { text: bytes.iter().map(|&b| char::from(b)).collect(), lossy: false });
816 }
817 Some("windows-1252" | "cp1252" | "x-cp1252") => {
818 return Some(DecodedText { text: bytes.iter().map(|&b| windows_1252_char(b)).collect(), lossy: false });
819 }
820 _ => {}
821 }
822 match std::str::from_utf8(bytes) {
823 Ok(text) => Some(DecodedText { text: text.to_owned(), lossy: false }),
824 Err(_) if is_textual(content_type) || charset.is_some() => {
825 Some(DecodedText { text: String::from_utf8_lossy(bytes).into_owned(), lossy: true })
826 }
827 Err(_) => None,
828 }
829}
830
831fn charset_of(content_type: &str) -> Option<String> {
833 content_type.split(';').skip(1).find_map(|param| {
834 let (name, value) = param.split_once('=')?;
835 name.trim().eq_ignore_ascii_case("charset").then(|| value.trim().trim_matches('"').to_ascii_lowercase())
836 })
837}
838
839fn windows_1252_char(b: u8) -> char {
841 const HIGH: [char; 32] = [
842 '\u{20AC}', '\u{FFFD}', '\u{201A}', '\u{0192}', '\u{201E}', '\u{2026}', '\u{2020}', '\u{2021}', '\u{02C6}',
843 '\u{2030}', '\u{0160}', '\u{2039}', '\u{0152}', '\u{FFFD}', '\u{017D}', '\u{FFFD}', '\u{FFFD}', '\u{2018}',
844 '\u{2019}', '\u{201C}', '\u{201D}', '\u{2022}', '\u{2013}', '\u{2014}', '\u{02DC}', '\u{2122}', '\u{0161}',
845 '\u{203A}', '\u{0153}', '\u{FFFD}', '\u{017E}', '\u{0178}',
846 ];
847 match b {
848 0x80..=0x9F => HIGH.get(usize::from(b - 0x80)).copied().unwrap_or('\u{FFFD}'),
849 _ => char::from(b),
850 }
851}
852
853fn is_textual(content_type: &str) -> bool {
855 let ct = content_type.split(';').next().unwrap_or_default().trim();
856 ct.starts_with("text/")
857 || ct.contains("json")
858 || ct.contains("xml")
859 || ct.contains("javascript")
860 || ct == "application/x-www-form-urlencoded"
861}
862
863fn is_json_type(content_type: &str) -> bool {
864 content_type.split(';').next().unwrap_or_default().trim().contains("json")
865}
866
867fn is_binary(content_type: &str, disposition: &str) -> bool {
868 if disposition.to_ascii_lowercase().contains("attachment") {
869 return true;
870 }
871 let ct = content_type.split(';').next().unwrap_or_default().trim();
872 if ct.is_empty()
873 || ct.starts_with("text/")
874 || ct.contains("json")
875 || ct.contains("xml")
876 || ct.contains("javascript")
877 {
878 return false;
879 }
880 ct.starts_with("image/")
881 || ct.starts_with("audio/")
882 || ct.starts_with("video/")
883 || ct.starts_with("font/")
884 || matches!(
885 ct,
886 "application/octet-stream"
887 | "application/pdf"
888 | "application/zip"
889 | "application/gzip"
890 | "application/x-tar"
891 | "application/msword"
892 )
893 || ct.starts_with("application/vnd.")
894}
895
896fn file_kind(mime: &str) -> FileKind {
897 if mime.starts_with("image/") {
898 FileKind::Image
899 } else if mime.starts_with("audio/") {
900 FileKind::Audio
901 } else if mime.starts_with("video/") {
902 FileKind::Video
903 } else if mime == "application/pdf" || mime.starts_with("application/vnd.") || mime == "application/msword" {
904 FileKind::Document
905 } else {
906 FileKind::Custom
907 }
908}
909
910fn filename_from_disposition(disposition: &str) -> Option<String> {
911 disposition.split(';').find_map(|part| {
912 let part = part.trim();
913 let value = part.strip_prefix("filename=")?;
914 let name = value.trim_matches('"').rsplit(['/', '\\']).next().unwrap_or_default().trim();
915 (!name.is_empty()).then(|| name.to_owned())
916 })
917}
918
919fn parse_retry_after_ms(value: &str) -> Option<u64> {
922 let seconds: u64 = value.trim().parse().ok()?;
923 Some(seconds.min(3600).saturating_mul(1000))
924}
925
926fn too_large(cap: usize) -> NodeError {
927 NodeError::fatal("response_too_large", format!("response body exceeds the limit of {cap} bytes"))
928}
929
930fn egress_error(e: EgressError) -> NodeError {
931 NodeError::fatal("egress_blocked", e.to_string())
932}
933
934fn find_egress_error<'a>(error: &'a (dyn std::error::Error + 'static)) -> Option<&'a EgressError> {
935 let mut current: Option<&(dyn std::error::Error + 'static)> = Some(error);
936 while let Some(e) = current {
937 if let Some(egress) = e.downcast_ref::<EgressError>() {
938 return Some(egress);
939 }
940 current = e.source();
941 }
942 None
943}
944
945fn error_chain(error: &(dyn std::error::Error + 'static)) -> String {
946 let mut parts = vec![error.to_string()];
947 let mut current = error.source();
948 while let Some(e) = current {
949 let text = e.to_string();
950 if parts.last() != Some(&text) {
951 parts.push(text);
952 }
953 current = e.source();
954 }
955 parts.join(": ")
956}
957
958#[cfg(test)]
959mod tests {
960 #[test]
961 fn retry_after_parsing() {
962 assert_eq!(super::parse_retry_after_ms(" 2 "), Some(2000));
963 assert_eq!(super::parse_retry_after_ms("999999"), Some(3_600_000));
964 assert_eq!(super::parse_retry_after_ms("Wed, 21 Oct 2015 07:28:00 GMT"), None);
965 }
966
967 use super::*;
968
969 fn doc(config: serde_json::Value) -> NodeDoc {
970 NodeDoc::new("h", KIND, config)
971 }
972
973 #[test]
974 fn kv_text_form() {
975 let spec = KvSpec::Text("a: 1\n\nb:two:three\nflag".into());
976 let pool = VariablePool::new();
977 let pairs: Vec<(String, String)> =
978 spec.pairs().iter().map(|(k, v)| (k.render(&pool), v.render(&pool))).collect();
979 assert_eq!(pairs, vec![("a".into(), "1".into()), ("b".into(), "two:three".into()), ("flag".into(), "".into())]);
980 }
981
982 #[test]
983 fn kv_list_and_map_forms() {
984 let list: KvSpec = serde_json::from_value(serde_json::json!([{"key": "x", "value": "{{#n.v#}}"}])).unwrap();
985 let map: KvSpec = serde_json::from_value(serde_json::json!({"x": "{{#n.v#}}"})).unwrap();
986 let mut pool = VariablePool::new();
987 pool.insert("n", "v", Value::from("val"));
988 for spec in [list, map] {
989 let pairs = spec.pairs();
990 assert_eq!(pairs.len(), 1);
991 assert_eq!(pairs[0].1.render(&pool), "val");
992 }
993 }
994
995 #[test]
996 fn config_validation() {
997 assert!(HttpRequestNode::from_doc(&doc(serde_json::json!({"url": "https://x", "method": "TRACE"}))).is_err());
998 assert!(HttpRequestNode::from_doc(&doc(serde_json::json!({"url": " "}))).is_err());
999 assert!(
1000 HttpRequestNode::from_doc(&doc(serde_json::json!({
1001 "url": "https://x",
1002 "authorization": {"type": "api-key", "config": {"type": "custom", "api_key": "k"}}
1003 })))
1004 .is_err()
1005 );
1006 let n = HttpRequestNode::from_doc(&doc(serde_json::json!({
1007 "url": "https://x", "method": "options", "timeout": {"connect": 1, "read": 2.5, "write": 0}
1008 })))
1009 .unwrap();
1010 assert_eq!(n.method, Method::OPTIONS);
1011 assert_eq!(n.timeout, Some(Duration::from_millis(3500)));
1012 }
1013
1014 #[test]
1015 fn binary_classification() {
1016 assert!(is_binary("image/png", ""));
1017 assert!(is_binary("application/pdf", ""));
1018 assert!(is_binary("text/plain", "attachment; filename=a.txt"));
1019 assert!(!is_binary("application/json; charset=utf-8", ""));
1020 assert!(!is_binary("text/html", ""));
1021 assert!(!is_binary("", ""));
1022 assert_eq!(filename_from_disposition("attachment; filename=\"../r.pdf\""), Some("r.pdf".into()));
1023 }
1024}