Skip to main content

graphrt_http/
node.rs

1//! The `http-request` node.
2
3use std::sync::{Arc, OnceLock};
4use std::time::Duration;
5
6use base64::Engine as _;
7use base64::engine::general_purpose::STANDARD as BASE64;
8use futures::future::BoxFuture;
9use indexmap::IndexMap;
10use reqwest::Method;
11use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
12use serde::Deserialize;
13
14use graphrt_core::dsl::NodeDoc;
15use graphrt_core::node::{ConfigError, Node, NodeContext, NodeError, NodeOutput, NodeResult, parse_config};
16use graphrt_core::template::Template;
17use graphrt_core::value::{File, FileKind, FileTransfer, SECRET_MASK};
18use graphrt_core::{Selector, Value, VariablePool};
19
20use crate::body::{MultipartField, data_url, encode_multipart, inline_file_bytes, render_json};
21use crate::client::HttpClient;
22use crate::policy::{EgressError, EgressPolicy};
23
24/// Node kind handled by [`HttpRequestNode`].
25pub const KIND: &str = "http-request";
26
27/// Binary responses up to this size are returned inline as `data:` URL files.
28pub const INLINE_FILE_LIMIT: usize = 1024 * 1024;
29
30/// Key/value list accepted as `"k: v\nk2: v2"`, `[{key, value}]` or `{k: v}`.
31#[derive(Clone, Debug, Deserialize)]
32#[serde(untagged)]
33enum KvSpec {
34    Text(String),
35    List(Vec<KvItem>),
36    Map(IndexMap<String, Template>),
37}
38
39impl Default for KvSpec {
40    fn default() -> Self {
41        KvSpec::List(Vec::new())
42    }
43}
44
45#[derive(Clone, Debug, Deserialize)]
46struct KvItem {
47    key: Template,
48    #[serde(default)]
49    value: Template,
50}
51
52impl KvSpec {
53    /// Parsed pairs of templates (legacy text lines are split on the first `:`).
54    fn pairs(&self) -> Vec<(Template, Template)> {
55        match self {
56            KvSpec::Text(text) => text
57                .lines()
58                .filter(|line| !line.trim().is_empty())
59                .map(|line| match line.split_once(':') {
60                    Some((k, v)) => (Template::parse(k.trim()), Template::parse(v.trim())),
61                    None => (Template::parse(line.trim()), Template::parse("")),
62                })
63                .collect(),
64            KvSpec::List(items) => items.iter().map(|i| (i.key.clone(), i.value.clone())).collect(),
65            KvSpec::Map(map) => map.iter().map(|(k, v)| (Template::parse(k), v.clone())).collect(),
66        }
67    }
68}
69
70#[derive(Clone, Debug, Default, Deserialize)]
71struct Authorization {
72    #[serde(rename = "type", default)]
73    kind: AuthKind,
74    #[serde(default)]
75    config: Option<AuthConfig>,
76}
77
78#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
79#[serde(rename_all = "kebab-case")]
80enum AuthKind {
81    #[default]
82    NoAuth,
83    ApiKey,
84}
85
86#[derive(Clone, Debug, Deserialize)]
87struct AuthConfig {
88    #[serde(rename = "type", default)]
89    scheme: AuthScheme,
90    #[serde(default)]
91    api_key: Template,
92    #[serde(default)]
93    header: Option<String>,
94}
95
96#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
97#[serde(rename_all = "lowercase")]
98enum AuthScheme {
99    #[default]
100    Bearer,
101    Basic,
102    Custom,
103}
104
105#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
106#[serde(rename_all = "kebab-case")]
107enum BodyKind {
108    #[default]
109    None,
110    Json,
111    RawText,
112    FormData,
113    XWwwFormUrlencoded,
114    Binary,
115}
116
117#[derive(Clone, Debug, Default, Deserialize)]
118struct BodyConfig {
119    #[serde(rename = "type", default)]
120    kind: BodyKind,
121    #[serde(default)]
122    data: BodyData,
123}
124
125#[derive(Clone, Debug, Deserialize)]
126#[serde(untagged)]
127enum BodyData {
128    Text(String),
129    Items(Vec<BodyItem>),
130}
131
132impl Default for BodyData {
133    fn default() -> Self {
134        BodyData::Items(Vec::new())
135    }
136}
137
138#[derive(Clone, Debug, Deserialize)]
139struct BodyItem {
140    #[serde(default)]
141    key: Template,
142    #[serde(rename = "type", default)]
143    kind: Option<String>,
144    #[serde(default)]
145    value: Template,
146    #[serde(default)]
147    file: Option<Selector>,
148}
149
150impl BodyData {
151    /// Raw template for json / raw-text bodies.
152    fn raw(&self) -> Template {
153        match self {
154            BodyData::Text(t) => Template::parse(t),
155            BodyData::Items(items) => items.first().map(|i| i.value.clone()).unwrap_or_default(),
156        }
157    }
158
159    fn items(&self) -> &[BodyItem] {
160        match self {
161            BodyData::Text(_) => &[],
162            BodyData::Items(items) => items,
163        }
164    }
165}
166
167#[derive(Clone, Copy, Debug, Default, Deserialize)]
168struct TimeoutConfig {
169    #[serde(default, alias = "max_connect_timeout")]
170    connect: Option<f64>,
171    #[serde(default, alias = "max_read_timeout")]
172    read: Option<f64>,
173    #[serde(default, alias = "max_write_timeout")]
174    write: Option<f64>,
175}
176
177#[derive(Clone, Debug, Deserialize)]
178struct HttpConfig {
179    #[serde(default = "default_method")]
180    method: String,
181    url: Template,
182    #[serde(default)]
183    headers: KvSpec,
184    #[serde(default)]
185    params: KvSpec,
186    #[serde(default)]
187    authorization: Authorization,
188    #[serde(default)]
189    body: Option<BodyConfig>,
190    #[serde(default)]
191    timeout: Option<TimeoutConfig>,
192    #[serde(default = "yes")]
193    ssl_verify: bool,
194    #[serde(default = "yes")]
195    fail_on_error_status: bool,
196    #[serde(default)]
197    max_response_bytes: Option<usize>,
198    #[serde(default)]
199    retry_non_idempotent: bool,
200    #[serde(default)]
201    parse_json: ParseJson,
202}
203
204/// How a JSON response body that fails to parse is handled.
205#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
206#[serde(rename_all = "lowercase")]
207enum ParseJson {
208    /// Keep `body`, output `json: null` and record `json_error` in metadata.
209    #[default]
210    Lenient,
211    /// Fail the node with `invalid_response`.
212    Strict,
213}
214
215fn default_method() -> String {
216    "get".to_owned()
217}
218
219fn yes() -> bool {
220    true
221}
222
223/// Makes an HTTP request.
224///
225/// Outputs: `status_code`, `body` (text; empty for binary responses), `headers`
226/// (object, lower-case names), `files` (binary responses as inline files) and, for
227/// JSON content types, `json` (the parsed body, or `null` when the body is not valid
228/// JSON — the parse error is reported in metadata `json_error`; set
229/// `parse_json: strict` to fail instead).
230///
231/// Text bodies honour the `charset` content-type parameter for UTF-8, ISO-8859-1 and
232/// windows-1252; other undecodable text is decoded lossily and flagged with
233/// metadata `body_lossy: true`.
234///
235/// Redirects are followed by the node itself (up to the policy's `max_redirects`,
236/// each hop re-validated). When a hop changes origin (scheme, host or port), every
237/// credential header is dropped: `authorization`, `cookie`, `proxy-authorization`,
238/// the configured custom auth header and any header whose value came from a secret.
239///
240/// Node-level `timeout` and `max_response_bytes` can only lower the limits of the
241/// host's [`EgressPolicy`], never raise them.
242#[derive(Debug)]
243pub struct HttpRequestNode {
244    method: Method,
245    url: Template,
246    headers: Vec<(Template, Template)>,
247    params: Vec<(Template, Template)>,
248    auth: Option<AuthConfig>,
249    body: BodyConfig,
250    timeout: Option<Duration>,
251    ssl_verify: bool,
252    fail_on_error_status: bool,
253    max_response_bytes: Option<usize>,
254    retry_non_idempotent: bool,
255    parse_json: ParseJson,
256    default_client: OnceLock<Result<HttpClient, String>>,
257}
258
259/// Headers that always carry credentials.
260const CREDENTIAL_HEADERS: &[&str] = &["authorization", "cookie", "proxy-authorization"];
261
262/// A rendered request, ready to send (possibly several times across redirects).
263struct PreparedRequest {
264    method: Method,
265    url: reqwest::Url,
266    headers: HeaderMap,
267    /// Lower-case names of headers to drop when a redirect changes origin.
268    sensitive: Vec<String>,
269    body: Option<Vec<u8>>,
270}
271
272impl HttpRequestNode {
273    /// Build from a node document.
274    pub fn from_doc(doc: &NodeDoc) -> Result<Self, ConfigError> {
275        let cfg: HttpConfig = parse_config(doc)?;
276        let method = parse_method(&cfg.method)
277            .ok_or_else(|| ConfigError::new(doc, format!("unsupported HTTP method `{}`", cfg.method)))?;
278        if cfg.url.source().trim().is_empty() {
279            return Err(ConfigError::new(doc, "`url` must not be empty"));
280        }
281        let auth = match cfg.authorization.kind {
282            AuthKind::NoAuth => None,
283            AuthKind::ApiKey => {
284                let config = cfg
285                    .authorization
286                    .config
287                    .ok_or_else(|| ConfigError::new(doc, "api-key authorization requires `config`"))?;
288                if config.scheme == AuthScheme::Custom && config.header.as_deref().is_none_or(|h| h.trim().is_empty()) {
289                    return Err(ConfigError::new(doc, "custom authorization requires a `header` name"));
290                }
291                Some(config)
292            }
293        };
294        let timeout = cfg.timeout.and_then(|t| {
295            let total: f64 = [t.connect, t.read, t.write].iter().flatten().filter(|s| s.is_finite() && **s > 0.0).sum();
296            (total > 0.0).then(|| Duration::from_secs_f64(total.min(86_400.0)))
297        });
298        Ok(Self {
299            method,
300            url: cfg.url,
301            headers: cfg.headers.pairs(),
302            params: cfg.params.pairs(),
303            auth,
304            body: cfg.body.unwrap_or_default(),
305            timeout,
306            ssl_verify: cfg.ssl_verify,
307            fail_on_error_status: cfg.fail_on_error_status,
308            max_response_bytes: cfg.max_response_bytes,
309            retry_non_idempotent: cfg.retry_non_idempotent,
310            parse_json: cfg.parse_json,
311            default_client: OnceLock::new(),
312        })
313    }
314
315    fn client<'a>(&'a self, ctx: &'a NodeContext) -> Result<&'a HttpClient, NodeError> {
316        if let Some(client) = ctx.service::<HttpClient>() {
317            return Ok(client);
318        }
319        self.default_client
320            .get_or_init(|| HttpClient::new(EgressPolicy::default()).map_err(|e| e.to_string()))
321            .as_ref()
322            .map_err(|e| NodeError::fatal("http_client", e.clone()))
323    }
324
325    fn may_retry(&self) -> bool {
326        self.retry_non_idempotent || is_idempotent(&self.method)
327    }
328
329    async fn execute(&self, ctx: &NodeContext) -> Result<NodeOutput, NodeError> {
330        let pool = &ctx.pool;
331        let client = self.client(ctx)?;
332        let policy = client.policy();
333
334        // URL and query parameters.
335        let raw_url = self.url.render_exposed(pool);
336        let mut url = reqwest::Url::parse(raw_url.trim())
337            .map_err(|e| NodeError::fatal("invalid_input", format!("invalid URL `{}`: {e}", self.url.render(pool))))?;
338        if !self.params.is_empty() {
339            let mut query = url.query_pairs_mut();
340            for (k, v) in &self.params {
341                if let Some(key) = render_key(k, pool, "query parameter")? {
342                    query.append_pair(&key, &v.render_exposed(pool));
343                }
344            }
345        }
346        client.check_url(&url).map_err(egress_error)?;
347
348        // Headers: user headers first, then authorization (which wins).
349        let mut headers = HeaderMap::new();
350        let mut sensitive: Vec<String> = CREDENTIAL_HEADERS.iter().map(|h| (*h).to_owned()).collect();
351        let mut shown_headers = IndexMap::new();
352        for (k, v) in &self.headers {
353            let Some(name) = render_key(k, pool, "header")? else { continue };
354            let name = name.trim().to_ascii_lowercase();
355            let exposed = v.render_exposed(pool);
356            let shown = v.render(pool);
357            if exposed != shown {
358                // The value contains a secret: never forward it to another origin.
359                sensitive.push(name.clone());
360            }
361            insert_header(&mut headers, &name, &exposed)?;
362            shown_headers.insert(name, Value::from(shown));
363        }
364        if let Some(auth) = &self.auth {
365            let key = auth.api_key.render_exposed(pool);
366            let (name, value) = match auth.scheme {
367                AuthScheme::Bearer => ("authorization".to_owned(), format!("Bearer {key}")),
368                AuthScheme::Basic => {
369                    let encoded = if key.contains(':') { BASE64.encode(key.as_bytes()) } else { key };
370                    ("authorization".to_owned(), format!("Basic {encoded}"))
371                }
372                AuthScheme::Custom => (auth.header.clone().unwrap_or_default().trim().to_ascii_lowercase(), key),
373            };
374            insert_header(&mut headers, &name, &value)?;
375            sensitive.push(name.clone());
376            shown_headers.insert(name, Value::from(SECRET_MASK));
377        }
378
379        // Body.
380        let body = self.build_body(ctx)?;
381        let body_len = body.as_ref().map_or(0, |(_, b)| b.len());
382        let body = match body {
383            Some((content_type, bytes)) => {
384                if !headers.contains_key(reqwest::header::CONTENT_TYPE) {
385                    insert_header(&mut headers, "content-type", &content_type)?;
386                }
387                Some(bytes)
388            }
389            None => None,
390        };
391
392        let mut shown_url = self.url.render(pool);
393        if url.query().is_some() && !self.params.is_empty() {
394            shown_url = format!("{shown_url}?{}", masked_query(&self.params, pool));
395        }
396
397        // Limits from node config can only lower the operator's caps.
398        let total_timeout = self.timeout.map_or(policy.total_timeout, |t| t.min(policy.total_timeout));
399        let total_timeout = match ctx.remaining() {
400            Some(remaining) => total_timeout.min(remaining),
401            None => total_timeout,
402        };
403        let cap = self.max_response_bytes.map_or(policy.max_response_bytes, |c| c.min(policy.max_response_bytes));
404
405        let prepared = PreparedRequest { method: self.method.clone(), url, headers, sensitive, body };
406        let (response, final_url, redirects) = self.send_following_redirects(client, prepared, total_timeout).await?;
407
408        let request_meta = Value::object([
409            ("method", Value::from(self.method.as_str())),
410            ("url", Value::from(shown_url)),
411            ("headers", Value::Object(Arc::new(shown_headers.into_iter().collect()))),
412            ("body_bytes", Value::Int(i64::try_from(body_len).unwrap_or(i64::MAX))),
413            ("redirects", Value::Int(i64::try_from(redirects).unwrap_or(i64::MAX))),
414        ]);
415        let output = self.read_response(response, cap, &final_url).await?;
416        Ok(output.meta("request", request_meta))
417    }
418
419    /// Send the request, following redirects manually so credentials never cross origins.
420    /// Returns the final response, its URL and the number of redirects followed.
421    async fn send_following_redirects(
422        &self,
423        client: &HttpClient,
424        mut request: PreparedRequest,
425        total_timeout: Duration,
426    ) -> Result<(reqwest::Response, reqwest::Url, usize), NodeError> {
427        let started = std::time::Instant::now();
428        let max_redirects = client.policy().max_redirects;
429        let http = client.manual_redirect_client(self.ssl_verify);
430        let mut redirects = 0usize;
431        loop {
432            let remaining = total_timeout.saturating_sub(started.elapsed());
433            if remaining.is_zero() {
434                let mut err = NodeError::new("timeout", "HTTP request timed out while following redirects");
435                err.retryable = self.may_retry();
436                return Err(err);
437            }
438            let mut builder = http
439                .request(request.method.clone(), request.url.clone())
440                .headers(request.headers.clone())
441                .timeout(remaining);
442            if let Some(bytes) = &request.body {
443                builder = builder.body(bytes.clone());
444            }
445            let response = builder.send().await.map_err(|e| self.transport_error(e))?;
446
447            let status = response.status();
448            let location = response.headers().get(reqwest::header::LOCATION).and_then(|v| v.to_str().ok());
449            let Some(location) = location.filter(|_| status.is_redirection() && max_redirects > 0) else {
450                return Ok((response, request.url, redirects));
451            };
452            if redirects >= max_redirects {
453                return Err(NodeError::fatal(
454                    "too_many_redirects",
455                    format!("stopped after {max_redirects} redirects (policy limit)"),
456                ));
457            }
458            let next = request.url.join(location).map_err(|_| {
459                NodeError::fatal("invalid_response", format!("HTTP {status} with an invalid Location header"))
460            })?;
461            client.check_url(&next).map_err(egress_error)?;
462
463            if next.origin() != request.url.origin() {
464                for name in &request.sensitive {
465                    request.headers.remove(name.as_str());
466                }
467            }
468            let code = status.as_u16();
469            let becomes_get = code == 303 || (matches!(code, 301 | 302) && request.method == Method::POST);
470            if becomes_get {
471                if request.method != Method::HEAD {
472                    request.method = Method::GET;
473                }
474                request.body = None;
475                request.headers.remove(reqwest::header::CONTENT_TYPE);
476                request.headers.remove(reqwest::header::CONTENT_LENGTH);
477            }
478            request.url = next;
479            redirects += 1;
480        }
481    }
482
483    fn build_body(&self, ctx: &NodeContext) -> Result<Option<(String, Vec<u8>)>, NodeError> {
484        let pool = &ctx.pool;
485        Ok(match self.body.kind {
486            BodyKind::None => None,
487            BodyKind::Json => {
488                let value = render_json(&self.body.data.raw(), pool)?;
489                if value.is_null() && self.body.data.raw().source().trim().is_empty() {
490                    None
491                } else {
492                    let bytes = serde_json::to_vec(&value)
493                        .map_err(|e| NodeError::fatal("invalid_input", format!("cannot encode JSON body: {e}")))?;
494                    Some(("application/json".to_owned(), bytes))
495                }
496            }
497            BodyKind::RawText => {
498                Some(("text/plain; charset=utf-8".to_owned(), self.body.data.raw().render_exposed(pool).into_bytes()))
499            }
500            BodyKind::XWwwFormUrlencoded => {
501                let mut form = url::form_urlencoded::Serializer::new(String::new());
502                for item in self.body.data.items() {
503                    if let Some(key) = render_key(&item.key, pool, "form field")? {
504                        form.append_pair(&key, &item.value.render_exposed(pool));
505                    }
506                }
507                Some(("application/x-www-form-urlencoded".to_owned(), form.finish().into_bytes()))
508            }
509            BodyKind::FormData => {
510                let mut fields = Vec::new();
511                for item in self.body.data.items() {
512                    let Some(name) = render_key(&item.key, pool, "form field")? else { continue };
513                    if item.kind.as_deref() == Some("file") {
514                        for file in files_at(pool, item.file.as_ref())? {
515                            fields.push(MultipartField::File {
516                                name: name.clone(),
517                                filename: file.name.clone(),
518                                mime: file.mime_type.clone().unwrap_or_else(|| "application/octet-stream".to_owned()),
519                                bytes: inline_file_bytes(&file)?,
520                            });
521                        }
522                    } else {
523                        fields.push(MultipartField::Text { name, value: item.value.render_exposed(pool) });
524                    }
525                }
526                let boundary = format!("graphrt-{}", ctx.exec_id.replace(|c: char| !c.is_ascii_alphanumeric(), ""));
527                let (content_type, body) = encode_multipart(&fields, &boundary);
528                Some((content_type, body))
529            }
530            BodyKind::Binary => {
531                let selector = self.body.data.items().first().and_then(|i| i.file.as_ref());
532                let file = files_at(pool, selector)?
533                    .into_iter()
534                    .next()
535                    .ok_or_else(|| NodeError::fatal("invalid_input", "binary body requires a file"))?;
536                let mime = file.mime_type.clone().unwrap_or_else(|| "application/octet-stream".to_owned());
537                Some((mime, inline_file_bytes(&file)?))
538            }
539        })
540    }
541
542    async fn read_response(
543        &self,
544        mut response: reqwest::Response,
545        cap: usize,
546        url: &reqwest::Url,
547    ) -> Result<NodeOutput, NodeError> {
548        let status = response.status();
549        // Error responses only need a snippet; reading stops at the cap instead of failing,
550        // so a large 5xx page is still reported as a (retryable) status error.
551        let error_status = self.fail_on_error_status && !status.is_success();
552        if let Some(len) = response.content_length() {
553            if !error_status && usize::try_from(len).map_or(true, |l| l > cap) {
554                return Err(too_large(cap));
555            }
556        }
557        let mut header_map: IndexMap<String, Value> = IndexMap::new();
558        for (name, value) in response.headers() {
559            let text = String::from_utf8_lossy(value.as_bytes()).into_owned();
560            let key = name.as_str().to_owned();
561            let merged = match header_map.get(&key).and_then(Value::as_str) {
562                Some(prev) => format!("{prev}, {text}"),
563                None => text,
564            };
565            header_map.insert(key, Value::from(merged));
566        }
567        let content_type =
568            header_map.get("content-type").and_then(Value::as_str).unwrap_or_default().to_ascii_lowercase();
569        let disposition = header_map.get("content-disposition").and_then(Value::as_str).unwrap_or_default().to_owned();
570        let retry_after_ms = header_map.get("retry-after").and_then(Value::as_str).and_then(parse_retry_after_ms);
571
572        let mut bytes: Vec<u8> = Vec::new();
573        loop {
574            match response.chunk().await {
575                Ok(Some(chunk)) => {
576                    if bytes.len().saturating_add(chunk.len()) > cap {
577                        if error_status {
578                            break;
579                        }
580                        return Err(too_large(cap));
581                    }
582                    bytes.extend_from_slice(&chunk);
583                }
584                Ok(None) => break,
585                Err(e) => return Err(self.transport_error(e)),
586            }
587        }
588
589        if error_status {
590            let snippet: String = String::from_utf8_lossy(&bytes).chars().take(500).collect();
591            let message = format!("HTTP {status} from {}: {snippet}", url.host_str().unwrap_or_default());
592            let retryable_status = status.is_server_error() || matches!(status.as_u16(), 408 | 429);
593            let mut err = NodeError::new("http_status", message);
594            err.retryable = retryable_status && self.may_retry();
595            if matches!(status.as_u16(), 429 | 503) {
596                err.retry_after_ms = retry_after_ms;
597            }
598            return Err(err);
599        }
600
601        let status_code = Value::Int(i64::from(status.as_u16()));
602        let headers = Value::Object(Arc::new(header_map.into_iter().collect()));
603        let mut output = NodeOutput::new().with("status_code", status_code).with("headers", headers);
604
605        let decoded = if is_binary(&content_type, &disposition) { None } else { decode_text(&bytes, &content_type) };
606        match decoded {
607            Some(DecodedText { text, lossy }) => {
608                if is_json_type(&content_type) {
609                    match serde_json::from_str::<serde_json::Value>(&text) {
610                        Ok(json) => output = output.with("json", Value::from_json(json)),
611                        Err(e) if self.parse_json == ParseJson::Strict => {
612                            return Err(NodeError::fatal(
613                                "invalid_response",
614                                format!("response declared as JSON is not valid JSON: {e}"),
615                            ));
616                        }
617                        Err(e) => {
618                            output = output.with("json", Value::Null).meta("json_error", e.to_string());
619                        }
620                    }
621                }
622                if lossy {
623                    output = output.meta("body_lossy", true);
624                }
625                output = output.with("body", text).with("files", Value::array([]));
626            }
627            None => {
628                if bytes.len() > INLINE_FILE_LIMIT {
629                    return Err(NodeError::fatal(
630                        "response_too_large",
631                        format!(
632                            "binary response of {} bytes exceeds the inline file limit of {INLINE_FILE_LIMIT} bytes",
633                            bytes.len()
634                        ),
635                    ));
636                }
637                let mime = content_type.split(';').next().unwrap_or_default().trim();
638                let mime = if mime.is_empty() { "application/octet-stream" } else { mime };
639                let name = filename_from_disposition(&disposition)
640                    .or_else(|| {
641                        url.path_segments().and_then(|mut s| s.next_back()).filter(|s| !s.is_empty()).map(str::to_owned)
642                    })
643                    .unwrap_or_else(|| "download".to_owned());
644                let file = File {
645                    marker: true,
646                    id: None,
647                    extension: name.rsplit_once('.').map(|(_, ext)| ext.to_ascii_lowercase()),
648                    name,
649                    kind: file_kind(mime),
650                    transfer: FileTransfer::Inline,
651                    url: Some(data_url(mime, &bytes)),
652                    reference: None,
653                    mime_type: Some(mime.to_owned()),
654                    size: u64::try_from(bytes.len()).ok(),
655                };
656                output = output.with("body", "").with("files", Value::array([Value::from(file)]));
657            }
658        }
659        Ok(output)
660    }
661
662    /// Map a transport error. The URL is removed from the message: it may contain secrets.
663    fn transport_error(&self, error: reqwest::Error) -> NodeError {
664        let error = error.without_url();
665        if let Some(egress) = find_egress_error(&error) {
666            return egress_error(egress.clone());
667        }
668        if error.is_redirect() {
669            return NodeError::fatal("egress_blocked", error_chain(&error));
670        }
671        let mut err = if error.is_timeout() {
672            NodeError::new("timeout", error_chain(&error))
673        } else if error.is_connect() {
674            // Nothing was sent: retrying is safe for every method.
675            return NodeError::new("connection", error_chain(&error));
676        } else {
677            NodeError::new("http", error_chain(&error))
678        };
679        err.retryable = self.may_retry();
680        err
681    }
682}
683
684impl Node for HttpRequestNode {
685    fn run(&self, ctx: NodeContext) -> BoxFuture<'_, NodeResult> {
686        Box::pin(async move { Ok(self.execute(&ctx).await?.into()) })
687    }
688
689    fn inputs(&self) -> Vec<Selector> {
690        let mut out: Vec<Selector> = self.url.selectors().cloned().collect();
691        for (k, v) in self.headers.iter().chain(self.params.iter()) {
692            out.extend(k.selectors().cloned());
693            out.extend(v.selectors().cloned());
694        }
695        if let Some(auth) = &self.auth {
696            out.extend(auth.api_key.selectors().cloned());
697        }
698        match &self.body.data {
699            BodyData::Text(text) => out.extend(Template::parse(text).selectors().cloned()),
700            BodyData::Items(items) => {
701                for item in items {
702                    out.extend(item.key.selectors().cloned());
703                    out.extend(item.value.selectors().cloned());
704                    out.extend(item.file.iter().cloned());
705                }
706            }
707        }
708        out
709    }
710}
711
712fn parse_method(method: &str) -> Option<Method> {
713    Some(match method.trim().to_ascii_uppercase().as_str() {
714        "GET" => Method::GET,
715        "POST" => Method::POST,
716        "PUT" => Method::PUT,
717        "PATCH" => Method::PATCH,
718        "DELETE" => Method::DELETE,
719        "HEAD" => Method::HEAD,
720        "OPTIONS" => Method::OPTIONS,
721        _ => return None,
722    })
723}
724
725fn is_idempotent(method: &Method) -> bool {
726    matches!(*method, Method::GET | Method::HEAD | Method::OPTIONS | Method::PUT | Method::DELETE)
727}
728
729fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(), NodeError> {
730    let name = HeaderName::from_bytes(name.as_bytes())
731        .map_err(|_| NodeError::fatal("invalid_input", format!("invalid header name `{name}`")))?;
732    let value = HeaderValue::from_str(value.trim())
733        .map_err(|_| NodeError::fatal("invalid_input", format!("invalid value for header `{name}`")))?;
734    headers.insert(name, value);
735    Ok(())
736}
737
738/// Files referenced by a selector (a single file or an array of files).
739///
740/// A `null` value means "no file" (optional input). A selector that does not resolve
741/// at all, or array items that are not files, are errors rather than silently
742/// sending no file part.
743fn files_at(pool: &VariablePool, selector: Option<&Selector>) -> Result<Vec<File>, NodeError> {
744    let selector = selector.ok_or_else(|| NodeError::fatal("invalid_input", "file field has no `file` selector"))?;
745    match pool.get(selector) {
746        Some(Value::File(f)) => Ok(vec![(*f).clone()]),
747        Some(Value::Array(items)) => items
748            .iter()
749            .map(|v| match v {
750                Value::File(f) => Ok((**f).clone()),
751                other => Err(NodeError::fatal(
752                    "invalid_input",
753                    format!("`{selector}` contains a {}, expected only files", other.type_name()),
754                )),
755            })
756            .collect(),
757        Some(Value::Null) => Ok(Vec::new()),
758        None => Err(NodeError::fatal("invalid_input", format!("file variable `{selector}` not found"))),
759        Some(other) => {
760            Err(NodeError::fatal("invalid_input", format!("`{selector}` is a {}, not a file", other.type_name())))
761        }
762    }
763}
764
765/// Query string for display: secret-derived keys and values are masked.
766fn masked_query(params: &[(Template, Template)], pool: &VariablePool) -> String {
767    let mut q = url::form_urlencoded::Serializer::new(String::new());
768    for (k, v) in params {
769        let key = k.render(pool);
770        if !key.is_empty() {
771            q.append_pair(&key, &v.render(pool));
772        }
773    }
774    q.finish()
775}
776
777/// Render the key of a header / parameter / form field.
778///
779/// Returns `Ok(None)` for rows whose key template is blank (empty rows left in a
780/// form), and an error when a non-blank key template renders empty — that usually
781/// means a misspelled or missing variable, which must not silently drop the field.
782fn render_key(template: &Template, pool: &VariablePool, what: &str) -> Result<Option<String>, NodeError> {
783    if template.source().trim().is_empty() {
784        return Ok(None);
785    }
786    let key = template.render_exposed(pool);
787    if key.trim().is_empty() {
788        return Err(NodeError::fatal(
789            "invalid_input",
790            format!("{what} name `{}` rendered empty (missing variable?)", template.source()),
791        ));
792    }
793    Ok(Some(key))
794}
795
796/// Decoded text body.
797struct DecodedText {
798    text: String,
799    /// Some bytes could not be decoded and were replaced.
800    lossy: bool,
801}
802
803/// Decode a response body as text.
804///
805/// Honours the `charset` parameter for UTF-8, ISO-8859-1 / latin-1 and windows-1252.
806/// Bodies with a textual content type are always decoded (lossily when needed); bodies
807/// with an unknown or missing type are text only when they are valid UTF-8, otherwise
808/// `None` (treated as binary).
809fn decode_text(bytes: &[u8], content_type: &str) -> Option<DecodedText> {
810    let charset = charset_of(content_type);
811    match charset.as_deref() {
812        Some("iso-8859-1" | "iso8859-1" | "latin1" | "latin-1" | "l1" | "us-ascii" | "ascii") => {
813            return Some(DecodedText { text: bytes.iter().map(|&b| char::from(b)).collect(), lossy: false });
814        }
815        Some("windows-1252" | "cp1252" | "x-cp1252") => {
816            return Some(DecodedText { text: bytes.iter().map(|&b| windows_1252_char(b)).collect(), lossy: false });
817        }
818        _ => {}
819    }
820    match std::str::from_utf8(bytes) {
821        Ok(text) => Some(DecodedText { text: text.to_owned(), lossy: false }),
822        Err(_) if is_textual(content_type) || charset.is_some() => {
823            Some(DecodedText { text: String::from_utf8_lossy(bytes).into_owned(), lossy: true })
824        }
825        Err(_) => None,
826    }
827}
828
829/// The lower-cased `charset` parameter of a content type.
830fn charset_of(content_type: &str) -> Option<String> {
831    content_type.split(';').skip(1).find_map(|param| {
832        let (name, value) = param.split_once('=')?;
833        name.trim().eq_ignore_ascii_case("charset").then(|| value.trim().trim_matches('"').to_ascii_lowercase())
834    })
835}
836
837/// windows-1252 differs from latin-1 only in 0x80..=0x9F.
838fn windows_1252_char(b: u8) -> char {
839    const HIGH: [char; 32] = [
840        '\u{20AC}', '\u{FFFD}', '\u{201A}', '\u{0192}', '\u{201E}', '\u{2026}', '\u{2020}', '\u{2021}', '\u{02C6}',
841        '\u{2030}', '\u{0160}', '\u{2039}', '\u{0152}', '\u{FFFD}', '\u{017D}', '\u{FFFD}', '\u{FFFD}', '\u{2018}',
842        '\u{2019}', '\u{201C}', '\u{201D}', '\u{2022}', '\u{2013}', '\u{2014}', '\u{02DC}', '\u{2122}', '\u{0161}',
843        '\u{203A}', '\u{0153}', '\u{FFFD}', '\u{017E}', '\u{0178}',
844    ];
845    match b {
846        0x80..=0x9F => HIGH.get(usize::from(b - 0x80)).copied().unwrap_or('\u{FFFD}'),
847        _ => char::from(b),
848    }
849}
850
851/// Content types that are text by definition.
852fn is_textual(content_type: &str) -> bool {
853    let ct = content_type.split(';').next().unwrap_or_default().trim();
854    ct.starts_with("text/")
855        || ct.contains("json")
856        || ct.contains("xml")
857        || ct.contains("javascript")
858        || ct == "application/x-www-form-urlencoded"
859}
860
861fn is_json_type(content_type: &str) -> bool {
862    content_type.split(';').next().unwrap_or_default().trim().contains("json")
863}
864
865fn is_binary(content_type: &str, disposition: &str) -> bool {
866    if disposition.to_ascii_lowercase().contains("attachment") {
867        return true;
868    }
869    let ct = content_type.split(';').next().unwrap_or_default().trim();
870    if ct.is_empty()
871        || ct.starts_with("text/")
872        || ct.contains("json")
873        || ct.contains("xml")
874        || ct.contains("javascript")
875    {
876        return false;
877    }
878    ct.starts_with("image/")
879        || ct.starts_with("audio/")
880        || ct.starts_with("video/")
881        || ct.starts_with("font/")
882        || matches!(
883            ct,
884            "application/octet-stream"
885                | "application/pdf"
886                | "application/zip"
887                | "application/gzip"
888                | "application/x-tar"
889                | "application/msword"
890        )
891        || ct.starts_with("application/vnd.")
892}
893
894fn file_kind(mime: &str) -> FileKind {
895    if mime.starts_with("image/") {
896        FileKind::Image
897    } else if mime.starts_with("audio/") {
898        FileKind::Audio
899    } else if mime.starts_with("video/") {
900        FileKind::Video
901    } else if mime == "application/pdf" || mime.starts_with("application/vnd.") || mime == "application/msword" {
902        FileKind::Document
903    } else {
904        FileKind::Custom
905    }
906}
907
908fn filename_from_disposition(disposition: &str) -> Option<String> {
909    disposition.split(';').find_map(|part| {
910        let part = part.trim();
911        let value = part.strip_prefix("filename=")?;
912        let name = value.trim_matches('"').rsplit(['/', '\\']).next().unwrap_or_default().trim();
913        (!name.is_empty()).then(|| name.to_owned())
914    })
915}
916
917/// Parse a `Retry-After` header given in delay-seconds (the HTTP-date form is
918/// ignored: the node's own retry policy then applies). Capped at one hour.
919fn parse_retry_after_ms(value: &str) -> Option<u64> {
920    let seconds: u64 = value.trim().parse().ok()?;
921    Some(seconds.min(3600).saturating_mul(1000))
922}
923
924fn too_large(cap: usize) -> NodeError {
925    NodeError::fatal("response_too_large", format!("response body exceeds the limit of {cap} bytes"))
926}
927
928fn egress_error(e: EgressError) -> NodeError {
929    NodeError::fatal("egress_blocked", e.to_string())
930}
931
932fn find_egress_error<'a>(error: &'a (dyn std::error::Error + 'static)) -> Option<&'a EgressError> {
933    let mut current: Option<&(dyn std::error::Error + 'static)> = Some(error);
934    while let Some(e) = current {
935        if let Some(egress) = e.downcast_ref::<EgressError>() {
936            return Some(egress);
937        }
938        current = e.source();
939    }
940    None
941}
942
943fn error_chain(error: &(dyn std::error::Error + 'static)) -> String {
944    let mut parts = vec![error.to_string()];
945    let mut current = error.source();
946    while let Some(e) = current {
947        let text = e.to_string();
948        if parts.last() != Some(&text) {
949            parts.push(text);
950        }
951        current = e.source();
952    }
953    parts.join(": ")
954}
955
956#[cfg(test)]
957mod tests {
958    #[test]
959    fn retry_after_parsing() {
960        assert_eq!(super::parse_retry_after_ms(" 2 "), Some(2000));
961        assert_eq!(super::parse_retry_after_ms("999999"), Some(3_600_000));
962        assert_eq!(super::parse_retry_after_ms("Wed, 21 Oct 2015 07:28:00 GMT"), None);
963    }
964
965    use super::*;
966
967    fn doc(config: serde_json::Value) -> NodeDoc {
968        NodeDoc::new("h", KIND, config)
969    }
970
971    #[test]
972    fn kv_text_form() {
973        let spec = KvSpec::Text("a: 1\n\nb:two:three\nflag".into());
974        let pool = VariablePool::new();
975        let pairs: Vec<(String, String)> =
976            spec.pairs().iter().map(|(k, v)| (k.render(&pool), v.render(&pool))).collect();
977        assert_eq!(pairs, vec![("a".into(), "1".into()), ("b".into(), "two:three".into()), ("flag".into(), "".into())]);
978    }
979
980    #[test]
981    fn kv_list_and_map_forms() {
982        let list: KvSpec = serde_json::from_value(serde_json::json!([{"key": "x", "value": "{{#n.v#}}"}])).unwrap();
983        let map: KvSpec = serde_json::from_value(serde_json::json!({"x": "{{#n.v#}}"})).unwrap();
984        let mut pool = VariablePool::new();
985        pool.insert("n", "v", Value::from("val"));
986        for spec in [list, map] {
987            let pairs = spec.pairs();
988            assert_eq!(pairs.len(), 1);
989            assert_eq!(pairs[0].1.render(&pool), "val");
990        }
991    }
992
993    #[test]
994    fn config_validation() {
995        assert!(HttpRequestNode::from_doc(&doc(serde_json::json!({"url": "https://x", "method": "TRACE"}))).is_err());
996        assert!(HttpRequestNode::from_doc(&doc(serde_json::json!({"url": " "}))).is_err());
997        assert!(
998            HttpRequestNode::from_doc(&doc(serde_json::json!({
999                "url": "https://x",
1000                "authorization": {"type": "api-key", "config": {"type": "custom", "api_key": "k"}}
1001            })))
1002            .is_err()
1003        );
1004        let n = HttpRequestNode::from_doc(&doc(serde_json::json!({
1005            "url": "https://x", "method": "options", "timeout": {"connect": 1, "read": 2.5, "write": 0}
1006        })))
1007        .unwrap();
1008        assert_eq!(n.method, Method::OPTIONS);
1009        assert_eq!(n.timeout, Some(Duration::from_millis(3500)));
1010    }
1011
1012    #[test]
1013    fn binary_classification() {
1014        assert!(is_binary("image/png", ""));
1015        assert!(is_binary("application/pdf", ""));
1016        assert!(is_binary("text/plain", "attachment; filename=a.txt"));
1017        assert!(!is_binary("application/json; charset=utf-8", ""));
1018        assert!(!is_binary("text/html", ""));
1019        assert!(!is_binary("", ""));
1020        assert_eq!(filename_from_disposition("attachment; filename=\"../r.pdf\""), Some("r.pdf".into()));
1021    }
1022}