Skip to main content

graphrt_http/
node.rs

1//! The `http-request` node.
2
3use std::sync::{Arc, OnceLock};
4use std::time::Duration;
5
6use base64::Engine as _;
7use base64::engine::general_purpose::STANDARD as BASE64;
8use futures::future::BoxFuture;
9use indexmap::IndexMap;
10use reqwest::Method;
11use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
12use serde::Deserialize;
13
14use graphrt_core::dsl::NodeDoc;
15use graphrt_core::node::{ConfigError, Node, NodeContext, NodeError, NodeOutput, NodeResult, parse_config};
16use graphrt_core::template::Template;
17use graphrt_core::value::{File, FileKind, FileTransfer, SECRET_MASK};
18use graphrt_core::{Selector, Value, VariablePool};
19
20use crate::body::{MultipartField, data_url, encode_multipart, inline_file_bytes, render_json};
21use crate::client::HttpClient;
22use crate::policy::{EgressError, EgressPolicy};
23
24/// Node kind handled by [`HttpRequestNode`].
25pub const KIND: &str = "http-request";
26
27/// Binary responses up to this size are returned inline as `data:` URL files.
28pub const INLINE_FILE_LIMIT: usize = 1024 * 1024;
29
30/// Key/value list accepted as `"k: v\nk2: v2"`, `[{key, value}]` or `{k: v}`.
31#[derive(Clone, Debug, Deserialize)]
32#[serde(untagged)]
33enum KvSpec {
34    Text(String),
35    List(Vec<KvItem>),
36    Map(IndexMap<String, Template>),
37}
38
39impl Default for KvSpec {
40    fn default() -> Self {
41        KvSpec::List(Vec::new())
42    }
43}
44
45#[derive(Clone, Debug, Deserialize)]
46struct KvItem {
47    key: Template,
48    #[serde(default)]
49    value: Template,
50}
51
52impl KvSpec {
53    /// Parsed pairs of templates (legacy text lines are split on the first `:`).
54    fn pairs(&self) -> Vec<(Template, Template)> {
55        match self {
56            KvSpec::Text(text) => text
57                .lines()
58                .filter(|line| !line.trim().is_empty())
59                .map(|line| match line.split_once(':') {
60                    Some((k, v)) => (Template::parse(k.trim()), Template::parse(v.trim())),
61                    None => (Template::parse(line.trim()), Template::parse("")),
62                })
63                .collect(),
64            KvSpec::List(items) => items.iter().map(|i| (i.key.clone(), i.value.clone())).collect(),
65            KvSpec::Map(map) => map.iter().map(|(k, v)| (Template::parse(k), v.clone())).collect(),
66        }
67    }
68}
69
70#[derive(Clone, Debug, Default, Deserialize)]
71struct Authorization {
72    #[serde(rename = "type", default)]
73    kind: AuthKind,
74    #[serde(default)]
75    config: Option<AuthConfig>,
76}
77
78#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
79#[serde(rename_all = "kebab-case")]
80enum AuthKind {
81    #[default]
82    NoAuth,
83    ApiKey,
84}
85
86#[derive(Clone, Debug, Deserialize)]
87struct AuthConfig {
88    #[serde(rename = "type", default)]
89    scheme: AuthScheme,
90    #[serde(default)]
91    api_key: Template,
92    #[serde(default)]
93    header: Option<String>,
94}
95
96#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
97#[serde(rename_all = "lowercase")]
98enum AuthScheme {
99    #[default]
100    Bearer,
101    Basic,
102    Custom,
103}
104
105#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
106#[serde(rename_all = "kebab-case")]
107enum BodyKind {
108    #[default]
109    None,
110    Json,
111    RawText,
112    FormData,
113    XWwwFormUrlencoded,
114    Binary,
115}
116
117#[derive(Clone, Debug, Default, Deserialize)]
118struct BodyConfig {
119    #[serde(rename = "type", default)]
120    kind: BodyKind,
121    #[serde(default)]
122    data: BodyData,
123}
124
125#[derive(Clone, Debug, Deserialize)]
126#[serde(untagged)]
127enum BodyData {
128    Text(String),
129    Items(Vec<BodyItem>),
130}
131
132impl Default for BodyData {
133    fn default() -> Self {
134        BodyData::Items(Vec::new())
135    }
136}
137
138#[derive(Clone, Debug, Deserialize)]
139struct BodyItem {
140    #[serde(default)]
141    key: Template,
142    #[serde(rename = "type", default)]
143    kind: Option<String>,
144    #[serde(default)]
145    value: Template,
146    #[serde(default)]
147    file: Option<Selector>,
148}
149
150impl BodyData {
151    /// Raw template for json / raw-text bodies.
152    fn raw(&self) -> Template {
153        match self {
154            BodyData::Text(t) => Template::parse(t),
155            BodyData::Items(items) => items.first().map(|i| i.value.clone()).unwrap_or_default(),
156        }
157    }
158
159    fn items(&self) -> &[BodyItem] {
160        match self {
161            BodyData::Text(_) => &[],
162            BodyData::Items(items) => items,
163        }
164    }
165}
166
167#[derive(Clone, Copy, Debug, Default, Deserialize)]
168struct TimeoutConfig {
169    #[serde(default, alias = "max_connect_timeout")]
170    connect: Option<f64>,
171    #[serde(default, alias = "max_read_timeout")]
172    read: Option<f64>,
173    #[serde(default, alias = "max_write_timeout")]
174    write: Option<f64>,
175}
176
177#[derive(Clone, Debug, Deserialize)]
178struct HttpConfig {
179    #[serde(default = "default_method")]
180    method: String,
181    url: Template,
182    #[serde(default)]
183    headers: KvSpec,
184    #[serde(default)]
185    params: KvSpec,
186    #[serde(default)]
187    authorization: Authorization,
188    #[serde(default)]
189    body: Option<BodyConfig>,
190    #[serde(default)]
191    timeout: Option<TimeoutConfig>,
192    #[serde(default = "yes")]
193    ssl_verify: bool,
194    #[serde(default = "yes")]
195    fail_on_error_status: bool,
196    #[serde(default)]
197    max_response_bytes: Option<usize>,
198    #[serde(default)]
199    retry_non_idempotent: bool,
200    #[serde(default)]
201    parse_json: ParseJson,
202}
203
204/// How a JSON response body that fails to parse is handled.
205#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)]
206#[serde(rename_all = "lowercase")]
207enum ParseJson {
208    /// Keep `body`, output `json: null` and record `json_error` in metadata.
209    #[default]
210    Lenient,
211    /// Fail the node with `invalid_response`.
212    Strict,
213}
214
215fn default_method() -> String {
216    "get".to_owned()
217}
218
219fn yes() -> bool {
220    true
221}
222
223/// Makes an HTTP request.
224///
225/// Outputs: `status_code`, `body` (text; empty for binary responses), `headers`
226/// (object, lower-case names), `files` (binary responses as inline files) and, for
227/// JSON content types, `json` (the parsed body, or `null` when the body is not valid
228/// JSON — the parse error is reported in metadata `json_error`; set
229/// `parse_json: strict` to fail instead).
230///
231/// Text bodies honour the `charset` content-type parameter for UTF-8, ISO-8859-1 and
232/// windows-1252; other undecodable text is decoded lossily and flagged with
233/// metadata `body_lossy: true`.
234///
235/// Redirects are followed by the node itself (up to the policy's `max_redirects`,
236/// each hop re-validated). When a hop changes origin (scheme, host or port), every
237/// credential header is dropped: `authorization`, `cookie`, `proxy-authorization`,
238/// the configured custom auth header and any header whose value came from a secret.
239///
240/// Node-level `timeout` and `max_response_bytes` can only lower the limits of the
241/// host's [`EgressPolicy`], never raise them.
242#[derive(Debug)]
243pub struct HttpRequestNode {
244    method: Method,
245    url: Template,
246    headers: Vec<(Template, Template)>,
247    params: Vec<(Template, Template)>,
248    auth: Option<AuthConfig>,
249    body: BodyConfig,
250    timeout: Option<Duration>,
251    ssl_verify: bool,
252    fail_on_error_status: bool,
253    max_response_bytes: Option<usize>,
254    retry_non_idempotent: bool,
255    parse_json: ParseJson,
256    default_client: OnceLock<Result<HttpClient, String>>,
257}
258
259/// Headers that always carry credentials.
260const CREDENTIAL_HEADERS: &[&str] = &["authorization", "cookie", "proxy-authorization"];
261
262/// A rendered request, ready to send (possibly several times across redirects).
263struct PreparedRequest {
264    method: Method,
265    url: reqwest::Url,
266    headers: HeaderMap,
267    /// Lower-case names of headers to drop when a redirect changes origin.
268    sensitive: Vec<String>,
269    body: Option<Vec<u8>>,
270}
271
272impl HttpRequestNode {
273    /// Build from a node document.
274    pub fn from_doc(doc: &NodeDoc) -> Result<Self, ConfigError> {
275        let cfg: HttpConfig = parse_config(doc)?;
276        let method = parse_method(&cfg.method)
277            .ok_or_else(|| ConfigError::new(doc, format!("unsupported HTTP method `{}`", cfg.method)))?;
278        if cfg.url.source().trim().is_empty() {
279            return Err(ConfigError::new(doc, "`url` must not be empty"));
280        }
281        let auth = match cfg.authorization.kind {
282            AuthKind::NoAuth => None,
283            AuthKind::ApiKey => {
284                let config = cfg
285                    .authorization
286                    .config
287                    .ok_or_else(|| ConfigError::new(doc, "api-key authorization requires `config`"))?;
288                if config.scheme == AuthScheme::Custom && config.header.as_deref().is_none_or(|h| h.trim().is_empty()) {
289                    return Err(ConfigError::new(doc, "custom authorization requires a `header` name"));
290                }
291                Some(config)
292            }
293        };
294        let timeout = cfg.timeout.and_then(|t| {
295            let total: f64 = [t.connect, t.read, t.write].iter().flatten().filter(|s| s.is_finite() && **s > 0.0).sum();
296            (total > 0.0).then(|| Duration::from_secs_f64(total.min(86_400.0)))
297        });
298        Ok(Self {
299            method,
300            url: cfg.url,
301            headers: cfg.headers.pairs(),
302            params: cfg.params.pairs(),
303            auth,
304            body: cfg.body.unwrap_or_default(),
305            timeout,
306            ssl_verify: cfg.ssl_verify,
307            fail_on_error_status: cfg.fail_on_error_status,
308            max_response_bytes: cfg.max_response_bytes,
309            retry_non_idempotent: cfg.retry_non_idempotent,
310            parse_json: cfg.parse_json,
311            default_client: OnceLock::new(),
312        })
313    }
314
315    fn client<'a>(&'a self, ctx: &'a NodeContext) -> Result<&'a HttpClient, NodeError> {
316        if let Some(client) = ctx.service::<HttpClient>() {
317            return Ok(client);
318        }
319        self.default_client
320            .get_or_init(|| HttpClient::new(EgressPolicy::default()).map_err(|e| e.to_string()))
321            .as_ref()
322            .map_err(|e| NodeError::fatal("http_client", e.clone()))
323    }
324
325    fn may_retry(&self) -> bool {
326        self.retry_non_idempotent || is_idempotent(&self.method)
327    }
328
329    async fn execute(&self, ctx: &NodeContext) -> Result<NodeOutput, NodeError> {
330        let pool = &ctx.pool;
331        let client = self.client(ctx)?;
332        let policy = client.policy();
333
334        // URL and query parameters.
335        let raw_url = self.url.render_exposed(pool);
336        let mut url = reqwest::Url::parse(raw_url.trim())
337            .map_err(|e| NodeError::fatal("invalid_input", format!("invalid URL `{}`: {e}", self.url.render(pool))))?;
338        if !self.params.is_empty() {
339            let mut query = url.query_pairs_mut();
340            for (k, v) in &self.params {
341                if let Some(key) = render_key(k, pool, "query parameter")? {
342                    query.append_pair(&key, &v.render_exposed(pool));
343                }
344            }
345        }
346        client.check_url(&url).map_err(egress_error)?;
347
348        // Headers: user headers first, then authorization (which wins).
349        let mut headers = HeaderMap::new();
350        let mut sensitive: Vec<String> = CREDENTIAL_HEADERS.iter().map(|h| (*h).to_owned()).collect();
351        let mut shown_headers = IndexMap::new();
352        for (k, v) in &self.headers {
353            let Some(name) = render_key(k, pool, "header")? else { continue };
354            let name = name.trim().to_ascii_lowercase();
355            let exposed = v.render_exposed(pool);
356            let shown = v.render(pool);
357            if exposed != shown {
358                // The value contains a secret: never forward it to another origin.
359                sensitive.push(name.clone());
360            }
361            insert_header(&mut headers, &name, &exposed)?;
362            shown_headers.insert(name, Value::from(shown));
363        }
364        if let Some(auth) = &self.auth {
365            let key = auth.api_key.render_exposed(pool);
366            let (name, value) = match auth.scheme {
367                AuthScheme::Bearer => ("authorization".to_owned(), format!("Bearer {key}")),
368                AuthScheme::Basic => {
369                    let encoded = if key.contains(':') { BASE64.encode(key.as_bytes()) } else { key };
370                    ("authorization".to_owned(), format!("Basic {encoded}"))
371                }
372                AuthScheme::Custom => (auth.header.clone().unwrap_or_default().trim().to_ascii_lowercase(), key),
373            };
374            insert_header(&mut headers, &name, &value)?;
375            sensitive.push(name.clone());
376            shown_headers.insert(name, Value::from(SECRET_MASK));
377        }
378
379        // Body.
380        let body = self.build_body(ctx)?;
381        let body_len = body.as_ref().map_or(0, |(_, b)| b.len());
382        let body = match body {
383            Some((content_type, bytes)) => {
384                if !headers.contains_key(reqwest::header::CONTENT_TYPE) {
385                    insert_header(&mut headers, "content-type", &content_type)?;
386                }
387                Some(bytes)
388            }
389            None => None,
390        };
391
392        let mut shown_url = self.url.render(pool);
393        if url.query().is_some() && !self.params.is_empty() {
394            shown_url = format!("{shown_url}?{}", masked_query(&self.params, pool));
395        }
396
397        // Limits from node config can only lower the operator's caps.
398        let total_timeout = self.timeout.map_or(policy.total_timeout, |t| t.min(policy.total_timeout));
399        let total_timeout = match ctx.remaining() {
400            Some(remaining) => total_timeout.min(remaining),
401            None => total_timeout,
402        };
403        let cap = self.max_response_bytes.map_or(policy.max_response_bytes, |c| c.min(policy.max_response_bytes));
404
405        let prepared = PreparedRequest { method: self.method.clone(), url, headers, sensitive, body };
406        let (response, final_url, redirects) = self.send_following_redirects(client, prepared, total_timeout).await?;
407
408        let request_meta = Value::object([
409            ("method", Value::from(self.method.as_str())),
410            ("url", Value::from(shown_url)),
411            ("headers", Value::Object(Arc::new(shown_headers.into_iter().collect()))),
412            ("body_bytes", Value::Int(i64::try_from(body_len).unwrap_or(i64::MAX))),
413            ("redirects", Value::Int(i64::try_from(redirects).unwrap_or(i64::MAX))),
414        ]);
415        let output = self.read_response(response, cap, &final_url).await?;
416        Ok(output.meta("request", request_meta))
417    }
418
419    /// Send the request, following redirects manually so credentials never cross origins.
420    /// Returns the final response, its URL and the number of redirects followed.
421    async fn send_following_redirects(
422        &self,
423        client: &HttpClient,
424        mut request: PreparedRequest,
425        total_timeout: Duration,
426    ) -> Result<(reqwest::Response, reqwest::Url, usize), NodeError> {
427        let started = std::time::Instant::now();
428        let max_redirects = client.policy().max_redirects;
429        let http = client
430            .manual_redirect_client(self.ssl_verify)
431            .map_err(|e| NodeError::fatal("http_client", e.to_string()))?;
432        let mut redirects = 0usize;
433        loop {
434            let remaining = total_timeout.saturating_sub(started.elapsed());
435            if remaining.is_zero() {
436                let mut err = NodeError::new("timeout", "HTTP request timed out while following redirects");
437                err.retryable = self.may_retry();
438                return Err(err);
439            }
440            let mut builder = http
441                .request(request.method.clone(), request.url.clone())
442                .headers(request.headers.clone())
443                .timeout(remaining);
444            if let Some(bytes) = &request.body {
445                builder = builder.body(bytes.clone());
446            }
447            let response = builder.send().await.map_err(|e| self.transport_error(e))?;
448
449            let status = response.status();
450            let location = response.headers().get(reqwest::header::LOCATION).and_then(|v| v.to_str().ok());
451            let Some(location) = location.filter(|_| status.is_redirection() && max_redirects > 0) else {
452                return Ok((response, request.url, redirects));
453            };
454            if redirects >= max_redirects {
455                return Err(NodeError::fatal(
456                    "too_many_redirects",
457                    format!("stopped after {max_redirects} redirects (policy limit)"),
458                ));
459            }
460            let next = request.url.join(location).map_err(|_| {
461                NodeError::fatal("invalid_response", format!("HTTP {status} with an invalid Location header"))
462            })?;
463            client.check_url(&next).map_err(egress_error)?;
464
465            if next.origin() != request.url.origin() {
466                for name in &request.sensitive {
467                    request.headers.remove(name.as_str());
468                }
469            }
470            let code = status.as_u16();
471            let becomes_get = code == 303 || (matches!(code, 301 | 302) && request.method == Method::POST);
472            if becomes_get {
473                if request.method != Method::HEAD {
474                    request.method = Method::GET;
475                }
476                request.body = None;
477                request.headers.remove(reqwest::header::CONTENT_TYPE);
478                request.headers.remove(reqwest::header::CONTENT_LENGTH);
479            }
480            request.url = next;
481            redirects += 1;
482        }
483    }
484
485    fn build_body(&self, ctx: &NodeContext) -> Result<Option<(String, Vec<u8>)>, NodeError> {
486        let pool = &ctx.pool;
487        Ok(match self.body.kind {
488            BodyKind::None => None,
489            BodyKind::Json => {
490                let value = render_json(&self.body.data.raw(), pool)?;
491                if value.is_null() && self.body.data.raw().source().trim().is_empty() {
492                    None
493                } else {
494                    let bytes = serde_json::to_vec(&value)
495                        .map_err(|e| NodeError::fatal("invalid_input", format!("cannot encode JSON body: {e}")))?;
496                    Some(("application/json".to_owned(), bytes))
497                }
498            }
499            BodyKind::RawText => {
500                Some(("text/plain; charset=utf-8".to_owned(), self.body.data.raw().render_exposed(pool).into_bytes()))
501            }
502            BodyKind::XWwwFormUrlencoded => {
503                let mut form = url::form_urlencoded::Serializer::new(String::new());
504                for item in self.body.data.items() {
505                    if let Some(key) = render_key(&item.key, pool, "form field")? {
506                        form.append_pair(&key, &item.value.render_exposed(pool));
507                    }
508                }
509                Some(("application/x-www-form-urlencoded".to_owned(), form.finish().into_bytes()))
510            }
511            BodyKind::FormData => {
512                let mut fields = Vec::new();
513                for item in self.body.data.items() {
514                    let Some(name) = render_key(&item.key, pool, "form field")? else { continue };
515                    if item.kind.as_deref() == Some("file") {
516                        for file in files_at(pool, item.file.as_ref())? {
517                            fields.push(MultipartField::File {
518                                name: name.clone(),
519                                filename: file.name.clone(),
520                                mime: file.mime_type.clone().unwrap_or_else(|| "application/octet-stream".to_owned()),
521                                bytes: inline_file_bytes(&file)?,
522                            });
523                        }
524                    } else {
525                        fields.push(MultipartField::Text { name, value: item.value.render_exposed(pool) });
526                    }
527                }
528                let boundary = format!("graphrt-{}", ctx.exec_id.replace(|c: char| !c.is_ascii_alphanumeric(), ""));
529                let (content_type, body) = encode_multipart(&fields, &boundary);
530                Some((content_type, body))
531            }
532            BodyKind::Binary => {
533                let selector = self.body.data.items().first().and_then(|i| i.file.as_ref());
534                let file = files_at(pool, selector)?
535                    .into_iter()
536                    .next()
537                    .ok_or_else(|| NodeError::fatal("invalid_input", "binary body requires a file"))?;
538                let mime = file.mime_type.clone().unwrap_or_else(|| "application/octet-stream".to_owned());
539                Some((mime, inline_file_bytes(&file)?))
540            }
541        })
542    }
543
544    async fn read_response(
545        &self,
546        mut response: reqwest::Response,
547        cap: usize,
548        url: &reqwest::Url,
549    ) -> Result<NodeOutput, NodeError> {
550        let status = response.status();
551        // Error responses only need a snippet; reading stops at the cap instead of failing,
552        // so a large 5xx page is still reported as a (retryable) status error.
553        let error_status = self.fail_on_error_status && !status.is_success();
554        if let Some(len) = response.content_length() {
555            if !error_status && usize::try_from(len).map_or(true, |l| l > cap) {
556                return Err(too_large(cap));
557            }
558        }
559        let mut header_map: IndexMap<String, Value> = IndexMap::new();
560        for (name, value) in response.headers() {
561            let text = String::from_utf8_lossy(value.as_bytes()).into_owned();
562            let key = name.as_str().to_owned();
563            let merged = match header_map.get(&key).and_then(Value::as_str) {
564                Some(prev) => format!("{prev}, {text}"),
565                None => text,
566            };
567            header_map.insert(key, Value::from(merged));
568        }
569        let content_type =
570            header_map.get("content-type").and_then(Value::as_str).unwrap_or_default().to_ascii_lowercase();
571        let disposition = header_map.get("content-disposition").and_then(Value::as_str).unwrap_or_default().to_owned();
572        let retry_after_ms = header_map.get("retry-after").and_then(Value::as_str).and_then(parse_retry_after_ms);
573
574        let mut bytes: Vec<u8> = Vec::new();
575        loop {
576            match response.chunk().await {
577                Ok(Some(chunk)) => {
578                    if bytes.len().saturating_add(chunk.len()) > cap {
579                        if error_status {
580                            break;
581                        }
582                        return Err(too_large(cap));
583                    }
584                    bytes.extend_from_slice(&chunk);
585                }
586                Ok(None) => break,
587                Err(e) => return Err(self.transport_error(e)),
588            }
589        }
590
591        if error_status {
592            let snippet: String = String::from_utf8_lossy(&bytes).chars().take(500).collect();
593            let message = format!("HTTP {status} from {}: {snippet}", url.host_str().unwrap_or_default());
594            let retryable_status = status.is_server_error() || matches!(status.as_u16(), 408 | 429);
595            let mut err = NodeError::new("http_status", message);
596            err.retryable = retryable_status && self.may_retry();
597            if matches!(status.as_u16(), 429 | 503) {
598                err.retry_after_ms = retry_after_ms;
599            }
600            return Err(err);
601        }
602
603        let status_code = Value::Int(i64::from(status.as_u16()));
604        let headers = Value::Object(Arc::new(header_map.into_iter().collect()));
605        let mut output = NodeOutput::new().with("status_code", status_code).with("headers", headers);
606
607        let decoded = if is_binary(&content_type, &disposition) { None } else { decode_text(&bytes, &content_type) };
608        match decoded {
609            Some(DecodedText { text, lossy }) => {
610                if is_json_type(&content_type) {
611                    match serde_json::from_str::<serde_json::Value>(&text) {
612                        Ok(json) => output = output.with("json", Value::from_json(json)),
613                        Err(e) if self.parse_json == ParseJson::Strict => {
614                            return Err(NodeError::fatal(
615                                "invalid_response",
616                                format!("response declared as JSON is not valid JSON: {e}"),
617                            ));
618                        }
619                        Err(e) => {
620                            output = output.with("json", Value::Null).meta("json_error", e.to_string());
621                        }
622                    }
623                }
624                if lossy {
625                    output = output.meta("body_lossy", true);
626                }
627                output = output.with("body", text).with("files", Value::array([]));
628            }
629            None => {
630                if bytes.len() > INLINE_FILE_LIMIT {
631                    return Err(NodeError::fatal(
632                        "response_too_large",
633                        format!(
634                            "binary response of {} bytes exceeds the inline file limit of {INLINE_FILE_LIMIT} bytes",
635                            bytes.len()
636                        ),
637                    ));
638                }
639                let mime = content_type.split(';').next().unwrap_or_default().trim();
640                let mime = if mime.is_empty() { "application/octet-stream" } else { mime };
641                let name = filename_from_disposition(&disposition)
642                    .or_else(|| {
643                        url.path_segments().and_then(|mut s| s.next_back()).filter(|s| !s.is_empty()).map(str::to_owned)
644                    })
645                    .unwrap_or_else(|| "download".to_owned());
646                let file = File {
647                    marker: true,
648                    id: None,
649                    extension: name.rsplit_once('.').map(|(_, ext)| ext.to_ascii_lowercase()),
650                    name,
651                    kind: file_kind(mime),
652                    transfer: FileTransfer::Inline,
653                    url: Some(data_url(mime, &bytes)),
654                    reference: None,
655                    mime_type: Some(mime.to_owned()),
656                    size: u64::try_from(bytes.len()).ok(),
657                };
658                output = output.with("body", "").with("files", Value::array([Value::from(file)]));
659            }
660        }
661        Ok(output)
662    }
663
664    /// Map a transport error. The URL is removed from the message: it may contain secrets.
665    fn transport_error(&self, error: reqwest::Error) -> NodeError {
666        let error = error.without_url();
667        if let Some(egress) = find_egress_error(&error) {
668            return egress_error(egress.clone());
669        }
670        if error.is_redirect() {
671            return NodeError::fatal("egress_blocked", error_chain(&error));
672        }
673        let mut err = if error.is_timeout() {
674            NodeError::new("timeout", error_chain(&error))
675        } else if error.is_connect() {
676            // Nothing was sent: retrying is safe for every method.
677            return NodeError::new("connection", error_chain(&error));
678        } else {
679            NodeError::new("http", error_chain(&error))
680        };
681        err.retryable = self.may_retry();
682        err
683    }
684}
685
686impl Node for HttpRequestNode {
687    fn run(&self, ctx: NodeContext) -> BoxFuture<'_, NodeResult> {
688        Box::pin(async move { Ok(self.execute(&ctx).await?.into()) })
689    }
690
691    fn inputs(&self) -> Vec<Selector> {
692        let mut out: Vec<Selector> = self.url.selectors().cloned().collect();
693        for (k, v) in self.headers.iter().chain(self.params.iter()) {
694            out.extend(k.selectors().cloned());
695            out.extend(v.selectors().cloned());
696        }
697        if let Some(auth) = &self.auth {
698            out.extend(auth.api_key.selectors().cloned());
699        }
700        match &self.body.data {
701            BodyData::Text(text) => out.extend(Template::parse(text).selectors().cloned()),
702            BodyData::Items(items) => {
703                for item in items {
704                    out.extend(item.key.selectors().cloned());
705                    out.extend(item.value.selectors().cloned());
706                    out.extend(item.file.iter().cloned());
707                }
708            }
709        }
710        out
711    }
712}
713
714fn parse_method(method: &str) -> Option<Method> {
715    Some(match method.trim().to_ascii_uppercase().as_str() {
716        "GET" => Method::GET,
717        "POST" => Method::POST,
718        "PUT" => Method::PUT,
719        "PATCH" => Method::PATCH,
720        "DELETE" => Method::DELETE,
721        "HEAD" => Method::HEAD,
722        "OPTIONS" => Method::OPTIONS,
723        _ => return None,
724    })
725}
726
727fn is_idempotent(method: &Method) -> bool {
728    matches!(*method, Method::GET | Method::HEAD | Method::OPTIONS | Method::PUT | Method::DELETE)
729}
730
731fn insert_header(headers: &mut HeaderMap, name: &str, value: &str) -> Result<(), NodeError> {
732    let name = HeaderName::from_bytes(name.as_bytes())
733        .map_err(|_| NodeError::fatal("invalid_input", format!("invalid header name `{name}`")))?;
734    let value = HeaderValue::from_str(value.trim())
735        .map_err(|_| NodeError::fatal("invalid_input", format!("invalid value for header `{name}`")))?;
736    headers.insert(name, value);
737    Ok(())
738}
739
740/// Files referenced by a selector (a single file or an array of files).
741///
742/// A `null` value means "no file" (optional input). A selector that does not resolve
743/// at all, or array items that are not files, are errors rather than silently
744/// sending no file part.
745fn files_at(pool: &VariablePool, selector: Option<&Selector>) -> Result<Vec<File>, NodeError> {
746    let selector = selector.ok_or_else(|| NodeError::fatal("invalid_input", "file field has no `file` selector"))?;
747    match pool.get(selector) {
748        Some(Value::File(f)) => Ok(vec![(*f).clone()]),
749        Some(Value::Array(items)) => items
750            .iter()
751            .map(|v| match v {
752                Value::File(f) => Ok((**f).clone()),
753                other => Err(NodeError::fatal(
754                    "invalid_input",
755                    format!("`{selector}` contains a {}, expected only files", other.type_name()),
756                )),
757            })
758            .collect(),
759        Some(Value::Null) => Ok(Vec::new()),
760        None => Err(NodeError::fatal("invalid_input", format!("file variable `{selector}` not found"))),
761        Some(other) => {
762            Err(NodeError::fatal("invalid_input", format!("`{selector}` is a {}, not a file", other.type_name())))
763        }
764    }
765}
766
767/// Query string for display: secret-derived keys and values are masked.
768fn masked_query(params: &[(Template, Template)], pool: &VariablePool) -> String {
769    let mut q = url::form_urlencoded::Serializer::new(String::new());
770    for (k, v) in params {
771        let key = k.render(pool);
772        if !key.is_empty() {
773            q.append_pair(&key, &v.render(pool));
774        }
775    }
776    q.finish()
777}
778
779/// Render the key of a header / parameter / form field.
780///
781/// Returns `Ok(None)` for rows whose key template is blank (empty rows left in a
782/// form), and an error when a non-blank key template renders empty — that usually
783/// means a misspelled or missing variable, which must not silently drop the field.
784fn render_key(template: &Template, pool: &VariablePool, what: &str) -> Result<Option<String>, NodeError> {
785    if template.source().trim().is_empty() {
786        return Ok(None);
787    }
788    let key = template.render_exposed(pool);
789    if key.trim().is_empty() {
790        return Err(NodeError::fatal(
791            "invalid_input",
792            format!("{what} name `{}` rendered empty (missing variable?)", template.source()),
793        ));
794    }
795    Ok(Some(key))
796}
797
798/// Decoded text body.
799struct DecodedText {
800    text: String,
801    /// Some bytes could not be decoded and were replaced.
802    lossy: bool,
803}
804
805/// Decode a response body as text.
806///
807/// Honours the `charset` parameter for UTF-8, ISO-8859-1 / latin-1 and windows-1252.
808/// Bodies with a textual content type are always decoded (lossily when needed); bodies
809/// with an unknown or missing type are text only when they are valid UTF-8, otherwise
810/// `None` (treated as binary).
811fn decode_text(bytes: &[u8], content_type: &str) -> Option<DecodedText> {
812    let charset = charset_of(content_type);
813    match charset.as_deref() {
814        Some("iso-8859-1" | "iso8859-1" | "latin1" | "latin-1" | "l1" | "us-ascii" | "ascii") => {
815            return Some(DecodedText { text: bytes.iter().map(|&b| char::from(b)).collect(), lossy: false });
816        }
817        Some("windows-1252" | "cp1252" | "x-cp1252") => {
818            return Some(DecodedText { text: bytes.iter().map(|&b| windows_1252_char(b)).collect(), lossy: false });
819        }
820        _ => {}
821    }
822    match std::str::from_utf8(bytes) {
823        Ok(text) => Some(DecodedText { text: text.to_owned(), lossy: false }),
824        Err(_) if is_textual(content_type) || charset.is_some() => {
825            Some(DecodedText { text: String::from_utf8_lossy(bytes).into_owned(), lossy: true })
826        }
827        Err(_) => None,
828    }
829}
830
831/// The lower-cased `charset` parameter of a content type.
832fn charset_of(content_type: &str) -> Option<String> {
833    content_type.split(';').skip(1).find_map(|param| {
834        let (name, value) = param.split_once('=')?;
835        name.trim().eq_ignore_ascii_case("charset").then(|| value.trim().trim_matches('"').to_ascii_lowercase())
836    })
837}
838
839/// windows-1252 differs from latin-1 only in 0x80..=0x9F.
840fn windows_1252_char(b: u8) -> char {
841    const HIGH: [char; 32] = [
842        '\u{20AC}', '\u{FFFD}', '\u{201A}', '\u{0192}', '\u{201E}', '\u{2026}', '\u{2020}', '\u{2021}', '\u{02C6}',
843        '\u{2030}', '\u{0160}', '\u{2039}', '\u{0152}', '\u{FFFD}', '\u{017D}', '\u{FFFD}', '\u{FFFD}', '\u{2018}',
844        '\u{2019}', '\u{201C}', '\u{201D}', '\u{2022}', '\u{2013}', '\u{2014}', '\u{02DC}', '\u{2122}', '\u{0161}',
845        '\u{203A}', '\u{0153}', '\u{FFFD}', '\u{017E}', '\u{0178}',
846    ];
847    match b {
848        0x80..=0x9F => HIGH.get(usize::from(b - 0x80)).copied().unwrap_or('\u{FFFD}'),
849        _ => char::from(b),
850    }
851}
852
853/// Content types that are text by definition.
854fn is_textual(content_type: &str) -> bool {
855    let ct = content_type.split(';').next().unwrap_or_default().trim();
856    ct.starts_with("text/")
857        || ct.contains("json")
858        || ct.contains("xml")
859        || ct.contains("javascript")
860        || ct == "application/x-www-form-urlencoded"
861}
862
863fn is_json_type(content_type: &str) -> bool {
864    content_type.split(';').next().unwrap_or_default().trim().contains("json")
865}
866
867fn is_binary(content_type: &str, disposition: &str) -> bool {
868    if disposition.to_ascii_lowercase().contains("attachment") {
869        return true;
870    }
871    let ct = content_type.split(';').next().unwrap_or_default().trim();
872    if ct.is_empty()
873        || ct.starts_with("text/")
874        || ct.contains("json")
875        || ct.contains("xml")
876        || ct.contains("javascript")
877    {
878        return false;
879    }
880    ct.starts_with("image/")
881        || ct.starts_with("audio/")
882        || ct.starts_with("video/")
883        || ct.starts_with("font/")
884        || matches!(
885            ct,
886            "application/octet-stream"
887                | "application/pdf"
888                | "application/zip"
889                | "application/gzip"
890                | "application/x-tar"
891                | "application/msword"
892        )
893        || ct.starts_with("application/vnd.")
894}
895
896fn file_kind(mime: &str) -> FileKind {
897    if mime.starts_with("image/") {
898        FileKind::Image
899    } else if mime.starts_with("audio/") {
900        FileKind::Audio
901    } else if mime.starts_with("video/") {
902        FileKind::Video
903    } else if mime == "application/pdf" || mime.starts_with("application/vnd.") || mime == "application/msword" {
904        FileKind::Document
905    } else {
906        FileKind::Custom
907    }
908}
909
910fn filename_from_disposition(disposition: &str) -> Option<String> {
911    disposition.split(';').find_map(|part| {
912        let part = part.trim();
913        let value = part.strip_prefix("filename=")?;
914        let name = value.trim_matches('"').rsplit(['/', '\\']).next().unwrap_or_default().trim();
915        (!name.is_empty()).then(|| name.to_owned())
916    })
917}
918
919/// Parse a `Retry-After` header given in delay-seconds (the HTTP-date form is
920/// ignored: the node's own retry policy then applies). Capped at one hour.
921fn parse_retry_after_ms(value: &str) -> Option<u64> {
922    let seconds: u64 = value.trim().parse().ok()?;
923    Some(seconds.min(3600).saturating_mul(1000))
924}
925
926fn too_large(cap: usize) -> NodeError {
927    NodeError::fatal("response_too_large", format!("response body exceeds the limit of {cap} bytes"))
928}
929
930fn egress_error(e: EgressError) -> NodeError {
931    NodeError::fatal("egress_blocked", e.to_string())
932}
933
934fn find_egress_error<'a>(error: &'a (dyn std::error::Error + 'static)) -> Option<&'a EgressError> {
935    let mut current: Option<&(dyn std::error::Error + 'static)> = Some(error);
936    while let Some(e) = current {
937        if let Some(egress) = e.downcast_ref::<EgressError>() {
938            return Some(egress);
939        }
940        current = e.source();
941    }
942    None
943}
944
945fn error_chain(error: &(dyn std::error::Error + 'static)) -> String {
946    let mut parts = vec![error.to_string()];
947    let mut current = error.source();
948    while let Some(e) = current {
949        let text = e.to_string();
950        if parts.last() != Some(&text) {
951            parts.push(text);
952        }
953        current = e.source();
954    }
955    parts.join(": ")
956}
957
958#[cfg(test)]
959mod tests {
960    #[test]
961    fn retry_after_parsing() {
962        assert_eq!(super::parse_retry_after_ms(" 2 "), Some(2000));
963        assert_eq!(super::parse_retry_after_ms("999999"), Some(3_600_000));
964        assert_eq!(super::parse_retry_after_ms("Wed, 21 Oct 2015 07:28:00 GMT"), None);
965    }
966
967    use super::*;
968
969    fn doc(config: serde_json::Value) -> NodeDoc {
970        NodeDoc::new("h", KIND, config)
971    }
972
973    #[test]
974    fn kv_text_form() {
975        let spec = KvSpec::Text("a: 1\n\nb:two:three\nflag".into());
976        let pool = VariablePool::new();
977        let pairs: Vec<(String, String)> =
978            spec.pairs().iter().map(|(k, v)| (k.render(&pool), v.render(&pool))).collect();
979        assert_eq!(pairs, vec![("a".into(), "1".into()), ("b".into(), "two:three".into()), ("flag".into(), "".into())]);
980    }
981
982    #[test]
983    fn kv_list_and_map_forms() {
984        let list: KvSpec = serde_json::from_value(serde_json::json!([{"key": "x", "value": "{{#n.v#}}"}])).unwrap();
985        let map: KvSpec = serde_json::from_value(serde_json::json!({"x": "{{#n.v#}}"})).unwrap();
986        let mut pool = VariablePool::new();
987        pool.insert("n", "v", Value::from("val"));
988        for spec in [list, map] {
989            let pairs = spec.pairs();
990            assert_eq!(pairs.len(), 1);
991            assert_eq!(pairs[0].1.render(&pool), "val");
992        }
993    }
994
995    #[test]
996    fn config_validation() {
997        assert!(HttpRequestNode::from_doc(&doc(serde_json::json!({"url": "https://x", "method": "TRACE"}))).is_err());
998        assert!(HttpRequestNode::from_doc(&doc(serde_json::json!({"url": " "}))).is_err());
999        assert!(
1000            HttpRequestNode::from_doc(&doc(serde_json::json!({
1001                "url": "https://x",
1002                "authorization": {"type": "api-key", "config": {"type": "custom", "api_key": "k"}}
1003            })))
1004            .is_err()
1005        );
1006        let n = HttpRequestNode::from_doc(&doc(serde_json::json!({
1007            "url": "https://x", "method": "options", "timeout": {"connect": 1, "read": 2.5, "write": 0}
1008        })))
1009        .unwrap();
1010        assert_eq!(n.method, Method::OPTIONS);
1011        assert_eq!(n.timeout, Some(Duration::from_millis(3500)));
1012    }
1013
1014    #[test]
1015    fn binary_classification() {
1016        assert!(is_binary("image/png", ""));
1017        assert!(is_binary("application/pdf", ""));
1018        assert!(is_binary("text/plain", "attachment; filename=a.txt"));
1019        assert!(!is_binary("application/json; charset=utf-8", ""));
1020        assert!(!is_binary("text/html", ""));
1021        assert!(!is_binary("", ""));
1022        assert_eq!(filename_from_disposition("attachment; filename=\"../r.pdf\""), Some("r.pdf".into()));
1023    }
1024}