nu_command/network/http/
client.rs

1use crate::{
2    formats::value_to_json_value,
3    network::{http::timeout_extractor_reader::UreqTimeoutExtractorReader, tls::tls_config},
4};
5use base64::{
6    Engine, alphabet,
7    engine::{GeneralPurpose, general_purpose::PAD},
8};
9use http::StatusCode;
10use log::error;
11use multipart_rs::MultipartWriter;
12use nu_engine::command_prelude::*;
13use nu_protocol::{ByteStream, LabeledError, PipelineMetadata, Signals, shell_error::io::IoError};
14use serde_json::Value as JsonValue;
15use std::{
16    collections::HashMap,
17    io::Cursor,
18    path::PathBuf,
19    str::FromStr,
20    sync::mpsc::{self, RecvTimeoutError},
21    time::Duration,
22};
23use ureq::{
24    Body, Error, RequestBuilder, ResponseExt, SendBody,
25    typestate::{WithBody, WithoutBody},
26};
27use url::Url;
28
29const HTTP_DOCS: &str = "https://www.nushell.sh/cookbook/http.html";
30
31type Response = http::Response<Body>;
32
33type ContentType = String;
34
35#[derive(Debug, PartialEq, Eq)]
36pub enum BodyType {
37    Json,
38    Form,
39    Multipart,
40    Unknown(Option<ContentType>),
41}
42
43impl From<Option<ContentType>> for BodyType {
44    fn from(content_type: Option<ContentType>) -> Self {
45        match content_type {
46            Some(it) if it.contains("application/json") => BodyType::Json,
47            Some(it) if it.contains("application/x-www-form-urlencoded") => BodyType::Form,
48            Some(it) if it.contains("multipart/form-data") => BodyType::Multipart,
49            Some(it) => BodyType::Unknown(Some(it)),
50            None => BodyType::Unknown(None),
51        }
52    }
53}
54
55trait GetHeader {
56    fn header(&self, key: &str) -> Option<&str>;
57}
58
59impl GetHeader for Response {
60    fn header(&self, key: &str) -> Option<&str> {
61        self.headers().get(key).and_then(|v| {
62            v.to_str()
63                .map_err(|e| log::error!("Invalid header {e:?}"))
64                .ok()
65        })
66    }
67}
68
69#[derive(Clone, Copy, PartialEq)]
70pub enum RedirectMode {
71    Follow,
72    Error,
73    Manual,
74}
75
76impl RedirectMode {
77    pub(crate) const MODES: &[&str] = &["follow", "error", "manual"];
78}
79
80pub fn http_client(
81    allow_insecure: bool,
82    redirect_mode: RedirectMode,
83    engine_state: &EngineState,
84    stack: &mut Stack,
85) -> Result<ureq::Agent, ShellError> {
86    let mut config_builder = ureq::config::Config::builder()
87        .user_agent("nushell")
88        .save_redirect_history(true)
89        .http_status_as_error(false)
90        .max_redirects_will_error(false);
91
92    if let RedirectMode::Manual | RedirectMode::Error = redirect_mode {
93        config_builder = config_builder.max_redirects(0);
94    }
95
96    if let Some(http_proxy) = retrieve_http_proxy_from_env(engine_state, stack)
97        && let Ok(proxy) = ureq::Proxy::new(&http_proxy)
98    {
99        config_builder = config_builder.proxy(Some(proxy));
100    };
101
102    config_builder = config_builder.tls_config(tls_config(allow_insecure)?);
103    Ok(ureq::Agent::new_with_config(config_builder.build()))
104}
105
106pub fn http_parse_url(
107    call: &Call,
108    span: Span,
109    raw_url: Value,
110) -> Result<(String, Url), ShellError> {
111    let mut requested_url = raw_url.coerce_into_string()?;
112    if requested_url.starts_with(':') {
113        requested_url = format!("http://localhost{requested_url}");
114    } else if !requested_url.contains("://") {
115        requested_url = format!("http://{requested_url}");
116    }
117
118    let url = match url::Url::parse(&requested_url) {
119        Ok(u) => u,
120        Err(_e) => {
121            return Err(ShellError::UnsupportedInput {
122                msg: "Incomplete or incorrect URL. Expected a full URL, e.g., https://www.example.com".to_string(),
123                input: format!("value: '{requested_url:?}'"),
124                msg_span: call.head,
125                input_span: span,
126            });
127        }
128    };
129
130    Ok((requested_url, url))
131}
132
133pub fn http_parse_redirect_mode(mode: Option<Spanned<String>>) -> Result<RedirectMode, ShellError> {
134    mode.map_or(Ok(RedirectMode::Follow), |v| match &v.item[..] {
135        "follow" | "f" => Ok(RedirectMode::Follow),
136        "error" | "e" => Ok(RedirectMode::Error),
137        "manual" | "m" => Ok(RedirectMode::Manual),
138        _ => Err(ShellError::TypeMismatch {
139            err_message: "Invalid redirect handling mode".to_string(),
140            span: v.span,
141        }),
142    })
143}
144
145pub fn response_to_buffer(
146    response: Response,
147    engine_state: &EngineState,
148    span: Span,
149) -> PipelineData {
150    // Try to get the size of the file to be downloaded.
151    // This is helpful to show the progress of the stream.
152    let buffer_size = match response.header("content-length") {
153        Some(content_length) => {
154            let content_length = content_length.parse::<u64>().unwrap_or_default();
155
156            if content_length == 0 {
157                None
158            } else {
159                Some(content_length)
160            }
161        }
162        _ => None,
163    };
164
165    // Try to guess whether the response is definitely intended to binary or definitely intended to
166    // be UTF-8 text. Otherwise specify `None` and just guess. This doesn't have to be thorough.
167    let content_type_lowercase = response.header("content-type").map(|s| s.to_lowercase());
168    let response_type = match content_type_lowercase.as_deref() {
169        Some("application/octet-stream") => ByteStreamType::Binary,
170        Some(h) if h.contains("charset=utf-8") => ByteStreamType::String,
171        _ => ByteStreamType::Unknown,
172    };
173
174    // Extract response metadata before consuming the body
175    let metadata = extract_response_metadata(&response, span);
176
177    let reader = UreqTimeoutExtractorReader {
178        r: response.into_body().into_reader(),
179    };
180
181    PipelineData::byte_stream(
182        ByteStream::read(reader, span, engine_state.signals().clone(), response_type)
183            .with_known_size(buffer_size),
184        Some(metadata),
185    )
186}
187
188fn extract_response_metadata(response: &Response, span: Span) -> PipelineMetadata {
189    let status = Value::int(response.status().as_u16().into(), span);
190
191    let headers_value = headers_to_nu(&extract_response_headers(response), span)
192        .and_then(|data| data.into_value(span))
193        .unwrap_or(Value::nothing(span));
194
195    let urls = Value::list(
196        response
197            .get_redirect_history()
198            .into_iter()
199            .flatten()
200            .map(|v| Value::string(v.to_string(), span))
201            .collect(),
202        span,
203    );
204
205    let http_response = Value::record(
206        record! {
207            "status" => status,
208            "headers" => headers_value,
209            "urls" => urls,
210        },
211        span,
212    );
213
214    let mut metadata = PipelineMetadata::default();
215    metadata
216        .custom
217        .insert("http_response".to_string(), http_response);
218    metadata
219}
220
221pub fn request_add_authorization_header<B>(
222    user: Option<String>,
223    password: Option<String>,
224    mut request: RequestBuilder<B>,
225) -> RequestBuilder<B> {
226    let base64_engine = GeneralPurpose::new(&alphabet::STANDARD, PAD);
227
228    let login = match (user, password) {
229        (Some(user), Some(password)) => {
230            let mut enc_str = String::new();
231            base64_engine.encode_string(format!("{user}:{password}"), &mut enc_str);
232            Some(enc_str)
233        }
234        (Some(user), _) => {
235            let mut enc_str = String::new();
236            base64_engine.encode_string(format!("{user}:"), &mut enc_str);
237            Some(enc_str)
238        }
239        (_, Some(password)) => {
240            let mut enc_str = String::new();
241            base64_engine.encode_string(format!(":{password}"), &mut enc_str);
242            Some(enc_str)
243        }
244        _ => None,
245    };
246
247    if let Some(login) = login {
248        request = request.header("Authorization", &format!("Basic {login}"));
249    }
250
251    request
252}
253
254#[derive(Debug)]
255#[allow(clippy::large_enum_variant)]
256pub enum ShellErrorOrRequestError {
257    ShellError(ShellError),
258    RequestError(String, Box<Error>),
259}
260
261impl From<ShellError> for ShellErrorOrRequestError {
262    fn from(error: ShellError) -> Self {
263        ShellErrorOrRequestError::ShellError(error)
264    }
265}
266
267#[derive(Debug)]
268pub enum HttpBody {
269    Value(Value),
270    ByteStream(ByteStream),
271}
272
273pub fn send_request_no_body(
274    request: RequestBuilder<WithoutBody>,
275    span: Span,
276    signals: &Signals,
277) -> (Result<Response, ShellError>, Headers) {
278    let headers = extract_request_headers(&request);
279    let request_url = request.uri_ref().cloned().unwrap_or_default().to_string();
280    let result = send_cancellable_request(&request_url, Box::new(|| request.call()), span, signals)
281        .map_err(|e| request_error_to_shell_error(span, e));
282
283    (result, headers.unwrap_or_default())
284}
285
286// remove once all commands have been migrated
287pub fn send_request(
288    engine_state: &EngineState,
289    request: RequestBuilder<WithBody>,
290    body: HttpBody,
291    content_type: Option<String>,
292    span: Span,
293    signals: &Signals,
294) -> (Result<Response, ShellError>, Headers) {
295    let mut request_headers = Headers::new();
296    let request_url = request.uri_ref().cloned().unwrap_or_default().to_string();
297    // hard code serialize_types to false because closures probably shouldn't be
298    // deserialized for send_request but it's required by send_json_request
299    let serialize_types = false;
300    let response = match body {
301        HttpBody::ByteStream(byte_stream) => {
302            let req = if let Some(content_type) = content_type {
303                request.header("Content-Type", &content_type)
304            } else {
305                request
306            };
307            if let Some(h) = extract_request_headers(&req) {
308                request_headers = h;
309            }
310            send_cancellable_request_bytes(&request_url, req, byte_stream, span, signals)
311        }
312        HttpBody::Value(body) => {
313            let body_type = BodyType::from(content_type);
314
315            // We should set the content_type if there is one available
316            // when the content type is unknown
317            let req = if let BodyType::Unknown(Some(content_type)) = &body_type {
318                request.header("Content-Type", content_type)
319            } else {
320                request
321            };
322
323            if let Some(h) = extract_request_headers(&req) {
324                request_headers = h;
325            }
326
327            match body_type {
328                BodyType::Json => send_json_request(
329                    engine_state,
330                    &request_url,
331                    body,
332                    req,
333                    span,
334                    signals,
335                    serialize_types,
336                ),
337                BodyType::Form => send_form_request(&request_url, body, req, span, signals),
338                BodyType::Multipart => {
339                    send_multipart_request(&request_url, body, req, span, signals)
340                }
341                BodyType::Unknown(_) => {
342                    send_default_request(&request_url, body, req, span, signals)
343                }
344            }
345        }
346    };
347
348    let response = response.map_err(|e| request_error_to_shell_error(span, e));
349
350    (response, request_headers)
351}
352
353fn send_json_request(
354    engine_state: &EngineState,
355    request_url: &str,
356    body: Value,
357    req: RequestBuilder<WithBody>,
358    span: Span,
359    signals: &Signals,
360    serialize_types: bool,
361) -> Result<Response, ShellErrorOrRequestError> {
362    match body {
363        Value::Int { .. } | Value::Float { .. } | Value::List { .. } | Value::Record { .. } => {
364            let data = value_to_json_value(engine_state, &body, span, serialize_types)?;
365            send_cancellable_request(request_url, Box::new(|| req.send_json(data)), span, signals)
366        }
367        // If the body type is string, assume it is string json content.
368        // If parsing fails, just send the raw string
369        Value::String { val: s, .. } => {
370            if let Ok(jvalue) = serde_json::from_str::<JsonValue>(&s) {
371                send_cancellable_request(
372                    request_url,
373                    Box::new(|| req.send_json(jvalue)),
374                    span,
375                    signals,
376                )
377            } else {
378                let data = serde_json::from_str(&s).unwrap_or_else(|_| nu_json::Value::String(s));
379                send_cancellable_request(
380                    request_url,
381                    Box::new(|| req.send_json(data)),
382                    span,
383                    signals,
384                )
385            }
386        }
387        _ => Err(ShellErrorOrRequestError::ShellError(
388            ShellError::TypeMismatch {
389                err_message: format!(
390                    "Accepted types: [int, float, list, string, record]. Check: {HTTP_DOCS}"
391                ),
392                span: body.span(),
393            },
394        )),
395    }
396}
397
398fn send_form_request(
399    request_url: &str,
400    body: Value,
401    req: RequestBuilder<WithBody>,
402    span: Span,
403    signals: &Signals,
404) -> Result<Response, ShellErrorOrRequestError> {
405    let build_request_fn = |data: Vec<(String, String)>| {
406        // coerce `data` into a shape that send_form() is happy with
407        let data = data
408            .iter()
409            .map(|(a, b)| (a.as_str(), b.as_str()))
410            .collect::<Vec<(&str, &str)>>();
411        req.send_form(data)
412    };
413
414    match body {
415        Value::List { ref vals, .. } => {
416            if vals.len() % 2 != 0 {
417                return Err(ShellErrorOrRequestError::ShellError(ShellError::IncorrectValue {
418                    msg: "Body type 'list' for form requests requires paired values. E.g.: [foo, 10]".into(),
419                    val_span: body.span(),
420                    call_span: span,
421                }));
422            }
423
424            let data = vals
425                .chunks(2)
426                .map(|it| Ok((it[0].coerce_string()?, it[1].coerce_string()?)))
427                .collect::<Result<Vec<(String, String)>, ShellErrorOrRequestError>>()?;
428
429            let request_fn = Box::new(|| build_request_fn(data));
430            send_cancellable_request(request_url, request_fn, span, signals)
431        }
432        Value::Record { val, .. } => {
433            let mut data: Vec<(String, String)> = Vec::with_capacity(val.len());
434
435            for (col, val) in val.into_owned() {
436                data.push((col, val.coerce_into_string()?))
437            }
438
439            let request_fn = Box::new(|| build_request_fn(data));
440            send_cancellable_request(request_url, request_fn, span, signals)
441        }
442        _ => Err(ShellErrorOrRequestError::ShellError(
443            ShellError::TypeMismatch {
444                err_message: format!("Accepted types: [list, record]. Check: {HTTP_DOCS}"),
445                span: body.span(),
446            },
447        )),
448    }
449}
450
451fn send_multipart_request(
452    request_url: &str,
453    body: Value,
454    req: RequestBuilder<WithBody>,
455    span: Span,
456    signals: &Signals,
457) -> Result<Response, ShellErrorOrRequestError> {
458    let request_fn = match body {
459        Value::Record { val, .. } => {
460            let mut builder = MultipartWriter::new();
461
462            let err = |e: std::io::Error| {
463                ShellErrorOrRequestError::ShellError(IoError::new(e, span, None).into())
464            };
465
466            for (col, val) in val.into_owned() {
467                if let Value::Binary { val, .. } = val {
468                    let headers = [
469                        "Content-Type: application/octet-stream".to_string(),
470                        "Content-Transfer-Encoding: binary".to_string(),
471                        format!(
472                            "Content-Disposition: form-data; name=\"{col}\"; filename=\"{col}\""
473                        ),
474                        format!("Content-Length: {}", val.len()),
475                    ];
476                    builder
477                        .add(&mut Cursor::new(val), &headers.join("\r\n"))
478                        .map_err(err)?;
479                } else {
480                    let headers = format!(r#"Content-Disposition: form-data; name="{col}""#);
481                    builder
482                        .add(val.coerce_into_string()?.as_bytes(), &headers)
483                        .map_err(err)?;
484                }
485            }
486            builder.finish();
487
488            let (boundary, data) = (builder.boundary, builder.data);
489            let content_type = format!("multipart/form-data; boundary={boundary}");
490
491            move || req.header("Content-Type", &content_type).send(&data)
492        }
493        _ => {
494            return Err(ShellErrorOrRequestError::ShellError(
495                ShellError::TypeMismatch {
496                    err_message: format!("Accepted types: [record]. Check: {HTTP_DOCS}"),
497                    span: body.span(),
498                },
499            ));
500        }
501    };
502    send_cancellable_request(request_url, Box::new(request_fn), span, signals)
503}
504
505fn send_default_request(
506    request_url: &str,
507    body: Value,
508    req: RequestBuilder<WithBody>,
509    span: Span,
510    signals: &Signals,
511) -> Result<Response, ShellErrorOrRequestError> {
512    match body {
513        Value::Binary { val, .. } => {
514            send_cancellable_request(request_url, Box::new(move || req.send(&val)), span, signals)
515        }
516        Value::String { val, .. } => {
517            send_cancellable_request(request_url, Box::new(move || req.send(&val)), span, signals)
518        }
519        _ => Err(ShellErrorOrRequestError::ShellError(
520            ShellError::TypeMismatch {
521                err_message: format!("Accepted types: [binary, string]. Check: {HTTP_DOCS}"),
522                span: body.span(),
523            },
524        )),
525    }
526}
527
528// Helper method used to make blocking HTTP request calls cancellable with ctrl+c
529// ureq functions can block for a long time (default 30s?) while attempting to make an HTTP connection
530fn send_cancellable_request(
531    request_url: &str,
532    request_fn: Box<dyn FnOnce() -> Result<Response, Error> + Sync + Send>,
533    span: Span,
534    signals: &Signals,
535) -> Result<Response, ShellErrorOrRequestError> {
536    let (tx, rx) = mpsc::channel::<Result<Response, Error>>();
537
538    // Make the blocking request on a background thread...
539    std::thread::Builder::new()
540        .name("HTTP requester".to_string())
541        .spawn(move || {
542            let ret = request_fn();
543            let _ = tx.send(ret); // may fail if the user has cancelled the operation
544        })
545        .map_err(|err| {
546            IoError::new_with_additional_context(err, span, None, "Could not spawn HTTP requester")
547        })
548        .map_err(ShellError::from)?;
549
550    // ...and poll the channel for responses
551    loop {
552        signals.check(&span)?;
553
554        // 100ms wait time chosen arbitrarily
555        match rx.recv_timeout(Duration::from_millis(100)) {
556            Ok(result) => {
557                return result.map_err(|e| {
558                    ShellErrorOrRequestError::RequestError(request_url.to_string(), Box::new(e))
559                });
560            }
561            Err(RecvTimeoutError::Timeout) => continue,
562            Err(RecvTimeoutError::Disconnected) => panic!("http response channel disconnected"),
563        }
564    }
565}
566
567// Helper method used to make blocking HTTP request calls cancellable with ctrl+c
568// ureq functions can block for a long time (default 30s?) while attempting to make an HTTP connection
569fn send_cancellable_request_bytes(
570    request_url: &str,
571    request: ureq::RequestBuilder<WithBody>,
572    byte_stream: ByteStream,
573    span: Span,
574    signals: &Signals,
575) -> Result<Response, ShellErrorOrRequestError> {
576    let (tx, rx) = mpsc::channel::<Result<Response, ShellErrorOrRequestError>>();
577    let request_url_string = request_url.to_string();
578
579    // Make the blocking request on a background thread...
580    std::thread::Builder::new()
581        .name("HTTP requester".to_string())
582        .spawn(move || {
583            let ret = byte_stream
584                .reader()
585                .ok_or_else(|| {
586                    ShellErrorOrRequestError::ShellError(ShellError::GenericError {
587                        error: "Could not read byte stream".to_string(),
588                        msg: "".into(),
589                        span: None,
590                        help: None,
591                        inner: vec![],
592                    })
593                })
594                .and_then(|reader| {
595                    request
596                        .send(SendBody::from_owned_reader(reader))
597                        .map_err(|e| {
598                            ShellErrorOrRequestError::RequestError(request_url_string, Box::new(e))
599                        })
600                });
601
602            // may fail if the user has cancelled the operation
603            let _ = tx.send(ret);
604        })
605        .map_err(|err| {
606            IoError::new_with_additional_context(err, span, None, "Could not spawn HTTP requester")
607        })
608        .map_err(ShellError::from)?;
609
610    // ...and poll the channel for responses
611    loop {
612        signals.check(&span)?;
613
614        // 100ms wait time chosen arbitrarily
615        match rx.recv_timeout(Duration::from_millis(100)) {
616            Ok(result) => return result,
617            Err(RecvTimeoutError::Timeout) => continue,
618            Err(RecvTimeoutError::Disconnected) => panic!("http response channel disconnected"),
619        }
620    }
621}
622
623pub fn request_set_timeout<B>(
624    timeout: Option<Value>,
625    mut request: RequestBuilder<B>,
626) -> Result<RequestBuilder<B>, ShellError> {
627    if let Some(timeout) = timeout {
628        let val = timeout.as_duration()?;
629        if val.is_negative() || val < 1 {
630            return Err(ShellError::TypeMismatch {
631                err_message: "Timeout value must be an int and larger than 0".to_string(),
632                span: timeout.span(),
633            });
634        }
635
636        request = request
637            .config()
638            .timeout_global(Some(Duration::from_nanos(val as u64)))
639            .build()
640    }
641
642    Ok(request)
643}
644
645pub fn request_add_custom_headers<B>(
646    headers: Option<Value>,
647    mut request: RequestBuilder<B>,
648) -> Result<RequestBuilder<B>, ShellError> {
649    if let Some(headers) = headers {
650        let mut custom_headers: HashMap<String, Value> = HashMap::new();
651
652        match &headers {
653            Value::Record { val, .. } => {
654                for (k, v) in &**val {
655                    custom_headers.insert(k.to_string(), v.clone());
656                }
657            }
658
659            Value::List { vals: table, .. } => {
660                if table.len() == 1 {
661                    // single row([key1 key2]; [val1 val2])
662                    match &table[0] {
663                        Value::Record { val, .. } => {
664                            for (k, v) in &**val {
665                                custom_headers.insert(k.to_string(), v.clone());
666                            }
667                        }
668
669                        x => {
670                            return Err(ShellError::CantConvert {
671                                to_type: "string list or single row".into(),
672                                from_type: x.get_type().to_string(),
673                                span: headers.span(),
674                                help: None,
675                            });
676                        }
677                    }
678                } else {
679                    // primitive values ([key1 val1 key2 val2])
680                    for row in table.chunks(2) {
681                        if row.len() == 2 {
682                            custom_headers.insert(row[0].coerce_string()?, row[1].clone());
683                        }
684                    }
685                }
686            }
687
688            x => {
689                return Err(ShellError::CantConvert {
690                    to_type: "string list or single row".into(),
691                    from_type: x.get_type().to_string(),
692                    span: headers.span(),
693                    help: None,
694                });
695            }
696        };
697
698        for (k, v) in custom_headers {
699            if let Ok(s) = v.coerce_into_string() {
700                request = request.header(&k, &s);
701            }
702        }
703    }
704
705    Ok(request)
706}
707
708fn handle_status_error(span: Span, requested_url: &str, status: StatusCode) -> ShellError {
709    match status {
710        StatusCode::MOVED_PERMANENTLY => ShellError::NetworkFailure {
711            msg: format!("Resource moved permanently (301): {requested_url:?}"),
712            span,
713        },
714        StatusCode::BAD_REQUEST => ShellError::NetworkFailure {
715            msg: format!("Bad request (400) to {requested_url:?}"),
716            span,
717        },
718        StatusCode::FORBIDDEN => ShellError::NetworkFailure {
719            msg: format!("Access forbidden (403) to {requested_url:?}"),
720            span,
721        },
722        StatusCode::NOT_FOUND => ShellError::NetworkFailure {
723            msg: format!("Requested file not found (404): {requested_url:?}"),
724            span,
725        },
726        StatusCode::REQUEST_TIMEOUT => ShellError::NetworkFailure {
727            msg: format!("Request timeout (408): {requested_url:?}"),
728            span,
729        },
730        c => ShellError::NetworkFailure {
731            msg: format!(
732                "Cannot make request to {:?}. Error is {:?}",
733                requested_url,
734                c.to_string()
735            ),
736            span,
737        },
738    }
739}
740
741fn handle_response_error(span: Span, requested_url: &str, response_err: Error) -> ShellError {
742    match response_err {
743        Error::ConnectionFailed => ShellError::NetworkFailure {
744            msg: format!(
745                "Cannot make request to {requested_url}, there was an error establishing a connection.",
746            ),
747            span,
748        },
749        Error::Timeout(..) => ShellError::Io(IoError::new(
750            ErrorKind::from_std(std::io::ErrorKind::TimedOut),
751            span,
752            None,
753        )),
754        Error::Io(error) => ShellError::Io(IoError::new(error, span, None)),
755        e => ShellError::NetworkFailure {
756            msg: e.to_string(),
757            span,
758        },
759    }
760}
761
762pub struct RequestFlags {
763    pub allow_errors: bool,
764    pub raw: bool,
765    pub full: bool,
766}
767
768fn transform_response_using_content_type(
769    engine_state: &EngineState,
770    stack: &mut Stack,
771    span: Span,
772    requested_url: &str,
773    flags: &RequestFlags,
774    resp: Response,
775    content_type: &str,
776) -> Result<PipelineData, ShellError> {
777    let content_type = mime::Mime::from_str(content_type)
778        // there are invalid content types in the wild, so we try to recover
779        // Example: `Content-Type: "text/plain"; charset="utf8"` (note the quotes)
780        .or_else(|_| mime::Mime::from_str(&content_type.replace('"', "")))
781        .or_else(|_| mime::Mime::from_str("text/plain"))
782        .expect("Failed to parse content type, and failed to default to text/plain");
783
784    let ext = match (content_type.type_(), content_type.subtype()) {
785        (mime::TEXT, mime::PLAIN) => url::Url::parse(requested_url)
786            .map_err(|err| {
787                LabeledError::new(err.to_string())
788                    .with_help("cannot parse")
789                    .with_label(
790                        format!("Cannot parse URL: {requested_url}"),
791                        Span::unknown(),
792                    )
793            })?
794            .path_segments()
795            .and_then(|mut segments| segments.next_back())
796            .and_then(|name| if name.is_empty() { None } else { Some(name) })
797            .and_then(|name| {
798                PathBuf::from(name)
799                    .extension()
800                    .map(|name| name.to_string_lossy().to_string())
801            }),
802        _ => Some(content_type.subtype().to_string()),
803    };
804
805    let output = response_to_buffer(resp, engine_state, span);
806    if flags.raw {
807        Ok(output)
808    } else if let Some(ext) = ext {
809        match engine_state.find_decl(format!("from {ext}").as_bytes(), &[]) {
810            Some(converter_id) => engine_state.get_decl(converter_id).run(
811                engine_state,
812                stack,
813                &Call::new(span),
814                output,
815            ),
816            None => Ok(output),
817        }
818    } else {
819        Ok(output)
820    }
821}
822
823pub fn check_response_redirection(
824    redirect_mode: RedirectMode,
825    span: Span,
826    resp: &Response,
827) -> Result<(), ShellError> {
828    if RedirectMode::Error == redirect_mode && (300..400).contains(&resp.status().as_u16()) {
829        return Err(ShellError::NetworkFailure {
830            msg: format!(
831                "Redirect encountered when redirect handling mode was 'error' ({})",
832                resp.status()
833            ),
834            span,
835        });
836    }
837
838    Ok(())
839}
840
841pub(crate) fn handle_response_status(
842    resp: &Response,
843    redirect_mode: RedirectMode,
844    requested_url: &str,
845    span: Span,
846    allow_errors: bool,
847) -> Result<(), ShellError> {
848    let manual_redirect = redirect_mode == RedirectMode::Manual;
849
850    let is_success = resp.status().is_success()
851        || allow_errors
852        || (resp.status().is_redirection() && manual_redirect);
853    if is_success {
854        Ok(())
855    } else {
856        Err(handle_status_error(span, requested_url, resp.status()))
857    }
858}
859
860pub(crate) struct RequestMetadata<'a> {
861    pub requested_url: &'a str,
862    pub span: Span,
863    pub headers: Headers,
864    pub redirect_mode: RedirectMode,
865    pub flags: RequestFlags,
866}
867
868pub(crate) fn request_handle_response(
869    engine_state: &EngineState,
870    stack: &mut Stack,
871    RequestMetadata {
872        requested_url,
873        span,
874        headers,
875        redirect_mode,
876        flags,
877    }: RequestMetadata,
878
879    resp: Response,
880) -> Result<PipelineData, ShellError> {
881    // #response_to_buffer moves "resp" making it impossible to read headers later.
882    // Wrapping it into a closure to call when needed
883    let mut consume_response_body = |response: Response| {
884        let content_type = response.header("content-type").map(|s| s.to_owned());
885
886        match content_type {
887            Some(content_type) => transform_response_using_content_type(
888                engine_state,
889                stack,
890                span,
891                requested_url,
892                &flags,
893                response,
894                &content_type,
895            ),
896            None => Ok(response_to_buffer(response, engine_state, span)),
897        }
898    };
899    handle_response_status(
900        &resp,
901        redirect_mode,
902        requested_url,
903        span,
904        flags.allow_errors,
905    )?;
906
907    if flags.full {
908        let response_status = resp.status();
909
910        let request_headers_value = headers_to_nu(&headers, span)
911            .and_then(|data| data.into_value(span))
912            .unwrap_or(Value::nothing(span));
913
914        let response_headers_value = headers_to_nu(&extract_response_headers(&resp), span)
915            .and_then(|data| data.into_value(span))
916            .unwrap_or(Value::nothing(span));
917
918        let headers = record! {
919            "request" => request_headers_value,
920            "response" => response_headers_value,
921        };
922        let urls = Value::list(
923            resp.get_redirect_history()
924                .into_iter()
925                .flatten()
926                .map(|v| Value::string(v.to_string(), span))
927                .collect(),
928            span,
929        );
930        let body = consume_response_body(resp)?.into_value(span)?;
931
932        let full_response = Value::record(
933            record! {
934                "urls" => urls,
935                "headers" => Value::record(headers, span),
936                "body" => body,
937                "status" => Value::int(response_status.as_u16().into(), span),
938
939            },
940            span,
941        );
942
943        Ok(full_response.into_pipeline_data())
944    } else {
945        Ok(consume_response_body(resp)?)
946    }
947}
948
949type Headers = HashMap<String, Vec<String>>;
950
951fn extract_request_headers<B>(request: &RequestBuilder<B>) -> Option<Headers> {
952    let headers = request.headers_ref()?;
953    let headers_str = headers
954        .keys()
955        .map(|name| {
956            (
957                name.to_string().clone(),
958                headers
959                    .get_all(name)
960                    .iter()
961                    .filter_map(|v| {
962                        v.to_str()
963                            .map_err(|e| {
964                                error!("Invalid header {name:?}: {e:?}");
965                            })
966                            .ok()
967                            .map(|s| s.to_string())
968                    })
969                    .collect(),
970            )
971        })
972        .collect();
973    Some(headers_str)
974}
975
976pub(crate) fn extract_response_headers(response: &Response) -> Headers {
977    let header_map = response.headers();
978    header_map
979        .keys()
980        .map(|name| {
981            (
982                name.to_string().clone(),
983                header_map
984                    .get_all(name)
985                    .iter()
986                    .filter_map(|v| {
987                        v.to_str()
988                            .map_err(|e| {
989                                error!("Invalid header {name:?}: {e:?}");
990                            })
991                            .ok()
992                            .map(|s| s.to_string())
993                    })
994                    .collect(),
995            )
996        })
997        .collect()
998}
999
1000pub(crate) fn headers_to_nu(headers: &Headers, span: Span) -> Result<PipelineData, ShellError> {
1001    let mut vals = Vec::with_capacity(headers.len());
1002
1003    for (name, values) in headers {
1004        let is_duplicate = vals.iter().any(|val| {
1005            if let Value::Record { val, .. } = val
1006                && let Some((
1007                    _col,
1008                    Value::String {
1009                        val: header_name, ..
1010                    },
1011                )) = val.get_index(0)
1012            {
1013                return name == header_name;
1014            }
1015            false
1016        });
1017        if !is_duplicate {
1018            // A single header can hold multiple values
1019            // This interface is why we needed to check if we've already parsed this header name.
1020            for str_value in values {
1021                let record = record! {
1022                    "name" => Value::string(name, span),
1023                    "value" => Value::string(str_value, span),
1024                };
1025                vals.push(Value::record(record, span));
1026            }
1027        }
1028    }
1029
1030    Ok(Value::list(vals, span).into_pipeline_data())
1031}
1032
1033pub(crate) fn request_error_to_shell_error(span: Span, e: ShellErrorOrRequestError) -> ShellError {
1034    match e {
1035        ShellErrorOrRequestError::ShellError(e) => e,
1036        ShellErrorOrRequestError::RequestError(requested_url, e) => {
1037            handle_response_error(span, &requested_url, *e)
1038        }
1039    }
1040}
1041
1042fn retrieve_http_proxy_from_env(engine_state: &EngineState, stack: &mut Stack) -> Option<String> {
1043    stack
1044        .get_env_var(engine_state, "http_proxy")
1045        .or(stack.get_env_var(engine_state, "HTTP_PROXY"))
1046        .or(stack.get_env_var(engine_state, "https_proxy"))
1047        .or(stack.get_env_var(engine_state, "HTTPS_PROXY"))
1048        .or(stack.get_env_var(engine_state, "ALL_PROXY"))
1049        .cloned()
1050        .and_then(|proxy| proxy.coerce_into_string().ok())
1051}
1052
1053#[cfg(test)]
1054mod test {
1055    use super::*;
1056
1057    #[test]
1058    fn test_body_type_from_content_type() {
1059        let json = Some("application/json".to_string());
1060        assert_eq!(BodyType::Json, BodyType::from(json));
1061
1062        // while the charset wont' be passed as we are allowing serde and the library to control
1063        // this, it still shouldn't be missed as json if passed in.
1064        let json_with_charset = Some("application/json; charset=utf-8".to_string());
1065        assert_eq!(BodyType::Json, BodyType::from(json_with_charset));
1066
1067        let form = Some("application/x-www-form-urlencoded".to_string());
1068        assert_eq!(BodyType::Form, BodyType::from(form));
1069
1070        let multipart = Some("multipart/form-data".to_string());
1071        assert_eq!(BodyType::Multipart, BodyType::from(multipart));
1072
1073        let unknown = Some("application/octet-stream".to_string());
1074        assert_eq!(BodyType::Unknown(unknown.clone()), BodyType::from(unknown));
1075
1076        let none = None;
1077        assert_eq!(BodyType::Unknown(none.clone()), BodyType::from(none));
1078    }
1079}
nu_command/network/http/client.rs

nu_command/network/http/
client.rs
