shell-download 0.20.0

Zero-dependency Rust library for downloading a remote URL to a file, string or bytes using commonly-available shell tools.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

use std::path::Path;
use std::process::Command;
use std::sync::{Arc, atomic::AtomicBool};
use std::thread::JoinHandle;

use crate::{DownloadResult, RequestBuilder, ResponseError, StartError, drivers::Driver, util};

#[derive(Debug, Clone, Copy)]
/// `wget` backend.
pub(crate) struct WgetDriver;

impl Driver for WgetDriver {
    /// Start a download using `wget`.
    fn start(
        &self,
        req: RequestBuilder,
        out_path: &Path,
        cancel: Arc<AtomicBool>,
    ) -> Result<JoinHandle<Result<DownloadResult, ResponseError>>, StartError> {
        let mut cmd = Command::new("wget");
        cmd.arg("-O")
            .arg(out_path)
            .arg("--server-response")
            // Avoid reusing a single TCP connection across redirects; ELBs (e.g. httpbin)
            // sometimes return 502 on a stale keep-alive after a redirect chain.
            .arg("--no-http-keep-alive")
            .arg(&req.url);
        if !req.follow_redirects {
            cmd.arg("--max-redirect=0");
        } else {
            // Some wget builds differ in default redirect behavior; set explicitly.
            cmd.arg("--max-redirect=10");
        }
        for (k, v) in util::add_common_headers(&req) {
            cmd.arg("--header").arg(format!("{k}: {v}"));
        }

        let child = util::spawn_child_for_output(cmd, "wget")?;

        Ok(util::spawn_download_thread(
            req,
            out_path,
            cancel,
            move |req, _out, cancel| {
                let output = util::wait_child_with_output(child, cancel, "wget", req.quiet)?;
                let stderr = String::from_utf8_lossy(&output.stderr);
                let mut last_code: Option<u16> = None;
                for line in stderr.lines() {
                    let line = line.trim();
                    if let Some(rest) = line.strip_prefix("HTTP/") {
                        let parts: Vec<&str> = rest.split_whitespace().collect();
                        if parts.len() >= 2 {
                            if let Ok(code) = parts[1].parse::<u16>() {
                                last_code = Some(code);
                            }
                        }
                    }
                }
                Ok((last_code.unwrap_or(200), None))
            },
        ))
    }
}