urls2disk 0.1.1

Http client to download urls to disk concurrently, optionally converting them to PDF
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98

use std::sync::Arc;

use num_cpus;
use reqwest;

use client::Client;
use error::Result;
use semaphore::Semaphore;
use wkhtmltopdf;

/// A `ClientBuilder` can be used to create a `Client` with custom configuration.
#[derive(Clone, Debug)]
pub struct ClientBuilder {
    max_requests_per_second: usize,
    max_threads_cpu: usize,
    max_threads_io: usize,
    reqwest_client: Option<reqwest::Client>,
    wkhtmltopdf_settings: wkhtmltopdf::Settings,
}

impl Default for ClientBuilder {
    /// Creates a `ClientBuilder` with the following default settings:
    /// * `max_requests_per_second` = `10`
    /// * `max_threads_cpu` = number of logical cores on your machine
    /// * `max_threads_io` = `100`
    /// * `reqwest_client` = default `reqwest::Client` plus `gzip` set to `false` and `timeout` set to `None`
    /// * `wkhtmltopdf_zoom` = `"3.5"` on macOS and `"1.0"` on any other system
    fn default() -> ClientBuilder {
        ClientBuilder {
            max_requests_per_second: 10,
            max_threads_cpu: num_cpus::get(),
            max_threads_io: 100,
            reqwest_client: None,
            wkhtmltopdf_settings: wkhtmltopdf::Settings::default(),
        }
    }
}

impl ClientBuilder {
    /// Set the maximum number of requests per second.
    pub fn set_max_requests_per_second(mut self, max_requests_per_second: usize) -> ClientBuilder {
        self.max_requests_per_second = max_requests_per_second;
        self
    }

    /// Set the maximum number of cpu threads (those used for PDF conversion).
    pub fn set_max_threads_cpu(mut self, max_threads_cpu: usize) -> ClientBuilder {
        self.max_threads_cpu = max_threads_cpu;
        self
    }

    /// Set the maximum number of io threads (those used for downloading bytes).
    pub fn set_max_threads_io(mut self, max_threads_io: usize) -> ClientBuilder {
        self.max_threads_io = max_threads_io;
        self
    }

    /// Provide your own customized `reqwest::Client`.
    pub fn set_reqwest_client(mut self, reqwest_client: reqwest::Client) -> ClientBuilder {
        self.reqwest_client = Some(reqwest_client);
        self
    }

    /// Set wkhtmltopdf setting.
    pub fn set_wkhtmltopdf_setting(mut self, setting: wkhtmltopdf::Setting) -> ClientBuilder {
        self.wkhtmltopdf_settings.set(setting);
        self
    }

    /// Set wkhtmltopdf settings based on provided `Vec` of `wkhtmltopdf::Setting`.
    pub fn set_wkhtmltopdf_settings(mut self, settings: Vec<wkhtmltopdf::Setting>) -> ClientBuilder {
        for setting in settings {
            self.wkhtmltopdf_settings.set(setting);
        }
        self
    }

    /// Returns a `Client` that uses this `ClientBuilder` configuration.
    pub fn build(self) -> Result<Client> {
        let reqwest_client = match self.reqwest_client {
            Some(reqwest_client) => reqwest_client,
            None => reqwest::ClientBuilder::new()
                .gzip(false)
                .timeout(None)
                .build()?,
        };
        let semaphore = Semaphore::new(
            self.max_requests_per_second,
            self.max_threads_cpu,
            self.max_threads_io,
        );
        Ok(Client {
            inner: reqwest_client,
            semaphore: Arc::new(semaphore),
            wkhtmltopdf_settings: self.wkhtmltopdf_settings,
        })
    }
}