typst_kit/
download.rs

1// Acknowledgement:
2// Closely modelled after rustup's `DownloadTracker`.
3// https://github.com/rust-lang/rustup/blob/master/src/cli/download_tracker.rs
4
5//! Helpers for making various web requests with status reporting. These are
6//! primarily used for communicating with package registries.
7
8use std::collections::VecDeque;
9use std::fmt::Debug;
10use std::io::{self, ErrorKind, Read};
11use std::path::PathBuf;
12use std::sync::Arc;
13use std::time::{Duration, Instant};
14
15use ecow::EcoString;
16use native_tls::{Certificate, TlsConnector};
17use once_cell::sync::OnceCell;
18use ureq::Response;
19
20/// Manages progress reporting for downloads.
21pub trait Progress {
22    /// Invoked when a download is started.
23    fn print_start(&mut self);
24
25    /// Invoked repeatedly while a download is ongoing.
26    fn print_progress(&mut self, state: &DownloadState);
27
28    /// Invoked when a download is finished.
29    fn print_finish(&mut self, state: &DownloadState);
30}
31
32/// An implementation of [`Progress`] with no-op reporting, i.e., reporting
33/// events are swallowed.
34pub struct ProgressSink;
35
36impl Progress for ProgressSink {
37    fn print_start(&mut self) {}
38    fn print_progress(&mut self, _: &DownloadState) {}
39    fn print_finish(&mut self, _: &DownloadState) {}
40}
41
42/// The current state of an in progress or finished download.
43#[derive(Debug)]
44pub struct DownloadState {
45    /// The expected amount of bytes to download, `None` if the response header
46    /// was not set.
47    pub content_len: Option<usize>,
48    /// The total amount of downloaded bytes until now.
49    pub total_downloaded: usize,
50    /// A backlog of the amount of downloaded bytes each second.
51    pub bytes_per_second: VecDeque<usize>,
52    /// The download starting instant.
53    pub start_time: Instant,
54}
55
56/// A minimal https client for downloading various resources.
57pub struct Downloader {
58    user_agent: EcoString,
59    cert_path: Option<PathBuf>,
60    cert: OnceCell<Certificate>,
61}
62
63impl Downloader {
64    /// Crates a new downloader with the given user agent and no certificate.
65    pub fn new(user_agent: impl Into<EcoString>) -> Self {
66        Self {
67            user_agent: user_agent.into(),
68            cert_path: None,
69            cert: OnceCell::new(),
70        }
71    }
72
73    /// Crates a new downloader with the given user agent and certificate path.
74    ///
75    /// If the certificate cannot be read it is set to `None`.
76    pub fn with_path(user_agent: impl Into<EcoString>, cert_path: PathBuf) -> Self {
77        Self {
78            user_agent: user_agent.into(),
79            cert_path: Some(cert_path),
80            cert: OnceCell::new(),
81        }
82    }
83
84    /// Crates a new downloader with the given user agent and certificate.
85    pub fn with_cert(user_agent: impl Into<EcoString>, cert: Certificate) -> Self {
86        Self {
87            user_agent: user_agent.into(),
88            cert_path: None,
89            cert: OnceCell::with_value(cert),
90        }
91    }
92
93    /// Returns the certificate this client is using, if a custom certificate
94    /// is used it is loaded on first access.
95    ///
96    /// - Returns `None` if `--cert` and `TYPST_CERT` are not set.
97    /// - Returns `Some(Ok(cert))` if the certificate was loaded successfully.
98    /// - Returns `Some(Err(err))` if an error occurred while loading the certificate.
99    pub fn cert(&self) -> Option<io::Result<&Certificate>> {
100        self.cert_path.as_ref().map(|path| {
101            self.cert.get_or_try_init(|| {
102                let pem = std::fs::read(path)?;
103                Certificate::from_pem(&pem).map_err(io::Error::other)
104            })
105        })
106    }
107
108    /// Download binary data from the given url.
109    #[allow(clippy::result_large_err)]
110    pub fn download(&self, url: &str) -> Result<ureq::Response, ureq::Error> {
111        let mut builder = ureq::AgentBuilder::new();
112        let mut tls = TlsConnector::builder();
113
114        // Set user agent.
115        builder = builder.user_agent(&self.user_agent);
116
117        // Get the network proxy config from the environment and apply it.
118        if let Some(proxy) = env_proxy::for_url_str(url)
119            .to_url()
120            .and_then(|url| ureq::Proxy::new(url).ok())
121        {
122            builder = builder.proxy(proxy);
123        }
124
125        // Apply a custom CA certificate if present.
126        if let Some(cert) = self.cert() {
127            tls.add_root_certificate(cert?.clone());
128        }
129
130        // Configure native TLS.
131        let connector =
132            tls.build().map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
133        builder = builder.tls_connector(Arc::new(connector));
134
135        builder.build().get(url).call()
136    }
137
138    /// Download binary data from the given url and report its progress.
139    #[allow(clippy::result_large_err)]
140    pub fn download_with_progress(
141        &self,
142        url: &str,
143        progress: &mut dyn Progress,
144    ) -> Result<Vec<u8>, ureq::Error> {
145        progress.print_start();
146        let response = self.download(url)?;
147        Ok(RemoteReader::from_response(response, progress).download()?)
148    }
149}
150
151impl Debug for Downloader {
152    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
153        f.debug_struct("Downloader")
154            .field("user_agent", &self.user_agent)
155            .field("cert_path", &self.cert_path)
156            .field(
157                "cert",
158                &self
159                    .cert
160                    .get()
161                    .map(|_| typst_utils::debug(|f| write!(f, "Certificate(..)"))),
162            )
163            .finish()
164    }
165}
166
167/// Keep track of this many download speed samples.
168const SAMPLES: usize = 5;
169
170/// A wrapper around [`ureq::Response`] that reads the response body in chunks
171/// over a websocket and reports its progress.
172struct RemoteReader<'p> {
173    /// The reader returned by the ureq::Response.
174    reader: Box<dyn Read + Send + Sync + 'static>,
175    /// The download state, holding download metadata for progress reporting.
176    state: DownloadState,
177    /// The instant at which progress was last reported.
178    last_progress: Option<Instant>,
179    /// A trait object used to report download progress.
180    progress: &'p mut dyn Progress,
181}
182
183impl<'p> RemoteReader<'p> {
184    /// Wraps a [`ureq::Response`] and prepares it for downloading.
185    ///
186    /// The 'Content-Length' header is used as a size hint for read
187    /// optimization, if present.
188    fn from_response(response: Response, progress: &'p mut dyn Progress) -> Self {
189        let content_len: Option<usize> = response
190            .header("Content-Length")
191            .and_then(|header| header.parse().ok());
192
193        Self {
194            reader: response.into_reader(),
195            last_progress: None,
196            state: DownloadState {
197                content_len,
198                total_downloaded: 0,
199                bytes_per_second: VecDeque::with_capacity(SAMPLES),
200                start_time: Instant::now(),
201            },
202            progress,
203        }
204    }
205
206    /// Download the body's content as raw bytes while reporting download
207    /// progress.
208    fn download(mut self) -> io::Result<Vec<u8>> {
209        let mut buffer = vec![0; 8192];
210        let mut data = match self.state.content_len {
211            Some(content_len) => Vec::with_capacity(content_len),
212            None => Vec::with_capacity(8192),
213        };
214
215        let mut downloaded_this_sec = 0;
216        loop {
217            let read = match self.reader.read(&mut buffer) {
218                Ok(0) => break,
219                Ok(n) => n,
220                // If the data is not yet ready but will be available eventually
221                // keep trying until we either get an actual error, receive data
222                // or an Ok(0).
223                Err(ref e) if e.kind() == ErrorKind::Interrupted => continue,
224                Err(e) => return Err(e),
225            };
226
227            data.extend(&buffer[..read]);
228
229            let last_printed = match self.last_progress {
230                Some(prev) => prev,
231                None => {
232                    let current_time = Instant::now();
233                    self.last_progress = Some(current_time);
234                    current_time
235                }
236            };
237            let elapsed = Instant::now().saturating_duration_since(last_printed);
238
239            downloaded_this_sec += read;
240            self.state.total_downloaded += read;
241
242            if elapsed >= Duration::from_secs(1) {
243                if self.state.bytes_per_second.len() == SAMPLES {
244                    self.state.bytes_per_second.pop_back();
245                }
246
247                self.state.bytes_per_second.push_front(downloaded_this_sec);
248                downloaded_this_sec = 0;
249
250                self.progress.print_progress(&self.state);
251                self.last_progress = Some(Instant::now());
252            }
253        }
254
255        self.progress.print_finish(&self.state);
256
257        Ok(data)
258    }
259}
typst_kit/download.rs

typst_kit/
download.rs
