typst_kit/
download.rs

1// Acknowledgement:
2// Closely modelled after rustup's `DownloadTracker`.
3// https://github.com/rust-lang/rustup/blob/master/src/cli/download_tracker.rs
4
5//! Helpers for making various web requests with status reporting. These are
6//! primarily used for communicating with package registries.
7
8use std::collections::VecDeque;
9use std::fmt::Debug;
10use std::io::{self, ErrorKind, Read};
11use std::path::PathBuf;
12use std::sync::Arc;
13use std::time::{Duration, Instant};
14
15use ecow::EcoString;
16use native_tls::{Certificate, TlsConnector};
17use once_cell::sync::OnceCell;
18use ureq::Response;
19
20/// Manages progress reporting for downloads.
21pub trait Progress {
22    /// Invoked when a download is started.
23    fn print_start(&mut self);
24
25    /// Invoked repeatedly while a download is ongoing.
26    fn print_progress(&mut self, state: &DownloadState);
27
28    /// Invoked when a download is finished.
29    fn print_finish(&mut self, state: &DownloadState);
30}
31
32/// An implementation of [`Progress`] with no-op reporting, i.e., reporting
33/// events are swallowed.
34pub struct ProgressSink;
35
36impl Progress for ProgressSink {
37    fn print_start(&mut self) {}
38    fn print_progress(&mut self, _: &DownloadState) {}
39    fn print_finish(&mut self, _: &DownloadState) {}
40}
41
42/// The current state of an in progress or finished download.
43#[derive(Debug)]
44pub struct DownloadState {
45    /// The expected amount of bytes to download, `None` if the response header
46    /// was not set.
47    pub content_len: Option<usize>,
48    /// The total amount of downloaded bytes until now.
49    pub total_downloaded: usize,
50    /// A backlog of the amount of downloaded bytes each second.
51    pub bytes_per_second: VecDeque<usize>,
52    /// The download starting instant.
53    pub start_time: Instant,
54}
55
56/// A minimal https client for downloading various resources.
57pub struct Downloader {
58    user_agent: EcoString,
59    cert_path: Option<PathBuf>,
60    cert: OnceCell<Certificate>,
61}
62
63impl Downloader {
64    /// Crates a new downloader with the given user agent and no certificate.
65    pub fn new(user_agent: impl Into<EcoString>) -> Self {
66        Self {
67            user_agent: user_agent.into(),
68            cert_path: None,
69            cert: OnceCell::new(),
70        }
71    }
72
73    /// Crates a new downloader with the given user agent and certificate path.
74    ///
75    /// If the certificate cannot be read it is set to `None`.
76    pub fn with_path(user_agent: impl Into<EcoString>, cert_path: PathBuf) -> Self {
77        Self {
78            user_agent: user_agent.into(),
79            cert_path: Some(cert_path),
80            cert: OnceCell::new(),
81        }
82    }
83
84    /// Crates a new downloader with the given user agent and certificate.
85    pub fn with_cert(user_agent: impl Into<EcoString>, cert: Certificate) -> Self {
86        Self {
87            user_agent: user_agent.into(),
88            cert_path: None,
89            cert: OnceCell::with_value(cert),
90        }
91    }
92
93    /// Returns the certificate this client is using, if a custom certificate
94    /// is used it is loaded on first access.
95    ///
96    /// - Returns `None` if `--cert` and `TYPST_CERT` are not set.
97    /// - Returns `Some(Ok(cert))` if the certificate was loaded successfully.
98    /// - Returns `Some(Err(err))` if an error occurred while loading the certificate.
99    pub fn cert(&self) -> Option<io::Result<&Certificate>> {
100        self.cert_path.as_ref().map(|path| {
101            self.cert.get_or_try_init(|| {
102                let pem = std::fs::read(path)?;
103                Certificate::from_pem(&pem).map_err(io::Error::other)
104            })
105        })
106    }
107
108    /// Download binary data from the given url.
109    #[allow(clippy::result_large_err)]
110    pub fn download(&self, url: &str) -> Result<ureq::Response, ureq::Error> {
111        let mut builder = ureq::AgentBuilder::new();
112        let mut tls = TlsConnector::builder();
113
114        // Set user agent.
115        builder = builder.user_agent(&self.user_agent);
116
117        // Get the network proxy config from the environment and apply it.
118        if let Some(proxy) = env_proxy::for_url_str(url)
119            .to_url()
120            .and_then(|url| ureq::Proxy::new(url).ok())
121        {
122            builder = builder.proxy(proxy);
123        }
124
125        // Apply a custom CA certificate if present.
126        if let Some(cert) = self.cert() {
127            tls.add_root_certificate(cert?.clone());
128        }
129
130        // Configure native TLS.
131        let connector = tls.build().map_err(io::Error::other)?;
132        builder = builder.tls_connector(Arc::new(connector));
133
134        builder.build().get(url).call()
135    }
136
137    /// Download binary data from the given url and report its progress.
138    #[allow(clippy::result_large_err)]
139    pub fn download_with_progress(
140        &self,
141        url: &str,
142        progress: &mut dyn Progress,
143    ) -> Result<Vec<u8>, ureq::Error> {
144        progress.print_start();
145        let response = self.download(url)?;
146        Ok(RemoteReader::from_response(response, progress).download()?)
147    }
148}
149
150impl Debug for Downloader {
151    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
152        f.debug_struct("Downloader")
153            .field("user_agent", &self.user_agent)
154            .field("cert_path", &self.cert_path)
155            .field(
156                "cert",
157                &self
158                    .cert
159                    .get()
160                    .map(|_| typst_utils::debug(|f| write!(f, "Certificate(..)"))),
161            )
162            .finish()
163    }
164}
165
166/// Keep track of this many download speed samples.
167const SAMPLES: usize = 5;
168
169/// A wrapper around [`ureq::Response`] that reads the response body in chunks
170/// over a websocket and reports its progress.
171struct RemoteReader<'p> {
172    /// The reader returned by the ureq::Response.
173    reader: Box<dyn Read + Send + Sync + 'static>,
174    /// The download state, holding download metadata for progress reporting.
175    state: DownloadState,
176    /// The instant at which progress was last reported.
177    last_progress: Option<Instant>,
178    /// A trait object used to report download progress.
179    progress: &'p mut dyn Progress,
180}
181
182impl<'p> RemoteReader<'p> {
183    /// Wraps a [`ureq::Response`] and prepares it for downloading.
184    ///
185    /// The 'Content-Length' header is used as a size hint for read
186    /// optimization, if present.
187    fn from_response(response: Response, progress: &'p mut dyn Progress) -> Self {
188        let content_len: Option<usize> = response
189            .header("Content-Length")
190            .and_then(|header| header.parse().ok());
191
192        Self {
193            reader: response.into_reader(),
194            last_progress: None,
195            state: DownloadState {
196                content_len,
197                total_downloaded: 0,
198                bytes_per_second: VecDeque::with_capacity(SAMPLES),
199                start_time: Instant::now(),
200            },
201            progress,
202        }
203    }
204
205    /// Download the body's content as raw bytes while reporting download
206    /// progress.
207    fn download(mut self) -> io::Result<Vec<u8>> {
208        let mut buffer = vec![0; 8192];
209        let mut data = match self.state.content_len {
210            Some(content_len) => Vec::with_capacity(content_len),
211            None => Vec::with_capacity(8192),
212        };
213
214        let mut downloaded_this_sec = 0;
215        loop {
216            let read = match self.reader.read(&mut buffer) {
217                Ok(0) => break,
218                Ok(n) => n,
219                // If the data is not yet ready but will be available eventually
220                // keep trying until we either get an actual error, receive data
221                // or an Ok(0).
222                Err(ref e) if e.kind() == ErrorKind::Interrupted => continue,
223                Err(e) => return Err(e),
224            };
225
226            data.extend(&buffer[..read]);
227
228            let last_printed = match self.last_progress {
229                Some(prev) => prev,
230                None => {
231                    let current_time = Instant::now();
232                    self.last_progress = Some(current_time);
233                    current_time
234                }
235            };
236            let elapsed = Instant::now().saturating_duration_since(last_printed);
237
238            downloaded_this_sec += read;
239            self.state.total_downloaded += read;
240
241            if elapsed >= Duration::from_secs(1) {
242                if self.state.bytes_per_second.len() == SAMPLES {
243                    self.state.bytes_per_second.pop_back();
244                }
245
246                self.state.bytes_per_second.push_front(downloaded_this_sec);
247                downloaded_this_sec = 0;
248
249                self.progress.print_progress(&self.state);
250                self.last_progress = Some(Instant::now());
251            }
252        }
253
254        self.progress.print_finish(&self.state);
255
256        Ok(data)
257    }
258}
typst_kit/download.rs

typst_kit/
download.rs
