vivacity_core/
fetch.rs

1//! Dist downloads, interoperable with Composer's cache: same layout
2//! (`<cache>/files/<vendor>/<pkg>/<sha1-of-url>.zip`), both read AND fed, so
3//! a cache warmed by one serves the other. Minimal v1 auth: `github-oauth`,
4//! `http-basic`, `bearer` (project auth.json, COMPOSER_AUTH, then the
5//! COMPOSER_HOME auth.json). The lock's shasum, when present, is checked on
6//! download AND when reading back from the cache (meta-analysis F7: a shared
7//! cache is read back with suspicion).
8
9use crate::error::{Error, Result};
10use serde_json::Value;
11use sha1::{Digest, Sha1};
12use std::collections::BTreeMap;
13use std::path::{Path, PathBuf};
14
15#[derive(Debug, Default, Clone)]
16pub struct Auth {
17    pub github_oauth: BTreeMap<String, String>,
18    pub http_basic: BTreeMap<String, (String, String)>,
19    pub bearer: BTreeMap<String, String>,
20}
21
22impl Auth {
23    /// Merges (from lowest to highest priority): the COMPOSER_HOME auth.json,
24    /// the COMPOSER_AUTH variable, the project auth.json.
25    pub fn load(project_dir: &Path) -> Auth {
26        let mut auth = Auth::default();
27        if let Some(home) = composer_home() {
28            auth.merge_json_file(&home.join("auth.json"));
29        }
30        if let Ok(env) = std::env::var("COMPOSER_AUTH") {
31            if let Ok(v) = serde_json::from_str::<Value>(&env) {
32                auth.merge_value(&v);
33            }
34        }
35        auth.merge_json_file(&project_dir.join("auth.json"));
36        auth
37    }
38
39    fn merge_json_file(&mut self, path: &Path) {
40        if let Ok(text) = std::fs::read_to_string(path) {
41            if let Ok(v) = serde_json::from_str::<Value>(&text) {
42                self.merge_value(&v);
43            }
44        }
45    }
46
47    fn merge_value(&mut self, v: &Value) {
48        if let Some(map) = v.get("github-oauth").and_then(Value::as_object) {
49            for (host, tok) in map {
50                if let Some(t) = tok.as_str() {
51                    self.github_oauth
52                        .insert(host.to_ascii_lowercase(), t.to_owned());
53                }
54            }
55        }
56        if let Some(map) = v.get("bearer").and_then(Value::as_object) {
57            for (host, tok) in map {
58                if let Some(t) = tok.as_str() {
59                    self.bearer.insert(host.to_ascii_lowercase(), t.to_owned());
60                }
61            }
62        }
63        if let Some(map) = v.get("http-basic").and_then(Value::as_object) {
64            for (host, creds) in map {
65                if let (Some(u), Some(p)) = (
66                    creds.get("username").and_then(Value::as_str),
67                    creds.get("password").and_then(Value::as_str),
68                ) {
69                    self.http_basic
70                        .insert(host.to_ascii_lowercase(), (u.to_owned(), p.to_owned()));
71                }
72            }
73        }
74    }
75
76    /// Value of the Authorization header for this host, if any.
77    pub fn authorization_for(&self, host: &str) -> Option<String> {
78        let host = host.to_ascii_lowercase();
79        // GitHub dists go through api.github.com / codeload.github.com
80        // but the token is stored under github.com.
81        if host == "github.com" || host.ends_with(".github.com") {
82            if let Some(t) = self.github_oauth.get("github.com") {
83                return Some(format!("token {t}"));
84            }
85        }
86        if let Some(t) = self.github_oauth.get(&host) {
87            return Some(format!("token {t}"));
88        }
89        if let Some(t) = self.bearer.get(&host) {
90            return Some(format!("Bearer {t}"));
91        }
92        if let Some((u, p)) = self.http_basic.get(&host) {
93            use base64::Engine as _;
94            let encoded = base64::engine::general_purpose::STANDARD.encode(format!("{u}:{p}"));
95            return Some(format!("Basic {encoded}"));
96        }
97        None
98    }
99}
100
101/// `Factory::useXdg`: true as soon as any `XDG_*` environment variable exists.
102#[cfg(not(windows))]
103fn use_xdg() -> bool {
104    std::env::vars_os().any(|(k, _)| k.to_string_lossy().starts_with("XDG_"))
105}
106
107#[cfg(not(windows))]
108fn user_dir() -> Option<PathBuf> {
109    std::env::var("HOME")
110        .ok()
111        .map(|h| PathBuf::from(h.trim_end_matches('/')))
112}
113
114/// `Factory::getHomeDir` (docs/reference/Factory.php): COMPOSER_HOME, else
115/// on Windows `%APPDATA%/Composer`, else the first existing directory among
116/// `$XDG_CONFIG_HOME/composer` (if XDG is in use) and `~/.composer`, else
117/// the first candidate.
118pub fn composer_home() -> Option<PathBuf> {
119    if let Ok(h) = std::env::var("COMPOSER_HOME") {
120        if !h.is_empty() {
121            return Some(PathBuf::from(h));
122        }
123    }
124    #[cfg(windows)]
125    {
126        // Composer requires APPDATA on Windows (throws otherwise); here it
127        // is None, and the layers above (auth.json, config.json) cope
128        // without it.
129        std::env::var("APPDATA")
130            .ok()
131            .filter(|s| !s.is_empty())
132            .map(|a| PathBuf::from(a.trim_end_matches(['/', '\\'])).join("Composer"))
133    }
134    #[cfg(not(windows))]
135    {
136        let user = user_dir()?;
137        let mut dirs: Vec<PathBuf> = Vec::new();
138        if use_xdg() {
139            let xdg = std::env::var("XDG_CONFIG_HOME")
140                .ok()
141                .filter(|s| !s.is_empty())
142                .map(PathBuf::from)
143                .unwrap_or_else(|| user.join(".config"));
144            dirs.push(xdg.join("composer"));
145        }
146        dirs.push(user.join(".composer"));
147        dirs.iter()
148            .find(|d| d.is_dir())
149            .cloned()
150            .or_else(|| dirs.first().cloned())
151    }
152}
153
154/// `Factory::getCacheDir`: COMPOSER_CACHE_DIR; else `$COMPOSER_HOME/cache`
155/// if COMPOSER_HOME is set; Windows -> `%LOCALAPPDATA%/Composer` (else
156/// `<home>/cache`); Darwin -> `~/Library/Caches/composer`;
157/// `~/.composer/cache` if it exists; XDG -> `$XDG_CACHE_HOME/composer`;
158/// else `<home>/cache`.
159pub fn composer_cache_dir() -> PathBuf {
160    if let Ok(d) = std::env::var("COMPOSER_CACHE_DIR") {
161        if !d.is_empty() {
162            return PathBuf::from(d);
163        }
164    }
165    if let Ok(h) = std::env::var("COMPOSER_HOME") {
166        if !h.is_empty() {
167            return PathBuf::from(h).join("cache");
168        }
169    }
170    #[cfg(windows)]
171    {
172        if let Ok(l) = std::env::var("LOCALAPPDATA") {
173            if !l.is_empty() {
174                return PathBuf::from(l.trim_end_matches(['/', '\\'])).join("Composer");
175            }
176        }
177        composer_home()
178            .unwrap_or_else(|| PathBuf::from("."))
179            .join("cache")
180    }
181    #[cfg(not(windows))]
182    {
183        let user = user_dir().unwrap_or_else(|| PathBuf::from("."));
184        let home = composer_home().unwrap_or_else(|| user.join(".composer"));
185        if cfg!(target_os = "macos") {
186            return user.join("Library/Caches/composer");
187        }
188        if home == user.join(".composer") && home.join("cache").is_dir() {
189            return home.join("cache");
190        }
191        if use_xdg() {
192            let xdg = std::env::var("XDG_CACHE_HOME")
193                .ok()
194                .filter(|s| !s.is_empty())
195                .map(PathBuf::from)
196                .unwrap_or_else(|| user.join(".cache"));
197            return xdg.join("composer");
198        }
199        home.join("cache")
200    }
201}
202
203/// Cache path of a dist, identical to Composer's: sha1 of the FULL URL
204/// (deliberate in Composer: prevents cross-repository poisoning), key
205/// sanitised to `[a-z0-9._/-]`.
206pub fn dist_cache_path(cache_root: &Path, name: &str, url: &str) -> PathBuf {
207    let mut h = Sha1::new();
208    h.update(url.as_bytes());
209    let sha: String = h.finalize().iter().map(|b| format!("{b:02x}")).collect();
210    let sane_name: String = name
211        .chars()
212        .map(|c| {
213            let c = c.to_ascii_lowercase();
214            if c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '.' | '_' | '/' | '-') {
215                c
216            } else {
217                '-'
218            }
219        })
220        .collect();
221    cache_root
222        .join("files")
223        .join(sane_name)
224        .join(format!("{sha}.zip"))
225}
226
227fn sha1_hex(bytes: &[u8]) -> String {
228    let mut h = Sha1::new();
229    h.update(bytes);
230    h.finalize().iter().map(|b| format!("{b:02x}")).collect()
231}
232
233/// Response of a conditional metadata GET.
234#[derive(Debug, Clone)]
235pub enum MetadataResponse {
236    NotModified,
237    NotFound,
238    Body {
239        bytes: Vec<u8>,
240        last_modified: Option<String>,
241    },
242}
243
244pub struct Fetcher {
245    client: reqwest::Client,
246    cache_root: PathBuf,
247    auth: Auth,
248}
249
250#[derive(Debug, Clone, Copy, PartialEq, Eq)]
251pub enum Provenance {
252    Cache,
253    Network,
254}
255
256impl Fetcher {
257    pub fn new(cache_root: PathBuf, auth: Auth) -> Result<Fetcher> {
258        let client = reqwest::Client::builder()
259            .user_agent(format!("vivacity/{}", env!("CARGO_PKG_VERSION")))
260            .build()
261            .map_err(|e| Error::Http {
262                url: "client".to_owned(),
263                message: e.to_string(),
264            })?;
265        Ok(Fetcher {
266            client,
267            cache_root,
268            auth,
269        })
270    }
271
272    /// Bytes of the dist: cache first (shasum re-checked), network otherwise
273    /// (3 attempts, backoff), cache fed through temp+rename.
274    pub async fn dist_bytes(
275        &self,
276        name: &str,
277        url: &str,
278        expected_sha1: Option<&str>,
279        offline: bool,
280    ) -> Result<(Vec<u8>, Provenance)> {
281        let cache_path = dist_cache_path(&self.cache_root, name, url);
282        if let Ok(bytes) = std::fs::read(&cache_path) {
283            match expected_sha1 {
284                Some(exp) if sha1_hex(&bytes) != exp => {
285                    // Corrupted/poisoned cache entry: throw it away.
286                    let _ = std::fs::remove_file(&cache_path);
287                }
288                _ => return Ok((bytes, Provenance::Cache)),
289            }
290        }
291        if offline {
292            return Err(Error::Http {
293                url: url.to_owned(),
294                message: format!("missing cache for {name} in offline mode"),
295            });
296        }
297
298        let mut last_err = String::new();
299        for attempt in 0..3u32 {
300            if attempt > 0 {
301                tokio::time::sleep(std::time::Duration::from_millis(250 * (1 << attempt))).await;
302            }
303            match self.try_download(url).await {
304                Ok(bytes) => {
305                    if let Some(exp) = expected_sha1 {
306                        let actual = sha1_hex(&bytes);
307                        if actual != exp {
308                            return Err(Error::ShasumMismatch {
309                                name: name.to_owned(),
310                                expected: exp.to_owned(),
311                                actual,
312                            });
313                        }
314                    }
315                    if let Some(parent) = cache_path.parent() {
316                        if std::fs::create_dir_all(parent).is_ok() {
317                            let tmp = cache_path.with_extension("zip.vivacity-tmp");
318                            if std::fs::write(&tmp, &bytes).is_ok() {
319                                let _ = std::fs::rename(&tmp, &cache_path);
320                            }
321                        }
322                    }
323                    return Ok((bytes, Provenance::Network));
324                }
325                Err(e) => last_err = e,
326            }
327        }
328        Err(Error::Http {
329            url: url.to_owned(),
330            message: format!("failed after 3 attempts: {last_err}"),
331        })
332    }
333
334    /// Metadata GET (packages.json, p2 files): `Ok(None)` on 404 (unknown
335    /// package, tolerated by Composer), error otherwise; 3 attempts on
336    /// transport errors.
337    pub async fn metadata_bytes(&self, url: &str) -> Result<Option<Vec<u8>>> {
338        match self.metadata_fetch(url, None).await? {
339            MetadataResponse::Body { bytes, .. } => Ok(Some(bytes)),
340            MetadataResponse::NotFound | MetadataResponse::NotModified => Ok(None),
341        }
342    }
343
344    /// `application/x-www-form-urlencoded` POST (the security advisories
345    /// API: `packages[]=...`), 10 s timeout like Composer, a single attempt;
346    /// 404 -> `NotFound`.
347    pub async fn post_form(&self, url: &str, body: &str) -> Result<MetadataResponse> {
348        let mut req = self
349            .client
350            .post(url)
351            .header(
352                reqwest::header::CONTENT_TYPE,
353                "application/x-www-form-urlencoded",
354            )
355            .timeout(std::time::Duration::from_secs(10))
356            .body(body.to_owned());
357        if let Ok(parsed) = reqwest::Url::parse(url) {
358            if let Some(host) = parsed.host_str() {
359                if let Some(authz) = self.auth.authorization_for(host) {
360                    req = req.header(reqwest::header::AUTHORIZATION, authz);
361                }
362            }
363        }
364        let resp = req.send().await.map_err(|e| Error::Http {
365            url: url.to_owned(),
366            message: e.to_string(),
367        })?;
368        if resp.status() == reqwest::StatusCode::NOT_FOUND {
369            return Ok(MetadataResponse::NotFound);
370        }
371        let resp = resp.error_for_status().map_err(|e| Error::Http {
372            url: url.to_owned(),
373            message: e.to_string(),
374        })?;
375        let bytes = resp.bytes().await.map_err(|e| Error::Http {
376            url: url.to_owned(),
377            message: e.to_string(),
378        })?;
379        Ok(MetadataResponse::Body {
380            bytes: bytes.to_vec(),
381            last_modified: None,
382        })
383    }
384
385    /// Conditional metadata GET: `If-Modified-Since` when the cache has a
386    /// date, 304 -> `NotModified`, 404 -> `NotFound`, else the body and the
387    /// `Last-Modified` header; 3 attempts on transport errors.
388    pub async fn metadata_fetch(
389        &self,
390        url: &str,
391        if_modified_since: Option<&str>,
392    ) -> Result<MetadataResponse> {
393        let mut last_err = String::new();
394        for attempt in 0..3u32 {
395            if attempt > 0 {
396                tokio::time::sleep(std::time::Duration::from_millis(250 * (1 << attempt))).await;
397            }
398            let mut req = self.client.get(url);
399            if let Ok(parsed) = reqwest::Url::parse(url) {
400                if let Some(host) = parsed.host_str() {
401                    if let Some(authz) = self.auth.authorization_for(host) {
402                        req = req.header(reqwest::header::AUTHORIZATION, authz);
403                    }
404                }
405            }
406            if let Some(ims) = if_modified_since {
407                req = req.header(reqwest::header::IF_MODIFIED_SINCE, ims);
408            }
409            match req.send().await {
410                Ok(resp) => {
411                    if resp.status() == reqwest::StatusCode::NOT_FOUND {
412                        return Ok(MetadataResponse::NotFound);
413                    }
414                    if resp.status() == reqwest::StatusCode::NOT_MODIFIED {
415                        return Ok(MetadataResponse::NotModified);
416                    }
417                    let last_modified = resp
418                        .headers()
419                        .get(reqwest::header::LAST_MODIFIED)
420                        .and_then(|v| v.to_str().ok())
421                        .map(str::to_owned);
422                    match resp.error_for_status() {
423                        Ok(resp) => match resp.bytes().await {
424                            Ok(b) => {
425                                return Ok(MetadataResponse::Body {
426                                    bytes: b.to_vec(),
427                                    last_modified,
428                                })
429                            }
430                            Err(e) => last_err = e.to_string(),
431                        },
432                        Err(e) => {
433                            // 4xx other than 404 are not retried.
434                            if e.status().is_some_and(|s| s.is_client_error()) {
435                                return Err(Error::Http {
436                                    url: url.to_owned(),
437                                    message: e.to_string(),
438                                });
439                            }
440                            last_err = e.to_string();
441                        }
442                    }
443                }
444                Err(e) => last_err = e.to_string(),
445            }
446        }
447        Err(Error::Http {
448            url: url.to_owned(),
449            message: format!("failed after 3 attempts: {last_err}"),
450        })
451    }
452
453    async fn try_download(&self, url: &str) -> std::result::Result<Vec<u8>, String> {
454        let mut req = self.client.get(url);
455        if let Ok(parsed) = reqwest::Url::parse(url) {
456            if let Some(host) = parsed.host_str() {
457                if let Some(authz) = self.auth.authorization_for(host) {
458                    req = req.header(reqwest::header::AUTHORIZATION, authz);
459                }
460            }
461        }
462        let resp = req.send().await.map_err(|e| e.to_string())?;
463        let resp = resp.error_for_status().map_err(|e| e.to_string())?;
464        Ok(resp.bytes().await.map_err(|e| e.to_string())?.to_vec())
465    }
466}
467
468#[cfg(test)]
469mod tests {
470    use super::*;
471
472    #[test]
473    fn cache_layout_matches_composer() {
474        // Checked against Composer's real cache in M0: the key is
475        // files/<name>/<sha1(url)>.zip.
476        let p = dist_cache_path(
477            Path::new("/c"),
478            "monolog/monolog",
479            "https://api.github.com/repos/Seldaek/monolog/zipball/abc",
480        );
481        // `join` separates with `\` on Windows: compare in normalized form.
482        let s = p.to_string_lossy().replace('\\', "/");
483        assert!(s.starts_with("/c/files/monolog/monolog/"));
484        assert!(s.ends_with(".zip"));
485        assert_eq!(sha1_hex(b"abc"), "a9993e364706816aba3e25717850c26c9cd0d89d");
486    }
487
488    #[test]
489    fn auth_header_selection() {
490        let mut auth = Auth::default();
491        auth.github_oauth
492            .insert("github.com".into(), "ghtok".into());
493        auth.bearer
494            .insert("repo.example.com".into(), "beartok".into());
495        auth.http_basic
496            .insert("basic.example.com".into(), ("user".into(), "pass".into()));
497
498        assert_eq!(
499            auth.authorization_for("github.com").as_deref(),
500            Some("token ghtok")
501        );
502        assert_eq!(
503            auth.authorization_for("codeload.github.com").as_deref(),
504            Some("token ghtok"),
505            "github dists go through codeload"
506        );
507        assert_eq!(
508            auth.authorization_for("api.github.com").as_deref(),
509            Some("token ghtok")
510        );
511        assert_eq!(
512            auth.authorization_for("repo.example.com").as_deref(),
513            Some("Bearer beartok")
514        );
515        assert_eq!(
516            auth.authorization_for("basic.example.com").as_deref(),
517            Some("Basic dXNlcjpwYXNz")
518        );
519        assert_eq!(auth.authorization_for("unknown.example.com"), None);
520    }
521
522    #[test]
523    fn composer_auth_env_shape_is_parsed() {
524        let mut auth = Auth::default();
525        auth.merge_value(&serde_json::json!({
526            "github-oauth": {"github.com": "t1"},
527            "http-basic": {"h": {"username": "u", "password": "p"}},
528            "bearer": {"b": "tk"}
529        }));
530        assert_eq!(auth.github_oauth.len(), 1);
531        assert_eq!(auth.http_basic.len(), 1);
532        assert_eq!(auth.bearer.len(), 1);
533    }
534}
vivacity_core/fetch.rs

vivacity_core/
fetch.rs
