vivacity_core/
fetch.rs

1//! Dist downloads, interoperable with Composer's cache: same layout
2//! (`<cache>/files/<vendor>/<pkg>/<sha1-of-url>.zip`), both read AND fed, so
3//! a cache warmed by one serves the other. Minimal v1 auth: `github-oauth`,
4//! `http-basic`, `bearer` (project auth.json, COMPOSER_AUTH, then the
5//! COMPOSER_HOME auth.json). The lock's shasum, when present, is checked on
6//! download AND when reading back from the cache (meta-analysis F7: a shared
7//! cache is read back with suspicion).
8
9use crate::error::{Error, Result};
10use serde_json::Value;
11use sha1::{Digest, Sha1};
12use std::collections::BTreeMap;
13use std::path::{Path, PathBuf};
14
15#[derive(Debug, Default, Clone)]
16pub struct Auth {
17    pub github_oauth: BTreeMap<String, String>,
18    pub http_basic: BTreeMap<String, (String, String)>,
19    pub bearer: BTreeMap<String, String>,
20}
21
22impl Auth {
23    /// Merges (from lowest to highest priority): the COMPOSER_HOME auth.json,
24    /// the COMPOSER_AUTH variable, the project auth.json.
25    pub fn load(project_dir: &Path) -> Auth {
26        let mut auth = Auth::default();
27        if let Some(home) = composer_home() {
28            auth.merge_json_file(&home.join("auth.json"));
29        }
30        if let Ok(env) = std::env::var("COMPOSER_AUTH") {
31            if let Ok(v) = serde_json::from_str::<Value>(&env) {
32                auth.merge_value(&v);
33            }
34        }
35        auth.merge_json_file(&project_dir.join("auth.json"));
36        auth
37    }
38
39    fn merge_json_file(&mut self, path: &Path) {
40        if let Ok(text) = std::fs::read_to_string(path) {
41            if let Ok(v) = serde_json::from_str::<Value>(&text) {
42                self.merge_value(&v);
43            }
44        }
45    }
46
47    fn merge_value(&mut self, v: &Value) {
48        if let Some(map) = v.get("github-oauth").and_then(Value::as_object) {
49            for (host, tok) in map {
50                if let Some(t) = tok.as_str() {
51                    self.github_oauth
52                        .insert(host.to_ascii_lowercase(), t.to_owned());
53                }
54            }
55        }
56        if let Some(map) = v.get("bearer").and_then(Value::as_object) {
57            for (host, tok) in map {
58                if let Some(t) = tok.as_str() {
59                    self.bearer.insert(host.to_ascii_lowercase(), t.to_owned());
60                }
61            }
62        }
63        if let Some(map) = v.get("http-basic").and_then(Value::as_object) {
64            for (host, creds) in map {
65                if let (Some(u), Some(p)) = (
66                    creds.get("username").and_then(Value::as_str),
67                    creds.get("password").and_then(Value::as_str),
68                ) {
69                    self.http_basic
70                        .insert(host.to_ascii_lowercase(), (u.to_owned(), p.to_owned()));
71                }
72            }
73        }
74    }
75
76    /// Value of the Authorization header for this host, if any.
77    pub fn authorization_for(&self, host: &str) -> Option<String> {
78        let host = host.to_ascii_lowercase();
79        // GitHub dists go through api.github.com / codeload.github.com
80        // but the token is stored under github.com.
81        if host == "github.com" || host.ends_with(".github.com") {
82            if let Some(t) = self.github_oauth.get("github.com") {
83                return Some(format!("token {t}"));
84            }
85        }
86        if let Some(t) = self.github_oauth.get(&host) {
87            return Some(format!("token {t}"));
88        }
89        if let Some(t) = self.bearer.get(&host) {
90            return Some(format!("Bearer {t}"));
91        }
92        if let Some((u, p)) = self.http_basic.get(&host) {
93            use base64::Engine as _;
94            let encoded = base64::engine::general_purpose::STANDARD.encode(format!("{u}:{p}"));
95            return Some(format!("Basic {encoded}"));
96        }
97        None
98    }
99}
100
101/// `Factory::useXdg`: true as soon as any `XDG_*` environment variable exists.
102#[cfg(not(windows))]
103fn use_xdg() -> bool {
104    std::env::vars_os().any(|(k, _)| k.to_string_lossy().starts_with("XDG_"))
105}
106
107#[cfg(not(windows))]
108fn user_dir() -> Option<PathBuf> {
109    std::env::var("HOME")
110        .ok()
111        .map(|h| PathBuf::from(h.trim_end_matches('/')))
112}
113
114/// `Factory::getHomeDir` (docs/reference/Factory.php): COMPOSER_HOME, else
115/// on Windows `%APPDATA%/Composer`, else the first existing directory among
116/// `$XDG_CONFIG_HOME/composer` (if XDG is in use) and `~/.composer`, else
117/// the first candidate.
118pub fn composer_home() -> Option<PathBuf> {
119    if let Ok(h) = std::env::var("COMPOSER_HOME") {
120        if !h.is_empty() {
121            return Some(PathBuf::from(h));
122        }
123    }
124    #[cfg(windows)]
125    {
126        // Composer requires APPDATA on Windows (throws otherwise); here it
127        // is None, and the layers above (auth.json, config.json) cope
128        // without it.
129        std::env::var("APPDATA")
130            .ok()
131            .filter(|s| !s.is_empty())
132            .map(|a| PathBuf::from(a.trim_end_matches(['/', '\\'])).join("Composer"))
133    }
134    #[cfg(not(windows))]
135    {
136        let user = user_dir()?;
137        let mut dirs: Vec<PathBuf> = Vec::new();
138        if use_xdg() {
139            let xdg = std::env::var("XDG_CONFIG_HOME")
140                .ok()
141                .filter(|s| !s.is_empty())
142                .map(PathBuf::from)
143                .unwrap_or_else(|| user.join(".config"));
144            dirs.push(xdg.join("composer"));
145        }
146        dirs.push(user.join(".composer"));
147        dirs.iter()
148            .find(|d| d.is_dir())
149            .cloned()
150            .or_else(|| dirs.first().cloned())
151    }
152}
153
154/// `Factory::getCacheDir`: COMPOSER_CACHE_DIR; else `$COMPOSER_HOME/cache`
155/// if COMPOSER_HOME is set; Windows -> `%LOCALAPPDATA%/Composer` (else
156/// `<home>/cache`); Darwin -> `~/Library/Caches/composer`;
157/// `~/.composer/cache` if it exists; XDG -> `$XDG_CACHE_HOME/composer`;
158/// else `<home>/cache`.
159pub fn composer_cache_dir() -> PathBuf {
160    if let Ok(d) = std::env::var("COMPOSER_CACHE_DIR") {
161        if !d.is_empty() {
162            return PathBuf::from(d);
163        }
164    }
165    if let Ok(h) = std::env::var("COMPOSER_HOME") {
166        if !h.is_empty() {
167            return PathBuf::from(h).join("cache");
168        }
169    }
170    #[cfg(windows)]
171    {
172        if let Ok(l) = std::env::var("LOCALAPPDATA") {
173            if !l.is_empty() {
174                return PathBuf::from(l.trim_end_matches(['/', '\\'])).join("Composer");
175            }
176        }
177        composer_home()
178            .unwrap_or_else(|| PathBuf::from("."))
179            .join("cache")
180    }
181    #[cfg(not(windows))]
182    {
183        let user = user_dir().unwrap_or_else(|| PathBuf::from("."));
184        let home = composer_home().unwrap_or_else(|| user.join(".composer"));
185        if cfg!(target_os = "macos") {
186            return user.join("Library/Caches/composer");
187        }
188        if home == user.join(".composer") && home.join("cache").is_dir() {
189            return home.join("cache");
190        }
191        if use_xdg() {
192            let xdg = std::env::var("XDG_CACHE_HOME")
193                .ok()
194                .filter(|s| !s.is_empty())
195                .map(PathBuf::from)
196                .unwrap_or_else(|| user.join(".cache"));
197            return xdg.join("composer");
198        }
199        home.join("cache")
200    }
201}
202
203/// Cache path of a dist, identical to Composer's: sha1 of the FULL URL
204/// (deliberate in Composer: prevents cross-repository poisoning), key
205/// sanitised to `[a-z0-9._/-]`.
206pub fn dist_cache_path(cache_root: &Path, name: &str, url: &str) -> PathBuf {
207    dist_cache_path_of(cache_root, name, url, "zip")
208}
209
210/// `FileDownloader`'s cache key: `<name>/<sha1(url)>.<dist type>` — a
211/// `tar` dist is cached as `.tar`, so Composer and vivacity find each
212/// other's files.
213pub fn dist_cache_path_of(cache_root: &Path, name: &str, url: &str, dist_type: &str) -> PathBuf {
214    let mut h = Sha1::new();
215    h.update(url.as_bytes());
216    let sha: String = h.finalize().iter().map(|b| format!("{b:02x}")).collect();
217    let sane_name: String = name
218        .chars()
219        .map(|c| {
220            let c = c.to_ascii_lowercase();
221            if c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '.' | '_' | '/' | '-') {
222                c
223            } else {
224                '-'
225            }
226        })
227        .collect();
228    cache_root
229        .join("files")
230        .join(sane_name)
231        .join(format!("{sha}.{dist_type}"))
232}
233
234fn sha1_hex(bytes: &[u8]) -> String {
235    let mut h = Sha1::new();
236    h.update(bytes);
237    h.finalize().iter().map(|b| format!("{b:02x}")).collect()
238}
239
240/// Response of a conditional metadata GET.
241#[derive(Debug, Clone)]
242pub enum MetadataResponse {
243    NotModified,
244    NotFound,
245    Body {
246        bytes: Vec<u8>,
247        last_modified: Option<String>,
248    },
249}
250
251pub struct Fetcher {
252    client: reqwest::Client,
253    cache_root: PathBuf,
254    auth: Auth,
255}
256
257#[derive(Debug, Clone, Copy, PartialEq, Eq)]
258pub enum Provenance {
259    Cache,
260    Network,
261}
262
263impl Fetcher {
264    pub fn new(cache_root: PathBuf, auth: Auth) -> Result<Fetcher> {
265        let client = reqwest::Client::builder()
266            .user_agent(format!("vivacity/{}", env!("CARGO_PKG_VERSION")))
267            .build()
268            .map_err(|e| Error::Http {
269                url: "client".to_owned(),
270                message: e.to_string(),
271            })?;
272        Ok(Fetcher {
273            client,
274            cache_root,
275            auth,
276        })
277    }
278
279    /// Bytes of the dist: cache first (shasum re-checked), network otherwise
280    /// (3 attempts, backoff), cache fed through temp+rename.
281    pub async fn dist_bytes(
282        &self,
283        name: &str,
284        url: &str,
285        expected_sha1: Option<&str>,
286        offline: bool,
287    ) -> Result<(Vec<u8>, Provenance)> {
288        self.dist_bytes_of(name, url, "zip", expected_sha1, offline)
289            .await
290    }
291
292    /// `dist_bytes` for a given dist type (the cache file's extension).
293    pub async fn dist_bytes_of(
294        &self,
295        name: &str,
296        url: &str,
297        dist_type: &str,
298        expected_sha1: Option<&str>,
299        offline: bool,
300    ) -> Result<(Vec<u8>, Provenance)> {
301        let cache_path = dist_cache_path_of(&self.cache_root, name, url, dist_type);
302        if let Ok(bytes) = std::fs::read(&cache_path) {
303            match expected_sha1 {
304                Some(exp) if sha1_hex(&bytes) != exp => {
305                    // Corrupted/poisoned cache entry: throw it away.
306                    let _ = std::fs::remove_file(&cache_path);
307                }
308                _ => return Ok((bytes, Provenance::Cache)),
309            }
310        }
311        if offline {
312            return Err(Error::Http {
313                url: url.to_owned(),
314                message: format!("missing cache for {name} in offline mode"),
315            });
316        }
317
318        let mut last_err = String::new();
319        for attempt in 0..3u32 {
320            if attempt > 0 {
321                tokio::time::sleep(std::time::Duration::from_millis(250 * (1 << attempt))).await;
322            }
323            match self.try_download(url).await {
324                Ok(bytes) => {
325                    if let Some(exp) = expected_sha1 {
326                        let actual = sha1_hex(&bytes);
327                        if actual != exp {
328                            return Err(Error::ShasumMismatch {
329                                name: name.to_owned(),
330                                expected: exp.to_owned(),
331                                actual,
332                            });
333                        }
334                    }
335                    if let Some(parent) = cache_path.parent() {
336                        if std::fs::create_dir_all(parent).is_ok() {
337                            let tmp = cache_path.with_extension("zip.vivacity-tmp");
338                            if std::fs::write(&tmp, &bytes).is_ok() {
339                                let _ = std::fs::rename(&tmp, &cache_path);
340                            }
341                        }
342                    }
343                    return Ok((bytes, Provenance::Network));
344                }
345                Err(e) => last_err = e,
346            }
347        }
348        Err(Error::Http {
349            url: url.to_owned(),
350            message: format!("failed after 3 attempts: {last_err}"),
351        })
352    }
353
354    /// Metadata GET (packages.json, p2 files): `Ok(None)` on 404 (unknown
355    /// package, tolerated by Composer), error otherwise; 3 attempts on
356    /// transport errors.
357    pub async fn metadata_bytes(&self, url: &str) -> Result<Option<Vec<u8>>> {
358        match self.metadata_fetch(url, None).await? {
359            MetadataResponse::Body { bytes, .. } => Ok(Some(bytes)),
360            MetadataResponse::NotFound | MetadataResponse::NotModified => Ok(None),
361        }
362    }
363
364    /// `application/x-www-form-urlencoded` POST (the security advisories
365    /// API: `packages[]=...`), 10 s timeout like Composer, a single attempt;
366    /// 404 -> `NotFound`.
367    pub async fn post_form(&self, url: &str, body: &str) -> Result<MetadataResponse> {
368        let mut req = self
369            .client
370            .post(url)
371            .header(
372                reqwest::header::CONTENT_TYPE,
373                "application/x-www-form-urlencoded",
374            )
375            .timeout(std::time::Duration::from_secs(10))
376            .body(body.to_owned());
377        if let Ok(parsed) = reqwest::Url::parse(url) {
378            if let Some(host) = parsed.host_str() {
379                if let Some(authz) = self.auth.authorization_for(host) {
380                    req = req.header(reqwest::header::AUTHORIZATION, authz);
381                }
382            }
383        }
384        let resp = req.send().await.map_err(|e| Error::Http {
385            url: url.to_owned(),
386            message: e.to_string(),
387        })?;
388        if resp.status() == reqwest::StatusCode::NOT_FOUND {
389            return Ok(MetadataResponse::NotFound);
390        }
391        let resp = resp.error_for_status().map_err(|e| Error::Http {
392            url: url.to_owned(),
393            message: e.to_string(),
394        })?;
395        let bytes = resp.bytes().await.map_err(|e| Error::Http {
396            url: url.to_owned(),
397            message: e.to_string(),
398        })?;
399        Ok(MetadataResponse::Body {
400            bytes: bytes.to_vec(),
401            last_modified: None,
402        })
403    }
404
405    /// Conditional metadata GET: `If-Modified-Since` when the cache has a
406    /// date, 304 -> `NotModified`, 404 -> `NotFound`, else the body and the
407    /// `Last-Modified` header; 3 attempts on transport errors.
408    pub async fn metadata_fetch(
409        &self,
410        url: &str,
411        if_modified_since: Option<&str>,
412    ) -> Result<MetadataResponse> {
413        let mut last_err = String::new();
414        for attempt in 0..3u32 {
415            if attempt > 0 {
416                tokio::time::sleep(std::time::Duration::from_millis(250 * (1 << attempt))).await;
417            }
418            let mut req = self.client.get(url);
419            if let Ok(parsed) = reqwest::Url::parse(url) {
420                if let Some(host) = parsed.host_str() {
421                    if let Some(authz) = self.auth.authorization_for(host) {
422                        req = req.header(reqwest::header::AUTHORIZATION, authz);
423                    }
424                }
425            }
426            if let Some(ims) = if_modified_since {
427                req = req.header(reqwest::header::IF_MODIFIED_SINCE, ims);
428            }
429            match req.send().await {
430                Ok(resp) => {
431                    if resp.status() == reqwest::StatusCode::NOT_FOUND {
432                        return Ok(MetadataResponse::NotFound);
433                    }
434                    if resp.status() == reqwest::StatusCode::NOT_MODIFIED {
435                        return Ok(MetadataResponse::NotModified);
436                    }
437                    let last_modified = resp
438                        .headers()
439                        .get(reqwest::header::LAST_MODIFIED)
440                        .and_then(|v| v.to_str().ok())
441                        .map(str::to_owned);
442                    match resp.error_for_status() {
443                        Ok(resp) => match resp.bytes().await {
444                            Ok(b) => {
445                                return Ok(MetadataResponse::Body {
446                                    bytes: b.to_vec(),
447                                    last_modified,
448                                })
449                            }
450                            Err(e) => last_err = e.to_string(),
451                        },
452                        Err(e) => {
453                            // 4xx other than 404 are not retried.
454                            if e.status().is_some_and(|s| s.is_client_error()) {
455                                return Err(Error::Http {
456                                    url: url.to_owned(),
457                                    message: e.to_string(),
458                                });
459                            }
460                            last_err = e.to_string();
461                        }
462                    }
463                }
464                Err(e) => last_err = e.to_string(),
465            }
466        }
467        Err(Error::Http {
468            url: url.to_owned(),
469            message: format!("failed after 3 attempts: {last_err}"),
470        })
471    }
472
473    async fn try_download(&self, url: &str) -> std::result::Result<Vec<u8>, String> {
474        let mut req = self.client.get(url);
475        if let Ok(parsed) = reqwest::Url::parse(url) {
476            if let Some(host) = parsed.host_str() {
477                if let Some(authz) = self.auth.authorization_for(host) {
478                    req = req.header(reqwest::header::AUTHORIZATION, authz);
479                }
480            }
481        }
482        let resp = req.send().await.map_err(|e| e.to_string())?;
483        let resp = resp.error_for_status().map_err(|e| e.to_string())?;
484        Ok(resp.bytes().await.map_err(|e| e.to_string())?.to_vec())
485    }
486}
487
488#[cfg(test)]
489mod tests {
490    use super::*;
491
492    #[test]
493    fn cache_layout_matches_composer() {
494        // Checked against Composer's real cache in M0: the key is
495        // files/<name>/<sha1(url)>.zip.
496        let p = dist_cache_path(
497            Path::new("/c"),
498            "monolog/monolog",
499            "https://api.github.com/repos/Seldaek/monolog/zipball/abc",
500        );
501        // `join` separates with `\` on Windows: compare in normalized form.
502        let s = p.to_string_lossy().replace('\\', "/");
503        assert!(s.starts_with("/c/files/monolog/monolog/"));
504        assert!(s.ends_with(".zip"));
505        assert_eq!(sha1_hex(b"abc"), "a9993e364706816aba3e25717850c26c9cd0d89d");
506    }
507
508    #[test]
509    fn auth_header_selection() {
510        let mut auth = Auth::default();
511        auth.github_oauth
512            .insert("github.com".into(), "ghtok".into());
513        auth.bearer
514            .insert("repo.example.com".into(), "beartok".into());
515        auth.http_basic
516            .insert("basic.example.com".into(), ("user".into(), "pass".into()));
517
518        assert_eq!(
519            auth.authorization_for("github.com").as_deref(),
520            Some("token ghtok")
521        );
522        assert_eq!(
523            auth.authorization_for("codeload.github.com").as_deref(),
524            Some("token ghtok"),
525            "github dists go through codeload"
526        );
527        assert_eq!(
528            auth.authorization_for("api.github.com").as_deref(),
529            Some("token ghtok")
530        );
531        assert_eq!(
532            auth.authorization_for("repo.example.com").as_deref(),
533            Some("Bearer beartok")
534        );
535        assert_eq!(
536            auth.authorization_for("basic.example.com").as_deref(),
537            Some("Basic dXNlcjpwYXNz")
538        );
539        assert_eq!(auth.authorization_for("unknown.example.com"), None);
540    }
541
542    #[test]
543    fn composer_auth_env_shape_is_parsed() {
544        let mut auth = Auth::default();
545        auth.merge_value(&serde_json::json!({
546            "github-oauth": {"github.com": "t1"},
547            "http-basic": {"h": {"username": "u", "password": "p"}},
548            "bearer": {"b": "tk"}
549        }));
550        assert_eq!(auth.github_oauth.len(), 1);
551        assert_eq!(auth.http_basic.len(), 1);
552        assert_eq!(auth.bearer.len(), 1);
553    }
554}
vivacity_core/fetch.rs

vivacity_core/
fetch.rs
