vivacity_core/
fetch.rs

1//! Dist downloads, interoperable with Composer's cache: same layout
2//! (`<cache>/files/<vendor>/<pkg>/<sha1-of-url>.zip`), both read AND fed, so
3//! a cache warmed by one serves the other. Minimal v1 auth: `github-oauth`,
4//! `http-basic`, `bearer` (project auth.json, COMPOSER_AUTH, then the
5//! COMPOSER_HOME auth.json). The lock's shasum, when present, is checked on
6//! download AND when reading back from the cache (meta-analysis F7: a shared
7//! cache is read back with suspicion).
8
9use crate::error::{Error, Result};
10use serde_json::Value;
11use sha1::{Digest, Sha1};
12use std::collections::BTreeMap;
13use std::path::{Path, PathBuf};
14
15#[derive(Debug, Default, Clone)]
16pub struct Auth {
17    pub github_oauth: BTreeMap<String, String>,
18    pub http_basic: BTreeMap<String, (String, String)>,
19    pub bearer: BTreeMap<String, String>,
20}
21
22impl Auth {
23    /// Merges (from lowest to highest priority): the COMPOSER_HOME auth.json,
24    /// the COMPOSER_AUTH variable, the project auth.json.
25    pub fn load(project_dir: &Path) -> Auth {
26        let mut auth = Auth::default();
27        if let Some(home) = composer_home() {
28            auth.merge_json_file(&home.join("auth.json"));
29        }
30        if let Ok(env) = std::env::var("COMPOSER_AUTH") {
31            if let Ok(v) = serde_json::from_str::<Value>(&env) {
32                auth.merge_value(&v);
33            }
34        }
35        auth.merge_json_file(&project_dir.join("auth.json"));
36        auth
37    }
38
39    fn merge_json_file(&mut self, path: &Path) {
40        if let Ok(text) = std::fs::read_to_string(path) {
41            if let Ok(v) = serde_json::from_str::<Value>(&text) {
42                self.merge_value(&v);
43            }
44        }
45    }
46
47    fn merge_value(&mut self, v: &Value) {
48        if let Some(map) = v.get("github-oauth").and_then(Value::as_object) {
49            for (host, tok) in map {
50                if let Some(t) = tok.as_str() {
51                    self.github_oauth
52                        .insert(host.to_ascii_lowercase(), t.to_owned());
53                }
54            }
55        }
56        if let Some(map) = v.get("bearer").and_then(Value::as_object) {
57            for (host, tok) in map {
58                if let Some(t) = tok.as_str() {
59                    self.bearer.insert(host.to_ascii_lowercase(), t.to_owned());
60                }
61            }
62        }
63        if let Some(map) = v.get("http-basic").and_then(Value::as_object) {
64            for (host, creds) in map {
65                if let (Some(u), Some(p)) = (
66                    creds.get("username").and_then(Value::as_str),
67                    creds.get("password").and_then(Value::as_str),
68                ) {
69                    self.http_basic
70                        .insert(host.to_ascii_lowercase(), (u.to_owned(), p.to_owned()));
71                }
72            }
73        }
74    }
75
76    /// Value of the Authorization header for this host, if any.
77    pub fn authorization_for(&self, host: &str) -> Option<String> {
78        let host = host.to_ascii_lowercase();
79        // GitHub dists go through api.github.com / codeload.github.com
80        // but the token is stored under github.com.
81        if host == "github.com" || host.ends_with(".github.com") {
82            if let Some(t) = self.github_oauth.get("github.com") {
83                return Some(format!("token {t}"));
84            }
85        }
86        if let Some(t) = self.github_oauth.get(&host) {
87            return Some(format!("token {t}"));
88        }
89        if let Some(t) = self.bearer.get(&host) {
90            return Some(format!("Bearer {t}"));
91        }
92        if let Some((u, p)) = self.http_basic.get(&host) {
93            use base64::Engine as _;
94            let encoded = base64::engine::general_purpose::STANDARD.encode(format!("{u}:{p}"));
95            return Some(format!("Basic {encoded}"));
96        }
97        None
98    }
99}
100
101/// `Factory::useXdg`: true as soon as any `XDG_*` environment variable exists.
102fn use_xdg() -> bool {
103    std::env::vars_os().any(|(k, _)| k.to_string_lossy().starts_with("XDG_"))
104}
105
106fn user_dir() -> Option<PathBuf> {
107    std::env::var("HOME")
108        .ok()
109        .map(|h| PathBuf::from(h.trim_end_matches('/')))
110}
111
112/// `Factory::getHomeDir` (docs/reference/Factory.php): COMPOSER_HOME, else
113/// the first existing directory among `$XDG_CONFIG_HOME/composer` (if XDG is
114/// in use) and `~/.composer`, else the first candidate.
115pub fn composer_home() -> Option<PathBuf> {
116    if let Ok(h) = std::env::var("COMPOSER_HOME") {
117        if !h.is_empty() {
118            return Some(PathBuf::from(h));
119        }
120    }
121    let user = user_dir()?;
122    let mut dirs: Vec<PathBuf> = Vec::new();
123    if use_xdg() {
124        let xdg = std::env::var("XDG_CONFIG_HOME")
125            .ok()
126            .filter(|s| !s.is_empty())
127            .map(PathBuf::from)
128            .unwrap_or_else(|| user.join(".config"));
129        dirs.push(xdg.join("composer"));
130    }
131    dirs.push(user.join(".composer"));
132    dirs.iter()
133        .find(|d| d.is_dir())
134        .cloned()
135        .or_else(|| dirs.first().cloned())
136}
137
138/// `Factory::getCacheDir`: COMPOSER_CACHE_DIR; else `$COMPOSER_HOME/cache`
139/// if COMPOSER_HOME is set; Darwin -> `~/Library/Caches/composer`;
140/// `~/.composer/cache` if it exists; XDG -> `$XDG_CACHE_HOME/composer`;
141/// else `<home>/cache`.
142pub fn composer_cache_dir() -> PathBuf {
143    if let Ok(d) = std::env::var("COMPOSER_CACHE_DIR") {
144        if !d.is_empty() {
145            return PathBuf::from(d);
146        }
147    }
148    if let Ok(h) = std::env::var("COMPOSER_HOME") {
149        if !h.is_empty() {
150            return PathBuf::from(h).join("cache");
151        }
152    }
153    let user = user_dir().unwrap_or_else(|| PathBuf::from("."));
154    let home = composer_home().unwrap_or_else(|| user.join(".composer"));
155    if cfg!(target_os = "macos") {
156        return user.join("Library/Caches/composer");
157    }
158    if home == user.join(".composer") && home.join("cache").is_dir() {
159        return home.join("cache");
160    }
161    if use_xdg() {
162        let xdg = std::env::var("XDG_CACHE_HOME")
163            .ok()
164            .filter(|s| !s.is_empty())
165            .map(PathBuf::from)
166            .unwrap_or_else(|| user.join(".cache"));
167        return xdg.join("composer");
168    }
169    home.join("cache")
170}
171
172/// Cache path of a dist, identical to Composer's: sha1 of the FULL URL
173/// (deliberate in Composer: prevents cross-repository poisoning), key
174/// sanitised to `[a-z0-9._/-]`.
175pub fn dist_cache_path(cache_root: &Path, name: &str, url: &str) -> PathBuf {
176    let mut h = Sha1::new();
177    h.update(url.as_bytes());
178    let sha: String = h.finalize().iter().map(|b| format!("{b:02x}")).collect();
179    let sane_name: String = name
180        .chars()
181        .map(|c| {
182            let c = c.to_ascii_lowercase();
183            if c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '.' | '_' | '/' | '-') {
184                c
185            } else {
186                '-'
187            }
188        })
189        .collect();
190    cache_root
191        .join("files")
192        .join(sane_name)
193        .join(format!("{sha}.zip"))
194}
195
196fn sha1_hex(bytes: &[u8]) -> String {
197    let mut h = Sha1::new();
198    h.update(bytes);
199    h.finalize().iter().map(|b| format!("{b:02x}")).collect()
200}
201
202/// Response of a conditional metadata GET.
203#[derive(Debug, Clone)]
204pub enum MetadataResponse {
205    NotModified,
206    NotFound,
207    Body {
208        bytes: Vec<u8>,
209        last_modified: Option<String>,
210    },
211}
212
213pub struct Fetcher {
214    client: reqwest::Client,
215    cache_root: PathBuf,
216    auth: Auth,
217}
218
219#[derive(Debug, Clone, Copy, PartialEq, Eq)]
220pub enum Provenance {
221    Cache,
222    Network,
223}
224
225impl Fetcher {
226    pub fn new(cache_root: PathBuf, auth: Auth) -> Result<Fetcher> {
227        let client = reqwest::Client::builder()
228            .user_agent(format!("vivacity/{}", env!("CARGO_PKG_VERSION")))
229            .build()
230            .map_err(|e| Error::Http {
231                url: "client".to_owned(),
232                message: e.to_string(),
233            })?;
234        Ok(Fetcher {
235            client,
236            cache_root,
237            auth,
238        })
239    }
240
241    /// Bytes of the dist: cache first (shasum re-checked), network otherwise
242    /// (3 attempts, backoff), cache fed through temp+rename.
243    pub async fn dist_bytes(
244        &self,
245        name: &str,
246        url: &str,
247        expected_sha1: Option<&str>,
248        offline: bool,
249    ) -> Result<(Vec<u8>, Provenance)> {
250        let cache_path = dist_cache_path(&self.cache_root, name, url);
251        if let Ok(bytes) = std::fs::read(&cache_path) {
252            match expected_sha1 {
253                Some(exp) if sha1_hex(&bytes) != exp => {
254                    // Corrupted/poisoned cache entry: throw it away.
255                    let _ = std::fs::remove_file(&cache_path);
256                }
257                _ => return Ok((bytes, Provenance::Cache)),
258            }
259        }
260        if offline {
261            return Err(Error::Http {
262                url: url.to_owned(),
263                message: format!("missing cache for {name} in offline mode"),
264            });
265        }
266
267        let mut last_err = String::new();
268        for attempt in 0..3u32 {
269            if attempt > 0 {
270                tokio::time::sleep(std::time::Duration::from_millis(250 * (1 << attempt))).await;
271            }
272            match self.try_download(url).await {
273                Ok(bytes) => {
274                    if let Some(exp) = expected_sha1 {
275                        let actual = sha1_hex(&bytes);
276                        if actual != exp {
277                            return Err(Error::ShasumMismatch {
278                                name: name.to_owned(),
279                                expected: exp.to_owned(),
280                                actual,
281                            });
282                        }
283                    }
284                    if let Some(parent) = cache_path.parent() {
285                        if std::fs::create_dir_all(parent).is_ok() {
286                            let tmp = cache_path.with_extension("zip.vivacity-tmp");
287                            if std::fs::write(&tmp, &bytes).is_ok() {
288                                let _ = std::fs::rename(&tmp, &cache_path);
289                            }
290                        }
291                    }
292                    return Ok((bytes, Provenance::Network));
293                }
294                Err(e) => last_err = e,
295            }
296        }
297        Err(Error::Http {
298            url: url.to_owned(),
299            message: format!("failed after 3 attempts: {last_err}"),
300        })
301    }
302
303    /// Metadata GET (packages.json, p2 files): `Ok(None)` on 404 (unknown
304    /// package, tolerated by Composer), error otherwise; 3 attempts on
305    /// transport errors.
306    pub async fn metadata_bytes(&self, url: &str) -> Result<Option<Vec<u8>>> {
307        match self.metadata_fetch(url, None).await? {
308            MetadataResponse::Body { bytes, .. } => Ok(Some(bytes)),
309            MetadataResponse::NotFound | MetadataResponse::NotModified => Ok(None),
310        }
311    }
312
313    /// `application/x-www-form-urlencoded` POST (the security advisories
314    /// API: `packages[]=...`), 10 s timeout like Composer, a single attempt;
315    /// 404 -> `NotFound`.
316    pub async fn post_form(&self, url: &str, body: &str) -> Result<MetadataResponse> {
317        let mut req = self
318            .client
319            .post(url)
320            .header(
321                reqwest::header::CONTENT_TYPE,
322                "application/x-www-form-urlencoded",
323            )
324            .timeout(std::time::Duration::from_secs(10))
325            .body(body.to_owned());
326        if let Ok(parsed) = reqwest::Url::parse(url) {
327            if let Some(host) = parsed.host_str() {
328                if let Some(authz) = self.auth.authorization_for(host) {
329                    req = req.header(reqwest::header::AUTHORIZATION, authz);
330                }
331            }
332        }
333        let resp = req.send().await.map_err(|e| Error::Http {
334            url: url.to_owned(),
335            message: e.to_string(),
336        })?;
337        if resp.status() == reqwest::StatusCode::NOT_FOUND {
338            return Ok(MetadataResponse::NotFound);
339        }
340        let resp = resp.error_for_status().map_err(|e| Error::Http {
341            url: url.to_owned(),
342            message: e.to_string(),
343        })?;
344        let bytes = resp.bytes().await.map_err(|e| Error::Http {
345            url: url.to_owned(),
346            message: e.to_string(),
347        })?;
348        Ok(MetadataResponse::Body {
349            bytes: bytes.to_vec(),
350            last_modified: None,
351        })
352    }
353
354    /// Conditional metadata GET: `If-Modified-Since` when the cache has a
355    /// date, 304 -> `NotModified`, 404 -> `NotFound`, else the body and the
356    /// `Last-Modified` header; 3 attempts on transport errors.
357    pub async fn metadata_fetch(
358        &self,
359        url: &str,
360        if_modified_since: Option<&str>,
361    ) -> Result<MetadataResponse> {
362        let mut last_err = String::new();
363        for attempt in 0..3u32 {
364            if attempt > 0 {
365                tokio::time::sleep(std::time::Duration::from_millis(250 * (1 << attempt))).await;
366            }
367            let mut req = self.client.get(url);
368            if let Ok(parsed) = reqwest::Url::parse(url) {
369                if let Some(host) = parsed.host_str() {
370                    if let Some(authz) = self.auth.authorization_for(host) {
371                        req = req.header(reqwest::header::AUTHORIZATION, authz);
372                    }
373                }
374            }
375            if let Some(ims) = if_modified_since {
376                req = req.header(reqwest::header::IF_MODIFIED_SINCE, ims);
377            }
378            match req.send().await {
379                Ok(resp) => {
380                    if resp.status() == reqwest::StatusCode::NOT_FOUND {
381                        return Ok(MetadataResponse::NotFound);
382                    }
383                    if resp.status() == reqwest::StatusCode::NOT_MODIFIED {
384                        return Ok(MetadataResponse::NotModified);
385                    }
386                    let last_modified = resp
387                        .headers()
388                        .get(reqwest::header::LAST_MODIFIED)
389                        .and_then(|v| v.to_str().ok())
390                        .map(str::to_owned);
391                    match resp.error_for_status() {
392                        Ok(resp) => match resp.bytes().await {
393                            Ok(b) => {
394                                return Ok(MetadataResponse::Body {
395                                    bytes: b.to_vec(),
396                                    last_modified,
397                                })
398                            }
399                            Err(e) => last_err = e.to_string(),
400                        },
401                        Err(e) => {
402                            // 4xx other than 404 are not retried.
403                            if e.status().is_some_and(|s| s.is_client_error()) {
404                                return Err(Error::Http {
405                                    url: url.to_owned(),
406                                    message: e.to_string(),
407                                });
408                            }
409                            last_err = e.to_string();
410                        }
411                    }
412                }
413                Err(e) => last_err = e.to_string(),
414            }
415        }
416        Err(Error::Http {
417            url: url.to_owned(),
418            message: format!("failed after 3 attempts: {last_err}"),
419        })
420    }
421
422    async fn try_download(&self, url: &str) -> std::result::Result<Vec<u8>, String> {
423        let mut req = self.client.get(url);
424        if let Ok(parsed) = reqwest::Url::parse(url) {
425            if let Some(host) = parsed.host_str() {
426                if let Some(authz) = self.auth.authorization_for(host) {
427                    req = req.header(reqwest::header::AUTHORIZATION, authz);
428                }
429            }
430        }
431        let resp = req.send().await.map_err(|e| e.to_string())?;
432        let resp = resp.error_for_status().map_err(|e| e.to_string())?;
433        Ok(resp.bytes().await.map_err(|e| e.to_string())?.to_vec())
434    }
435}
436
437#[cfg(test)]
438mod tests {
439    use super::*;
440
441    #[test]
442    fn cache_layout_matches_composer() {
443        // Checked against Composer's real cache in M0: the key is
444        // files/<name>/<sha1(url)>.zip.
445        let p = dist_cache_path(
446            Path::new("/c"),
447            "monolog/monolog",
448            "https://api.github.com/repos/Seldaek/monolog/zipball/abc",
449        );
450        let s = p.to_string_lossy();
451        assert!(s.starts_with("/c/files/monolog/monolog/"));
452        assert!(s.ends_with(".zip"));
453        assert_eq!(sha1_hex(b"abc"), "a9993e364706816aba3e25717850c26c9cd0d89d");
454    }
455
456    #[test]
457    fn auth_header_selection() {
458        let mut auth = Auth::default();
459        auth.github_oauth
460            .insert("github.com".into(), "ghtok".into());
461        auth.bearer
462            .insert("repo.example.com".into(), "beartok".into());
463        auth.http_basic
464            .insert("basic.example.com".into(), ("user".into(), "pass".into()));
465
466        assert_eq!(
467            auth.authorization_for("github.com").as_deref(),
468            Some("token ghtok")
469        );
470        assert_eq!(
471            auth.authorization_for("codeload.github.com").as_deref(),
472            Some("token ghtok"),
473            "github dists go through codeload"
474        );
475        assert_eq!(
476            auth.authorization_for("api.github.com").as_deref(),
477            Some("token ghtok")
478        );
479        assert_eq!(
480            auth.authorization_for("repo.example.com").as_deref(),
481            Some("Bearer beartok")
482        );
483        assert_eq!(
484            auth.authorization_for("basic.example.com").as_deref(),
485            Some("Basic dXNlcjpwYXNz")
486        );
487        assert_eq!(auth.authorization_for("unknown.example.com"), None);
488    }
489
490    #[test]
491    fn composer_auth_env_shape_is_parsed() {
492        let mut auth = Auth::default();
493        auth.merge_value(&serde_json::json!({
494            "github-oauth": {"github.com": "t1"},
495            "http-basic": {"h": {"username": "u", "password": "p"}},
496            "bearer": {"b": "tk"}
497        }));
498        assert_eq!(auth.github_oauth.len(), 1);
499        assert_eq!(auth.http_basic.len(), 1);
500        assert_eq!(auth.bearer.len(), 1);
501    }
502}
vivacity_core/fetch.rs

vivacity_core/
fetch.rs
