repro_env/resolver/
alpine.rs

1use crate::args;
2use crate::container::{self, Container};
3use crate::errors::*;
4use crate::http;
5use crate::lockfile::{ContainerLock, PackageLock};
6use crate::manifest::PackagesManifest;
7use crate::paths;
8use crate::utils;
9use data_encoding::BASE64;
10use flate2::bufread::GzDecoder;
11use sha1::Sha1;
12use sha2::{Digest, Sha256};
13use std::collections::{HashMap, HashSet};
14use std::io::{BufRead, BufReader, Read};
15use std::rc::Rc;
16use tokio::fs;
17
18pub fn decode_apk_checksum(checksum: &str) -> Result<Vec<u8>> {
19    let checksum = checksum
20        .strip_prefix("Q1")
21        .with_context(|| anyhow!("Only checksums starting with Q1 are supported: {checksum:?}"))?;
22    let checksum = BASE64
23        .decode(checksum.as_bytes())
24        .context("Failed to decode checksum as base64")?;
25    Ok(checksum)
26}
27
28#[derive(Debug, Default)]
29pub struct DatabaseCache {
30    repos: HashMap<String, Rc<String>>,
31    pkgs: HashMap<String, CacheEntry>,
32}
33
34#[derive(Debug)]
35pub struct CacheEntry {
36    name: String,
37    version: String,
38    arch: String,
39    provides: Vec<String>,
40    checksum: String,
41    repo_url: Rc<String>,
42}
43
44pub struct CacheEntryDraft {
45    pub name: Option<String>,
46    pub version: Option<String>,
47    pub arch: Option<String>,
48    pub provides: Vec<String>,
49    pub checksum: Option<String>,
50    pub repo_url: Rc<String>,
51}
52
53impl TryFrom<CacheEntryDraft> for CacheEntry {
54    type Error = Error;
55
56    fn try_from(draft: CacheEntryDraft) -> Result<Self> {
57        Ok(Self {
58            name: draft.name.context("Missing name field")?,
59            version: draft.version.context("Missing version field")?,
60            arch: draft.arch.context("Missing arch field")?,
61            provides: draft.provides,
62            checksum: draft.checksum.context("Missing checksum field")?,
63            repo_url: draft.repo_url,
64        })
65    }
66}
67
68impl CacheEntryDraft {
69    pub fn new(repo_url: Rc<String>) -> Self {
70        CacheEntryDraft {
71            name: None,
72            version: None,
73            arch: None,
74            provides: vec![],
75            checksum: None,
76            repo_url,
77        }
78    }
79}
80
81impl DatabaseCache {
82    pub fn get(&self, id: &str) -> Result<&CacheEntry> {
83        let entry = self
84            .pkgs
85            .get(id)
86            .context("Failed to find package database entry for: {id:?}")?;
87        Ok(entry)
88    }
89
90    pub fn read_apkindex_text<R: Read>(&mut self, r: R, repo_url: &Rc<String>) -> Result<()> {
91        let reader = BufReader::new(r);
92        let mut draft = CacheEntryDraft::new(repo_url.clone());
93        for line in reader.lines() {
94            let line = line?;
95            if line.is_empty() {
96                let mut new = CacheEntryDraft::new(repo_url.clone());
97                (new, draft) = (draft, new);
98                let pkg = CacheEntry::try_from(new)?;
99                let id = format!("{}-{}", pkg.name, pkg.version);
100                trace!("Inserting pkg into lookup table: {id:?} => {pkg:?}");
101                self.pkgs.insert(id, pkg);
102            } else if let Some((key, value)) = line.split_once(':') {
103                match key {
104                    "P" => {
105                        trace!("Package name: {value:?}");
106                        draft.name = Some(value.to_string());
107                    }
108                    "V" => {
109                        trace!("Package version: {value:?}");
110                        draft.version = Some(value.to_string());
111                    }
112                    "C" => {
113                        trace!("Package checksum: {value:?}");
114                        let checksum = decode_apk_checksum(value)?;
115                        draft.checksum = Some(hex::encode(checksum));
116                    }
117                    "A" => {
118                        trace!("Package architecture: {value:?}");
119                        draft.arch = Some(value.to_string());
120                    }
121                    "p" => {
122                        trace!("Package provides: {value:?}");
123                        for entry in value.split(' ') {
124                            let (name, _) = entry.split_once('=').unwrap_or((entry, ""));
125                            draft.provides.push(name.to_string());
126                        }
127                    }
128                    _ => trace!("Ignoring APKINDEX value key={key:?}, value={value:?}"),
129                }
130            } else {
131                bail!("Invalid line in index: {line:?}");
132            }
133        }
134        Ok(())
135    }
136
137    pub fn read_apkindex_container<R: Read>(&mut self, r: R, repo_url: &Rc<String>) -> Result<()> {
138        let mut r = BufReader::new(r);
139        utils::read_gzip_to_end(&mut r).context("Failed to strip signature")?;
140
141        let gz = GzDecoder::new(r);
142        let mut tar = tar::Archive::new(gz);
143
144        for entry in tar.entries()? {
145            let entry = entry?;
146            if entry.header().entry_type() == tar::EntryType::Regular {
147                let path = entry.path()?;
148                if path.to_str() == Some("APKINDEX") {
149                    self.read_apkindex_text(entry, repo_url)?;
150                }
151            }
152        }
153
154        Ok(())
155    }
156
157    pub fn import_from_container(&mut self, buf: &[u8]) -> Result<()> {
158        let mut tar = tar::Archive::new(buf);
159
160        for entry in tar.entries()? {
161            let entry = entry?;
162            if entry.header().entry_type() == tar::EntryType::Regular {
163                let path = entry.path()?;
164                let file_name = path
165                    .file_name()
166                    .context("Failed to detect filename")?
167                    .to_str()
168                    .unwrap_or("");
169                if let Some(repo_url) = self.repos.get(file_name).cloned() {
170                    debug!("Reading package index for repository: {repo_url:?} ({file_name:?})");
171                    self.read_apkindex_container(entry, &repo_url)?;
172                }
173            }
174        }
175
176        Ok(())
177    }
178
179    pub fn register_repo(&mut self, repo: String) {
180        let mut hasher = Sha1::new();
181        hasher.update(&repo);
182        let hash = hasher.finalize();
183        let sha1 = hex::encode(&hash[..4]);
184        self.repos
185            .insert(format!("APKINDEX.{sha1}.tar.gz"), Rc::new(repo));
186    }
187
188    pub fn init_repos_from_container(&mut self, buf: &[u8]) -> Result<()> {
189        let mut tar = tar::Archive::new(buf);
190        for entry in tar.entries()? {
191            let entry = entry?;
192            if entry.header().entry_type() == tar::EntryType::Regular {
193                let reader = BufReader::new(entry);
194                for repo in reader.lines() {
195                    let repo = repo?;
196                    debug!("Found repository in /etc/apk/repositories: {repo:?}");
197                    self.register_repo(repo);
198                }
199            }
200        }
201        Ok(())
202    }
203}
204
205pub fn calculate_checksum_for_apk(apk: &[u8]) -> Result<Vec<u8>> {
206    // the first gzip has no end-of-stream marker, only read one file from tar
207    let remaining = {
208        let gz = GzDecoder::new(apk);
209        let mut tar = tar::Archive::new(gz);
210        tar.entries()?.next();
211        tar.into_inner().into_inner()
212    };
213
214    // this is slightly chaotic, there's some over-read by GzDecoder that we need to correct
215    let sig = apk.len() - remaining.len() + 8;
216
217    // locate the start of the 3rd gzip stream
218    let mut r = &apk[sig..];
219    utils::read_gzip_to_end(&mut r)?;
220    let content = r.len();
221
222    // cut at the location of the 2nd gzip stream
223    let control_data = &apk[sig..(apk.len() - content)];
224
225    let mut sha1 = Sha1::new();
226    sha1.update(control_data);
227    let sha1 = sha1.finalize();
228    Ok(sha1.to_vec())
229}
230
231pub async fn detect_installed(container: &Container) -> Result<HashSet<String>> {
232    let buf = container
233        .exec(
234            &["apk", "info", "-v"],
235            container::Exec {
236                capture_stdout: true,
237                ..Default::default()
238            },
239        )
240        .await?;
241    let buf = String::from_utf8(buf).context("Failed to decode apk output as utf8")?;
242
243    let installed = buf.lines().map(String::from).collect();
244    Ok(installed)
245}
246
247pub async fn resolve_dependencies(
248    container: &Container,
249    manifest: &PackagesManifest,
250    dependencies: &mut Vec<PackageLock>,
251) -> Result<()> {
252    info!("Syncing package datatabase...");
253    container
254        .exec(&["apk", "update"], container::Exec::default())
255        .await?;
256
257    let mut dbs = DatabaseCache::default();
258    {
259        // we only need these files briefly, declare them in a small scope so they get free'd early
260        let repos = container.tar("/etc/apk/repositories").await?;
261        dbs.init_repos_from_container(&repos)?;
262
263        let tar = container.tar("/var/cache/apk").await?;
264        dbs.import_from_container(&tar)?;
265    }
266
267    info!("Resolving dependencies...");
268    let initial_packages = detect_installed(container).await?;
269
270    // upgrade and install
271    container
272        .exec(&["apk", "upgrade"], container::Exec::default())
273        .await?;
274
275    let mut cmd = vec!["apk", "add", "--"];
276    for dep in &manifest.dependencies {
277        cmd.push(dep.as_str());
278    }
279    container.exec(&cmd, container::Exec::default()).await?;
280
281    // detect dependencies
282    let packages_afterwards = detect_installed(container).await?;
283    let new_packages = packages_afterwards.difference(&initial_packages);
284
285    info!("Calculating package checksums...");
286    let client = http::Client::new()?;
287    let alpine_cache_dir = paths::alpine_cache_dir()?;
288    for pkg_identifier in new_packages {
289        let pkg = dbs.get(pkg_identifier)?;
290        debug!("Detected dependency: {pkg:?}");
291
292        let url = format!(
293            "{}/{}/{}-{}.apk",
294            pkg.repo_url, pkg.arch, pkg.name, pkg.version
295        );
296
297        let sha256 = if let Some(sha256) = alpine_cache_dir.sha1_read_link(&pkg.checksum).await? {
298            sha256
299        } else {
300            let mut buf = Vec::new();
301
302            let mut response = client
303                .request(&url)
304                .await
305                .with_context(|| anyhow!("Failed to download package from url: {:?}", url))?;
306
307            let mut sha256 = Sha256::new();
308            while let Some(chunk) = response
309                .chunk()
310                .await
311                .context("Failed to read from download stream")?
312            {
313                buf.extend(&chunk);
314                sha256.update(&chunk);
315            }
316
317            let sha256 = hex::encode(sha256.finalize());
318            let sha1 = hex::encode(&calculate_checksum_for_apk(&buf)?);
319
320            if sha1 != pkg.checksum {
321                bail!("Downloaded package (checksum={sha1:?} does not match checksum in APKINDEX (checksum={:?})",
322                    pkg.checksum
323                );
324            }
325
326            let (sha1_path, sha256_path) =
327                alpine_cache_dir.sha1_to_sha256(&pkg.checksum, &sha256)?;
328
329            let parent = sha1_path
330                .parent()
331                .context("Failed to determine parent directory")?;
332            fs::create_dir_all(parent).await.with_context(|| {
333                anyhow!("Failed to create parent directories for file: {sha1_path:?}")
334            })?;
335
336            fs::symlink(sha256_path, sha1_path)
337                .await
338                .context("Failed to create sha1 symlink")?;
339
340            sha256
341        };
342
343        // record provides if it mentions a dependency
344        let mut provides = Vec::new();
345        for value in &pkg.provides {
346            if manifest.dependencies.contains(value) {
347                provides.push(value.to_string());
348            }
349        }
350
351        dependencies.push(PackageLock {
352            name: pkg.name.to_string(),
353            version: pkg.version.to_string(),
354            system: "alpine".to_string(),
355            url,
356            provides,
357            sha256,
358            signature: None,
359            installed: false,
360        });
361    }
362
363    Ok(())
364}
365
366pub async fn resolve(
367    update: &args::Update,
368    manifest: &PackagesManifest,
369    container: &ContainerLock,
370    dependencies: &mut Vec<PackageLock>,
371) -> Result<()> {
372    let container = Container::create(
373        &container.image,
374        container::Config {
375            mounts: &[],
376            expose_fuse: false,
377        },
378    )
379    .await?;
380    container
381        .run(
382            resolve_dependencies(&container, manifest, dependencies),
383            update.keep,
384        )
385        .await
386}
387
388#[cfg(test)]
389mod tests {
390    use super::*;
391
392    #[test]
393    fn test_checksum_from_apk() -> Result<()> {
394        let checksum = decode_apk_checksum("Q10cGs1h9J5440p6BRXhZC8FO7pVg=")?;
395        let calculated = calculate_checksum_for_apk(crate::test_data::ALPINE_APK_EXAMPLE)?;
396        assert_eq!(checksum, calculated);
397        Ok(())
398    }
399}
repro_env/resolver/alpine.rs

repro_env/resolver/
alpine.rs
