1use crate::args;
2use crate::container::{self, Container};
3use crate::errors::*;
4use crate::http;
5use crate::lockfile::{ContainerLock, PackageLock};
6use crate::manifest::PackagesManifest;
7use crate::paths;
8use serde::Deserialize;
9use sha1::Sha1;
10use sha2::{Digest, Sha256};
11use std::collections::HashMap;
12use std::io::prelude::*;
13use std::io::Lines;
14use tokio::fs;
15
16#[derive(Debug, Deserialize)]
17pub struct JsonSnapshotInfo {
18 pub result: Vec<JsonSnapshotPkg>,
19}
20
21#[derive(Debug, Deserialize)]
22pub struct JsonSnapshotPkg {
23 pub archive_name: String,
24 pub first_seen: String,
25 pub name: String,
26 pub path: String,
27 pub size: i64,
28}
29
30#[derive(Debug, Clone, PartialEq)]
31pub struct PkgEntry {
32 name: String,
33 version: String,
34 provides: Vec<String>,
35 sha256: String,
36}
37
38#[derive(Debug, Default, PartialEq)]
39pub struct PkgDatabase {
40 pkgs: HashMap<String, PkgEntry>,
41}
42
43impl PkgDatabase {
44 pub fn import_lz4<R: Read>(&mut self, reader: R) -> Result<()> {
45 let rdr = lz4_flex::frame::FrameDecoder::new(reader);
46 self.import_lines_stream(rdr.lines())
47 }
48
49 pub fn import_lines_stream<R: BufRead>(&mut self, mut lines: Lines<R>) -> Result<()> {
50 while let Some(line) = lines.next() {
51 let line = line?;
52 trace!("Found line in debian package database: {line:?}");
53 let Some(name) = line.strip_prefix("Package: ") else {
54 bail!("Unexpected line in database (expected `Package: `): {line:?}")
55 };
56 let mut version = None;
57 let mut filename = None;
58 let mut provides = Vec::new();
59 let mut sha256 = None;
60
61 for line in &mut lines {
62 let line = line?;
63 trace!("Found line in debian package database: {line:?}");
64
65 if line.is_empty() {
66 break;
67 } else if let Some(value) = line.strip_prefix("Version: ") {
68 version = Some(value.to_string());
69 } else if let Some(value) = line.strip_prefix("Filename: ") {
70 let value = value
71 .rsplit_once('/')
72 .map(|(_, filename)| filename)
73 .unwrap_or(value);
74 filename = Some(value.to_string());
75 } else if let Some(value) = line.strip_prefix("Provides: ") {
76 for entry in value.split(", ") {
77 let (name, _) = entry.split_once(' ').unwrap_or((entry, ""));
78 provides.push(name.to_string());
79 }
80 } else if let Some(value) = line.strip_prefix("SHA256: ") {
81 sha256 = Some(value.to_string());
82 }
83 }
84
85 let filename = filename.context("Package database entry is missing filename")?;
86 let new = PkgEntry {
87 name: name.to_string(),
88 version: version.context("Package database entry is missing version")?,
89 provides,
90 sha256: sha256.context("Package database entry is missing sha256")?,
91 };
92 let old = self.pkgs.insert(filename.to_string(), new.clone());
93
94 if let Some(old) = old {
95 if old != new {
97 bail!("Filename is not unique in package database: filename={filename:?}, old={old:?}, new={new:?}");
98 }
99 }
100 }
101
102 Ok(())
103 }
104
105 pub fn import_tar(buf: &[u8]) -> Result<Self> {
106 let mut tar = tar::Archive::new(buf);
107
108 let mut db = Self::default();
109 for entry in tar.entries()? {
110 let entry = entry?;
111 let path = entry
112 .header()
113 .path()
114 .context("Filename was not valid utf-8")?;
115 let Some(extension) = path.extension() else {
116 continue;
117 };
118
119 if extension.to_str() == Some("lz4") {
120 db.import_lz4(entry)?;
121 }
122 }
123
124 Ok(db)
125 }
126
127 pub fn find_by_filename(&self, filename: &str) -> Result<&PkgEntry> {
128 let entry = self
129 .pkgs
130 .get(filename)
131 .context("Failed to find package database entry for: {filename:?}")?;
132 Ok(entry)
133 }
134
135 pub fn find_by_apt_output(&self, line: &str) -> Result<(String, &PkgEntry)> {
136 let mut line = line.split(' ');
137 let url = line.next().context("Missing url in apt output")?;
138 let filename = line.next().context("Missing filename in apt output")?;
139 let _size = line.next().context("Missing size in apt output")?;
140 let _md5sum = line.next().context("Missing md5sum in apt output")?;
141
142 if let Some(trailing) = line.next() {
143 bail!("Trailing data in apt output: {trailing:?}");
144 }
145
146 let url = url.strip_prefix('\'').unwrap_or(url);
147 let url = url.strip_suffix('\'').unwrap_or(url);
148 debug!("Detected dependency filename={filename:?} url={url:?}");
149
150 let package = {
151 let url = url
152 .parse::<reqwest::Url>()
153 .context("Failed to parse as url")?;
154 let filename = url
155 .path_segments()
156 .context("Failed to get path from url")?
157 .last()
158 .context("Failed to get filename from url")?;
159 let filename =
160 urlencoding::decode(filename).context("Failed to url decode filename")?;
161 self.find_by_filename(&filename).with_context(|| {
162 anyhow!("Failed to find package database entry for file: {filename:?}")
163 })?
164 };
165
166 Ok((url.to_string(), package))
167 }
168}
169
170pub async fn resolve_dependencies(
171 container: &Container,
172 manifest: &PackagesManifest,
173 dependencies: &mut Vec<PackageLock>,
174) -> Result<()> {
175 info!("Update package datatabase...");
176 container
177 .exec(&["apt-get", "update"], container::Exec::default())
178 .await?;
179
180 info!("Importing package database...");
181 let tar = container.tar("/var/lib/apt/lists").await?;
182 let db = PkgDatabase::import_tar(&tar)?;
183
184 info!("Resolving dependencies...");
185 let mut cmd = vec![
186 "apt-get",
187 "-qq",
188 "--print-uris",
189 "--no-install-recommends",
190 "upgrade",
191 "--",
192 ];
193 for dep in &manifest.dependencies {
194 cmd.push(dep.as_str());
195 }
196 let buf = container
197 .exec(
198 &cmd,
199 container::Exec {
200 capture_stdout: true,
201 ..Default::default()
202 },
203 )
204 .await?;
205 let buf = String::from_utf8(buf).context("Failed to decode apt output as utf8")?;
206
207 let client = http::Client::new()?;
208 let pkgs_cache_dir = paths::pkgs_cache_dir()?;
209 for line in buf.lines() {
210 let (url, package) = db.find_by_apt_output(line)?;
211
212 let path = pkgs_cache_dir.sha256_path(&package.sha256)?;
213 let buf = if path.exists() {
214 fs::read(path).await?
215 } else {
216 let buf = client.fetch(&url).await?.to_vec();
217
218 let mut hasher = Sha256::new();
219 hasher.update(&buf);
220 let result = hex::encode(hasher.finalize());
221
222 if result != package.sha256 {
223 bail!(
224 "Mismatch of sha256 checksum, expected={}, downloaded={}",
225 package.sha256,
226 result
227 );
228 }
229
230 buf
231 };
232
233 let mut hasher = Sha1::new();
234 hasher.update(&buf);
235 let sha1 = hex::encode(hasher.finalize());
236
237 let url = format!("https://snapshot.debian.org/mr/file/{sha1}/info");
238 let buf = client
239 .fetch(&url)
240 .await
241 .context("Failed to lookup pkg hash on snapshot.debian.org")?;
242
243 let info = serde_json::from_slice::<JsonSnapshotInfo>(&buf)
244 .context("Failed to decode snapshot.debian.org json response")?;
245
246 let pkg = info
247 .result
248 .first()
249 .context("Could not find package in any snapshots")?;
250
251 let archive_name = &pkg.archive_name;
252 let first_seen = &pkg.first_seen;
253 let path = &pkg.path;
254 let name = &pkg.name;
255
256 let url =
257 format!("https://snapshot.debian.org/archive/{archive_name}/{first_seen}{path}/{name}");
258
259 let mut provides = Vec::new();
261 for value in &package.provides {
262 if manifest.dependencies.contains(value) {
263 provides.push(value.to_string());
264 }
265 }
266
267 dependencies.push(PackageLock {
268 name: package.name.to_string(),
269 version: package.version.to_string(),
270 system: "debian".to_string(),
271 url,
272 provides,
273 sha256: package.sha256.to_string(),
274 signature: None,
275 installed: false,
276 });
277 }
278
279 Ok(())
280}
281
282pub async fn resolve(
283 update: &args::Update,
284 manifest: &PackagesManifest,
285 container: &ContainerLock,
286 dependencies: &mut Vec<PackageLock>,
287) -> Result<()> {
288 let container = Container::create(
289 &container.image,
290 container::Config {
291 mounts: &[],
292 expose_fuse: false,
293 },
294 )
295 .await?;
296 container
297 .run(
298 resolve_dependencies(&container, manifest, dependencies),
299 update.keep,
300 )
301 .await
302}
303
304#[cfg(test)]
305mod tests {
306 use super::*;
307 use std::io::BufReader;
308
309 #[test]
310 fn test_pkg_database() -> Result<()> {
311 let lz4 = {
312 let mut w = lz4_flex::frame::FrameEncoder::new(Vec::new());
313 w.write_all(br#"Package: binutils-aarch64-linux-gnu
314Source: binutils
315Version: 2.40-2
316Installed-Size: 19242
317Maintainer: Matthias Klose <doko@debian.org>
318Architecture: amd64
319Replaces: binutils (<< 2.29-6), binutils-dev (<< 2.38.50.20220609-2)
320Depends: binutils-common (= 2.40-2), libbinutils (>= 2.39.50), libc6 (>= 2.36), libgcc-s1 (>= 4.2), libjansson4 (>= 2.14), libzstd1 (>= 1.5.2), zlib1g (>= 1:1.1.4)
321Suggests: binutils-doc (= 2.40-2)
322Breaks: binutils (<< 2.29-6), binutils-dev (<< 2.38.50.20220609-2)
323Description: GNU binary utilities, for aarch64-linux-gnu target
324Multi-Arch: allowed
325Homepage: https://www.gnu.org/software/binutils/
326Description-md5: 102820197d11c3672c0cd4ce0becb720
327Section: devel
328Priority: optional
329Filename: pool/main/b/binutils/binutils-aarch64-linux-gnu_2.40-2_amd64.deb
330Size: 3352924
331MD5sum: 2c02fdb8d4455ace16be0bb922eb8502
332SHA256: 3d6f64a7a4ed6d73719f8fa2e85fd896f58ff7f211a6683942ba93de690aaa66
333
334Package: rustc
335Version: 1.63.0+dfsg1-2
336Installed-Size: 7753
337Maintainer: Debian Rust Maintainers <pkg-rust-maintainers@alioth-lists.debian.net>
338Architecture: amd64
339Replaces: libstd-rust-dev (<< 1.25.0+dfsg1-2~~)
340Depends: libc6 (>= 2.34), libgcc-s1 (>= 3.0), libstd-rust-dev (= 1.63.0+dfsg1-2), gcc, libc-dev, binutils (>= 2.26)
341Recommends: cargo (>= 0.64.0~~), cargo (<< 0.65.0~~), llvm-14
342Suggests: lld-14, clang-14
343Breaks: libstd-rust-dev (<< 1.25.0+dfsg1-2~~)
344Description: Rust systems programming language
345Multi-Arch: allowed
346Homepage: http://www.rust-lang.org/
347Description-md5: 67ca6080eea53dc7f3cdf73bc6b8521e
348Section: rust
349Priority: optional
350Filename: pool/main/r/rustc/rustc_1.63.0+dfsg1-2_amd64.deb
351Size: 2612712
352MD5sum: 5eaa6969388c512a206377bf813ab531
353SHA256: 26dd439266153e38d3e6fbe0fe2dbbb41f20994afa688faa71f38427348589ed
354"#)?;
355 w.finish()?
356 };
357
358 let tar = {
359 let mut tar = tar::Builder::new(Vec::new());
360 let mut header = tar::Header::new_gnu();
361 header.set_path("deb.debian.org_debian_dists_stable_main_binary-amd64_Packages.lz4")?;
362 header.set_size(lz4.len() as u64);
363 header.set_cksum();
364 tar.append(&header, &lz4[..])?;
365 tar.into_inner()?
366 };
367
368 let db = PkgDatabase::import_tar(&tar)?;
369 let pkgs = {
370 let mut pkgs = HashMap::new();
371 pkgs.insert(
372 "binutils-aarch64-linux-gnu_2.40-2_amd64.deb".to_string(),
373 PkgEntry {
374 name: "binutils-aarch64-linux-gnu".to_string(),
375 version: "2.40-2".to_string(),
376 provides: vec![],
377 sha256: "3d6f64a7a4ed6d73719f8fa2e85fd896f58ff7f211a6683942ba93de690aaa66"
378 .to_string(),
379 },
380 );
381 pkgs.insert(
382 "rustc_1.63.0+dfsg1-2_amd64.deb".to_string(),
383 PkgEntry {
384 name: "rustc".to_string(),
385 version: "1.63.0+dfsg1-2".to_string(),
386 provides: vec![],
387 sha256: "26dd439266153e38d3e6fbe0fe2dbbb41f20994afa688faa71f38427348589ed"
388 .to_string(),
389 },
390 );
391 pkgs
392 };
393 assert_eq!(db, PkgDatabase { pkgs });
394
395 Ok(())
396 }
397
398 #[test]
399 fn test_pkg_database_apt_output_parser() -> Result<()> {
400 let mut db = PkgDatabase::default();
401 db.pkgs.insert(
402 "rustc_1.63.0+dfsg1-2_amd64.deb".to_string(),
403 PkgEntry {
404 name: "rustc".to_string(),
405 version: "1.63.0+dfsg1-2".to_string(),
406 provides: vec![],
407 sha256: "26dd439266153e38d3e6fbe0fe2dbbb41f20994afa688faa71f38427348589ed"
408 .to_string(),
409 },
410 );
411
412 let result = db.find_by_apt_output("'http://deb.debian.org/debian/pool/main/r/rustc/rustc_1.63.0%2bdfsg1-2_amd64.deb' rustc_1.63.0+dfsg1-2_amd64.deb 2612712 MD5Sum:5eaa6969388c512a206377bf813ab531")?;
413 assert_eq!(
414 result,
415 (
416 "http://deb.debian.org/debian/pool/main/r/rustc/rustc_1.63.0%2bdfsg1-2_amd64.deb"
417 .to_string(),
418 &PkgEntry {
419 name: "rustc".to_string(),
420 version: "1.63.0+dfsg1-2".to_string(),
421 provides: vec![],
422 sha256: "26dd439266153e38d3e6fbe0fe2dbbb41f20994afa688faa71f38427348589ed"
423 .to_string(),
424 }
425 )
426 );
427
428 let result = db.find_by_apt_output("'http://deb.debian.org/debian/pool/main/n/non-existant/non-existant_1.2.3_amd64.deb' non-existant_1.2.3_amd64.deb 2612712 MD5Sum:5eaa6969388c512a206377bf813ab531");
429 assert!(result.is_err());
430
431 Ok(())
432 }
433
434 #[test]
435 fn test_parse_provides() -> Result<()> {
436 let foo = BufReader::new(r#"Package: librust-repro-env-dev
437Source: rust-repro-env
438Version: 0.3.2-1
439Installed-Size: 175
440Maintainer: Debian Rust Maintainers <pkg-rust-maintainers@alioth-lists.debian.net>
441Architecture: amd64
442Provides: librust-repro-env+default-dev (= 0.3.2-1), librust-repro-env-0+default-dev (= 0.3.2-1), librust-repro-env-0-dev (= 0.3.2-1), librust-repro-env-0.3+default-dev (= 0.3.2-1), librust-repro-env-0.3-dev (= 0.3.2-1), librust-repro-env-0.3.2+default-dev (= 0.3.2-1), librust-repro-env-0.3.2-dev (= 0.3.2-1)
443Depends: librust-anyhow-1+default-dev (>= 1.0.71-~~), librust-ar-0.9+default-dev, librust-bytes-1+default-dev (>= 1.4.0-~~), librust-clap-4+default-dev, librust-clap-4+derive-dev, librust-clap-complete-4+default-dev, librust-clone-file-0.1+default-dev, librust-data-encoding-2+default-dev (>= 2.4.0-~~), librust-dirs-5+default-dev (>= 5.0.1-~~), librust-env-logger-0.10+default-dev, librust-fd-lock-3+default-dev, librust-flate2-1+default-dev (>= 1.0.26-~~), librust-hex-0.4+default-dev (>= 0.4.3-~~), librust-log-0.4+default-dev (>= 0.4.19-~~), librust-lz4-flex-0.11+default-dev (>= 0.11.1-~~), librust-lzma-rs-0.3+default-dev, librust-memchr-2+default-dev (>= 2.5.0-~~), librust-nix-0.26+sched-dev, librust-peekread-0.1+default-dev (>= 0.1.1-~~), librust-reqwest-0.11+rustls-tls-native-roots-dev (>= 0.11.18-~~), librust-reqwest-0.11+stream-dev (>= 0.11.18-~~), librust-reqwest-0.11+tokio-socks-dev (>= 0.11.18-~~), librust-ruzstd-0.4+default-dev, librust-serde-1+default-dev, librust-serde-1+derive-dev, librust-serde-json-1+default-dev, librust-sha1-0.10+default-dev (>= 0.10.5-~~), librust-sha2-0.10+default-dev (>= 0.10.7-~~), librust-tar-0.4+default-dev (>= 0.4.38-~~), librust-tempfile-3+default-dev (>= 3.6.0-~~), librust-tokio-1+default-dev, librust-tokio-1+fs-dev, librust-tokio-1+macros-dev, librust-tokio-1+process-dev, librust-tokio-1+rt-multi-thread-dev, librust-tokio-1+signal-dev, librust-toml-0.7+default-dev, librust-urlencoding-2+default-dev (>= 2.1.2-~~)
444Description: Dependency lockfiles for reproducible build environments 📦🔒 - Rust source code
445Multi-Arch: same
446Description-md5: 1023d39707057b0b09f9d6bf7deeb14e
447Section: utils
448Priority: optional
449Filename: pool/main/r/rust-repro-env/librust-repro-env-dev_0.3.2-1_amd64.deb
450Size: 40344
451MD5sum: 4dafbbe511b9a068728930e6811a0bf0
452SHA256: 2bb1befee1b89f0462b74d519be9b8c94c038d7f8a074d050d62985f47ec4164
453"#.as_bytes());
454 let mut db = PkgDatabase::default();
455 db.import_lines_stream(foo.lines())?;
456
457 let pkgs = {
458 let mut pkgs = HashMap::new();
459 pkgs.insert(
460 "librust-repro-env-dev_0.3.2-1_amd64.deb".to_string(),
461 PkgEntry {
462 name: "librust-repro-env-dev".to_string(),
463 version: "0.3.2-1".to_string(),
464 provides: vec![
465 "librust-repro-env+default-dev".to_string(),
466 "librust-repro-env-0+default-dev".to_string(),
467 "librust-repro-env-0-dev".to_string(),
468 "librust-repro-env-0.3+default-dev".to_string(),
469 "librust-repro-env-0.3-dev".to_string(),
470 "librust-repro-env-0.3.2+default-dev".to_string(),
471 "librust-repro-env-0.3.2-dev".to_string(),
472 ],
473 sha256: "2bb1befee1b89f0462b74d519be9b8c94c038d7f8a074d050d62985f47ec4164"
474 .to_string(),
475 },
476 );
477 pkgs
478 };
479 assert_eq!(db, PkgDatabase { pkgs });
480 Ok(())
481 }
482}