1use std::path::Path;
14
15#[derive(Debug, Clone)]
18pub struct CrateEntry {
19 pub name: String,
20 pub version: String,
21 pub cksum: String,
23 pub bytes: Vec<u8>,
24}
25
26pub fn index_path(name: &str) -> String {
29 let n = name.to_lowercase();
30 let take =
35 |from: usize, to: usize| -> String { n.chars().skip(from).take(to - from).collect() };
36 match n.chars().count() {
37 1 => format!("1/{n}"),
38 2 => format!("2/{n}"),
39 3 => format!("3/{}/{n}", take(0, 1)),
40 _ => format!("{}/{}/{n}", take(0, 2), take(2, 4)),
41 }
42}
43
44pub fn validate_crate_name(name: &str) -> Result<(), CrateExportError> {
49 if name.is_empty() {
50 return Err(CrateExportError::UnrepresentableName {
51 name: name.to_string(),
52 why: "empty".into(),
53 });
54 }
55 if let Some(bad) = name
56 .chars()
57 .find(|c| !(c.is_ascii_alphanumeric() || *c == '-' || *c == '_'))
58 {
59 return Err(CrateExportError::UnrepresentableName {
60 name: name.to_string(),
61 why: format!("contains {bad:?}; Cargo names are ASCII alphanumeric, '-' or '_'"),
62 });
63 }
64 Ok(())
65}
66
67pub fn validate_crate_version(version: &str) -> Result<(), CrateExportError> {
70 if version.is_empty() {
71 return Err(CrateExportError::UnrepresentableVersion {
72 version: version.to_string(),
73 why: "empty".into(),
74 });
75 }
76 if let Some(bad) = version
77 .chars()
78 .find(|c| !(c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '+')))
79 {
80 return Err(CrateExportError::UnrepresentableVersion {
81 version: version.to_string(),
82 why: format!("contains {bad:?}; semver is ASCII alphanumeric, '.', '-' or '+'"),
83 });
84 }
85 Ok(())
86}
87
88fn validate_entries(crates: &[CrateEntry]) -> Result<(), CrateExportError> {
91 for e in crates {
92 validate_crate_name(&e.name)?;
93 validate_crate_version(&e.version)?;
94 if e.cksum.len() != 64 || !e.cksum.chars().all(|c| c.is_ascii_hexdigit()) {
99 return Err(CrateExportError::UnrepresentableCksum {
100 name: e.name.clone(),
101 cksum: e.cksum.clone(),
102 });
103 }
104 }
105 Ok(())
106}
107
108pub fn index_line(entry: &CrateEntry) -> String {
112 format!(
114 r#"{{"name":"{}","vers":"{}","deps":[],"cksum":"{}","features":{{}},"yanked":false}}"#,
115 entry.name, entry.version, entry.cksum
116 )
117}
118
119pub fn cargo_config_toml(registry_dir: &Path) -> String {
122 format!(
123 "# Generated by `varve export-cargo` (REQ-CRATE-001).\n\
124 # Redirects crates.io to a varve-verified local registry; build --offline.\n\
125 [source.crates-io]\n\
126 replace-with = \"varve\"\n\n\
127 [source.varve]\n\
128 local-registry = \"{}\"\n",
129 registry_dir.display()
130 )
131}
132
133#[derive(Debug, thiserror::Error)]
134pub enum CrateExportError {
135 #[error("io error at {path}: {source}")]
136 Io {
137 path: String,
138 #[source]
139 source: std::io::Error,
140 },
141 #[error("crate name {name:?} cannot be exported: {why}")]
142 UnrepresentableName { name: String, why: String },
143 #[error("crate version {version:?} cannot be exported: {why}")]
144 UnrepresentableVersion { version: String, why: String },
145 #[error("crate {name:?} has a cksum that is not a bare sha256 hex digest: {cksum:?}")]
146 UnrepresentableCksum { name: String, cksum: String },
147}
148
149pub fn cargo_checksum_json(cksum: &str) -> String {
155 format!(r#"{{"files":{{}},"package":"{cksum}"}}"#)
156}
157
158pub fn vendored_config_toml(vendor_dir: &Path) -> String {
164 format!(
165 "# Generated by `varve export-crates-vendor` (REQ-VENDOR-001).\n\
166 [source.crates-io]\n\
167 replace-with = \"vendored-sources\"\n\n\
168 [source.vendored-sources]\n\
169 directory = \"{}\"\n",
170 vendor_dir.display()
171 )
172}
173
174pub fn export_vendor_dir(
180 crates: &[CrateEntry],
181 vendor_dir: &Path,
182) -> Result<usize, CrateExportError> {
183 validate_entries(crates)?;
186 let io = |path: &Path, source: std::io::Error| CrateExportError::Io {
187 path: path.display().to_string(),
188 source,
189 };
190 std::fs::create_dir_all(vendor_dir).map_err(|e| io(vendor_dir, e))?;
191 for entry in crates {
192 let mut archive = tar::Archive::new(flate2::read::GzDecoder::new(entry.bytes.as_slice()));
194 archive.unpack(vendor_dir).map_err(|e| io(vendor_dir, e))?;
195 let crate_dir = vendor_dir.join(format!("{}-{}", entry.name, entry.version));
196 let checksum = crate_dir.join(".cargo-checksum.json");
197 std::fs::write(&checksum, cargo_checksum_json(&entry.cksum))
198 .map_err(|e| io(&checksum, e))?;
199 }
200 Ok(crates.len())
201}
202
203pub fn export_local_registry(
206 crates: &[CrateEntry],
207 registry_dir: &Path,
208) -> Result<usize, CrateExportError> {
209 validate_entries(crates)?;
212 let io = |path: &Path, source: std::io::Error| CrateExportError::Io {
213 path: path.display().to_string(),
214 source,
215 };
216 std::fs::create_dir_all(registry_dir).map_err(|e| io(registry_dir, e))?;
217 for entry in crates {
218 let crate_file = registry_dir.join(format!("{}-{}.crate", entry.name, entry.version));
220 std::fs::write(&crate_file, &entry.bytes).map_err(|e| io(&crate_file, e))?;
221
222 let idx = registry_dir.join("index").join(index_path(&entry.name));
224 if let Some(parent) = idx.parent() {
225 std::fs::create_dir_all(parent).map_err(|e| io(parent, e))?;
226 }
227 let mut line = index_line(entry);
228 line.push('\n');
229 let existing = std::fs::read_to_string(&idx).unwrap_or_default();
231 let prefix = format!(r#"{{"name":"{}","vers":"{}""#, entry.name, entry.version);
232 let mut kept: Vec<String> = existing
233 .lines()
234 .filter(|l| !l.starts_with(&prefix))
235 .map(str::to_string)
236 .collect();
237 kept.push(line.trim_end().to_string());
238 std::fs::write(&idx, kept.join("\n") + "\n").map_err(|e| io(&idx, e))?;
239 }
240 Ok(crates.len())
241}
242
243pub fn export_distdir(crates: &[CrateEntry], distdir: &Path) -> Result<usize, CrateExportError> {
254 validate_entries(crates)?;
257 let io = |path: &Path, source: std::io::Error| CrateExportError::Io {
258 path: path.display().to_string(),
259 source,
260 };
261 std::fs::create_dir_all(distdir).map_err(|e| io(distdir, e))?;
262 for entry in crates {
263 let file = distdir.join(format!("{}-{}.crate", entry.name, entry.version));
266 std::fs::write(&file, &entry.bytes).map_err(|e| io(&file, e))?;
267 }
268 Ok(crates.len())
269}
270
271#[cfg(test)]
272mod tests {
273 use super::*;
274
275 #[test]
277 fn index_paths_follow_cargos_layout() {
278 assert_eq!(index_path("a"), "1/a");
279 assert_eq!(index_path("ab"), "2/ab");
280 assert_eq!(index_path("abc"), "3/a/abc");
281 assert_eq!(index_path("serde"), "se/rd/serde");
282 assert_eq!(index_path("Varve-SDK"), "va/rv/varve-sdk"); }
284
285 #[test]
287 fn a_non_ascii_crate_name_is_an_error_not_a_panic() {
288 for bad in ["日本語", "ααα", "café-utils"] {
291 assert!(
292 validate_crate_name(bad).is_err(),
293 "{bad} must be refused, not sliced"
294 );
295 let _ = index_path(bad);
297 }
298 }
299
300 #[test]
302 fn a_name_or_version_that_would_corrupt_the_index_json_is_refused() {
303 assert!(validate_crate_name("evil\"name").is_err());
306 assert!(validate_crate_name("back\\slash").is_err());
307 assert!(validate_crate_name("").is_err());
308 assert!(validate_crate_version("1.0.0\"").is_err());
309 assert!(validate_crate_name("serde_json").is_ok());
310 assert!(validate_crate_name("varve-core").is_ok());
311 assert!(validate_crate_version("0.1.0-alpha.1+build.2").is_ok());
312 }
313
314 #[test]
316 fn export_refuses_an_unrepresentable_crate_name() {
317 let dir = tempfile::tempdir().unwrap();
318 let bad = [CrateEntry {
319 name: "café-utils".into(),
320 version: "0.1.0".into(),
321 cksum: "a".repeat(64),
322 bytes: vec![],
323 }];
324 assert!(export_local_registry(&bad, dir.path()).is_err());
326 assert!(export_vendor_dir(&bad, dir.path()).is_err());
327 assert!(export_distdir(&bad, dir.path()).is_err());
328 }
329
330 #[test]
332 fn an_index_line_carries_the_cksum_cargo_will_verify() {
333 let e = CrateEntry {
334 name: "demo".into(),
335 version: "0.1.0".into(),
336 cksum: "b".repeat(64),
337 bytes: vec![],
338 };
339 let line = index_line(&e);
340 assert!(line.contains(r#""name":"demo""#));
341 assert!(line.contains(r#""vers":"0.1.0""#));
342 assert!(line.contains(&format!(r#""cksum":"{}""#, "b".repeat(64))));
343 assert!(line.contains(r#""yanked":false"#));
344 }
345
346 #[test]
348 fn vendoring_never_writes_outside_the_vendor_directory() {
349 use std::io::Write;
354 let dir = tempfile::tempdir().unwrap();
355 let outside = dir.path().join("OUTSIDE");
356 std::fs::create_dir_all(&outside).unwrap();
357 let vendor = dir.path().join("vendor");
358
359 let mut tar_bytes = Vec::new();
360 {
361 let mut b = tar::Builder::new(&mut tar_bytes);
362 let mut link = tar::Header::new_gnu();
364 link.set_entry_type(tar::EntryType::Symlink);
365 link.set_size(0);
366 link.set_mode(0o777);
367 b.append_link(&mut link, "escape-0.1.0/link", &outside)
368 .unwrap();
369 let payload = b"PWNED";
371 let mut f = tar::Header::new_gnu();
372 f.set_size(payload.len() as u64);
373 f.set_mode(0o644);
374 b.append_data(&mut f, "escape-0.1.0/link/pwned.txt", &payload[..])
375 .unwrap();
376 b.finish().unwrap();
377 }
378 let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast());
379 gz.write_all(&tar_bytes).unwrap();
380 let evil = gz.finish().unwrap();
381
382 let entries = [CrateEntry {
383 name: "escape".into(),
384 version: "0.1.0".into(),
385 cksum: "c".repeat(64),
386 bytes: evil,
387 }];
388 let _ = export_vendor_dir(&entries, &vendor);
391 assert!(
392 !outside.join("pwned.txt").exists(),
393 "a crate tarball escaped the vendor directory"
394 );
395 assert!(
396 std::fs::read_dir(&outside).unwrap().next().is_none(),
397 "nothing may be written outside the vendor directory"
398 );
399 }
400
401 #[test]
403 fn vendoring_unpacks_the_crate_and_preserves_the_upstream_hash() {
404 let mut builder = tar::Builder::new(flate2::write::GzEncoder::new(
406 Vec::new(),
407 flate2::Compression::default(),
408 ));
409 for (name, body) in [
410 ("demo-0.1.0/Cargo.toml", "[package]\nname=\"demo\"\n"),
411 ("demo-0.1.0/src/lib.rs", "pub fn f() {}\n"),
412 ] {
413 let mut h = tar::Header::new_gnu();
414 h.set_size(body.len() as u64);
415 h.set_mode(0o644);
416 h.set_cksum();
417 builder.append_data(&mut h, name, body.as_bytes()).unwrap();
418 }
419 let targz = builder.into_inner().unwrap().finish().unwrap();
420
421 let tmp = tempfile::tempdir().unwrap();
422 let vendor = tmp.path().join("vendor");
423 let e = CrateEntry {
424 name: "demo".into(),
425 version: "0.1.0".into(),
426 cksum: "d".repeat(64),
427 bytes: targz,
428 };
429 assert_eq!(
430 export_vendor_dir(std::slice::from_ref(&e), &vendor).unwrap(),
431 1
432 );
433 assert!(vendor.join("demo-0.1.0/Cargo.toml").is_file());
435 assert!(vendor.join("demo-0.1.0/src/lib.rs").is_file());
436 let checksum =
438 std::fs::read_to_string(vendor.join("demo-0.1.0/.cargo-checksum.json")).unwrap();
439 assert_eq!(
440 checksum,
441 format!(r#"{{"files":{{}},"package":"{}"}}"#, "d".repeat(64))
442 );
443 }
444
445 #[test]
447 fn a_distdir_holds_the_verified_crate_bytes_keyed_for_bazel() {
448 let tmp = tempfile::tempdir().unwrap();
449 let dd = tmp.path().join("distdir");
450 let bytes = b"the-verified-crate-tarball-bytes".to_vec();
451 let cksum = {
453 use sha2::{Digest, Sha256};
454 hex::encode(Sha256::digest(&bytes))
455 };
456 let e = CrateEntry {
457 name: "cfg-if".into(),
458 version: "1.0.0".into(),
459 cksum: cksum.clone(),
460 bytes: bytes.clone(),
461 };
462 assert_eq!(export_distdir(std::slice::from_ref(&e), &dd).unwrap(), 1);
463 let file = dd.join("cfg-if-1.0.0.crate");
464 assert_eq!(std::fs::read(&file).unwrap(), bytes);
466 let on_disk = {
468 use sha2::{Digest, Sha256};
469 hex::encode(Sha256::digest(std::fs::read(&file).unwrap()))
470 };
471 assert_eq!(
472 on_disk, cksum,
473 "distdir file sha256 must equal the crate_universe pin"
474 );
475 }
476
477 #[test]
479 fn the_vendored_config_replaces_with_a_directory_source() {
480 let cfg = vendored_config_toml(Path::new("/v/dir"));
481 assert!(cfg.contains(r#"replace-with = "vendored-sources""#));
482 assert!(cfg.contains(r#"directory = "/v/dir""#));
483 }
484
485 #[test]
487 fn config_redirects_crates_io_to_the_local_registry() {
488 let cfg = cargo_config_toml(Path::new("/verified/reg"));
489 assert!(cfg.contains(r#"replace-with = "varve""#));
490 assert!(cfg.contains(r#"local-registry = "/verified/reg""#));
491 }
492
493 #[test]
495 fn materialising_writes_the_crate_and_a_matching_index_entry() {
496 let tmp = tempfile::tempdir().unwrap();
497 let reg = tmp.path().join("registry");
498 let e = CrateEntry {
499 name: "demo".into(),
500 version: "0.1.0".into(),
501 cksum: "e".repeat(64),
502 bytes: b"crate-tarball-bytes".to_vec(),
503 };
504 assert_eq!(
505 export_local_registry(std::slice::from_ref(&e), ®).unwrap(),
506 1
507 );
508 assert_eq!(
510 std::fs::read(reg.join("demo-0.1.0.crate")).unwrap(),
511 b"crate-tarball-bytes"
512 );
513 let idx = std::fs::read_to_string(reg.join("index/de/mo/demo")).unwrap();
515 assert!(idx.contains(&format!(r#""cksum":"{}""#, "e".repeat(64))));
516
517 export_local_registry(std::slice::from_ref(&e), ®).unwrap();
519 let idx2 = std::fs::read_to_string(reg.join("index/de/mo/demo")).unwrap();
520 assert_eq!(idx2.lines().count(), 1, "one line per (name, version)");
521 }
522
523 #[test]
525 fn a_registry_exported_from_a_layer_offers_every_version_it_pins() {
526 use sha2::{Digest, Sha256};
532 let tmp = tempfile::tempdir().unwrap();
533 let reg = tmp.path().join("registry");
534 let bytes = |v: &str| format!("serde-{v}-crate-tarball").into_bytes();
535 let entry = |v: &str| CrateEntry {
536 name: "serde".into(),
537 version: v.into(),
538 cksum: hex::encode(Sha256::digest(bytes(v))),
539 bytes: bytes(v),
540 };
541 let crates = [entry("1.0.200"), entry("1.0.210")];
542 export_local_registry(&crates, ®).unwrap();
543
544 for v in ["1.0.200", "1.0.210"] {
546 assert_eq!(
547 std::fs::read(reg.join(format!("serde-{v}.crate"))).unwrap(),
548 bytes(v),
549 "version {v} must export its own bytes"
550 );
551 }
552
553 let idx = std::fs::read_to_string(reg.join("index/se/rd/serde")).unwrap();
556 let lines: Vec<serde_json::Value> = idx
557 .lines()
558 .filter(|l| !l.trim().is_empty())
559 .map(|l| serde_json::from_str(l).expect("each index line is JSON Cargo can parse"))
560 .collect();
561 let mut offered: Vec<(String, String)> = lines
562 .iter()
563 .map(|l| {
564 (
565 l["vers"].as_str().unwrap().to_string(),
566 l["cksum"].as_str().unwrap().to_string(),
567 )
568 })
569 .collect();
570 offered.sort();
571 assert_eq!(
572 offered,
573 vec![
574 (
575 "1.0.200".to_string(),
576 hex::encode(Sha256::digest(bytes("1.0.200")))
577 ),
578 (
579 "1.0.210".to_string(),
580 hex::encode(Sha256::digest(bytes("1.0.210")))
581 ),
582 ],
583 "the index must offer BOTH versions, each bound to its own bytes"
584 );
585 assert!(lines.iter().all(|l| l["name"] == "serde"));
586
587 let dd = tmp.path().join("distdir");
590 export_distdir(&crates, &dd).unwrap();
591 assert_eq!(
592 std::fs::read(dd.join("serde-1.0.200.crate")).unwrap(),
593 bytes("1.0.200")
594 );
595 assert_eq!(
596 std::fs::read(dd.join("serde-1.0.210.crate")).unwrap(),
597 bytes("1.0.210")
598 );
599 }
600}