use std::path::{Path, PathBuf};
use async_trait::async_trait;
use crate::runner::RunnerError;
pub const MAX_CONTEXT_BYTES: u64 = 512 * 1024 * 1024;
#[async_trait]
pub trait BuildContextPublisher: Send + Sync {
async fn publish(&self, key: &str, tarball: Vec<u8>) -> Result<String, RunnerError>;
async fn discard(&self, key: &str);
}
pub struct NoBuildContextPublisher;
#[async_trait]
impl BuildContextPublisher for NoBuildContextPublisher {
async fn publish(&self, _key: &str, _tarball: Vec<u8>) -> Result<String, RunnerError> {
Err(RunnerError::InvalidConfig(
"this build-image step must run on a different host than qed, so its build \
context has to be fetched by the worker rather than bind-mounted — but no \
build-context publisher is wired into this runner. The `yah` CLI wires an \
R2-backed one; a bare `yah-qed` embedding must supply its own via \
`PipelineRunner::with_build_context_publisher`."
.into(),
))
}
async fn discard(&self, _key: &str) {}
}
pub fn pack_context(
context_dir: &Path,
extra: &[(String, Vec<u8>)],
) -> Result<Vec<u8>, RunnerError> {
let mut budget = MAX_CONTEXT_BYTES;
let gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast());
let mut tar = tar::Builder::new(gz);
tar.follow_symlinks(true);
append_dir(&mut tar, context_dir, Path::new(""), &mut budget)?;
for (name, bytes) in extra {
let mut header = tar::Header::new_gnu();
header.set_size(bytes.len() as u64);
header.set_mode(0o644);
header.set_mtime(0);
header.set_cksum();
tar.append_data(&mut header, name, bytes.as_slice())
.map_err(|e| pack_err(format!("adding {name} to the context tar: {e}")))?;
}
let gz = tar
.into_inner()
.map_err(|e| pack_err(format!("finishing the context tar: {e}")))?;
gz.finish()
.map_err(|e| pack_err(format!("compressing the context tar: {e}")))
}
pub const SOURCE_CONTEXT_URL_ENV: &str = "YAH_SOURCE_CONTEXT_URL";
pub fn pack_source_context(camp_root: &Path, paths: &[PathBuf]) -> Result<Vec<u8>, RunnerError> {
let rel = source_context_files(camp_root, paths)?;
let mut budget = MAX_CONTEXT_BYTES;
let gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast());
let mut tar = tar::Builder::new(gz);
tar.follow_symlinks(true);
for entry in &rel {
let abs = camp_root.join(entry);
let meta = match std::fs::metadata(&abs) {
Ok(meta) => meta,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
Err(e) => return Err(source_err(format!("stat {}: {e}", abs.display()))),
};
if meta.is_dir() {
continue;
}
let len = meta.len();
budget = budget.checked_sub(len).ok_or_else(|| {
source_err(format!(
"source context exceeds the {} MiB cross-host limit (tripped at {}). \
Narrow `source_context` to the subtrees the build actually compiles.",
MAX_CONTEXT_BYTES / (1024 * 1024),
entry.display(),
))
})?;
let mut file = std::fs::File::open(&abs)
.map_err(|e| source_err(format!("opening {}: {e}", abs.display())))?;
tar.append_file(entry, &mut file).map_err(|e| {
source_err(format!("adding {} to the source tar: {e}", entry.display()))
})?;
}
let gz = tar
.into_inner()
.map_err(|e| source_err(format!("finishing the source tar: {e}")))?;
gz.finish()
.map_err(|e| source_err(format!("compressing the source tar: {e}")))
}
fn source_context_files(camp_root: &Path, paths: &[PathBuf]) -> Result<Vec<PathBuf>, RunnerError> {
if paths.is_empty() {
return Err(source_err(
"source context requested with no paths".to_string(),
));
}
let mut cmd = std::process::Command::new("git");
cmd.arg("-C").arg(camp_root).args(["ls-files", "-z", "--"]);
for path in paths {
cmd.arg(path);
}
let out = cmd.output().map_err(|e| {
source_err(format!(
"running git ls-files in {}: {e}",
camp_root.display()
))
})?;
if !out.status.success() {
return Err(source_err(format!(
"git ls-files in {} failed: {}",
camp_root.display(),
String::from_utf8_lossy(&out.stderr).trim(),
)));
}
let mut rel: Vec<PathBuf> = out
.stdout
.split(|b| *b == 0)
.filter(|s| !s.is_empty())
.map(|s| PathBuf::from(String::from_utf8_lossy(s).into_owned()))
.collect();
rel.sort();
if rel.is_empty() {
return Err(source_err(format!(
"`source_context` matched no git-tracked files under {:?} — the step would \
fetch an empty source tree and fail at the build. Check the paths are \
camp-root-relative and tracked.",
paths,
)));
}
Ok(rel)
}
pub fn source_context_fingerprint(
camp_root: &Path,
paths: &[PathBuf],
) -> Result<String, RunnerError> {
let rel = source_context_files(camp_root, paths)?;
let mut acc = blake3::Hasher::new();
for entry in &rel {
let abs = camp_root.join(entry);
let meta = match std::fs::metadata(&abs) {
Ok(meta) => meta,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue,
Err(e) => return Err(source_err(format!("stat {}: {e}", abs.display()))),
};
if meta.is_dir() {
continue;
}
let bytes = std::fs::read(&abs)
.map_err(|e| source_err(format!("reading {} for fingerprint: {e}", abs.display())))?;
let path_bytes = entry.to_string_lossy();
acc.update(&(path_bytes.len() as u64).to_le_bytes());
acc.update(path_bytes.as_bytes());
acc.update(blake3::hash(&bytes).as_bytes());
}
Ok(acc.finalize().to_hex().to_string())
}
fn append_dir<W: std::io::Write>(
tar: &mut tar::Builder<W>,
dir: &Path,
prefix: &Path,
budget: &mut u64,
) -> Result<(), RunnerError> {
let entries = std::fs::read_dir(dir)
.map_err(|e| pack_err(format!("reading build context {}: {e}", dir.display())))?;
let mut names: Vec<std::ffi::OsString> = Vec::new();
for entry in entries {
let entry = entry.map_err(|e| pack_err(format!("walking {}: {e}", dir.display())))?;
names.push(entry.file_name());
}
names.sort();
for name in names {
let path = dir.join(&name);
let rel = prefix.join(&name);
let meta = std::fs::metadata(&path)
.map_err(|e| pack_err(format!("stat {}: {e}", path.display())))?;
if meta.is_dir() {
append_dir(tar, &path, &rel, budget)?;
continue;
}
let len = meta.len();
*budget = budget
.checked_sub(len)
.ok_or_else(|| RunnerError::StepFailed {
step: "build-image".into(),
msg: format!(
"build context {} exceeds the {} MiB cross-host limit (tripped at {}). \
A remote build ships its context over the network, so the whole \
directory has to travel — set `context = \"<dir>\"` on the step to the \
directory the Dockerfile actually COPYs from instead of the camp root.",
path.display(),
MAX_CONTEXT_BYTES / (1024 * 1024),
rel.display(),
),
})?;
let mut file = std::fs::File::open(&path)
.map_err(|e| pack_err(format!("opening {}: {e}", path.display())))?;
tar.append_file(&rel, &mut file)
.map_err(|e| pack_err(format!("adding {} to the context tar: {e}", rel.display())))?;
}
Ok(())
}
fn pack_err(msg: String) -> RunnerError {
RunnerError::StepFailed {
step: "build-image".into(),
msg,
}
}
fn source_err(msg: String) -> RunnerError {
RunnerError::StepFailed {
step: "source-context".into(),
msg,
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Read;
fn entries_of(tarball: &[u8]) -> Vec<(String, Vec<u8>)> {
let gz = flate2::read::GzDecoder::new(tarball);
let mut archive = tar::Archive::new(gz);
let mut out = Vec::new();
for entry in archive.entries().unwrap() {
let mut entry = entry.unwrap();
let name = entry.path().unwrap().to_string_lossy().into_owned();
let mut bytes = Vec::new();
entry.read_to_end(&mut bytes).unwrap();
out.push((name, bytes));
}
out
}
#[test]
fn packs_relative_to_the_context_root() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("build-v8.sh"), b"#!/bin/sh\n").unwrap();
std::fs::create_dir(dir.path().join("patches")).unwrap();
std::fs::write(dir.path().join("patches/a.patch"), b"diff\n").unwrap();
let names: Vec<String> = entries_of(&pack_context(dir.path(), &[]).unwrap())
.into_iter()
.map(|(n, _)| n)
.collect();
assert_eq!(names, vec!["build-v8.sh", "patches/a.patch"]);
}
#[test]
fn packs_without_extended_attributes() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("verify-consumer.sh"), b"#!/bin/sh\n").unwrap();
#[cfg(target_os = "macos")]
{
let path = dir.path().join("verify-consumer.sh");
let _ = std::process::Command::new("xattr")
.args(["-w", "com.apple.provenance", "x"])
.arg(&path)
.status();
}
let tarball = pack_context(dir.path(), &[]).unwrap();
let gz = flate2::read::GzDecoder::new(tarball.as_slice());
let mut archive = tar::Archive::new(gz);
for entry in archive.entries().unwrap() {
let mut entry = entry.unwrap();
let name = entry.path().unwrap().to_string_lossy().into_owned();
assert!(
!name.starts_with("._"),
"AppleDouble sidecar leaked into the context tar: {name}"
);
let pax: Vec<String> = entry
.pax_extensions()
.unwrap()
.into_iter()
.flatten()
.map(|e| e.unwrap().key().unwrap().to_string())
.collect();
assert!(
!pax.iter().any(|k| k.contains("xattr")),
"{name} carries xattr pax headers {pax:?} — BuildKit will fail \
the remote-context unpack with lsetxattr: operation not supported"
);
}
}
#[test]
fn extra_entries_override_same_named_context_files() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("x.Dockerfile"), b"FROM stale\n").unwrap();
let tarball = pack_context(
dir.path(),
&[("x.Dockerfile".to_string(), b"FROM compiled\n".to_vec())],
)
.unwrap();
let entries = entries_of(&tarball);
let last = entries.last().unwrap();
assert_eq!(last.0, "x.Dockerfile");
assert_eq!(last.1, b"FROM compiled\n");
}
#[test]
fn packs_in_sorted_order() {
let dir = tempfile::tempdir().unwrap();
for name in ["zeta", "alpha", "mid"] {
std::fs::write(dir.path().join(name), name.as_bytes()).unwrap();
}
let names: Vec<String> = entries_of(&pack_context(dir.path(), &[]).unwrap())
.into_iter()
.map(|(n, _)| n)
.collect();
assert_eq!(names, vec!["alpha", "mid", "zeta"]);
}
#[test]
fn oversized_context_is_refused_with_an_actionable_message() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("big.bin"), vec![0u8; 4096]).unwrap();
let mut budget = 1024u64;
let err = append_dir(
&mut tar::Builder::new(Vec::new()),
dir.path(),
Path::new(""),
&mut budget,
)
.unwrap_err();
let msg = err.to_string();
assert!(msg.contains("big.bin"), "must name the file: {msg}");
assert!(msg.contains("context ="), "must name the fix: {msg}");
}
fn source_repo() -> tempfile::TempDir {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
let git = |args: &[&str]| {
let out = std::process::Command::new("git")
.arg("-C")
.arg(root)
.args(args)
.output()
.unwrap();
assert!(out.status.success(), "git {args:?}: {out:?}");
};
git(&["init", "-q"]);
git(&["config", "user.email", "t@example.com"]);
git(&["config", "user.name", "t"]);
std::fs::create_dir_all(root.join("pkg/src")).unwrap();
std::fs::create_dir_all(root.join("pkg/target/debug")).unwrap();
std::fs::create_dir_all(root.join("sibling/src")).unwrap();
std::fs::create_dir_all(root.join("elsewhere")).unwrap();
std::fs::write(root.join(".gitignore"), "target/\n").unwrap();
std::fs::write(root.join("pkg/src/lib.rs"), b"// lib\n").unwrap();
std::fs::write(root.join("pkg/Cargo.toml"), b"[package]\n").unwrap();
std::fs::write(root.join("pkg/target/debug/huge.bin"), vec![0u8; 4096]).unwrap();
std::fs::write(root.join("sibling/src/dep.rs"), b"// dep\n").unwrap();
std::fs::write(root.join("elsewhere/nope.rs"), b"// nope\n").unwrap();
git(&["add", "-A"]);
git(&["commit", "-qm", "init"]);
dir
}
#[test]
fn source_context_ships_tracked_files_at_camp_relative_paths() {
let repo = source_repo();
let tarball = pack_source_context(
repo.path(),
&[PathBuf::from("pkg"), PathBuf::from("sibling")],
)
.unwrap();
let names: Vec<String> = entries_of(&tarball).into_iter().map(|(n, _)| n).collect();
assert_eq!(
names,
vec!["pkg/Cargo.toml", "pkg/src/lib.rs", "sibling/src/dep.rs"],
"camp-relative paths, sorted, tracked-only",
);
assert!(
!names.iter().any(|n| n.contains("target/")),
"an ignored build dir must not travel: {names:?}",
);
assert!(
!names.iter().any(|n| n.starts_with("elsewhere/")),
"only the named subtrees travel: {names:?}",
);
}
#[test]
fn source_context_packs_deterministically() {
let repo = source_repo();
let paths = [PathBuf::from("pkg"), PathBuf::from("sibling")];
assert_eq!(
pack_source_context(repo.path(), &paths).unwrap(),
pack_source_context(repo.path(), &paths).unwrap(),
);
}
#[test]
fn source_context_skips_files_deleted_in_the_working_tree() {
let repo = source_repo();
std::fs::remove_file(repo.path().join("pkg/src/lib.rs")).unwrap();
let names: Vec<String> =
entries_of(&pack_source_context(repo.path(), &[PathBuf::from("pkg")]).unwrap())
.into_iter()
.map(|(n, _)| n)
.collect();
assert_eq!(names, vec!["pkg/Cargo.toml"]);
}
#[test]
fn source_context_matching_nothing_is_refused_with_an_actionable_message() {
let repo = source_repo();
let err = pack_source_context(repo.path(), &[PathBuf::from("oss/typo")]).unwrap_err();
let msg = err.to_string();
assert!(msg.contains("no git-tracked files"), "{msg}");
assert!(msg.contains("camp-root-relative"), "{msg}");
}
#[test]
fn fingerprint_ignores_mtime() {
let repo = source_repo();
let paths = [PathBuf::from("pkg"), PathBuf::from("sibling")];
let before = source_context_fingerprint(repo.path(), &paths).unwrap();
let file = repo.path().join("pkg/src/lib.rs");
let content = std::fs::read(&file).unwrap();
std::fs::remove_file(&file).unwrap();
std::fs::write(&file, &content).unwrap();
assert_eq!(
before,
source_context_fingerprint(repo.path(), &paths).unwrap()
);
assert_ne!(
before,
blake3::hash(&pack_source_context(repo.path(), &paths).unwrap())
.to_hex()
.to_string(),
);
}
#[test]
fn fingerprint_moves_on_a_content_change() {
let repo = source_repo();
let paths = [PathBuf::from("pkg")];
let before = source_context_fingerprint(repo.path(), &paths).unwrap();
std::fs::write(repo.path().join("pkg/src/lib.rs"), b"pub fn f() {}\n").unwrap();
assert_ne!(
before,
source_context_fingerprint(repo.path(), &paths).unwrap()
);
}
#[test]
fn fingerprint_ignores_untracked_files() {
let repo = source_repo();
let paths = [PathBuf::from("pkg")];
let before = source_context_fingerprint(repo.path(), &paths).unwrap();
std::fs::create_dir_all(repo.path().join("pkg/target")).unwrap();
std::fs::write(repo.path().join("pkg/target/junk.bin"), b"\x00\x01").unwrap();
assert_eq!(
before,
source_context_fingerprint(repo.path(), &paths).unwrap()
);
}
#[test]
fn fingerprint_matching_nothing_is_refused() {
let repo = source_repo();
let err =
source_context_fingerprint(repo.path(), &[PathBuf::from("oss/typo")]).unwrap_err();
assert!(err.to_string().contains("no git-tracked files"), "{err}");
}
#[tokio::test]
async fn no_publisher_refuses_with_the_wiring_instruction() {
let err = NoBuildContextPublisher
.publish("k", vec![])
.await
.unwrap_err();
let msg = err.to_string();
assert!(msg.contains("with_build_context_publisher"), "{msg}");
}
}