use std::path::{Path, PathBuf};
use anyhow::{Context, Result};
use gunnar_store::git_storage_trait::{GitServe, ObjType};
use crate::stores::ServerStores;
pub const RECIPE_FILE: &str = "gunnar-build.toml";
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Recipe {
pub name: String,
pub command: String,
pub args: Vec<String>,
pub artifacts: Vec<String>,
}
impl Recipe {
pub fn cargo_default() -> Recipe {
Recipe {
name: "cargo build --release --locked".to_owned(),
command: "cargo".to_owned(),
args: vec![
"build".to_owned(),
"--release".to_owned(),
"--locked".to_owned(),
],
artifacts: Vec::new(),
}
}
pub fn parse(text: &str) -> Result<Recipe> {
let mut recipe = Recipe {
name: String::new(),
command: String::new(),
args: Vec::new(),
artifacts: Vec::new(),
};
for line in text.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') || line.starts_with('[') {
continue;
}
let Some((key, value)) = line.split_once('=') else {
continue;
};
let value = value.trim();
let list = |value: &str| -> Vec<String> {
value
.trim_start_matches('[')
.trim_end_matches(']')
.split(',')
.map(|item| item.trim().trim_matches('"').to_owned())
.filter(|item| !item.is_empty())
.collect()
};
match key.trim() {
"name" => recipe.name = value.trim_matches('"').to_owned(),
"command" => recipe.command = value.trim_matches('"').to_owned(),
"args" => recipe.args = list(value),
"artifacts" => recipe.artifacts = list(value),
_ => {}
}
}
anyhow::ensure!(
!recipe.command.is_empty(),
"{RECIPE_FILE} names no `command`, so there is nothing to run. A recipe that builds \
nothing is not a smaller recipe — it is a build that reports success having done \
nothing."
);
if recipe.name.is_empty() {
recipe.name = recipe.command.clone();
}
Ok(recipe)
}
}
#[derive(Debug, Clone)]
pub struct Built {
pub record: gunnar_core::BuildRecord,
pub workdir: PathBuf,
}
fn checkout(
git: &dyn GitServe,
hash: gix_hash::Kind,
tree: gix_hash::ObjectId,
into: &Path,
) -> Result<usize> {
std::fs::create_dir_all(into).with_context(|| format!("creating {}", into.display()))?;
let (kind, bytes) = git
.read(tree.as_bytes())?
.with_context(|| format!("tree {tree} is not in this repository"))?;
anyhow::ensure!(kind == ObjType::Tree, "{tree} is not a tree");
let mut files = 0usize;
for entry in gunnar_store::object::TreeIter::from_bytes(&bytes, hash).named() {
let entry = entry.with_context(|| format!("decoding tree {tree}"))?;
let name = String::from_utf8_lossy(entry.name).into_owned();
anyhow::ensure!(
name != "." && name != ".." && !name.contains('/'),
"tree {tree} carries an entry named {name:?}, which would write outside the checkout"
);
let path = into.join(&name);
match entry.kind {
gunnar_store::object::EntryKind::Tree => {
files += checkout(git, hash, entry.oid, &path)?;
}
gunnar_store::object::EntryKind::Commit => {
continue;
}
kind => {
let Some((ObjType::Blob, blob)) = git.read(entry.oid.as_bytes())? else {
anyhow::bail!("{} is not a blob in this repository", entry.oid);
};
if kind == gunnar_store::object::EntryKind::Link {
let target = String::from_utf8_lossy(&blob).into_owned();
let _ = std::fs::remove_file(&path);
std::os::unix::fs::symlink(&target, &path)
.with_context(|| format!("linking {}", path.display()))?;
} else {
std::fs::write(&path, &blob)
.with_context(|| format!("writing {}", path.display()))?;
if kind == gunnar_store::object::EntryKind::BlobExecutable {
use std::os::unix::fs::PermissionsExt as _;
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755))
.with_context(|| format!("chmod {}", path.display()))?;
}
}
files += 1;
}
}
}
Ok(files)
}
fn recipe_for(checkout: &Path) -> Result<Recipe> {
let path = checkout.join(RECIPE_FILE);
if path.exists() {
let text = std::fs::read_to_string(&path)
.with_context(|| format!("reading {}", path.display()))?;
return Recipe::parse(&text)
.with_context(|| format!("{} is not a usable recipe", path.display()));
}
anyhow::ensure!(
checkout.join("Cargo.toml").exists(),
"this repository carries no {RECIPE_FILE} and is not a cargo workspace, so there is no \
recipe and no default that fits. Add a {RECIPE_FILE} naming a `command`."
);
Ok(Recipe::cargo_default())
}
fn collect(checkout: &Path, recipe: &Recipe) -> Result<Vec<PathBuf>> {
if !recipe.artifacts.is_empty() {
let mut out = Vec::new();
for named in &recipe.artifacts {
let path = checkout.join(named);
anyhow::ensure!(
path.is_file(),
"the recipe names {named} and the build did not produce it. A build that \
reports success without its own declared output is a build nobody checked."
);
out.push(path);
}
return Ok(out);
}
use std::os::unix::fs::PermissionsExt as _;
let dir = checkout.join("target").join("release");
let mut out = Vec::new();
if let Ok(entries) = std::fs::read_dir(&dir) {
for entry in entries.flatten() {
let path = entry.path();
let Ok(meta) = entry.metadata() else { continue };
if meta.is_file() && meta.permissions().mode() & 0o111 != 0 {
out.push(path);
}
}
}
out.sort();
anyhow::ensure!(
!out.is_empty(),
"the build produced no executable in {}. Either it built nothing, or its output is \
somewhere the default recipe does not look — name the artifacts in {RECIPE_FILE}.",
dir.display()
);
Ok(out)
}
pub fn build_commit(
stores: &ServerStores,
repo: &gunnar_core::RepoRef,
commit: gix_hash::ObjectId,
workspace: &Path,
builder: &str,
allow_network: bool,
) -> Result<Built> {
let entry = stores
.catalog()
.get(repo)?
.with_context(|| format!("no such repository: {repo}"))?;
let store = stores.repo_store(&entry)?;
let git = store.serve();
let hash = store.hash_kind();
let (kind, bytes) = git
.read(commit.as_bytes())?
.with_context(|| format!("commit {commit} is not in {repo}"))?;
anyhow::ensure!(kind == ObjType::Commit, "{commit} is not a commit");
let tree = gunnar_store::object::commit(&bytes, hash)
.with_context(|| format!("decoding commit {commit}"))?
.tree;
let workdir = workspace
.join(repo.namespace())
.join(repo.name())
.join(commit.to_string());
let _ = std::fs::remove_dir_all(&workdir);
let files = checkout(git, hash, tree, &workdir)?;
anyhow::ensure!(
files > 0,
"commit {commit} has an empty tree; there is nothing to build"
);
let recipe = recipe_for(&workdir)?;
if !allow_network {
if let Err(err) = crate::sandbox::available() {
anyhow::bail!(
"this build was to be hermetic and the sandbox cannot be entered: {}\n\n\
Refusing rather than building with the network up. A record that says \
`network: false` has to mean it on every machine. Pass --allow-network to \
build anyway; the record will say so.",
err.explain()
);
}
}
let started = gunnar_grpc::bus::now_unix_ms();
let mut command = hardened_command(&recipe.command, &workdir, allow_network);
command
.args(&recipe.args)
.env("CARGO_NET_OFFLINE", "true")
.env("GUNNAR_BUILD_COMMIT", commit.to_string())
.env("GUNNAR_BUILD_REPO", repo.to_string());
let output = command
.output()
.with_context(|| format!("running {}", recipe.command))?;
let finished = gunnar_grpc::bus::now_unix_ms();
if let Some(err) = output
.status
.code()
.and_then(crate::sandbox::error_from_exit)
{
anyhow::bail!("the build never started: {}", err.explain());
}
anyhow::ensure!(
output.status.success(),
"the build failed ({}):\n{}",
output.status,
String::from_utf8_lossy(&output.stderr)
);
let artifacts = collect(&workdir, &recipe)?;
let subjects = artifacts
.iter()
.map(|path| -> Result<gunnar_core::Subject> {
Ok(gunnar_core::Subject {
name: path
.strip_prefix(&workdir)
.unwrap_or(path)
.display()
.to_string(),
sha256: gunnar_core::provenance::digest_file(path)
.with_context(|| format!("hashing {}", path.display()))?,
})
})
.collect::<Result<Vec<_>>>()?;
let mut materials = Vec::new();
let lock = workdir.join("Cargo.lock");
if lock.is_file() {
materials.push(gunnar_core::Subject {
name: "Cargo.lock".to_owned(),
sha256: gunnar_core::provenance::digest_file(&lock)?,
});
}
let record = gunnar_core::BuildRecord {
builder: builder.to_owned(),
source_repo: repo.to_string(),
source_commit: commit.to_string(),
recipe: recipe.name.clone(),
toolchain: toolchain(&recipe.command, &workdir, allow_network),
network: allow_network,
materials,
subjects,
started_at_unix_ms: started,
finished_at_unix_ms: finished,
};
let key = stores.provenance_key()?;
let signed = gunnar_core::provenance::sign_build(&key, record.clone())
.context("signing the build record")?;
let mut line = serde_json::to_string(&signed)?;
line.push('\n');
let _framed = gunnar_core::txlog::sink::append_line(stores.build_log().records_path(), line.trim_end_matches('\n'))?;
use std::io::Write as _;
let mut file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(stores.build_log().records_path())?;
file.write_all(line.as_bytes())?;
Ok(Built { record, workdir })
}
fn hardened_command(program: &str, workdir: &Path, allow_network: bool) -> std::process::Command {
let mut command = std::process::Command::new(program);
command
.current_dir(workdir)
.env_clear()
.env("PATH", std::env::var_os("PATH").unwrap_or_default())
.env("HOME", std::env::var_os("HOME").unwrap_or_default());
if !allow_network {
use std::os::unix::process::CommandExt as _;
let enter_sandbox = || {
if let Err(err) = unsafe { crate::sandbox::enter() } {
crate::sandbox::leave_child(crate::sandbox::exit_code(err));
}
Ok(())
};
unsafe {
command.pre_exec(enter_sandbox);
}
}
command
}
fn toolchain(command: &str, workdir: &Path, allow_network: bool) -> String {
hardened_command(command, workdir, allow_network)
.arg("--version")
.output()
.ok()
.filter(|out| out.status.success())
.map(|out| String::from_utf8_lossy(&out.stdout).trim().to_owned())
.unwrap_or_default()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_watch_that_ends_is_said_and_an_error_is_an_alert() {
let (level, words) = said_end(&Err(anyhow::anyhow!("building a runtime for the runner's subscription: no threads")));
assert_eq!(level, tracing::Level::ERROR);
assert!(words.starts_with("ALERT runner"), "{words}");
assert!(words.contains("no threads") && words.contains("no longer built"), "{words}");
let (level, words) = said_end(&Ok(3));
assert_eq!(level, tracing::Level::WARN);
assert!(words.contains("3 build") && words.contains("no longer built"), "{words}");
}
#[test]
fn the_watch_returns_its_end_for_the_server_to_say() {
let dir = tempfile::tempdir().expect("a temp dir");
let stores = ServerStores::open(dir.path()).expect("stores");
let bus = gunnar_grpc::EventBus::new(4);
let config = WatchConfig { workspace: dir.path().join("ws"), builder: "runner".into(), allow_network: true, refs: Vec::new(), max_builds: Some(0) };
let end = watch(&stores, &bus, &config, |_| {});
assert_eq!(said_end(&end).0, tracing::Level::WARN, "{:?}", said_end(&end));
}
#[test]
fn a_recipe_parses_and_one_with_no_command_is_refused() {
let recipe = Recipe::parse(
r#"
name = "the appliance"
command = "make"
args = ["-j", "4"]
artifacts = ["out/gunnar"]
"#,
)
.expect("a usable recipe");
assert_eq!(recipe.name, "the appliance");
assert_eq!(recipe.command, "make");
assert_eq!(recipe.args, vec!["-j", "4"]);
assert_eq!(recipe.artifacts, vec!["out/gunnar"]);
let err = Recipe::parse("name = \"empty\"").expect_err("no command");
assert!(format!("{err}").contains("command"), "{err}");
}
#[test]
fn a_recipe_with_no_name_is_named_after_its_command() {
let recipe = Recipe::parse("command = \"cargo\"").expect("usable");
assert_eq!(recipe.name, "cargo");
}
#[test]
fn the_runner_has_exactly_one_process_constructor() {
let source = include_str!("runner.rs");
let constructors = source
.lines()
.filter(|line| !line.trim_start().starts_with("//"))
.filter(|line| !line.contains(char::from(34)))
.filter(|line| line.contains("std::process::Command::new("))
.count();
assert_eq!(
constructors, 1,
"every exec a build performs must go through `hardened_command`; \
found {constructors} fully-qualified process constructors in runner.rs"
);
assert!(
!source.contains(concat!("use std::process::", "Command;")),
"runner.rs imports the process builder by name; the count above \
only sees the fully-qualified spelling"
);
}
#[test]
fn the_hardened_command_clears_the_environment_and_sets_the_directory() {
let dir = tempfile::tempdir().expect("a temp dir");
let seen = |program: &str| -> Option<String> {
let out = hardened_command(program, dir.path(), true)
.env("GUNNAR_TEST_LEAK", "this must not cross")
.output()
.ok()?;
out.status
.success()
.then(|| String::from_utf8_lossy(&out.stdout).into_owned())
};
let Some(environment) = seen("/usr/bin/env") else {
return; };
assert!(
environment.contains("GUNNAR_TEST_LEAK=this must not cross"),
"the fixture cannot see the child's environment at all: {environment:?}"
);
for line in environment.lines().filter(|l| !l.is_empty()) {
let name = line.split('=').next().unwrap_or_default();
assert!(
matches!(name, "PATH" | "HOME" | "GUNNAR_TEST_LEAK"),
"the build child inherited {name} from the operator's shell"
);
}
if let Some(cwd) = seen("/bin/pwd") {
assert_eq!(
std::fs::canonicalize(cwd.trim()).ok(),
std::fs::canonicalize(dir.path()).ok(),
"the build child did not start in the checkout"
);
}
}
#[test]
fn the_default_recipe_locks_its_dependencies() {
assert!(
Recipe::cargo_default()
.args
.contains(&"--locked".to_owned()),
"the default cargo recipe does not pass --locked"
);
}
}
#[derive(Debug, Clone)]
pub struct WatchConfig {
pub workspace: PathBuf,
pub builder: String,
pub allow_network: bool,
pub refs: Vec<String>,
pub max_builds: Option<usize>,
}
#[derive(Debug)]
pub enum Watched {
Built(Box<Built>),
Skipped {
repo: String,
r#ref: String,
why: String,
},
Failed {
repo: String,
commit: String,
why: String,
},
}
fn wanted(config: &WatchConfig, ref_name: &str) -> bool {
if !ref_name.starts_with("refs/heads/") {
return false;
}
config.refs.is_empty() || config.refs.iter().any(|want| want == ref_name)
}
pub fn said_end(end: &Result<usize>) -> (tracing::Level, String) {
match end {
Err(err) => (tracing::Level::ERROR, format!("ALERT runner: the build runner stopped watching this server's pushes: {err:#}; pushes are no longer built until the server restarts")),
Ok(built) => (tracing::Level::WARN, format!("the build runner stopped watching this server's pushes after {built} build(s); pushes are no longer built until the server restarts")),
}
}
pub fn watch(
stores: &ServerStores,
bus: &gunnar_grpc::EventBus,
config: &WatchConfig,
mut report: impl FnMut(Watched),
) -> Result<usize> {
use tokio_stream::StreamExt as _;
let filter = gunnar_grpc::EventFilter {
namespaces: Vec::new(),
repos: Vec::new(),
kinds: vec![gunnar_proto::EventKind::RefUpdated],
include_attestation_object_ids: false,
};
let mut stream = bus.subscribe(filter);
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.context("building a runtime for the runner's subscription")?;
let mut built = 0usize;
while config.max_builds.is_none_or(|max| built < max) {
let Some(event) = runtime.block_on(stream.next()) else {
break;
};
let Ok(event) = event else { break };
let Some(gunnar_proto::event::Payload::RefUpdated(update)) = &event.payload else {
continue;
};
let Some(repo) = update.repo.as_ref() else {
continue;
};
let key = format!("{}/{}", repo.namespace, repo.repo);
if !wanted(config, &update.ref_name) {
report(Watched::Skipped {
repo: key,
r#ref: update.ref_name.clone(),
why: "not a branch this runner builds".to_owned(),
});
continue;
}
let Ok(commit) = gix_hash::ObjectId::from_hex(update.new_oid.as_bytes()) else {
report(Watched::Skipped {
repo: key,
r#ref: update.ref_name.clone(),
why: format!("{} is not an object id", update.new_oid),
});
continue;
};
let Ok(repo_ref) = gunnar_core::RepoRef::new(&repo.namespace, &repo.repo) else {
continue;
};
match build_commit(
stores,
&repo_ref,
commit,
&config.workspace,
&config.builder,
config.allow_network,
) {
Ok(outcome) => {
built += 1;
report(Watched::Built(Box::new(outcome)));
}
Err(err) => report(Watched::Failed {
repo: key,
commit: commit.to_string(),
why: format!("{err:#}"),
}),
}
}
Ok(built)
}