jam_pvm_builder/

lib.rs

//! Builder logic for creating PVM code blobs for execution on the JAM PVM instances (service code
//! and authorizer code).
//!
//! # Reproducibility
//!
//! Two pieces here drive cross-machine reproducibility of the produced blob:
//!
//! - **Path remapping (`build_encoded_rustflags`).** Adds `--remap-path-prefix` directives that
//!   rewrite `$HOME`, `$RUSTUP_HOME`, `$CARGO_HOME`, and each workspace-member path to stable roots
//!   (`~/`, `~/.rustup`, `~/.cargo`, `/crate/<name>`). This prevents the user's actual home, cargo
//!   cache, and workspace location from leaking into `file!()` strings and embedded debuginfo.
//!   Sufficient to make builds byte-identical across different machines running the same host OS
//!   (tested on Linux).
//!
//! - **`RUSTC_WRAPPER` forwarding (`build_pvm_blob`).** The inner cargo invocation does
//!   `env_clear()` and then selectively forwards a few env vars; `RUSTC_WRAPPER` is one of them
//!   when the caller sets it. Used to additionally normalise cross-OS (Linux vs macOS) blob output
//!   via a custom wrapper that rewrites cargo's per-crate `-Cmetadata` to a host-independent value.
//!   Wrapper lives at `scripts/rustc-wrapper-fixed-metadata.sh` in this repository; see its header
//!   for details. Set it like:
//!
//!   ```text
//!   RUSTC_WRAPPER=$(pwd)/scripts/rustc-wrapper-fixed-metadata.sh cargo build -p polkajam-fuzz
//!   ```
//!
//!   Default builds (no `RUSTC_WRAPPER` exported) only get the path-remap form of
//!   reproducibility — that's the same-OS guarantee.

#![allow(clippy::unwrap_used)]

// If you update this, you should also update the toolchain installed by .github/workflows/rust.yml
const TOOLCHAIN: &str = "nightly-2025-05-10";

use codec::Encode;
use jam_program_blob_common::{ConventionalMetadata, CoreVmProgramBlob, CrateInfo, ProgramBlob};
use std::{
	fmt::Display,
	fs,
	path::{Path, PathBuf},
	process::Command,
	sync::OnceLock,
};

/// Program blob type.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum BlobType {
	/// JAM service (`jam_pvm_common::Service`).
	Service,
	/// JAM authorizer (`jam_pvm_common::Authorizer`).
	Authorizer,
	/// CoreVM guest program (`corevm_guest` crate).
	CoreVmGuest,
}

impl Display for BlobType {
	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
		std::fmt::Debug::fmt(self, f)
	}
}

impl BlobType {
	pub fn dispatch_table(&self) -> Vec<Vec<u8>> {
		match self {
			Self::Service => vec![b"refine_ext".into(), b"accumulate_ext".into()],
			Self::Authorizer => vec![b"is_authorized_ext".into()],
			Self::CoreVmGuest => Vec::new(),
		}
	}

	/// Get output file path for the specified crate name and output directory.
	pub fn output_file(&self, out_dir: &Path, crate_name: &str) -> PathBuf {
		let suffix = match self {
			Self::Service | Self::Authorizer => "jam",
			Self::CoreVmGuest => "corevm",
		};
		out_dir.join(format!("{crate_name}.{suffix}"))
	}
}

pub enum ProfileType {
	Debug,
	Release,
	Other(&'static str),
}
impl ProfileType {
	fn as_str(&self) -> &'static str {
		match self {
			ProfileType::Debug => "debug",
			ProfileType::Release => "release",
			ProfileType::Other(s) => s,
		}
	}
	fn to_arg(&self) -> String {
		match self {
			ProfileType::Debug => "--debug".into(),
			ProfileType::Release => "--release".into(),
			ProfileType::Other(s) => format!("--profile={s}"),
		}
	}

	/// Whether this profile should optimise for a small, opaque blob.
	/// Disabled for debug builds.
	fn is_release_like(&self) -> bool {
		!matches!(self, ProfileType::Debug)
	}
}

fn build_pvm_blob_in_build_script(crate_dir: &Path, blob_type: BlobType) {
	let out_dir: PathBuf = std::env::var("OUT_DIR").expect("No OUT_DIR").into();
	println!("cargo:rerun-if-env-changed=SKIP_PVM_BUILDS");
	println!("cargo:rerun-if-env-changed=PVM_BUILDER_STRIP");
	if std::env::var_os("SKIP_PVM_BUILDS").is_some() {
		let crate_name = get_crate_info(crate_dir).name;
		let output_file = blob_type.output_file(&out_dir, &crate_name);
		fs::write(&output_file, []).expect("error creating dummy program blob");
		println!("cargo:rustc-env=PVM_BINARY_{crate_name}={}", output_file.display());
		let hash_output_file = out_dir.join("{crate_name}.hash");
		fs::write(&hash_output_file, [0_u8; 32]).expect("error creating dummy program blob hash");
		println!("cargo:rustc-env=PVM_BINARY_HASH_{crate_name}={}", hash_output_file.display());
	} else {
		println!("cargo:rerun-if-changed={}", crate_dir.to_str().unwrap());
		let (crate_name, output_file, hash_output_file) =
			build_pvm_blob(crate_dir, blob_type, &out_dir, false, ProfileType::Other("production"));
		println!("cargo:rustc-env=PVM_BINARY_{crate_name}={}", output_file.display());
		println!("cargo:rustc-env=PVM_BINARY_HASH_{crate_name}={}", hash_output_file.display());
	}
}

/// Build the service crate in `crate_dir` for PVM.
///
/// Outputs
/// - `{out_dir}/{crate_name}.jam` - JAM program blob,
/// - `{out_dir}/{crate_name}.polkavm` - PolkaVM program blob for debugging.
///
/// The blob may be included in the relevant crate by using the [`pvm_binary`] macro.
pub fn build_service(crate_dir: &Path) {
	build_pvm_blob_in_build_script(crate_dir, BlobType::Service);
}

/// Build the authorizer crate in `crate_dir` for PVM.
///
/// Outputs
/// - `{out_dir}/{crate_name}.jam` - JAM program blob,
/// - `{out_dir}/{crate_name}.polkavm` - PolkaVM program blob for debugging.
///
/// The blob may be included in the relevant crate by using the [`pvm_binary`] macro.
pub fn build_authorizer(crate_dir: &Path) {
	build_pvm_blob_in_build_script(crate_dir, BlobType::Authorizer);
}

/// Build the CoreVM guest program crate in `crate_dir` for PVM.
///
/// Outputs
/// - `{out_dir}/{crate_name}.corevm` - CoreVM program blob,
/// - `{out_dir}/{crate_name}.polkavm` - PolkaVM program blob for debugging.
///
/// The blob may be included in the relevant crate by using the [`pvm_binary`] macro.
pub fn build_corevm_guest(crate_dir: &Path) {
	build_pvm_blob_in_build_script(crate_dir, BlobType::CoreVmGuest);
}

/// Build the `CARGO_ENCODED_RUSTFLAGS` value for the PVM build.
///
/// `--remap-path-prefix` directives prevent absolute paths (rust-src, cargo registry, workspace)
/// from leaking into `file!()` strings embedded in the PVM blob. Without them the blob - and
/// therefore its hash - is not reproducible across machines or toolchain installs.
///
/// Remap targets are chosen so PolkaVM's `source_cache` can resolve embedded paths back to local
/// files on whatever machine later inspects the blob: it strips a leading `~/` and joins with the
/// running process's own `$HOME`, so an embedded `~/.rustup/toolchains/<channel>/...` finds the
/// equivalent file under any user who has that toolchain installed via rustup. Workspace members
/// each get a stable `/crate/<name>` root.
///
/// Note: in release-like builds `panic_immediate_abort` (set in `build-std-features` below)
/// lowers panics to `abort()` without consuming the `&Location` propagated by `#[track_caller]`,
/// so the file/line/column literals are DCE'd. In debug builds those literals reach rodata, but
/// always in remapped form, which keeps cross-host reproducibility.
fn build_encoded_rustflags(crate_dir: &Path) -> String {
	let mut flags: Vec<String> = vec!["-C".into(), "panic=abort".into()];

	// Order matters: rustc applies the LAST matching `--remap-path-prefix`, so the broadest
	// catch-all goes first and the most specific override goes last.
	let home = std::env::var("HOME").ok();
	if let Some(h) = home.as_deref() {
		flags.push(format!("--remap-path-prefix={h}=~"));
	}
	let rustup = std::env::var("RUSTUP_HOME")
		.ok()
		.or_else(|| home.as_deref().map(|h| format!("{h}/.rustup")));
	if let Some(p) = rustup {
		flags.push(format!("--remap-path-prefix={p}=~/.rustup"));
	}
	let cargo = std::env::var("CARGO_HOME")
		.ok()
		.or_else(|| home.as_deref().map(|h| format!("{h}/.cargo")));
	if let Some(p) = cargo {
		flags.push(format!("--remap-path-prefix={p}=~/.cargo"));
	}
	for (name, path) in workspace_members(crate_dir) {
		flags.push(format!("--remap-path-prefix={}=/crate/{name}", path.display()));
	}

	flags.join("\x1f")
}

/// Enumerate workspace members visible from `crate_dir` as `(name, manifest_dir)` pairs.
fn workspace_members(crate_dir: &Path) -> impl Iterator<Item = (String, PathBuf)> {
	let packages = (|| -> Option<Vec<serde_json::Value>> {
		let output = Command::new("cargo")
			.current_dir(crate_dir)
			.args(["metadata", "--no-deps", "--format-version", "1"])
			.output()
			.ok()
			.filter(|o| o.status.success())?;
		let mut meta: serde_json::Value = serde_json::from_slice(&output.stdout).ok()?;
		match meta.get_mut("packages")?.take() {
			serde_json::Value::Array(arr) => Some(arr),
			_ => None,
		}
	})()
	.unwrap_or_default();

	packages.into_iter().filter_map(|pkg| {
		let name = pkg.get("name")?.as_str()?.to_string();
		let manifest = pkg.get("manifest_path")?.as_str()?;
		let path = Path::new(manifest).parent()?.to_path_buf();
		Some((name, path))
	})
}

fn get_crate_info(crate_dir: &Path) -> CrateInfo {
	let read_manifest_output = Command::new("cargo")
		.current_dir(crate_dir)
		.arg("read-manifest")
		.output()
		.unwrap_or_else(|err| {
			panic!("Failed to run `cargo read-manifest` in {}: {err}", crate_dir.display());
		});
	if !read_manifest_output.status.success() {
		panic!(
			"Failed to read Cargo.toml manifest in {}:\n{}",
			crate_dir.display(),
			String::from_utf8_lossy(&read_manifest_output.stderr)
		);
	}
	let man = serde_json::from_slice::<serde_json::Value>(&read_manifest_output.stdout).unwrap();
	// read-manifest output should always contain a valid name/version
	let name = man.get("name").unwrap().as_str().unwrap().to_string();
	let version = man.get("version").unwrap().as_str().unwrap().to_string();
	// read-manifest output contains "license": null when no license is specified in the Cargo.toml
	let license = man
		.get("license")
		.unwrap()
		.as_str()
		.unwrap_or_else(|| {
			panic!("No license specified in Cargo.toml manifest in {}", crate_dir.display());
		})
		.to_string();
	// read-manifest output should always contain a valid authors list
	let authors = man
		.get("authors")
		.unwrap()
		.as_array()
		.unwrap()
		.iter()
		.map(|x| x.as_str().unwrap().to_owned())
		.collect::<Vec<String>>();
	CrateInfo { name, version, license, authors }
}

/// Build the PVM crate in `crate_dir` for the RISCV target.
///
/// Outputs
/// - depending on the `blob_type` either [JAM program blob](jam_program_blob_common::ProgramBlob)
///   or [CoreVM program blob](jam_program_blob_common::CoreVmProgramBlob) as
///   `{out_dir}/{crate_name}.jam` or `{out_dir}/{crate_name}.corevm` respectively;
/// - [PolkaVM program blob](polkavm_linker::ProgramBlob) as `{out_dir}/{crate_name}.polkavm` for
///   debugging.
/// - Blake2b hash of the PolkaVM program blob as `{out_dir}/{crate_name}.hash`.
///
/// `out_dir` is used to store any intermediate build files.
pub fn build_pvm_blob(
	crate_dir: &Path,
	blob_type: BlobType,
	out_dir: &Path,
	install_rustc: bool,
	profile: ProfileType,
) -> (String, PathBuf, PathBuf) {
	let mut args = polkavm_linker::TargetJsonArgs::default();
	args.is_64_bit = true;
	args.rustc_version = polkavm_linker::RustcVersion::Legacy;

	let (target_name, target_json_path) =
		("riscv64emac-unknown-none-polkavm", polkavm_linker::target_json_path(args).unwrap());

	println!("🪤 PVM module type: {blob_type}");
	println!("🎯 Target name: {target_name}");

	let rustup_installed = if Command::new("rustup").output().is_ok() {
		let output = Command::new("rustup")
			.args(["component", "list", "--toolchain", TOOLCHAIN, "--installed"])
			.output()
			.unwrap_or_else(|_| {
				panic!(
				"Failed to execute `rustup component list --toolchain {TOOLCHAIN} --installed`.\n\
		Please install `rustup` to continue.",
			)
			});

		if !output.status.success() ||
			!output.stdout.split(|x| *x == b'\n').any(|x| x[..] == b"rust-src"[..])
		{
			if install_rustc {
				println!("Installing rustc dependencies...");
				let mut child = Command::new("rustup")
					.args(["toolchain", "install", TOOLCHAIN, "-c", "rust-src"])
					.stdout(std::process::Stdio::inherit())
					.stderr(std::process::Stdio::inherit())
					.spawn()
					.unwrap_or_else(|_| {
						panic!(
						"Failed to execute `rustup toolchain install {TOOLCHAIN} -c rust-src`.\n\
				Please install `rustup` to continue."
					)
					});
				if !child.wait().expect("Failed to execute rustup process").success() {
					panic!("Failed to install `rust-src` component of {TOOLCHAIN}.");
				}
			} else {
				panic!("`rust-src` component of {TOOLCHAIN} is required to build the PVM binary.",);
			}
		}
		println!("ℹ️ `rustup` and toolchain installed. Continuing build process...");

		true
	} else {
		println!("ℹ️ `rustup` not installed, here be dragons. Continuing build process...");

		false
	};

	let info = get_crate_info(crate_dir);
	println!("📦 Crate name: {}", info.name);
	println!("🏷️ Build profile: {}", profile.as_str());

	let mut child = Command::new("cargo");

	child
		.current_dir(crate_dir)
		.env_clear()
		.env("PATH", std::env::var("PATH").unwrap())
		.env("CARGO_ENCODED_RUSTFLAGS", build_encoded_rustflags(crate_dir))
		.env("CARGO_TARGET_DIR", out_dir)
		// Support building on stable. (required for `-Zbuild-std`)
		.env("RUSTC_BOOTSTRAP", "1");

	// Forward `RUSTC_WRAPPER` if set in the parent env. Used by callers that need
	// cross-host-reproducible PVM blobs
	// (e.g. `RUSTC_WRAPPER=scripts/rustc-wrapper-fixed-metadata.sh cargo build -p polkajam-fuzz`)
	// the wrapper can rewrite `-Cmetadata` so rustc's `StableCrateId` is identical on Linux and
	// macOS. Default builds inherit no wrapper.
	if let Some(w) = std::env::var_os("RUSTC_WRAPPER") {
		child.env("RUSTC_WRAPPER", w);
	}

	if rustup_installed {
		child.arg(format!("+{TOOLCHAIN}"));
	}

	child.args(["rustc", "--lib", "--crate-type=cdylib", "-Z", "build-std=core,alloc"]);
	if profile.is_release_like() {
		// Lowers every panic site to a direct `intrinsics::abort()`,
		// so panic messages and `track_caller` `&Location` data get DCE'd out of rodata.
		child.args(["-Z", "build-std-features=panic_immediate_abort"]);
	}
	child.arg(profile.to_arg()).arg("--target").arg(target_json_path);

	// Use job server to not oversubscribe CPU cores when compiling multiple PVM binaries in
	// parallel.
	if let Some(client) = get_job_server_client() {
		client.configure(&mut child);
	}

	let mut child = child.spawn().expect("Failed to execute cargo process");
	let status = child.wait().expect("Failed to execute cargo process");

	if !status.success() {
		eprintln!("Failed to build RISC-V ELF due to cargo execution error");
		std::process::exit(1);
	}

	// Post processing
	println!("Converting RISC-V ELF to PVM blob...");
	let mut config = polkavm_linker::Config::default();
	config.set_strip(std::env::var("PVM_BUILDER_STRIP").map(|value| value == "1").unwrap_or(true));
	config.set_dispatch_table(blob_type.dispatch_table());

	let input_root = &out_dir.join(target_name).join(profile.as_str());
	let input_path_bin = input_root.join(&info.name);
	let input_path_cdylib = input_root.join(format!("{}.elf", info.name.replace("-", "_")));

	let input_path = if input_path_cdylib.exists() {
		if input_path_bin.exists() {
			eprintln!(
				"Both {} and {} exist; run 'cargo clean' to get rid of old artifacts!",
				input_path_cdylib.display(),
				input_path_bin.display()
			);
			std::process::exit(1);
		}
		input_path_cdylib
	} else if input_path_bin.exists() {
		input_path_bin
	} else {
		eprintln!(
			"Failed to build: neither {} nor {} exist",
			input_path_cdylib.display(),
			input_path_bin.display()
		);
		std::process::exit(1);
	};

	let orig =
		fs::read(&input_path).unwrap_or_else(|e| panic!("Failed to read {input_path:?} :{e:?}"));
	let linked = polkavm_linker::program_from_elf(
		config,
		polkavm_linker::TargetInstructionSet::JamV1,
		orig.as_ref(),
	)
	.expect("Failed to link pvm program:");

	// Write out a full `.polkavm` blob for debugging/inspection.
	let output_path_pvm = out_dir.join(format!("{}.polkavm", info.name));
	let hash_output_file = out_dir.join(format!("{}.hash", info.name));
	fs::write(&output_path_pvm, &linked).expect("Error writing resulting binary");
	let name = info.name.clone();
	let metadata = ConventionalMetadata::Info(info).encode().into();
	let output_file = blob_type.output_file(out_dir, &name);
	let blob = if !matches!(blob_type, BlobType::CoreVmGuest) {
		let parts = polkavm_linker::ProgramParts::from_bytes(linked.into())
			.expect("failed to deserialize linked PolkaVM program");
		let blob = ProgramBlob::from_pvm(&parts, metadata)
			.to_vec()
			.expect("error serializing the .jam blob");
		fs::write(&output_file, &blob).expect("error writing the .jam blob");
		blob
	} else {
		let blob = CoreVmProgramBlob { metadata, pvm_blob: linked.into() }
			.to_vec()
			.expect("error serializing the CoreVM blob");
		fs::write(&output_file, &blob).expect("error writing the CoreVM blob");
		blob
	};
	let hash = code_hash(&blob);
	fs::write(&hash_output_file, hash).expect("error writing blob hash");

	(name, output_file, hash_output_file)
}

/// Returns the hash of the code blob.
///
/// Should produce the same value as [`pvm_binary_hash`] macro.
pub fn code_hash(data: &[u8]) -> [u8; 32] {
	let h = blake2b_simd::Params::new().hash_length(32).hash(data);
	h.as_bytes().try_into().expect("Hash length set to 32")
}

fn get_job_server_client() -> Option<&'static jobserver::Client> {
	static CLIENT: OnceLock<Option<jobserver::Client>> = OnceLock::new();
	CLIENT.get_or_init(|| unsafe { jobserver::Client::from_env() }).as_ref()
}

/// Returns the resulting blob as a byte slice.
#[macro_export]
macro_rules! pvm_binary {
	($name:literal) => {
		include_bytes!(env!(concat!("PVM_BINARY_", $name)))
	};
}

/// Returns the resulting blob hash as a byte slice.
///
/// Should produce the same value as [`code_hash`] but at compile time, i.e. `pvm_binary_hash!(name)
/// == &code_hash(pvm_binary!(name))`.
#[macro_export]
macro_rules! pvm_binary_hash {
	($name:literal) => {
		include_bytes!(env!(concat!("PVM_BINARY_HASH_", $name)))
	};
}