pub mod vulkan_host;
use std::path::Path;
use buildlang::ast;
use buildlang::codegen::{CodeGenerator, Target};
use buildlang::lexer::{Lexer, SourceFile};
use buildlang::parser::Parser;
use buildlang::types::{TypeChecker, TypeContext};
const N: usize = 1024;
const LOCAL_SIZE_X: u32 = 64;
const TOLERANCE: f32 = 1e-6;
const MM_M: usize = 64;
const MM_K: usize = 64;
const MM_N: usize = 64;
const LOCAL_SIZE_2D: (u32, u32) = (16, 16);
const STENCIL_N: usize = 1024;
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
enum ScalarKind {
F32,
F64,
U32,
}
#[derive(Clone, Debug)]
enum KernelParam {
Scalar { name: String, kind: ScalarKind },
Buffer {
name: String,
writable: bool,
elem: ScalarKind,
},
}
#[derive(Clone, Debug)]
struct KernelSig {
entry: String,
params: Vec<KernelParam>,
}
impl KernelSig {
fn buffers(&self) -> impl Iterator<Item = (&str, bool, ScalarKind)> {
self.params.iter().filter_map(|p| match p {
KernelParam::Buffer {
name,
writable,
elem,
} => Some((name.as_str(), *writable, *elem)),
KernelParam::Scalar { .. } => None,
})
}
fn scalars(&self) -> impl Iterator<Item = (&str, ScalarKind)> {
self.params.iter().filter_map(|p| match p {
KernelParam::Scalar { name, kind } => Some((name.as_str(), *kind)),
KernelParam::Buffer { .. } => None,
})
}
fn is_stencil(&self) -> bool {
let u32_scalars = self
.scalars()
.filter(|(_, k)| *k == ScalarKind::U32)
.count();
let other_scalars = self
.scalars()
.filter(|(_, k)| *k != ScalarKind::U32)
.count();
let inputs = self.buffers().filter(|(_, w, _)| !*w).count();
let outputs = self.buffers().filter(|(_, w, _)| *w).count();
u32_scalars == 1 && other_scalars == 0 && inputs == 1 && outputs == 1
}
fn output_buffer_index(&self) -> Result<usize, String> {
let writable: Vec<usize> = self
.buffers()
.enumerate()
.filter(|(_, (_, w, _))| *w)
.map(|(i, _)| i)
.collect();
match writable.as_slice() {
[i] => Ok(*i),
[] => Err("kernel has no `&mut [_]` output buffer".to_string()),
_ => Err(
"Phase-1 elementwise GPU path supports exactly one `&mut [_]` output \
buffer"
.to_string(),
),
}
}
}
fn ast_scalar_kind(ty: &ast::Type) -> Option<ScalarKind> {
if let ast::TypeKind::Path(path) = &ty.kind {
if let Some(ident) = path.last_ident() {
return match ident.name.as_ref() {
"f32" => Some(ScalarKind::F32),
"f64" => Some(ScalarKind::F64),
"u32" => Some(ScalarKind::U32),
_ => None,
};
}
}
None
}
fn discover_kernel_sig(source_path: &Path) -> Result<KernelSig, String> {
let text = std::fs::read_to_string(source_path)
.map_err(|e| format!("read {}: {e}", source_path.display()))?;
let source_file = SourceFile::new(source_path.to_string_lossy(), text);
let mut lexer = Lexer::new(&source_file);
let tokens = lexer.tokenize().map_err(|e| format!("lex: {e}"))?;
let mut parser = Parser::new(&source_file, tokens);
let module = parser.parse().map_err(|e| format!("parse: {e}"))?;
let mut found: Option<KernelSig> = None;
for item in &module.items {
let ast::ItemKind::Function(f) = &item.kind else {
continue;
};
let is_compute = item.attrs.iter().any(|a| {
a.path
.segments
.first()
.map(|s| s.ident.name.as_ref() == "compute")
.unwrap_or(false)
});
if !is_compute {
continue;
}
if found.is_some() {
return Err(
"more than one `#[compute]` kernel in the file; the GPU path \
expects exactly one"
.to_string(),
);
}
let entry = f.name.name.to_string();
let mut params = Vec::with_capacity(f.sig.params.len());
for p in &f.sig.params {
let name = match &p.pattern.kind {
ast::PatternKind::Ident { name, .. } => name.name.to_string(),
_ => return Err("compute kernel parameters must be simple identifiers".to_string()),
};
match &p.ty.kind {
ast::TypeKind::Ref { mutability, ty, .. }
if matches!(ty.kind, ast::TypeKind::Slice(_)) =>
{
let ast::TypeKind::Slice(elem) = &ty.kind else {
unreachable!()
};
let elem = ast_scalar_kind(elem).filter(|k| *k != ScalarKind::U32);
let elem = elem.ok_or_else(|| {
format!("buffer parameter `{name}` must be a float slice (`&[f32]`)")
})?;
params.push(KernelParam::Buffer {
name,
writable: matches!(mutability, ast::Mutability::Mutable),
elem,
});
}
_ => {
let kind = ast_scalar_kind(&p.ty).ok_or_else(|| {
format!(
"parameter `{name}` has an unsupported type for the GPU path \
(Phase 1: float scalars or float slices)"
)
})?;
params.push(KernelParam::Scalar { name, kind });
}
}
}
found = Some(KernelSig { entry, params });
}
found.ok_or_else(|| "no `#[compute]` kernel found in the file".to_string())
}
fn compile_to(source_path: &Path, target: Target) -> Result<Vec<u8>, String> {
let text = std::fs::read_to_string(source_path)
.map_err(|e| format!("read {}: {e}", source_path.display()))?;
let source_file = SourceFile::new(source_path.to_string_lossy(), text);
let mut lexer = Lexer::new(&source_file);
let tokens = lexer.tokenize().map_err(|e| format!("lex: {e}"))?;
let mut parser = Parser::new(&source_file, tokens);
let ast = parser.parse().map_err(|e| format!("parse: {e}"))?;
let mut ctx = TypeContext::new();
let mut checker = TypeChecker::new(&mut ctx);
checker.set_source_file(&source_file);
checker.check_module(&ast);
if checker.has_errors() {
let msgs: Vec<String> = checker
.errors()
.iter()
.map(|e| e.error.to_string())
.collect();
return Err(format!("type errors: {}", msgs.join("; ")));
}
let mut codegen = CodeGenerator::with_source(&ctx, target, source_file.source().into());
let generated = codegen
.generate(&ast)
.map_err(|e| format!("codegen: {e}"))?;
Ok(generated.data)
}
fn bytes_to_words(bytes: &[u8]) -> Result<Vec<u32>, String> {
if bytes.len() % 4 != 0 {
return Err("SPIR-V blob length is not a multiple of 4".to_string());
}
Ok(bytes
.chunks_exact(4)
.map(|c| u32::from_le_bytes([c[0], c[1], c[2], c[3]]))
.collect())
}
fn extract_c_function(c_source: &str, name: &str) -> Option<String> {
let needle = format!("static void {}(", name);
let mut search_from = 0;
loop {
let rel = c_source[search_from..].find(&needle)?;
let start = search_from + rel;
let after_sig = &c_source[start..];
let brace = after_sig.find('{');
let semi = after_sig.find(';');
match (brace, semi) {
(Some(b), Some(s)) if s < b => {
search_from = start + s + 1;
continue;
}
(Some(b), _) => {
let body_start = start + b;
let mut depth = 0i32;
for (i, ch) in c_source[body_start..].char_indices() {
match ch {
'{' => depth += 1,
'}' => {
depth -= 1;
if depth == 0 {
let end = body_start + i + 1;
return Some(c_source[start..end].to_string());
}
}
_ => {}
}
}
return None;
}
_ => return None,
}
}
}
fn normalize_slice_type(kernel_fn: &str) -> String {
let exact = "struct { float* ptr; size_t len; }";
if kernel_fn.contains(exact) {
return kernel_fn.replace(exact, "bl_slice_f32");
}
let mut result = String::with_capacity(kernel_fn.len());
let mut i = 0;
while i < kernel_fn.len() {
if kernel_fn[i..].starts_with("struct") {
if let Some(open_rel) = kernel_fn[i..].find('{') {
let open = i + open_rel;
let mut depth = 0i32;
let mut close = None;
for (j, ch) in kernel_fn[open..].char_indices() {
match ch {
'{' => depth += 1,
'}' => {
depth -= 1;
if depth == 0 {
close = Some(open + j);
break;
}
}
_ => {}
}
}
if let Some(close) = close {
let interior = &kernel_fn[open + 1..close];
if interior.contains("ptr") && interior.contains("len") {
result.push_str("bl_slice_f32");
i = close + 1;
continue;
}
}
}
}
result.push_str(&kernel_fn[i..i + 1]);
i += 1;
}
result
}
fn cpu_c_reference(
sig: &KernelSig,
kernel_fn_c: &str,
buffer_data: &[(&str, Vec<f32>)],
scalar_vals: &[(&str, f32)],
n: usize,
) -> Result<Vec<f32>, String> {
let fmt_arr = |v: &[f32]| -> String {
v.iter()
.map(|x| format!("{:.9}f", x))
.collect::<Vec<_>>()
.join(", ")
};
let mut decls = String::new();
for (name, data) in buffer_data {
decls.push_str(&format!(
" static float {name}_data[{n}] = {{ {init} }};\n",
name = name,
n = n,
init = fmt_arr(data),
));
decls.push_str(&format!(
" bl_slice_f32 {name} = {{ {name}_data, {n} }};\n",
name = name,
n = n,
));
}
for (name, val) in scalar_vals {
decls.push_str(&format!(
" float {name} = {val:.9}f;\n",
name = name,
val = val
));
}
let call_args: Vec<String> = sig
.params
.iter()
.map(|p| match p {
KernelParam::Scalar { name, .. } => name.clone(),
KernelParam::Buffer { name, .. } => format!("&{name}"),
})
.collect();
let out_name = sig
.buffers()
.find(|(_, w, _)| *w)
.map(|(n, _, _)| n.to_string())
.ok_or_else(|| "kernel has no writable output buffer".to_string())?;
let driver = format!(
r#"{prelude}
{kernel}
int main(void) {{
{decls} for (uint32_t i = 0; i < {n}; ++i) {{
buildc_gl_global_invocation_x = i;
{entry}({args});
}}
for (size_t i = 0; i < {n}; ++i) {{
printf("%.9g\n", (double){out}_data[i]);
}}
return 0;
}}
"#,
kernel = kernel_fn_c,
prelude = driver_prelude(),
decls = decls,
entry = sig.entry,
args = call_args.join(", "),
out = out_name,
n = n,
);
let dir = std::env::temp_dir().join(format!("buildlang_gpu_cpu_{}", std::process::id()));
std::fs::create_dir_all(&dir).map_err(|e| format!("create temp dir: {e}"))?;
let c_path = dir.join("cpu_ref.c");
std::fs::write(&c_path, driver).map_err(|e| format!("write cpu_ref.c: {e}"))?;
let exe_path = dir.join(if cfg!(windows) {
"cpu_ref.exe"
} else {
"cpu_ref"
});
compile_c(&c_path, &exe_path)?;
let output = std::process::Command::new(&exe_path)
.output()
.map_err(|e| format!("run cpu_ref: {e}"))?;
if !output.status.success() {
return Err(format!(
"cpu_ref exited non-zero: {}",
String::from_utf8_lossy(&output.stderr)
));
}
let stdout = String::from_utf8_lossy(&output.stdout);
let values: Vec<f32> = stdout
.lines()
.filter(|l| !l.trim().is_empty())
.map(|l| {
l.trim()
.parse::<f32>()
.map_err(|e| format!("parse cpu output '{l}': {e}"))
})
.collect::<Result<Vec<_>, _>>()?;
if values.len() != n {
return Err(format!(
"cpu_ref produced {} values, expected {}",
values.len(),
n
));
}
Ok(values)
}
fn driver_prelude() -> &'static str {
r#"#include <stdio.h>
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
/* Ambient GPU thread-index vars the kernel body reads; the driver sets them per invocation. */
uint32_t buildc_gl_global_invocation_x;
uint32_t buildc_gl_global_invocation_y;
typedef struct { float* ptr; size_t len; } bl_slice_f32;
typedef bl_slice_f32 BlSlice_f32;
"#
}
fn extract_kernel_with_runtime(c_source: &str, entry: &str) -> Option<String> {
let marker = c_source.find("// End BuildLang Runtime")?;
let line_end = |from: usize| c_source[from..].find('\n').map(|i| from + i + 1);
let runtime_end = line_end(line_end(marker)?)?;
let kernel = normalize_slice_type(&extract_c_function(c_source, entry)?);
Some(format!("{}\n{}", &c_source[..runtime_end], kernel))
}
fn compile_c(c_path: &Path, exe_path: &Path) -> Result<(), String> {
let compiler = crate::find_c_compiler()
.ok_or_else(|| "no C compiler found to build the CPU reference".to_string())?;
crate::invoke_c_compiler(&compiler, c_path, exe_path, false, &[])
.map_err(|code| format!("C compiler failed (exit {code}) building the CPU reference"))
}
fn kernel_is_2d(source_path: &Path) -> Result<bool, String> {
let text = std::fs::read_to_string(source_path)
.map_err(|e| format!("read {}: {e}", source_path.display()))?;
Ok(text.contains("gl_GlobalInvocationID.y"))
}
#[allow(clippy::too_many_arguments)]
fn cpu_c_matmul_reference(
sig: &KernelSig,
kernel_fn_c: &str,
a: &[f32],
b: &[f32],
m: usize,
k: usize,
n: usize,
) -> Result<Vec<f32>, String> {
let fmt_arr = |v: &[f32]| -> String {
v.iter()
.map(|x| format!("{:.9}f", x))
.collect::<Vec<_>>()
.join(", ")
};
let buffers: Vec<(&str, bool)> = sig.buffers().map(|(name, w, _)| (name, w)).collect();
if buffers.len() != 3 {
return Err(format!(
"matmul cross-check expects exactly three buffers (a, b, c); found {}",
buffers.len()
));
}
let (a_name, _) = buffers[0];
let (b_name, _) = buffers[1];
let (c_name, c_writable) = buffers[2];
if !c_writable {
return Err("matmul output buffer (third) must be `&mut [f32]`".to_string());
}
let mut decls = String::new();
decls.push_str(&format!(
" static float {a_name}_data[{la}] = {{ {ai} }};\n bl_slice_f32 {a_name} = {{ {a_name}_data, {la} }};\n",
la = m * k,
ai = fmt_arr(a),
));
decls.push_str(&format!(
" static float {b_name}_data[{lb}] = {{ {bi} }};\n bl_slice_f32 {b_name} = {{ {b_name}_data, {lb} }};\n",
lb = k * n,
bi = fmt_arr(b),
));
decls.push_str(&format!(
" static float {c_name}_data[{lc}] = {{ 0 }};\n bl_slice_f32 {c_name} = {{ {c_name}_data, {lc} }};\n",
lc = m * n,
));
decls.push_str(&format!(
" uint32_t m = {m}; uint32_t k = {k}; uint32_t n = {n};\n"
));
let call_args: Vec<String> = sig
.params
.iter()
.map(|p| match p {
KernelParam::Scalar { name, .. } => name.clone(),
KernelParam::Buffer { name, .. } => format!("&{name}"),
})
.collect();
let driver = format!(
r#"{prelude}
{kernel}
int main(void) {{
{decls} for (uint32_t gy = 0; gy < m; ++gy) {{
for (uint32_t gx = 0; gx < n; ++gx) {{
buildc_gl_global_invocation_y = gy;
buildc_gl_global_invocation_x = gx;
{entry}({args});
}}
}}
for (size_t i = 0; i < (size_t)m * (size_t)n; ++i) {{
printf("%.9g\n", (double){out}_data[i]);
}}
return 0;
}}
"#,
kernel = kernel_fn_c,
prelude = driver_prelude(),
decls = decls,
entry = sig.entry,
args = call_args.join(", "),
out = c_name,
);
let dir = std::env::temp_dir().join(format!("buildlang_gpu_mm_{}", std::process::id()));
std::fs::create_dir_all(&dir).map_err(|e| format!("create temp dir: {e}"))?;
let c_path = dir.join("cpu_ref_mm.c");
std::fs::write(&c_path, driver).map_err(|e| format!("write cpu_ref_mm.c: {e}"))?;
let exe_path = dir.join(if cfg!(windows) {
"cpu_ref_mm.exe"
} else {
"cpu_ref_mm"
});
compile_c(&c_path, &exe_path)?;
let output = std::process::Command::new(&exe_path)
.output()
.map_err(|e| format!("run cpu_ref_mm: {e}"))?;
if !output.status.success() {
return Err(format!(
"cpu_ref_mm exited non-zero: {}",
String::from_utf8_lossy(&output.stderr)
));
}
let stdout = String::from_utf8_lossy(&output.stdout);
let values: Vec<f32> = stdout
.lines()
.filter(|l| !l.trim().is_empty())
.map(|l| {
l.trim()
.parse::<f32>()
.map_err(|e| format!("parse cpu output '{l}': {e}"))
})
.collect::<Result<Vec<_>, _>>()?;
if values.len() != m * n {
return Err(format!(
"cpu_ref_mm produced {} values, expected {}",
values.len(),
m * n
));
}
Ok(values)
}
fn parse_mm_dims_override() -> Option<(usize, usize, usize)> {
let raw = std::env::var("BUILDLANG_MM_DIMS").ok()?;
let parts: Vec<&str> = raw.split(['x', 'X']).collect();
if parts.len() != 3 {
return None;
}
let m = parts[0].trim().parse::<usize>().ok()?;
let k = parts[1].trim().parse::<usize>().ok()?;
let n = parts[2].trim().parse::<usize>().ok()?;
if m == 0 || k == 0 || n == 0 {
return None;
}
Some((m, k, n))
}
fn run_matmul_cross_check(
file: &Path,
sig: &KernelSig,
emit_receipt: Option<&Path>,
) -> Result<(), i32> {
let (m, k, n) = parse_mm_dims_override().unwrap_or((MM_M, MM_K, MM_N));
let mut a = vec![0.0f32; m * k];
for i in 0..m.min(k) {
a[i * k + i] = 1.0;
}
let b: Vec<f32> = (0..k * n).map(|i| (i as f32) + 1.0).collect();
let scalar_names: Vec<&str> = sig
.scalars()
.map(|(name, kind)| {
debug_assert_eq!(kind, ScalarKind::U32);
name
})
.collect();
if scalar_names.len() != 3 {
eprintln!(
"GPU: matmul kernel must have exactly three u32 shape scalars (m, k, n); found {}",
scalar_names.len()
);
return Err(1);
}
let spirv_bytes = compile_to(file, Target::SpirV).map_err(|e| {
eprintln!("GPU: failed to compile matmul kernel to SPIR-V: {e}");
1
})?;
let words = bytes_to_words(&spirv_bytes).map_err(|e| {
eprintln!("GPU: {e}");
1
})?;
let c_zero = vec![0.0f32; m * n];
vulkan_host::validate_matmul_shapes(m, k, n, a.len(), b.len(), c_zero.len(), LOCAL_SIZE_2D)
.map_err(|e| {
eprintln!("GPU: {e}");
1
})?;
let buffer_args = vec![
vulkan_host::BufferArg {
data: &a,
writable: false,
},
vulkan_host::BufferArg {
data: &b,
writable: false,
},
vulkan_host::BufferArg {
data: &c_zero,
writable: true,
},
];
let mut push_bytes: Vec<u8> = Vec::with_capacity(12);
for v in [m as u32, k as u32, n as u32] {
push_bytes.extend_from_slice(&v.to_le_bytes());
}
let gpu_out = vulkan_host::dispatch_compute(
&words,
&sig.entry,
&buffer_args,
&push_bytes,
vulkan_host::Grid::D2 { gx: n, gy: m },
LOCAL_SIZE_2D,
)
.map_err(|e| {
eprintln!("GPU: matmul device dispatch failed: {e}");
1
})?;
let c_bytes = compile_to(file, Target::C).map_err(|e| {
eprintln!("GPU: failed to compile matmul kernel to C: {e}");
1
})?;
let c_source = String::from_utf8_lossy(&c_bytes);
let kernel_fn = extract_kernel_with_runtime(&c_source, &sig.entry).ok_or_else(|| {
eprintln!(
"GPU: could not extract the `{}` function from the emitted C",
sig.entry
);
1
})?;
let cpu_out = cpu_c_matmul_reference(sig, &kernel_fn, &a, &b, m, k, n).map_err(|e| {
eprintln!("GPU: CPU-C matmul reference failed: {e}");
1
})?;
let mut gpu_out = gpu_out;
if std::env::var("BUILDLANG_GPU_CORRUPT_READBACK").is_ok() && !gpu_out.is_empty() {
gpu_out[0] += 1.0;
}
let elems = m * n;
if std::env::var("BUILDLANG_GPU_CORRUPT_READBACK").is_err() {
let mut max_id_dev = 0.0f32;
for i in 0..elems {
let dev = (cpu_out[i] - b[i]).abs();
if dev > max_id_dev {
max_id_dev = dev;
}
}
if max_id_dev > TOLERANCE {
eprintln!(
"matmul identity sanity: FAIL (identity x B != B; max abs deviation \
{max_id_dev:.3e} > tol {TOLERANCE:.3e}) -- the kernel does not compute matmul"
);
return Err(1);
}
println!("matmul identity sanity: PASS (identity({m}) x B == B within tol)");
}
let mut max_dev = 0.0f32;
for i in 0..elems {
let dev = (gpu_out[i] - cpu_out[i]).abs();
if dev > max_dev {
max_dev = dev;
}
}
if max_dev <= TOLERANCE {
println!(
"gpu-cpu agreement: PASS (matmul {m}x{k}x{n}, N={elems}, max abs deviation \
{max_dev:.3e} <= tol {TOLERANCE:.3e})"
);
} else {
eprintln!(
"gpu-cpu agreement: FAIL (matmul {m}x{k}x{n}, N={elems}, max abs deviation \
{max_dev:.3e} > tol {TOLERANCE:.3e})"
);
return Err(1);
}
if let Some(receipt_path) = emit_receipt {
crate::gpu_receipt::emit_gpu_receipt(
file,
receipt_path,
&gpu_out,
&cpu_out,
max_dev,
TOLERANCE,
)
.map_err(|e| {
eprintln!("GPU: failed to emit receipt: {e}");
1
})?;
println!("wrote gpu receipt to {}", receipt_path.display());
}
Ok(())
}
fn parse_stencil_n_override() -> Option<usize> {
let n = std::env::var("BUILDLANG_STENCIL_N")
.ok()?
.trim()
.parse::<usize>()
.ok()?;
if n >= 2 {
Some(n)
} else {
None
}
}
fn cpu_c_stencil_reference(
sig: &KernelSig,
kernel_fn_c: &str,
a: &[f32],
n: usize,
) -> Result<Vec<f32>, String> {
let fmt_arr = |v: &[f32]| -> String {
v.iter()
.map(|x| format!("{:.9}f", x))
.collect::<Vec<_>>()
.join(", ")
};
let buffers: Vec<(&str, bool)> = sig.buffers().map(|(name, w, _)| (name, w)).collect();
if buffers.len() != 2 {
return Err(format!(
"stencil cross-check expects exactly two buffers (a, out); found {}",
buffers.len()
));
}
let (a_name, _) = buffers[0];
let (out_name, out_writable) = buffers[1];
if !out_writable {
return Err("stencil output buffer (second) must be `&mut [f32]`".to_string());
}
let n_name = sig
.scalars()
.find(|(_, k)| *k == ScalarKind::U32)
.map(|(name, _)| name.to_string())
.ok_or_else(|| "stencil kernel has no u32 length scalar".to_string())?;
let mut decls = String::new();
decls.push_str(&format!(
" static float {a_name}_data[{n}] = {{ {ai} }};\n bl_slice_f32 {a_name} = {{ {a_name}_data, {n} }};\n",
ai = fmt_arr(a),
));
decls.push_str(&format!(
" static float {out_name}_data[{n}] = {{ 0 }};\n bl_slice_f32 {out_name} = {{ {out_name}_data, {n} }};\n",
));
decls.push_str(&format!(" uint32_t {n_name} = {n};\n"));
let call_args: Vec<String> = sig
.params
.iter()
.map(|p| match p {
KernelParam::Scalar { name, .. } => name.clone(),
KernelParam::Buffer { name, .. } => format!("&{name}"),
})
.collect();
let driver = format!(
r#"{prelude}
{kernel}
int main(void) {{
{decls} for (uint32_t i = 0; i < {n}; ++i) {{
buildc_gl_global_invocation_x = i;
{entry}({args});
}}
for (size_t i = 0; i < {n}; ++i) {{
printf("%.9g\n", (double){out}_data[i]);
}}
return 0;
}}
"#,
kernel = kernel_fn_c,
prelude = driver_prelude(),
decls = decls,
entry = sig.entry,
args = call_args.join(", "),
out = out_name,
);
let dir = std::env::temp_dir().join(format!("buildlang_gpu_stencil_{}", std::process::id()));
std::fs::create_dir_all(&dir).map_err(|e| format!("create temp dir: {e}"))?;
let c_path = dir.join("cpu_ref_stencil.c");
std::fs::write(&c_path, driver).map_err(|e| format!("write cpu_ref_stencil.c: {e}"))?;
let exe_path = dir.join(if cfg!(windows) {
"cpu_ref_stencil.exe"
} else {
"cpu_ref_stencil"
});
compile_c(&c_path, &exe_path)?;
let output = std::process::Command::new(&exe_path)
.output()
.map_err(|e| format!("run cpu_ref_stencil: {e}"))?;
if !output.status.success() {
return Err(format!(
"cpu_ref_stencil exited non-zero: {}",
String::from_utf8_lossy(&output.stderr)
));
}
let stdout = String::from_utf8_lossy(&output.stdout);
let values: Vec<f32> = stdout
.lines()
.filter(|l| !l.trim().is_empty())
.map(|l| {
l.trim()
.parse::<f32>()
.map_err(|e| format!("parse cpu output '{l}': {e}"))
})
.collect::<Result<Vec<_>, _>>()?;
if values.len() != n {
return Err(format!(
"cpu_ref_stencil produced {} values, expected {}",
values.len(),
n
));
}
Ok(values)
}
fn run_stencil_cross_check(
file: &Path,
sig: &KernelSig,
emit_receipt: Option<&Path>,
) -> Result<(), i32> {
let n = parse_stencil_n_override().unwrap_or(STENCIL_N);
let a: Vec<f32> = (0..n).map(|i| (i as f32) + 1.0).collect();
let out_zero = vec![0.0f32; n];
let spirv_bytes = compile_to(file, Target::SpirV).map_err(|e| {
eprintln!("GPU: failed to compile stencil kernel to SPIR-V: {e}");
1
})?;
let words = bytes_to_words(&spirv_bytes).map_err(|e| {
eprintln!("GPU: {e}");
1
})?;
let buffer_args = vec![
vulkan_host::BufferArg {
data: &a,
writable: false,
},
vulkan_host::BufferArg {
data: &out_zero,
writable: true,
},
];
let push_bytes: Vec<u8> = (n as u32).to_le_bytes().to_vec();
let gpu_out = vulkan_host::dispatch_compute(
&words,
&sig.entry,
&buffer_args,
&push_bytes,
vulkan_host::Grid::D1(n),
(LOCAL_SIZE_X, 1),
)
.map_err(|e| {
eprintln!("GPU: stencil device dispatch failed: {e}");
1
})?;
let c_bytes = compile_to(file, Target::C).map_err(|e| {
eprintln!("GPU: failed to compile stencil kernel to C: {e}");
1
})?;
let c_source = String::from_utf8_lossy(&c_bytes);
let kernel_fn = extract_kernel_with_runtime(&c_source, &sig.entry).ok_or_else(|| {
eprintln!(
"GPU: could not extract the `{}` function from the emitted C",
sig.entry
);
1
})?;
let cpu_out = cpu_c_stencil_reference(sig, &kernel_fn, &a, n).map_err(|e| {
eprintln!("GPU: CPU-C stencil reference failed: {e}");
1
})?;
let mut gpu_out = gpu_out;
if std::env::var("BUILDLANG_GPU_CORRUPT_READBACK").is_ok() && !gpu_out.is_empty() {
gpu_out[0] += 1.0;
}
if std::env::var("BUILDLANG_GPU_CORRUPT_READBACK").is_err() {
let expected_0 = (a[0] + a[0] + a[1]) / 3.0;
let expected_last = (a[n - 2] + a[n - 1] + a[n - 1]) / 3.0;
let dev_0 = (cpu_out[0] - expected_0).abs();
let dev_last = (cpu_out[n - 1] - expected_last).abs();
if dev_0 > TOLERANCE || dev_last > TOLERANCE {
eprintln!(
"stencil clamped-edge sanity: FAIL (out[0]={} vs clamped {expected_0}, \
out[n-1]={} vs clamped {expected_last}) -- the kernel did not clamp the edges",
cpu_out[0],
cpu_out[n - 1],
);
return Err(1);
}
println!("stencil clamped-edge sanity: PASS (clamped out[0] and out[n-1] match closed form within tol)");
println!(
"stencil boundary out[0]={:.9} out[n-1]={:.9}",
gpu_out[0],
gpu_out[n - 1],
);
}
let mut max_dev = 0.0f32;
for i in 0..n {
let dev = (gpu_out[i] - cpu_out[i]).abs();
if dev > max_dev {
max_dev = dev;
}
}
if max_dev <= TOLERANCE {
println!(
"gpu-cpu agreement: PASS (stencil n={n}, max abs deviation {max_dev:.3e} <= tol {TOLERANCE:.3e})"
);
} else {
eprintln!(
"gpu-cpu agreement: FAIL (stencil n={n}, max abs deviation {max_dev:.3e} > tol {TOLERANCE:.3e})"
);
return Err(1);
}
if let Some(receipt_path) = emit_receipt {
crate::gpu_receipt::emit_gpu_receipt(
file,
receipt_path,
&gpu_out,
&cpu_out,
max_dev,
TOLERANCE,
)
.map_err(|e| {
eprintln!("GPU: failed to emit receipt: {e}");
1
})?;
println!("wrote gpu receipt to {}", receipt_path.display());
}
Ok(())
}
pub fn run_gpu_cross_check(file: &Path, emit_receipt: Option<&Path>) -> Result<(), i32> {
let sig = discover_kernel_sig(file).map_err(|e| {
eprintln!("GPU: {e}");
1
})?;
if sig.params.iter().any(|p| {
matches!(
p,
KernelParam::Scalar {
kind: ScalarKind::F64,
..
} | KernelParam::Buffer {
elem: ScalarKind::F64,
..
}
)
}) {
eprintln!(
"GPU: the GPU path is f32; this kernel uses f64. Declare its parameters `f32` \
(the device path does not run f64 -- values would be silently coerced)."
);
return Err(1);
}
let _out_idx = sig.output_buffer_index().map_err(|e| {
eprintln!("GPU: {e}");
1
})?;
if kernel_is_2d(file).map_err(|e| {
eprintln!("GPU: {e}");
1
})? {
return run_matmul_cross_check(file, &sig, emit_receipt);
}
if sig.is_stencil() {
return run_stencil_cross_check(file, &sig, emit_receipt);
}
let mut buffer_data: Vec<(&str, Vec<f32>)> = Vec::new();
for (idx, (name, writable, _elem)) in sig.buffers().enumerate() {
let data: Vec<f32> = if writable {
vec![0.0f32; N]
} else {
let base = (idx as f32) * (N as f32);
(0..N).map(|i| base + (i + 1) as f32).collect()
};
buffer_data.push((name, data));
}
let scalar_vals: Vec<(&str, f32)> = sig.scalars().map(|(name, _)| (name, 2.0f32)).collect();
let spirv_bytes = compile_to(file, Target::SpirV).map_err(|e| {
eprintln!("GPU: failed to compile kernel to SPIR-V: {e}");
1
})?;
let words = bytes_to_words(&spirv_bytes).map_err(|e| {
eprintln!("GPU: {e}");
1
})?;
let buffer_args: Vec<vulkan_host::BufferArg<'_>> = sig
.buffers()
.zip(buffer_data.iter())
.map(|((_, writable, _), (_, data))| vulkan_host::BufferArg { data, writable })
.collect();
let mut push_bytes: Vec<u8> = Vec::new();
for (name, kind) in sig.scalars() {
let val = scalar_vals
.iter()
.find(|(n, _)| *n == name)
.map(|(_, v)| *v)
.unwrap_or(0.0);
match kind {
ScalarKind::U32 => push_bytes.extend_from_slice(&(val as u32).to_le_bytes()),
ScalarKind::F32 | ScalarKind::F64 => push_bytes.extend_from_slice(&val.to_le_bytes()),
}
}
let gpu_out = vulkan_host::dispatch_compute(
&words,
&sig.entry,
&buffer_args,
&push_bytes,
vulkan_host::Grid::D1(N),
(LOCAL_SIZE_X, 1),
)
.map_err(|e| {
eprintln!("GPU: device dispatch failed: {e}");
1
})?;
let c_bytes = compile_to(file, Target::C).map_err(|e| {
eprintln!("GPU: failed to compile kernel to C for the cross-check: {e}");
1
})?;
let c_source = String::from_utf8_lossy(&c_bytes);
let kernel_fn = extract_kernel_with_runtime(&c_source, &sig.entry).ok_or_else(|| {
eprintln!(
"GPU: could not extract the `{}` function from the emitted C",
sig.entry
);
1
})?;
let cpu_out =
cpu_c_reference(&sig, &kernel_fn, &buffer_data, &scalar_vals, N).map_err(|e| {
eprintln!("GPU: CPU-C reference failed: {e}");
1
})?;
let mut gpu_out = gpu_out;
if std::env::var("BUILDLANG_GPU_CORRUPT_READBACK").is_ok() && !gpu_out.is_empty() {
gpu_out[0] += 1.0;
}
let mut max_dev = 0.0f32;
for i in 0..N {
let dev = (gpu_out[i] - cpu_out[i]).abs();
if dev > max_dev {
max_dev = dev;
}
}
if max_dev <= TOLERANCE {
println!(
"gpu-cpu agreement: PASS (N={N}, max abs deviation {max_dev:.3e} <= tol {TOLERANCE:.3e})"
);
} else {
eprintln!(
"gpu-cpu agreement: FAIL (N={N}, max abs deviation {max_dev:.3e} > tol {TOLERANCE:.3e})"
);
return Err(1);
}
if let Some(receipt_path) = emit_receipt {
crate::gpu_receipt::emit_gpu_receipt(
file,
receipt_path,
&gpu_out,
&cpu_out,
max_dev,
TOLERANCE,
)
.map_err(|e| {
eprintln!("GPU: failed to emit receipt: {e}");
1
})?;
println!("wrote gpu receipt to {}", receipt_path.display());
}
Ok(())
}
#[cfg(test)]
mod cpu_ref_tests;