use crate::{
dispatch::{
CompilationOptions, DebugCompilationOptions, GpuBackend, GpuBufferBackend,
GpuKernelBackend, PollStatus, TargetCompilationOptions, TargetFlags,
backend::{
Axis, DType, MetaId, NodeId, Op, Param, ParamTy, ValueId, ValueState,
kernel::{Dependencies, RawKernel, Redirect},
},
},
errors::{Error, ErrorKind},
tensor::{ToBuffer, build_dims, calc_grid},
};
use briny::raw::cast::cast_slice;
use core::{fmt::Write, num::NonZeroU64, str::FromStr};
use std::{string::String, vec::Vec};
pub use wgpu::{
BackendOptions, Backends, BindGroup, BindGroupDescriptor, BindGroupEntry,
BindGroupLayoutDescriptor, BindGroupLayoutEntry, BindingType, Buffer, BufferBindingType,
BufferDescriptor, BufferUsages, CommandBuffer, CommandEncoder, CommandEncoderDescriptor,
ComputePass, ComputePassDescriptor, ComputePipeline, ComputePipelineDescriptor, Device,
DeviceDescriptor, Dx12BackendOptions, Dx12Compiler, Dx12SwapchainKind,
Dx12UseFrameLatencyWaitableObject, ExperimentalFeatures, Features, ForceShaderModelToken,
GlBackendOptions, GlDebugFns, GlFenceBehavior, Gles3MinorVersion, Instance, InstanceDescriptor,
InstanceFlags, Limits, MemoryBudgetThresholds, MemoryHints, NoopBackendOptions,
PipelineCompilationOptions, PipelineLayout, PipelineLayoutDescriptor, PollType,
PowerPreference, Queue, RequestAdapterError, RequestAdapterOptions, ShaderModuleDescriptor,
ShaderSource, ShaderStages, SubmissionIndex, Trace,
util::{BufferInitDescriptor, DeviceExt},
};
#[derive(Debug)]
pub struct GpuContext {
device: Device,
queue: Queue,
}
impl GpuContext {
pub async fn new() -> Result<Self, crate::errors::Error> {
let instance = Instance::new(InstanceDescriptor {
backends: Backends::all(),
flags: InstanceFlags::empty(),
memory_budget_thresholds: MemoryBudgetThresholds::default(),
backend_options: BackendOptions {
gl: GlBackendOptions {
gles_minor_version: Gles3MinorVersion::Automatic,
fence_behavior: GlFenceBehavior::AutoFinish,
debug_fns: GlDebugFns::Disabled,
},
dx12: Dx12BackendOptions {
shader_compiler: Dx12Compiler::Auto,
presentation_system: Dx12SwapchainKind::default(),
latency_waitable_object: Dx12UseFrameLatencyWaitableObject::None,
force_shader_model: ForceShaderModelToken::default(),
agility_sdk: None,
},
noop: NoopBackendOptions::default(),
},
display: None,
});
let adapter = instance
.request_adapter(&RequestAdapterOptions {
power_preference: PowerPreference::HighPerformance,
force_fallback_adapter: false,
compatible_surface: None,
apply_limit_buckets: false,
})
.await
.map_err(|e| match e {
RequestAdapterError::EnvNotSet => Error {
msg: "adapter environment variables not set",
kind: ErrorKind::EnvNotSet,
ctx: (),
},
_ => Error {
msg: "no adapter found",
kind: ErrorKind::AdapterNotFound,
ctx: (),
},
})?;
let adapter_limits = adapter.limits();
let (device, queue) = match adapter
.request_device(&DeviceDescriptor {
label: Some("device"),
required_limits: adapter_limits,
required_features: Features::SHADER_F16,
experimental_features: ExperimentalFeatures::disabled(),
memory_hints: MemoryHints::Performance,
trace: Trace::Off,
})
.await
{
Ok(dev) => dev,
Err(_) => adapter
.request_device(&DeviceDescriptor {
label: Some("device"),
required_limits: Limits::defaults(),
required_features: Features::SHADER_F16,
experimental_features: ExperimentalFeatures::disabled(),
memory_hints: MemoryHints::Performance,
trace: Trace::Off,
})
.await
.map_err(|_| Error {
msg: "failed to request device",
kind: ErrorKind::DeviceNotFound,
ctx: (),
})?,
};
Ok(Self { device, queue })
}
}
#[derive(Debug, Clone)]
pub struct GpuKernel {
kernel: ComputePipeline,
iter_space: Vec<MetaId>,
block: [u32; 3],
}
impl GpuKernelBackend for GpuKernel {
fn iteration_space(&self) -> &[MetaId] {
&self.iter_space
}
fn block(&self) -> &[u32; 3] {
&self.block
}
}
pub struct Schedule {
kernels: Vec<(ComputePipeline, [u32; 3], BindGroup)>,
}
impl GpuBackend for GpuContext {
type Buffer = GpuBuffer;
type Kernel = GpuKernel;
type SubmissionIndex = SubmissionIndex;
type ParamLayout = PipelineLayout;
type Schedule = Schedule;
type SyncSubmissions = CommandBuffer;
type Batcher<'a> = ComputePass<'a>;
type BatchState = CommandEncoder;
fn target_spec(&self) -> TargetCompilationOptions {
TargetCompilationOptions {
flags: TargetFlags::empty(),
}
}
#[inline]
fn alloc(&self, len: usize) -> Self::Buffer {
GpuBuffer(self.device.create_buffer(&BufferDescriptor {
label: Some("gpu_tensor"),
size: len as u64,
usage: BufferUsages::STORAGE | BufferUsages::COPY_SRC | BufferUsages::COPY_DST,
mapped_at_creation: false,
}))
}
#[inline]
fn alloc_init(&self, contents: &[u8]) -> Self::Buffer {
GpuBuffer(self.device.create_buffer_init(&BufferInitDescriptor {
label: Some("gpu_tensor"),
contents,
usage: BufferUsages::STORAGE | BufferUsages::COPY_SRC | BufferUsages::COPY_DST,
}))
}
#[inline]
fn alloc_meta(&self, data: &[u32]) -> Self::Buffer {
let offset = (4 - (data.len() % 4)) % 4;
let new_len = data.len() + offset;
let mut aligned = std::vec![u32::MAX; new_len];
aligned[..data.len()].copy_from_slice(data);
GpuBuffer(self.device.create_buffer_init(&BufferInitDescriptor {
label: Some("gpu_meta"),
contents: cast_slice(&aligned),
usage: BufferUsages::UNIFORM,
}))
}
#[inline]
fn upload(
&self,
buffer: &Self::Buffer,
data: &[u8],
dst_off: u32,
) -> Result<Self::SubmissionIndex, Error> {
if buffer.size_bytes().saturating_sub(dst_off) as usize > data.len() {
return Err(Error {
msg: "CPU buffer of smaller size than GPU buffer during upload",
kind: ErrorKind::FailedBufferCopy,
ctx: (),
});
}
let mut encoder = self
.device
.create_command_encoder(&CommandEncoderDescriptor::default());
let src = self.device.create_buffer_init(&BufferInitDescriptor {
label: Some("gpu_tensor"),
contents: data,
usage: BufferUsages::STORAGE | BufferUsages::COPY_SRC,
});
encoder.copy_buffer_to_buffer(&src, 0, &buffer.0, dst_off as u64, Some(data.len() as u64));
Ok(self.queue.submit(Some(encoder.finish())))
}
fn pipe(&self, src: &Self::Buffer, dst: &Self::Buffer) -> Result<Self::SubmissionIndex, Error> {
let src_size = src.0.size();
if src_size != dst.0.size() {
return Err(Error {
msg: "buffers of unequal sizes during pipe",
kind: ErrorKind::FailedBufferCopy,
ctx: (),
});
}
let mut encoder = self
.device
.create_command_encoder(&CommandEncoderDescriptor::default());
encoder.copy_buffer_to_buffer(&src.0, 0, &dst.0, 0, src_size);
Ok(self.queue.submit(Some(encoder.finish())))
}
#[inline]
fn download(&self, buffer: &Self::Buffer, data: &mut [u8]) -> Result<(), Error> {
if buffer.size_bytes() as usize > data.len() {
return Err(Error {
msg: "insufficient CPU memory allocated for GPU download",
kind: ErrorKind::FailedBufferCopy,
ctx: (),
});
}
let mut encoder = self
.device
.create_command_encoder(&CommandEncoderDescriptor::default());
let dst = self.device.create_buffer(&BufferDescriptor {
label: Some("download"),
size: buffer.0.size(),
usage: BufferUsages::MAP_READ | BufferUsages::COPY_DST,
mapped_at_creation: false,
});
encoder.copy_buffer_to_buffer(&buffer.0, 0, &dst, 0, buffer.0.size());
let submission_index = self.queue.submit(Some(encoder.finish()));
let buffer_slice = dst.slice(..);
let (send, recv) = std::sync::mpsc::channel();
buffer_slice.map_async(wgpu::MapMode::Read, move |res| {
if res.is_ok() {
let _ = send.send(());
}
});
let _ = self.device.poll(PollType::Wait {
submission_index: Some(submission_index),
timeout: None,
});
let _ = recv.recv();
let len = data.len().min(buffer.size_bytes() as usize);
data[..len].copy_from_slice(
&buffer_slice.get_mapped_range().map_err(|_| Error {
msg: "failed to map GPU memory to CPU",
kind: ErrorKind::FailedBufferCopy,
ctx: (),
})?[..len],
);
dst.unmap();
Ok(())
}
#[inline]
fn compile(
&self,
src: &RawKernel,
params: &[Param],
options: &CompilationOptions,
) -> Result<Self::Kernel, Error> {
let entries = generate_layout_desc(params);
let desc = BindGroupLayoutDescriptor {
label: Some("bind_group_layout"),
entries: &entries,
};
let bind_group_layout = self.device.create_bind_group_layout(&desc);
let bind_group_layouts = &[Some(&bind_group_layout)];
let desc = PipelineLayoutDescriptor {
label: Some("pipeline_layout"),
bind_group_layouts,
immediate_size: 0,
};
let pipeline_layout = self.device.create_pipeline_layout(&desc);
let source = generate_wgsl(
src,
params,
options
.debug
.contains(DebugCompilationOptions::PRETTY_PRINT_IR),
)?;
let shader = self.device.create_shader_module(ShaderModuleDescriptor {
label: Some("shader"),
source: ShaderSource::Wgsl(source.into()),
});
Ok(GpuKernel {
kernel: self
.device
.create_compute_pipeline(&ComputePipelineDescriptor {
label: Some("pipeline"),
layout: Some(&pipeline_layout),
module: &shader,
entry_point: Some("main"),
compilation_options: PipelineCompilationOptions::default(),
cache: None,
}),
iter_space: src.iter_space.clone(),
block: src.block,
})
}
fn schedule(
&self,
kernels: Vec<Dependencies<Redirect<(Self::Kernel, NodeId, &[bool])>>>,
bindings: &[&Self::Buffer],
meta: &[u32],
) -> Result<Self::Schedule, Error> {
let mut resolved = Vec::new();
let mut tmp_res = Vec::new();
let mut scheduled_kernels = Vec::new();
while resolved.len() < kernels.len() {
for kernel in &kernels {
let dep = &kernel.dep;
let kernel = match &kernel.val {
Redirect::Unmasked(kernel_data) => kernel_data,
Redirect::Redirected(idx) => match &kernels[*idx].val {
Redirect::Unmasked(kernel) => kernel,
Redirect::Redirected(_) => {
return Err(Error {
msg: "double redirection or loop encountered in kernel resolution",
kind: ErrorKind::UnresolvedRedirection,
ctx: (),
});
}
},
};
let (kernel, idx, params) = kernel;
if resolved.contains(idx) {
continue;
}
if dep.iter().all(|x| resolved.contains(x)) {
tmp_res.push(*idx);
let iter_space = build_dims(kernel.iteration_space(), meta);
let grid = calc_grid(&iter_space, *kernel.block());
let kernel_bindings = bindings
.iter()
.enumerate()
.filter_map(|(i, buf)| {
if params[i] {
let entry = BindGroupEntry {
binding: i as u32,
resource: buf.0.as_entire_binding(),
};
Some(entry)
} else {
None
}
})
.collect::<Vec<_>>();
let kernel = &kernel.kernel;
let bind_group = self.device.create_bind_group(&BindGroupDescriptor {
layout: &kernel.get_bind_group_layout(0),
entries: &kernel_bindings,
label: Some("bind_group"),
});
scheduled_kernels.push((kernel.clone(), grid, bind_group));
}
}
resolved.append(&mut tmp_res);
}
Ok(Schedule {
kernels: scheduled_kernels,
})
}
fn prepare_batch(&self) -> Self::BatchState {
self.device
.create_command_encoder(&CommandEncoderDescriptor {
label: Some("encoder"),
})
}
fn start_batch<'a>(&self, encoder: &'a mut Self::BatchState) -> Self::Batcher<'a> {
encoder.begin_compute_pass(&ComputePassDescriptor {
label: Some("pass"),
timestamp_writes: None,
})
}
fn dispatch_schedule(&self, pass: &mut Self::Batcher<'_>, schedule: &Self::Schedule) {
for (kernel, wg, bind_group) in &schedule.kernels {
pass.set_pipeline(kernel);
pass.set_bind_group(0, bind_group, &[]);
pass.dispatch_workgroups(wg[0], wg[1], wg[2]);
}
}
#[inline]
fn dispatch_kernel(
&self,
pass: &mut Self::Batcher<'_>,
kernel: &Self::Kernel,
wg: [u32; 3],
bindings: &[&Self::Buffer],
) {
let kernel = &kernel.kernel;
let entries = bindings
.iter()
.enumerate()
.map(|(i, x)| BindGroupEntry {
binding: i as u32,
resource: x.0.as_entire_binding(),
})
.collect::<Vec<_>>();
let bind_group = self.device.create_bind_group(&BindGroupDescriptor {
layout: &kernel.get_bind_group_layout(0),
entries: &entries,
label: Some("bind_group"),
});
pass.set_pipeline(kernel);
pass.set_bind_group(0, &bind_group, &[]);
pass.dispatch_workgroups(wg[0], wg[1], wg[2]);
}
#[inline]
fn sync(&self, submission_index: Self::SubmissionIndex) {
let _ = self.device.poll(PollType::Wait {
submission_index: Some(submission_index),
timeout: None,
});
}
fn encode(&self, state: Self::BatchState) -> Self::SyncSubmissions {
state.finish()
}
fn submit(&self, submission: Self::SyncSubmissions) -> Self::SubmissionIndex {
self.queue.submit([submission])
}
#[inline]
fn poll(&self) -> PollStatus {
let res = self.device.poll(PollType::Poll);
match res {
Err(_) => PollStatus::Failed,
Ok(wgpu::PollStatus::Poll) => PollStatus::Pending,
Ok(wgpu::PollStatus::WaitSucceeded) => PollStatus::Ready,
Ok(wgpu::PollStatus::QueueEmpty) => PollStatus::QueueEmpty,
}
}
}
#[repr(transparent)]
#[derive(Debug, Clone)]
pub struct GpuBuffer(Buffer);
impl GpuBufferBackend for GpuBuffer {
#[inline]
fn size_bytes(&self) -> u32 {
self.0.size() as u32
}
#[inline]
fn size(&self) -> u32 {
(self.0.size() / (size_of::<f32>() as u64)) as u32
}
}
impl ToBuffer<GpuContext> for GpuBuffer {
#[inline]
fn as_buffer(&self) -> &Self {
self
}
#[inline]
fn to_buffer(self) -> Self {
self
}
}
#[inline]
fn generate_layout_desc(params: &[Param]) -> Vec<BindGroupLayoutEntry> {
const MIN_BIND_SIZE: NonZeroU64 = NonZeroU64::new(1024).unwrap();
params
.iter()
.map(|x| {
let ty = match x.ty {
ParamTy::Uniform => BufferBindingType::Uniform,
ParamTy::ReadOnly => BufferBindingType::Storage { read_only: true },
ParamTy::ReadWrite => BufferBindingType::Storage { read_only: false },
};
BindGroupLayoutEntry {
binding: x.pid as u32,
visibility: ShaderStages::COMPUTE,
ty: BindingType::Buffer {
ty,
has_dynamic_offset: false,
min_binding_size: match ty {
BufferBindingType::Uniform => None,
BufferBindingType::Storage { .. } => Some(MIN_BIND_SIZE),
},
},
count: None,
}
})
.collect::<Vec<_>>()
}
#[inline]
fn generate_wgsl(
kernel: &RawKernel,
params: &[Param],
pretty_print: bool,
) -> Result<String, Error> {
let mut out = String::from_str("enable f16;").map_err(|_| Error {
msg: "infallible",
kind: ErrorKind::InternalError,
ctx: (),
})?;
newline(pretty_print, &mut out, 0);
emit_bindings(kernel, params, &mut out, pretty_print)?;
newline(pretty_print, &mut out, 0);
emit_entry(kernel, &mut out, pretty_print)?;
Ok(out)
}
#[inline]
const fn get_axis(axis: Axis) -> &'static str {
match axis {
Axis::X => "x",
Axis::Y => "y",
Axis::Z => "z",
}
}
#[inline]
const fn get_dtype(dtype: DType) -> Result<&'static str, Error> {
match dtype {
DType::F64 => Ok("f64"),
DType::F32 => Ok("f32"),
DType::F16 => Ok("f16"),
DType::BF16 => Err(Error {
msg: "bf16 not supported",
kind: ErrorKind::InvalidDType,
ctx: (),
}),
DType::Bool => Ok("bool"),
DType::I32 => Ok("i32"),
DType::U32 => Ok("u32"),
}
}
#[inline]
fn newline(pretty_print: bool, out: &mut String, nesting: usize) {
if pretty_print {
let _ = write!(out, "\n{}", " ".repeat(nesting));
} else {
let _ = write!(out, " ");
}
}
#[inline]
fn tab(pretty_print: bool, out: &mut String) {
if pretty_print {
let _ = write!(out, " ");
}
}
#[inline]
fn emit_bindings(
kernel: &RawKernel,
params: &[Param],
out: &mut String,
pretty_print: bool,
) -> Result<(), Error> {
let _ = write!(out, "struct Meta {{");
newline(pretty_print, out, 0);
let _ = write!(out, "f0: f32,");
for f in 1..kernel.meta.fields {
newline(pretty_print, out, 1);
let _ = write!(out, "f{f}: u32,");
}
newline(pretty_print, out, 0);
let _ = write!(out, "}}");
newline(pretty_print, out, 0);
for p in params {
let var_type = match p.ty {
ParamTy::ReadOnly => "var<storage, read>",
ParamTy::ReadWrite => "var<storage, read_write>",
ParamTy::Uniform => "var<uniform>",
};
newline(pretty_print, out, 0);
let pid = p.pid;
if p.ty == ParamTy::Uniform && p.dtype == DType::U32 {
let _ = write!(
out,
"@group(0) @binding({pid}) {var_type} param{pid}: Meta;"
);
} else {
let _ = write!(
out,
"@group(0) @binding({pid}) {var_type} param{pid}: array<{}>;",
get_dtype(p.dtype)?,
);
}
}
newline(pretty_print, out, 0);
for (i, s) in kernel.shared.iter().enumerate() {
newline(pretty_print, out, 0);
let _ = write!(
out,
"var<workgroup> shared{i}: array<{}, ({})>;",
get_dtype(s.dtype)?,
s.size
);
}
Ok(())
}
#[inline]
fn emit_entry(kernel: &RawKernel, out: &mut String, pretty_print: bool) -> Result<(), Error> {
newline(pretty_print, out, 0);
let _ = write!(
out,
"@compute @workgroup_size({}, ({}), ({})) ",
kernel.block[0], kernel.block[1], kernel.block[2]
);
newline(pretty_print, out, 0);
let _ = write!(out, "fn main(");
newline(pretty_print, out, 1);
let _ = write!(out, "@builtin(local_invocation_id) lid: vec3<u32>,");
newline(pretty_print, out, 1);
let _ = write!(out, "@builtin(workgroup_id) bid: vec3<u32>,");
newline(pretty_print, out, 1);
let _ = write!(out, "@builtin(global_invocation_id) gid: vec3<u32>,");
newline(pretty_print, out, 0);
let _ = write!(out, ") {{");
emit_ops(kernel, out, pretty_print)?;
newline(pretty_print, out, 0);
let _ = write!(out, "}}");
Ok(())
}
#[inline]
fn emit_ops(kernel: &RawKernel, out: &mut String, pretty_print: bool) -> Result<(), Error> {
let mut nesting = 0;
for op in &kernel.ops {
if let Op::DefineVar { id } = op
&& matches!(
kernel.values[*id].state,
ValueState::Inline | ValueState::Masked
)
{
continue;
}
newline(pretty_print, out, nesting);
if *op != Op::EndScope {
tab(pretty_print, out);
}
process_op(out, op, &mut nesting, kernel)?;
}
Ok(())
}
fn process_op(
out: &mut String,
op: &Op,
nesting: &mut usize,
kernel: &RawKernel,
) -> Result<(), Error> {
match op {
Op::Nop => {}
Op::DefineVar { id } => {
let val = &kernel.values[*id];
match val.state {
ValueState::Masked | ValueState::Inline => {}
ValueState::Const => {
let _ = write!(out, "const v{id}: {}", get_dtype(val.dtype)?);
if let Some(op) = &val.init {
let _ = out.write_str(" = ");
process_op(out, op, nesting, kernel)?;
}
let _ = out.write_char(';');
}
var => {
if var == ValueState::Immut {
let _ = write!(out, "let v{id}: {}", get_dtype(val.dtype)?);
} else {
let _ = write!(out, "var v{id}: {}", get_dtype(val.dtype)?);
}
if let Some(op) = &val.init {
let _ = out.write_str(" = ");
process_op(out, op, nesting, kernel)?;
}
let _ = out.write_char(';');
}
}
}
Op::OverwriteVar { id, val } => {
let _ = write!(out, "v{id} = {};", render_val(*val, kernel)?);
}
Op::AddAssign { id, val } => {
let _ = write!(out, "v{id} += {};", render_val(*val, kernel)?);
}
Op::MulAssign { id, val } => {
let _ = write!(out, "v{id} *= {};", render_val(*val, kernel)?);
}
Op::DivAssign { id, val } => {
let _ = write!(out, "v{id} /= {};", render_val(*val, kernel)?);
}
Op::SubAssign { id, val } => {
let _ = write!(out, "v{id} -= {};", render_val(*val, kernel)?);
}
Op::ShlAssign { id, val } => {
let _ = write!(out, "v{id} <<= {};", render_val(*val, kernel)?);
}
Op::ShrAssign { id, val } => {
let _ = write!(out, "v{id} >>= {};", render_val(*val, kernel)?);
}
Op::CopyVar { id } => {
let _ = out.write_str(&render_val(*id, kernel)?);
}
Op::ConstF64 { value } => {
let _ = write!(out, "{value}d");
}
Op::ConstF32 { value } => {
let _ = write!(out, "{value}f");
}
Op::ConstF16 { value } => {
let _ = write!(out, "{value}h");
}
Op::ConstBf16 { value: _ } => {
return Err(Error {
msg: "cannot evaluate constant `bf16`",
kind: ErrorKind::InvalidDType,
ctx: (),
});
}
Op::ConstU32 { value } => {
let _ = write!(out, "{value}u");
}
Op::ConstI32 { value } => {
let _ = write!(out, "{value}");
}
Op::ReadMeta { param, field } => {
let _ = write!(out, "param{param}.f{field}");
}
Op::Eq { a, b } => {
let _ = write!(
out,
"({}) == ({})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Ne { a, b } => {
let _ = write!(
out,
"({}) != ({})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Lt { a, b } => {
let _ = write!(
out,
"({}) < ({})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Gt { a, b } => {
let _ = write!(
out,
"({}) > ({})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Le { a, b } => {
let _ = write!(
out,
"({}) <= ({})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Ge { a, b } => {
let _ = write!(
out,
"({}) >= ({})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::LocalId { axis } => {
let _ = write!(out, "lid.{}", get_axis(*axis));
}
Op::BlockId { axis } => {
let _ = write!(out, "bid.{}", get_axis(*axis));
}
Op::GlobalId { axis } => {
let _ = write!(out, "gid.{}", get_axis(*axis));
}
Op::Add { a, b } => {
let _ = write!(
out,
"({}) + ({})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Sub { a, b } => {
let _ = write!(
out,
"({}) - ({})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Mul { a, b } => {
let _ = write!(
out,
"({}) * ({})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Div { a, b } => {
let _ = write!(
out,
"({}) / ({})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Mod { a, b } => {
let _ = write!(
out,
"({}) % ({})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Pow { a, b } => {
let _ = write!(
out,
"pow({}, {})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Shl { a, b } => {
let _ = write!(
out,
"({}) << ({})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Shr { a, b } => {
let _ = write!(
out,
"({}) >> ({})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Fma { a, b, c } => {
let _ = write!(
out,
"fma({}, {}, {})",
render_val(*a, kernel)?,
render_val(*b, kernel)?,
render_val(*c, kernel)?
);
}
Op::Max { a, b } => {
let _ = write!(
out,
"max({}, {})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Min { a, b } => {
let _ = write!(
out,
"min({}, {})",
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Select { cond, a, b } => {
let _ = write!(
out,
"select({}, {}, {})",
render_val(*cond, kernel)?,
render_val(*a, kernel)?,
render_val(*b, kernel)?
);
}
Op::Exp { x } => {
let _ = write!(out, "exp({})", render_val(*x, kernel)?);
}
Op::Abs { x } => {
let _ = write!(out, "abs({})", render_val(*x, kernel)?);
}
Op::Neg { x } => {
let _ = write!(out, "-({})", render_val(*x, kernel)?);
}
Op::Log { x } => {
let _ = write!(out, "log({})", render_val(*x, kernel)?);
}
Op::Tanh { x } => {
let _ = write!(out, "tanh({})", render_val(*x, kernel)?);
}
Op::Sqrt { x } => {
let _ = write!(out, "sqrt({})", render_val(*x, kernel)?);
}
Op::ParamLoad { param, index } => {
let _ = write!(out, "param{param}[{}]", render_val(*index, kernel)?);
}
Op::Not { cond } => {
let _ = write!(out, "!({})", render_val(*cond, kernel)?);
}
Op::CastF64 { id } => {
let _ = write!(out, "f64({})", render_val(*id, kernel)?);
}
Op::CastF32 { id } => {
let _ = write!(out, "f32({})", render_val(*id, kernel)?);
}
Op::CastF16 { id } => {
let _ = write!(out, "f16({})", render_val(*id, kernel)?);
}
Op::CastU32 { id } => {
let _ = write!(out, "u32({})", render_val(*id, kernel)?);
}
Op::CastI32 { id } => {
let _ = write!(out, "i32({})", render_val(*id, kernel)?);
}
Op::CastBF16 { .. } => {
return Err(Error {
msg: "bf16 not supported",
kind: ErrorKind::InvalidDType,
ctx: (),
});
}
Op::ParamStore {
param,
index,
value,
} => {
let _ = write!(
out,
"param{param}[{}] = {};",
render_val(*index, kernel)?,
render_val(*value, kernel)?
);
}
Op::ParamAccum {
param,
index,
value,
} => {
let _ = write!(
out,
"param{param}[{}] += {};",
render_val(*index, kernel)?,
render_val(*value, kernel)?
);
}
Op::ParamMul {
param,
index,
value,
} => {
let _ = write!(
out,
"param{param}[{}] *= {};",
render_val(*index, kernel)?,
render_val(*value, kernel)?
);
}
Op::ParamDiv {
param,
index,
value,
} => {
let _ = write!(
out,
"param{param}[{}] /= {};",
render_val(*index, kernel)?,
render_val(*value, kernel)?
);
}
Op::ParamSub {
param,
index,
value,
} => {
let _ = write!(
out,
"param{param}[{}] -= {};",
render_val(*index, kernel)?,
render_val(*value, kernel)?
);
}
Op::ParamShl {
param,
index,
value,
} => {
let _ = write!(
out,
"param{param}[{}] <<= {};",
render_val(*index, kernel)?,
render_val(*value, kernel)?
);
}
Op::ParamShr {
param,
index,
value,
} => {
let _ = write!(
out,
"param{param}[{}] >>= {};",
render_val(*index, kernel)?,
render_val(*value, kernel)?
);
}
Op::SharedLoad { mem, index } => {
let _ = write!(out, "shared{mem}[{}]", render_val(*index, kernel)?);
}
Op::SharedStore { mem, index, value } => {
let _ = write!(
out,
"shared{mem}[{}] = {};",
render_val(*index, kernel)?,
render_val(*value, kernel)?
);
}
Op::SharedAccum { mem, index, value } => {
let _ = write!(
out,
"shared{mem}[{}] += {};",
render_val(*index, kernel)?,
render_val(*value, kernel)?
);
}
Op::SharedMul { mem, index, value } => {
let _ = write!(
out,
"shared{mem}[{}] *= {};",
render_val(*index, kernel)?,
render_val(*value, kernel)?
);
}
Op::SharedDiv { mem, index, value } => {
let _ = write!(
out,
"shared{mem}[{}] /= {};",
render_val(*index, kernel)?,
render_val(*value, kernel)?
);
}
Op::SharedSub { mem, index, value } => {
let _ = write!(
out,
"shared{mem}[{}] -= {};",
render_val(*index, kernel)?,
render_val(*value, kernel)?
);
}
Op::SharedShl { mem, index, value } => {
let _ = write!(
out,
"shared{mem}[{}] <<= {};",
render_val(*index, kernel)?,
render_val(*value, kernel)?
);
}
Op::SharedShr { mem, index, value } => {
let _ = write!(
out,
"shared{mem}[{}] >>= {};",
render_val(*index, kernel)?,
render_val(*value, kernel)?
);
}
Op::ForLoopBegin { index, end, step } => {
let _ = write!(
out,
"for (; v{index} < {}; v{index} += {}) {{",
render_val(*end, kernel)?,
render_val(*step, kernel)?,
);
*nesting += 1;
}
Op::ForeverLoopBegin => {
let _ = write!(out, "loop {{");
*nesting += 1;
}
Op::IfBegin { cond } => {
let _ = write!(out, "if ({}) {{", render_val(*cond, kernel)?);
*nesting += 1;
}
Op::ElseBegin => {
let _ = write!(out, "else {{");
*nesting += 1;
}
Op::StartScope => {
let _ = write!(out, "{{");
}
Op::EndScope => {
let _ = write!(out, "}}");
*nesting -= 1;
}
Op::Barrier => {
let _ = write!(out, "workgroupBarrier();");
}
Op::Return => {
let _ = write!(out, "return;");
}
Op::Continue => {
let _ = write!(out, "continue;");
}
Op::Break => {
let _ = write!(out, "break;");
}
}
Ok(())
}
fn render_val(id: ValueId, kernel: &RawKernel) -> Result<String, Error> {
let mut out = String::new();
let val = &kernel.values[id];
match val.state {
ValueState::Inline => {
if let Some(op) = &val.init {
process_op(&mut out, op, &mut 0, kernel)?;
}
}
ValueState::Masked => {}
_ => {
let _ = write!(out, "v{id}");
}
}
Ok(out)
}