1#![allow(dead_code, clippy::many_single_char_names)]
2use std::collections::HashMap;
13
14pub struct LayerActivations {
19 pub attn_norm_out: wgpu::Buffer,
21 pub attn_output: wgpu::Buffer,
23 pub ffn_norm_out: wgpu::Buffer,
25 pub silu_gate_output: wgpu::Buffer,
27 pub rstd_attn: wgpu::Buffer,
29 pub rstd_ffn: wgpu::Buffer,
31 pub softmax_logsumexp: wgpu::Buffer,
33}
34
35pub struct QkvLoRA<'a> {
37 pub q_a: &'a wgpu::Buffer,
38 pub q_b: &'a wgpu::Buffer,
39 pub k_a: &'a wgpu::Buffer,
40 pub k_b: &'a wgpu::Buffer,
41 pub v_a: &'a wgpu::Buffer,
42 pub v_b: &'a wgpu::Buffer,
43 pub rank: u32,
44 pub scale: f32,
45 pub in_dim: u32,
46 pub q_dim: u32,
47 pub kv_dim: u32,
48 pub lora_pipeline: &'a wgpu::ComputePipeline,
49 pub lora_bgl: &'a wgpu::BindGroupLayout,
50}
51
52pub struct WgslForwardPass {
55 device: wgpu::Device,
56 queue: wgpu::Queue,
57
58 matmul_pipeline: wgpu::ComputePipeline,
60 tiled_matmul_pipeline: wgpu::ComputePipeline,
62 gemv_pipeline: wgpu::ComputePipeline,
64 q4k_gemv_pipeline: wgpu::ComputePipeline,
66 attention_pipeline: wgpu::ComputePipeline,
68 attention_bgl: wgpu::BindGroupLayout,
69 rmsnorm_pipeline: wgpu::ComputePipeline,
70 silu_mul_pipeline: wgpu::ComputePipeline,
71 rope_pipeline: wgpu::ComputePipeline,
72 batch_rope_pipeline: wgpu::ComputePipeline,
73 batch_rope_bgl: wgpu::BindGroupLayout,
74 residual_pipeline: wgpu::ComputePipeline,
75
76 matmul_bgl: wgpu::BindGroupLayout,
78 elementwise_bgl: wgpu::BindGroupLayout,
79
80 weight_buffers: HashMap<String, wgpu::Buffer>,
82 q4k_weights: HashMap<String, wgpu::Buffer>,
84 cpu_biases: HashMap<String, Vec<f32>>,
86 kv_cache_k: Vec<wgpu::Buffer>,
88 kv_cache_v: Vec<wgpu::Buffer>,
90
91 hidden_buf: wgpu::Buffer, q_buf: wgpu::Buffer, k_buf: wgpu::Buffer, v_buf: wgpu::Buffer, attn_out_buf: wgpu::Buffer, ffn_gate_buf: wgpu::Buffer, ffn_up_buf: wgpu::Buffer, ffn_silu_buf: wgpu::Buffer, ffn_out_buf: wgpu::Buffer, norm_buf: wgpu::Buffer, staging_buf: wgpu::Buffer, hidden_dim: u32,
107 num_heads: u32,
108 num_kv_heads: u32,
109 head_dim: u32,
110 intermediate_dim: u32,
111 rms_norm_eps: f32,
115 rope_theta: f64,
117}
118
119pub const DEFAULT_RMS_NORM_EPS: f32 = 1e-6;
121
122pub const DEFAULT_ROPE_THETA: f64 = 1_000_000.0;
124
125fn rope_rotate_heads(
127 data: &mut [f32],
128 heads: usize,
129 head_dim: usize,
130 position: usize,
131 rope_theta: f64,
132) {
133 let half = head_dim / 2;
134 for h in 0..heads {
135 let offset = h * head_dim;
136 for i in 0..half {
137 let theta = rope_theta.powf(-((2 * i) as f64) / head_dim as f64);
138 let angle = position as f64 * theta;
139 let (sin_a, cos_a) = (angle.sin() as f32, angle.cos() as f32);
140 let x0 = data[offset + i];
141 let x1 = data[offset + i + half];
142 data[offset + i] = x0 * cos_a - x1 * sin_a;
143 data[offset + i + half] = x0 * sin_a + x1 * cos_a;
144 }
145 }
146}
147
148fn rmsnorm_params(dim: u32, eps: f32) -> [u32; 4] {
150 [dim, eps.to_bits(), 0, 0]
151}
152
153const RMSNORM_SHADER: &str = r#"
156@group(0) @binding(0) var<storage, read> input: array<f32>;
157@group(0) @binding(1) var<storage, read> weight: array<f32>;
158@group(0) @binding(2) var<storage, read_write> output: array<f32>;
159@group(0) @binding(3) var<uniform> params: vec4<u32>; // (dim, bitcast<u32>(eps), 0, 0)
160
161var<workgroup> shared_sum: array<f32, 256>;
162
163@compute @workgroup_size(256)
164fn main(@builtin(local_invocation_id) lid: vec3<u32>,
165 @builtin(workgroup_id) wg_id: vec3<u32>) {
166 let dim = params.x;
167 let row = wg_id.y;
168 let base = row * dim;
169 let tid = lid.x;
170
171 // Compute sum of squares (reduction) for this row
172 var local_sum: f32 = 0.0;
173 var i = tid;
174 while (i < dim) {
175 let val = input[base + i];
176 local_sum += val * val;
177 i += 256u;
178 }
179 shared_sum[tid] = local_sum;
180 workgroupBarrier();
181
182 // Tree reduction
183 var stride = 128u;
184 while (stride > 0u) {
185 if (tid < stride) {
186 shared_sum[tid] += shared_sum[tid + stride];
187 }
188 workgroupBarrier();
189 stride >>= 1u;
190 }
191
192 let eps = bitcast<f32>(params.y);
193 let rms = sqrt(shared_sum[0] / f32(dim) + eps);
194
195 // Normalize and scale
196 i = tid;
197 while (i < dim) {
198 output[base + i] = (input[base + i] / rms) * weight[i];
199 i += 256u;
200 }
201}
202"#;
203
204const SILU_MUL_SHADER: &str = r#"
206@group(0) @binding(0) var<storage, read> gate: array<f32>;
207@group(0) @binding(1) var<storage, read> up: array<f32>;
208@group(0) @binding(2) var<storage, read_write> output: array<f32>;
209@group(0) @binding(3) var<uniform> params: vec4<u32>; // (dim, 0, 0, 0)
210
211@compute @workgroup_size(256)
212fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
213 let idx = gid.x;
214 if (idx >= params.x) { return; }
215 let g = gate[idx];
216 let silu_g = g / (1.0 + exp(-g));
217 output[idx] = silu_g * up[idx];
218}
219"#;
220
221const RESIDUAL_SHADER: &str = r#"
223@group(0) @binding(0) var<storage, read> a: array<f32>;
224@group(0) @binding(1) var<storage, read> b: array<f32>;
225@group(0) @binding(2) var<storage, read_write> output: array<f32>;
226@group(0) @binding(3) var<uniform> params: vec4<u32>; // (dim, 0, 0, 0)
227
228@compute @workgroup_size(256)
229fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
230 let idx = gid.x;
231 if (idx >= params.x) { return; }
232 output[idx] = a[idx] + b[idx];
233}
234"#;
235
236const BATCH_ROPE_SHADER: &str = r#"
240@group(0) @binding(0) var<storage, read_write> qk: array<f32>;
241
242struct RopeParams {
243 seq_len: u32,
244 num_heads: u32,
245 head_dim: u32,
246 _pad: u32,
247}
248
249@group(0) @binding(1) var<uniform> params: RopeParams;
250
251@compute @workgroup_size(256)
252fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
253 let idx = gid.x;
254 let total = params.seq_len * params.num_heads * params.head_dim;
255 if (idx >= total) { return; }
256
257 let head_dim = params.head_dim;
258 let half_hd = head_dim / 2u;
259
260 // Decompose idx into (position, head, pos_in_head)
261 let elements_per_pos = params.num_heads * head_dim;
262 let position = idx / elements_per_pos;
263 let within_pos = idx % elements_per_pos;
264 let head_idx = within_pos / head_dim;
265 let pos_in_head = within_pos % head_dim;
266
267 // Only process the first half of each head (pairs with second half)
268 if (pos_in_head >= half_hd) { return; }
269
270 let theta = pow(1000000.0, -f32(pos_in_head * 2u) / f32(head_dim));
271 let angle = f32(position) * theta;
272 let cos_a = cos(angle);
273 let sin_a = sin(angle);
274
275 let base = position * elements_per_pos + head_idx * head_dim;
276 let i0 = base + pos_in_head;
277 let i1 = i0 + half_hd;
278
279 let x0 = qk[i0];
280 let x1 = qk[i1];
281 qk[i0] = x0 * cos_a - x1 * sin_a;
282 qk[i1] = x0 * sin_a + x1 * cos_a;
283}
284"#;
285
286const ROPE_SHADER: &str = r#"
288@group(0) @binding(0) var<storage, read_write> qk: array<f32>;
289@group(0) @binding(1) var<uniform> params: vec4<u32>; // (dim, position, num_heads, head_dim)
290
291@compute @workgroup_size(256)
292fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
293 let idx = gid.x;
294 let dim = params.x;
295 let position = params.y;
296 let head_dim = params.w;
297
298 if (idx >= dim) { return; }
299
300 let half_hd = head_dim / 2u;
301 let head_idx = idx / head_dim;
302 let pos_in_head = idx % head_dim;
303
304 if (pos_in_head >= half_hd) { return; }
305
306 let theta = pow(1000000.0, -f32(pos_in_head * 2u) / f32(head_dim));
307 let angle = f32(position) * theta;
308 let cos_a = cos(angle);
309 let sin_a = sin(angle);
310
311 let i0 = head_idx * head_dim + pos_in_head;
312 let i1 = i0 + half_hd;
313
314 let x0 = qk[i0];
315 let x1 = qk[i1];
316 qk[i0] = x0 * cos_a - x1 * sin_a;
317 qk[i1] = x0 * sin_a + x1 * cos_a;
318}
319"#;
320
321impl WgslForwardPass {
322 pub fn rmsnorm_shader() -> &'static str {
324 RMSNORM_SHADER
325 }
326 pub fn silu_mul_shader() -> &'static str {
327 SILU_MUL_SHADER
328 }
329 pub fn residual_shader() -> &'static str {
330 RESIDUAL_SHADER
331 }
332 pub fn rope_shader() -> &'static str {
333 ROPE_SHADER
334 }
335
336 pub fn new(
341 device: wgpu::Device,
342 queue: wgpu::Queue,
343 hidden_dim: usize,
344 num_heads: usize,
345 num_kv_heads: usize,
346 head_dim: usize,
347 intermediate_dim: usize,
348 ) -> Self {
349 let q_dim = num_heads * head_dim;
350 let kv_dim = num_kv_heads * head_dim;
351
352 let matmul_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
354 label: Some("matmul"),
355 source: wgpu::ShaderSource::Wgsl(crate::backends::gpu::shaders::MATMUL_SHADER.into()),
356 });
357 let rmsnorm_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
358 label: Some("rmsnorm"),
359 source: wgpu::ShaderSource::Wgsl(RMSNORM_SHADER.into()),
360 });
361 let silu_mul_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
362 label: Some("silu_mul"),
363 source: wgpu::ShaderSource::Wgsl(SILU_MUL_SHADER.into()),
364 });
365 let rope_shader_mod = device.create_shader_module(wgpu::ShaderModuleDescriptor {
366 label: Some("rope"),
367 source: wgpu::ShaderSource::Wgsl(ROPE_SHADER.into()),
368 });
369 let residual_shader_mod = device.create_shader_module(wgpu::ShaderModuleDescriptor {
370 label: Some("residual"),
371 source: wgpu::ShaderSource::Wgsl(RESIDUAL_SHADER.into()),
372 });
373
374 let matmul_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
376 label: Some("matmul_bgl"),
377 entries: &[
378 bgl_storage(0, true),
379 bgl_storage(1, true),
380 bgl_storage(2, false),
381 bgl_uniform(3),
382 ],
383 });
384 let elementwise_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
385 label: Some("ew_bgl"),
386 entries: &[
387 bgl_storage(0, true),
388 bgl_storage(1, true),
389 bgl_storage(2, false),
390 bgl_uniform(3),
391 ],
392 });
393
394 let make_pipeline =
396 |shader: &wgpu::ShaderModule, bgl: &wgpu::BindGroupLayout, label: &str| {
397 let pl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
398 label: Some(label),
399 bind_group_layouts: &[bgl],
400 push_constant_ranges: &[],
401 });
402 device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
403 label: Some(label),
404 layout: Some(&pl),
405 module: shader,
406 entry_point: Some("main"),
407 compilation_options: Default::default(),
408 cache: None,
409 })
410 };
411
412 let matmul_pipeline = make_pipeline(&matmul_shader, &matmul_bgl, "matmul_pipe");
413
414 let tiled_matmul_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
416 label: Some("tiled_matmul"),
417 source: wgpu::ShaderSource::Wgsl(
418 crate::backends::gpu::shaders::TILED_GEMM_SHADER.into(),
419 ),
420 });
421 let tiled_matmul_pipeline =
422 make_pipeline(&tiled_matmul_shader, &matmul_bgl, "tiled_matmul_pipe");
423
424 let attention_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
426 label: Some("causal_attention"),
427 source: wgpu::ShaderSource::Wgsl(
428 crate::backends::gpu::shaders::CAUSAL_ATTENTION_SHADER.into(),
429 ),
430 });
431 let attention_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
432 label: Some("attn_bgl"),
433 entries: &[
434 bgl_storage(0, true), bgl_storage(1, true), bgl_storage(2, true), bgl_storage(3, false), bgl_uniform(4), ],
440 });
441 let attention_pl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
442 label: Some("attn_pl"),
443 bind_group_layouts: &[&attention_bgl],
444 push_constant_ranges: &[],
445 });
446 let attention_pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
447 label: Some("attn_pipe"),
448 layout: Some(&attention_pl),
449 module: &attention_shader,
450 entry_point: Some("main"),
451 compilation_options: Default::default(),
452 cache: None,
453 });
454
455 let gemv_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
457 label: Some("gemv"),
458 source: wgpu::ShaderSource::Wgsl(crate::backends::gpu::shaders::GEMV_SHADER.into()),
459 });
460 let gemv_pipeline = make_pipeline(&gemv_shader, &matmul_bgl, "gemv_pipe");
461
462 let q4k_gemv_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
464 label: Some("q4k_gemv"),
465 source: wgpu::ShaderSource::Wgsl(crate::backends::gpu::shaders::Q4K_GEMV_SHADER.into()),
466 });
467 let q4k_gemv_pipeline = make_pipeline(&q4k_gemv_shader, &matmul_bgl, "q4k_gemv_pipe");
468
469 let rmsnorm_pipeline = make_pipeline(&rmsnorm_shader, &elementwise_bgl, "rmsnorm_pipe");
470 let silu_mul_pipeline = make_pipeline(&silu_mul_shader, &elementwise_bgl, "silu_pipe");
471 let residual_pipeline = make_pipeline(&residual_shader_mod, &elementwise_bgl, "res_pipe");
472
473 let rope_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
475 label: Some("rope_bgl"),
476 entries: &[bgl_storage(0, false), bgl_uniform(1)],
477 });
478 let rope_pipeline = {
479 let pl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
480 label: Some("rope_pl"),
481 bind_group_layouts: &[&rope_bgl],
482 push_constant_ranges: &[],
483 });
484 device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
485 label: Some("rope_pipe"),
486 layout: Some(&pl),
487 module: &rope_shader_mod,
488 entry_point: Some("main"),
489 compilation_options: Default::default(),
490 cache: None,
491 })
492 };
493
494 let batch_rope_shader_mod = device.create_shader_module(wgpu::ShaderModuleDescriptor {
496 label: Some("batch_rope"),
497 source: wgpu::ShaderSource::Wgsl(BATCH_ROPE_SHADER.into()),
498 });
499 let batch_rope_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
500 label: Some("batch_rope_bgl"),
501 entries: &[bgl_storage(0, false), bgl_uniform(1)],
502 });
503 let batch_rope_pipeline = {
504 let pl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
505 label: Some("batch_rope_pl"),
506 bind_group_layouts: &[&batch_rope_bgl],
507 push_constant_ranges: &[],
508 });
509 device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
510 label: Some("batch_rope_pipe"),
511 layout: Some(&pl),
512 module: &batch_rope_shader_mod,
513 entry_point: Some("main"),
514 compilation_options: Default::default(),
515 cache: None,
516 })
517 };
518
519 let buf = |size: usize, label: &str| -> wgpu::Buffer {
521 device.create_buffer(&wgpu::BufferDescriptor {
522 label: Some(label),
523 size: (size * 4) as u64,
524 usage: wgpu::BufferUsages::STORAGE
525 | wgpu::BufferUsages::COPY_SRC
526 | wgpu::BufferUsages::COPY_DST,
527 mapped_at_creation: false,
528 })
529 };
530
531 let max_seq = 2048;
535 let hidden_buf = buf(max_seq * hidden_dim, "hidden");
536 let q_buf = buf(max_seq * q_dim, "q");
537 let k_buf = buf(max_seq * kv_dim, "k");
538 let v_buf = buf(max_seq * kv_dim, "v");
539 let attn_out_buf = buf(max_seq * hidden_dim, "attn_out");
540 let ffn_gate_buf = buf(max_seq * intermediate_dim, "ffn_gate");
541 let ffn_up_buf = buf(max_seq * intermediate_dim, "ffn_up");
542 let ffn_silu_buf = buf(max_seq * intermediate_dim, "ffn_silu");
543 let ffn_out_buf = buf(max_seq * hidden_dim, "ffn_out");
544 let norm_buf = buf(max_seq * hidden_dim, "norm");
545
546 let max_out = max_seq * hidden_dim.max(intermediate_dim);
547 let staging_buf = device.create_buffer(&wgpu::BufferDescriptor {
548 label: Some("staging"),
549 size: (max_out * 4) as u64,
550 usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
551 mapped_at_creation: false,
552 });
553
554 Self {
555 device,
556 queue,
557 matmul_pipeline,
558 tiled_matmul_pipeline,
559 attention_pipeline,
560 attention_bgl,
561 gemv_pipeline,
562 q4k_gemv_pipeline,
563 rmsnorm_pipeline,
564 silu_mul_pipeline,
565 rope_pipeline,
566 batch_rope_pipeline,
567 batch_rope_bgl,
568 residual_pipeline,
569 matmul_bgl,
570 elementwise_bgl,
571 weight_buffers: HashMap::new(),
572 q4k_weights: HashMap::new(),
573 kv_cache_k: Vec::new(),
574 kv_cache_v: Vec::new(),
575 cpu_biases: HashMap::new(),
576 hidden_buf,
577 q_buf,
578 k_buf,
579 v_buf,
580 attn_out_buf,
581 ffn_gate_buf,
582 ffn_up_buf,
583 ffn_silu_buf,
584 ffn_out_buf,
585 norm_buf,
586 staging_buf,
587 hidden_dim: hidden_dim as u32,
588 num_heads: num_heads as u32,
589 num_kv_heads: num_kv_heads as u32,
590 head_dim: head_dim as u32,
591 intermediate_dim: intermediate_dim as u32,
592 rms_norm_eps: DEFAULT_RMS_NORM_EPS,
593 rope_theta: DEFAULT_ROPE_THETA,
594 }
595 }
596
597 pub fn set_rms_norm_eps(&mut self, eps: f32) {
600 self.rms_norm_eps = eps;
601 }
602
603 pub fn set_rope_theta(&mut self, theta: f64) {
606 self.rope_theta = theta;
607 }
608
609 #[must_use]
611 pub fn rms_norm_eps(&self) -> f32 {
612 self.rms_norm_eps
613 }
614
615 pub fn upload_weight(&mut self, name: &str, data: &[f32]) {
618 if name.contains("bias") {
619 self.cpu_biases.insert(name.to_string(), data.to_vec());
621 return;
622 }
623 let size_bytes = (data.len() * 4) as u64;
625 let max_binding = self.device.limits().max_storage_buffer_binding_size as u64;
626 if size_bytes > max_binding {
627 eprintln!(
628 "[wgpu] Skipping weight '{}' ({:.1} MB > {:.1} MB limit) — CPU fallback",
629 name,
630 size_bytes as f64 / 1e6,
631 max_binding as f64 / 1e6
632 );
633 return;
634 }
635 use wgpu::util::DeviceExt;
636 let buffer = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
637 label: Some(name),
638 contents: bytemuck::cast_slice(data),
639 usage: wgpu::BufferUsages::STORAGE,
640 });
641 self.weight_buffers.insert(name.to_string(), buffer);
642 }
643
644 pub fn upload_q4k_weight(&mut self, name: &str, data: &[u8]) {
646 use wgpu::util::DeviceExt;
647 let buffer = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
648 label: Some(name),
649 contents: data,
650 usage: wgpu::BufferUsages::STORAGE,
651 });
652 self.q4k_weights.insert(name.to_string(), buffer);
653 }
654
655 pub fn init_kv_cache(&mut self, num_layers: usize) {
657 let kv_dim = (self.num_kv_heads * self.head_dim) as u64;
658 let max_seq = 2048u64;
659 for _ in 0..num_layers {
660 let k = self.device.create_buffer(&wgpu::BufferDescriptor {
661 label: Some("kv_cache_k"),
662 size: max_seq * kv_dim * 4,
663 usage: wgpu::BufferUsages::STORAGE
664 | wgpu::BufferUsages::COPY_DST
665 | wgpu::BufferUsages::COPY_SRC,
666 mapped_at_creation: false,
667 });
668 let v = self.device.create_buffer(&wgpu::BufferDescriptor {
669 label: Some("kv_cache_v"),
670 size: max_seq * kv_dim * 4,
671 usage: wgpu::BufferUsages::STORAGE
672 | wgpu::BufferUsages::COPY_DST
673 | wgpu::BufferUsages::COPY_SRC,
674 mapped_at_creation: false,
675 });
676 self.kv_cache_k.push(k);
677 self.kv_cache_v.push(v);
678 }
679 }
680
681 pub fn weight_count(&self) -> usize {
683 self.weight_buffers.len()
684 }
685
686 pub fn weight_buffer(&self, name: &str) -> Option<&wgpu::Buffer> {
689 self.weight_buffers.get(name)
690 }
691
692 pub fn device_ref(&self) -> &wgpu::Device {
694 &self.device
695 }
696
697 pub fn queue_ref(&self) -> &wgpu::Queue {
699 &self.queue
700 }
701
702 pub fn hidden_buffer(&self) -> &wgpu::Buffer {
704 &self.hidden_buf
705 }
706
707 pub fn q_buffer(&self) -> &wgpu::Buffer {
709 &self.q_buf
710 }
711
712 pub fn k_buffer(&self) -> &wgpu::Buffer {
714 &self.k_buf
715 }
716
717 pub fn v_buffer(&self) -> &wgpu::Buffer {
719 &self.v_buf
720 }
721
722 pub fn gpu_residual_add(
724 &self,
725 a: &wgpu::Buffer,
726 b: &wgpu::Buffer,
727 output: &wgpu::Buffer,
728 len: u32,
729 ) {
730 let mut encoder = self.device.create_command_encoder(&Default::default());
731 self.encode_residual(&mut encoder, a, b, output, len);
732 self.queue.submit(Some(encoder.finish()));
733 }
734
735 pub fn gpu_rmsnorm(&self, weight: &wgpu::Buffer, output: &wgpu::Buffer, _seq_len: u32) {
738 let mut encoder = self.device.create_command_encoder(&Default::default());
739 self.encode_rmsnorm(&mut encoder, &self.hidden_buf, weight, output, self.hidden_dim);
740 self.queue.submit(Some(encoder.finish()));
741 }
742
743 pub fn download_hidden(&self, len: usize) -> Vec<f32> {
745 let size = (len * 4) as u64;
746 let staging = self.device.create_buffer(&wgpu::BufferDescriptor {
747 label: Some("hidden_download"),
748 size,
749 usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
750 mapped_at_creation: false,
751 });
752 let mut encoder = self.device.create_command_encoder(&Default::default());
753 encoder.copy_buffer_to_buffer(&self.hidden_buf, 0, &staging, 0, size);
754 self.queue.submit(Some(encoder.finish()));
755
756 let slice = staging.slice(..size);
757 let (tx, rx) = std::sync::mpsc::channel();
758 slice.map_async(wgpu::MapMode::Read, move |r| {
759 tx.send(r).ok();
760 });
761 self.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: None }).ok();
762 rx.recv()
763 .expect("GPU map_async callback channel disconnected")
764 .expect("GPU buffer mapping failed");
765
766 let data = slice.get_mapped_range();
767 let result: Vec<f32> = bytemuck::cast_slice(&data)[..len].to_vec();
768 drop(data);
769 staging.unmap();
770 result
771 }
772
773 pub fn total_vram_bytes(&self) -> usize {
775 let weight_bytes: usize = self.weight_buffers.values().map(|b| b.size() as usize).sum();
776 let intermediate_bytes = (self.hidden_dim as usize * 4) * 4 + (self.num_heads as usize * self.head_dim as usize * 4) + (self.num_kv_heads as usize * self.head_dim as usize * 4) * 2 + (self.intermediate_dim as usize * 4) * 2; weight_bytes + intermediate_bytes
781 }
782
783 #[provable_contracts_macros::contract("wgpu-forward-pass-v1", equation = "rmsnorm_correctness")]
789 pub fn forward_model(
790 &self,
791 token_id: u32,
792 position: usize,
793 num_layers: usize,
794 token_embedding: &[f32],
795 output_norm_weight: &[f32],
796 lm_head_weight: &[f32],
797 vocab_size: usize,
798 eps: f32,
799 kv_caches: &mut Vec<(Vec<f32>, Vec<f32>)>,
800 ) -> Result<Vec<f32>, String> {
801 let hd = self.hidden_dim as usize;
802
803 let embed_start = token_id as usize * hd;
805 if embed_start + hd > token_embedding.len() {
806 return Err(format!(
807 "Token {} out of range (embedding size {})",
808 token_id,
809 token_embedding.len() / hd
810 ));
811 }
812 let mut hidden: Vec<f32> = token_embedding[embed_start..embed_start + hd].to_vec();
813
814 while kv_caches.len() < num_layers {
817 kv_caches.push((Vec::new(), Vec::new()));
818 }
819 for layer_idx in 0..num_layers {
820 let prefix = format!("layer.{layer_idx}");
821 let (ref mut k_cache, ref mut v_cache) = kv_caches[layer_idx];
822 self.forward_layer(&mut hidden, &prefix, position, k_cache, v_cache)?;
823 }
824
825 let rms = (hidden.iter().map(|x| x * x).sum::<f32>() / hd as f32 + eps).sqrt();
827 for i in 0..hd {
828 hidden[i] = (hidden[i] / rms) * output_norm_weight[i];
829 }
830
831 let mut logits = vec![0.0f32; vocab_size];
837 for v in 0..vocab_size {
838 let mut sum = 0.0f32;
839 let row_start = v * hd;
840 for j in 0..hd {
841 sum += lm_head_weight[row_start + j] * hidden[j];
842 }
843 logits[v] = sum;
844 }
845 Ok(logits)
846 }
847
848 pub fn forward_layer(
855 &self,
856 hidden: &mut [f32],
857 layer_prefix: &str,
858 _position: usize,
859 kv_cache_k: &mut Vec<f32>, kv_cache_v: &mut Vec<f32>, ) -> Result<(), String> {
862 let hd = self.hidden_dim;
863
864 self.queue.write_buffer(&self.hidden_buf, 0, bytemuck::cast_slice(hidden));
866
867 let mut encoder = self.device.create_command_encoder(&Default::default());
868
869 let norm_w = self
871 .weight_buffers
872 .get(&format!("{layer_prefix}.attn_norm"))
873 .ok_or_else(|| format!("Missing {layer_prefix}.attn_norm"))?;
874 self.encode_rmsnorm(&mut encoder, &self.hidden_buf, norm_w, &self.norm_buf, hd);
875
876 let q_dim = self.num_heads * self.head_dim;
878 let kv_dim = self.num_kv_heads * self.head_dim;
879
880 self.encode_matmul(
881 &mut encoder,
882 &self.norm_buf,
883 layer_prefix,
884 "q_proj",
885 &self.q_buf,
886 1,
887 hd,
888 q_dim,
889 );
890 self.encode_matmul(
891 &mut encoder,
892 &self.norm_buf,
893 layer_prefix,
894 "k_proj",
895 &self.k_buf,
896 1,
897 hd,
898 kv_dim,
899 );
900 self.encode_matmul(
901 &mut encoder,
902 &self.norm_buf,
903 layer_prefix,
904 "v_proj",
905 &self.v_buf,
906 1,
907 hd,
908 kv_dim,
909 );
910
911 let q_bytes = (q_dim * 4) as u64;
914 let kv_bytes = (kv_dim * 4) as u64;
915
916 let q_staging = self.device.create_buffer(&wgpu::BufferDescriptor {
918 label: Some("q_stg"),
919 size: q_bytes,
920 usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
921 mapped_at_creation: false,
922 });
923 let k_staging = self.device.create_buffer(&wgpu::BufferDescriptor {
924 label: Some("k_stg"),
925 size: kv_bytes,
926 usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
927 mapped_at_creation: false,
928 });
929 let v_staging = self.device.create_buffer(&wgpu::BufferDescriptor {
930 label: Some("v_stg"),
931 size: kv_bytes,
932 usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
933 mapped_at_creation: false,
934 });
935 encoder.copy_buffer_to_buffer(&self.q_buf, 0, &q_staging, 0, q_bytes);
936 encoder.copy_buffer_to_buffer(&self.k_buf, 0, &k_staging, 0, kv_bytes);
937 encoder.copy_buffer_to_buffer(&self.v_buf, 0, &v_staging, 0, kv_bytes);
938 self.queue.submit(Some(encoder.finish()));
939
940 let mut q_data = vec![0.0f32; q_dim as usize];
942 {
943 let slice = q_staging.slice(..q_bytes);
944 let (tx, rx) = std::sync::mpsc::channel();
945 slice.map_async(wgpu::MapMode::Read, move |r| {
946 tx.send(r).ok();
947 });
948 self.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: None }).ok();
949 rx.recv().map_err(|e| format!("q recv: {e}"))?.map_err(|e| format!("q map: {e:?}"))?;
950 let data = slice.get_mapped_range();
951 q_data.copy_from_slice(&bytemuck::cast_slice::<u8, f32>(&data)[..q_dim as usize]);
952 }
953 q_staging.unmap();
954
955 let mut k_data = vec![0.0f32; kv_dim as usize];
957 {
958 let slice = k_staging.slice(..kv_bytes);
959 let (tx, rx) = std::sync::mpsc::channel();
960 slice.map_async(wgpu::MapMode::Read, move |r| {
961 tx.send(r).ok();
962 });
963 self.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: None }).ok();
964 rx.recv().map_err(|e| format!("k recv: {e}"))?.map_err(|e| format!("k map: {e:?}"))?;
965 let data = slice.get_mapped_range();
966 k_data.copy_from_slice(&bytemuck::cast_slice::<u8, f32>(&data)[..kv_dim as usize]);
967 }
968 k_staging.unmap();
969
970 let mut v_data = vec![0.0f32; kv_dim as usize];
972 {
973 let slice = v_staging.slice(..kv_bytes);
974 let (tx, rx) = std::sync::mpsc::channel();
975 slice.map_async(wgpu::MapMode::Read, move |r| {
976 tx.send(r).ok();
977 });
978 self.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: None }).ok();
979 rx.recv().map_err(|e| format!("v recv: {e}"))?.map_err(|e| format!("v map: {e:?}"))?;
980 let data = slice.get_mapped_range();
981 v_data.copy_from_slice(&bytemuck::cast_slice::<u8, f32>(&data)[..kv_dim as usize]);
982 }
983 v_staging.unmap();
984
985 if let Some(q_bias) = self.cpu_biases.get(&format!("{layer_prefix}.q_bias")) {
987 for (q, b) in q_data.iter_mut().zip(q_bias.iter()) {
988 *q += *b;
989 }
990 }
991 if let Some(k_bias) = self.cpu_biases.get(&format!("{layer_prefix}.k_bias")) {
992 for (k, b) in k_data.iter_mut().zip(k_bias.iter()) {
993 *k += *b;
994 }
995 }
996 if let Some(v_bias) = self.cpu_biases.get(&format!("{layer_prefix}.v_bias")) {
997 for (v, b) in v_data.iter_mut().zip(v_bias.iter()) {
998 *v += *b;
999 }
1000 }
1001
1002 let head_dim = self.head_dim as usize;
1004 let position = _position; rope_rotate_heads(
1006 &mut q_data,
1007 self.num_heads as usize,
1008 head_dim,
1009 position,
1010 self.rope_theta,
1011 );
1012 rope_rotate_heads(
1013 &mut k_data,
1014 self.num_kv_heads as usize,
1015 head_dim,
1016 position,
1017 self.rope_theta,
1018 );
1019
1020 let head_dim = self.head_dim as usize;
1022 let num_heads = self.num_heads as usize;
1023 let num_kv_heads = self.num_kv_heads as usize;
1024 let kv_dim_usize = kv_dim as usize;
1025
1026 kv_cache_k.extend_from_slice(&k_data);
1027 kv_cache_v.extend_from_slice(&v_data);
1028 let seq_len = kv_cache_k.len() / kv_dim_usize;
1029
1030 let kv_group = num_heads / num_kv_heads;
1032 let scale = 1.0 / (head_dim as f32).sqrt();
1033 let mut attn_out = vec![0.0f32; q_dim as usize];
1034
1035 for h in 0..num_heads {
1036 let kv_h = h / kv_group;
1037 let q_offset = h * head_dim;
1038 let kv_offset = kv_h * head_dim;
1039
1040 let scores = Self::attention_scores(
1042 &q_data[q_offset..q_offset + head_dim],
1043 kv_cache_k,
1044 kv_offset,
1045 kv_dim_usize,
1046 seq_len,
1047 scale,
1048 );
1049
1050 let out_offset = h * head_dim;
1052 Self::attention_weighted_v(
1053 &scores,
1054 kv_cache_v,
1055 &mut attn_out[out_offset..out_offset + head_dim],
1056 kv_offset,
1057 kv_dim_usize,
1058 );
1059 }
1060
1061 self.queue.write_buffer(&self.q_buf, 0, bytemuck::cast_slice(&attn_out));
1063
1064 let mut encoder = self.device.create_command_encoder(&Default::default());
1066
1067 self.encode_matmul(
1069 &mut encoder,
1070 &self.q_buf,
1071 layer_prefix,
1072 "o_proj",
1073 &self.attn_out_buf,
1074 1,
1075 q_dim,
1076 hd,
1077 );
1078
1079 self.encode_residual(
1081 &mut encoder,
1082 &self.hidden_buf,
1083 &self.attn_out_buf,
1084 &self.ffn_out_buf,
1085 hd,
1086 );
1087
1088 let ffn_norm_w = self
1090 .weight_buffers
1091 .get(&format!("{layer_prefix}.ffn_norm"))
1092 .ok_or_else(|| format!("Missing {layer_prefix}.ffn_norm"))?;
1093 self.encode_rmsnorm(&mut encoder, &self.ffn_out_buf, ffn_norm_w, &self.norm_buf, hd);
1094
1095 let inter = self.intermediate_dim;
1097 self.encode_matmul(
1098 &mut encoder,
1099 &self.norm_buf,
1100 layer_prefix,
1101 "gate_proj",
1102 &self.ffn_gate_buf,
1103 1,
1104 hd,
1105 inter,
1106 );
1107 self.encode_matmul(
1108 &mut encoder,
1109 &self.norm_buf,
1110 layer_prefix,
1111 "up_proj",
1112 &self.ffn_up_buf,
1113 1,
1114 hd,
1115 inter,
1116 );
1117
1118 self.encode_silu_mul(
1124 &mut encoder,
1125 &self.ffn_gate_buf,
1126 &self.ffn_up_buf,
1127 &self.ffn_silu_buf,
1128 inter,
1129 );
1130
1131 self.encode_matmul(
1133 &mut encoder,
1134 &self.ffn_silu_buf,
1135 layer_prefix,
1136 "down_proj",
1137 &self.norm_buf,
1138 1,
1139 inter,
1140 hd,
1141 );
1142
1143 self.encode_residual(&mut encoder, &self.ffn_out_buf, &self.norm_buf, &self.hidden_buf, hd);
1145
1146 encoder.copy_buffer_to_buffer(&self.hidden_buf, 0, &self.staging_buf, 0, (hd * 4) as u64);
1148 self.queue.submit(Some(encoder.finish()));
1149
1150 let slice = self.staging_buf.slice(..(hd as u64 * 4));
1152 let (tx, rx) = std::sync::mpsc::channel();
1153 slice.map_async(wgpu::MapMode::Read, move |r| {
1154 tx.send(r).ok();
1155 });
1156 self.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: None }).ok();
1157 rx.recv().map_err(|e| format!("recv: {e}"))?.map_err(|e| format!("map: {e:?}"))?;
1158 {
1159 let data = slice.get_mapped_range();
1160 hidden.copy_from_slice(
1161 &bytemuck::cast_slice::<u8, f32>(&data)[..self.hidden_dim as usize],
1162 );
1163 }
1164 self.staging_buf.unmap();
1165
1166 Ok(())
1167 }
1168
1169 fn attention_scores(
1174 q_head: &[f32],
1175 kv_cache_k: &[f32],
1176 kv_offset: usize,
1177 kv_dim: usize,
1178 seq_len: usize,
1179 scale: f32,
1180 ) -> Vec<f32> {
1181 let mut scores = vec![0.0f32; seq_len];
1182 for (s, score) in scores.iter_mut().enumerate() {
1183 let k_offset = s * kv_dim + kv_offset;
1184 let mut dot = 0.0f32;
1185 for (d, q) in q_head.iter().enumerate() {
1186 dot += q * kv_cache_k[k_offset + d];
1187 }
1188 *score = dot * scale;
1189 }
1190
1191 let max_score = scores.iter().copied().fold(f32::NEG_INFINITY, f32::max);
1193 let mut sum = 0.0f32;
1194 for s in scores.iter_mut() {
1195 *s = (*s - max_score).exp();
1196 sum += *s;
1197 }
1198 if sum > 0.0 {
1199 for s in scores.iter_mut() {
1200 *s /= sum;
1201 }
1202 }
1203 scores
1204 }
1205
1206 fn attention_weighted_v(
1210 scores: &[f32],
1211 kv_cache_v: &[f32],
1212 out_head: &mut [f32],
1213 kv_offset: usize,
1214 kv_dim: usize,
1215 ) {
1216 for (d, out) in out_head.iter_mut().enumerate() {
1217 let mut val = 0.0f32;
1218 for (s, score) in scores.iter().enumerate() {
1219 let v_offset = s * kv_dim + kv_offset;
1220 val += score * kv_cache_v[v_offset + d];
1221 }
1222 *out = val;
1223 }
1224 }
1225
1226 pub fn encode_forward_layer_training(
1242 &self,
1243 encoder: &mut wgpu::CommandEncoder,
1244 seq_len: u32,
1245 layer_prefix: &str,
1246 saved: &LayerActivations,
1247 lora: Option<&QkvLoRA<'_>>,
1248 ) -> Result<(), String> {
1249 let hd = self.hidden_dim;
1250 let q_dim = self.num_heads * self.head_dim;
1251 let kv_dim = self.num_kv_heads * self.head_dim;
1252 let inter = self.intermediate_dim;
1253 let s = seq_len as usize;
1254
1255 let norm_w = self
1257 .weight_buffers
1258 .get(&format!("{layer_prefix}.attn_norm"))
1259 .ok_or_else(|| format!("Missing {layer_prefix}.attn_norm"))?;
1260 self.encode_rmsnorm(encoder, &self.hidden_buf, norm_w, &self.norm_buf, hd);
1261
1262 encoder.copy_buffer_to_buffer(
1264 &self.norm_buf,
1265 0,
1266 &saved.attn_norm_out,
1267 0,
1268 (s * hd as usize * 4) as u64,
1269 );
1270
1271 self.encode_matmul(
1273 encoder,
1274 &self.norm_buf,
1275 layer_prefix,
1276 "q_proj",
1277 &self.q_buf,
1278 seq_len,
1279 hd,
1280 q_dim,
1281 );
1282 self.encode_matmul(
1283 encoder,
1284 &self.norm_buf,
1285 layer_prefix,
1286 "k_proj",
1287 &self.k_buf,
1288 seq_len,
1289 hd,
1290 kv_dim,
1291 );
1292 self.encode_matmul(
1293 encoder,
1294 &self.norm_buf,
1295 layer_prefix,
1296 "v_proj",
1297 &self.v_buf,
1298 seq_len,
1299 hd,
1300 kv_dim,
1301 );
1302
1303 if let Some(lora) = lora {
1306 self.encode_lora_addmm(
1307 encoder,
1308 &saved.attn_norm_out,
1309 lora.q_a,
1310 lora.q_b,
1311 &self.q_buf,
1312 seq_len,
1313 lora.in_dim,
1314 lora.rank,
1315 lora.q_dim,
1316 lora.scale,
1317 lora.lora_pipeline,
1318 lora.lora_bgl,
1319 );
1320 self.encode_lora_addmm(
1321 encoder,
1322 &saved.attn_norm_out,
1323 lora.k_a,
1324 lora.k_b,
1325 &self.k_buf,
1326 seq_len,
1327 lora.in_dim,
1328 lora.rank,
1329 lora.kv_dim,
1330 lora.scale,
1331 lora.lora_pipeline,
1332 lora.lora_bgl,
1333 );
1334 self.encode_lora_addmm(
1335 encoder,
1336 &saved.attn_norm_out,
1337 lora.v_a,
1338 lora.v_b,
1339 &self.v_buf,
1340 seq_len,
1341 lora.in_dim,
1342 lora.rank,
1343 lora.kv_dim,
1344 lora.scale,
1345 lora.lora_pipeline,
1346 lora.lora_bgl,
1347 );
1348 }
1349
1350 if let Some(q_bias) = self.cpu_biases.get(&format!("{layer_prefix}.q_bias")) {
1352 self.encode_broadcast_bias(encoder, &self.q_buf, q_bias, seq_len);
1353 }
1354 if let Some(k_bias) = self.cpu_biases.get(&format!("{layer_prefix}.k_bias")) {
1355 self.encode_broadcast_bias(encoder, &self.k_buf, k_bias, seq_len);
1356 }
1357 if let Some(v_bias) = self.cpu_biases.get(&format!("{layer_prefix}.v_bias")) {
1358 self.encode_broadcast_bias(encoder, &self.v_buf, v_bias, seq_len);
1359 }
1360
1361 self.encode_batch_rope(encoder, &self.q_buf, seq_len, self.num_heads, self.head_dim);
1363 self.encode_batch_rope(encoder, &self.k_buf, seq_len, self.num_kv_heads, self.head_dim);
1364
1365 self.encode_attention(encoder, seq_len);
1367
1368 encoder.copy_buffer_to_buffer(
1370 &self.attn_out_buf,
1371 0,
1372 &saved.attn_output,
1373 0,
1374 (s * q_dim as usize * 4) as u64,
1375 );
1376
1377 self.encode_matmul(
1379 encoder,
1380 &self.attn_out_buf,
1381 layer_prefix,
1382 "o_proj",
1383 &self.q_buf,
1384 seq_len,
1385 q_dim,
1386 hd,
1387 );
1388
1389 self.encode_residual(
1391 encoder,
1392 &self.hidden_buf,
1393 &self.q_buf,
1394 &self.ffn_out_buf,
1395 hd * seq_len,
1396 );
1397
1398 let ffn_norm_w = self
1400 .weight_buffers
1401 .get(&format!("{layer_prefix}.ffn_norm"))
1402 .ok_or_else(|| format!("Missing {layer_prefix}.ffn_norm"))?;
1403 self.encode_rmsnorm(encoder, &self.ffn_out_buf, ffn_norm_w, &self.norm_buf, hd);
1404
1405 encoder.copy_buffer_to_buffer(
1407 &self.norm_buf,
1408 0,
1409 &saved.ffn_norm_out,
1410 0,
1411 (s * hd as usize * 4) as u64,
1412 );
1413
1414 self.encode_matmul(
1416 encoder,
1417 &self.norm_buf,
1418 layer_prefix,
1419 "gate_proj",
1420 &self.ffn_gate_buf,
1421 seq_len,
1422 hd,
1423 inter,
1424 );
1425 self.encode_matmul(
1426 encoder,
1427 &self.norm_buf,
1428 layer_prefix,
1429 "up_proj",
1430 &self.ffn_up_buf,
1431 seq_len,
1432 hd,
1433 inter,
1434 );
1435
1436 self.encode_silu_mul(
1438 encoder,
1439 &self.ffn_gate_buf,
1440 &self.ffn_up_buf,
1441 &self.ffn_silu_buf,
1442 inter * seq_len,
1443 );
1444
1445 encoder.copy_buffer_to_buffer(
1447 &self.ffn_silu_buf,
1448 0,
1449 &saved.silu_gate_output,
1450 0,
1451 (s * inter as usize * 4) as u64,
1452 );
1453
1454 self.encode_matmul(
1456 encoder,
1457 &self.ffn_silu_buf,
1458 layer_prefix,
1459 "down_proj",
1460 &self.norm_buf,
1461 seq_len,
1462 inter,
1463 hd,
1464 );
1465
1466 self.encode_residual(
1468 encoder,
1469 &self.ffn_out_buf,
1470 &self.norm_buf,
1471 &self.hidden_buf,
1472 hd * seq_len,
1473 );
1474
1475 Ok(())
1476 }
1477
1478 pub fn forward_layer_traced(
1481 &self,
1482 seq_len: u32,
1483 layer_prefix: &str,
1484 saved: &LayerActivations,
1485 lora: Option<&QkvLoRA<'_>>,
1486 ) -> Result<(), String> {
1487 let hd = self.hidden_dim;
1488 let q_dim = self.num_heads * self.head_dim;
1489 let kv_dim = self.num_kv_heads * self.head_dim;
1490 let inter = self.intermediate_dim;
1491 let s = seq_len as usize;
1492
1493 let norm_w = self
1494 .weight_buffers
1495 .get(&format!("{layer_prefix}.attn_norm"))
1496 .ok_or_else(|| format!("Missing {layer_prefix}.attn_norm"))?;
1497
1498 let mut trace = Vec::new();
1499 let mut run = |name: &str, f: &dyn Fn(&mut wgpu::CommandEncoder)| {
1500 let mut enc = self.device.create_command_encoder(&Default::default());
1501 f(&mut enc);
1502 self.queue.submit(Some(enc.finish()));
1503 let t = std::time::Instant::now();
1504 self.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: None }).ok();
1505 trace.push((name.to_string(), t.elapsed().as_millis() as u64));
1506 };
1507
1508 run("rmsnorm1", &|e| self.encode_rmsnorm(e, &self.hidden_buf, norm_w, &self.norm_buf, hd));
1509 {
1510 let mut e = self.device.create_command_encoder(&Default::default());
1511 e.copy_buffer_to_buffer(
1512 &self.norm_buf,
1513 0,
1514 &saved.attn_norm_out,
1515 0,
1516 (s * hd as usize * 4) as u64,
1517 );
1518 self.queue.submit(Some(e.finish()));
1519 }
1520 run("q_proj", &|e| {
1521 self.encode_matmul(
1522 e,
1523 &self.norm_buf,
1524 layer_prefix,
1525 "q_proj",
1526 &self.q_buf,
1527 seq_len,
1528 hd,
1529 q_dim,
1530 )
1531 });
1532 run("k_proj", &|e| {
1533 self.encode_matmul(
1534 e,
1535 &self.norm_buf,
1536 layer_prefix,
1537 "k_proj",
1538 &self.k_buf,
1539 seq_len,
1540 hd,
1541 kv_dim,
1542 )
1543 });
1544 run("v_proj", &|e| {
1545 self.encode_matmul(
1546 e,
1547 &self.norm_buf,
1548 layer_prefix,
1549 "v_proj",
1550 &self.v_buf,
1551 seq_len,
1552 hd,
1553 kv_dim,
1554 )
1555 });
1556 if let Some(lr) = lora {
1557 run("lora_qkv", &|e| {
1558 self.encode_lora_addmm(
1559 e,
1560 &saved.attn_norm_out,
1561 lr.q_a,
1562 lr.q_b,
1563 &self.q_buf,
1564 seq_len,
1565 lr.in_dim,
1566 lr.rank,
1567 lr.q_dim,
1568 lr.scale,
1569 lr.lora_pipeline,
1570 lr.lora_bgl,
1571 );
1572 self.encode_lora_addmm(
1573 e,
1574 &saved.attn_norm_out,
1575 lr.k_a,
1576 lr.k_b,
1577 &self.k_buf,
1578 seq_len,
1579 lr.in_dim,
1580 lr.rank,
1581 lr.kv_dim,
1582 lr.scale,
1583 lr.lora_pipeline,
1584 lr.lora_bgl,
1585 );
1586 self.encode_lora_addmm(
1587 e,
1588 &saved.attn_norm_out,
1589 lr.v_a,
1590 lr.v_b,
1591 &self.v_buf,
1592 seq_len,
1593 lr.in_dim,
1594 lr.rank,
1595 lr.kv_dim,
1596 lr.scale,
1597 lr.lora_pipeline,
1598 lr.lora_bgl,
1599 );
1600 });
1601 }
1602 if let Some(q_bias) = self.cpu_biases.get(&format!("{layer_prefix}.q_bias")) {
1604 run("q_bias", &|e| self.encode_broadcast_bias(e, &self.q_buf, q_bias, seq_len));
1605 }
1606 if let Some(k_bias) = self.cpu_biases.get(&format!("{layer_prefix}.k_bias")) {
1607 run("k_bias", &|e| self.encode_broadcast_bias(e, &self.k_buf, k_bias, seq_len));
1608 }
1609 if let Some(v_bias) = self.cpu_biases.get(&format!("{layer_prefix}.v_bias")) {
1610 run("v_bias", &|e| self.encode_broadcast_bias(e, &self.v_buf, v_bias, seq_len));
1611 }
1612 run("rope_q", &|e| {
1613 self.encode_batch_rope(e, &self.q_buf, seq_len, self.num_heads, self.head_dim)
1614 });
1615 run("rope_k", &|e| {
1616 self.encode_batch_rope(e, &self.k_buf, seq_len, self.num_kv_heads, self.head_dim)
1617 });
1618 run("attention", &|e| self.encode_attention(e, seq_len));
1619 {
1620 let mut e = self.device.create_command_encoder(&Default::default());
1621 e.copy_buffer_to_buffer(
1622 &self.attn_out_buf,
1623 0,
1624 &saved.attn_output,
1625 0,
1626 (s * q_dim as usize * 4) as u64,
1627 );
1628 self.queue.submit(Some(e.finish()));
1629 }
1630 run("o_proj", &|e| {
1631 self.encode_matmul(
1632 e,
1633 &self.attn_out_buf,
1634 layer_prefix,
1635 "o_proj",
1636 &self.q_buf,
1637 seq_len,
1638 q_dim,
1639 hd,
1640 )
1641 });
1642 run("residual1", &|e| {
1643 self.encode_residual(e, &self.hidden_buf, &self.q_buf, &self.ffn_out_buf, hd * seq_len)
1644 });
1645 let ffn_norm_w = self
1646 .weight_buffers
1647 .get(&format!("{layer_prefix}.ffn_norm"))
1648 .ok_or_else(|| format!("Missing {layer_prefix}.ffn_norm"))?;
1649 run("rmsnorm2", &|e| {
1650 self.encode_rmsnorm(e, &self.ffn_out_buf, ffn_norm_w, &self.norm_buf, hd)
1651 });
1652 {
1653 let mut e = self.device.create_command_encoder(&Default::default());
1654 e.copy_buffer_to_buffer(
1655 &self.norm_buf,
1656 0,
1657 &saved.ffn_norm_out,
1658 0,
1659 (s * hd as usize * 4) as u64,
1660 );
1661 self.queue.submit(Some(e.finish()));
1662 }
1663 run("gate_proj", &|e| {
1664 self.encode_matmul(
1665 e,
1666 &self.norm_buf,
1667 layer_prefix,
1668 "gate_proj",
1669 &self.ffn_gate_buf,
1670 seq_len,
1671 hd,
1672 inter,
1673 )
1674 });
1675 run("up_proj", &|e| {
1676 self.encode_matmul(
1677 e,
1678 &self.norm_buf,
1679 layer_prefix,
1680 "up_proj",
1681 &self.ffn_up_buf,
1682 seq_len,
1683 hd,
1684 inter,
1685 )
1686 });
1687 run("silu", &|e| {
1688 self.encode_silu_mul(
1689 e,
1690 &self.ffn_gate_buf,
1691 &self.ffn_up_buf,
1692 &self.ffn_silu_buf,
1693 inter * seq_len,
1694 )
1695 });
1696 {
1697 let mut e = self.device.create_command_encoder(&Default::default());
1698 e.copy_buffer_to_buffer(
1699 &self.ffn_silu_buf,
1700 0,
1701 &saved.silu_gate_output,
1702 0,
1703 (s * inter as usize * 4) as u64,
1704 );
1705 self.queue.submit(Some(e.finish()));
1706 }
1707 run("down_proj", &|e| {
1708 self.encode_matmul(
1709 e,
1710 &self.ffn_silu_buf,
1711 layer_prefix,
1712 "down_proj",
1713 &self.norm_buf,
1714 seq_len,
1715 inter,
1716 hd,
1717 )
1718 });
1719 run("residual2", &|e| {
1720 self.encode_residual(
1721 e,
1722 &self.ffn_out_buf,
1723 &self.norm_buf,
1724 &self.hidden_buf,
1725 hd * seq_len,
1726 )
1727 });
1728
1729 let total: u64 = trace.iter().map(|(_, ms)| ms).sum();
1730 let parts: Vec<String> = trace.iter().map(|(n, ms)| format!("{n}={ms}")).collect();
1731 eprintln!("[OP-TRACE] layer {} total={}ms: {}", layer_prefix, total, parts.join(" "));
1732 Ok(())
1733 }
1734
1735 pub fn alloc_layer_activations(&self, seq_len: u32) -> LayerActivations {
1737 let s = seq_len as usize;
1738 let buf = |size: usize, label: &str| -> wgpu::Buffer {
1739 self.device.create_buffer(&wgpu::BufferDescriptor {
1740 label: Some(label),
1741 size: (size * 4) as u64,
1742 usage: wgpu::BufferUsages::STORAGE
1743 | wgpu::BufferUsages::COPY_SRC
1744 | wgpu::BufferUsages::COPY_DST,
1745 mapped_at_creation: false,
1746 })
1747 };
1748 LayerActivations {
1749 attn_norm_out: buf(s * self.hidden_dim as usize, "saved_attn_norm"),
1750 attn_output: buf(s * (self.num_heads * self.head_dim) as usize, "saved_attn_out"),
1751 ffn_norm_out: buf(s * self.hidden_dim as usize, "saved_ffn_norm"),
1752 silu_gate_output: buf(s * self.intermediate_dim as usize, "saved_silu"),
1753 rstd_attn: buf(s, "saved_rstd_attn"),
1754 rstd_ffn: buf(s, "saved_rstd_ffn"),
1755 softmax_logsumexp: buf(self.num_heads as usize * s, "saved_logsumexp"),
1756 }
1757 }
1758
1759 pub fn forward_layer_training(
1761 &self,
1762 seq_len: u32,
1763 layer_prefix: &str,
1764 ) -> Result<LayerActivations, String> {
1765 let saved = self.alloc_layer_activations(seq_len);
1766 let mut encoder = self.device.create_command_encoder(&Default::default());
1767 self.encode_forward_layer_training(&mut encoder, seq_len, layer_prefix, &saved, None)?;
1768 self.queue.submit(Some(encoder.finish()));
1769 Ok(saved)
1770 }
1771
1772 pub fn forward_all_layers_training(
1774 &self,
1775 seq_len: u32,
1776 num_layers: usize,
1777 ) -> Result<Vec<LayerActivations>, String> {
1778 let mut encoder = self.device.create_command_encoder(&Default::default());
1779 let mut all_saved = Vec::with_capacity(num_layers);
1780
1781 for layer_idx in 0..num_layers {
1782 let prefix = format!("layer.{layer_idx}");
1783 let saved = self.alloc_layer_activations(seq_len);
1784 self.encode_forward_layer_training(&mut encoder, seq_len, &prefix, &saved, None)?;
1785 all_saved.push(saved);
1786 }
1787
1788 self.queue.submit(Some(encoder.finish()));
1790 Ok(all_saved)
1791 }
1792 pub fn encode_broadcast_bias(
1800 &self,
1801 encoder: &mut wgpu::CommandEncoder,
1802 buf: &wgpu::Buffer,
1803 bias: &[f32],
1804 seq_len: u32,
1805 ) {
1806 let dim = bias.len();
1807 let mut full_bias = Vec::with_capacity(seq_len as usize * dim);
1809 for _ in 0..seq_len {
1810 full_bias.extend_from_slice(bias);
1811 }
1812 let bias_buf = self.device.create_buffer(&wgpu::BufferDescriptor {
1813 label: Some("broadcast_bias"),
1814 size: (full_bias.len() * 4) as u64,
1815 usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST,
1816 mapped_at_creation: false,
1817 });
1818 self.queue.write_buffer(&bias_buf, 0, bytemuck::cast_slice(&full_bias));
1819
1820 let total = seq_len * dim as u32;
1822 let tmp = self.device.create_buffer(&wgpu::BufferDescriptor {
1823 label: Some("bias_tmp"),
1824 size: (total as usize * 4) as u64,
1825 usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
1826 mapped_at_creation: false,
1827 });
1828 self.encode_residual(encoder, buf, &bias_buf, &tmp, total);
1829 encoder.copy_buffer_to_buffer(&tmp, 0, buf, 0, (total as u64) * 4);
1830 }
1831
1832 fn encode_batch_rope(
1835 &self,
1836 encoder: &mut wgpu::CommandEncoder,
1837 qk_buf: &wgpu::Buffer,
1838 seq_len: u32,
1839 num_heads: u32,
1840 head_dim: u32,
1841 ) {
1842 #[repr(C)]
1843 #[derive(Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)]
1844 struct RopeParams {
1845 seq_len: u32,
1846 num_heads: u32,
1847 head_dim: u32,
1848 _pad: u32,
1849 }
1850 let params = RopeParams { seq_len, num_heads, head_dim, _pad: 0 };
1851 let params_buf = self.device.create_buffer(&wgpu::BufferDescriptor {
1852 label: Some("batch_rope_params"),
1853 size: 16,
1854 usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
1855 mapped_at_creation: false,
1856 });
1857 self.queue.write_buffer(¶ms_buf, 0, bytemuck::bytes_of(¶ms));
1858
1859 let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
1860 label: Some("batch_rope_bg"),
1861 layout: &self.batch_rope_bgl,
1862 entries: &[
1863 wgpu::BindGroupEntry { binding: 0, resource: qk_buf.as_entire_binding() },
1864 wgpu::BindGroupEntry { binding: 1, resource: params_buf.as_entire_binding() },
1865 ],
1866 });
1867 let total = seq_len * num_heads * head_dim;
1868 let wg = total.div_ceil(256);
1869 let mut pass = encoder.begin_compute_pass(&Default::default());
1870 pass.set_pipeline(&self.batch_rope_pipeline);
1871 pass.set_bind_group(0, &bg, &[]);
1872 pass.dispatch_workgroups(wg, 1, 1);
1873 }
1874
1875 fn encode_attention(&self, encoder: &mut wgpu::CommandEncoder, seq_len: u32) {
1876 let params = [seq_len, self.num_heads, self.num_kv_heads, self.head_dim];
1877 let params_buf = self.make_uniform(¶ms);
1878 let _q_dim = self.num_heads * self.head_dim;
1879
1880 let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
1885 label: None,
1886 layout: &self.attention_bgl,
1887 entries: &[
1888 wgpu::BindGroupEntry { binding: 0, resource: self.q_buf.as_entire_binding() },
1889 wgpu::BindGroupEntry { binding: 1, resource: self.k_buf.as_entire_binding() },
1890 wgpu::BindGroupEntry { binding: 2, resource: self.v_buf.as_entire_binding() },
1891 wgpu::BindGroupEntry {
1892 binding: 3,
1893 resource: self.attn_out_buf.as_entire_binding(),
1894 },
1895 wgpu::BindGroupEntry { binding: 4, resource: params_buf.as_entire_binding() },
1896 ],
1897 });
1898 let mut pass = encoder.begin_compute_pass(&Default::default());
1899 pass.set_pipeline(&self.attention_pipeline);
1900 pass.set_bind_group(0, &bg, &[]);
1901 pass.dispatch_workgroups(self.num_heads, seq_len, 1);
1903 }
1904
1905 #[allow(clippy::too_many_arguments)]
1915 fn encode_lora_addmm(
1916 &self,
1917 encoder: &mut wgpu::CommandEncoder,
1918 input: &wgpu::Buffer,
1919 lora_a: &wgpu::Buffer,
1920 lora_b: &wgpu::Buffer,
1921 output: &wgpu::Buffer,
1922 seq_len: u32,
1923 in_dim: u32,
1924 rank: u32,
1925 out_dim: u32,
1926 scale: f32,
1927 _pipeline: &wgpu::ComputePipeline,
1928 _bgl: &wgpu::BindGroupLayout,
1929 ) {
1930 let temp_size = (seq_len * rank) as u64 * 4;
1932 let temp = self.device.create_buffer(&wgpu::BufferDescriptor {
1933 label: Some("lora_temp"),
1934 size: temp_size,
1935 usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
1936 mapped_at_creation: false,
1937 });
1938 self.encode_tiled_gemm(encoder, input, lora_a, &temp, seq_len, in_dim, rank, 1.0);
1939
1940 let delta_size = (seq_len * out_dim) as u64 * 4;
1942 let delta = self.device.create_buffer(&wgpu::BufferDescriptor {
1943 label: Some("lora_delta"),
1944 size: delta_size,
1945 usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
1946 mapped_at_creation: false,
1947 });
1948 self.encode_tiled_gemm(encoder, &temp, lora_b, &delta, seq_len, rank, out_dim, scale);
1949
1950 let sum_buf = self.device.create_buffer(&wgpu::BufferDescriptor {
1952 label: Some("lora_sum"),
1953 size: delta_size,
1954 usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
1955 mapped_at_creation: false,
1956 });
1957 self.encode_residual(encoder, output, &delta, &sum_buf, seq_len * out_dim);
1958 encoder.copy_buffer_to_buffer(&sum_buf, 0, output, 0, delta_size);
1959 }
1960
1961 fn encode_tiled_gemm(
1963 &self,
1964 encoder: &mut wgpu::CommandEncoder,
1965 a: &wgpu::Buffer,
1966 b: &wgpu::Buffer,
1967 c: &wgpu::Buffer,
1968 m: u32,
1969 k: u32,
1970 n: u32,
1971 alpha: f32,
1972 ) {
1973 let params = [m, k, n, alpha.to_bits()];
1974 let params_buf = self.make_uniform(¶ms);
1975 let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
1976 label: None,
1977 layout: &self.matmul_bgl,
1978 entries: &[
1979 wgpu::BindGroupEntry { binding: 0, resource: a.as_entire_binding() },
1980 wgpu::BindGroupEntry { binding: 1, resource: b.as_entire_binding() },
1981 wgpu::BindGroupEntry { binding: 2, resource: c.as_entire_binding() },
1982 wgpu::BindGroupEntry { binding: 3, resource: params_buf.as_entire_binding() },
1983 ],
1984 });
1985 let mut pass = encoder.begin_compute_pass(&Default::default());
1986 pass.set_pipeline(&self.tiled_matmul_pipeline);
1987 pass.set_bind_group(0, &bg, &[]);
1988 pass.dispatch_workgroups(n.div_ceil(64), m.div_ceil(64), 1);
1989 }
1990
1991 fn encode_rmsnorm(
1992 &self,
1993 encoder: &mut wgpu::CommandEncoder,
1994 input: &wgpu::Buffer,
1995 weight: &wgpu::Buffer,
1996 output: &wgpu::Buffer,
1997 dim: u32,
1998 ) {
1999 let params = rmsnorm_params(dim, self.rms_norm_eps);
2000 let params_buf = self.make_uniform(¶ms);
2001 let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
2002 label: None,
2003 layout: &self.elementwise_bgl,
2004 entries: &[
2005 wgpu::BindGroupEntry { binding: 0, resource: input.as_entire_binding() },
2006 wgpu::BindGroupEntry { binding: 1, resource: weight.as_entire_binding() },
2007 wgpu::BindGroupEntry { binding: 2, resource: output.as_entire_binding() },
2008 wgpu::BindGroupEntry { binding: 3, resource: params_buf.as_entire_binding() },
2009 ],
2010 });
2011 let num_rows = (input.size() / (dim as u64 * 4)).max(1) as u32;
2014 let mut pass = encoder.begin_compute_pass(&Default::default());
2015 pass.set_pipeline(&self.rmsnorm_pipeline);
2016 pass.set_bind_group(0, &bg, &[]);
2017 pass.dispatch_workgroups(1, num_rows, 1);
2018 }
2019
2020 fn encode_matmul(
2021 &self,
2022 encoder: &mut wgpu::CommandEncoder,
2023 input: &wgpu::Buffer,
2024 layer_prefix: &str,
2025 proj_name: &str,
2026 output: &wgpu::Buffer,
2027 m: u32,
2028 k: u32,
2029 n: u32,
2030 ) {
2031 if m == 1 && self.encode_q4k_gemv(encoder, input, output, layer_prefix, proj_name, n, k) {
2033 return;
2034 }
2035 let weight_key = format!("{layer_prefix}.{proj_name}");
2036 let weight = match self.weight_buffers.get(&weight_key) {
2037 Some(w) => w,
2038 None => return, };
2040 let params = if m == 1 { [n, k, 0u32, 0u32] } else { [m, k, n, 1.0_f32.to_bits()] };
2045 let params_buf = self.make_uniform(¶ms);
2046 let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
2047 label: None,
2048 layout: &self.matmul_bgl,
2049 entries: &[
2050 wgpu::BindGroupEntry { binding: 0, resource: input.as_entire_binding() },
2051 wgpu::BindGroupEntry { binding: 1, resource: weight.as_entire_binding() },
2052 wgpu::BindGroupEntry { binding: 2, resource: output.as_entire_binding() },
2053 wgpu::BindGroupEntry { binding: 3, resource: params_buf.as_entire_binding() },
2054 ],
2055 });
2056 let mut pass = encoder.begin_compute_pass(&Default::default());
2057 if m == 1 {
2058 pass.set_pipeline(&self.gemv_pipeline);
2060 pass.set_bind_group(0, &bg, &[]);
2061 pass.dispatch_workgroups(n, 1, 1);
2062 } else if m >= 4 {
2063 pass.set_pipeline(&self.tiled_matmul_pipeline);
2066 pass.set_bind_group(0, &bg, &[]);
2067 pass.dispatch_workgroups(n.div_ceil(64), m.div_ceil(64), 1);
2068 } else {
2069 pass.set_pipeline(&self.matmul_pipeline);
2071 pass.set_bind_group(0, &bg, &[]);
2072 pass.dispatch_workgroups(m.div_ceil(16), n.div_ceil(16), 1);
2073 }
2074 }
2075
2076 fn encode_q4k_gemv(
2080 &self,
2081 encoder: &mut wgpu::CommandEncoder,
2082 input: &wgpu::Buffer,
2083 output: &wgpu::Buffer,
2084 layer_prefix: &str,
2085 proj_name: &str,
2086 n: u32,
2087 k: u32,
2088 ) -> bool {
2089 let weight_key = format!("{layer_prefix}.{proj_name}");
2090 let weight = match self.q4k_weights.get(&weight_key) {
2091 Some(w) => w,
2092 None => return false,
2093 };
2094 let num_superblocks = (k + 255) / 256;
2095 let params = [n, k, num_superblocks, 0u32];
2096 let params_buf = self.make_uniform(¶ms);
2097 let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
2098 label: None,
2099 layout: &self.matmul_bgl,
2100 entries: &[
2101 wgpu::BindGroupEntry { binding: 0, resource: input.as_entire_binding() },
2102 wgpu::BindGroupEntry { binding: 1, resource: weight.as_entire_binding() },
2103 wgpu::BindGroupEntry { binding: 2, resource: output.as_entire_binding() },
2104 wgpu::BindGroupEntry { binding: 3, resource: params_buf.as_entire_binding() },
2105 ],
2106 });
2107 let mut pass = encoder.begin_compute_pass(&Default::default());
2108 pass.set_pipeline(&self.q4k_gemv_pipeline);
2109 pass.set_bind_group(0, &bg, &[]);
2110 pass.dispatch_workgroups(n, 1, 1);
2111 true
2112 }
2113
2114 fn encode_silu_mul(
2115 &self,
2116 encoder: &mut wgpu::CommandEncoder,
2117 gate: &wgpu::Buffer,
2118 up: &wgpu::Buffer,
2119 output: &wgpu::Buffer,
2120 dim: u32,
2121 ) {
2122 let params = [dim, 0u32, 0, 0];
2123 let params_buf = self.make_uniform(¶ms);
2124 let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
2125 label: None,
2126 layout: &self.elementwise_bgl,
2127 entries: &[
2128 wgpu::BindGroupEntry { binding: 0, resource: gate.as_entire_binding() },
2129 wgpu::BindGroupEntry { binding: 1, resource: up.as_entire_binding() },
2130 wgpu::BindGroupEntry { binding: 2, resource: output.as_entire_binding() },
2131 wgpu::BindGroupEntry { binding: 3, resource: params_buf.as_entire_binding() },
2132 ],
2133 });
2134 let mut pass = encoder.begin_compute_pass(&Default::default());
2135 pass.set_pipeline(&self.silu_mul_pipeline);
2136 pass.set_bind_group(0, &bg, &[]);
2137 pass.dispatch_workgroups(dim.div_ceil(256), 1, 1);
2138 }
2139
2140 fn encode_residual(
2141 &self,
2142 encoder: &mut wgpu::CommandEncoder,
2143 a: &wgpu::Buffer,
2144 b: &wgpu::Buffer,
2145 output: &wgpu::Buffer,
2146 dim: u32,
2147 ) {
2148 let params = [dim, 0u32, 0, 0];
2149 let params_buf = self.make_uniform(¶ms);
2150 let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
2151 label: None,
2152 layout: &self.elementwise_bgl,
2153 entries: &[
2154 wgpu::BindGroupEntry { binding: 0, resource: a.as_entire_binding() },
2155 wgpu::BindGroupEntry { binding: 1, resource: b.as_entire_binding() },
2156 wgpu::BindGroupEntry { binding: 2, resource: output.as_entire_binding() },
2157 wgpu::BindGroupEntry { binding: 3, resource: params_buf.as_entire_binding() },
2158 ],
2159 });
2160 let mut pass = encoder.begin_compute_pass(&Default::default());
2161 pass.set_pipeline(&self.residual_pipeline);
2162 pass.set_bind_group(0, &bg, &[]);
2163 pass.dispatch_workgroups(dim.div_ceil(256), 1, 1);
2164 }
2165
2166 fn make_uniform(&self, data: &[u32; 4]) -> wgpu::Buffer {
2167 use wgpu::util::DeviceExt;
2168 self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
2169 label: None,
2170 contents: bytemuck::cast_slice(data),
2171 usage: wgpu::BufferUsages::UNIFORM,
2172 })
2173 }
2174}
2175
2176fn bgl_storage(binding: u32, read_only: bool) -> wgpu::BindGroupLayoutEntry {
2177 wgpu::BindGroupLayoutEntry {
2178 binding,
2179 visibility: wgpu::ShaderStages::COMPUTE,
2180 ty: wgpu::BindingType::Buffer {
2181 ty: wgpu::BufferBindingType::Storage { read_only },
2182 has_dynamic_offset: false,
2183 min_binding_size: None,
2184 },
2185 count: None,
2186 }
2187}
2188
2189fn bgl_uniform(binding: u32) -> wgpu::BindGroupLayoutEntry {
2190 wgpu::BindGroupLayoutEntry {
2191 binding,
2192 visibility: wgpu::ShaderStages::COMPUTE,
2193 ty: wgpu::BindingType::Buffer {
2194 ty: wgpu::BufferBindingType::Uniform,
2195 has_dynamic_offset: false,
2196 min_binding_size: None,
2197 },
2198 count: None,
2199 }
2200}
2201
2202#[cfg(test)]
2203mod rmsnorm_eps_tests {
2204 use super::{rmsnorm_params, DEFAULT_RMS_NORM_EPS};
2205
2206 #[test]
2209 fn rmsnorm_params_carry_eps_bits() {
2210 let p = rmsnorm_params(896, 1e-5);
2211 assert_eq!(p[0], 896);
2212 assert_eq!(f32::from_bits(p[1]), 1e-5);
2213 assert_ne!(f32::from_bits(p[1]), DEFAULT_RMS_NORM_EPS);
2214 assert_eq!(f32::from_bits(rmsnorm_params(1, DEFAULT_RMS_NORM_EPS)[1]), 1e-6);
2215 }
2216
2217 #[test]
2220 fn rmsnorm_shader_validates_on_device() {
2221 let Ok(gpu) = crate::backends::gpu::GpuDevice::new() else {
2222 return;
2223 };
2224 gpu.device.push_error_scope(wgpu::ErrorFilter::Validation);
2225 let _module = gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor {
2226 label: Some("rmsnorm_eps_test"),
2227 source: wgpu::ShaderSource::Wgsl(super::RMSNORM_SHADER.into()),
2228 });
2229 let err = pollster::block_on(gpu.device.pop_error_scope());
2230 assert!(err.is_none(), "RMSNORM_SHADER failed validation: {err:?}");
2231 }
2232}
2233
2234#[cfg(test)]
2235mod rope_theta_tests {
2236 use super::{rope_rotate_heads, DEFAULT_ROPE_THETA};
2237
2238 fn rotated(theta: f64, position: usize) -> Vec<f32> {
2239 let mut v = vec![1.0, 0.5, 0.25, 2.0, -1.0, 0.75, 0.1, 0.9];
2240 rope_rotate_heads(&mut v, 1, 8, position, theta);
2241 v
2242 }
2243
2244 #[test]
2247 fn rope_follows_the_configured_theta() {
2248 assert_eq!(rotated(1e4, 0), rotated(1e6, 0));
2249 assert_ne!(rotated(1e4, 5), rotated(1e6, 5));
2250 assert_eq!(rotated(DEFAULT_ROPE_THETA, 5), rotated(1e6, 5));
2251 }
2252
2253 #[test]
2255 fn rope_matches_the_closed_form() {
2256 let mut v = vec![1.0, 0.0, 0.0, 0.0];
2257 rope_rotate_heads(&mut v, 1, 4, 3, 1e4);
2258 let a = 3.0f64; assert!((f64::from(v[0]) - a.cos()).abs() < 1e-6);
2260 assert!((f64::from(v[2]) - a.sin()).abs() < 1e-6);
2261 }
2262}