Skip to main content

trueno/backends/gpu/device/linalg/
wgsl_forward.rs

1#![allow(dead_code, clippy::many_single_char_names)]
2//! PMAT-324: WGSL transformer forward pass — multi-pass single submission.
3//!
4//! Instead of one matmul per CPU call (2ms roundtrip each), this encodes
5//! ALL operations for one transformer layer into a single command encoder.
6//! Only one submit + one readback per layer (or per full forward pass).
7//!
8//! Architecture: separate WGSL kernels per operation type, dispatched
9//! sequentially within one command encoder. All intermediate data stays
10//! GPU-resident in persistent buffers.
11
12use std::collections::HashMap;
13
14/// Saved activations for one transformer layer's backward pass.
15///
16/// Contains the 7 tensors needed for LoRA gradient computation
17/// without replaying the forward pass (§26.11.5, falsification-verified).
18pub struct LayerActivations {
19    /// Input to Q/K/V projections (RMSNorm output). [seq, hidden]
20    pub attn_norm_out: wgpu::Buffer,
21    /// Input to O projection (attention output). [seq, q_dim]
22    pub attn_output: wgpu::Buffer,
23    /// Input to gate/up/down projections (FFN RMSNorm output). [seq, hidden]
24    pub ffn_norm_out: wgpu::Buffer,
25    /// Input to down projection (SiLU(gate)×up). [seq, intermediate]
26    pub silu_gate_output: wgpu::Buffer,
27    /// RMSNorm reciprocal std for attention norm. [seq]
28    pub rstd_attn: wgpu::Buffer,
29    /// RMSNorm reciprocal std for FFN norm. [seq]
30    pub rstd_ffn: wgpu::Buffer,
31    /// Softmax logsumexp for attention backward. [num_heads, seq]
32    pub softmax_logsumexp: wgpu::Buffer,
33}
34
35/// Optional LoRA buffers for Q/K/V projections in a layer's forward pass.
36pub struct QkvLoRA<'a> {
37    pub q_a: &'a wgpu::Buffer,
38    pub q_b: &'a wgpu::Buffer,
39    pub k_a: &'a wgpu::Buffer,
40    pub k_b: &'a wgpu::Buffer,
41    pub v_a: &'a wgpu::Buffer,
42    pub v_b: &'a wgpu::Buffer,
43    pub rank: u32,
44    pub scale: f32,
45    pub in_dim: u32,
46    pub q_dim: u32,
47    pub kv_dim: u32,
48    pub lora_pipeline: &'a wgpu::ComputePipeline,
49    pub lora_bgl: &'a wgpu::BindGroupLayout,
50}
51
52/// GPU-resident transformer layer state.
53/// All buffers persist across tokens — only input/output change per step.
54pub struct WgslForwardPass {
55    device: wgpu::Device,
56    queue: wgpu::Queue,
57
58    // Kernels (compiled once)
59    matmul_pipeline: wgpu::ComputePipeline,
60    /// CUTLASS-style tiled GEMM for M>=4 (training batch, prefill)
61    tiled_matmul_pipeline: wgpu::ComputePipeline,
62    /// PMAT-327: GEMV pipeline for M=1 decode (cooperative K-reduction)
63    gemv_pipeline: wgpu::ComputePipeline,
64    /// C-WGPU-Q4K-001: Q4K GEMV pipeline — dequantize-on-the-fly, no F32 weights
65    q4k_gemv_pipeline: wgpu::ComputePipeline,
66    /// Causal attention pipeline for training (full sequence, no KV cache)
67    attention_pipeline: wgpu::ComputePipeline,
68    attention_bgl: wgpu::BindGroupLayout,
69    rmsnorm_pipeline: wgpu::ComputePipeline,
70    silu_mul_pipeline: wgpu::ComputePipeline,
71    rope_pipeline: wgpu::ComputePipeline,
72    batch_rope_pipeline: wgpu::ComputePipeline,
73    batch_rope_bgl: wgpu::BindGroupLayout,
74    residual_pipeline: wgpu::ComputePipeline,
75
76    // Bind group layouts
77    matmul_bgl: wgpu::BindGroupLayout,
78    elementwise_bgl: wgpu::BindGroupLayout,
79
80    // Weight buffers (persistent, uploaded once)
81    weight_buffers: HashMap<String, wgpu::Buffer>,
82    /// GH-560: Raw Q4K weight buffers for fused dequant+GEMV.
83    q4k_weights: HashMap<String, wgpu::Buffer>,
84    /// PMAT-342: CPU-side bias data (small, not worth GPU dispatch)
85    cpu_biases: HashMap<String, Vec<f32>>,
86    /// GH-560: Per-layer GPU KV cache buffers.
87    kv_cache_k: Vec<wgpu::Buffer>,
88    /// GH-560: Per-layer GPU KV cache buffers (values).
89    kv_cache_v: Vec<wgpu::Buffer>,
90
91    // Intermediate buffers (persistent, reused across calls)
92    // For 1.5B: hidden=1536, kv=256, intermediate=8960
93    hidden_buf: wgpu::Buffer,   // [hidden_dim] working state
94    q_buf: wgpu::Buffer,        // [q_dim]
95    k_buf: wgpu::Buffer,        // [kv_dim]
96    v_buf: wgpu::Buffer,        // [kv_dim]
97    attn_out_buf: wgpu::Buffer, // [hidden_dim]
98    ffn_gate_buf: wgpu::Buffer, // [intermediate_dim]
99    ffn_up_buf: wgpu::Buffer,   // [intermediate_dim]
100    ffn_silu_buf: wgpu::Buffer, // [intermediate_dim] — SiLU(gate)×up output (can't alias inputs)
101    ffn_out_buf: wgpu::Buffer,  // [hidden_dim]
102    norm_buf: wgpu::Buffer,     // [hidden_dim] for RMSNorm output
103    staging_buf: wgpu::Buffer,  // readback
104
105    // Config
106    hidden_dim: u32,
107    num_heads: u32,
108    num_kv_heads: u32,
109    head_dim: u32,
110    intermediate_dim: u32,
111    /// RMSNorm epsilon, passed to the shader as `params.y` (f32 bits). Defaults to
112    /// [`DEFAULT_RMS_NORM_EPS`]; callers set the model's value with
113    /// [`WgslForwardPass::set_rms_norm_eps`] (#4056: Llama-family models use 1e-5).
114    rms_norm_eps: f32,
115    /// RoPE base for the CPU-side rotation; set with [`WgslForwardPass::set_rope_theta`] (#5503).
116    rope_theta: f64,
117}
118
119/// Default RMSNorm epsilon: the constant the shader hardcoded before #4056.
120pub const DEFAULT_RMS_NORM_EPS: f32 = 1e-6;
121
122/// RoPE base used until the model's configured `rope_theta` is set (Qwen2: 1e6; Llama: 1e4).
123pub const DEFAULT_ROPE_THETA: f64 = 1_000_000.0;
124
125/// Rotate `heads` consecutive `head_dim` rows of `data` (NeoX half-split) for `position`.
126fn rope_rotate_heads(
127    data: &mut [f32],
128    heads: usize,
129    head_dim: usize,
130    position: usize,
131    rope_theta: f64,
132) {
133    let half = head_dim / 2;
134    for h in 0..heads {
135        let offset = h * head_dim;
136        for i in 0..half {
137            let theta = rope_theta.powf(-((2 * i) as f64) / head_dim as f64);
138            let angle = position as f64 * theta;
139            let (sin_a, cos_a) = (angle.sin() as f32, angle.cos() as f32);
140            let x0 = data[offset + i];
141            let x1 = data[offset + i + half];
142            data[offset + i] = x0 * cos_a - x1 * sin_a;
143            data[offset + i + half] = x0 * sin_a + x1 * cos_a;
144        }
145    }
146}
147
148/// Uniform params for the RMSNorm shader: `(dim, eps as f32 bits, 0, 0)`.
149fn rmsnorm_params(dim: u32, eps: f32) -> [u32; 4] {
150    [dim, eps.to_bits(), 0, 0]
151}
152
153// WGSL shader source for RMSNorm (multi-row via workgroup_id.y)
154// Dispatch: (1, seq_len, 1) — one workgroup per row.
155const RMSNORM_SHADER: &str = r#"
156@group(0) @binding(0) var<storage, read> input: array<f32>;
157@group(0) @binding(1) var<storage, read> weight: array<f32>;
158@group(0) @binding(2) var<storage, read_write> output: array<f32>;
159@group(0) @binding(3) var<uniform> params: vec4<u32>; // (dim, bitcast<u32>(eps), 0, 0)
160
161var<workgroup> shared_sum: array<f32, 256>;
162
163@compute @workgroup_size(256)
164fn main(@builtin(local_invocation_id) lid: vec3<u32>,
165        @builtin(workgroup_id) wg_id: vec3<u32>) {
166    let dim = params.x;
167    let row = wg_id.y;
168    let base = row * dim;
169    let tid = lid.x;
170
171    // Compute sum of squares (reduction) for this row
172    var local_sum: f32 = 0.0;
173    var i = tid;
174    while (i < dim) {
175        let val = input[base + i];
176        local_sum += val * val;
177        i += 256u;
178    }
179    shared_sum[tid] = local_sum;
180    workgroupBarrier();
181
182    // Tree reduction
183    var stride = 128u;
184    while (stride > 0u) {
185        if (tid < stride) {
186            shared_sum[tid] += shared_sum[tid + stride];
187        }
188        workgroupBarrier();
189        stride >>= 1u;
190    }
191
192    let eps = bitcast<f32>(params.y);
193    let rms = sqrt(shared_sum[0] / f32(dim) + eps);
194
195    // Normalize and scale
196    i = tid;
197    while (i < dim) {
198        output[base + i] = (input[base + i] / rms) * weight[i];
199        i += 256u;
200    }
201}
202"#;
203
204// WGSL shader for SiLU(gate) * up
205const SILU_MUL_SHADER: &str = r#"
206@group(0) @binding(0) var<storage, read> gate: array<f32>;
207@group(0) @binding(1) var<storage, read> up: array<f32>;
208@group(0) @binding(2) var<storage, read_write> output: array<f32>;
209@group(0) @binding(3) var<uniform> params: vec4<u32>; // (dim, 0, 0, 0)
210
211@compute @workgroup_size(256)
212fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
213    let idx = gid.x;
214    if (idx >= params.x) { return; }
215    let g = gate[idx];
216    let silu_g = g / (1.0 + exp(-g));
217    output[idx] = silu_g * up[idx];
218}
219"#;
220
221// WGSL shader for residual add: output = a + b
222const RESIDUAL_SHADER: &str = r#"
223@group(0) @binding(0) var<storage, read> a: array<f32>;
224@group(0) @binding(1) var<storage, read> b: array<f32>;
225@group(0) @binding(2) var<storage, read_write> output: array<f32>;
226@group(0) @binding(3) var<uniform> params: vec4<u32>; // (dim, 0, 0, 0)
227
228@compute @workgroup_size(256)
229fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
230    let idx = gid.x;
231    if (idx >= params.x) { return; }
232    output[idx] = a[idx] + b[idx];
233}
234"#;
235
236// Batch RoPE shader — applies RoPE to all positions in a sequence at once.
237// PMAT-509: Training forward path was missing RoPE entirely, causing loss > random.
238// Input: qk[seq_len * num_heads * head_dim], applies position-dependent rotation.
239const BATCH_ROPE_SHADER: &str = r#"
240@group(0) @binding(0) var<storage, read_write> qk: array<f32>;
241
242struct RopeParams {
243    seq_len: u32,
244    num_heads: u32,
245    head_dim: u32,
246    _pad: u32,
247}
248
249@group(0) @binding(1) var<uniform> params: RopeParams;
250
251@compute @workgroup_size(256)
252fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
253    let idx = gid.x;
254    let total = params.seq_len * params.num_heads * params.head_dim;
255    if (idx >= total) { return; }
256
257    let head_dim = params.head_dim;
258    let half_hd = head_dim / 2u;
259
260    // Decompose idx into (position, head, pos_in_head)
261    let elements_per_pos = params.num_heads * head_dim;
262    let position = idx / elements_per_pos;
263    let within_pos = idx % elements_per_pos;
264    let head_idx = within_pos / head_dim;
265    let pos_in_head = within_pos % head_dim;
266
267    // Only process the first half of each head (pairs with second half)
268    if (pos_in_head >= half_hd) { return; }
269
270    let theta = pow(1000000.0, -f32(pos_in_head * 2u) / f32(head_dim));
271    let angle = f32(position) * theta;
272    let cos_a = cos(angle);
273    let sin_a = sin(angle);
274
275    let base = position * elements_per_pos + head_idx * head_dim;
276    let i0 = base + pos_in_head;
277    let i1 = i0 + half_hd;
278
279    let x0 = qk[i0];
280    let x1 = qk[i1];
281    qk[i0] = x0 * cos_a - x1 * sin_a;
282    qk[i1] = x0 * sin_a + x1 * cos_a;
283}
284"#;
285
286// RoPE shader (NeoX-style interleaved) — single position (inference)
287const ROPE_SHADER: &str = r#"
288@group(0) @binding(0) var<storage, read_write> qk: array<f32>;
289@group(0) @binding(1) var<uniform> params: vec4<u32>; // (dim, position, num_heads, head_dim)
290
291@compute @workgroup_size(256)
292fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
293    let idx = gid.x;
294    let dim = params.x;
295    let position = params.y;
296    let head_dim = params.w;
297
298    if (idx >= dim) { return; }
299
300    let half_hd = head_dim / 2u;
301    let head_idx = idx / head_dim;
302    let pos_in_head = idx % head_dim;
303
304    if (pos_in_head >= half_hd) { return; }
305
306    let theta = pow(1000000.0, -f32(pos_in_head * 2u) / f32(head_dim));
307    let angle = f32(position) * theta;
308    let cos_a = cos(angle);
309    let sin_a = sin(angle);
310
311    let i0 = head_idx * head_dim + pos_in_head;
312    let i1 = i0 + half_hd;
313
314    let x0 = qk[i0];
315    let x1 = qk[i1];
316    qk[i0] = x0 * cos_a - x1 * sin_a;
317    qk[i1] = x0 * sin_a + x1 * cos_a;
318}
319"#;
320
321impl WgslForwardPass {
322    /// Get the shader sources for external inspection/testing
323    pub fn rmsnorm_shader() -> &'static str {
324        RMSNORM_SHADER
325    }
326    pub fn silu_mul_shader() -> &'static str {
327        SILU_MUL_SHADER
328    }
329    pub fn residual_shader() -> &'static str {
330        RESIDUAL_SHADER
331    }
332    pub fn rope_shader() -> &'static str {
333        ROPE_SHADER
334    }
335
336    /// PMAT-325: Create a new WGSL forward pass context.
337    ///
338    /// Compiles all shader pipelines and allocates persistent intermediate buffers.
339    /// Call once at model init. All GPU resources persist until dropped.
340    pub fn new(
341        device: wgpu::Device,
342        queue: wgpu::Queue,
343        hidden_dim: usize,
344        num_heads: usize,
345        num_kv_heads: usize,
346        head_dim: usize,
347        intermediate_dim: usize,
348    ) -> Self {
349        let q_dim = num_heads * head_dim;
350        let kv_dim = num_kv_heads * head_dim;
351
352        // Compile shaders
353        let matmul_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
354            label: Some("matmul"),
355            source: wgpu::ShaderSource::Wgsl(crate::backends::gpu::shaders::MATMUL_SHADER.into()),
356        });
357        let rmsnorm_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
358            label: Some("rmsnorm"),
359            source: wgpu::ShaderSource::Wgsl(RMSNORM_SHADER.into()),
360        });
361        let silu_mul_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
362            label: Some("silu_mul"),
363            source: wgpu::ShaderSource::Wgsl(SILU_MUL_SHADER.into()),
364        });
365        let rope_shader_mod = device.create_shader_module(wgpu::ShaderModuleDescriptor {
366            label: Some("rope"),
367            source: wgpu::ShaderSource::Wgsl(ROPE_SHADER.into()),
368        });
369        let residual_shader_mod = device.create_shader_module(wgpu::ShaderModuleDescriptor {
370            label: Some("residual"),
371            source: wgpu::ShaderSource::Wgsl(RESIDUAL_SHADER.into()),
372        });
373
374        // Bind group layouts
375        let matmul_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
376            label: Some("matmul_bgl"),
377            entries: &[
378                bgl_storage(0, true),
379                bgl_storage(1, true),
380                bgl_storage(2, false),
381                bgl_uniform(3),
382            ],
383        });
384        let elementwise_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
385            label: Some("ew_bgl"),
386            entries: &[
387                bgl_storage(0, true),
388                bgl_storage(1, true),
389                bgl_storage(2, false),
390                bgl_uniform(3),
391            ],
392        });
393
394        // Pipelines
395        let make_pipeline =
396            |shader: &wgpu::ShaderModule, bgl: &wgpu::BindGroupLayout, label: &str| {
397                let pl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
398                    label: Some(label),
399                    bind_group_layouts: &[bgl],
400                    push_constant_ranges: &[],
401                });
402                device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
403                    label: Some(label),
404                    layout: Some(&pl),
405                    module: shader,
406                    entry_point: Some("main"),
407                    compilation_options: Default::default(),
408                    cache: None,
409                })
410            };
411
412        let matmul_pipeline = make_pipeline(&matmul_shader, &matmul_bgl, "matmul_pipe");
413
414        // CUTLASS-style tiled GEMM for M>=4 (training batch, prefill)
415        let tiled_matmul_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
416            label: Some("tiled_matmul"),
417            source: wgpu::ShaderSource::Wgsl(
418                crate::backends::gpu::shaders::TILED_GEMM_SHADER.into(),
419            ),
420        });
421        let tiled_matmul_pipeline =
422            make_pipeline(&tiled_matmul_shader, &matmul_bgl, "tiled_matmul_pipe");
423
424        // Causal attention pipeline for training
425        let attention_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
426            label: Some("causal_attention"),
427            source: wgpu::ShaderSource::Wgsl(
428                crate::backends::gpu::shaders::CAUSAL_ATTENTION_SHADER.into(),
429            ),
430        });
431        let attention_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
432            label: Some("attn_bgl"),
433            entries: &[
434                bgl_storage(0, true),  // Q
435                bgl_storage(1, true),  // K
436                bgl_storage(2, true),  // V
437                bgl_storage(3, false), // output
438                bgl_uniform(4),        // params
439            ],
440        });
441        let attention_pl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
442            label: Some("attn_pl"),
443            bind_group_layouts: &[&attention_bgl],
444            push_constant_ranges: &[],
445        });
446        let attention_pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
447            label: Some("attn_pipe"),
448            layout: Some(&attention_pl),
449            module: &attention_shader,
450            entry_point: Some("main"),
451            compilation_options: Default::default(),
452            cache: None,
453        });
454
455        // PMAT-327: GEMV pipeline — same bind group layout as matmul but cooperative reduction
456        let gemv_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
457            label: Some("gemv"),
458            source: wgpu::ShaderSource::Wgsl(crate::backends::gpu::shaders::GEMV_SHADER.into()),
459        });
460        let gemv_pipeline = make_pipeline(&gemv_shader, &matmul_bgl, "gemv_pipe");
461
462        // C-WGPU-Q4K-001: Q4K GEMV — dequantize on-the-fly, no F32 weight buffer
463        let q4k_gemv_shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
464            label: Some("q4k_gemv"),
465            source: wgpu::ShaderSource::Wgsl(crate::backends::gpu::shaders::Q4K_GEMV_SHADER.into()),
466        });
467        let q4k_gemv_pipeline = make_pipeline(&q4k_gemv_shader, &matmul_bgl, "q4k_gemv_pipe");
468
469        let rmsnorm_pipeline = make_pipeline(&rmsnorm_shader, &elementwise_bgl, "rmsnorm_pipe");
470        let silu_mul_pipeline = make_pipeline(&silu_mul_shader, &elementwise_bgl, "silu_pipe");
471        let residual_pipeline = make_pipeline(&residual_shader_mod, &elementwise_bgl, "res_pipe");
472
473        // RoPE has a 2-binding layout (in-place + uniform)
474        let rope_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
475            label: Some("rope_bgl"),
476            entries: &[bgl_storage(0, false), bgl_uniform(1)],
477        });
478        let rope_pipeline = {
479            let pl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
480                label: Some("rope_pl"),
481                bind_group_layouts: &[&rope_bgl],
482                push_constant_ranges: &[],
483            });
484            device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
485                label: Some("rope_pipe"),
486                layout: Some(&pl),
487                module: &rope_shader_mod,
488                entry_point: Some("main"),
489                compilation_options: Default::default(),
490                cache: None,
491            })
492        };
493
494        // PMAT-509: Batch RoPE for training (all positions at once)
495        let batch_rope_shader_mod = device.create_shader_module(wgpu::ShaderModuleDescriptor {
496            label: Some("batch_rope"),
497            source: wgpu::ShaderSource::Wgsl(BATCH_ROPE_SHADER.into()),
498        });
499        let batch_rope_bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
500            label: Some("batch_rope_bgl"),
501            entries: &[bgl_storage(0, false), bgl_uniform(1)],
502        });
503        let batch_rope_pipeline = {
504            let pl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
505                label: Some("batch_rope_pl"),
506                bind_group_layouts: &[&batch_rope_bgl],
507                push_constant_ranges: &[],
508            });
509            device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
510                label: Some("batch_rope_pipe"),
511                layout: Some(&pl),
512                module: &batch_rope_shader_mod,
513                entry_point: Some("main"),
514                compilation_options: Default::default(),
515                cache: None,
516            })
517        };
518
519        // Allocate persistent intermediate buffers
520        let buf = |size: usize, label: &str| -> wgpu::Buffer {
521            device.create_buffer(&wgpu::BufferDescriptor {
522                label: Some(label),
523                size: (size * 4) as u64,
524                usage: wgpu::BufferUsages::STORAGE
525                    | wgpu::BufferUsages::COPY_SRC
526                    | wgpu::BufferUsages::COPY_DST,
527                mapped_at_creation: false,
528            })
529        };
530
531        // Buffer sizes: max_seq × dim for training, or 1 × dim for inference.
532        // Training calls forward_layer_training with seq_len > 1.
533        // Allocate for max_seq=2048 to support both.
534        let max_seq = 2048;
535        let hidden_buf = buf(max_seq * hidden_dim, "hidden");
536        let q_buf = buf(max_seq * q_dim, "q");
537        let k_buf = buf(max_seq * kv_dim, "k");
538        let v_buf = buf(max_seq * kv_dim, "v");
539        let attn_out_buf = buf(max_seq * hidden_dim, "attn_out");
540        let ffn_gate_buf = buf(max_seq * intermediate_dim, "ffn_gate");
541        let ffn_up_buf = buf(max_seq * intermediate_dim, "ffn_up");
542        let ffn_silu_buf = buf(max_seq * intermediate_dim, "ffn_silu");
543        let ffn_out_buf = buf(max_seq * hidden_dim, "ffn_out");
544        let norm_buf = buf(max_seq * hidden_dim, "norm");
545
546        let max_out = max_seq * hidden_dim.max(intermediate_dim);
547        let staging_buf = device.create_buffer(&wgpu::BufferDescriptor {
548            label: Some("staging"),
549            size: (max_out * 4) as u64,
550            usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
551            mapped_at_creation: false,
552        });
553
554        Self {
555            device,
556            queue,
557            matmul_pipeline,
558            tiled_matmul_pipeline,
559            attention_pipeline,
560            attention_bgl,
561            gemv_pipeline,
562            q4k_gemv_pipeline,
563            rmsnorm_pipeline,
564            silu_mul_pipeline,
565            rope_pipeline,
566            batch_rope_pipeline,
567            batch_rope_bgl,
568            residual_pipeline,
569            matmul_bgl,
570            elementwise_bgl,
571            weight_buffers: HashMap::new(),
572            q4k_weights: HashMap::new(),
573            kv_cache_k: Vec::new(),
574            kv_cache_v: Vec::new(),
575            cpu_biases: HashMap::new(),
576            hidden_buf,
577            q_buf,
578            k_buf,
579            v_buf,
580            attn_out_buf,
581            ffn_gate_buf,
582            ffn_up_buf,
583            ffn_silu_buf,
584            ffn_out_buf,
585            norm_buf,
586            staging_buf,
587            hidden_dim: hidden_dim as u32,
588            num_heads: num_heads as u32,
589            num_kv_heads: num_kv_heads as u32,
590            head_dim: head_dim as u32,
591            intermediate_dim: intermediate_dim as u32,
592            rms_norm_eps: DEFAULT_RMS_NORM_EPS,
593            rope_theta: DEFAULT_ROPE_THETA,
594        }
595    }
596
597    /// Set the RMSNorm epsilon every norm in this forward pass uses — pass the
598    /// model's configured `rms_norm_eps`. Default: [`DEFAULT_RMS_NORM_EPS`].
599    pub fn set_rms_norm_eps(&mut self, eps: f32) {
600        self.rms_norm_eps = eps;
601    }
602
603    /// Set the RoPE base the position>0 rotation uses — pass the model's configured
604    /// `rope_theta`. Default: [`DEFAULT_ROPE_THETA`].
605    pub fn set_rope_theta(&mut self, theta: f64) {
606        self.rope_theta = theta;
607    }
608
609    /// The RMSNorm epsilon this forward pass uses.
610    #[must_use]
611    pub fn rms_norm_eps(&self) -> f32 {
612        self.rms_norm_eps
613    }
614
615    /// Upload a weight matrix (call once per layer at init).
616    /// PMAT-342: Bias weights (name contains "bias") are stored CPU-side.
617    pub fn upload_weight(&mut self, name: &str, data: &[f32]) {
618        if name.contains("bias") {
619            // Biases are small, keep on CPU for easy access in attention
620            self.cpu_biases.insert(name.to_string(), data.to_vec());
621            return;
622        }
623        // Skip weights that exceed the device's max buffer binding size (e.g., lm_head > 2 GB)
624        let size_bytes = (data.len() * 4) as u64;
625        let max_binding = self.device.limits().max_storage_buffer_binding_size as u64;
626        if size_bytes > max_binding {
627            eprintln!(
628                "[wgpu] Skipping weight '{}' ({:.1} MB > {:.1} MB limit) — CPU fallback",
629                name,
630                size_bytes as f64 / 1e6,
631                max_binding as f64 / 1e6
632            );
633            return;
634        }
635        use wgpu::util::DeviceExt;
636        let buffer = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
637            label: Some(name),
638            contents: bytemuck::cast_slice(data),
639            usage: wgpu::BufferUsages::STORAGE,
640        });
641        self.weight_buffers.insert(name.to_string(), buffer);
642    }
643
644    /// GH-560: Upload raw Q4K weight bytes for fused dequant+GEMV on GPU.
645    pub fn upload_q4k_weight(&mut self, name: &str, data: &[u8]) {
646        use wgpu::util::DeviceExt;
647        let buffer = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
648            label: Some(name),
649            contents: data,
650            usage: wgpu::BufferUsages::STORAGE,
651        });
652        self.q4k_weights.insert(name.to_string(), buffer);
653    }
654
655    /// GH-560: Initialize per-layer KV cache buffers on GPU.
656    pub fn init_kv_cache(&mut self, num_layers: usize) {
657        let kv_dim = (self.num_kv_heads * self.head_dim) as u64;
658        let max_seq = 2048u64;
659        for _ in 0..num_layers {
660            let k = self.device.create_buffer(&wgpu::BufferDescriptor {
661                label: Some("kv_cache_k"),
662                size: max_seq * kv_dim * 4,
663                usage: wgpu::BufferUsages::STORAGE
664                    | wgpu::BufferUsages::COPY_DST
665                    | wgpu::BufferUsages::COPY_SRC,
666                mapped_at_creation: false,
667            });
668            let v = self.device.create_buffer(&wgpu::BufferDescriptor {
669                label: Some("kv_cache_v"),
670                size: max_seq * kv_dim * 4,
671                usage: wgpu::BufferUsages::STORAGE
672                    | wgpu::BufferUsages::COPY_DST
673                    | wgpu::BufferUsages::COPY_SRC,
674                mapped_at_creation: false,
675            });
676            self.kv_cache_k.push(k);
677            self.kv_cache_v.push(v);
678        }
679    }
680
681    /// Number of uploaded weight buffers.
682    pub fn weight_count(&self) -> usize {
683        self.weight_buffers.len()
684    }
685
686    /// Access a dequantized weight buffer by name (e.g. "layer.0.down_proj").
687    /// Used by backward pass for gradient propagation through frozen base weights.
688    pub fn weight_buffer(&self, name: &str) -> Option<&wgpu::Buffer> {
689        self.weight_buffers.get(name)
690    }
691
692    /// Reference to the wgpu device.
693    pub fn device_ref(&self) -> &wgpu::Device {
694        &self.device
695    }
696
697    /// Reference to the wgpu queue.
698    pub fn queue_ref(&self) -> &wgpu::Queue {
699        &self.queue
700    }
701
702    /// Reference to the hidden state buffer (for writing input).
703    pub fn hidden_buffer(&self) -> &wgpu::Buffer {
704        &self.hidden_buf
705    }
706
707    /// Reference to Q buffer (for LoRA addmm after Q projection).
708    pub fn q_buffer(&self) -> &wgpu::Buffer {
709        &self.q_buf
710    }
711
712    /// Reference to K buffer.
713    pub fn k_buffer(&self) -> &wgpu::Buffer {
714        &self.k_buf
715    }
716
717    /// Reference to V buffer.
718    pub fn v_buffer(&self) -> &wgpu::Buffer {
719        &self.v_buf
720    }
721
722    /// Elementwise add: output = a + b. Dispatches residual add shader.
723    pub fn gpu_residual_add(
724        &self,
725        a: &wgpu::Buffer,
726        b: &wgpu::Buffer,
727        output: &wgpu::Buffer,
728        len: u32,
729    ) {
730        let mut encoder = self.device.create_command_encoder(&Default::default());
731        self.encode_residual(&mut encoder, a, b, output, len);
732        self.queue.submit(Some(encoder.finish()));
733    }
734
735    /// Apply RMSNorm on GPU: normed = rmsnorm(hidden_buf, weight) → output_buf.
736    /// Contract: gpu-output-norm-v1 / gpu_resident — hidden state never leaves GPU.
737    pub fn gpu_rmsnorm(&self, weight: &wgpu::Buffer, output: &wgpu::Buffer, _seq_len: u32) {
738        let mut encoder = self.device.create_command_encoder(&Default::default());
739        self.encode_rmsnorm(&mut encoder, &self.hidden_buf, weight, output, self.hidden_dim);
740        self.queue.submit(Some(encoder.finish()));
741    }
742
743    /// Download hidden state from GPU.
744    pub fn download_hidden(&self, len: usize) -> Vec<f32> {
745        let size = (len * 4) as u64;
746        let staging = self.device.create_buffer(&wgpu::BufferDescriptor {
747            label: Some("hidden_download"),
748            size,
749            usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
750            mapped_at_creation: false,
751        });
752        let mut encoder = self.device.create_command_encoder(&Default::default());
753        encoder.copy_buffer_to_buffer(&self.hidden_buf, 0, &staging, 0, size);
754        self.queue.submit(Some(encoder.finish()));
755
756        let slice = staging.slice(..size);
757        let (tx, rx) = std::sync::mpsc::channel();
758        slice.map_async(wgpu::MapMode::Read, move |r| {
759            tx.send(r).ok();
760        });
761        self.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: None }).ok();
762        rx.recv()
763            .expect("GPU map_async callback channel disconnected")
764            .expect("GPU buffer mapping failed");
765
766        let data = slice.get_mapped_range();
767        let result: Vec<f32> = bytemuck::cast_slice(&data)[..len].to_vec();
768        drop(data);
769        staging.unmap();
770        result
771    }
772
773    /// Total VRAM used by all buffers (bytes).
774    pub fn total_vram_bytes(&self) -> usize {
775        let weight_bytes: usize = self.weight_buffers.values().map(|b| b.size() as usize).sum();
776        let intermediate_bytes = (self.hidden_dim as usize * 4) * 4  // hidden, attn_out, ffn_out, norm
777            + (self.num_heads as usize * self.head_dim as usize * 4) // q
778            + (self.num_kv_heads as usize * self.head_dim as usize * 4) * 2 // k, v
779            + (self.intermediate_dim as usize * 4) * 2; // gate, up
780        weight_bytes + intermediate_bytes
781    }
782
783    /// PMAT-336: Full model forward — embedding + all layers + output norm + LM head.
784    ///
785    /// Returns logits [vocab_size] for the given token at the given position.
786    /// Embedding lookup and final LM head are CPU-side (not yet GPU-accelerated).
787    /// PMAT-344: Added kv_caches for multi-token context
788    #[provable_contracts_macros::contract("wgpu-forward-pass-v1", equation = "rmsnorm_correctness")]
789    pub fn forward_model(
790        &self,
791        token_id: u32,
792        position: usize,
793        num_layers: usize,
794        token_embedding: &[f32],
795        output_norm_weight: &[f32],
796        lm_head_weight: &[f32],
797        vocab_size: usize,
798        eps: f32,
799        kv_caches: &mut Vec<(Vec<f32>, Vec<f32>)>,
800    ) -> Result<Vec<f32>, String> {
801        let hd = self.hidden_dim as usize;
802
803        // 1. Embedding lookup (CPU)
804        let embed_start = token_id as usize * hd;
805        if embed_start + hd > token_embedding.len() {
806            return Err(format!(
807                "Token {} out of range (embedding size {})",
808                token_id,
809                token_embedding.len() / hd
810            ));
811        }
812        let mut hidden: Vec<f32> = token_embedding[embed_start..embed_start + hd].to_vec();
813
814        // 2. Transformer layers (GPU via forward_layer with KV cache)
815        // Initialize KV caches if empty
816        while kv_caches.len() < num_layers {
817            kv_caches.push((Vec::new(), Vec::new()));
818        }
819        for layer_idx in 0..num_layers {
820            let prefix = format!("layer.{layer_idx}");
821            let (ref mut k_cache, ref mut v_cache) = kv_caches[layer_idx];
822            self.forward_layer(&mut hidden, &prefix, position, k_cache, v_cache)?;
823        }
824
825        // 3. Output RMSNorm (CPU — small, not worth GPU dispatch)
826        let rms = (hidden.iter().map(|x| x * x).sum::<f32>() / hd as f32 + eps).sqrt();
827        for i in 0..hd {
828            hidden[i] = (hidden[i] / rms) * output_norm_weight[i];
829        }
830
831        // 4. LM head — CPU matmul
832        // PMAT-346: GPU tiled GEMM expects weight in [K,N] layout but lm_head is [N,K].
833        // CPU path reads weight[v * hd + j] which matches the [vocab, hidden] layout.
834        // GPU LM head via GEMV is blocked by vocab > 65535 dispatch limit.
835        // Deferred (PMAT-751): add upload_weight_transposed() for GPU-accelerated LM head.
836        let mut logits = vec![0.0f32; vocab_size];
837        for v in 0..vocab_size {
838            let mut sum = 0.0f32;
839            let row_start = v * hd;
840            for j in 0..hd {
841                sum += lm_head_weight[row_start + j] * hidden[j];
842            }
843            logits[v] = sum;
844        }
845        Ok(logits)
846    }
847
848    /// PMAT-325: Execute one transformer layer — 14 passes, 1 submit, 1 readback.
849    ///
850    /// Input: hidden state [hidden_dim] on CPU.
851    /// Output: updated hidden state [hidden_dim] on CPU.
852    /// All intermediate computation stays GPU-resident.
853    /// PMAT-344: KV cache parameters for multi-token context
854    pub fn forward_layer(
855        &self,
856        hidden: &mut [f32],
857        layer_prefix: &str,
858        _position: usize,
859        kv_cache_k: &mut Vec<f32>, // accumulated K: [seq_len * kv_dim]
860        kv_cache_v: &mut Vec<f32>, // accumulated V: [seq_len * kv_dim]
861    ) -> Result<(), String> {
862        let hd = self.hidden_dim;
863
864        // Upload hidden state
865        self.queue.write_buffer(&self.hidden_buf, 0, bytemuck::cast_slice(hidden));
866
867        let mut encoder = self.device.create_command_encoder(&Default::default());
868
869        // Pass 1: RMSNorm(hidden → norm_buf)
870        let norm_w = self
871            .weight_buffers
872            .get(&format!("{layer_prefix}.attn_norm"))
873            .ok_or_else(|| format!("Missing {layer_prefix}.attn_norm"))?;
874        self.encode_rmsnorm(&mut encoder, &self.hidden_buf, norm_w, &self.norm_buf, hd);
875
876        // Passes 2-4: Q/K/V projections (norm_buf × W → q/k/v_buf)
877        let q_dim = self.num_heads * self.head_dim;
878        let kv_dim = self.num_kv_heads * self.head_dim;
879
880        self.encode_matmul(
881            &mut encoder,
882            &self.norm_buf,
883            layer_prefix,
884            "q_proj",
885            &self.q_buf,
886            1,
887            hd,
888            q_dim,
889        );
890        self.encode_matmul(
891            &mut encoder,
892            &self.norm_buf,
893            layer_prefix,
894            "k_proj",
895            &self.k_buf,
896            1,
897            hd,
898            kv_dim,
899        );
900        self.encode_matmul(
901            &mut encoder,
902            &self.norm_buf,
903            layer_prefix,
904            "v_proj",
905            &self.v_buf,
906            1,
907            hd,
908            kv_dim,
909        );
910
911        // PMAT-342: Submit Q/K/V projections, readback, do attention on CPU
912        // GPU handles the heavy matmuls; CPU handles attention (small at M=1)
913        let q_bytes = (q_dim * 4) as u64;
914        let kv_bytes = (kv_dim * 4) as u64;
915
916        // Readback Q/K/V from GPU
917        let q_staging = self.device.create_buffer(&wgpu::BufferDescriptor {
918            label: Some("q_stg"),
919            size: q_bytes,
920            usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
921            mapped_at_creation: false,
922        });
923        let k_staging = self.device.create_buffer(&wgpu::BufferDescriptor {
924            label: Some("k_stg"),
925            size: kv_bytes,
926            usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
927            mapped_at_creation: false,
928        });
929        let v_staging = self.device.create_buffer(&wgpu::BufferDescriptor {
930            label: Some("v_stg"),
931            size: kv_bytes,
932            usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST,
933            mapped_at_creation: false,
934        });
935        encoder.copy_buffer_to_buffer(&self.q_buf, 0, &q_staging, 0, q_bytes);
936        encoder.copy_buffer_to_buffer(&self.k_buf, 0, &k_staging, 0, kv_bytes);
937        encoder.copy_buffer_to_buffer(&self.v_buf, 0, &v_staging, 0, kv_bytes);
938        self.queue.submit(Some(encoder.finish()));
939
940        // Readback Q
941        let mut q_data = vec![0.0f32; q_dim as usize];
942        {
943            let slice = q_staging.slice(..q_bytes);
944            let (tx, rx) = std::sync::mpsc::channel();
945            slice.map_async(wgpu::MapMode::Read, move |r| {
946                tx.send(r).ok();
947            });
948            self.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: None }).ok();
949            rx.recv().map_err(|e| format!("q recv: {e}"))?.map_err(|e| format!("q map: {e:?}"))?;
950            let data = slice.get_mapped_range();
951            q_data.copy_from_slice(&bytemuck::cast_slice::<u8, f32>(&data)[..q_dim as usize]);
952        }
953        q_staging.unmap();
954
955        // Readback K
956        let mut k_data = vec![0.0f32; kv_dim as usize];
957        {
958            let slice = k_staging.slice(..kv_bytes);
959            let (tx, rx) = std::sync::mpsc::channel();
960            slice.map_async(wgpu::MapMode::Read, move |r| {
961                tx.send(r).ok();
962            });
963            self.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: None }).ok();
964            rx.recv().map_err(|e| format!("k recv: {e}"))?.map_err(|e| format!("k map: {e:?}"))?;
965            let data = slice.get_mapped_range();
966            k_data.copy_from_slice(&bytemuck::cast_slice::<u8, f32>(&data)[..kv_dim as usize]);
967        }
968        k_staging.unmap();
969
970        // Readback V
971        let mut v_data = vec![0.0f32; kv_dim as usize];
972        {
973            let slice = v_staging.slice(..kv_bytes);
974            let (tx, rx) = std::sync::mpsc::channel();
975            slice.map_async(wgpu::MapMode::Read, move |r| {
976                tx.send(r).ok();
977            });
978            self.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: None }).ok();
979            rx.recv().map_err(|e| format!("v recv: {e}"))?.map_err(|e| format!("v map: {e:?}"))?;
980            let data = slice.get_mapped_range();
981            v_data.copy_from_slice(&bytemuck::cast_slice::<u8, f32>(&data)[..kv_dim as usize]);
982        }
983        v_staging.unmap();
984
985        // PMAT-342: Add QKV biases (required for Qwen2)
986        if let Some(q_bias) = self.cpu_biases.get(&format!("{layer_prefix}.q_bias")) {
987            for (q, b) in q_data.iter_mut().zip(q_bias.iter()) {
988                *q += *b;
989            }
990        }
991        if let Some(k_bias) = self.cpu_biases.get(&format!("{layer_prefix}.k_bias")) {
992            for (k, b) in k_data.iter_mut().zip(k_bias.iter()) {
993                *k += *b;
994            }
995        }
996        if let Some(v_bias) = self.cpu_biases.get(&format!("{layer_prefix}.v_bias")) {
997            for (v, b) in v_data.iter_mut().zip(v_bias.iter()) {
998                *v += *b;
999            }
1000        }
1001
1002        // PMAT-343: Apply RoPE (NeoX-style interleaved) to Q and K
1003        let head_dim = self.head_dim as usize;
1004        let position = _position; // Use the position parameter
1005        rope_rotate_heads(
1006            &mut q_data,
1007            self.num_heads as usize,
1008            head_dim,
1009            position,
1010            self.rope_theta,
1011        );
1012        rope_rotate_heads(
1013            &mut k_data,
1014            self.num_kv_heads as usize,
1015            head_dim,
1016            position,
1017            self.rope_theta,
1018        );
1019
1020        // PMAT-344: Append K,V to cache and compute full attention
1021        let head_dim = self.head_dim as usize;
1022        let num_heads = self.num_heads as usize;
1023        let num_kv_heads = self.num_kv_heads as usize;
1024        let kv_dim_usize = kv_dim as usize;
1025
1026        kv_cache_k.extend_from_slice(&k_data);
1027        kv_cache_v.extend_from_slice(&v_data);
1028        let seq_len = kv_cache_k.len() / kv_dim_usize;
1029
1030        // Scaled dot-product attention with GQA
1031        let kv_group = num_heads / num_kv_heads;
1032        let scale = 1.0 / (head_dim as f32).sqrt();
1033        let mut attn_out = vec![0.0f32; q_dim as usize];
1034
1035        for h in 0..num_heads {
1036            let kv_h = h / kv_group;
1037            let q_offset = h * head_dim;
1038            let kv_offset = kv_h * head_dim;
1039
1040            // Compute attention scores: Q[h] · K[kv_h, :seq_len]^T / sqrt(d)
1041            let scores = Self::attention_scores(
1042                &q_data[q_offset..q_offset + head_dim],
1043                kv_cache_k,
1044                kv_offset,
1045                kv_dim_usize,
1046                seq_len,
1047                scale,
1048            );
1049
1050            // Weighted sum of V
1051            let out_offset = h * head_dim;
1052            Self::attention_weighted_v(
1053                &scores,
1054                kv_cache_v,
1055                &mut attn_out[out_offset..out_offset + head_dim],
1056                kv_offset,
1057                kv_dim_usize,
1058            );
1059        }
1060
1061        // Upload attention output back to GPU for O projection
1062        self.queue.write_buffer(&self.q_buf, 0, bytemuck::cast_slice(&attn_out));
1063
1064        // New encoder for remaining passes
1065        let mut encoder = self.device.create_command_encoder(&Default::default());
1066
1067        // Pass 7: O projection (attn_out × W_o → attn_out_buf)
1068        self.encode_matmul(
1069            &mut encoder,
1070            &self.q_buf,
1071            layer_prefix,
1072            "o_proj",
1073            &self.attn_out_buf,
1074            1,
1075            q_dim,
1076            hd,
1077        );
1078
1079        // Pass 8: Residual(hidden + attn_out → hidden)
1080        self.encode_residual(
1081            &mut encoder,
1082            &self.hidden_buf,
1083            &self.attn_out_buf,
1084            &self.ffn_out_buf,
1085            hd,
1086        );
1087
1088        // Pass 9: FFN RMSNorm(ffn_out → norm_buf)
1089        let ffn_norm_w = self
1090            .weight_buffers
1091            .get(&format!("{layer_prefix}.ffn_norm"))
1092            .ok_or_else(|| format!("Missing {layer_prefix}.ffn_norm"))?;
1093        self.encode_rmsnorm(&mut encoder, &self.ffn_out_buf, ffn_norm_w, &self.norm_buf, hd);
1094
1095        // Passes 10-11: Gate + Up projections
1096        let inter = self.intermediate_dim;
1097        self.encode_matmul(
1098            &mut encoder,
1099            &self.norm_buf,
1100            layer_prefix,
1101            "gate_proj",
1102            &self.ffn_gate_buf,
1103            1,
1104            hd,
1105            inter,
1106        );
1107        self.encode_matmul(
1108            &mut encoder,
1109            &self.norm_buf,
1110            layer_prefix,
1111            "up_proj",
1112            &self.ffn_up_buf,
1113            1,
1114            hd,
1115            inter,
1116        );
1117
1118        // Pass 12: SiLU(gate) × up → ffn_silu_buf [intermediate_dim]
1119        // BUG FIX: was writing to attn_out_buf (hidden_dim=3584) but needs intermediate_dim=18944.
1120        // attn_out_buf is only hidden_dim — wgpu robustness silently drops OOB writes,
1121        // then down_proj reads zeros past hidden_dim → 81% of FFN truncated → garbage output.
1122        // Cannot alias gate/up buffers (WGSL read/write aliasing UB), so use dedicated buffer.
1123        self.encode_silu_mul(
1124            &mut encoder,
1125            &self.ffn_gate_buf,
1126            &self.ffn_up_buf,
1127            &self.ffn_silu_buf,
1128            inter,
1129        );
1130
1131        // Pass 13: Down projection (reads ffn_silu_buf [intermediate_dim] → norm_buf [hidden_dim])
1132        self.encode_matmul(
1133            &mut encoder,
1134            &self.ffn_silu_buf,
1135            layer_prefix,
1136            "down_proj",
1137            &self.norm_buf,
1138            1,
1139            inter,
1140            hd,
1141        );
1142
1143        // Pass 14: Residual(ffn_out + down → hidden)
1144        self.encode_residual(&mut encoder, &self.ffn_out_buf, &self.norm_buf, &self.hidden_buf, hd);
1145
1146        // Single readback
1147        encoder.copy_buffer_to_buffer(&self.hidden_buf, 0, &self.staging_buf, 0, (hd * 4) as u64);
1148        self.queue.submit(Some(encoder.finish()));
1149
1150        // Readback
1151        let slice = self.staging_buf.slice(..(hd as u64 * 4));
1152        let (tx, rx) = std::sync::mpsc::channel();
1153        slice.map_async(wgpu::MapMode::Read, move |r| {
1154            tx.send(r).ok();
1155        });
1156        self.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: None }).ok();
1157        rx.recv().map_err(|e| format!("recv: {e}"))?.map_err(|e| format!("map: {e:?}"))?;
1158        {
1159            let data = slice.get_mapped_range();
1160            hidden.copy_from_slice(
1161                &bytemuck::cast_slice::<u8, f32>(&data)[..self.hidden_dim as usize],
1162            );
1163        }
1164        self.staging_buf.unmap();
1165
1166        Ok(())
1167    }
1168
1169    /// Softmaxed attention scores for one head over `seq_len` cached steps.
1170    ///
1171    /// `kv_offset` is this head's offset inside each cached step, and each
1172    /// cached step is `kv_dim` floats wide. Head dimension is `q_head.len()`.
1173    fn attention_scores(
1174        q_head: &[f32],
1175        kv_cache_k: &[f32],
1176        kv_offset: usize,
1177        kv_dim: usize,
1178        seq_len: usize,
1179        scale: f32,
1180    ) -> Vec<f32> {
1181        let mut scores = vec![0.0f32; seq_len];
1182        for (s, score) in scores.iter_mut().enumerate() {
1183            let k_offset = s * kv_dim + kv_offset;
1184            let mut dot = 0.0f32;
1185            for (d, q) in q_head.iter().enumerate() {
1186                dot += q * kv_cache_k[k_offset + d];
1187            }
1188            *score = dot * scale;
1189        }
1190
1191        // Softmax
1192        let max_score = scores.iter().copied().fold(f32::NEG_INFINITY, f32::max);
1193        let mut sum = 0.0f32;
1194        for s in scores.iter_mut() {
1195            *s = (*s - max_score).exp();
1196            sum += *s;
1197        }
1198        if sum > 0.0 {
1199            for s in scores.iter_mut() {
1200                *s /= sum;
1201            }
1202        }
1203        scores
1204    }
1205
1206    /// Weighted sum of V for one head, written into `out_head`.
1207    ///
1208    /// `kv_offset` and `kv_dim` are as in `attention_scores`.
1209    fn attention_weighted_v(
1210        scores: &[f32],
1211        kv_cache_v: &[f32],
1212        out_head: &mut [f32],
1213        kv_offset: usize,
1214        kv_dim: usize,
1215    ) {
1216        for (d, out) in out_head.iter_mut().enumerate() {
1217            let mut val = 0.0f32;
1218            for (s, score) in scores.iter().enumerate() {
1219                let v_offset = s * kv_dim + kv_offset;
1220                val += score * kv_cache_v[v_offset + d];
1221            }
1222            *out = val;
1223        }
1224    }
1225
1226    /// Training forward pass for a single transformer layer.
1227    ///
1228    /// Unlike `forward_layer` (M=1 decode), this processes the full sequence
1229    /// at once (M=seq_len) and keeps everything on GPU. No CPU readback.
1230    ///
1231    /// Saves `norm_output` (pre-projection activations) for backward pass.
1232    ///
1233    /// # Arguments
1234    /// - `seq_len`: number of tokens in the sequence
1235    /// - `layer_prefix`: e.g. "model.layers.0"
1236    /// - `saved_norm_attn`: OUTPUT — saved pre-attention norm for backward (wgpu::Buffer, [seq×hidden])
1237    /// - `saved_norm_ffn`: OUTPUT — saved pre-FFN norm for backward (wgpu::Buffer, [seq×hidden])
1238    ///
1239    /// Forward one layer into an EXISTING encoder (no submit).
1240    /// Caller batches multiple layers into one encoder, submits once.
1241    pub fn encode_forward_layer_training(
1242        &self,
1243        encoder: &mut wgpu::CommandEncoder,
1244        seq_len: u32,
1245        layer_prefix: &str,
1246        saved: &LayerActivations,
1247        lora: Option<&QkvLoRA<'_>>,
1248    ) -> Result<(), String> {
1249        let hd = self.hidden_dim;
1250        let q_dim = self.num_heads * self.head_dim;
1251        let kv_dim = self.num_kv_heads * self.head_dim;
1252        let inter = self.intermediate_dim;
1253        let s = seq_len as usize;
1254
1255        // Pass 1: RMSNorm
1256        let norm_w = self
1257            .weight_buffers
1258            .get(&format!("{layer_prefix}.attn_norm"))
1259            .ok_or_else(|| format!("Missing {layer_prefix}.attn_norm"))?;
1260        self.encode_rmsnorm(encoder, &self.hidden_buf, norm_w, &self.norm_buf, hd);
1261
1262        // SAVE attn_norm_out
1263        encoder.copy_buffer_to_buffer(
1264            &self.norm_buf,
1265            0,
1266            &saved.attn_norm_out,
1267            0,
1268            (s * hd as usize * 4) as u64,
1269        );
1270
1271        // Q/K/V projections
1272        self.encode_matmul(
1273            encoder,
1274            &self.norm_buf,
1275            layer_prefix,
1276            "q_proj",
1277            &self.q_buf,
1278            seq_len,
1279            hd,
1280            q_dim,
1281        );
1282        self.encode_matmul(
1283            encoder,
1284            &self.norm_buf,
1285            layer_prefix,
1286            "k_proj",
1287            &self.k_buf,
1288            seq_len,
1289            hd,
1290            kv_dim,
1291        );
1292        self.encode_matmul(
1293            encoder,
1294            &self.norm_buf,
1295            layer_prefix,
1296            "v_proj",
1297            &self.v_buf,
1298            seq_len,
1299            hd,
1300            kv_dim,
1301        );
1302
1303        // LoRA addmm on Q/K/V: output += (saved_input @ A) @ B * scale
1304        // Must happen BEFORE attention consumes Q/K/V buffers.
1305        if let Some(lora) = lora {
1306            self.encode_lora_addmm(
1307                encoder,
1308                &saved.attn_norm_out,
1309                lora.q_a,
1310                lora.q_b,
1311                &self.q_buf,
1312                seq_len,
1313                lora.in_dim,
1314                lora.rank,
1315                lora.q_dim,
1316                lora.scale,
1317                lora.lora_pipeline,
1318                lora.lora_bgl,
1319            );
1320            self.encode_lora_addmm(
1321                encoder,
1322                &saved.attn_norm_out,
1323                lora.k_a,
1324                lora.k_b,
1325                &self.k_buf,
1326                seq_len,
1327                lora.in_dim,
1328                lora.rank,
1329                lora.kv_dim,
1330                lora.scale,
1331                lora.lora_pipeline,
1332                lora.lora_bgl,
1333            );
1334            self.encode_lora_addmm(
1335                encoder,
1336                &saved.attn_norm_out,
1337                lora.v_a,
1338                lora.v_b,
1339                &self.v_buf,
1340                seq_len,
1341                lora.in_dim,
1342                lora.rank,
1343                lora.kv_dim,
1344                lora.scale,
1345                lora.lora_pipeline,
1346                lora.lora_bgl,
1347            );
1348        }
1349
1350        // PMAT-509: Apply QKV biases (required for Qwen2)
1351        if let Some(q_bias) = self.cpu_biases.get(&format!("{layer_prefix}.q_bias")) {
1352            self.encode_broadcast_bias(encoder, &self.q_buf, q_bias, seq_len);
1353        }
1354        if let Some(k_bias) = self.cpu_biases.get(&format!("{layer_prefix}.k_bias")) {
1355            self.encode_broadcast_bias(encoder, &self.k_buf, k_bias, seq_len);
1356        }
1357        if let Some(v_bias) = self.cpu_biases.get(&format!("{layer_prefix}.v_bias")) {
1358            self.encode_broadcast_bias(encoder, &self.v_buf, v_bias, seq_len);
1359        }
1360
1361        // PMAT-509: Apply RoPE to Q and K before attention.
1362        self.encode_batch_rope(encoder, &self.q_buf, seq_len, self.num_heads, self.head_dim);
1363        self.encode_batch_rope(encoder, &self.k_buf, seq_len, self.num_kv_heads, self.head_dim);
1364
1365        // Attention — wgpu handles execution ordering within the encoder.
1366        self.encode_attention(encoder, seq_len);
1367
1368        // SAVE attn_output
1369        encoder.copy_buffer_to_buffer(
1370            &self.attn_out_buf,
1371            0,
1372            &saved.attn_output,
1373            0,
1374            (s * q_dim as usize * 4) as u64,
1375        );
1376
1377        // O projection
1378        self.encode_matmul(
1379            encoder,
1380            &self.attn_out_buf,
1381            layer_prefix,
1382            "o_proj",
1383            &self.q_buf,
1384            seq_len,
1385            q_dim,
1386            hd,
1387        );
1388
1389        // Residual
1390        self.encode_residual(
1391            encoder,
1392            &self.hidden_buf,
1393            &self.q_buf,
1394            &self.ffn_out_buf,
1395            hd * seq_len,
1396        );
1397
1398        // FFN RMSNorm
1399        let ffn_norm_w = self
1400            .weight_buffers
1401            .get(&format!("{layer_prefix}.ffn_norm"))
1402            .ok_or_else(|| format!("Missing {layer_prefix}.ffn_norm"))?;
1403        self.encode_rmsnorm(encoder, &self.ffn_out_buf, ffn_norm_w, &self.norm_buf, hd);
1404
1405        // SAVE ffn_norm_out
1406        encoder.copy_buffer_to_buffer(
1407            &self.norm_buf,
1408            0,
1409            &saved.ffn_norm_out,
1410            0,
1411            (s * hd as usize * 4) as u64,
1412        );
1413
1414        // Gate + Up
1415        self.encode_matmul(
1416            encoder,
1417            &self.norm_buf,
1418            layer_prefix,
1419            "gate_proj",
1420            &self.ffn_gate_buf,
1421            seq_len,
1422            hd,
1423            inter,
1424        );
1425        self.encode_matmul(
1426            encoder,
1427            &self.norm_buf,
1428            layer_prefix,
1429            "up_proj",
1430            &self.ffn_up_buf,
1431            seq_len,
1432            hd,
1433            inter,
1434        );
1435
1436        // SiLU
1437        self.encode_silu_mul(
1438            encoder,
1439            &self.ffn_gate_buf,
1440            &self.ffn_up_buf,
1441            &self.ffn_silu_buf,
1442            inter * seq_len,
1443        );
1444
1445        // SAVE silu_gate_output
1446        encoder.copy_buffer_to_buffer(
1447            &self.ffn_silu_buf,
1448            0,
1449            &saved.silu_gate_output,
1450            0,
1451            (s * inter as usize * 4) as u64,
1452        );
1453
1454        // Down projection
1455        self.encode_matmul(
1456            encoder,
1457            &self.ffn_silu_buf,
1458            layer_prefix,
1459            "down_proj",
1460            &self.norm_buf,
1461            seq_len,
1462            inter,
1463            hd,
1464        );
1465
1466        // Residual
1467        self.encode_residual(
1468            encoder,
1469            &self.ffn_out_buf,
1470            &self.norm_buf,
1471            &self.hidden_buf,
1472            hd * seq_len,
1473        );
1474
1475        Ok(())
1476    }
1477
1478    /// Run one layer with per-operation GPU timing (submit+poll between each op group).
1479    /// Contract: forward-pass-perf-v1 / bottleneck_identified
1480    pub fn forward_layer_traced(
1481        &self,
1482        seq_len: u32,
1483        layer_prefix: &str,
1484        saved: &LayerActivations,
1485        lora: Option<&QkvLoRA<'_>>,
1486    ) -> Result<(), String> {
1487        let hd = self.hidden_dim;
1488        let q_dim = self.num_heads * self.head_dim;
1489        let kv_dim = self.num_kv_heads * self.head_dim;
1490        let inter = self.intermediate_dim;
1491        let s = seq_len as usize;
1492
1493        let norm_w = self
1494            .weight_buffers
1495            .get(&format!("{layer_prefix}.attn_norm"))
1496            .ok_or_else(|| format!("Missing {layer_prefix}.attn_norm"))?;
1497
1498        let mut trace = Vec::new();
1499        let mut run = |name: &str, f: &dyn Fn(&mut wgpu::CommandEncoder)| {
1500            let mut enc = self.device.create_command_encoder(&Default::default());
1501            f(&mut enc);
1502            self.queue.submit(Some(enc.finish()));
1503            let t = std::time::Instant::now();
1504            self.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: None }).ok();
1505            trace.push((name.to_string(), t.elapsed().as_millis() as u64));
1506        };
1507
1508        run("rmsnorm1", &|e| self.encode_rmsnorm(e, &self.hidden_buf, norm_w, &self.norm_buf, hd));
1509        {
1510            let mut e = self.device.create_command_encoder(&Default::default());
1511            e.copy_buffer_to_buffer(
1512                &self.norm_buf,
1513                0,
1514                &saved.attn_norm_out,
1515                0,
1516                (s * hd as usize * 4) as u64,
1517            );
1518            self.queue.submit(Some(e.finish()));
1519        }
1520        run("q_proj", &|e| {
1521            self.encode_matmul(
1522                e,
1523                &self.norm_buf,
1524                layer_prefix,
1525                "q_proj",
1526                &self.q_buf,
1527                seq_len,
1528                hd,
1529                q_dim,
1530            )
1531        });
1532        run("k_proj", &|e| {
1533            self.encode_matmul(
1534                e,
1535                &self.norm_buf,
1536                layer_prefix,
1537                "k_proj",
1538                &self.k_buf,
1539                seq_len,
1540                hd,
1541                kv_dim,
1542            )
1543        });
1544        run("v_proj", &|e| {
1545            self.encode_matmul(
1546                e,
1547                &self.norm_buf,
1548                layer_prefix,
1549                "v_proj",
1550                &self.v_buf,
1551                seq_len,
1552                hd,
1553                kv_dim,
1554            )
1555        });
1556        if let Some(lr) = lora {
1557            run("lora_qkv", &|e| {
1558                self.encode_lora_addmm(
1559                    e,
1560                    &saved.attn_norm_out,
1561                    lr.q_a,
1562                    lr.q_b,
1563                    &self.q_buf,
1564                    seq_len,
1565                    lr.in_dim,
1566                    lr.rank,
1567                    lr.q_dim,
1568                    lr.scale,
1569                    lr.lora_pipeline,
1570                    lr.lora_bgl,
1571                );
1572                self.encode_lora_addmm(
1573                    e,
1574                    &saved.attn_norm_out,
1575                    lr.k_a,
1576                    lr.k_b,
1577                    &self.k_buf,
1578                    seq_len,
1579                    lr.in_dim,
1580                    lr.rank,
1581                    lr.kv_dim,
1582                    lr.scale,
1583                    lr.lora_pipeline,
1584                    lr.lora_bgl,
1585                );
1586                self.encode_lora_addmm(
1587                    e,
1588                    &saved.attn_norm_out,
1589                    lr.v_a,
1590                    lr.v_b,
1591                    &self.v_buf,
1592                    seq_len,
1593                    lr.in_dim,
1594                    lr.rank,
1595                    lr.kv_dim,
1596                    lr.scale,
1597                    lr.lora_pipeline,
1598                    lr.lora_bgl,
1599                );
1600            });
1601        }
1602        // PMAT-509: QKV biases + RoPE before attention
1603        if let Some(q_bias) = self.cpu_biases.get(&format!("{layer_prefix}.q_bias")) {
1604            run("q_bias", &|e| self.encode_broadcast_bias(e, &self.q_buf, q_bias, seq_len));
1605        }
1606        if let Some(k_bias) = self.cpu_biases.get(&format!("{layer_prefix}.k_bias")) {
1607            run("k_bias", &|e| self.encode_broadcast_bias(e, &self.k_buf, k_bias, seq_len));
1608        }
1609        if let Some(v_bias) = self.cpu_biases.get(&format!("{layer_prefix}.v_bias")) {
1610            run("v_bias", &|e| self.encode_broadcast_bias(e, &self.v_buf, v_bias, seq_len));
1611        }
1612        run("rope_q", &|e| {
1613            self.encode_batch_rope(e, &self.q_buf, seq_len, self.num_heads, self.head_dim)
1614        });
1615        run("rope_k", &|e| {
1616            self.encode_batch_rope(e, &self.k_buf, seq_len, self.num_kv_heads, self.head_dim)
1617        });
1618        run("attention", &|e| self.encode_attention(e, seq_len));
1619        {
1620            let mut e = self.device.create_command_encoder(&Default::default());
1621            e.copy_buffer_to_buffer(
1622                &self.attn_out_buf,
1623                0,
1624                &saved.attn_output,
1625                0,
1626                (s * q_dim as usize * 4) as u64,
1627            );
1628            self.queue.submit(Some(e.finish()));
1629        }
1630        run("o_proj", &|e| {
1631            self.encode_matmul(
1632                e,
1633                &self.attn_out_buf,
1634                layer_prefix,
1635                "o_proj",
1636                &self.q_buf,
1637                seq_len,
1638                q_dim,
1639                hd,
1640            )
1641        });
1642        run("residual1", &|e| {
1643            self.encode_residual(e, &self.hidden_buf, &self.q_buf, &self.ffn_out_buf, hd * seq_len)
1644        });
1645        let ffn_norm_w = self
1646            .weight_buffers
1647            .get(&format!("{layer_prefix}.ffn_norm"))
1648            .ok_or_else(|| format!("Missing {layer_prefix}.ffn_norm"))?;
1649        run("rmsnorm2", &|e| {
1650            self.encode_rmsnorm(e, &self.ffn_out_buf, ffn_norm_w, &self.norm_buf, hd)
1651        });
1652        {
1653            let mut e = self.device.create_command_encoder(&Default::default());
1654            e.copy_buffer_to_buffer(
1655                &self.norm_buf,
1656                0,
1657                &saved.ffn_norm_out,
1658                0,
1659                (s * hd as usize * 4) as u64,
1660            );
1661            self.queue.submit(Some(e.finish()));
1662        }
1663        run("gate_proj", &|e| {
1664            self.encode_matmul(
1665                e,
1666                &self.norm_buf,
1667                layer_prefix,
1668                "gate_proj",
1669                &self.ffn_gate_buf,
1670                seq_len,
1671                hd,
1672                inter,
1673            )
1674        });
1675        run("up_proj", &|e| {
1676            self.encode_matmul(
1677                e,
1678                &self.norm_buf,
1679                layer_prefix,
1680                "up_proj",
1681                &self.ffn_up_buf,
1682                seq_len,
1683                hd,
1684                inter,
1685            )
1686        });
1687        run("silu", &|e| {
1688            self.encode_silu_mul(
1689                e,
1690                &self.ffn_gate_buf,
1691                &self.ffn_up_buf,
1692                &self.ffn_silu_buf,
1693                inter * seq_len,
1694            )
1695        });
1696        {
1697            let mut e = self.device.create_command_encoder(&Default::default());
1698            e.copy_buffer_to_buffer(
1699                &self.ffn_silu_buf,
1700                0,
1701                &saved.silu_gate_output,
1702                0,
1703                (s * inter as usize * 4) as u64,
1704            );
1705            self.queue.submit(Some(e.finish()));
1706        }
1707        run("down_proj", &|e| {
1708            self.encode_matmul(
1709                e,
1710                &self.ffn_silu_buf,
1711                layer_prefix,
1712                "down_proj",
1713                &self.norm_buf,
1714                seq_len,
1715                inter,
1716                hd,
1717            )
1718        });
1719        run("residual2", &|e| {
1720            self.encode_residual(
1721                e,
1722                &self.ffn_out_buf,
1723                &self.norm_buf,
1724                &self.hidden_buf,
1725                hd * seq_len,
1726            )
1727        });
1728
1729        let total: u64 = trace.iter().map(|(_, ms)| ms).sum();
1730        let parts: Vec<String> = trace.iter().map(|(n, ms)| format!("{n}={ms}")).collect();
1731        eprintln!("[OP-TRACE] layer {} total={}ms: {}", layer_prefix, total, parts.join(" "));
1732        Ok(())
1733    }
1734
1735    /// Allocate saved activations for one layer.
1736    pub fn alloc_layer_activations(&self, seq_len: u32) -> LayerActivations {
1737        let s = seq_len as usize;
1738        let buf = |size: usize, label: &str| -> wgpu::Buffer {
1739            self.device.create_buffer(&wgpu::BufferDescriptor {
1740                label: Some(label),
1741                size: (size * 4) as u64,
1742                usage: wgpu::BufferUsages::STORAGE
1743                    | wgpu::BufferUsages::COPY_SRC
1744                    | wgpu::BufferUsages::COPY_DST,
1745                mapped_at_creation: false,
1746            })
1747        };
1748        LayerActivations {
1749            attn_norm_out: buf(s * self.hidden_dim as usize, "saved_attn_norm"),
1750            attn_output: buf(s * (self.num_heads * self.head_dim) as usize, "saved_attn_out"),
1751            ffn_norm_out: buf(s * self.hidden_dim as usize, "saved_ffn_norm"),
1752            silu_gate_output: buf(s * self.intermediate_dim as usize, "saved_silu"),
1753            rstd_attn: buf(s, "saved_rstd_attn"),
1754            rstd_ffn: buf(s, "saved_rstd_ffn"),
1755            softmax_logsumexp: buf(self.num_heads as usize * s, "saved_logsumexp"),
1756        }
1757    }
1758
1759    /// Forward one layer with its own encoder + submit (original API, kept for compat).
1760    pub fn forward_layer_training(
1761        &self,
1762        seq_len: u32,
1763        layer_prefix: &str,
1764    ) -> Result<LayerActivations, String> {
1765        let saved = self.alloc_layer_activations(seq_len);
1766        let mut encoder = self.device.create_command_encoder(&Default::default());
1767        self.encode_forward_layer_training(&mut encoder, seq_len, layer_prefix, &saved, None)?;
1768        self.queue.submit(Some(encoder.finish()));
1769        Ok(saved)
1770    }
1771
1772    /// Forward ALL layers in one encoder submit. 28 layers → 1 GPU sync.
1773    pub fn forward_all_layers_training(
1774        &self,
1775        seq_len: u32,
1776        num_layers: usize,
1777    ) -> Result<Vec<LayerActivations>, String> {
1778        let mut encoder = self.device.create_command_encoder(&Default::default());
1779        let mut all_saved = Vec::with_capacity(num_layers);
1780
1781        for layer_idx in 0..num_layers {
1782            let prefix = format!("layer.{layer_idx}");
1783            let saved = self.alloc_layer_activations(seq_len);
1784            self.encode_forward_layer_training(&mut encoder, seq_len, &prefix, &saved, None)?;
1785            all_saved.push(saved);
1786        }
1787
1788        // ONE submit for all 28 layers — eliminates 27 GPU sync barriers
1789        self.queue.submit(Some(encoder.finish()));
1790        Ok(all_saved)
1791    }
1792    // --- Encode helpers (add compute passes to an existing encoder) ---
1793
1794    /// Encode causal multi-head attention on GPU.
1795    /// Q: [seq_len, num_heads * head_dim], K/V: [seq_len, num_kv_heads * head_dim]
1796    /// Output written to q_buf (reused as attn output).
1797    /// PMAT-509: Add broadcast bias to a [seq_len, dim] buffer.
1798    /// bias has shape [dim], applied to each of seq_len rows.
1799    pub fn encode_broadcast_bias(
1800        &self,
1801        encoder: &mut wgpu::CommandEncoder,
1802        buf: &wgpu::Buffer,
1803        bias: &[f32],
1804        seq_len: u32,
1805    ) {
1806        let dim = bias.len();
1807        // Create a full-size bias buffer by repeating the bias per position
1808        let mut full_bias = Vec::with_capacity(seq_len as usize * dim);
1809        for _ in 0..seq_len {
1810            full_bias.extend_from_slice(bias);
1811        }
1812        let bias_buf = self.device.create_buffer(&wgpu::BufferDescriptor {
1813            label: Some("broadcast_bias"),
1814            size: (full_bias.len() * 4) as u64,
1815            usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_DST,
1816            mapped_at_creation: false,
1817        });
1818        self.queue.write_buffer(&bias_buf, 0, bytemuck::cast_slice(&full_bias));
1819
1820        // Use existing residual: out = buf + bias_buf (into a temp, then copy back)
1821        let total = seq_len * dim as u32;
1822        let tmp = self.device.create_buffer(&wgpu::BufferDescriptor {
1823            label: Some("bias_tmp"),
1824            size: (total as usize * 4) as u64,
1825            usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
1826            mapped_at_creation: false,
1827        });
1828        self.encode_residual(encoder, buf, &bias_buf, &tmp, total);
1829        encoder.copy_buffer_to_buffer(&tmp, 0, buf, 0, (total as u64) * 4);
1830    }
1831
1832    /// PMAT-509: Encode batch RoPE for all positions in a sequence.
1833    /// Applies position-dependent rotation to Q or K buffer in-place.
1834    fn encode_batch_rope(
1835        &self,
1836        encoder: &mut wgpu::CommandEncoder,
1837        qk_buf: &wgpu::Buffer,
1838        seq_len: u32,
1839        num_heads: u32,
1840        head_dim: u32,
1841    ) {
1842        #[repr(C)]
1843        #[derive(Copy, Clone, bytemuck::Pod, bytemuck::Zeroable)]
1844        struct RopeParams {
1845            seq_len: u32,
1846            num_heads: u32,
1847            head_dim: u32,
1848            _pad: u32,
1849        }
1850        let params = RopeParams { seq_len, num_heads, head_dim, _pad: 0 };
1851        let params_buf = self.device.create_buffer(&wgpu::BufferDescriptor {
1852            label: Some("batch_rope_params"),
1853            size: 16,
1854            usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
1855            mapped_at_creation: false,
1856        });
1857        self.queue.write_buffer(&params_buf, 0, bytemuck::bytes_of(&params));
1858
1859        let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
1860            label: Some("batch_rope_bg"),
1861            layout: &self.batch_rope_bgl,
1862            entries: &[
1863                wgpu::BindGroupEntry { binding: 0, resource: qk_buf.as_entire_binding() },
1864                wgpu::BindGroupEntry { binding: 1, resource: params_buf.as_entire_binding() },
1865            ],
1866        });
1867        let total = seq_len * num_heads * head_dim;
1868        let wg = total.div_ceil(256);
1869        let mut pass = encoder.begin_compute_pass(&Default::default());
1870        pass.set_pipeline(&self.batch_rope_pipeline);
1871        pass.set_bind_group(0, &bg, &[]);
1872        pass.dispatch_workgroups(wg, 1, 1);
1873    }
1874
1875    fn encode_attention(&self, encoder: &mut wgpu::CommandEncoder, seq_len: u32) {
1876        let params = [seq_len, self.num_heads, self.num_kv_heads, self.head_dim];
1877        let params_buf = self.make_uniform(&params);
1878        let _q_dim = self.num_heads * self.head_dim;
1879
1880        // Attention reads Q and writes to attn_out_buf.
1881        // Then O projection reads attn_out_buf → writes to another buffer.
1882        // We can safely write to norm_buf here since it's not read during attention.
1883        // After attention, we'll copy norm_buf → q_buf for the O projection to read.
1884        let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
1885            label: None,
1886            layout: &self.attention_bgl,
1887            entries: &[
1888                wgpu::BindGroupEntry { binding: 0, resource: self.q_buf.as_entire_binding() },
1889                wgpu::BindGroupEntry { binding: 1, resource: self.k_buf.as_entire_binding() },
1890                wgpu::BindGroupEntry { binding: 2, resource: self.v_buf.as_entire_binding() },
1891                wgpu::BindGroupEntry {
1892                    binding: 3,
1893                    resource: self.attn_out_buf.as_entire_binding(),
1894                },
1895                wgpu::BindGroupEntry { binding: 4, resource: params_buf.as_entire_binding() },
1896            ],
1897        });
1898        let mut pass = encoder.begin_compute_pass(&Default::default());
1899        pass.set_pipeline(&self.attention_pipeline);
1900        pass.set_bind_group(0, &bg, &[]);
1901        // One workgroup per (head, position)
1902        pass.dispatch_workgroups(self.num_heads, seq_len, 1);
1903    }
1904
1905    /// Encode LoRA addmm: output += (input @ A) @ B * scale
1906    ///
1907    /// KAIZEN: replaced fused shader (0.11 GFLOPS) with two tiled GEMM dispatches (1000+ GFLOPS).
1908    /// Step 1: temp = input @ A  [seq, rank] via tiled GEMM
1909    /// Step 2: output += scale * (temp @ B) [seq, out_dim] via tiled GEMM with alpha=scale
1910    ///
1911    /// The second GEMM uses alpha=scale in the tiled GEMM shader (C = alpha * A @ B).
1912    /// But we need ADD (+=), not overwrite (=). We use a temp buffer for the delta,
1913    /// then add to output via an elementwise shader.
1914    #[allow(clippy::too_many_arguments)]
1915    fn encode_lora_addmm(
1916        &self,
1917        encoder: &mut wgpu::CommandEncoder,
1918        input: &wgpu::Buffer,
1919        lora_a: &wgpu::Buffer,
1920        lora_b: &wgpu::Buffer,
1921        output: &wgpu::Buffer,
1922        seq_len: u32,
1923        in_dim: u32,
1924        rank: u32,
1925        out_dim: u32,
1926        scale: f32,
1927        _pipeline: &wgpu::ComputePipeline,
1928        _bgl: &wgpu::BindGroupLayout,
1929    ) {
1930        // Step 1: temp[seq, rank] = input[seq, in_dim] @ A[in_dim, rank]
1931        let temp_size = (seq_len * rank) as u64 * 4;
1932        let temp = self.device.create_buffer(&wgpu::BufferDescriptor {
1933            label: Some("lora_temp"),
1934            size: temp_size,
1935            usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
1936            mapped_at_creation: false,
1937        });
1938        self.encode_tiled_gemm(encoder, input, lora_a, &temp, seq_len, in_dim, rank, 1.0);
1939
1940        // Step 2: delta[seq, out_dim] = scale * temp[seq, rank] @ B[rank, out_dim]
1941        let delta_size = (seq_len * out_dim) as u64 * 4;
1942        let delta = self.device.create_buffer(&wgpu::BufferDescriptor {
1943            label: Some("lora_delta"),
1944            size: delta_size,
1945            usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
1946            mapped_at_creation: false,
1947        });
1948        self.encode_tiled_gemm(encoder, &temp, lora_b, &delta, seq_len, rank, out_dim, scale);
1949
1950        // Step 3: output += delta (elementwise add, via temp to avoid aliasing)
1951        let sum_buf = self.device.create_buffer(&wgpu::BufferDescriptor {
1952            label: Some("lora_sum"),
1953            size: delta_size,
1954            usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC,
1955            mapped_at_creation: false,
1956        });
1957        self.encode_residual(encoder, output, &delta, &sum_buf, seq_len * out_dim);
1958        encoder.copy_buffer_to_buffer(&sum_buf, 0, output, 0, delta_size);
1959    }
1960
1961    /// Encode tiled GEMM: C = alpha * A[M,K] @ B[K,N]. Uses CUTLASS-style 64×64 tiles.
1962    fn encode_tiled_gemm(
1963        &self,
1964        encoder: &mut wgpu::CommandEncoder,
1965        a: &wgpu::Buffer,
1966        b: &wgpu::Buffer,
1967        c: &wgpu::Buffer,
1968        m: u32,
1969        k: u32,
1970        n: u32,
1971        alpha: f32,
1972    ) {
1973        let params = [m, k, n, alpha.to_bits()];
1974        let params_buf = self.make_uniform(&params);
1975        let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
1976            label: None,
1977            layout: &self.matmul_bgl,
1978            entries: &[
1979                wgpu::BindGroupEntry { binding: 0, resource: a.as_entire_binding() },
1980                wgpu::BindGroupEntry { binding: 1, resource: b.as_entire_binding() },
1981                wgpu::BindGroupEntry { binding: 2, resource: c.as_entire_binding() },
1982                wgpu::BindGroupEntry { binding: 3, resource: params_buf.as_entire_binding() },
1983            ],
1984        });
1985        let mut pass = encoder.begin_compute_pass(&Default::default());
1986        pass.set_pipeline(&self.tiled_matmul_pipeline);
1987        pass.set_bind_group(0, &bg, &[]);
1988        pass.dispatch_workgroups(n.div_ceil(64), m.div_ceil(64), 1);
1989    }
1990
1991    fn encode_rmsnorm(
1992        &self,
1993        encoder: &mut wgpu::CommandEncoder,
1994        input: &wgpu::Buffer,
1995        weight: &wgpu::Buffer,
1996        output: &wgpu::Buffer,
1997        dim: u32,
1998    ) {
1999        let params = rmsnorm_params(dim, self.rms_norm_eps);
2000        let params_buf = self.make_uniform(&params);
2001        let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
2002            label: None,
2003            layout: &self.elementwise_bgl,
2004            entries: &[
2005                wgpu::BindGroupEntry { binding: 0, resource: input.as_entire_binding() },
2006                wgpu::BindGroupEntry { binding: 1, resource: weight.as_entire_binding() },
2007                wgpu::BindGroupEntry { binding: 2, resource: output.as_entire_binding() },
2008                wgpu::BindGroupEntry { binding: 3, resource: params_buf.as_entire_binding() },
2009            ],
2010        });
2011        // Dispatch: (1, num_rows, 1). Each workgroup processes one row via wg_id.y.
2012        // For inference (M=1): dispatch (1,1,1). For training (M=seq_len): dispatch (1,seq_len,1).
2013        let num_rows = (input.size() / (dim as u64 * 4)).max(1) as u32;
2014        let mut pass = encoder.begin_compute_pass(&Default::default());
2015        pass.set_pipeline(&self.rmsnorm_pipeline);
2016        pass.set_bind_group(0, &bg, &[]);
2017        pass.dispatch_workgroups(1, num_rows, 1);
2018    }
2019
2020    fn encode_matmul(
2021        &self,
2022        encoder: &mut wgpu::CommandEncoder,
2023        input: &wgpu::Buffer,
2024        layer_prefix: &str,
2025        proj_name: &str,
2026        output: &wgpu::Buffer,
2027        m: u32,
2028        k: u32,
2029        n: u32,
2030    ) {
2031        // C-WGPU-Q4K-001: Try Q4K GEMV first for M=1 decode (7x less VRAM)
2032        if m == 1 && self.encode_q4k_gemv(encoder, input, output, layer_prefix, proj_name, n, k) {
2033            return;
2034        }
2035        let weight_key = format!("{layer_prefix}.{proj_name}");
2036        let weight = match self.weight_buffers.get(&weight_key) {
2037            Some(w) => w,
2038            None => return, // Skip missing weights silently
2039        };
2040        // PMAT-346: GEMV and matmul have different uniform struct layouts.
2041        // GEMV: Params { n (output dim), k (input dim), _, _ }
2042        // Matmul: Dimensions { M, K, N, _ }
2043        // Tiled GEMM: Dimensions { M, K, N, alpha_bits }
2044        let params = if m == 1 { [n, k, 0u32, 0u32] } else { [m, k, n, 1.0_f32.to_bits()] };
2045        let params_buf = self.make_uniform(&params);
2046        let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
2047            label: None,
2048            layout: &self.matmul_bgl,
2049            entries: &[
2050                wgpu::BindGroupEntry { binding: 0, resource: input.as_entire_binding() },
2051                wgpu::BindGroupEntry { binding: 1, resource: weight.as_entire_binding() },
2052                wgpu::BindGroupEntry { binding: 2, resource: output.as_entire_binding() },
2053                wgpu::BindGroupEntry { binding: 3, resource: params_buf.as_entire_binding() },
2054            ],
2055        });
2056        let mut pass = encoder.begin_compute_pass(&Default::default());
2057        if m == 1 {
2058            // PMAT-327: GEMV for M=1 — cooperative K-reduction, N workgroups
2059            pass.set_pipeline(&self.gemv_pipeline);
2060            pass.set_bind_group(0, &bg, &[]);
2061            pass.dispatch_workgroups(n, 1, 1);
2062        } else if m >= 4 {
2063            // CUTLASS-style tiled GEMM for M>=4 (training batch, prefill)
2064            // 64×64 tiles, 4×4 thread micro-tiles, 10-30x faster than naive
2065            pass.set_pipeline(&self.tiled_matmul_pipeline);
2066            pass.set_bind_group(0, &bg, &[]);
2067            pass.dispatch_workgroups(n.div_ceil(64), m.div_ceil(64), 1);
2068        } else {
2069            // Naive 16×16 GEMM for small M (2-3)
2070            pass.set_pipeline(&self.matmul_pipeline);
2071            pass.set_bind_group(0, &bg, &[]);
2072            pass.dispatch_workgroups(m.div_ceil(16), n.div_ceil(16), 1);
2073        }
2074    }
2075
2076    /// C-WGPU-Q4K-001: Encode Q4K GEMV — reads raw Q4K weight bytes, dequantizes on-the-fly.
2077    /// Falls back to F32 GEMV if no Q4K weight found for this layer.
2078    /// Returns true if Q4K path was used.
2079    fn encode_q4k_gemv(
2080        &self,
2081        encoder: &mut wgpu::CommandEncoder,
2082        input: &wgpu::Buffer,
2083        output: &wgpu::Buffer,
2084        layer_prefix: &str,
2085        proj_name: &str,
2086        n: u32,
2087        k: u32,
2088    ) -> bool {
2089        let weight_key = format!("{layer_prefix}.{proj_name}");
2090        let weight = match self.q4k_weights.get(&weight_key) {
2091            Some(w) => w,
2092            None => return false,
2093        };
2094        let num_superblocks = (k + 255) / 256;
2095        let params = [n, k, num_superblocks, 0u32];
2096        let params_buf = self.make_uniform(&params);
2097        let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
2098            label: None,
2099            layout: &self.matmul_bgl,
2100            entries: &[
2101                wgpu::BindGroupEntry { binding: 0, resource: input.as_entire_binding() },
2102                wgpu::BindGroupEntry { binding: 1, resource: weight.as_entire_binding() },
2103                wgpu::BindGroupEntry { binding: 2, resource: output.as_entire_binding() },
2104                wgpu::BindGroupEntry { binding: 3, resource: params_buf.as_entire_binding() },
2105            ],
2106        });
2107        let mut pass = encoder.begin_compute_pass(&Default::default());
2108        pass.set_pipeline(&self.q4k_gemv_pipeline);
2109        pass.set_bind_group(0, &bg, &[]);
2110        pass.dispatch_workgroups(n, 1, 1);
2111        true
2112    }
2113
2114    fn encode_silu_mul(
2115        &self,
2116        encoder: &mut wgpu::CommandEncoder,
2117        gate: &wgpu::Buffer,
2118        up: &wgpu::Buffer,
2119        output: &wgpu::Buffer,
2120        dim: u32,
2121    ) {
2122        let params = [dim, 0u32, 0, 0];
2123        let params_buf = self.make_uniform(&params);
2124        let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
2125            label: None,
2126            layout: &self.elementwise_bgl,
2127            entries: &[
2128                wgpu::BindGroupEntry { binding: 0, resource: gate.as_entire_binding() },
2129                wgpu::BindGroupEntry { binding: 1, resource: up.as_entire_binding() },
2130                wgpu::BindGroupEntry { binding: 2, resource: output.as_entire_binding() },
2131                wgpu::BindGroupEntry { binding: 3, resource: params_buf.as_entire_binding() },
2132            ],
2133        });
2134        let mut pass = encoder.begin_compute_pass(&Default::default());
2135        pass.set_pipeline(&self.silu_mul_pipeline);
2136        pass.set_bind_group(0, &bg, &[]);
2137        pass.dispatch_workgroups(dim.div_ceil(256), 1, 1);
2138    }
2139
2140    fn encode_residual(
2141        &self,
2142        encoder: &mut wgpu::CommandEncoder,
2143        a: &wgpu::Buffer,
2144        b: &wgpu::Buffer,
2145        output: &wgpu::Buffer,
2146        dim: u32,
2147    ) {
2148        let params = [dim, 0u32, 0, 0];
2149        let params_buf = self.make_uniform(&params);
2150        let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
2151            label: None,
2152            layout: &self.elementwise_bgl,
2153            entries: &[
2154                wgpu::BindGroupEntry { binding: 0, resource: a.as_entire_binding() },
2155                wgpu::BindGroupEntry { binding: 1, resource: b.as_entire_binding() },
2156                wgpu::BindGroupEntry { binding: 2, resource: output.as_entire_binding() },
2157                wgpu::BindGroupEntry { binding: 3, resource: params_buf.as_entire_binding() },
2158            ],
2159        });
2160        let mut pass = encoder.begin_compute_pass(&Default::default());
2161        pass.set_pipeline(&self.residual_pipeline);
2162        pass.set_bind_group(0, &bg, &[]);
2163        pass.dispatch_workgroups(dim.div_ceil(256), 1, 1);
2164    }
2165
2166    fn make_uniform(&self, data: &[u32; 4]) -> wgpu::Buffer {
2167        use wgpu::util::DeviceExt;
2168        self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
2169            label: None,
2170            contents: bytemuck::cast_slice(data),
2171            usage: wgpu::BufferUsages::UNIFORM,
2172        })
2173    }
2174}
2175
2176fn bgl_storage(binding: u32, read_only: bool) -> wgpu::BindGroupLayoutEntry {
2177    wgpu::BindGroupLayoutEntry {
2178        binding,
2179        visibility: wgpu::ShaderStages::COMPUTE,
2180        ty: wgpu::BindingType::Buffer {
2181            ty: wgpu::BufferBindingType::Storage { read_only },
2182            has_dynamic_offset: false,
2183            min_binding_size: None,
2184        },
2185        count: None,
2186    }
2187}
2188
2189fn bgl_uniform(binding: u32) -> wgpu::BindGroupLayoutEntry {
2190    wgpu::BindGroupLayoutEntry {
2191        binding,
2192        visibility: wgpu::ShaderStages::COMPUTE,
2193        ty: wgpu::BindingType::Buffer {
2194            ty: wgpu::BufferBindingType::Uniform,
2195            has_dynamic_offset: false,
2196            min_binding_size: None,
2197        },
2198        count: None,
2199    }
2200}
2201
2202#[cfg(test)]
2203mod rmsnorm_eps_tests {
2204    use super::{rmsnorm_params, DEFAULT_RMS_NORM_EPS};
2205
2206    /// #4056: the shader reads eps from `params.y`; it must carry the configured
2207    /// value bit-exactly, not the old hardcoded 1e-6.
2208    #[test]
2209    fn rmsnorm_params_carry_eps_bits() {
2210        let p = rmsnorm_params(896, 1e-5);
2211        assert_eq!(p[0], 896);
2212        assert_eq!(f32::from_bits(p[1]), 1e-5);
2213        assert_ne!(f32::from_bits(p[1]), DEFAULT_RMS_NORM_EPS);
2214        assert_eq!(f32::from_bits(rmsnorm_params(1, DEFAULT_RMS_NORM_EPS)[1]), 1e-6);
2215    }
2216
2217    /// #4056: the shader now reads `bitcast<f32>(params.y)`; naga must accept it.
2218    /// Skips (returns) when no GPU adapter is present.
2219    #[test]
2220    fn rmsnorm_shader_validates_on_device() {
2221        let Ok(gpu) = crate::backends::gpu::GpuDevice::new() else {
2222            return;
2223        };
2224        gpu.device.push_error_scope(wgpu::ErrorFilter::Validation);
2225        let _module = gpu.device.create_shader_module(wgpu::ShaderModuleDescriptor {
2226            label: Some("rmsnorm_eps_test"),
2227            source: wgpu::ShaderSource::Wgsl(super::RMSNORM_SHADER.into()),
2228        });
2229        let err = pollster::block_on(gpu.device.pop_error_scope());
2230        assert!(err.is_none(), "RMSNORM_SHADER failed validation: {err:?}");
2231    }
2232}
2233
2234#[cfg(test)]
2235mod rope_theta_tests {
2236    use super::{rope_rotate_heads, DEFAULT_ROPE_THETA};
2237
2238    fn rotated(theta: f64, position: usize) -> Vec<f32> {
2239        let mut v = vec![1.0, 0.5, 0.25, 2.0, -1.0, 0.75, 0.1, 0.9];
2240        rope_rotate_heads(&mut v, 1, 8, position, theta);
2241        v
2242    }
2243
2244    /// #5503: the rotation follows the configured base — Llama 1e4 differs from Qwen 1e6
2245    /// at position > 0, and the two agree at position 0 (angle 0).
2246    #[test]
2247    fn rope_follows_the_configured_theta() {
2248        assert_eq!(rotated(1e4, 0), rotated(1e6, 0));
2249        assert_ne!(rotated(1e4, 5), rotated(1e6, 5));
2250        assert_eq!(rotated(DEFAULT_ROPE_THETA, 5), rotated(1e6, 5));
2251    }
2252
2253    /// Pair i rotates by position * theta^(-2i/d): checked against the closed form.
2254    #[test]
2255    fn rope_matches_the_closed_form() {
2256        let mut v = vec![1.0, 0.0, 0.0, 0.0];
2257        rope_rotate_heads(&mut v, 1, 4, 3, 1e4);
2258        let a = 3.0f64; // i = 0: theta^0 = 1
2259        assert!((f64::from(v[0]) - a.cos()).abs() < 1e-6);
2260        assert!((f64::from(v[2]) - a.sin()).abs() < 1e-6);
2261    }
2262}