ferrotorch-diffusion 0.5.8

#![cfg(feature = "cuda")]
//! GPU UNet2DConditionModel forward path for SD-1.5.
//!
//! [`GpuUNet2DConditional`] mirrors [`crate::unet::UNet2DConditionModel`]
//! op-for-op, resident in VRAM. Every conv / group-norm / layer-norm /
//! linear / attention / SiLU / GELU / upsample call is dispatched
//! through the matching `ferrotorch-gpu` kernel; the only host-side
//! traffic is the one-shot weight upload at construction, the small
//! `Timesteps` sinusoidal encoding (parameter-free arithmetic, ~640 f32
//! per call), the up-block skip-cat host bounce (correctness first; the
//! buffer counts are tiny vs. the rest of the forward) and the final
//! download at [`Self::forward`].
//!
//! The forward path is the SD-1.5 UNet topology described in
//! [`crate::unet`]:
//!
//! ```text
//! t  = time_proj(timestep)
//! t  = time_embedding(t)                    # SiLU + 2 Linear
//! h  = conv_in(sample)
//! skips = [h]
//! for db in down_blocks:                    # 3× CrossAttnDown + 1× Down
//!     h, block_skips = db(h, t, ehs)
//!     skips.extend(block_skips)
//! h  = mid_block(h, t, ehs)                 # resnet, attn, resnet
//! for ub in up_blocks:                      # 1× Up + 3× CrossAttnUp
//!     h = ub(h, popped_skips, t, ehs)
//! h  = silu(conv_norm_out(h))
//! eps = conv_out(h)                         # [B, 4, H, W]
//! ```
//!
//! **Critical architectural detail** (from the CPU code, see
//! [`crate::unet::CrossAttnDownBlock2D::new`]'s comment): the diffusers
//! `attention_head_dim` config entry is actually the **number of
//! heads**, not the per-head dimension. The per-head dim is computed as
//! `out_channels / num_heads`. This GPU code uses the same convention.
//!
//! Attention activations:
//! - SiLU for resnets and the time-embedding MLP.
//! - GELU (exact erf, not the tanh approx, not QuickGELU) for the
//!   GEGLU `FeedForward` inside each `BasicTransformerBlock`.

use ferrotorch_core::{FerrotorchError, FerrotorchResult, Tensor, TensorStorage};
use ferrotorch_gpu::{
    CudaBuffer, GpuDevice, GpuError, gpu_bmm_f32, gpu_conv2d_f32, gpu_group_norm_f32,
    gpu_layernorm, gpu_matmul_f32, gpu_nearest_upsample2x_f32, gpu_softmax,
    kernels::{gpu_add, gpu_broadcast_add, gpu_gelu_erf, gpu_scale, gpu_silu},
    transfer::{cpu_to_gpu, gpu_to_cpu},
};
use ferrotorch_nn::module::{Module, StateDict};

use crate::safetensors_loader::DropReport;
use crate::time_embedding::Timesteps;
use crate::unet::UNet2DConditionModel;
use crate::unet_config::UNet2DConditionConfig;

// ---------------------------------------------------------------------------
// Per-layer buffer bundles
// ---------------------------------------------------------------------------

/// `Conv2d(in, out, kernel, stride, padding, bias=true)` resident on the
/// GPU. Weight shape `[out, in, kH, kW]` (PyTorch convention).
#[derive(Debug)]
struct GpuConv2d {
    weight: CudaBuffer<f32>,
    bias: CudaBuffer<f32>,
    in_channels: usize,
    out_channels: usize,
    kernel: (usize, usize),
    stride: (usize, usize),
    padding: (usize, usize),
}

/// `GroupNorm(num_groups, num_channels, eps, affine=true)` resident on
/// the GPU.
#[derive(Debug)]
struct GpuGroupNorm {
    weight: CudaBuffer<f32>,
    bias: CudaBuffer<f32>,
    num_groups: usize,
    num_channels: usize,
    eps: f32,
}

/// `LayerNorm(normalized_shape=[features], eps, affine=true)` resident
/// on the GPU.
#[derive(Debug)]
struct GpuLayerNorm {
    weight: CudaBuffer<f32>,
    bias: CudaBuffer<f32>,
    normalized_shape: usize,
    eps: f32,
}

/// `Linear(in, out, bias)`, stored as `W^T` so a row-major matmul
/// `[m, in] @ [in, out] -> [m, out]` is `(W^T)`-then-add-bias. Matches
/// the F.2 CLIP convention.
#[derive(Debug)]
struct GpuLinearT {
    weight_t: CudaBuffer<f32>,
    bias: Option<CudaBuffer<f32>>,
    in_features: usize,
    out_features: usize,
}

/// `ResnetBlock2DTime`: GN -> SiLU -> Conv -> (+temb_proj) -> GN ->
/// SiLU -> Conv + residual (optional 1x1 shortcut).
#[derive(Debug)]
struct GpuResnetTime {
    norm1: GpuGroupNorm,
    conv1: GpuConv2d,
    time_emb_proj: GpuLinearT,
    norm2: GpuGroupNorm,
    conv2: GpuConv2d,
    /// Present iff `in_channels != out_channels`.
    conv_shortcut: Option<GpuConv2d>,
    in_channels: usize,
    out_channels: usize,
}

/// One head of multi-head attention. `to_q` / `to_k` / `to_v` carry the
/// (possibly-different) `kv_dim` channel count for cross-attn; for
/// self-attn `kv_dim == query_dim`. SD-1.5 sets `bias=False` on
/// q/k/v and `bias=True` on `to_out.0`.
#[derive(Debug)]
struct GpuAttention {
    to_q: GpuLinearT,
    to_k: GpuLinearT,
    to_v: GpuLinearT,
    to_out_0: GpuLinearT,
    heads: usize,
    dim_head: usize,
    inner_dim: usize,
}

/// `FeedForward` (GEGLU): `Linear(dim -> 2*dim_ff) -> chunk(2) -> x *
/// gelu(gate) -> Linear(dim_ff -> dim)`.
#[derive(Debug)]
struct GpuFeedForwardGEGLU {
    net_0_proj: GpuLinearT,
    net_2: GpuLinearT,
    dim: usize,
    dim_ff: usize,
}

/// One `BasicTransformerBlock`: pre-LN -> self-attn -> residual ->
/// pre-LN -> cross-attn -> residual -> pre-LN -> GEGLU FF -> residual.
#[derive(Debug)]
struct GpuBasicTransformerBlock {
    norm1: GpuLayerNorm,
    attn1: GpuAttention,
    norm2: GpuLayerNorm,
    attn2: GpuAttention,
    norm3: GpuLayerNorm,
    ff: GpuFeedForwardGEGLU,
    dim: usize,
}

/// `Transformer2DModel`: GroupNorm + Conv2d(k=1) + N transformer blocks
/// + Conv2d(k=1) + residual.
#[derive(Debug)]
struct GpuTransformer2D {
    norm: GpuGroupNorm,
    proj_in: GpuConv2d,
    blocks: Vec<GpuBasicTransformerBlock>,
    proj_out: GpuConv2d,
    channels: usize,
    inner_dim: usize,
}

/// `Upsample2D` for the UNet (nearest-2x then Conv2d k=3 pad=1).
#[derive(Debug)]
struct GpuUpsample2D {
    conv: GpuConv2d,
    channels: usize,
}

/// `Downsample2D` for the UNet (Conv2d k=3 stride=2 pad=1).
#[derive(Debug)]
struct GpuDownsample2D {
    conv: GpuConv2d,
    channels: usize,
}

/// `CrossAttnDownBlock2D`: `layers_per_block` pairs of
/// (`ResnetBlock2DTime`, `Transformer2DModel`) + optional `Downsample2D`.
#[derive(Debug)]
struct GpuCrossAttnDownBlock {
    resnets: Vec<GpuResnetTime>,
    attentions: Vec<GpuTransformer2D>,
    downsampler: Option<GpuDownsample2D>,
}

/// `DownBlock2D`: `layers_per_block` resnets + optional `Downsample2D`.
#[derive(Debug)]
struct GpuDownBlock {
    resnets: Vec<GpuResnetTime>,
    downsampler: Option<GpuDownsample2D>,
}

/// Discriminated down-block.
#[derive(Debug)]
enum AnyGpuDown {
    CrossAttn(GpuCrossAttnDownBlock),
    Plain(GpuDownBlock),
}

/// `UNetMidBlock2DCrossAttn`: resnet0 -> transformer -> resnet1.
#[derive(Debug)]
struct GpuMidBlock {
    resnet0: GpuResnetTime,
    attn0: GpuTransformer2D,
    resnet1: GpuResnetTime,
}

/// `CrossAttnUpBlock2D`: `layers_per_block + 1` pairs of (resnet, attn) +
/// optional `Upsample2D`. Each resnet's input is `cat([h, skip], dim=1)`.
#[derive(Debug)]
struct GpuCrossAttnUpBlock {
    resnets: Vec<GpuResnetTime>,
    attentions: Vec<GpuTransformer2D>,
    upsampler: Option<GpuUpsample2D>,
}

/// `UpBlock2D`: `layers_per_block + 1` resnets + optional `Upsample2D`.
#[derive(Debug)]
struct GpuUpBlock {
    resnets: Vec<GpuResnetTime>,
    upsampler: Option<GpuUpsample2D>,
}

/// Discriminated up-block.
#[derive(Debug)]
enum AnyGpuUp {
    CrossAttn(GpuCrossAttnUpBlock),
    Plain(GpuUpBlock),
}

// ---------------------------------------------------------------------------
// GpuUNet2DConditional
// ---------------------------------------------------------------------------

/// SD-1.5 UNet2DConditionModel forward path resident on a single CUDA
/// device.
///
/// Constructed from a [`UNet2DConditionConfig`] and a host-side
/// [`StateDict<f32>`] (the standard diffusers `unet.*` key layout
/// produced by [`crate::load_unet`]). Every parameter tensor is
/// uploaded once into GPU memory; the host copy is dropped after
/// construction.
///
/// # Example
///
/// ```ignore
/// let device = GpuDevice::new(0)?;
/// let (cpu_unet, _drop) = load_unet::<f32>(weights, cfg.clone(), false)?;
/// let (gpu, _drop) = GpuUNet2DConditional::from_module(&cpu_unet, &device)?;
/// let noise = gpu.forward(&latent, timestep, &text_emb)?;
/// // noise: [1, 4, 64, 64]
/// ```
#[derive(Debug)]
pub struct GpuUNet2DConditional {
    time_proj: Timesteps,
    time_emb_lin1: GpuLinearT,
    time_emb_lin2: GpuLinearT,
    conv_in: GpuConv2d,
    down_blocks: Vec<AnyGpuDown>,
    mid_block: GpuMidBlock,
    up_blocks: Vec<AnyGpuUp>,
    conv_norm_out: GpuGroupNorm,
    conv_out: GpuConv2d,
    config: UNet2DConditionConfig,
    device: GpuDevice,
}

impl GpuUNet2DConditional {
    /// Build the GPU UNet from a config + state-dict.
    ///
    /// The state-dict is expected in the same shape as the CPU
    /// `UNet2DConditionModel` produces. Tensors are checked for length
    /// (the GPU code re-uses the CPU shape contract) and uploaded once
    /// to VRAM.
    ///
    /// Unmapped keys are returned via [`DropReport`].
    ///
    /// # Errors
    ///
    /// Returns:
    /// - [`FerrotorchError::InvalidArgument`] for invalid config or a
    ///   missing tensor key.
    /// - [`FerrotorchError::ShapeMismatch`] when a tensor's element
    ///   count does not match the architectural shape implied by
    ///   `config`.
    /// - Any GPU error surfaced by `cpu_to_gpu` during upload (wrapped
    ///   in `FerrotorchError::InvalidArgument`).
    pub fn new(
        config: UNet2DConditionConfig,
        mut state: StateDict<f32>,
        device: GpuDevice,
    ) -> FerrotorchResult<(Self, DropReport)> {
        config.validate()?;
        let groups = config.norm_num_groups;
        let temb_channels = config.time_embed_dim();
        let bocs = &config.block_out_channels;
        let num_blocks = bocs.len();
        let cross_dim = config.cross_attention_dim;
        let heads = config.attention_head_dim; // diffusers footgun: this is the *count* of heads.
        let transformer_layers = config.transformer_layers_per_block;
        // Resnet GroupNorm eps is `1e-5` in the CPU code; final
        // conv_norm_out is also 1e-5. Group-norm-after-attention in the
        // transformer uses 1e-6 (matching diffusers' `Transformer2DModel`
        // default).
        let resnet_eps = 1e-5_f32;
        let transformer_eps = 1e-6_f32;

        // ---- time_proj (parameter-free) + time_embedding MLP ----------
        let time_proj = Timesteps::new(bocs[0], config.flip_sin_to_cos, config.freq_shift)?;
        let time_emb_lin1 = pop_linear(
            &mut state,
            "time_embedding.linear_1",
            bocs[0],
            temb_channels,
            true,
            &device,
        )?;
        let time_emb_lin2 = pop_linear(
            &mut state,
            "time_embedding.linear_2",
            temb_channels,
            temb_channels,
            true,
            &device,
        )?;

        // ---- conv_in --------------------------------------------------
        let conv_in = pop_conv(
            &mut state,
            "conv_in",
            config.in_channels,
            bocs[0],
            (3, 3),
            (1, 1),
            (1, 1),
            true,
            &device,
        )?;

        // ---- Down blocks ----------------------------------------------
        let mut down_blocks: Vec<AnyGpuDown> = Vec::with_capacity(num_blocks);
        let mut prev = bocs[0];
        for i in 0..num_blocks {
            let out_c = bocs[i];
            let is_final = i == num_blocks - 1;
            let add_downsample = !is_final;
            let dim_head = out_c / heads;
            let block_prefix = format!("down_blocks.{i}");
            if config.down_block_has_attn[i] {
                let mut resnets = Vec::with_capacity(config.layers_per_block);
                let mut attentions = Vec::with_capacity(config.layers_per_block);
                for j in 0..config.layers_per_block {
                    let in_c = if j == 0 { prev } else { out_c };
                    resnets.push(pop_resnet_time(
                        &mut state,
                        &format!("{block_prefix}.resnets.{j}"),
                        in_c,
                        out_c,
                        temb_channels,
                        groups,
                        resnet_eps,
                        &device,
                    )?);
                    attentions.push(pop_transformer_2d(
                        &mut state,
                        &format!("{block_prefix}.attentions.{j}"),
                        out_c,
                        heads,
                        dim_head,
                        transformer_layers,
                        cross_dim,
                        groups,
                        transformer_eps,
                        &device,
                    )?);
                }
                let downsampler = if add_downsample {
                    Some(pop_downsample(
                        &mut state,
                        &format!("{block_prefix}.downsamplers.0"),
                        out_c,
                        &device,
                    )?)
                } else {
                    None
                };
                down_blocks.push(AnyGpuDown::CrossAttn(GpuCrossAttnDownBlock {
                    resnets,
                    attentions,
                    downsampler,
                }));
            } else {
                let mut resnets = Vec::with_capacity(config.layers_per_block);
                for j in 0..config.layers_per_block {
                    let in_c = if j == 0 { prev } else { out_c };
                    resnets.push(pop_resnet_time(
                        &mut state,
                        &format!("{block_prefix}.resnets.{j}"),
                        in_c,
                        out_c,
                        temb_channels,
                        groups,
                        resnet_eps,
                        &device,
                    )?);
                }
                let downsampler = if add_downsample {
                    Some(pop_downsample(
                        &mut state,
                        &format!("{block_prefix}.downsamplers.0"),
                        out_c,
                        &device,
                    )?)
                } else {
                    None
                };
                down_blocks.push(AnyGpuDown::Plain(GpuDownBlock {
                    resnets,
                    downsampler,
                }));
            }
            prev = out_c;
        }

        // ---- Mid block ------------------------------------------------
        let mid_channels = bocs[num_blocks - 1];
        let mid_dim_head = mid_channels / heads;
        let mid_resnet0 = pop_resnet_time(
            &mut state,
            "mid_block.resnets.0",
            mid_channels,
            mid_channels,
            temb_channels,
            groups,
            resnet_eps,
            &device,
        )?;
        let mid_attn0 = pop_transformer_2d(
            &mut state,
            "mid_block.attentions.0",
            mid_channels,
            heads,
            mid_dim_head,
            transformer_layers,
            cross_dim,
            groups,
            transformer_eps,
            &device,
        )?;
        let mid_resnet1 = pop_resnet_time(
            &mut state,
            "mid_block.resnets.1",
            mid_channels,
            mid_channels,
            temb_channels,
            groups,
            resnet_eps,
            &device,
        )?;
        let mid_block = GpuMidBlock {
            resnet0: mid_resnet0,
            attn0: mid_attn0,
            resnet1: mid_resnet1,
        };

        // ---- Up blocks (reversed block_out_channels) ------------------
        let mut up_blocks: Vec<AnyGpuUp> = Vec::with_capacity(num_blocks);
        let reversed: Vec<usize> = bocs.iter().rev().copied().collect();
        let mut prev_up = mid_channels;
        let up_layers = config.layers_per_block + 1;
        for i in 0..num_blocks {
            let out_c = reversed[i];
            let in_c = reversed[(i + 1).min(num_blocks - 1)];
            let is_final = i == num_blocks - 1;
            let add_upsample = !is_final;
            let dim_head = out_c / heads;
            let block_prefix = format!("up_blocks.{i}");
            // Resnet input widths: see CPU CrossAttnUpBlock2D::new.
            //   resnet[j].in_c = (prev_up if j==0 else out_c) + res_skip
            //   where res_skip = in_c if j == up_layers-1 else out_c.
            if config.up_block_has_attn[i] {
                let mut resnets = Vec::with_capacity(up_layers);
                let mut attentions = Vec::with_capacity(up_layers);
                for j in 0..up_layers {
                    let res_skip = if j == up_layers - 1 { in_c } else { out_c };
                    let resnet_in = if j == 0 {
                        prev_up + res_skip
                    } else {
                        out_c + res_skip
                    };
                    resnets.push(pop_resnet_time(
                        &mut state,
                        &format!("{block_prefix}.resnets.{j}"),
                        resnet_in,
                        out_c,
                        temb_channels,
                        groups,
                        resnet_eps,
                        &device,
                    )?);
                    attentions.push(pop_transformer_2d(
                        &mut state,
                        &format!("{block_prefix}.attentions.{j}"),
                        out_c,
                        heads,
                        dim_head,
                        transformer_layers,
                        cross_dim,
                        groups,
                        transformer_eps,
                        &device,
                    )?);
                }
                let upsampler = if add_upsample {
                    Some(pop_upsample(
                        &mut state,
                        &format!("{block_prefix}.upsamplers.0"),
                        out_c,
                        &device,
                    )?)
                } else {
                    None
                };
                up_blocks.push(AnyGpuUp::CrossAttn(GpuCrossAttnUpBlock {
                    resnets,
                    attentions,
                    upsampler,
                }));
            } else {
                let mut resnets = Vec::with_capacity(up_layers);
                for j in 0..up_layers {
                    let res_skip = if j == up_layers - 1 { in_c } else { out_c };
                    let resnet_in = if j == 0 {
                        prev_up + res_skip
                    } else {
                        out_c + res_skip
                    };
                    resnets.push(pop_resnet_time(
                        &mut state,
                        &format!("{block_prefix}.resnets.{j}"),
                        resnet_in,
                        out_c,
                        temb_channels,
                        groups,
                        resnet_eps,
                        &device,
                    )?);
                }
                let upsampler = if add_upsample {
                    Some(pop_upsample(
                        &mut state,
                        &format!("{block_prefix}.upsamplers.0"),
                        out_c,
                        &device,
                    )?)
                } else {
                    None
                };
                up_blocks.push(AnyGpuUp::Plain(GpuUpBlock { resnets, upsampler }));
            }
            prev_up = out_c;
        }

        // ---- Output head ---------------------------------------------
        let conv_norm_out = pop_groupnorm(
            &mut state,
            "conv_norm_out",
            groups,
            bocs[0],
            resnet_eps,
            &device,
        )?;
        let conv_out = pop_conv(
            &mut state,
            "conv_out",
            bocs[0],
            config.out_channels,
            (3, 3),
            (1, 1),
            (1, 1),
            true,
            &device,
        )?;

        // Whatever remains is unmapped — surface as DropReport.
        let mut dropped: Vec<String> = state.keys().cloned().collect();
        dropped.sort();
        let report = DropReport { dropped };

        Ok((
            Self {
                time_proj,
                time_emb_lin1,
                time_emb_lin2,
                conv_in,
                down_blocks,
                mid_block,
                up_blocks,
                conv_norm_out,
                conv_out,
                config,
                device,
            },
            report,
        ))
    }

    /// Convenience constructor: build a [`GpuUNet2DConditional`] from
    /// an already-loaded CPU [`UNet2DConditionModel<f32>`].
    ///
    /// Equivalent to extracting `cpu.state_dict()` and calling
    /// [`Self::new`] on a clone of the device handle.
    ///
    /// # Errors
    ///
    /// Forwards every error from [`Self::new`].
    pub fn from_module(
        cpu: &UNet2DConditionModel<f32>,
        device: &GpuDevice,
    ) -> FerrotorchResult<(Self, DropReport)> {
        let state: StateDict<f32> = cpu.state_dict();
        Self::new(cpu.config.clone(), state, device.clone())
    }

    /// Run the UNet forward.
    ///
    /// - `sample`: `[B, in_channels, H, W]` noisy latent.
    /// - `timesteps`: `[B]` diffusion timestep per batch entry (read as
    ///   f32 host-side for the sinusoidal projection).
    /// - `encoder_hidden_states`: `[B, S, cross_attention_dim]` (text
    ///   conditioning).
    ///
    /// Returns predicted noise `[B, out_channels, H, W]`.
    ///
    /// # Errors
    ///
    /// - [`FerrotorchError::ShapeMismatch`] on bad input ranks.
    /// - GPU op errors wrapped in `FerrotorchError::InvalidArgument`.
    pub fn forward(
        &self,
        sample: &Tensor<f32>,
        timesteps: &Tensor<f32>,
        encoder_hidden_states: &Tensor<f32>,
    ) -> FerrotorchResult<Tensor<f32>> {
        let cfg = &self.config;
        let s_shape = sample.shape();
        if s_shape.len() != 4 || s_shape[1] != cfg.in_channels {
            return Err(FerrotorchError::ShapeMismatch {
                message: format!(
                    "GpuUNet2DConditional::forward: expected sample [B, {}, H, W], got {:?}",
                    cfg.in_channels, s_shape
                ),
            });
        }
        if timesteps.ndim() != 1 {
            return Err(FerrotorchError::ShapeMismatch {
                message: format!(
                    "GpuUNet2DConditional::forward: expected timesteps [B], got {:?}",
                    timesteps.shape()
                ),
            });
        }
        let eh_shape = encoder_hidden_states.shape();
        if eh_shape.len() != 3 || eh_shape[2] != cfg.cross_attention_dim {
            return Err(FerrotorchError::ShapeMismatch {
                message: format!(
                    "GpuUNet2DConditional::forward: expected encoder_hidden_states \
                     [B, S, {}], got {:?}",
                    cfg.cross_attention_dim, eh_shape
                ),
            });
        }
        let b = s_shape[0];
        let h_in = s_shape[2];
        let w_in = s_shape[3];
        let s_text = eh_shape[1];

        // ---- 1. Time embedding ----------------------------------------
        //
        // `Timesteps` is parameter-free arithmetic; we let the CPU
        // module produce `[B, bocs[0]]`, then upload + MLP through
        // Linear/SiLU/Linear on the GPU.
        let t_enc = self.time_proj.forward_t(timesteps)?;
        let t_enc_data = t_enc.data()?;
        let t_enc_gpu = cpu_to_gpu(t_enc_data, &self.device).map_err(gpu_err)?;
        let t1 = linear_forward(&self.time_emb_lin1, &t_enc_gpu, b, &self.device)?;
        let t1_act = gpu_silu(&t1, &self.device).map_err(gpu_err)?;
        let temb = linear_forward(&self.time_emb_lin2, &t1_act, b, &self.device)?;
        // temb buffer is `[B, temb_channels]` flat.

        // ---- 2. conv_in -----------------------------------------------
        let sample_data = sample.data()?;
        let x_in = cpu_to_gpu(sample_data, &self.device).map_err(gpu_err)?;
        let (h0_buf, h0_shape) = conv_forward(
            &self.conv_in,
            &x_in,
            [b, cfg.in_channels, h_in, w_in],
            &self.device,
        )?;

        // Encoder hidden states uploaded once for all cross-attns.
        let ehs_data = encoder_hidden_states.data()?;
        let ehs_gpu = cpu_to_gpu(ehs_data, &self.device).map_err(gpu_err)?;

        // skips holds (buf, shape) pairs to enable cat() with the right
        // channel counts on the up-side.
        let mut skips: Vec<(CudaBuffer<f32>, [usize; 4])> = Vec::new();
        skips.push((clone_buf(&h0_buf, &self.device)?, h0_shape));

        let mut h_buf = h0_buf;
        let mut h_shape = h0_shape;

        // ---- 3. Down blocks ------------------------------------------
        for db in &self.down_blocks {
            match db {
                AnyGpuDown::CrossAttn(blk) => {
                    for (r, a) in blk.resnets.iter().zip(blk.attentions.iter()) {
                        let (rb, rs) =
                            resnet_time_forward(r, &h_buf, h_shape, &temb, b, &self.device)?;
                        h_buf = rb;
                        h_shape = rs;
                        let (ab, asz) = transformer_2d_forward(
                            a,
                            &h_buf,
                            h_shape,
                            &ehs_gpu,
                            b,
                            s_text,
                            cfg.cross_attention_dim,
                            &self.device,
                        )?;
                        h_buf = ab;
                        h_shape = asz;
                        skips.push((clone_buf(&h_buf, &self.device)?, h_shape));
                    }
                    if let Some(ds) = &blk.downsampler {
                        let (db_out, ds_shape) =
                            downsample_forward(ds, &h_buf, h_shape, &self.device)?;
                        h_buf = db_out;
                        h_shape = ds_shape;
                        skips.push((clone_buf(&h_buf, &self.device)?, h_shape));
                    }
                }
                AnyGpuDown::Plain(blk) => {
                    for r in &blk.resnets {
                        let (rb, rs) =
                            resnet_time_forward(r, &h_buf, h_shape, &temb, b, &self.device)?;
                        h_buf = rb;
                        h_shape = rs;
                        skips.push((clone_buf(&h_buf, &self.device)?, h_shape));
                    }
                    if let Some(ds) = &blk.downsampler {
                        let (db_out, ds_shape) =
                            downsample_forward(ds, &h_buf, h_shape, &self.device)?;
                        h_buf = db_out;
                        h_shape = ds_shape;
                        skips.push((clone_buf(&h_buf, &self.device)?, h_shape));
                    }
                }
            }
        }

        // ---- 4. Mid block --------------------------------------------
        let (mr0, mr0_shape) = resnet_time_forward(
            &self.mid_block.resnet0,
            &h_buf,
            h_shape,
            &temb,
            b,
            &self.device,
        )?;
        let (ma0, ma0_shape) = transformer_2d_forward(
            &self.mid_block.attn0,
            &mr0,
            mr0_shape,
            &ehs_gpu,
            b,
            s_text,
            cfg.cross_attention_dim,
            &self.device,
        )?;
        let (mr1, mr1_shape) = resnet_time_forward(
            &self.mid_block.resnet1,
            &ma0,
            ma0_shape,
            &temb,
            b,
            &self.device,
        )?;
        h_buf = mr1;
        h_shape = mr1_shape;

        // ---- 5. Up blocks --------------------------------------------
        for ub in &self.up_blocks {
            let n = match ub {
                AnyGpuUp::CrossAttn(b) => b.resnets.len(),
                AnyGpuUp::Plain(b) => b.resnets.len(),
            };
            if skips.len() < n {
                return Err(FerrotorchError::InvalidArgument {
                    message: format!(
                        "GpuUNet2DConditional: up-block needs {n} skips, only {} left",
                        skips.len()
                    ),
                });
            }
            let split_at = skips.len() - n;
            let popped: Vec<(CudaBuffer<f32>, [usize; 4])> = skips.split_off(split_at);
            // CPU code reverses popped so resnet[0] gets the most
            // recently pushed skip.
            let popped_rev: Vec<(CudaBuffer<f32>, [usize; 4])> = popped.into_iter().rev().collect();

            match ub {
                AnyGpuUp::CrossAttn(blk) => {
                    for ((r, a), (skip_buf, skip_shape)) in blk
                        .resnets
                        .iter()
                        .zip(blk.attentions.iter())
                        .zip(popped_rev.iter())
                    {
                        let (cat_buf, cat_shape) =
                            cat_channels(&h_buf, h_shape, skip_buf, *skip_shape, &self.device)?;
                        let (rb, rs) =
                            resnet_time_forward(r, &cat_buf, cat_shape, &temb, b, &self.device)?;
                        let (ab, asz) = transformer_2d_forward(
                            a,
                            &rb,
                            rs,
                            &ehs_gpu,
                            b,
                            s_text,
                            cfg.cross_attention_dim,
                            &self.device,
                        )?;
                        h_buf = ab;
                        h_shape = asz;
                    }
                    if let Some(up) = &blk.upsampler {
                        let (ub_buf, ub_shape) =
                            upsample_forward(up, &h_buf, h_shape, &self.device)?;
                        h_buf = ub_buf;
                        h_shape = ub_shape;
                    }
                }
                AnyGpuUp::Plain(blk) => {
                    for (r, (skip_buf, skip_shape)) in blk.resnets.iter().zip(popped_rev.iter()) {
                        let (cat_buf, cat_shape) =
                            cat_channels(&h_buf, h_shape, skip_buf, *skip_shape, &self.device)?;
                        let (rb, rs) =
                            resnet_time_forward(r, &cat_buf, cat_shape, &temb, b, &self.device)?;
                        h_buf = rb;
                        h_shape = rs;
                    }
                    if let Some(up) = &blk.upsampler {
                        let (ub_buf, ub_shape) =
                            upsample_forward(up, &h_buf, h_shape, &self.device)?;
                        h_buf = ub_buf;
                        h_shape = ub_shape;
                    }
                }
            }
        }

        // ---- 6. Output head: GN -> SiLU -> conv_out -------------------
        h_buf = group_norm_forward(&self.conv_norm_out, &h_buf, h_shape, &self.device)?;
        h_buf = gpu_silu(&h_buf, &self.device).map_err(gpu_err)?;
        let (out_buf, out_shape) = conv_forward(&self.conv_out, &h_buf, h_shape, &self.device)?;

        let out_data = gpu_to_cpu(&out_buf, &self.device).map_err(gpu_err)?;
        Tensor::from_storage(TensorStorage::cpu(out_data), out_shape.to_vec(), false)
    }
}

// ===========================================================================
// Internal helpers — pop_* upload weights, *_forward run ops.
// ===========================================================================

fn gpu_err(e: GpuError) -> FerrotorchError {
    FerrotorchError::InvalidArgument {
        message: format!("GpuUNet2DConditional GPU op failed: {e}"),
    }
}

/// Round-trip a CUDA buffer through the host to get an independent
/// owned copy. SD-1.5 forward has ~25 skip allocations per call, mostly
/// `B*C*H*W` floats — for the deepest skip (b=1, c=1280, h=8, w=8) the
/// copy is 320 KB. Cumulative skip-clone traffic per forward is below
/// 20 MB which is well under the noise vs. the kernel compute. Perf
/// follow-up is a device-to-device clone; correctness first.
fn clone_buf(buf: &CudaBuffer<f32>, device: &GpuDevice) -> FerrotorchResult<CudaBuffer<f32>> {
    let host = gpu_to_cpu(buf, device).map_err(gpu_err)?;
    cpu_to_gpu(&host, device).map_err(gpu_err)
}

/// Remove a key from the state-dict and upload it as a CUDA buffer.
fn pop_tensor(
    state: &mut StateDict<f32>,
    key: &str,
    expected_len: usize,
    device: &GpuDevice,
) -> FerrotorchResult<CudaBuffer<f32>> {
    let t = state
        .remove(key)
        .ok_or_else(|| FerrotorchError::InvalidArgument {
            message: format!("GpuUNet2DConditional: missing tensor {key:?}"),
        })?;
    let data = t.data()?;
    if data.len() != expected_len {
        return Err(FerrotorchError::ShapeMismatch {
            message: format!(
                "GpuUNet2DConditional: tensor {key:?} length {} != expected {expected_len}",
                data.len()
            ),
        });
    }
    cpu_to_gpu(data, device).map_err(gpu_err)
}

#[allow(clippy::too_many_arguments)]
fn pop_conv(
    state: &mut StateDict<f32>,
    prefix: &str,
    in_c: usize,
    out_c: usize,
    kernel: (usize, usize),
    stride: (usize, usize),
    padding: (usize, usize),
    bias: bool,
    device: &GpuDevice,
) -> FerrotorchResult<GpuConv2d> {
    let w_len = out_c * in_c * kernel.0 * kernel.1;
    let weight = pop_tensor(state, &format!("{prefix}.weight"), w_len, device)?;
    let bias_buf = if bias {
        pop_tensor(state, &format!("{prefix}.bias"), out_c, device)?
    } else {
        // SD UNet conv layers all carry bias; this branch is here for
        // completeness only.
        return Err(FerrotorchError::InvalidArgument {
            message: format!("GpuUNet2DConditional: conv {prefix:?} expected bias=true"),
        });
    };
    Ok(GpuConv2d {
        weight,
        bias: bias_buf,
        in_channels: in_c,
        out_channels: out_c,
        kernel,
        stride,
        padding,
    })
}

fn pop_groupnorm(
    state: &mut StateDict<f32>,
    prefix: &str,
    groups: usize,
    channels: usize,
    eps: f32,
    device: &GpuDevice,
) -> FerrotorchResult<GpuGroupNorm> {
    let weight = pop_tensor(state, &format!("{prefix}.weight"), channels, device)?;
    let bias = pop_tensor(state, &format!("{prefix}.bias"), channels, device)?;
    Ok(GpuGroupNorm {
        weight,
        bias,
        num_groups: groups,
        num_channels: channels,
        eps,
    })
}

fn pop_layernorm(
    state: &mut StateDict<f32>,
    prefix: &str,
    features: usize,
    eps: f32,
    device: &GpuDevice,
) -> FerrotorchResult<GpuLayerNorm> {
    let weight = pop_tensor(state, &format!("{prefix}.weight"), features, device)?;
    let bias = pop_tensor(state, &format!("{prefix}.bias"), features, device)?;
    Ok(GpuLayerNorm {
        weight,
        bias,
        normalized_shape: features,
        eps,
    })
}

/// Pop a PyTorch `Linear` weight (stored `[out_f, in_f]`) and upload it
/// as `W^T` (`[in_f, out_f]`) so a row-major matmul
/// `[m, in_f] @ [in_f, out_f] -> [m, out_f]` is `gpu_matmul_f32(x,
/// W_t, m, in_f, out_f)`. Mirrors the F.2 CLIP layout.
fn pop_linear(
    state: &mut StateDict<f32>,
    prefix: &str,
    in_f: usize,
    out_f: usize,
    bias: bool,
    device: &GpuDevice,
) -> FerrotorchResult<GpuLinearT> {
    let w_key = format!("{prefix}.weight");
    let w = state
        .remove(&w_key)
        .ok_or_else(|| FerrotorchError::InvalidArgument {
            message: format!("GpuUNet2DConditional: missing tensor {w_key:?}"),
        })?;
    let w_data = w.data()?;
    if w_data.len() != out_f * in_f {
        return Err(FerrotorchError::ShapeMismatch {
            message: format!(
                "GpuUNet2DConditional: tensor {w_key:?} length {} != expected {}",
                w_data.len(),
                out_f * in_f
            ),
        });
    }
    let mut wt = vec![0.0_f32; in_f * out_f];
    for o in 0..out_f {
        for i in 0..in_f {
            wt[i * out_f + o] = w_data[o * in_f + i];
        }
    }
    let weight_t = cpu_to_gpu(&wt, device).map_err(gpu_err)?;
    let bias_buf = if bias {
        Some(pop_tensor(state, &format!("{prefix}.bias"), out_f, device)?)
    } else {
        None
    };
    Ok(GpuLinearT {
        weight_t,
        bias: bias_buf,
        in_features: in_f,
        out_features: out_f,
    })
}

#[allow(clippy::too_many_arguments)]
fn pop_resnet_time(
    state: &mut StateDict<f32>,
    prefix: &str,
    in_c: usize,
    out_c: usize,
    temb_channels: usize,
    groups: usize,
    eps: f32,
    device: &GpuDevice,
) -> FerrotorchResult<GpuResnetTime> {
    let norm1 = pop_groupnorm(state, &format!("{prefix}.norm1"), groups, in_c, eps, device)?;
    let conv1 = pop_conv(
        state,
        &format!("{prefix}.conv1"),
        in_c,
        out_c,
        (3, 3),
        (1, 1),
        (1, 1),
        true,
        device,
    )?;
    let time_emb_proj = pop_linear(
        state,
        &format!("{prefix}.time_emb_proj"),
        temb_channels,
        out_c,
        true,
        device,
    )?;
    let norm2 = pop_groupnorm(
        state,
        &format!("{prefix}.norm2"),
        groups,
        out_c,
        eps,
        device,
    )?;
    let conv2 = pop_conv(
        state,
        &format!("{prefix}.conv2"),
        out_c,
        out_c,
        (3, 3),
        (1, 1),
        (1, 1),
        true,
        device,
    )?;
    let conv_shortcut = if in_c == out_c {
        None
    } else {
        Some(pop_conv(
            state,
            &format!("{prefix}.conv_shortcut"),
            in_c,
            out_c,
            (1, 1),
            (1, 1),
            (0, 0),
            true,
            device,
        )?)
    };
    Ok(GpuResnetTime {
        norm1,
        conv1,
        time_emb_proj,
        norm2,
        conv2,
        conv_shortcut,
        in_channels: in_c,
        out_channels: out_c,
    })
}

fn pop_attention(
    state: &mut StateDict<f32>,
    prefix: &str,
    query_dim: usize,
    cross_attention_dim: Option<usize>,
    heads: usize,
    dim_head: usize,
    device: &GpuDevice,
) -> FerrotorchResult<GpuAttention> {
    let inner_dim = heads * dim_head;
    let kv_dim = cross_attention_dim.unwrap_or(query_dim);
    // SD-1.5 sets bias=False on q/k/v.
    let to_q = pop_linear(
        state,
        &format!("{prefix}.to_q"),
        query_dim,
        inner_dim,
        false,
        device,
    )?;
    let to_k = pop_linear(
        state,
        &format!("{prefix}.to_k"),
        kv_dim,
        inner_dim,
        false,
        device,
    )?;
    let to_v = pop_linear(
        state,
        &format!("{prefix}.to_v"),
        kv_dim,
        inner_dim,
        false,
        device,
    )?;
    // to_out.0 always has bias.
    let to_out_0 = pop_linear(
        state,
        &format!("{prefix}.to_out.0"),
        inner_dim,
        query_dim,
        true,
        device,
    )?;
    let _ = query_dim;
    let _ = kv_dim;
    Ok(GpuAttention {
        to_q,
        to_k,
        to_v,
        to_out_0,
        heads,
        dim_head,
        inner_dim,
    })
}

fn pop_feedforward_geglu(
    state: &mut StateDict<f32>,
    prefix: &str,
    dim: usize,
    mult: usize,
    device: &GpuDevice,
) -> FerrotorchResult<GpuFeedForwardGEGLU> {
    let dim_ff = dim * mult;
    let net_0_proj = pop_linear(
        state,
        &format!("{prefix}.net.0.proj"),
        dim,
        2 * dim_ff,
        true,
        device,
    )?;
    let net_2 = pop_linear(state, &format!("{prefix}.net.2"), dim_ff, dim, true, device)?;
    Ok(GpuFeedForwardGEGLU {
        net_0_proj,
        net_2,
        dim,
        dim_ff,
    })
}

fn pop_basic_transformer_block(
    state: &mut StateDict<f32>,
    prefix: &str,
    dim: usize,
    heads: usize,
    dim_head: usize,
    cross_dim: usize,
    device: &GpuDevice,
) -> FerrotorchResult<GpuBasicTransformerBlock> {
    let norm1 = pop_layernorm(state, &format!("{prefix}.norm1"), dim, 1e-5_f32, device)?;
    let attn1 = pop_attention(
        state,
        &format!("{prefix}.attn1"),
        dim,
        None,
        heads,
        dim_head,
        device,
    )?;
    let norm2 = pop_layernorm(state, &format!("{prefix}.norm2"), dim, 1e-5_f32, device)?;
    let attn2 = pop_attention(
        state,
        &format!("{prefix}.attn2"),
        dim,
        Some(cross_dim),
        heads,
        dim_head,
        device,
    )?;
    let norm3 = pop_layernorm(state, &format!("{prefix}.norm3"), dim, 1e-5_f32, device)?;
    let ff = pop_feedforward_geglu(state, &format!("{prefix}.ff"), dim, 4, device)?;
    Ok(GpuBasicTransformerBlock {
        norm1,
        attn1,
        norm2,
        attn2,
        norm3,
        ff,
        dim,
    })
}

#[allow(clippy::too_many_arguments)]
fn pop_transformer_2d(
    state: &mut StateDict<f32>,
    prefix: &str,
    in_channels: usize,
    heads: usize,
    dim_head: usize,
    num_layers: usize,
    cross_dim: usize,
    groups: usize,
    eps: f32,
    device: &GpuDevice,
) -> FerrotorchResult<GpuTransformer2D> {
    let inner_dim = heads * dim_head;
    let norm = pop_groupnorm(
        state,
        &format!("{prefix}.norm"),
        groups,
        in_channels,
        eps,
        device,
    )?;
    let proj_in = pop_conv(
        state,
        &format!("{prefix}.proj_in"),
        in_channels,
        inner_dim,
        (1, 1),
        (1, 1),
        (0, 0),
        true,
        device,
    )?;
    let proj_out = pop_conv(
        state,
        &format!("{prefix}.proj_out"),
        inner_dim,
        in_channels,
        (1, 1),
        (1, 1),
        (0, 0),
        true,
        device,
    )?;
    let mut blocks = Vec::with_capacity(num_layers);
    for j in 0..num_layers {
        blocks.push(pop_basic_transformer_block(
            state,
            &format!("{prefix}.transformer_blocks.{j}"),
            inner_dim,
            heads,
            dim_head,
            cross_dim,
            device,
        )?);
    }
    Ok(GpuTransformer2D {
        norm,
        proj_in,
        blocks,
        proj_out,
        channels: in_channels,
        inner_dim,
    })
}

fn pop_upsample(
    state: &mut StateDict<f32>,
    prefix: &str,
    channels: usize,
    device: &GpuDevice,
) -> FerrotorchResult<GpuUpsample2D> {
    let conv = pop_conv(
        state,
        &format!("{prefix}.conv"),
        channels,
        channels,
        (3, 3),
        (1, 1),
        (1, 1),
        true,
        device,
    )?;
    Ok(GpuUpsample2D { conv, channels })
}

fn pop_downsample(
    state: &mut StateDict<f32>,
    prefix: &str,
    channels: usize,
    device: &GpuDevice,
) -> FerrotorchResult<GpuDownsample2D> {
    // SD-1.5 Downsample2D is Conv2d k=3 stride=2 pad=1.
    let conv = pop_conv(
        state,
        &format!("{prefix}.conv"),
        channels,
        channels,
        (3, 3),
        (2, 2),
        (1, 1),
        true,
        device,
    )?;
    Ok(GpuDownsample2D { conv, channels })
}

// ===========================================================================
// Forward op dispatchers
// ===========================================================================

fn conv_forward(
    c: &GpuConv2d,
    x: &CudaBuffer<f32>,
    shape: [usize; 4],
    device: &GpuDevice,
) -> FerrotorchResult<(CudaBuffer<f32>, [usize; 4])> {
    let (out, out_shape) = gpu_conv2d_f32(
        x,
        &c.weight,
        Some(&c.bias),
        shape,
        [c.out_channels, c.in_channels, c.kernel.0, c.kernel.1],
        c.stride,
        c.padding,
        (1, 1),
        1,
        device,
    )
    .map_err(gpu_err)?;
    Ok((out, out_shape))
}

fn group_norm_forward(
    g: &GpuGroupNorm,
    x: &CudaBuffer<f32>,
    shape: [usize; 4],
    device: &GpuDevice,
) -> FerrotorchResult<CudaBuffer<f32>> {
    let [b, c, h, w] = shape;
    if c != g.num_channels {
        return Err(FerrotorchError::ShapeMismatch {
            message: format!(
                "GpuUNet2DConditional::group_norm: expected C={}, got {}",
                g.num_channels, c
            ),
        });
    }
    gpu_group_norm_f32(
        x,
        &g.weight,
        &g.bias,
        b,
        c,
        g.num_groups,
        h * w,
        g.eps,
        device,
    )
    .map_err(gpu_err)
}

fn layer_norm_forward(
    ln: &GpuLayerNorm,
    x: &CudaBuffer<f32>,
    rows: usize,
    cols: usize,
    device: &GpuDevice,
) -> FerrotorchResult<CudaBuffer<f32>> {
    if cols != ln.normalized_shape {
        return Err(FerrotorchError::ShapeMismatch {
            message: format!(
                "GpuUNet2DConditional::layer_norm: expected cols={}, got {}",
                ln.normalized_shape, cols
            ),
        });
    }
    gpu_layernorm(x, &ln.weight, &ln.bias, rows, cols, ln.eps, device).map_err(gpu_err)
}

/// `y = x @ W_t (+ b)` for `x: [m, in_f]` → `[m, out_f]`.
fn linear_forward(
    lin: &GpuLinearT,
    x: &CudaBuffer<f32>,
    m: usize,
    device: &GpuDevice,
) -> FerrotorchResult<CudaBuffer<f32>> {
    let y = gpu_matmul_f32(
        x,
        &lin.weight_t,
        m,
        lin.in_features,
        lin.out_features,
        device,
    )
    .map_err(gpu_err)?;
    if let Some(bias) = &lin.bias {
        gpu_broadcast_add(
            &y,
            bias,
            &[m, lin.out_features],
            &[1, lin.out_features],
            &[m, lin.out_features],
            device,
        )
        .map_err(gpu_err)
    } else {
        Ok(y)
    }
}

fn resnet_time_forward(
    r: &GpuResnetTime,
    x: &CudaBuffer<f32>,
    shape: [usize; 4],
    temb: &CudaBuffer<f32>,
    b: usize,
    device: &GpuDevice,
) -> FerrotorchResult<(CudaBuffer<f32>, [usize; 4])> {
    let c_in = shape[1];
    if c_in != r.in_channels {
        return Err(FerrotorchError::ShapeMismatch {
            message: format!(
                "GpuUNet2DConditional::resnet_time: expected C_in={}, got {}",
                r.in_channels, c_in
            ),
        });
    }
    // h = silu(norm1(x)); h = conv1(h)
    let mut h_buf = group_norm_forward(&r.norm1, x, shape, device)?;
    h_buf = gpu_silu(&h_buf, device).map_err(gpu_err)?;
    let (mut hb, mut hs) = conv_forward(&r.conv1, &h_buf, shape, device)?;

    // temb: silu -> Linear(temb_channels -> out_channels) -> add [B, C', 1, 1]
    let t_act = gpu_silu(temb, device).map_err(gpu_err)?;
    let t_proj = linear_forward(&r.time_emb_proj, &t_act, b, device)?;
    // t_proj: [B, out_channels] flat. We need broadcast-add into
    // [B, out_channels, H, W] across H*W. Treat `hb` as
    // `[B, out_channels, H*W]` flat with a `[B, out_channels, 1]`
    // bias-like operand.
    let hw = hs[2] * hs[3];
    hb = gpu_broadcast_add(
        &hb,
        &t_proj,
        &[b, r.out_channels, hw],
        &[b, r.out_channels, 1],
        &[b, r.out_channels, hw],
        device,
    )
    .map_err(gpu_err)?;

    // h = silu(norm2(h)); h = conv2(h)
    hb = group_norm_forward(&r.norm2, &hb, hs, device)?;
    hb = gpu_silu(&hb, device).map_err(gpu_err)?;
    (hb, hs) = conv_forward(&r.conv2, &hb, hs, device)?;

    // Residual
    if let Some(sc) = &r.conv_shortcut {
        let (sb, _) = conv_forward(sc, x, shape, device)?;
        hb = gpu_add(&hb, &sb, device).map_err(gpu_err)?;
    } else {
        hb = gpu_add(&hb, x, device).map_err(gpu_err)?;
    }
    let _ = r.out_channels;
    Ok((hb, hs))
}

/// Attention forward.
///
/// `query_buf` holds `[b, n, query_dim]` flat. When `kv_buf` is None,
/// the kv source is the query (self-attn); otherwise `kv_buf` holds
/// `[b, s, kv_dim]` flat (cross-attn).
fn attention_forward(
    a: &GpuAttention,
    query_buf: &CudaBuffer<f32>,
    b: usize,
    n: usize,
    kv_buf: Option<&CudaBuffer<f32>>,
    s: usize,
    device: &GpuDevice,
) -> FerrotorchResult<CudaBuffer<f32>> {
    let h = a.heads;
    let d = a.dim_head;
    let inner = a.inner_dim;
    let kv_src = kv_buf.unwrap_or(query_buf);
    let s_eff = if kv_buf.is_some() { s } else { n };

    // q: [B, N, inner], k: [B, S, inner], v: [B, S, inner]
    let q = linear_forward(&a.to_q, query_buf, b * n, device)?;
    let k = linear_forward(&a.to_k, kv_src, b * s_eff, device)?;
    let v = linear_forward(&a.to_v, kv_src, b * s_eff, device)?;

    // Reshape [B, N, H, D] -> [B*H, N, D] (transpose 1<->2).
    let q_h = reshape_bnhd_to_bhnd(&q, b, n, h, d, device)?;
    let k_h = reshape_bnhd_to_bhnd(&k, b, s_eff, h, d, device)?;
    let v_h = reshape_bnhd_to_bhnd(&v, b, s_eff, h, d, device)?;

    // scores = (q_h @ k_h^T) * scale -> [B*H, N, S]
    let k_h_t = transpose_last_two(&k_h, b * h, s_eff, d, device)?;
    let scores = gpu_bmm_f32(&q_h, &k_h_t, b * h, n, d, s_eff, device).map_err(gpu_err)?;
    let scale = (d as f64).sqrt().recip() as f32;
    let scaled = gpu_scale(&scores, scale, device).map_err(gpu_err)?;
    // Softmax over last dim (S). Rows = B*H*N.
    let probs = gpu_softmax(&scaled, b * h * n, s_eff, device).map_err(gpu_err)?;
    // attended = probs @ v_h : [B*H, N, S] @ [B*H, S, D] -> [B*H, N, D]
    let attended = gpu_bmm_f32(&probs, &v_h, b * h, n, s_eff, d, device).map_err(gpu_err)?;

    // Merge heads: [B*H, N, D] -> [B, N, inner]
    let merged = reshape_bhnd_to_bnhd(&attended, b, n, h, d, device)?;
    let _ = inner;

    // to_out.0
    linear_forward(&a.to_out_0, &merged, b * n, device)
}

fn basic_transformer_block_forward(
    blk: &GpuBasicTransformerBlock,
    x: &CudaBuffer<f32>,
    b: usize,
    n: usize,
    ehs: &CudaBuffer<f32>,
    s_text: usize,
    cross_dim: usize,
    device: &GpuDevice,
) -> FerrotorchResult<CudaBuffer<f32>> {
    let _ = blk.dim; // used for asserts in pop; runtime checks via layer_norm cols.
    let dim = blk.dim;

    // Sub-block 1: self-attn (pre-LN).
    let normed1 = layer_norm_forward(&blk.norm1, x, b * n, dim, device)?;
    let attn1_out = attention_forward(&blk.attn1, &normed1, b, n, None, n, device)?;
    let x1 = gpu_add(x, &attn1_out, device).map_err(gpu_err)?;

    // Sub-block 2: cross-attn (pre-LN).
    let normed2 = layer_norm_forward(&blk.norm2, &x1, b * n, dim, device)?;
    let _ = cross_dim;
    let attn2_out = attention_forward(&blk.attn2, &normed2, b, n, Some(ehs), s_text, device)?;
    let x2 = gpu_add(&x1, &attn2_out, device).map_err(gpu_err)?;

    // Sub-block 3: GEGLU FF (pre-LN).
    let normed3 = layer_norm_forward(&blk.norm3, &x2, b * n, dim, device)?;
    let ff_out = ff_geglu_forward(&blk.ff, &normed3, b * n, device)?;
    gpu_add(&x2, &ff_out, device).map_err(gpu_err)
}

fn ff_geglu_forward(
    ff: &GpuFeedForwardGEGLU,
    x: &CudaBuffer<f32>,
    m: usize,
    device: &GpuDevice,
) -> FerrotorchResult<CudaBuffer<f32>> {
    // proj: [m, 2*dim_ff]
    let proj = linear_forward(&ff.net_0_proj, x, m, device)?;
    // chunk(2, last): first half = x, second half = gate. Implement via
    // host bounce — at SD-1.5 hot-path scales the larger transformer
    // has m = b*HW = 4096, dim_ff = 1280, so 2*4096*1280*4 = 40 MB per
    // round trip × 16 transformer layers ≈ 640 MB host traffic per
    // forward. Acceptable for correctness; perf follow-up is a
    // dedicated chunk-or-strided-split kernel.
    let host = gpu_to_cpu(&proj, device).map_err(gpu_err)?;
    let dim_ff = ff.dim_ff;
    let mut x_part = vec![0.0_f32; m * dim_ff];
    let mut gate_part = vec![0.0_f32; m * dim_ff];
    for i in 0..m {
        let row = i * 2 * dim_ff;
        x_part[i * dim_ff..(i + 1) * dim_ff].copy_from_slice(&host[row..row + dim_ff]);
        gate_part[i * dim_ff..(i + 1) * dim_ff]
            .copy_from_slice(&host[row + dim_ff..row + 2 * dim_ff]);
    }
    let x_gpu = cpu_to_gpu(&x_part, device).map_err(gpu_err)?;
    let gate_gpu = cpu_to_gpu(&gate_part, device).map_err(gpu_err)?;
    // gate' = GELU(gate) (exact erf, matching diffusers' default).
    let gate_act = gpu_gelu_erf(&gate_gpu, device).map_err(gpu_err)?;
    let activated = ferrotorch_gpu::kernels::gpu_mul(&x_gpu, &gate_act, device).map_err(gpu_err)?;
    let _ = ff.dim;
    linear_forward(&ff.net_2, &activated, m, device)
}

#[allow(clippy::too_many_arguments)]
fn transformer_2d_forward(
    t: &GpuTransformer2D,
    x: &CudaBuffer<f32>,
    shape: [usize; 4],
    ehs: &CudaBuffer<f32>,
    b_ehs: usize,
    s_text: usize,
    cross_dim: usize,
    device: &GpuDevice,
) -> FerrotorchResult<(CudaBuffer<f32>, [usize; 4])> {
    let [b, c, h, w] = shape;
    if c != t.channels {
        return Err(FerrotorchError::ShapeMismatch {
            message: format!(
                "GpuUNet2DConditional::transformer_2d: expected C={}, got {}",
                t.channels, c
            ),
        });
    }
    if b != b_ehs {
        return Err(FerrotorchError::ShapeMismatch {
            message: format!(
                "GpuUNet2DConditional::transformer_2d: batch mismatch sample B={b} vs ehs B={b_ehs}"
            ),
        });
    }
    let hw = h * w;
    let inner = t.inner_dim;

    // norm + proj_in (Conv2d k=1) keeps [B, inner, H, W]
    let normed = group_norm_forward(&t.norm, x, shape, device)?;
    let (proj_in_buf, proj_in_shape) = conv_forward(&t.proj_in, &normed, shape, device)?;

    // Reshape [B, inner, H, W] -> [B, HW, inner] via host transpose
    // (C <-> HW). Done host-side for correctness; perf follow-up.
    let mut hidden_seq = transpose_bchw_to_bnc(&proj_in_buf, b, inner, hw, device)?;

    for block in &t.blocks {
        hidden_seq = basic_transformer_block_forward(
            block,
            &hidden_seq,
            b,
            hw,
            ehs,
            s_text,
            cross_dim,
            device,
        )?;
    }

    // Back to spatial: [B, HW, inner] -> [B, inner, H, W]
    let hidden_back = transpose_bnc_to_bchw(&hidden_seq, b, inner, hw, device)?;
    let (proj_out_buf, _) = conv_forward(&t.proj_out, &hidden_back, [b, inner, h, w], device)?;

    // Residual is `x` (not `normed`) — matches diffusers.
    let summed = gpu_add(&proj_out_buf, x, device).map_err(gpu_err)?;
    Ok((summed, proj_in_shape))
}

fn upsample_forward(
    u: &GpuUpsample2D,
    x: &CudaBuffer<f32>,
    shape: [usize; 4],
    device: &GpuDevice,
) -> FerrotorchResult<(CudaBuffer<f32>, [usize; 4])> {
    let [b, c, h, w] = shape;
    if c != u.channels {
        return Err(FerrotorchError::ShapeMismatch {
            message: format!(
                "GpuUNet2DConditional::upsample: expected C={}, got {}",
                u.channels, c
            ),
        });
    }
    let upsampled = gpu_nearest_upsample2x_f32(x, b, c, h, w, device).map_err(gpu_err)?;
    let new_shape = [b, c, h * 2, w * 2];
    conv_forward(&u.conv, &upsampled, new_shape, device)
}

fn downsample_forward(
    d: &GpuDownsample2D,
    x: &CudaBuffer<f32>,
    shape: [usize; 4],
    device: &GpuDevice,
) -> FerrotorchResult<(CudaBuffer<f32>, [usize; 4])> {
    let [_, c, _, _] = shape;
    if c != d.channels {
        return Err(FerrotorchError::ShapeMismatch {
            message: format!(
                "GpuUNet2DConditional::downsample: expected C={}, got {}",
                d.channels, c
            ),
        });
    }
    conv_forward(&d.conv, x, shape, device)
}

// ---------------------------------------------------------------------------
// Shape utilities (host bounces — correctness first; perf is a separate
// follow-up via dedicated transpose / chunk kernels).
// ---------------------------------------------------------------------------

/// `[B, N, H*D]` -> `[B*H, N, D]` (split heads, transpose 1<->2).
fn reshape_bnhd_to_bhnd(
    x: &CudaBuffer<f32>,
    b: usize,
    n: usize,
    h: usize,
    d: usize,
    device: &GpuDevice,
) -> FerrotorchResult<CudaBuffer<f32>> {
    let host = gpu_to_cpu(x, device).map_err(gpu_err)?;
    let mut out = vec![0.0_f32; b * h * n * d];
    for bi in 0..b {
        for ni in 0..n {
            for hi in 0..h {
                for di in 0..d {
                    let src = ((bi * n + ni) * h + hi) * d + di;
                    let dst = ((bi * h + hi) * n + ni) * d + di;
                    out[dst] = host[src];
                }
            }
        }
    }
    cpu_to_gpu(&out, device).map_err(gpu_err)
}

/// Inverse of `reshape_bnhd_to_bhnd`: `[B*H, N, D]` -> `[B, N, H*D]`.
fn reshape_bhnd_to_bnhd(
    x: &CudaBuffer<f32>,
    b: usize,
    n: usize,
    h: usize,
    d: usize,
    device: &GpuDevice,
) -> FerrotorchResult<CudaBuffer<f32>> {
    let host = gpu_to_cpu(x, device).map_err(gpu_err)?;
    let mut out = vec![0.0_f32; b * n * h * d];
    for bi in 0..b {
        for hi in 0..h {
            for ni in 0..n {
                for di in 0..d {
                    let src = ((bi * h + hi) * n + ni) * d + di;
                    let dst = ((bi * n + ni) * h + hi) * d + di;
                    out[dst] = host[src];
                }
            }
        }
    }
    cpu_to_gpu(&out, device).map_err(gpu_err)
}

/// `[batch, m, n]` -> `[batch, n, m]` via host bounce.
fn transpose_last_two(
    x: &CudaBuffer<f32>,
    batch: usize,
    m: usize,
    n: usize,
    device: &GpuDevice,
) -> FerrotorchResult<CudaBuffer<f32>> {
    let host = gpu_to_cpu(x, device).map_err(gpu_err)?;
    let mut out = vec![0.0_f32; batch * n * m];
    for bi in 0..batch {
        for mi in 0..m {
            for ni in 0..n {
                let src = bi * m * n + mi * n + ni;
                let dst = bi * n * m + ni * m + mi;
                out[dst] = host[src];
            }
        }
    }
    cpu_to_gpu(&out, device).map_err(gpu_err)
}

/// `[B, C, HW]` interpreted as `[B, C, H, W]` flattened to `[B, HW, C]`.
fn transpose_bchw_to_bnc(
    x: &CudaBuffer<f32>,
    b: usize,
    c: usize,
    hw: usize,
    device: &GpuDevice,
) -> FerrotorchResult<CudaBuffer<f32>> {
    let host = gpu_to_cpu(x, device).map_err(gpu_err)?;
    let mut out = vec![0.0_f32; b * hw * c];
    for bi in 0..b {
        for ci in 0..c {
            for hwi in 0..hw {
                let src = (bi * c + ci) * hw + hwi;
                let dst = (bi * hw + hwi) * c + ci;
                out[dst] = host[src];
            }
        }
    }
    cpu_to_gpu(&out, device).map_err(gpu_err)
}

/// Inverse: `[B, HW, C]` -> `[B, C, HW]`.
fn transpose_bnc_to_bchw(
    x: &CudaBuffer<f32>,
    b: usize,
    c: usize,
    hw: usize,
    device: &GpuDevice,
) -> FerrotorchResult<CudaBuffer<f32>> {
    let host = gpu_to_cpu(x, device).map_err(gpu_err)?;
    let mut out = vec![0.0_f32; b * c * hw];
    for bi in 0..b {
        for hwi in 0..hw {
            for ci in 0..c {
                let src = (bi * hw + hwi) * c + ci;
                let dst = (bi * c + ci) * hw + hwi;
                out[dst] = host[src];
            }
        }
    }
    cpu_to_gpu(&out, device).map_err(gpu_err)
}

/// Concatenate `[B, C1, H, W]` and `[B, C2, H, W]` along the channel
/// axis, producing `[B, C1+C2, H, W]`. Implemented via host bounce —
/// the up-side cat traffic per forward is ~40 MB cumulative which is
/// not the bottleneck.
fn cat_channels(
    a: &CudaBuffer<f32>,
    a_shape: [usize; 4],
    b: &CudaBuffer<f32>,
    b_shape: [usize; 4],
    device: &GpuDevice,
) -> FerrotorchResult<(CudaBuffer<f32>, [usize; 4])> {
    let [ba, ca, ha, wa] = a_shape;
    let [bb, cb, hb, wb] = b_shape;
    if ba != bb || ha != hb || wa != wb {
        return Err(FerrotorchError::ShapeMismatch {
            message: format!(
                "GpuUNet2DConditional::cat_channels: shape disagree {a_shape:?} vs {b_shape:?}"
            ),
        });
    }
    let a_host = gpu_to_cpu(a, device).map_err(gpu_err)?;
    let b_host = gpu_to_cpu(b, device).map_err(gpu_err)?;
    let c_out = ca + cb;
    let hw = ha * wa;
    let mut out = vec![0.0_f32; ba * c_out * hw];
    for bi in 0..ba {
        // a slice [ca, hw]
        for ci in 0..ca {
            let src = (bi * ca + ci) * hw;
            let dst = (bi * c_out + ci) * hw;
            out[dst..dst + hw].copy_from_slice(&a_host[src..src + hw]);
        }
        // b slice [cb, hw]
        for ci in 0..cb {
            let src = (bi * cb + ci) * hw;
            let dst = (bi * c_out + ca + ci) * hw;
            out[dst..dst + hw].copy_from_slice(&b_host[src..src + hw]);
        }
    }
    let out_gpu = cpu_to_gpu(&out, device).map_err(gpu_err)?;
    Ok((out_gpu, [ba, c_out, ha, wa]))
}

// ===========================================================================
// Tests
// ===========================================================================

#[cfg(all(test, feature = "cuda"))]
mod tests {
    use super::*;
    use crate::unet::UNet2DConditionModel;

    /// Tiny config: 4-block topology mirroring SD-1.5's down/up has-attn
    /// pattern but with small channels so the GPU↔CPU compare runs
    /// quickly. Exercises every op shape: CrossAttn down, Plain down,
    /// Mid resnet/attn/resnet, Plain up, CrossAttn up, skip cat, time
    /// embedding, conv_in, conv_norm_out + SiLU + conv_out.
    fn tiny_cfg() -> UNet2DConditionConfig {
        UNet2DConditionConfig {
            in_channels: 4,
            out_channels: 4,
            block_out_channels: vec![16, 32, 64, 64],
            layers_per_block: 1,
            attention_head_dim: 8,
            cross_attention_dim: 24,
            norm_num_groups: 4,
            sample_size: 8,
            flip_sin_to_cos: true,
            freq_shift: 0.0,
            transformer_layers_per_block: 1,
            down_block_has_attn: vec![true, true, true, false],
            up_block_has_attn: vec![false, true, true, true],
        }
    }

    #[test]
    fn gpu_unet_matches_cpu_tiny() {
        let Ok(device) = GpuDevice::new(0) else {
            return;
        };
        let cfg = tiny_cfg();
        let cpu = UNet2DConditionModel::<f32>::new(cfg.clone()).unwrap();
        let (gpu, report) = GpuUNet2DConditional::from_module(&cpu, &device).unwrap();
        assert!(
            report.dropped.is_empty(),
            "unexpected dropped keys: {:?}",
            report.dropped
        );

        let b = 1usize;
        let h_in = 8usize;
        let w_in = 8usize;
        let sample_data: Vec<f32> = (0..b * cfg.in_channels * h_in * w_in)
            .map(|i| ((i % 7) as f32) * 0.03 - 0.05)
            .collect();
        let sample = Tensor::from_storage(
            TensorStorage::cpu(sample_data),
            vec![b, cfg.in_channels, h_in, w_in],
            false,
        )
        .unwrap();
        let timesteps =
            Tensor::from_storage(TensorStorage::cpu(vec![5.0f32]), vec![b], false).unwrap();
        let s = 7usize;
        let ehs_data: Vec<f32> = (0..b * s * cfg.cross_attention_dim)
            .map(|i| ((i % 11) as f32) * 0.02 - 0.07)
            .collect();
        let ehs = Tensor::from_storage(
            TensorStorage::cpu(ehs_data),
            vec![b, s, cfg.cross_attention_dim],
            false,
        )
        .unwrap();

        let cpu_out = cpu.forward_t(&sample, &timesteps, &ehs).unwrap();
        let gpu_out = gpu.forward(&sample, &timesteps, &ehs).unwrap();
        assert_eq!(cpu_out.shape(), gpu_out.shape());
        let cpu_data = cpu_out.data().unwrap();
        let gpu_data = gpu_out.data().unwrap();
        let mut max_abs = 0.0_f32;
        for (a, c) in cpu_data.iter().zip(gpu_data.iter()) {
            let d = (a - c).abs();
            if d > max_abs {
                max_abs = d;
            }
        }
        assert!(max_abs < 1e-3, "gpu vs cpu tiny UNet max_abs = {max_abs}");
    }
}