use crate::gpu_model::{Act, Gpu};
use crate::gpu_transformer::{GpuContext, gemm_plain_batched, gemm_rhs_transposed_batched};
use crate::matrix::Matrix;
use crate::network::NetworkError;
use crate::t5::T5Encoder;
use cudarc::driver::CudaSlice;
use std::sync::Arc;
struct DeviceBlock {
attention_norm: CudaSlice<f32>,
qkv: Act,
output: Act,
mlp_norm: CudaSlice<f32>,
gate_up: Act,
mlp_out: Act,
}
pub struct DeviceT5 {
context: Arc<GpuContext>,
blocks: Vec<DeviceBlock>,
final_norm: CudaSlice<f32>,
narrow: bool,
}
impl std::fmt::Debug for DeviceT5 {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DeviceT5").finish_non_exhaustive()
}
}
impl DeviceT5 {
pub fn upload(context: &Arc<GpuContext>, encoder: &T5Encoder) -> Result<Self, NetworkError> {
if !encoder.config.gated {
return Err(NetworkError::InvalidConfig(
"the original ungated T5 has no device path; run it on the host".into(),
));
}
if context.half_accumulate {
return Err(NetworkError::InvalidConfig(
"T5 overflows FP16; use a BF16 context".into(),
));
}
let gpu = Gpu { context };
let pack = |parts: &[&crate::conv::Dense]| -> Result<Act, NetworkError> {
let inner = parts[0].in_dim();
let mut values = Vec::new();
for part in parts {
values.extend_from_slice(&part.weight_f32().data);
}
let units = values.len() / inner;
let wide = gpu.upload(&values)?;
match parts.iter().any(|part| part.is_quantized()) {
true => gpu.quantized(&wide.slice(..), units, inner),
false => gpu.narrowed(&wide.slice(..), units * inner, true),
}
};
let blocks = encoder
.blocks
.iter()
.map(|block| {
let gate = block.gate.as_ref().expect("checked gated above");
Ok(DeviceBlock {
attention_norm: gpu.upload(&block.attention_norm)?,
qkv: pack(&[&block.query, &block.key, &block.value])?,
output: pack(&[&block.output])?,
mlp_norm: gpu.upload(&block.mlp_norm)?,
gate_up: pack(&[&block.mlp_in, gate])?,
mlp_out: pack(&[&block.mlp_out])?,
})
})
.collect::<Result<_, NetworkError>>()?;
Ok(Self {
narrow: !encoder
.blocks
.iter()
.any(|block| block.query.is_quantized()),
final_norm: gpu.upload(&encoder.final_norm)?,
context: context.clone(),
blocks,
})
}
pub fn forward(&self, encoder: &T5Encoder, ids: &[u32]) -> Result<Matrix, NetworkError> {
let gpu = Gpu {
context: &self.context,
};
let config = &encoder.config;
let (tokens, d_model) = (ids.len(), config.d_model);
let (heads, head_dim) = (config.num_heads, config.head_dim);
let inner = heads * head_dim;
let width = config.mlp_dim;
let mut hidden = gpu.upload(&encoder.embed(ids)?.data)?;
let upload_bias = |layer: usize| gpu.upload(&encoder.bias(layer, tokens).concat());
let shared = match encoder.position_bias.len() {
1 => Some(upload_bias(0)?),
_ => None,
};
let square = tokens * tokens;
for (index, block) in self.blocks.iter().enumerate() {
let own;
let bias = match &shared {
Some(bias) => bias,
None => {
own = upload_bias(index)?;
&own
}
};
let (normed, _, _) = gpu.rmsnorm(
&hidden,
&block.attention_norm.as_view(),
tokens,
d_model,
config.eps,
self.narrow,
false,
)?;
let mut qkv = gpu.uninit(tokens * 3 * inner)?;
gpu.project(
&block.qkv,
3 * inner,
d_model,
&normed.all(),
self.narrow,
&mut qkv.slice_mut(..),
tokens,
0.0,
)?;
let mut scores = gpu.uninit(heads * square)?;
gemm_rhs_transposed_batched(
&self.context,
&qkv.slice(..),
3 * inner,
&qkv.slice(inner..),
3 * inner,
&mut scores,
tokens,
tokens,
tokens,
head_dim,
1.0,
0.0,
heads,
head_dim,
head_dim,
square,
)?;
gpu.add(&mut scores, bias, 0, heads * square, true)?;
let mut log_sum_exp = gpu.uninit(heads * tokens)?;
gpu.attention_softmax(
&mut scores,
&mut log_sum_exp,
heads * tokens,
tokens,
false,
tokens,
0,
)?;
let mut merged = gpu.uninit(tokens * inner)?;
gemm_plain_batched(
&self.context,
&scores,
tokens,
&qkv.slice(2 * inner..),
3 * inner,
&mut merged,
inner,
tokens,
head_dim,
tokens,
1.0,
0.0,
heads,
square,
head_dim,
head_dim,
)?;
let merged = gpu.narrowed(&merged.slice(..), tokens * inner, true)?;
gpu.project(
&block.output,
d_model,
inner,
&merged.all(),
true,
&mut hidden.slice_mut(..),
tokens,
1.0,
)?;
let (normed, _, _) = gpu.rmsnorm(
&hidden,
&block.mlp_norm.as_view(),
tokens,
d_model,
config.eps,
self.narrow,
false,
)?;
let mut gate_up = gpu.uninit(tokens * 2 * width)?;
gpu.project(
&block.gate_up,
2 * width,
d_model,
&normed.all(),
self.narrow,
&mut gate_up.slice_mut(..),
tokens,
0.0,
)?;
let wide = gpu.gated(&gate_up, tokens, width, self.narrow, true)?;
gpu.project(
&block.mlp_out,
d_model,
width,
&wide.all(),
self.narrow,
&mut hidden.slice_mut(..),
tokens,
1.0,
)?;
}
let (normed, _, _) = gpu.rmsnorm(
&hidden,
&self.final_norm.as_view(),
tokens,
d_model,
config.eps,
false,
false,
)?;
let values = self
.context
.stream
.clone_dtoh(&normed.wide())
.map_err(|error| NetworkError::Cuda(format!("device to host copy: {error}")))?;
Ok(Matrix::from_vec(tokens, d_model, values))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::cuda_training::cuda_doctor;
use crate::t5::tests::{checkpoint, checkpoint_with, tiny};
use crate::transformer::Precision;
fn cuda_or_skip() -> Option<Arc<GpuContext>> {
match cuda_doctor(0, 8192) {
Ok(_) => Some(GpuContext::new(0).expect("the CUDA doctor passed")),
Err(NetworkError::Cuda(message))
if message.contains("NO_DEVICE") || message.contains("no CUDA-capable device") =>
{
crate::cuda_training::skip_without_cuda();
None
}
Err(error) => panic!("CUDA is present but the CUDA doctor failed: {error}"),
}
}
#[test]
fn a_device_t5_matches_the_host_or_skips_without_device() {
let Some(context) = cuda_or_skip() else {
return;
};
let config = tiny();
let path = std::env::temp_dir().join("rusting_brain_cuda_t5.safetensors");
for (per_block, precision) in [
(false, Precision::F32),
(false, Precision::Q8),
(true, Precision::F32),
] {
checkpoint_with(&config, &path, per_block);
let encoder = T5Encoder::load_at(&path, config.clone(), precision).unwrap();
assert_eq!(encoder.position_bias.len() > 1, per_block);
let device = DeviceT5::upload(&context, &encoder).unwrap();
let ids = [5, 9, 3, 17, 2, 30, 1];
let expected = encoder.forward(&ids).unwrap();
let actual = device.forward(&encoder, &ids).unwrap();
assert_eq!((actual.rows, actual.cols), (expected.rows, expected.cols));
let scale = expected.data.iter().fold(0.0f32, |m, v| m.max(v.abs()));
let worst = actual
.data
.iter()
.zip(&expected.data)
.map(|(actual, expected)| (actual - expected).abs())
.fold(0.0f32, f32::max);
assert!(
worst < 2e-2 * scale,
"{precision:?}, per block {per_block}: worst difference {worst} of {scale}"
);
}
std::fs::remove_file(path).ok();
}
#[test]
#[ignore]
fn time_a_t5_base_sized_encoder() {
let Some(context) = cuda_or_skip() else {
return;
};
let config = crate::t5::T5Config {
d_model: 768,
head_dim: 64,
num_heads: 12,
layers: 12,
mlp_dim: 2048,
..tiny()
};
let path = std::env::temp_dir().join("rusting_brain_cuda_t5_base.safetensors");
checkpoint(&config, &path);
let encoder = T5Encoder::load_at(&path, config, Precision::F32).unwrap();
std::fs::remove_file(path).ok();
let device = DeviceT5::upload(&context, &encoder).unwrap();
let ids: Vec<u32> = (0..512).map(|index| index % 32).collect();
let started = std::time::Instant::now();
let expected = encoder.forward(&ids).unwrap();
let host = started.elapsed();
device.forward(&encoder, &ids).unwrap();
let mut device_times = Vec::new();
let mut actual = None;
for _ in 0..5 {
let started = std::time::Instant::now();
actual = Some(device.forward(&encoder, &ids).unwrap());
device_times.push(started.elapsed());
}
let actual = actual.unwrap();
let scale = expected.data.iter().fold(0.0f32, |m, v| m.max(v.abs()));
let worst = actual
.data
.iter()
.zip(&expected.data)
.map(|(actual, expected)| (actual - expected).abs())
.fold(0.0f32, f32::max);
println!("host {host:?}, device {device_times:?}, worst {worst} of {scale}");
}
}