use crate::conv::Dense;
use crate::matrix::Matrix;
use crate::mmdit::attention_over;
use crate::network::NetworkError;
use crate::safetensors::ShardedSafeTensors;
use crate::tokenizer::Bpe;
use crate::transformer::Precision;
use rayon::prelude::*;
#[derive(Clone, Debug, PartialEq)]
pub struct TextEncoderConfig {
pub vocab_size: usize,
pub d_model: usize,
pub layers: usize,
pub num_heads: usize,
pub kv_heads: usize,
pub head_dim: usize,
pub mlp_dim: usize,
pub rope_theta: f32,
pub eps: f32,
pub causal: bool,
}
impl TextEncoderConfig {
pub fn from_file(path: impl AsRef<std::path::Path>) -> Result<Self, NetworkError> {
let text = std::fs::read_to_string(path)?;
let json: serde_json::Value = serde_json::from_str(&text)
.map_err(|error| NetworkError::InvalidDataset(format!("config.json: {error}")))?;
let json = json.get("text_config").unwrap_or(&json).clone();
let number = |name: &str| -> Option<usize> {
json.get(name)
.and_then(|value| value.as_u64())
.map(|value| value as usize)
};
let missing =
|name: &str| NetworkError::InvalidDataset(format!("config.json has no {name}"));
let d_model = number("hidden_size").ok_or_else(|| missing("hidden_size"))?;
let num_heads =
number("num_attention_heads").ok_or_else(|| missing("num_attention_heads"))?;
Ok(Self {
vocab_size: number("vocab_size").ok_or_else(|| missing("vocab_size"))?,
d_model,
layers: number("num_hidden_layers").ok_or_else(|| missing("num_hidden_layers"))?,
num_heads,
kv_heads: number("num_key_value_heads").unwrap_or(num_heads),
head_dim: number("head_dim").unwrap_or(d_model / num_heads.max(1)),
mlp_dim: number("intermediate_size").ok_or_else(|| missing("intermediate_size"))?,
rope_theta: json
.get("rope_theta")
.and_then(|value| value.as_f64())
.unwrap_or(10_000.0) as f32,
eps: json
.get("rms_norm_eps")
.and_then(|value| value.as_f64())
.unwrap_or(1e-6) as f32,
causal: true,
})
}
}
pub(crate) struct Layer {
pub(crate) input_norm: Vec<f32>,
pub(crate) query: Dense,
pub(crate) key: Dense,
pub(crate) value: Dense,
pub(crate) output: Dense,
pub(crate) query_norm: Option<Vec<f32>>,
pub(crate) key_norm: Option<Vec<f32>>,
pub(crate) post_norm: Vec<f32>,
pub(crate) gate: Dense,
pub(crate) up: Dense,
pub(crate) down: Dense,
}
pub struct TextEncoder {
config: TextEncoderConfig,
embeddings: Matrix,
pub(crate) layers: Vec<Layer>,
final_norm: Vec<f32>,
#[cfg(feature = "cuda")]
device: Option<crate::cuda_text_encoder::DeviceTextEncoder>,
}
impl TextEncoder {
pub fn config(&self) -> &TextEncoderConfig {
&self.config
}
pub fn load(
path: impl AsRef<std::path::Path>,
prefix: &str,
config: TextEncoderConfig,
) -> Result<Self, NetworkError> {
Self::load_at(path, prefix, config, Precision::F32)
}
pub fn load_at(
path: impl AsRef<std::path::Path>,
prefix: &str,
config: TextEncoderConfig,
precision: Precision,
) -> Result<Self, NetworkError> {
let mut file = ShardedSafeTensors::open(path)?;
Self::read_at(&mut file, prefix, config, precision)
}
pub fn read(
file: &mut ShardedSafeTensors,
prefix: &str,
config: TextEncoderConfig,
) -> Result<Self, NetworkError> {
Self::read_at(file, prefix, config, Precision::F32)
}
pub fn read_at(
file: &mut ShardedSafeTensors,
prefix: &str,
config: TextEncoderConfig,
precision: Precision,
) -> Result<Self, NetworkError> {
let embeddings = file.matrix(&format!("{prefix}embed_tokens.weight"))?;
if embeddings.cols != config.d_model {
return Err(NetworkError::InvalidConfig(format!(
"the checkpoint's embeddings are {} wide and the configuration says {}",
embeddings.cols, config.d_model
)));
}
let layers = (0..config.layers)
.map(|index| {
let base = format!("{prefix}layers.{index}");
Ok(Layer {
input_norm: vector(file, &format!("{base}.input_layernorm.weight"))?,
query: dense(file, &format!("{base}.self_attn.q_proj"), precision)?,
key: dense(file, &format!("{base}.self_attn.k_proj"), precision)?,
value: dense(file, &format!("{base}.self_attn.v_proj"), precision)?,
output: dense(file, &format!("{base}.self_attn.o_proj"), precision)?,
query_norm: vector(file, &format!("{base}.self_attn.q_norm.weight")).ok(),
key_norm: vector(file, &format!("{base}.self_attn.k_norm.weight")).ok(),
post_norm: vector(file, &format!("{base}.post_attention_layernorm.weight"))?,
gate: dense(file, &format!("{base}.mlp.gate_proj"), precision)?,
up: dense(file, &format!("{base}.mlp.up_proj"), precision)?,
down: dense(file, &format!("{base}.mlp.down_proj"), precision)?,
})
})
.collect::<Result<Vec<_>, NetworkError>>()?;
Ok(Self {
final_norm: vector(file, &format!("{prefix}norm.weight"))?,
config,
embeddings,
layers,
#[cfg(feature = "cuda")]
device: None,
})
}
#[cfg(feature = "cuda")]
pub fn attach_device(
&mut self,
context: &std::sync::Arc<crate::gpu_transformer::GpuContext>,
) -> Result<(), NetworkError> {
self.device = Some(crate::cuda_text_encoder::DeviceTextEncoder::upload(
context, self,
)?);
Ok(())
}
#[cfg(feature = "cuda")]
pub fn on_device(&self) -> bool {
self.device.is_some()
}
#[cfg(feature = "cuda")]
pub fn detach_device(&mut self) {
self.device = None;
}
pub fn forward(&self, ids: &[u32]) -> Result<Matrix, NetworkError> {
#[cfg(feature = "cuda")]
if let Some(device) = &self.device {
let hidden = device.forward_layers(self, ids, ids.len(), &[self.layers.len()])?;
return Ok(rms_norm(&hidden, &self.final_norm, self.config.eps));
}
let hidden = self.run(ids, ids.len(), self.layers.len(), |_, _| {})?;
Ok(rms_norm(&hidden, &self.final_norm, self.config.eps))
}
pub fn forward_layers(
&self,
ids: &[u32],
real: usize,
layers: &[usize],
) -> Result<Matrix, NetworkError> {
let deepest = layers.iter().copied().max().unwrap_or(0);
if layers.is_empty() || layers.contains(&0) || deepest > self.layers.len() {
return Err(NetworkError::InvalidConfig(format!(
"this encoder has {} layers and the output asks for those after {layers:?}",
self.layers.len()
)));
}
#[cfg(feature = "cuda")]
if let Some(device) = &self.device {
return device.forward_layers(self, ids, real, layers);
}
let width = self.config.d_model;
let mut output = Matrix::new(ids.len(), layers.len() * width);
self.run(ids, real, deepest, |done, hidden| {
for (slot, _) in layers
.iter()
.enumerate()
.filter(|(_, layer)| **layer == done)
{
for token in 0..hidden.rows {
output.row_mut(token)[slot * width..(slot + 1) * width]
.copy_from_slice(hidden.row(token));
}
}
})?;
Ok(output)
}
fn run(
&self,
ids: &[u32],
real: usize,
depth: usize,
mut after: impl FnMut(usize, &Matrix),
) -> Result<Matrix, NetworkError> {
let mut hidden = self.embed(ids)?;
let (heads, kv_heads, head_dim) = (
self.config.num_heads,
self.config.kv_heads,
self.config.head_dim,
);
let rotary = Rotary::new(ids.len(), head_dim, self.config.rope_theta);
for (index, layer) in self.layers[..depth].iter().enumerate() {
let normed = rms_norm(&hidden, &layer.input_norm, self.config.eps);
let mut queries = layer.query.forward(&normed)?;
let mut keys = layer.key.forward(&normed)?;
let values = layer.value.forward(&normed)?;
if let Some(scale) = &layer.query_norm {
normalize_heads(&mut queries, scale, self.config.eps);
}
if let Some(scale) = &layer.key_norm {
normalize_heads(&mut keys, scale, self.config.eps);
}
rotary.apply(&mut queries, heads);
rotary.apply(&mut keys, kv_heads);
let attended = attention_over(
&queries,
&keys,
&values,
heads,
kv_heads,
self.config.causal,
real,
);
let projected = layer.output.forward(&attended)?;
for (value, delta) in hidden.data.iter_mut().zip(&projected.data) {
*value += delta;
}
let normed = rms_norm(&hidden, &layer.post_norm, self.config.eps);
let mut gate = layer.gate.forward(&normed)?;
let up = layer.up.forward(&normed)?;
gate.data
.par_iter_mut()
.zip(up.data.par_iter())
.for_each(|(gate, up)| *gate = crate::ffn::silu(*gate) * up);
let projected = layer.down.forward(&gate)?;
for (value, delta) in hidden.data.iter_mut().zip(&projected.data) {
*value += delta;
}
after(index + 1, &hidden);
}
Ok(hidden)
}
pub(crate) fn embed(&self, ids: &[u32]) -> Result<Matrix, NetworkError> {
if ids.is_empty() {
return Err(NetworkError::InvalidConfig(
"an empty prompt has no hidden states".into(),
));
}
let mut hidden = Matrix::new(ids.len(), self.config.d_model);
for (position, id) in ids.iter().enumerate() {
if *id as usize >= self.embeddings.rows {
return Err(NetworkError::InvalidTarget {
expected: self.embeddings.rows,
actual: *id as usize,
});
}
hidden
.row_mut(position)
.copy_from_slice(self.embeddings.row(*id as usize));
}
Ok(hidden)
}
pub fn encode(&self, tokenizer: &Bpe, prompt: &str) -> Result<Matrix, NetworkError> {
self.forward(&tokenizer.encode(prompt)?)
}
pub fn pool(&self, hidden: &Matrix) -> Vec<f32> {
hidden.row(hidden.rows - 1).to_vec()
}
}
pub(crate) struct Rotary {
pub(crate) cos: Vec<f32>,
pub(crate) sin: Vec<f32>,
half: usize,
}
impl Rotary {
pub(crate) fn new(tokens: usize, head_dim: usize, theta: f32) -> Self {
let half = head_dim / 2;
let mut cos = vec![0.0; tokens * half];
let mut sin = vec![0.0; tokens * half];
for position in 0..tokens {
for index in 0..half {
let frequency = theta.powf(-2.0 * index as f32 / head_dim as f32);
let angle = position as f32 * frequency;
cos[position * half + index] = angle.cos();
sin[position * half + index] = angle.sin();
}
}
Self { cos, sin, half }
}
pub(crate) fn apply(&self, values: &mut Matrix, heads: usize) {
let head_dim = self.half * 2;
let half = self.half;
values
.data
.par_chunks_mut(heads * head_dim)
.enumerate()
.for_each(|(position, row)| {
let cos = &self.cos[position * half..(position + 1) * half];
let sin = &self.sin[position * half..(position + 1) * half];
for head in 0..heads {
let head = &mut row[head * head_dim..(head + 1) * head_dim];
for index in 0..half {
let (first, second) = (head[index], head[index + half]);
head[index] = first * cos[index] - second * sin[index];
head[index + half] = second * cos[index] + first * sin[index];
}
}
});
}
}
pub(crate) fn rms_norm(tokens: &Matrix, weight: &[f32], eps: f32) -> Matrix {
let mut output = tokens.clone();
output.data.par_chunks_mut(tokens.cols).for_each(|row| {
let mean_square = row.iter().map(|value| value * value).sum::<f32>() / row.len() as f32;
let inverse = (mean_square + eps).sqrt().recip();
for (value, weight) in row.iter_mut().zip(weight) {
*value *= inverse * weight;
}
});
output
}
fn normalize_heads(values: &mut Matrix, weight: &[f32], eps: f32) {
let head_dim = weight.len();
values.data.par_chunks_mut(values.cols).for_each(|row| {
for head in row.chunks_mut(head_dim) {
let mean_square = head.iter().map(|value| value * value).sum::<f32>() / head_dim as f32;
let inverse = (mean_square + eps).sqrt().recip();
for (value, weight) in head.iter_mut().zip(weight) {
*value *= inverse * weight;
}
}
});
}
fn dense(
file: &mut ShardedSafeTensors,
name: &str,
precision: Precision,
) -> Result<Dense, NetworkError> {
let (values, shape) = file.tensor(&format!("{name}.weight"))?;
if shape.len() != 2 {
return Err(NetworkError::InvalidConfig(format!(
"{name}.weight is {shape:?}, which is not a linear layer"
)));
}
let bias = file
.tensor(&format!("{name}.bias"))
.ok()
.map(|(bias, _)| bias);
let mut layer = Dense::new(Matrix::from_vec(shape[0], shape[1], values), bias)?;
if precision == Precision::Q8 {
layer.quantize();
}
Ok(layer)
}
fn vector(file: &mut ShardedSafeTensors, name: &str) -> Result<Vec<f32>, NetworkError> {
Ok(file.tensor(name)?.0)
}
#[cfg(test)]
pub(crate) mod tests {
use super::*;
use std::collections::BTreeMap;
fn tiny() -> TextEncoderConfig {
TextEncoderConfig {
vocab_size: 16,
d_model: 16,
layers: 2,
num_heads: 4,
kv_heads: 2,
head_dim: 4,
mlp_dim: 32,
rope_theta: 10_000.0,
eps: 1e-6,
causal: true,
}
}
pub(crate) fn checkpoint(config: &TextEncoderConfig, path: &std::path::Path, query_norm: bool) {
let mut tensors: BTreeMap<String, (Vec<usize>, Vec<f32>)> = BTreeMap::new();
let mut add = |name: String, shape: Vec<usize>| {
let count = shape.iter().product::<usize>();
let values = (0..count)
.map(|index| ((index % 13) as f32 - 6.0) * 0.05)
.collect();
tensors.insert(name, (shape, values));
};
add(
"model.embed_tokens.weight".into(),
vec![config.vocab_size, config.d_model],
);
for index in 0..config.layers {
let base = format!("model.layers.{index}");
add(
format!("{base}.input_layernorm.weight"),
vec![config.d_model],
);
add(
format!("{base}.post_attention_layernorm.weight"),
vec![config.d_model],
);
let queries = config.num_heads * config.head_dim;
let keys = config.kv_heads * config.head_dim;
add(
format!("{base}.self_attn.q_proj.weight"),
vec![queries, config.d_model],
);
add(
format!("{base}.self_attn.k_proj.weight"),
vec![keys, config.d_model],
);
add(
format!("{base}.self_attn.v_proj.weight"),
vec![keys, config.d_model],
);
add(
format!("{base}.self_attn.o_proj.weight"),
vec![config.d_model, queries],
);
if query_norm {
add(
format!("{base}.self_attn.q_norm.weight"),
vec![config.head_dim],
);
add(
format!("{base}.self_attn.k_norm.weight"),
vec![config.head_dim],
);
}
add(
format!("{base}.mlp.gate_proj.weight"),
vec![config.mlp_dim, config.d_model],
);
add(
format!("{base}.mlp.up_proj.weight"),
vec![config.mlp_dim, config.d_model],
);
add(
format!("{base}.mlp.down_proj.weight"),
vec![config.d_model, config.mlp_dim],
);
}
add("model.norm.weight".into(), vec![config.d_model]);
crate::safetensors::write_checkpoint(path, &tensors);
}
fn encoder(config: &TextEncoderConfig, name: &str, query_norm: bool) -> TextEncoder {
let path = std::env::temp_dir().join(name);
checkpoint(config, &path, query_norm);
let encoder = TextEncoder::load(&path, "model.", config.clone()).unwrap();
std::fs::remove_file(path).ok();
encoder
}
#[test]
fn a_prompt_becomes_one_hidden_state_per_token() {
let config = tiny();
let encoder = encoder(&config, "rusting_brain_text_forward.safetensors", false);
let hidden = encoder.forward(&[1, 5, 9, 2]).unwrap();
assert_eq!((hidden.rows, hidden.cols), (4, config.d_model));
assert!(hidden.data.iter().all(|value| value.is_finite()));
assert_eq!(encoder.pool(&hidden), hidden.row(3).to_vec());
}
#[test]
fn a_causal_encoder_leaves_earlier_states_alone_when_the_prompt_grows() {
let encoder = encoder(&tiny(), "rusting_brain_text_causal.safetensors", false);
let short = encoder.forward(&[3, 7]).unwrap();
let long = encoder.forward(&[3, 7, 1, 4]).unwrap();
for row in 0..2 {
for (first, second) in short.row(row).iter().zip(long.row(row)) {
assert!((first - second).abs() < 1e-4, "{first} vs {second}");
}
}
assert_ne!(encoder.pool(&short), encoder.pool(&long));
}
#[test]
fn stacked_layers_are_the_states_after_each_depth_and_padding_is_not_read() {
let config = TextEncoderConfig {
layers: 3,
..tiny()
};
let encoder = encoder(&config, "rusting_brain_text_layers.safetensors", true);
let width = config.d_model;
let ids = [3, 7, 1, 4];
let stacked = encoder.forward_layers(&ids, ids.len(), &[1, 3]).unwrap();
assert_eq!((stacked.rows, stacked.cols), (4, 2 * width));
let mut last = Matrix::new(4, width);
for token in 0..4 {
last.row_mut(token)
.copy_from_slice(&stacked.row(token)[width..]);
}
let normed = rms_norm(&last, &encoder.final_norm, config.eps);
let plain = encoder.forward(&ids).unwrap();
for (first, second) in normed.data.iter().zip(&plain.data) {
assert!((first - second).abs() < 1e-5, "{first} vs {second}");
}
let first = encoder.forward_layers(&[3, 7, 9, 9], 2, &[2, 3]).unwrap();
let second = encoder.forward_layers(&[3, 7, 5, 9], 2, &[2, 3]).unwrap();
for token in [0, 1, 3] {
for (a, b) in first.row(token).iter().zip(second.row(token)) {
assert!((a - b).abs() < 1e-5, "token {token}: {a} vs {b}");
}
}
assert_ne!(first.row(2), second.row(2));
assert!(encoder.forward_layers(&ids, 4, &[4]).is_err());
}
#[test]
fn query_normalization_is_read_from_the_checkpoint_and_changes_the_output() {
let config = tiny();
let plain = encoder(&config, "rusting_brain_text_plain.safetensors", false);
let normalized = encoder(&config, "rusting_brain_text_qknorm.safetensors", true);
let ids = [2, 6, 4];
let first = plain.forward(&ids).unwrap();
let second = normalized.forward(&ids).unwrap();
assert!(
first
.data
.iter()
.zip(&second.data)
.any(|(first, second)| (first - second).abs() > 1e-5)
);
}
#[test]
fn an_impossible_prompt_is_refused() {
let encoder = encoder(&tiny(), "rusting_brain_text_refuse.safetensors", false);
assert!(encoder.forward(&[]).is_err());
assert!(encoder.forward(&[999]).is_err());
}
#[test]
fn a_hugging_face_configuration_reads_back() {
let path = std::env::temp_dir().join("rusting_brain_text_config.json");
std::fs::write(
&path,
r#"{"hidden_size": 2560, "num_hidden_layers": 36, "num_attention_heads": 32,
"num_key_value_heads": 8, "head_dim": 128, "intermediate_size": 9728,
"vocab_size": 151936, "rope_theta": 1000000.0, "rms_norm_eps": 1e-6}"#,
)
.unwrap();
let config = TextEncoderConfig::from_file(&path).unwrap();
assert_eq!(config.d_model, 2560);
assert_eq!(config.kv_heads, 8);
assert_eq!(config.head_dim, 128);
assert_eq!(config.rope_theta, 1e6);
assert!(config.causal);
std::fs::remove_file(path).ok();
assert!(TextEncoderConfig::from_file("/nonexistent/config.json").is_err());
}
}