use serde::{Deserialize, Serialize};
use crate::{Result, RltError};
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub struct RltConfig {
pub d_model: usize,
pub n_heads: usize,
pub d_ff: usize,
pub n_encoder_layers: usize,
pub n_decoder_layers: usize,
pub window: usize,
pub memory_groups: usize,
pub vocab_size: usize,
pub tied: bool,
pub feedback_alpha: f64,
pub max_seq_len: usize,
}
impl Default for RltConfig {
fn default() -> Self {
Self {
d_model: 256,
n_heads: 8,
d_ff: 1024,
n_encoder_layers: 4,
n_decoder_layers: 4,
window: 64,
memory_groups: 1,
vocab_size: 258,
tied: true,
feedback_alpha: 0.1,
max_seq_len: 4096,
}
}
}
impl RltConfig {
pub fn validate(&self) -> Result<()> {
let err = |m: &str| RltError::Config(m.to_string());
if self.n_heads == 0 {
return Err(err("n_heads must be >= 1"));
}
if self.d_model == 0 || !self.d_model.is_multiple_of(self.n_heads) {
return Err(err("n_heads must divide d_model"));
}
if self.n_encoder_layers == 0 || self.n_decoder_layers == 0 {
return Err(err("layer counts must be >= 1"));
}
if self.n_decoder_layers > self.n_encoder_layers && self.tied {
return Err(err(
"tied config requires n_decoder_layers <= n_encoder_layers",
));
}
if self.window == 0 {
return Err(err("window must be >= 1"));
}
if self.memory_groups == 0 {
return Err(err("memory_groups must be >= 1"));
}
if self.d_ff == 0 {
return Err(err("d_ff must be >= 1"));
}
if self.vocab_size < crate::tokenizer::MIN_VOCAB {
return Err(err("vocab_size too small for the byte-level tokenizer"));
}
if self.max_seq_len == 0 {
return Err(err("max_seq_len must be >= 1"));
}
if !self.head_dim().is_multiple_of(2) {
return Err(err("head_dim must be even (RoPE requirement)"));
}
if !self.feedback_alpha.is_finite() {
return Err(err("feedback_alpha must be finite"));
}
Ok(())
}
pub fn head_dim(&self) -> usize {
self.d_model / self.n_heads
}
pub fn memory_group_of(&self, layer: usize) -> usize {
layer % self.memory_groups
}
}