#[cfg(feature = "gpu")]
use crate::gpu::GpuContext;
use crate::model::{CausalLM, CausalLmConfig, ModelError};
use crate::{IndexTensor, Tensor};
use std::error::Error;
use std::fmt;
use std::sync::mpsc::{self, Receiver, SyncSender};
use std::sync::Arc;
use std::thread::{self, JoinHandle};
#[derive(Debug)]
pub enum DataParallelError {
Invalid(String),
Worker(String),
Model(ModelError),
}
impl fmt::Display for DataParallelError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Invalid(message) => write!(formatter, "data-parallel error: {message}"),
Self::Worker(message) => write!(formatter, "data-parallel worker error: {message}"),
Self::Model(error) => error.fmt(formatter),
}
}
}
impl Error for DataParallelError {}
impl From<ModelError> for DataParallelError {
fn from(error: ModelError) -> Self {
Self::Model(error)
}
}
struct StepRequest {
weights: Arc<Vec<Vec<f32>>>,
inputs: Vec<usize>,
targets: Vec<usize>,
batch_size: usize,
sequence_length: usize,
dropout_seed: u64,
activation_checkpointing: bool,
backward_scale: f32,
}
enum WorkerMessage {
Step(StepRequest),
Shutdown,
}
struct StepResult {
loss: f32,
grads: Vec<Vec<f32>>,
}
struct WorkerHandle {
request_tx: SyncSender<WorkerMessage>,
response_rx: Receiver<Result<StepResult, String>>,
join: Option<JoinHandle<()>>,
}
impl Drop for WorkerHandle {
fn drop(&mut self) {
let _ = self.request_tx.send(WorkerMessage::Shutdown);
if let Some(join) = self.join.take() {
let _ = join.join();
}
}
}
pub struct CpuDataParallelPool {
workers: usize,
handles: Vec<WorkerHandle>,
}
impl CpuDataParallelPool {
pub fn new(config: CausalLmConfig, workers: usize) -> Result<Self, DataParallelError> {
if workers < 2 {
return Err(DataParallelError::Invalid(
"CPU data-parallel pool requires at least 2 workers".into(),
));
}
let mut handles = Vec::with_capacity(workers);
for worker_index in 0..workers {
let (request_tx, request_rx) = mpsc::sync_channel::<WorkerMessage>(0);
let (response_tx, response_rx) = mpsc::sync_channel(0);
let (ready_tx, ready_rx) = mpsc::sync_channel::<Result<(), String>>(0);
let worker_config = config.clone();
let join = thread::Builder::new()
.name(format!("oxidelm-dp-{worker_index}"))
.spawn(move || worker_loop(worker_config, request_rx, response_tx, ready_tx))
.map_err(|error| {
DataParallelError::Invalid(format!("failed to spawn DP worker: {error}"))
})?;
match ready_rx.recv() {
Ok(Ok(())) => {}
Ok(Err(message)) => {
return Err(DataParallelError::Worker(message));
}
Err(_) => {
return Err(DataParallelError::Worker(
"worker exited during startup".into(),
));
}
}
handles.push(WorkerHandle {
request_tx,
response_rx,
join: Some(join),
});
}
Ok(Self { workers, handles })
}
pub fn workers(&self) -> usize {
self.workers
}
pub fn forward_backward(
&self,
weights: Arc<Vec<Vec<f32>>>,
input_ids: &IndexTensor,
target_ids: &IndexTensor,
dropout_seed: u64,
activation_checkpointing: bool,
backward_scale: f32,
) -> Result<(f32, Vec<Vec<f32>>), DataParallelError> {
let shards = split_batch_shards(input_ids, target_ids, self.workers)?;
for (handle, (inputs, targets, batch_size, sequence_length)) in
self.handles.iter().zip(shards.into_iter())
{
handle
.request_tx
.send(WorkerMessage::Step(StepRequest {
weights: Arc::clone(&weights),
inputs,
targets,
batch_size,
sequence_length,
dropout_seed,
activation_checkpointing,
backward_scale,
}))
.map_err(|_| DataParallelError::Worker("worker channel closed".into()))?;
}
let mut losses = Vec::with_capacity(self.workers);
let mut all_grads = Vec::with_capacity(self.workers);
for handle in &self.handles {
let result = handle
.response_rx
.recv()
.map_err(|_| DataParallelError::Worker("worker channel closed".into()))?
.map_err(DataParallelError::Worker)?;
losses.push(result.loss);
all_grads.push(result.grads);
}
let loss = losses.iter().sum::<f32>() / self.workers as f32;
let grads = mean_reduce_grads(&all_grads)?;
Ok((loss, grads))
}
}
fn worker_loop(
config: CausalLmConfig,
request_rx: Receiver<WorkerMessage>,
response_tx: SyncSender<Result<StepResult, String>>,
ready_tx: SyncSender<Result<(), String>>,
) {
let model = match CausalLM::new(config) {
Ok(model) => model,
Err(error) => {
let _ = ready_tx.send(Err(error.to_string()));
return;
}
};
if ready_tx.send(Ok(())).is_err() {
return;
}
while let Ok(message) = request_rx.recv() {
match message {
WorkerMessage::Shutdown => break,
WorkerMessage::Step(request) => {
let result = run_worker_step(&model, request);
if response_tx.send(result).is_err() {
break;
}
}
}
}
}
fn run_worker_step(model: &CausalLM, request: StepRequest) -> Result<StepResult, String> {
let parameters = model.parameters();
if parameters.len() != request.weights.len() {
return Err(format!(
"weight count mismatch: model has {}, got {}",
parameters.len(),
request.weights.len()
));
}
for (parameter, values) in parameters.iter().zip(request.weights.iter()) {
if parameter.numel() != values.len() {
return Err("parameter numel mismatch while loading replica weights".into());
}
parameter.set_data(values.clone());
}
model.zero_grad();
let shape = vec![request.batch_size, request.sequence_length];
let input_ids = IndexTensor::new(request.inputs, shape.clone());
let target_ids = IndexTensor::new(request.targets, shape);
let logits = model
.forward_training_with_checkpointing(
&input_ids,
request.dropout_seed,
request.activation_checkpointing,
)
.map_err(|error| error.to_string())?;
let loss = model
.compute_loss(&logits, &target_ids)
.map_err(|error| error.to_string())?;
let loss_value = loss.item();
if (request.backward_scale - 1.0).abs() < f32::EPSILON {
loss.backward();
} else {
loss.mul(&Tensor::scalar(request.backward_scale, false))
.backward();
}
let grads = parameters
.iter()
.map(|parameter| {
parameter.grad().ok_or_else(|| {
"parameter missing gradient after data-parallel backward".to_string()
})
})
.collect::<Result<Vec<_>, _>>()?;
Ok(StepResult {
loss: loss_value,
grads,
})
}
pub fn split_batch_shards(
input_ids: &IndexTensor,
target_ids: &IndexTensor,
workers: usize,
) -> Result<Vec<(Vec<usize>, Vec<usize>, usize, usize)>, DataParallelError> {
if workers == 0 {
return Err(DataParallelError::Invalid(
"data-parallel worker count must be positive".into(),
));
}
let input_shape = input_ids.shape();
let target_shape = target_ids.shape();
if input_shape != target_shape {
return Err(DataParallelError::Invalid(
"input and target shapes must match for data-parallel split".into(),
));
}
if input_shape.len() != 2 {
return Err(DataParallelError::Invalid(
"data-parallel split expects rank-2 [batch, sequence] tensors".into(),
));
}
let batch_size = input_shape[0];
let sequence_length = input_shape[1];
if batch_size < workers {
return Err(DataParallelError::Invalid(format!(
"batch size {batch_size} must be >= data-parallel workers {workers}"
)));
}
if batch_size % workers != 0 {
return Err(DataParallelError::Invalid(format!(
"batch size {batch_size} must be divisible by data-parallel workers {workers}"
)));
}
let shard_batch = batch_size / workers;
let inputs = input_ids.data();
let targets = target_ids.data();
let mut shards = Vec::with_capacity(workers);
for worker in 0..workers {
let start = worker * shard_batch * sequence_length;
let end = start + shard_batch * sequence_length;
shards.push((
inputs[start..end].to_vec(),
targets[start..end].to_vec(),
shard_batch,
sequence_length,
));
}
Ok(shards)
}
pub fn capture_parameter_weights(parameters: &[Tensor]) -> Result<Vec<Vec<f32>>, DataParallelError> {
parameters
.iter()
.map(|parameter| {
if parameter.device() != crate::Device::Cpu {
return Err(DataParallelError::Invalid(
"data-parallel weight capture requires CPU tensors; call to_cpu() first"
.into(),
));
}
Ok(parameter.data())
})
.collect()
}
pub fn mean_reduce_grads(worker_grads: &[Vec<Vec<f32>>]) -> Result<Vec<Vec<f32>>, DataParallelError> {
let Some(first) = worker_grads.first() else {
return Err(DataParallelError::Invalid(
"no worker gradients to reduce".into(),
));
};
let workers = worker_grads.len();
let inv = 1.0 / workers as f32;
let mut reduced = Vec::with_capacity(first.len());
for param_index in 0..first.len() {
let expected_len = first[param_index].len();
let mut average = vec![0.0f32; expected_len];
for worker in worker_grads {
if worker.len() != first.len() {
return Err(DataParallelError::Invalid(
"worker gradient parameter counts differ".into(),
));
}
let grad = &worker[param_index];
if grad.len() != expected_len {
return Err(DataParallelError::Invalid(
"worker gradient lengths differ".into(),
));
}
for (dst, src) in average.iter_mut().zip(grad.iter()) {
*dst += *src;
}
}
for value in &mut average {
*value *= inv;
}
reduced.push(average);
}
Ok(reduced)
}
pub fn accumulate_parameter_grads(parameters: &[Tensor], grads: &[Vec<f32>]) -> Result<(), DataParallelError> {
if parameters.len() != grads.len() {
return Err(DataParallelError::Invalid(
"parameter and gradient counts differ during data-parallel accumulate".into(),
));
}
for (parameter, grad) in parameters.iter().zip(grads.iter()) {
if parameter.numel() != grad.len() {
return Err(DataParallelError::Invalid(
"parameter numel mismatch during data-parallel accumulate".into(),
));
}
parameter.add_grad(grad);
}
Ok(())
}
#[cfg(feature = "gpu")]
pub fn accumulate_parameter_gpu_grads(
parameters: &[Tensor],
grads: &[Vec<f32>],
gpu: &GpuContext,
) -> Result<(), DataParallelError> {
if parameters.len() != grads.len() {
return Err(DataParallelError::Invalid(
"parameter and gradient counts differ during GPU data-parallel accumulate".into(),
));
}
for (parameter, grad) in parameters.iter().zip(grads.iter()) {
if parameter.numel() != grad.len() {
return Err(DataParallelError::Invalid(
"parameter numel mismatch during GPU data-parallel accumulate".into(),
));
}
let uploaded = gpu
.upload(grad, parameter.shape())
.map_err(|error| DataParallelError::Worker(error.to_string()))?;
if let Some(existing) = parameter.gpu_grad() {
let summed = gpu
.add(&existing, &uploaded)
.map_err(|error| DataParallelError::Worker(error.to_string()))?;
parameter.set_gpu_grad(Some(summed));
} else {
parameter.set_gpu_grad(Some(uploaded));
}
}
Ok(())
}
#[cfg(feature = "gpu")]
pub struct GpuDataParallelPool {
workers: usize,
handles: Vec<WorkerHandle>,
}
#[cfg(feature = "gpu")]
impl GpuDataParallelPool {
pub fn new(config: CausalLmConfig, workers: usize) -> Result<Self, DataParallelError> {
if workers < 2 {
return Err(DataParallelError::Invalid(
"GPU data-parallel pool requires at least 2 workers".into(),
));
}
let devices = GpuContext::create_devices(workers).map_err(|error| {
DataParallelError::Invalid(format!("failed to create GPU DP devices: {error}"))
})?;
let mut handles = Vec::with_capacity(workers);
for (worker_index, gpu) in devices.into_iter().enumerate() {
let (request_tx, request_rx) = mpsc::sync_channel::<WorkerMessage>(0);
let (response_tx, response_rx) = mpsc::sync_channel(0);
let (ready_tx, ready_rx) = mpsc::sync_channel::<Result<(), String>>(0);
let worker_config = config.clone();
let join = thread::Builder::new()
.name(format!("oxidelm-gpu-dp-{worker_index}"))
.spawn(move || {
gpu_worker_loop(worker_config, gpu, request_rx, response_tx, ready_tx)
})
.map_err(|error| {
DataParallelError::Invalid(format!("failed to spawn GPU DP worker: {error}"))
})?;
match ready_rx.recv() {
Ok(Ok(())) => {}
Ok(Err(message)) => {
return Err(DataParallelError::Worker(message));
}
Err(_) => {
return Err(DataParallelError::Worker(
"GPU worker exited during startup".into(),
));
}
}
handles.push(WorkerHandle {
request_tx,
response_rx,
join: Some(join),
});
}
Ok(Self { workers, handles })
}
pub fn workers(&self) -> usize {
self.workers
}
pub fn forward_backward(
&self,
weights: Arc<Vec<Vec<f32>>>,
input_ids: &IndexTensor,
target_ids: &IndexTensor,
dropout_seed: u64,
activation_checkpointing: bool,
backward_scale: f32,
) -> Result<(f32, Vec<Vec<f32>>), DataParallelError> {
let shards = split_batch_shards(input_ids, target_ids, self.workers)?;
for (handle, (inputs, targets, batch_size, sequence_length)) in
self.handles.iter().zip(shards.into_iter())
{
handle
.request_tx
.send(WorkerMessage::Step(StepRequest {
weights: Arc::clone(&weights),
inputs,
targets,
batch_size,
sequence_length,
dropout_seed,
activation_checkpointing,
backward_scale,
}))
.map_err(|_| DataParallelError::Worker("GPU worker channel closed".into()))?;
}
let mut losses = Vec::with_capacity(self.workers);
let mut all_grads = Vec::with_capacity(self.workers);
for handle in &self.handles {
let result = handle
.response_rx
.recv()
.map_err(|_| DataParallelError::Worker("GPU worker channel closed".into()))?
.map_err(DataParallelError::Worker)?;
losses.push(result.loss);
all_grads.push(result.grads);
}
let loss = losses.iter().sum::<f32>() / self.workers as f32;
let grads = mean_reduce_grads(&all_grads)?;
Ok((loss, grads))
}
}
#[cfg(feature = "gpu")]
fn gpu_worker_loop(
config: CausalLmConfig,
gpu: GpuContext,
request_rx: Receiver<WorkerMessage>,
response_tx: SyncSender<Result<StepResult, String>>,
ready_tx: SyncSender<Result<(), String>>,
) {
let model = match CausalLM::new(config) {
Ok(model) => model,
Err(error) => {
let _ = ready_tx.send(Err(error.to_string()));
return;
}
};
if ready_tx.send(Ok(())).is_err() {
return;
}
while let Ok(message) = request_rx.recv() {
match message {
WorkerMessage::Shutdown => break,
WorkerMessage::Step(request) => {
let result = run_gpu_worker_step(&model, &gpu, request);
if response_tx.send(result).is_err() {
break;
}
}
}
}
}
#[cfg(feature = "gpu")]
fn run_gpu_worker_step(
model: &CausalLM,
gpu: &GpuContext,
request: StepRequest,
) -> Result<StepResult, String> {
let parameters = model.parameters();
if parameters.len() != request.weights.len() {
return Err(format!(
"weight count mismatch: model has {}, got {}",
parameters.len(),
request.weights.len()
));
}
for (parameter, values) in parameters.iter().zip(request.weights.iter()) {
if parameter.numel() != values.len() {
return Err("parameter numel mismatch while loading GPU replica weights".into());
}
parameter.set_data(values.clone());
}
model.zero_grad();
let shape = vec![request.batch_size, request.sequence_length];
let input_ids = IndexTensor::new(request.inputs, shape.clone());
let target_ids = IndexTensor::new(request.targets, shape);
let logits = model
.forward_training_gpu_with_checkpointing(
&input_ids,
request.dropout_seed,
request.activation_checkpointing,
gpu,
)
.map_err(|error| error.to_string())?;
let loss = model
.compute_loss_gpu(&logits, &target_ids, gpu)
.map_err(|error| error.to_string())?;
let loss_value = loss.to_cpu().map_err(|error| error.to_string())?.item();
if (request.backward_scale - 1.0).abs() < f32::EPSILON {
loss.backward_gpu(gpu).map_err(|error| error.to_string())?;
} else {
loss.gpu_scale(request.backward_scale, gpu)
.map_err(|error| error.to_string())?
.backward_gpu(gpu)
.map_err(|error| error.to_string())?;
}
let grads = parameters
.iter()
.map(|parameter| {
parameter
.gpu_grad_to_cpu()
.map_err(|error| error.to_string())?
.ok_or_else(|| {
"parameter missing GPU gradient after data-parallel backward".to_string()
})
})
.collect::<Result<Vec<_>, _>>()?;
Ok(StepResult {
loss: loss_value,
grads,
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::model::CausalLmConfig;
use crate::tokenizer::{BpeTokenizer, FIRST_MERGE_ID};
use crate::IndexTensor;
fn tiny_model() -> CausalLM {
let text = "abababababababababababababababab";
let tokenizer = BpeTokenizer::train_from_bytes(text.as_bytes(), FIRST_MERGE_ID).unwrap();
CausalLM::new(CausalLmConfig {
vocab_size: tokenizer.vocab_size(),
hidden_size: 8,
num_layers: 1,
num_heads: 2,
num_key_value_heads: 1,
intermediate_size: 16,
max_sequence_length: 4,
rms_norm_epsilon: 1e-5,
rope_theta: 10_000.0,
rope_scaling: None,
eos_token_id: tokenizer.special_tokens.eos,
seed: 7,
initializer_std: 0.02,
tie_word_embeddings: true,
attention_dropout: 0.0,
residual_dropout: 0.0,
embedding_dropout: 0.0,
use_bias: false,
})
.unwrap()
}
#[test]
fn split_requires_divisible_batch() {
let inputs = IndexTensor::new(vec![1, 2, 3, 4, 5, 6], vec![3, 2]);
let targets = IndexTensor::new(vec![2, 3, 4, 5, 6, 7], vec![3, 2]);
assert!(split_batch_shards(&inputs, &targets, 2).is_err());
}
#[test]
fn pool_matches_single_worker_grads() {
let model = tiny_model();
let parameters = model.parameters();
let weights = Arc::new(capture_parameter_weights(¶meters).unwrap());
let batch_size = 4;
let sequence_length = 4;
let tokens: Vec<usize> = (0..batch_size * sequence_length)
.map(|index| index % model.config.vocab_size)
.collect();
let targets: Vec<usize> = (1..batch_size * sequence_length + 1)
.map(|index| index % model.config.vocab_size)
.collect();
let input_ids = IndexTensor::new(tokens.clone(), vec![batch_size, sequence_length]);
let target_ids = IndexTensor::new(targets.clone(), vec![batch_size, sequence_length]);
model.zero_grad();
let logits = model
.forward_training_with_checkpointing(&input_ids, 0, false)
.unwrap();
let loss = model.compute_loss(&logits, &target_ids).unwrap();
let single_loss = loss.item();
loss.backward();
let single_grads: Vec<Vec<f32>> = parameters.iter().map(|p| p.grad().unwrap()).collect();
let pool = CpuDataParallelPool::new(model.config.clone(), 2).unwrap();
let (dp_loss, dp_grads) = pool
.forward_backward(weights, &input_ids, &target_ids, 0, false, 1.0)
.unwrap();
assert!((single_loss - dp_loss).abs() < 1e-5, "{single_loss} vs {dp_loss}");
for (single, reduced) in single_grads.iter().zip(dp_grads.iter()) {
assert_eq!(single.len(), reduced.len());
for (a, b) in single.iter().zip(reduced.iter()) {
assert!((a - b).abs() < 1e-4, "{a} vs {b}");
}
}
}
#[cfg(feature = "gpu")]
#[test]
fn gpu_pool_matches_single_device_grads() {
let Ok(single_gpu) = GpuContext::new() else {
eprintln!("skipping gpu_pool_matches_single_device_grads: no GPU");
return;
};
let Ok(pool) = GpuDataParallelPool::new(tiny_model().config.clone(), 2) else {
eprintln!("skipping gpu_pool_matches_single_device_grads: create_devices(2) failed");
return;
};
let model = tiny_model();
let parameters = model.parameters();
let weights = Arc::new(capture_parameter_weights(¶meters).unwrap());
let batch_size = 4;
let sequence_length = 4;
let tokens: Vec<usize> = (0..batch_size * sequence_length)
.map(|index| index % model.config.vocab_size)
.collect();
let targets: Vec<usize> = (1..batch_size * sequence_length + 1)
.map(|index| index % model.config.vocab_size)
.collect();
let input_ids = IndexTensor::new(tokens, vec![batch_size, sequence_length]);
let target_ids = IndexTensor::new(targets, vec![batch_size, sequence_length]);
model.zero_grad();
let logits = model
.forward_training_gpu_with_checkpointing(&input_ids, 0, false, &single_gpu)
.unwrap();
let loss = model
.compute_loss_gpu(&logits, &target_ids, &single_gpu)
.unwrap();
let single_loss = loss.to_cpu().unwrap().item();
loss.backward_gpu(&single_gpu).unwrap();
let single_grads: Vec<Vec<f32>> = parameters
.iter()
.map(|p| p.gpu_grad_to_cpu().unwrap().unwrap())
.collect();
let (dp_loss, dp_grads) = pool
.forward_backward(weights, &input_ids, &target_ids, 0, false, 1.0)
.unwrap();
assert!(
(single_loss - dp_loss).abs() < 1e-3,
"{single_loss} vs {dp_loss}"
);
for (single, reduced) in single_grads.iter().zip(dp_grads.iter()) {
assert_eq!(single.len(), reduced.len());
for (a, b) in single.iter().zip(reduced.iter()) {
assert!((a - b).abs() < 5e-3, "{a} vs {b}");
}
}
}
}