use candle_core::{Tensor, D};
use crate::model::Rlt;
use crate::{Result, RltError};
#[derive(Clone, Debug)]
pub struct SamplingMetadata {
pub temperature: f32,
pub top_k: Option<usize>,
pub seed: u64,
}
#[derive(Clone, Debug)]
pub struct Rollout {
pub prompt_tokens: Vec<u32>,
pub response_tokens: Vec<u32>,
pub action_mask: Vec<bool>,
pub behavior_logprobs: Vec<f32>,
pub sampling: SamplingMetadata,
}
#[derive(Debug)]
pub struct ReplayResult {
pub current_logprobs: Tensor,
pub ratios: Vec<f32>,
}
pub fn replay(model: &Rlt, rollout: &Rollout) -> Result<ReplayResult> {
let n = rollout.response_tokens.len();
if rollout.action_mask.len() != n || rollout.behavior_logprobs.len() != n {
return Err(RltError::Replay(format!(
"response/mask/logprob length mismatch: {n} vs {} vs {}",
rollout.action_mask.len(),
rollout.behavior_logprobs.len()
)));
}
let (_, mut state) = model.prefill(&rollout.prompt_tokens)?;
let mut cur_logp: Vec<Tensor> = Vec::new();
let mut ratios: Vec<f32> = Vec::new();
for (i, &tok) in rollout.response_tokens.iter().enumerate() {
let logits = model.logits(&state.s)?; let lp = candle_nn::ops::log_softmax(&logits, D::Minus1)?
.gather(&Tensor::new(&[tok], &model.device)?.unsqueeze(1)?, 1)?
.squeeze(1)?; if rollout.action_mask[i] {
let scalar = lp.squeeze(0)?; let cur = scalar.to_scalar::<f32>()?;
ratios.push((cur - rollout.behavior_logprobs[i]).exp());
cur_logp.push(scalar);
}
model.step(tok, &mut state)?;
}
if cur_logp.is_empty() {
return Err(RltError::Replay("rollout has no action tokens".into()));
}
Ok(ReplayResult {
current_logprobs: Tensor::stack(&cur_logp, 0)?,
ratios,
})
}