use logit_loom::{
CoreError, GenerationPlan, Grammar, LogitBias, MAX_LOGIT_BIASES, MAX_STOP_SEQUENCE_BYTES,
MAX_STOP_SEQUENCES, ObserverSet, Pipeline, SamplingPlan, TokenId,
};
use crate::Result;
pub struct GenerationRequest<'controls> {
plan: GenerationPlan,
pipeline: Option<&'controls mut Pipeline>,
observers: Option<&'controls mut ObserverSet>,
}
impl<'controls> GenerationRequest<'controls> {
pub fn new(max_tokens: u32) -> Result<Self> {
Self::from_plan(GenerationPlan {
sampling: SamplingPlan::default(),
max_tokens,
biases: Vec::new(),
grammar: None,
stops: Vec::new(),
})
}
pub fn from_plan(plan: GenerationPlan) -> Result<Self> {
plan.validate()?;
Ok(Self {
plan,
pipeline: None,
observers: None,
})
}
pub fn sampling(mut self, sampling: SamplingPlan) -> Result<Self> {
self.plan.sampling = sampling;
self.plan.validate()?;
Ok(self)
}
pub fn bias(mut self, token: TokenId, bias: f32) -> Result<Self> {
if self.plan.biases.len() >= MAX_LOGIT_BIASES || !bias.is_finite() {
return Err(CoreError::invalid(
"logit biases",
"too many biases or a non-finite value",
)
.into());
}
if self.plan.biases.iter().any(|entry| entry.token == token) {
return Err(
CoreError::invalid("logit biases", "token identifiers must be unique").into(),
);
}
self.plan.biases.push(LogitBias { token, bias });
self.plan.validate()?;
Ok(self)
}
pub fn grammar(mut self, grammar: Grammar) -> Result<Self> {
self.plan.grammar = Some(grammar);
self.plan.validate()?;
Ok(self)
}
pub fn stop_bytes(mut self, stop: impl AsRef<[u8]>) -> Result<Self> {
let stop = stop.as_ref();
if self.plan.stops.len() >= MAX_STOP_SEQUENCES
|| stop.is_empty()
|| stop.len() > MAX_STOP_SEQUENCE_BYTES
{
return Err(CoreError::invalid(
"stop sequences",
format!(
"requires at most {MAX_STOP_SEQUENCES} non-empty sequences of at most \
{MAX_STOP_SEQUENCE_BYTES} bytes"
),
)
.into());
}
self.plan.stops.push(stop.to_vec());
self.plan.validate()?;
Ok(self)
}
pub fn pipeline(mut self, pipeline: &'controls mut Pipeline) -> Result<Self> {
if self.pipeline.is_some() {
return Err(CoreError::invalid(
"generation pipeline",
"only one pipeline may be attached",
)
.into());
}
self.pipeline = Some(pipeline);
Ok(self)
}
pub fn observers(mut self, observers: &'controls mut ObserverSet) -> Result<Self> {
if self.observers.is_some() {
return Err(CoreError::invalid(
"generation observers",
"only one observer set may be attached",
)
.into());
}
self.observers = Some(observers);
Ok(self)
}
pub const fn plan(&self) -> &GenerationPlan {
&self.plan
}
pub(crate) fn into_parts(
self,
) -> (
GenerationPlan,
Option<&'controls mut Pipeline>,
Option<&'controls mut ObserverSet>,
) {
(self.plan, self.pipeline, self.observers)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn maximum_tokens_are_required() {
assert!(GenerationRequest::new(0).is_err());
assert_eq!(GenerationRequest::new(1).unwrap().plan().max_tokens, 1);
}
#[test]
fn fluent_fields_preserve_exact_plan_mechanics() {
let token = TokenId::new(7).unwrap();
let request = GenerationRequest::new(12)
.unwrap()
.bias(token, 1.5)
.unwrap()
.stop_bytes([0xff, 0x00])
.unwrap();
assert_eq!(request.plan().biases, [LogitBias { token, bias: 1.5 }]);
assert_eq!(request.plan().stops, [vec![0xff, 0x00]]);
}
#[test]
fn duplicate_bias_and_empty_stop_fail_before_execution() {
let token = TokenId::new(7).unwrap();
assert!(
GenerationRequest::new(1)
.unwrap()
.bias(token, 1.0)
.unwrap()
.bias(token, 2.0)
.is_err()
);
assert!(
GenerationRequest::new(1)
.unwrap()
.bias(token, f32::NAN)
.is_err()
);
assert!(GenerationRequest::new(1).unwrap().stop_bytes([]).is_err());
let oversized = vec![0; MAX_STOP_SEQUENCE_BYTES + 1];
assert!(
GenerationRequest::new(1)
.unwrap()
.stop_bytes(&oversized)
.is_err()
);
}
#[test]
fn controls_cannot_be_attached_twice() {
let mut first = crate::PipelineBuilder::new(crate::CandidateMode::FullVocabulary, 1)
.unwrap()
.rank_bias(0, 1.0)
.unwrap()
.build()
.unwrap();
let mut second = crate::PipelineBuilder::new(crate::CandidateMode::FullVocabulary, 1)
.unwrap()
.rank_bias(1, 1.0)
.unwrap()
.build()
.unwrap();
let request = GenerationRequest::new(1)
.unwrap()
.pipeline(&mut first)
.unwrap();
assert!(request.pipeline(&mut second).is_err());
}
}