logit_loom_runtime/
request.rs1use logit_loom::{
6 CoreError, GenerationPlan, Grammar, LogitBias, MAX_LOGIT_BIASES, MAX_STOP_SEQUENCE_BYTES,
7 MAX_STOP_SEQUENCES, ObserverSet, Pipeline, SamplingPlan, TokenId,
8};
9
10use crate::Result;
11
12pub struct GenerationRequest<'controls> {
17 plan: GenerationPlan,
18 pipeline: Option<&'controls mut Pipeline>,
19 observers: Option<&'controls mut ObserverSet>,
20}
21
22impl<'controls> GenerationRequest<'controls> {
23 pub fn new(max_tokens: u32) -> Result<Self> {
29 Self::from_plan(GenerationPlan {
30 sampling: SamplingPlan::default(),
31 max_tokens,
32 biases: Vec::new(),
33 grammar: None,
34 stops: Vec::new(),
35 })
36 }
37
38 pub fn from_plan(plan: GenerationPlan) -> Result<Self> {
44 plan.validate()?;
45 Ok(Self {
46 plan,
47 pipeline: None,
48 observers: None,
49 })
50 }
51
52 pub fn sampling(mut self, sampling: SamplingPlan) -> Result<Self> {
58 self.plan.sampling = sampling;
59 self.plan.validate()?;
60 Ok(self)
61 }
62
63 pub fn bias(mut self, token: TokenId, bias: f32) -> Result<Self> {
70 if self.plan.biases.len() >= MAX_LOGIT_BIASES || !bias.is_finite() {
71 return Err(CoreError::invalid(
72 "logit biases",
73 "too many biases or a non-finite value",
74 )
75 .into());
76 }
77 if self.plan.biases.iter().any(|entry| entry.token == token) {
78 return Err(
79 CoreError::invalid("logit biases", "token identifiers must be unique").into(),
80 );
81 }
82 self.plan.biases.push(LogitBias { token, bias });
83 self.plan.validate()?;
84 Ok(self)
85 }
86
87 pub fn grammar(mut self, grammar: Grammar) -> Result<Self> {
93 self.plan.grammar = Some(grammar);
94 self.plan.validate()?;
95 Ok(self)
96 }
97
98 pub fn stop_bytes(mut self, stop: impl AsRef<[u8]>) -> Result<Self> {
105 let stop = stop.as_ref();
106 if self.plan.stops.len() >= MAX_STOP_SEQUENCES
107 || stop.is_empty()
108 || stop.len() > MAX_STOP_SEQUENCE_BYTES
109 {
110 return Err(CoreError::invalid(
111 "stop sequences",
112 format!(
113 "requires at most {MAX_STOP_SEQUENCES} non-empty sequences of at most \
114 {MAX_STOP_SEQUENCE_BYTES} bytes"
115 ),
116 )
117 .into());
118 }
119 self.plan.stops.push(stop.to_vec());
120 self.plan.validate()?;
121 Ok(self)
122 }
123
124 pub fn pipeline(mut self, pipeline: &'controls mut Pipeline) -> Result<Self> {
130 if self.pipeline.is_some() {
131 return Err(CoreError::invalid(
132 "generation pipeline",
133 "only one pipeline may be attached",
134 )
135 .into());
136 }
137 self.pipeline = Some(pipeline);
138 Ok(self)
139 }
140
141 pub fn observers(mut self, observers: &'controls mut ObserverSet) -> Result<Self> {
147 if self.observers.is_some() {
148 return Err(CoreError::invalid(
149 "generation observers",
150 "only one observer set may be attached",
151 )
152 .into());
153 }
154 self.observers = Some(observers);
155 Ok(self)
156 }
157
158 pub const fn plan(&self) -> &GenerationPlan {
160 &self.plan
161 }
162
163 pub(crate) fn into_parts(
164 self,
165 ) -> (
166 GenerationPlan,
167 Option<&'controls mut Pipeline>,
168 Option<&'controls mut ObserverSet>,
169 ) {
170 (self.plan, self.pipeline, self.observers)
171 }
172}
173
174#[cfg(test)]
175mod tests {
176 use super::*;
177
178 #[test]
179 fn maximum_tokens_are_required() {
180 assert!(GenerationRequest::new(0).is_err());
181 assert_eq!(GenerationRequest::new(1).unwrap().plan().max_tokens, 1);
182 }
183
184 #[test]
185 fn fluent_fields_preserve_exact_plan_mechanics() {
186 let token = TokenId::new(7).unwrap();
187 let request = GenerationRequest::new(12)
188 .unwrap()
189 .bias(token, 1.5)
190 .unwrap()
191 .stop_bytes([0xff, 0x00])
192 .unwrap();
193 assert_eq!(request.plan().biases, [LogitBias { token, bias: 1.5 }]);
194 assert_eq!(request.plan().stops, [vec![0xff, 0x00]]);
195 }
196
197 #[test]
198 fn duplicate_bias_and_empty_stop_fail_before_execution() {
199 let token = TokenId::new(7).unwrap();
200 assert!(
201 GenerationRequest::new(1)
202 .unwrap()
203 .bias(token, 1.0)
204 .unwrap()
205 .bias(token, 2.0)
206 .is_err()
207 );
208 assert!(
209 GenerationRequest::new(1)
210 .unwrap()
211 .bias(token, f32::NAN)
212 .is_err()
213 );
214 assert!(GenerationRequest::new(1).unwrap().stop_bytes([]).is_err());
215 let oversized = vec![0; MAX_STOP_SEQUENCE_BYTES + 1];
216 assert!(
217 GenerationRequest::new(1)
218 .unwrap()
219 .stop_bytes(&oversized)
220 .is_err()
221 );
222 }
223
224 #[test]
225 fn controls_cannot_be_attached_twice() {
226 let mut first = crate::PipelineBuilder::new(crate::CandidateMode::FullVocabulary, 1)
227 .unwrap()
228 .rank_bias(0, 1.0)
229 .unwrap()
230 .build()
231 .unwrap();
232 let mut second = crate::PipelineBuilder::new(crate::CandidateMode::FullVocabulary, 1)
233 .unwrap()
234 .rank_bias(1, 1.0)
235 .unwrap()
236 .build()
237 .unwrap();
238 let request = GenerationRequest::new(1)
239 .unwrap()
240 .pipeline(&mut first)
241 .unwrap();
242 assert!(request.pipeline(&mut second).is_err());
243 }
244}