ferrox_models/sampling/recommended.rs
1//! The sampling a **checkpoint recommends for itself**, and the
2//! precedence that resolves it against what one request asked for.
3//!
4//! Split out of `sampling.rs`: this is a policy about where a number
5//! comes from, not part of the sampler that consumes it.
6
7use super::SamplingParams;
8
9/// The sampling a **checkpoint recommends for itself**, one `Option`
10/// per field so that "this model says nothing about top_p" stays
11/// distinguishable from "this model recommends top_p = 1.0". This is
12/// sglang's `sampling_defaults='model'`, ported from FreeToken
13/// `python/freetoken/utils/hf.py:92 load_generation_sampling`.
14///
15/// Every field is `None` for a checkpoint that recommends nothing,
16/// which is the overwhelming majority, and
17/// [`RecommendedSampling::resolve`] then reproduces ferrox's existing
18/// defaults exactly -- a recommendation may only fill a gap the request
19/// left, never override it.
20///
21/// Why this exists at all: reasoning checkpoints are tuned for a
22/// specific sampler (Qwen3.5 ships temperature 1.0, top_k 20, top_p
23/// 0.95) and ship those numbers with the weights. Served under a
24/// generic greedy-or-0.8 default they fall into repetition loops --
25/// fluent output that never terminates -- which reads as a broken model
26/// rather than as a serving default nobody read off the file.
27#[derive(Debug, Clone, Copy, Default, PartialEq)]
28pub struct RecommendedSampling {
29 pub temperature: Option<f32>,
30 pub top_p: Option<f32>,
31 pub top_k: Option<usize>,
32}
33
34/// The sampling fields **one request** actually specified. `None` means
35/// the request said nothing about that field, so the checkpoint's
36/// recommendation (and then the framework default) may speak for it.
37///
38/// Collapsing this to a plain [`SamplingParams`] at the wire boundary
39/// -- `temperature: req.temperature.unwrap_or(0.0)` -- is what destroys
40/// the distinction: a request that omitted `temperature` becomes
41/// indistinguishable from one that explicitly asked for greedy, and no
42/// recommendation can ever apply.
43#[derive(Debug, Clone, Copy, Default, PartialEq)]
44pub struct RequestedSampling {
45 pub temperature: Option<f32>,
46 pub top_p: Option<f32>,
47 pub top_k: Option<usize>,
48}
49
50impl RecommendedSampling {
51 /// True when the checkpoint recommended nothing at all, i.e.
52 /// [`Self::resolve`] is guaranteed to return the framework defaults
53 /// for any request. Useful for telling an operator whether
54 /// "model defaults" had anything to act on.
55 pub fn is_empty(&self) -> bool {
56 *self == RecommendedSampling::default()
57 }
58
59 /// Precedence, exactly as FreeToken's `resolve_sampling.pick`
60 /// (`python/freetoken/server/generation.py:170`) applies it: the
61 /// **request's** own value, else the **checkpoint's**
62 /// recommendation, else the **framework** default carried by
63 /// `framework` (ferrox's `SamplingParams::default()` unless a caller
64 /// has its own).
65 ///
66 /// The penalty fields are taken from `framework` untouched: nothing
67 /// in the reference reads a recommended penalty, and inventing one
68 /// here would be this function changing generation on its own.
69 ///
70 /// Getting the order wrong in either direction is a silent
71 /// behaviour change: recommendation-over-request makes a client's
72 /// explicit `temperature: 0` unreachable on a model that recommends
73 /// 1.0, and framework-over-recommendation is the greedy repetition
74 /// loop this whole path exists to avoid.
75 pub fn resolve(
76 &self,
77 requested: RequestedSampling,
78 framework: SamplingParams,
79 ) -> SamplingParams {
80 SamplingParams {
81 temperature: requested
82 .temperature
83 .or(self.temperature)
84 .unwrap_or(framework.temperature),
85 top_p: requested.top_p.or(self.top_p).unwrap_or(framework.top_p),
86 top_k: requested.top_k.or(self.top_k).unwrap_or(framework.top_k),
87 ..framework
88 }
89 }
90
91 /// The recommendation in a HuggingFace-style `generation_config.json`
92 /// body.
93 ///
94 /// Two rules, both from the reference
95 /// (`hf.py:92 load_generation_sampling`):
96 ///
97 /// * `do_sample: false` means the checkpoint recommends **greedy**,
98 /// which is returned as `temperature = 0` and *nothing else* --
99 /// the top_k/top_p in such a file describe a sampler the model
100 /// asks not to be used.
101 /// * otherwise only the keys **actually present** are returned. An
102 /// absent key stays `None`; filling it with a house default (the
103 /// naive reading, and what HF's own `GenerationConfig` object does
104 /// for you) would turn silence into a recommendation and let a
105 /// file that says only `temperature: 0.6` also pin top_p to 1.0,
106 /// overriding the server's own default for a value the checkpoint
107 /// never expressed.
108 ///
109 /// A file that does not parse, or is not a JSON object, recommends
110 /// nothing -- a malformed sidecar must not be able to change how a
111 /// model is sampled.
112 pub fn from_generation_config(json: &str) -> Self {
113 let Ok(serde_json::Value::Object(map)) = serde_json::from_str::<serde_json::Value>(json)
114 else {
115 return RecommendedSampling::default();
116 };
117 if map.get("do_sample").and_then(|v| v.as_bool()) == Some(false) {
118 return RecommendedSampling {
119 temperature: Some(0.0),
120 ..RecommendedSampling::default()
121 };
122 }
123 RecommendedSampling {
124 temperature: map
125 .get("temperature")
126 .and_then(|v| v.as_f64())
127 .map(|v| v as f32),
128 top_p: map.get("top_p").and_then(|v| v.as_f64()).map(|v| v as f32),
129 top_k: map
130 .get("top_k")
131 .and_then(|v| v.as_u64())
132 .map(|v| v as usize),
133 }
134 }
135
136 /// [`Self::from_generation_config`] for the `generation_config.json`
137 /// beside a checkpoint's weights (an HF-format model directory).
138 ///
139 /// A directory with no such file recommends nothing, exactly like a
140 /// GGUF with no `general.sampling.*` keys: the absence of a
141 /// recommendation is the normal case and must never be an error that
142 /// stops a model from loading.
143 pub fn from_model_dir(dir: &std::path::Path) -> Self {
144 match std::fs::read_to_string(dir.join("generation_config.json")) {
145 Ok(text) => Self::from_generation_config(&text),
146 Err(_) => RecommendedSampling::default(),
147 }
148 }
149}
150
151#[cfg(test)]
152mod tests {
153 use super::*;
154
155 /// Only the keys the file actually carries become a recommendation.
156 ///
157 /// **This test fails if an absent key is filled with a house
158 /// default** (the naive reading, and what HF's own
159 /// `GenerationConfig` object does): `top_p` and `top_k` would come
160 /// back as `Some(1.0)` / `Some(0)` and would then override whatever
161 /// the server itself defaults to, for values this checkpoint never
162 /// expressed.
163 #[test]
164 fn an_absent_generation_config_key_stays_absent_rather_than_taking_a_default() {
165 let recommended = RecommendedSampling::from_generation_config(r#"{"temperature": 0.6}"#);
166 assert_eq!(recommended.temperature, Some(0.6));
167 assert_eq!(recommended.top_p, None, "top_p was not in the file");
168 assert_eq!(recommended.top_k, None, "top_k was not in the file");
169 // An explicit JSON null is silence too (the reference's
170 // `if val is not None`).
171 let nulled = RecommendedSampling::from_generation_config(r#"{"top_p": null}"#);
172 assert_eq!(nulled, RecommendedSampling::default());
173 }
174
175 /// A reasoning checkpoint's full recommendation survives intact --
176 /// the case the whole path exists for (Qwen3.5: temp 1.0, top_k 20,
177 /// top_p 0.95).
178 #[test]
179 fn every_generation_config_key_present_is_recommended() {
180 let recommended = RecommendedSampling::from_generation_config(
181 r#"{"do_sample": true, "temperature": 1.0, "top_k": 20, "top_p": 0.95}"#,
182 );
183 assert_eq!(
184 recommended,
185 RecommendedSampling {
186 temperature: Some(1.0),
187 top_p: Some(0.95),
188 top_k: Some(20),
189 }
190 );
191 }
192
193 /// `do_sample: false` recommends greedy, expressed as temperature 0
194 /// and *nothing else*: the top_k/top_p such a file also carries
195 /// describe a sampler it is asking not to be used, so returning them
196 /// would filter a distribution the model wants collapsed to its
197 /// argmax.
198 #[test]
199 fn do_sample_false_recommends_greedy_and_no_other_field() {
200 let recommended = RecommendedSampling::from_generation_config(
201 r#"{"do_sample": false, "temperature": 0.7, "top_k": 50, "top_p": 0.9}"#,
202 );
203 assert_eq!(recommended.temperature, Some(0.0));
204 assert_eq!(recommended.top_p, None);
205 assert_eq!(recommended.top_k, None);
206 }
207
208 /// A sidecar that does not parse must not be able to change how the
209 /// model is sampled.
210 #[test]
211 fn a_malformed_generation_config_recommends_nothing() {
212 for text in ["", "not json", "[1, 2, 3]", "null"] {
213 assert!(
214 RecommendedSampling::from_generation_config(text).is_empty(),
215 "{text:?} must recommend nothing"
216 );
217 }
218 }
219
220 /// Precedence: the request wins over the checkpoint, and the
221 /// checkpoint only fills what the request left unset. An explicit
222 /// `temperature: 0` from a client must stay reachable on a model
223 /// that recommends 1.0.
224 #[test]
225 fn a_request_outranks_the_recommendation_which_outranks_the_framework_default() {
226 let recommended = RecommendedSampling {
227 temperature: Some(1.0),
228 top_p: Some(0.95),
229 top_k: Some(20),
230 };
231 let resolved = recommended.resolve(
232 RequestedSampling {
233 temperature: Some(0.0),
234 ..RequestedSampling::default()
235 },
236 SamplingParams::default(),
237 );
238 assert_eq!(resolved.temperature, 0.0, "the request asked for greedy");
239 assert_eq!(resolved.top_p, 0.95, "the request said nothing about top_p");
240 assert_eq!(resolved.top_k, 20, "the request said nothing about top_k");
241 // Penalties are never recommended, only carried through.
242 assert_eq!(resolved.repetition_penalty, 1.0);
243 }
244
245 /// A checkpoint that recommends nothing must leave ferrox's existing
246 /// behaviour bit-identical: greedy, unfiltered, exactly
247 /// `SamplingParams::default()`.
248 #[test]
249 fn a_checkpoint_that_recommends_nothing_leaves_the_framework_defaults_alone() {
250 let resolved = RecommendedSampling::default()
251 .resolve(RequestedSampling::default(), SamplingParams::default());
252 let default = SamplingParams::default();
253 assert_eq!(resolved.temperature, default.temperature);
254 assert_eq!(resolved.top_p, default.top_p);
255 assert_eq!(resolved.top_k, default.top_k);
256 }
257
258 /// A model directory with no `generation_config.json` recommends
259 /// nothing rather than failing: the absence of a recommendation is
260 /// the normal case for most checkpoints.
261 #[test]
262 fn a_model_directory_without_a_generation_config_recommends_nothing() {
263 let dir = std::env::temp_dir().join(format!(
264 "ferrox_test_no_generation_config_{}",
265 std::process::id()
266 ));
267 std::fs::create_dir_all(&dir).unwrap();
268 assert!(RecommendedSampling::from_model_dir(&dir).is_empty());
269 std::fs::remove_dir_all(&dir).ok();
270 }
271
272 /// The sidecar is read from the directory beside the weights, the
273 /// same place HF's `GenerationConfig.from_pretrained` looks.
274 #[test]
275 fn a_model_directory_generation_config_is_read_from_beside_the_weights() {
276 let dir = std::env::temp_dir().join(format!(
277 "ferrox_test_generation_config_dir_{}",
278 std::process::id()
279 ));
280 std::fs::create_dir_all(&dir).unwrap();
281 std::fs::write(
282 dir.join("generation_config.json"),
283 r#"{"temperature": 0.6, "top_p": 0.95}"#,
284 )
285 .unwrap();
286 let recommended = RecommendedSampling::from_model_dir(&dir);
287 std::fs::remove_dir_all(&dir).ok();
288 assert_eq!(recommended.temperature, Some(0.6));
289 assert_eq!(recommended.top_p, Some(0.95));
290 assert_eq!(recommended.top_k, None);
291 }
292}