ferrox_models/sampling/params.rs
1//! [`SamplingParams`]: everything one generation request says about how
2//! the sampler chain should behave.
3//!
4//! Split out of `sampling.rs` because it is a different concept from the
5//! RNG and the chain runner that read it, and because it is the struct
6//! the whole workspace agrees about: the CLI flags build one, the two
7//! OpenAI routes and llama.cpp's native `/completion` build one, the
8//! response cache destructures one EXHAUSTIVELY (`ferrox_server::
9//! response_cache::sampling_key`) so that a knob added here and
10//! forgotten there stops that crate compiling.
11//!
12//! Every field's default is the value that makes its sampler a NO-OP,
13//! and where llama.cpp has a neutral value the two agree. llama.cpp's
14//! own CLI numbers (`--temp 0.8`, `--top-k 40`, `--min-p 0.05`) live on
15//! ferrox's CLI flags, where the person who typed them can see them, and
16//! not here: `SamplingParams::default()` is the "do nothing the caller
17//! did not ask for" baseline that an HTTP request with an empty body
18//! resolves to.
19
20use crate::dry::DryParams;
21use crate::sampler_order::{ChainStep, SamplerOrder};
22
23/// Sampling parameters for one generation request. `temperature <= 0.0`
24/// means "sample nothing, take the greedy argmax" -- the same
25/// deterministic behavior ferrox always had before this module existed.
26#[derive(Debug, Clone)]
27pub struct SamplingParams {
28 pub temperature: f32,
29 /// Nucleus sampling threshold in (0.0, 1.0]. 1.0 disables top-p
30 /// filtering (every token with nonzero probability is eligible).
31 pub top_p: f32,
32 /// Keep only candidates at least `min_p` times as likely as the most
33 /// likely one. `0.0` disables it; llama.cpp's `--min-p`, whose
34 /// default is **0.05** (`common/common.h:231`) rather than off.
35 ///
36 /// That default is why this is a parity item and not a feature:
37 /// llama.cpp truncates with min-p on every run nobody configured,
38 /// so without it ferrox could not reproduce llama.cpp's *own*
39 /// out-of-the-box output for any prompt.
40 ///
41 /// The struct default here stays `0.0` (disabled) for the same
42 /// reason `temperature` defaults to greedy: `SamplingParams::default`
43 /// is ferrox's "do nothing the caller did not ask for" baseline, and
44 /// llama.cpp's CLI numbers live on the CLI flags.
45 pub min_p: f32,
46 /// Keep only the `top_k` highest-probability tokens before
47 /// sampling. 0 disables top-k filtering.
48 pub top_k: usize,
49 /// Locally typical sampling, llama.cpp's `typ_p`
50 /// (`common/common.h:230`, default **1.0 = disabled**).
51 ///
52 /// Keeps the candidates whose surprisal is CLOSEST to the
53 /// distribution's entropy, from the middle outward, rather than the
54 /// most likely ones -- see
55 /// [`crate::sampler_chain::Candidates::typical_p`].
56 pub typical_p: f32,
57 /// Truncate at `n` standard deviations of the logits below the
58 /// maximum, llama.cpp's `top_n_sigma` (`common/common.h:250`,
59 /// default **-1.0 = disabled**).
60 pub top_n_sigma: f32,
61 /// The probability that XTC removes the top candidates on any one
62 /// token, llama.cpp's `xtc_probability` (`common/common.h:228`,
63 /// default **0.0 = disabled**).
64 pub xtc_probability: f32,
65 /// The probability a candidate must reach to be a candidate XTC
66 /// might remove, llama.cpp's `xtc_threshold` (`common/common.h:229`,
67 /// default 0.1). **Above 0.5 disables XTC**, which is upstream's
68 /// guard and not a range check: above 0.5 at most one candidate can
69 /// ever clear it, and XTC never removes the last one.
70 pub xtc_threshold: f32,
71 /// The DRY sequence-repetition penalty. Disabled by default; see
72 /// [`crate::dry`] for why its breakers are a type invariant rather
73 /// than four more `f32`s here.
74 pub dry: DryParams,
75 /// > 1.0 discourages repeating a token already in the
76 /// > [`crate::penalty_window::PenaltyWindow`] -- prompt included;
77 /// > 1.0 disables repetition penalty. Uses the standard convention
78 /// > (divide positive logits, multiply negative ones) so the penalty
79 /// > always pushes toward *less* likely, regardless of logit sign.
80 pub repetition_penalty: f32,
81 /// How many of the most recent tokens the penalties look at, as
82 /// llama.cpp's `penalty_last_n` (`common/common.h:238`, default 64).
83 ///
84 /// `0` disables the penalties entirely. ferrox had no window at all
85 /// and scanned the WHOLE history, so on a long generation it
86 /// penalised a steadily growing set of tokens where llama.cpp
87 /// penalises the last 64 -- the divergence grew with output length,
88 /// which is exactly when a repetition penalty matters most.
89 pub penalty_last_n: usize,
90 /// OpenAI-style presence penalty: subtract from logits of tokens
91 /// that already appeared in the window (once per distinct token).
92 pub presence_penalty: f32,
93 /// OpenAI-style frequency penalty: subtract `frequency_penalty *
94 /// count` from logits for each token id seen in the window.
95 pub frequency_penalty: f32,
96 /// The ORDER the chain above runs in, llama.cpp's `--samplers`.
97 ///
98 /// Not a cosmetic setting. Each filter renormalises over the
99 /// survivors of the last one, so moving a step changes which
100 /// candidates the next step can see -- ferrox has already shipped
101 /// that bug once, with temperature running first.
102 ///
103 /// The default is llama.cpp's own default chain
104 /// (`penalties;dry;top_n_sigma;top_k;typ_p;top_p;min_p;xtc;temperature`),
105 /// and every step ferrox added to it is a no-op at the neutral
106 /// values above. See [`crate::sampler_order`].
107 pub sampler_order: SamplerOrder,
108}
109
110impl Default for SamplingParams {
111 /// Greedy decoding: identical behavior to ferrox's original
112 /// argmax-only generation loop.
113 fn default() -> Self {
114 SamplingParams {
115 temperature: 0.0,
116 top_p: 1.0,
117 min_p: 0.0,
118 top_k: 0,
119 typical_p: 1.0,
120 top_n_sigma: -1.0,
121 xtc_probability: 0.0,
122 xtc_threshold: 0.1,
123 dry: DryParams::off(),
124 repetition_penalty: 1.0,
125 penalty_last_n: 64,
126 presence_penalty: 0.0,
127 frequency_penalty: 0.0,
128 sampler_order: SamplerOrder::default(),
129 }
130 }
131}
132
133impl SamplingParams {
134 /// The single predicate for "XTC can remove something".
135 ///
136 /// llama.cpp tests the same two conditions in two places --
137 /// `llama_sampler_init_xtc` returns an empty sampler at `:2208` and
138 /// `llama_sample_xtc_apply` returns early at `:2139` -- and this is
139 /// one function because ferrox reads it in two places too: the RNG
140 /// draw ([`super::Sampler::xtc_roll`]) and the filter itself. If
141 /// those disagreed, either the seeded stream would advance on a run
142 /// XTC never touched (making an existing generation irreproducible)
143 /// or XTC would ask for a draw nobody made.
144 pub fn xtc_can_fire(&self) -> bool {
145 self.xtc_probability > 0.0 && self.xtc_threshold <= 0.5
146 }
147
148 /// True when no step in this chain can move the argmax, so greedy
149 /// decoding may skip building the candidate list entirely -- and a
150 /// backend may fold `lm_head + argmax` into its decode stack.
151 ///
152 /// llama.cpp does not special-case `temp <= 0`: it runs the whole
153 /// chain and lets the temperature step set every logit but the
154 /// maximum to `-inf` (`src/llama-sampler.cpp:271-286`), so a filter
155 /// that removed the maximum changes greedy output. Exactly two do:
156 ///
157 /// * `xtc` removes the TOP candidates, by construction;
158 /// * `typ_p` selects outward from the distribution's entropy and can
159 /// drop the most likely token -- llama.cpp's own test case
160 /// `test_typical({0.4, 0.2, 0.2, 0.2}, {0.2, 0.2, 0.2}, 0.5)`
161 /// (`tests/test-sampling.cpp:346`) drops it.
162 ///
163 /// `dry` changes logits rather than removing candidates, but it can
164 /// change WHICH logit is the maximum, so it counts too. `top_k`,
165 /// `top_p`, `min_p` and `top_n_sigma` all keep the maximum by
166 /// construction, and `penalties` is applied to the whole vocabulary
167 /// before the candidate list exists.
168 ///
169 /// **One predicate, three readers**, because the alternative is this
170 /// repo's dominant defect: the sampler's own greedy shortcut
171 /// ([`super::greedy_choice`]), the Metal `lm_head + argmax` fold in
172 /// `ferrox_server::generate` and the same fold in `ferrox_cli::run`
173 /// must agree about it. A fold that ran while `xtc` was configured
174 /// would hand the sampler a single precomputed id with no
175 /// vocabulary left to remove anything from, and XTC would silently
176 /// not run.
177 ///
178 /// Chain membership is checked, not just the knob: a caller who set
179 /// `xtc_probability` but left `xtc` out of `--samplers` asked for no
180 /// XTC, and must keep the fast path.
181 pub fn greedy_equals_argmax(&self) -> bool {
182 let runs = |step| self.sampler_order.steps().contains(&step);
183 !(runs(ChainStep::Xtc) && self.xtc_can_fire())
184 && !(runs(ChainStep::TypP) && self.typical_p < 1.0)
185 && !(runs(ChainStep::Dry) && self.dry.is_enabled())
186 }
187}
188
189#[cfg(test)]
190mod tests {
191 use super::*;
192
193 /// Every knob this struct defaults to is the value that makes its
194 /// sampler do nothing, which is what lets llama.cpp's full default
195 /// chain be ferrox's default chain without changing any existing
196 /// run's output.
197 ///
198 /// The neutral values are llama.cpp's own (`common/common.h:228-250`):
199 /// `typ_p 1.00`, `top_n_sigma -1.00`, `xtc_probability 0.00`,
200 /// `dry_multiplier 0.0`. `xtc_threshold` defaults to upstream's
201 /// 0.10, which is NOT neutral on its own -- the probability is what
202 /// switches XTC off -- so it is pinned here rather than assumed.
203 #[test]
204 fn every_new_sampler_defaults_to_its_own_no_op() {
205 let d = SamplingParams::default();
206 assert_eq!(d.typical_p, 1.0, "1.0 disables typical-p");
207 assert_eq!(d.top_n_sigma, -1.0, "<= 0 disables top-n-sigma");
208 assert_eq!(d.xtc_probability, 0.0, "0.0 disables xtc");
209 assert_eq!(d.xtc_threshold, 0.1, "llama.cpp's default threshold");
210 assert!(!d.xtc_can_fire());
211 assert!(!d.dry.is_enabled(), "dry_multiplier 0.0 disables dry");
212 }
213
214 /// Both halves of the XTC guard, because only one of them is
215 /// obvious. A threshold above 0.5 disables XTC outright upstream: at
216 /// most one candidate can hold more than half the mass, and XTC
217 /// never removes the last candidate above the threshold.
218 #[test]
219 fn a_threshold_above_a_half_disables_xtc_as_surely_as_a_zero_probability() {
220 let live = SamplingParams {
221 xtc_probability: 0.5,
222 xtc_threshold: 0.1,
223 ..SamplingParams::default()
224 };
225 assert!(live.xtc_can_fire());
226 assert!(!SamplingParams {
227 xtc_threshold: 0.51,
228 ..live.clone()
229 }
230 .xtc_can_fire());
231 assert!(
232 SamplingParams {
233 xtc_threshold: 0.5,
234 ..live.clone()
235 }
236 .xtc_can_fire(),
237 "0.5 itself is still live"
238 );
239 assert!(!SamplingParams {
240 xtc_probability: 0.0,
241 ..live
242 }
243 .xtc_can_fire());
244 }
245}