ferrox_models/sampling/params.rs
1//! [`SamplingParams`]: everything one generation request says about how
2//! the sampler chain should behave.
3//!
4//! Split out of `sampling.rs` because it is a different concept from the
5//! RNG and the chain runner that read it, and because it is the struct
6//! the whole workspace agrees about: the CLI flags build one, the two
7//! OpenAI routes and llama.cpp's native `/completion` build one, the
8//! response cache destructures one EXHAUSTIVELY (`ferrox_server::
9//! response_cache::sampling_key`) so that a knob added here and
10//! forgotten there stops that crate compiling.
11//!
12//! Every field's default is the value that makes its sampler a NO-OP,
13//! and where llama.cpp has a neutral value the two agree. llama.cpp's
14//! own CLI numbers (`--temp 0.8`, `--top-k 40`, `--min-p 0.05`) live on
15//! ferrox's CLI flags, where the person who typed them can see them, and
16//! not here: `SamplingParams::default()` is the "do nothing the caller
17//! did not ask for" baseline that an HTTP request with an empty body
18//! resolves to.
19
20use crate::dry::DryParams;
21use crate::sampler_order::SamplerOrder;
22
23/// Sampling parameters for one generation request. `temperature <= 0.0`
24/// means "sample nothing, take the greedy argmax" -- the same
25/// deterministic behavior ferrox always had before this module existed.
26#[derive(Debug, Clone)]
27pub struct SamplingParams {
28 pub temperature: f32,
29 /// Nucleus sampling threshold in (0.0, 1.0]. 1.0 disables top-p
30 /// filtering (every token with nonzero probability is eligible).
31 pub top_p: f32,
32 /// Keep only candidates at least `min_p` times as likely as the most
33 /// likely one. `0.0` disables it; llama.cpp's `--min-p`, whose
34 /// default is **0.05** (`common/common.h:231`) rather than off.
35 ///
36 /// That default is why this is a parity item and not a feature:
37 /// llama.cpp truncates with min-p on every run nobody configured,
38 /// so without it ferrox could not reproduce llama.cpp's *own*
39 /// out-of-the-box output for any prompt.
40 ///
41 /// The struct default here stays `0.0` (disabled) for the same
42 /// reason `temperature` defaults to greedy: `SamplingParams::default`
43 /// is ferrox's "do nothing the caller did not ask for" baseline, and
44 /// llama.cpp's CLI numbers live on the CLI flags.
45 pub min_p: f32,
46 /// Keep only the `top_k` highest-probability tokens before
47 /// sampling. 0 disables top-k filtering.
48 pub top_k: usize,
49 /// Locally typical sampling, llama.cpp's `typ_p`
50 /// (`common/common.h:230`, default **1.0 = disabled**).
51 ///
52 /// Keeps the candidates whose surprisal is CLOSEST to the
53 /// distribution's entropy, from the middle outward, rather than the
54 /// most likely ones -- see
55 /// [`crate::sampler_chain::Candidates::typical_p`].
56 pub typical_p: f32,
57 /// Truncate at `n` standard deviations of the logits below the
58 /// maximum, llama.cpp's `top_n_sigma` (`common/common.h:250`,
59 /// default **-1.0 = disabled**).
60 pub top_n_sigma: f32,
61 /// The probability that XTC removes the top candidates on any one
62 /// token, llama.cpp's `xtc_probability` (`common/common.h:228`,
63 /// default **0.0 = disabled**).
64 pub xtc_probability: f32,
65 /// The probability a candidate must reach to be a candidate XTC
66 /// might remove, llama.cpp's `xtc_threshold` (`common/common.h:229`,
67 /// default 0.1). **Above 0.5 disables XTC**, which is upstream's
68 /// guard and not a range check: above 0.5 at most one candidate can
69 /// ever clear it, and XTC never removes the last one.
70 pub xtc_threshold: f32,
71 /// The DRY sequence-repetition penalty. Disabled by default; see
72 /// [`crate::dry`] for why its breakers are a type invariant rather
73 /// than four more `f32`s here.
74 pub dry: DryParams,
75 /// > 1.0 discourages repeating a token already in the
76 /// > [`crate::penalty_window::PenaltyWindow`] -- prompt included;
77 /// > 1.0 disables repetition penalty. Uses the standard convention
78 /// > (divide positive logits, multiply negative ones) so the penalty
79 /// > always pushes toward *less* likely, regardless of logit sign.
80 pub repetition_penalty: f32,
81 /// How many of the most recent tokens the penalties look at, as
82 /// llama.cpp's `penalty_last_n` (`common/common.h:238`, default 64).
83 ///
84 /// `0` disables the penalties entirely. ferrox had no window at all
85 /// and scanned the WHOLE history, so on a long generation it
86 /// penalised a steadily growing set of tokens where llama.cpp
87 /// penalises the last 64 -- the divergence grew with output length,
88 /// which is exactly when a repetition penalty matters most.
89 pub penalty_last_n: usize,
90 /// OpenAI-style presence penalty: subtract from logits of tokens
91 /// that already appeared in the window (once per distinct token).
92 pub presence_penalty: f32,
93 /// OpenAI-style frequency penalty: subtract `frequency_penalty *
94 /// count` from logits for each token id seen in the window.
95 pub frequency_penalty: f32,
96 /// The ORDER the chain above runs in, llama.cpp's `--samplers`.
97 ///
98 /// Not a cosmetic setting. Each filter renormalises over the
99 /// survivors of the last one, so moving a step changes which
100 /// candidates the next step can see -- ferrox has already shipped
101 /// that bug once, with temperature running first.
102 ///
103 /// The default is llama.cpp's own default chain
104 /// (`penalties;dry;top_n_sigma;top_k;typ_p;top_p;min_p;xtc;temperature`),
105 /// and every step ferrox added to it is a no-op at the neutral
106 /// values above. See [`crate::sampler_order`].
107 pub sampler_order: SamplerOrder,
108}
109
110impl Default for SamplingParams {
111 /// Greedy decoding: identical behavior to ferrox's original
112 /// argmax-only generation loop.
113 fn default() -> Self {
114 SamplingParams {
115 temperature: 0.0,
116 top_p: 1.0,
117 min_p: 0.0,
118 top_k: 0,
119 typical_p: 1.0,
120 top_n_sigma: -1.0,
121 xtc_probability: 0.0,
122 xtc_threshold: 0.1,
123 dry: DryParams::off(),
124 repetition_penalty: 1.0,
125 penalty_last_n: 64,
126 presence_penalty: 0.0,
127 frequency_penalty: 0.0,
128 sampler_order: SamplerOrder::default(),
129 }
130 }
131}
132
133impl SamplingParams {
134 /// The single predicate for "XTC can remove something".
135 ///
136 /// llama.cpp tests the same two conditions in two places --
137 /// `llama_sampler_init_xtc` returns an empty sampler at `:2208` and
138 /// `llama_sample_xtc_apply` returns early at `:2139` -- and this is
139 /// one function because ferrox reads it in two places too: the RNG
140 /// draw ([`super::Sampler::xtc_roll`]) and the filter itself. If
141 /// those disagreed, either the seeded stream would advance on a run
142 /// XTC never touched (making an existing generation irreproducible)
143 /// or XTC would ask for a draw nobody made.
144 pub fn xtc_can_fire(&self) -> bool {
145 self.xtc_probability > 0.0 && self.xtc_threshold <= 0.5
146 }
147
148 // The two "may an argmax stand in for the chain" predicates --
149 // [`Self::chain_keeps_the_argmax`] and
150 // [`Self::greedy_equals_raw_argmax`] -- are in
151 // [`super::greedy_equivalence`], with the exhaustive destructure of
152 // THIS struct that a knob added below must satisfy before the crate
153 // compiles. They used to be one function here, and it hand-listed
154 // three of the chain's nine steps: GitHub issue #170.
155}
156
157#[cfg(test)]
158mod tests {
159 use super::*;
160
161 /// Every knob this struct defaults to is the value that makes its
162 /// sampler do nothing, which is what lets llama.cpp's full default
163 /// chain be ferrox's default chain without changing any existing
164 /// run's output.
165 ///
166 /// The neutral values are llama.cpp's own (`common/common.h:228-250`):
167 /// `typ_p 1.00`, `top_n_sigma -1.00`, `xtc_probability 0.00`,
168 /// `dry_multiplier 0.0`. `xtc_threshold` defaults to upstream's
169 /// 0.10, which is NOT neutral on its own -- the probability is what
170 /// switches XTC off -- so it is pinned here rather than assumed.
171 #[test]
172 fn every_new_sampler_defaults_to_its_own_no_op() {
173 let d = SamplingParams::default();
174 assert_eq!(d.typical_p, 1.0, "1.0 disables typical-p");
175 assert_eq!(d.top_n_sigma, -1.0, "<= 0 disables top-n-sigma");
176 assert_eq!(d.xtc_probability, 0.0, "0.0 disables xtc");
177 assert_eq!(d.xtc_threshold, 0.1, "llama.cpp's default threshold");
178 assert!(!d.xtc_can_fire());
179 assert!(!d.dry.is_enabled(), "dry_multiplier 0.0 disables dry");
180 }
181
182 /// Both halves of the XTC guard, because only one of them is
183 /// obvious. A threshold above 0.5 disables XTC outright upstream: at
184 /// most one candidate can hold more than half the mass, and XTC
185 /// never removes the last candidate above the threshold.
186 #[test]
187 fn a_threshold_above_a_half_disables_xtc_as_surely_as_a_zero_probability() {
188 let live = SamplingParams {
189 xtc_probability: 0.5,
190 xtc_threshold: 0.1,
191 ..SamplingParams::default()
192 };
193 assert!(live.xtc_can_fire());
194 assert!(!SamplingParams {
195 xtc_threshold: 0.51,
196 ..live.clone()
197 }
198 .xtc_can_fire());
199 assert!(
200 SamplingParams {
201 xtc_threshold: 0.5,
202 ..live.clone()
203 }
204 .xtc_can_fire(),
205 "0.5 itself is still live"
206 );
207 assert!(!SamplingParams {
208 xtc_probability: 0.0,
209 ..live
210 }
211 .xtc_can_fire());
212 }
213}