use core::fmt;
pub const TALKER_KV_VALUES_PER_TOKEN: u64 = 57_344;
pub const MICRODECODER_KV_BYTES: u64 = 320 * 1024;
pub const CODEC_DECODER_KV_BYTES: u64 = 2_359_296;
pub const MAX_CONTEXT_TOKENS: u64 = 32_768;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub enum KvDtype {
Bf16,
F32,
}
impl KvDtype {
#[must_use]
pub const fn size_bytes(self) -> u64 {
match self {
Self::Bf16 => 2,
Self::F32 => 4,
}
}
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Bf16 => "bf16",
Self::F32 => "f32",
}
}
}
impl fmt::Display for KvDtype {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum BindingConstraint {
FrameCap,
ContextCeiling,
TextHeuristic,
}
impl BindingConstraint {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::FrameCap => "frame_cap",
Self::ContextCeiling => "context_ceiling",
Self::TextHeuristic => "text_heuristic",
}
}
}
pub const HEURISTIC_FRAMES_PER_PROMPT_TOKEN: u64 = 4;
pub const HEURISTIC_FRAME_HEADROOM: u64 = 64;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AdmissionRequest {
pub prompt_tokens: u64,
pub max_new_tokens: u64,
pub heuristic_eos_backstop: bool,
pub kv_dtype: KvDtype,
pub ring_buffer_bytes: u64,
pub weights_resident_bytes: u64,
pub budget_bytes: u64,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AdmissionPlan {
pub predicted_max_frames: u64,
pub binding_constraint: BindingConstraint,
pub kv_talker_bytes: u64,
pub bounded_state_bytes: u64,
pub weights_resident_bytes: u64,
pub predicted_peak_bytes: u64,
pub budget_bytes: u64,
}
impl AdmissionPlan {
#[must_use]
pub const fn shortfall_bytes(&self) -> u64 {
self.predicted_peak_bytes.saturating_sub(self.budget_bytes)
}
#[must_use]
pub const fn fits(&self) -> bool {
self.predicted_peak_bytes <= self.budget_bytes
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum AdmissionRejection {
PromptExceedsContext {
prompt_tokens: u64,
ceiling: u64,
},
NoFramesRequested,
BudgetExceeded {
plan: AdmissionPlan,
},
Overflow {
term: &'static str,
},
}
impl fmt::Display for AdmissionRejection {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::PromptExceedsContext {
prompt_tokens,
ceiling,
} => write!(
f,
"prompt is {prompt_tokens} tokens but the context ceiling is {ceiling}; \
no frames could be generated. Shorten the text or chunk it — raising the memory \
budget cannot help"
),
Self::NoFramesRequested => {
f.write_str("max_new_tokens is 0; there is nothing to synthesize")
}
Self::BudgetExceeded { plan } => write!(
f,
"predicted peak {} bytes exceeds the {} byte budget by {} \
(talker KV {} over {} frames, bounded state {}, weights {}; binding constraint: {}). \
Rejected before allocating, so nothing was committed",
plan.predicted_peak_bytes,
plan.budget_bytes,
plan.shortfall_bytes(),
plan.kv_talker_bytes,
plan.predicted_max_frames,
plan.bounded_state_bytes,
plan.weights_resident_bytes,
plan.binding_constraint.as_str(),
),
Self::Overflow { term } => write!(
f,
"admission arithmetic overflowed computing `{term}`; refusing rather than \
admitting on a wrapped total"
),
}
}
}
impl core::error::Error for AdmissionRejection {}
pub fn talker_kv_bytes(
prompt_tokens: u64,
frames: u64,
dtype: KvDtype,
) -> Result<u64, AdmissionRejection> {
prompt_tokens
.checked_add(frames)
.and_then(|tokens| tokens.checked_mul(TALKER_KV_VALUES_PER_TOKEN))
.and_then(|values| values.checked_mul(dtype.size_bytes()))
.ok_or(AdmissionRejection::Overflow {
term: "talker_kv_bytes",
})
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AdmissionPolicy {
pub budget_bytes: u64,
pub max_new_tokens: u64,
pub heuristic_eos_backstop: bool,
pub kv_dtype: KvDtype,
pub ring_buffer_bytes: u64,
pub weights_resident_bytes: u64,
}
pub const DEFAULT_BUDGET_BYTES: u64 = 2 * 1024 * 1024 * 1024;
pub const DEFAULT_MAX_NEW_TOKENS: u64 = 8_192;
impl Default for AdmissionPolicy {
fn default() -> Self {
Self {
budget_bytes: DEFAULT_BUDGET_BYTES,
max_new_tokens: DEFAULT_MAX_NEW_TOKENS,
heuristic_eos_backstop: true,
kv_dtype: KvDtype::Bf16,
ring_buffer_bytes: 0,
weights_resident_bytes: 0,
}
}
}
impl AdmissionPolicy {
#[must_use]
pub const fn request_for(&self, prompt_tokens: u64) -> AdmissionRequest {
AdmissionRequest {
prompt_tokens,
max_new_tokens: self.max_new_tokens,
heuristic_eos_backstop: self.heuristic_eos_backstop,
kv_dtype: self.kv_dtype,
ring_buffer_bytes: self.ring_buffer_bytes,
weights_resident_bytes: self.weights_resident_bytes,
budget_bytes: self.budget_bytes,
}
}
pub fn admit(&self, prompt_tokens: u64) -> Result<AdmissionPlan, AdmissionRejection> {
admit(&self.request_for(prompt_tokens))
}
}
pub fn admit(request: &AdmissionRequest) -> Result<AdmissionPlan, AdmissionRejection> {
if request.prompt_tokens >= MAX_CONTEXT_TOKENS {
return Err(AdmissionRejection::PromptExceedsContext {
prompt_tokens: request.prompt_tokens,
ceiling: MAX_CONTEXT_TOKENS,
});
}
if request.max_new_tokens == 0 {
return Err(AdmissionRejection::NoFramesRequested);
}
let headroom = MAX_CONTEXT_TOKENS - request.prompt_tokens;
let heuristic_cap = if request.heuristic_eos_backstop {
request
.prompt_tokens
.saturating_mul(HEURISTIC_FRAMES_PER_PROMPT_TOKEN)
.saturating_add(HEURISTIC_FRAME_HEADROOM)
} else {
u64::MAX
};
let predicted_max_frames = request.max_new_tokens.min(headroom).min(heuristic_cap);
let binding_constraint = if predicted_max_frames == heuristic_cap
&& heuristic_cap < request.max_new_tokens.min(headroom)
{
BindingConstraint::TextHeuristic
} else if request.max_new_tokens <= headroom {
BindingConstraint::FrameCap
} else {
BindingConstraint::ContextCeiling
};
let kv_talker_bytes = talker_kv_bytes(
request.prompt_tokens,
predicted_max_frames,
request.kv_dtype,
)?;
let bounded_state_bytes = MICRODECODER_KV_BYTES
.checked_add(CODEC_DECODER_KV_BYTES)
.and_then(|sum| sum.checked_add(request.ring_buffer_bytes))
.ok_or(AdmissionRejection::Overflow {
term: "bounded_state_bytes",
})?;
let predicted_peak_bytes = kv_talker_bytes
.checked_add(bounded_state_bytes)
.and_then(|sum| sum.checked_add(request.weights_resident_bytes))
.ok_or(AdmissionRejection::Overflow {
term: "predicted_peak_bytes",
})?;
let plan = AdmissionPlan {
predicted_max_frames,
binding_constraint,
kv_talker_bytes,
bounded_state_bytes,
weights_resident_bytes: request.weights_resident_bytes,
predicted_peak_bytes,
budget_bytes: request.budget_bytes,
};
if plan.fits() {
Ok(plan)
} else {
Err(AdmissionRejection::BudgetExceeded { plan })
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum StopReason {
EndOfSpeech,
FrameCapReached,
DurationLimitReached,
Cancelled,
}
impl StopReason {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::EndOfSpeech => "end_of_speech",
Self::FrameCapReached => "frame_cap_reached",
Self::DurationLimitReached => "duration_limit_reached",
Self::Cancelled => "cancelled",
}
}
#[must_use]
pub const fn is_truncated(self) -> bool {
matches!(self, Self::FrameCapReached | Self::DurationLimitReached)
}
#[must_use]
pub const fn is_clean_completion(self) -> bool {
matches!(self, Self::EndOfSpeech)
}
#[must_use]
pub const fn remedy(self) -> Option<&'static str> {
match self {
Self::EndOfSpeech | Self::Cancelled => None,
Self::FrameCapReached => Some(
"the utterance hit the frame cap before the model finished speaking; raise \
--max-frames or split the text into shorter chunks",
),
Self::DurationLimitReached => Some(
"the utterance hit the hard duration limit; raise it or split the text into \
shorter chunks",
),
}
}
}
impl fmt::Display for StopReason {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
#[cfg(test)]
mod tests {
use super::*;
const MIB: u64 = 1024 * 1024;
const GIB: u64 = 1024 * 1024 * 1024;
fn request(prompt_tokens: u64, max_new_tokens: u64, budget_bytes: u64) -> AdmissionRequest {
AdmissionRequest {
prompt_tokens,
max_new_tokens,
heuristic_eos_backstop: false,
kv_dtype: KvDtype::Bf16,
ring_buffer_bytes: 0,
weights_resident_bytes: 0,
budget_bytes,
}
}
#[test]
fn the_eos_backstop_binds_a_short_prompt_under_the_flat_default_cap() {
let mut with_backstop = request(28, DEFAULT_MAX_NEW_TOKENS, 2 * GIB);
with_backstop.heuristic_eos_backstop = true;
let plan = admit(&with_backstop).expect("fits easily");
assert_eq!(
plan.predicted_max_frames,
28 * HEURISTIC_FRAMES_PER_PROMPT_TOKEN + HEURISTIC_FRAME_HEADROOM
);
assert_eq!(plan.binding_constraint, BindingConstraint::TextHeuristic);
}
#[test]
fn an_explicit_cap_disables_the_eos_backstop_exactly() {
let explicit = request(28, 2_000, 2 * GIB);
let plan = admit(&explicit).expect("fits");
assert_eq!(plan.predicted_max_frames, 2_000);
assert_eq!(plan.binding_constraint, BindingConstraint::FrameCap);
}
#[test]
fn the_backstop_never_raises_a_smaller_explicit_cap() {
let mut small = request(1_000, 32, 2 * GIB);
small.heuristic_eos_backstop = true;
let plan = admit(&small).expect("fits");
assert_eq!(plan.predicted_max_frames, 32);
assert_eq!(plan.binding_constraint, BindingConstraint::FrameCap);
}
#[test]
fn talker_kv_matches_the_oq6_worked_points_exactly() {
assert_eq!(
talker_kv_bytes(512, 2048, KvDtype::Bf16).expect("no overflow"),
280 * MIB,
"512-token prompt + 2048-frame cap must be exactly 280 MiB"
);
assert_eq!(
talker_kv_bytes(512, 8192, KvDtype::Bf16).expect("no overflow"),
952 * MIB,
"512-token prompt + 8192-frame cap must be exactly 952 MiB"
);
assert_eq!(
talker_kv_bytes(0, MAX_CONTEXT_TOKENS, KvDtype::Bf16).expect("no overflow"),
7 * GIB / 2,
"the full 32768-token context must be exactly 3.50 GiB"
);
assert_eq!(
talker_kv_bytes(1, 0, KvDtype::Bf16).expect("no overflow"),
112 * 1024
);
assert_eq!(
talker_kv_bytes(512, 2048, KvDtype::F32).expect("no overflow"),
560 * MIB
);
}
#[test]
fn a_request_that_fits_is_admitted_with_its_full_prediction() {
let plan = admit(&request(512, 2048, 2 * GIB)).expect("must be admitted");
assert_eq!(plan.predicted_max_frames, 2048);
assert_eq!(plan.binding_constraint, BindingConstraint::FrameCap);
assert_eq!(plan.kv_talker_bytes, 280 * MIB);
assert_eq!(
plan.bounded_state_bytes,
MICRODECODER_KV_BYTES + CODEC_DECODER_KV_BYTES
);
assert!(plan.fits());
assert_eq!(plan.shortfall_bytes(), 0);
}
#[test]
fn an_over_budget_request_is_rejected_before_any_allocation_and_says_by_how_much() {
let error = admit(&request(512, 8192, 512 * MIB)).expect_err("must be rejected");
let AdmissionRejection::BudgetExceeded { plan } = error else {
panic!("expected a budget rejection, got {error}");
};
assert!(!plan.fits());
assert_eq!(plan.kv_talker_bytes, 952 * MIB);
assert!(plan.shortfall_bytes() > 0);
let rendered = error.to_string();
for expected in ["predicted peak", "budget", "exceeds", "before allocating"] {
assert!(
rendered.contains(expected),
"rejection is not actionable, missing `{expected}`: {rendered}"
);
}
}
#[test]
fn the_binding_constraint_is_reported_because_the_two_have_different_remedies() {
let plan = admit(&request(512, 8192, 8 * GIB)).expect("admitted");
assert_eq!(plan.binding_constraint, BindingConstraint::FrameCap);
assert_eq!(plan.predicted_max_frames, 8192);
let prompt = MAX_CONTEXT_TOKENS - 100;
let plan = admit(&request(prompt, 8192, 8 * GIB)).expect("admitted");
assert_eq!(plan.binding_constraint, BindingConstraint::ContextCeiling);
assert_eq!(plan.predicted_max_frames, 100);
let plan = admit(&request(24_000, 8192, 8 * GIB)).expect("admitted");
assert_eq!(plan.binding_constraint, BindingConstraint::FrameCap);
}
#[test]
fn a_prompt_at_or_past_the_ceiling_is_refused_as_unfixable_by_memory() {
for prompt in [MAX_CONTEXT_TOKENS, MAX_CONTEXT_TOKENS + 1, u64::MAX] {
let error = admit(&request(prompt, 1024, u64::MAX)).expect_err("must be rejected");
assert!(
matches!(error, AdmissionRejection::PromptExceedsContext { .. }),
"got {error}"
);
assert!(error.to_string().contains("cannot help"));
}
}
#[test]
fn zero_frames_is_refused_rather_than_admitted_as_a_no_op() {
let error = admit(&request(512, 0, u64::MAX)).expect_err("must be rejected");
assert_eq!(error, AdmissionRejection::NoFramesRequested);
}
#[test]
fn arithmetic_overflow_is_refused_never_wrapped_into_a_plausible_total() {
assert!(matches!(
talker_kv_bytes(u64::MAX, u64::MAX, KvDtype::F32),
Err(AdmissionRejection::Overflow { .. })
));
let over = AdmissionRequest {
prompt_tokens: 512,
max_new_tokens: 2048,
heuristic_eos_backstop: false,
kv_dtype: KvDtype::Bf16,
ring_buffer_bytes: u64::MAX,
weights_resident_bytes: u64::MAX,
budget_bytes: u64::MAX,
};
let error = admit(&over).expect_err("overflow must not be admitted");
assert!(
matches!(error, AdmissionRejection::Overflow { .. }),
"a wrapped total is exactly the failure admission exists to prevent, got {error}"
);
}
#[test]
fn admission_is_exactly_at_the_boundary_not_off_by_one() {
let peak = admit(&request(512, 2048, u64::MAX))
.expect("admitted")
.predicted_peak_bytes;
assert!(admit(&request(512, 2048, peak)).is_ok());
assert!(admit(&request(512, 2048, peak - 1)).is_err());
}
#[test]
fn only_end_of_speech_counts_as_a_clean_completion() {
assert!(StopReason::EndOfSpeech.is_clean_completion());
assert!(!StopReason::EndOfSpeech.is_truncated());
for cut in [
StopReason::FrameCapReached,
StopReason::DurationLimitReached,
] {
assert!(cut.is_truncated(), "{cut} must be reported as truncated");
assert!(
!cut.is_clean_completion(),
"{cut} must never be reported as an unqualified success — an agent cannot hear \
that the audio stopped mid-word"
);
assert!(
cut.remedy().is_some(),
"{cut} must tell the caller what to do"
);
}
assert!(!StopReason::Cancelled.is_clean_completion());
assert!(!StopReason::Cancelled.is_truncated());
}
#[test]
fn stop_reason_wire_strings_are_distinct_and_stable() {
let all = [
StopReason::EndOfSpeech,
StopReason::FrameCapReached,
StopReason::DurationLimitReached,
StopReason::Cancelled,
];
let mut seen: Vec<&str> = all.iter().map(|reason| reason.as_str()).collect();
let count = seen.len();
seen.sort_unstable();
seen.dedup();
assert_eq!(seen.len(), count, "two stop reasons share a wire string");
assert_eq!(StopReason::FrameCapReached.as_str(), "frame_cap_reached");
}
}