use std::io::BufRead;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::time::Duration;
use std::time::Instant;
static VERBOSE: AtomicBool = AtomicBool::new(false);
pub fn set_verbose(on: bool) {
VERBOSE.store(on, Ordering::Relaxed);
}
#[must_use]
pub fn verbose() -> bool {
VERBOSE.load(Ordering::Relaxed)
}
static PREFILL_TOK_S: AtomicU64 = AtomicU64::new(900);
static DECODE_TOK_S: AtomicU64 = AtomicU64::new(50);
static EXPECT_GEN: AtomicU64 = AtomicU64::new(1200);
static LAST_CACHED: AtomicBool = AtomicBool::new(false);
static LAST_LOAD_MS: AtomicU64 = AtomicU64::new(0);
#[must_use]
pub fn last_cached() -> bool {
LAST_CACHED.load(Ordering::Relaxed)
}
pub fn load_pace() {
let st = crate::state::State::load();
for (key, cell) in [
("prefill", &PREFILL_TOK_S),
("decode", &DECODE_TOK_S),
("gen", &EXPECT_GEN),
] {
if let Some(v) = st.get_u64("pace", key) {
cell.store(v.max(1), Ordering::Relaxed);
}
}
}
pub fn save_pace() {
let mut st = crate::state::State::load();
save_pace_in(&mut st);
st.save();
}
pub fn save_pace_in(st: &mut crate::state::State) {
st.set(
"pace",
"prefill",
PREFILL_TOK_S.load(Ordering::Relaxed).to_string(),
);
st.set(
"pace",
"decode",
DECODE_TOK_S.load(Ordering::Relaxed).to_string(),
);
st.set(
"pace",
"gen",
EXPECT_GEN.load(Ordering::Relaxed).to_string(),
);
}
#[must_use]
pub fn was_cached(prompt_tokens: u64, prefill_ms: u128) -> bool {
if prompt_tokens == 0 {
return false;
}
if prefill_ms == 0 {
return true;
}
let observed = prompt_tokens as f64 / (prefill_ms as f64 / 1000.0);
observed > 3_000.0
}
#[derive(Debug, Clone, Copy)]
pub struct Eta {
pub prefill_s: f64,
pub decode_s: f64,
pub gen_est: u64,
}
impl Eta {
#[must_use]
pub fn total_s(&self) -> f64 {
self.prefill_s + self.decode_s
}
#[must_use]
pub fn cached_s(&self) -> f64 {
self.decode_s
}
#[must_use]
pub fn total(&self) -> Duration {
Duration::from_secs_f64(self.total_s().max(1.0))
}
}
#[must_use]
pub fn bucket(prompt_tokens: u64) -> &'static str {
match prompt_tokens {
0..=1_999 => "b0",
2_000..=7_999 => "b2",
8_000..=31_999 => "b8",
_ => "b32",
}
}
pub fn record_obs_in(
st: &mut crate::state::State,
label: &str,
row: &Telemetry,
) {
const KEEP: usize = 200;
let line = row.line(label);
let key = crate::state::content_hash(line.as_bytes());
st.push(
crate::state::Log {
kind: "obs",
cap: KEEP,
},
&key,
line,
);
}
#[derive(Debug, Clone, Copy)]
pub struct Telemetry {
pub prompt_tok: u64,
pub prefill_ms: u128,
pub eval_tok: u64,
pub decode_ms: u128,
pub cold: bool,
}
impl Telemetry {
#[must_use]
pub fn line(&self, label: &str) -> String {
format!(
"{label} {} {} {} {} {}",
self.prompt_tok,
self.prefill_ms,
self.eval_tok,
self.decode_ms,
u8::from(self.cold)
)
}
}
pub fn record_obs(label: &str, row: &Telemetry) {
let mut st = crate::state::State::load();
record_obs_in(&mut st, label, row);
st.save();
}
#[must_use]
pub fn derived_prefill(prompt_tokens: u64) -> Option<f64> {
let want = bucket(prompt_tokens);
let st = crate::state::State::load();
let mut rates: Vec<f64> = st
.all("obs")
.iter()
.filter_map(|row| {
let f: Vec<&str> = row.split(' ').collect();
let [_, tok, ms, _, _, cached, ..] = f.as_slice() else {
return None;
};
if *cached == "1" {
return None;
}
let (tok, ms) = (tok.parse::<u64>().ok()?, ms.parse::<f64>().ok()?);
if bucket(tok) != want || ms <= 0.0 {
return None;
}
let r = tok as f64 / (ms / 1000.0);
if r > 3_000.0 { None } else { Some(r) }
})
.collect();
if rates.len() < 2 {
return None;
}
rates.sort_by(f64::total_cmp);
rates.get(rates.len() / 2).copied()
}
#[must_use]
pub fn predict(prompt_tokens: u64) -> Eta {
predict_for("", prompt_tokens)
}
#[must_use]
pub fn predict_for(label: &str, prompt_tokens: u64) -> Eta {
let pre = derived_prefill(prompt_tokens)
.unwrap_or_else(|| PREFILL_TOK_S.load(Ordering::Relaxed).max(1) as f64);
let dec = DECODE_TOK_S.load(Ordering::Relaxed).max(1) as f64;
let gen_tokens = if label.is_empty() {
EXPECT_GEN.load(Ordering::Relaxed)
} else {
crate::state::State::load()
.get_u64("gen", label)
.unwrap_or_else(|| EXPECT_GEN.load(Ordering::Relaxed))
};
Eta {
prefill_s: prompt_tokens as f64 / pre,
decode_s: gen_tokens as f64 / dec,
gen_est: gen_tokens,
}
}
pub fn raise_gen_floor_in(
st: &mut crate::state::State,
label: &str,
seen: u64,
) {
if label.is_empty() || seen == 0 {
return;
}
if st.get_u64("gen", label).unwrap_or(0) < seen {
st.set("gen", label, seen.to_string());
}
}
pub fn raise_gen_floor(label: &str, seen: u64) {
let mut st = crate::state::State::load();
raise_gen_floor_in(&mut st, label, seen);
st.save();
}
#[must_use]
pub fn last_load_ms() -> u64 {
LAST_LOAD_MS.load(Ordering::Relaxed)
}
pub fn observe_gen_in(
st: &mut crate::state::State,
label: &str,
eval_tokens: u64,
) {
if label.is_empty() || eval_tokens == 0 {
return;
}
let prev = st.get_u64("gen", label).unwrap_or(eval_tokens);
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
let next = (prev as f64 * 0.75 + eval_tokens as f64 * 0.25) as u64;
st.set("gen", label, next.to_string());
}
pub fn observe_gen(label: &str, eval_tokens: u64) {
let mut st = crate::state::State::load();
observe_gen_in(&mut st, label, eval_tokens);
st.save();
}
pub fn observe(r: &Reply, prefill_ms: u128, decode_ms: u128) {
let ewma = |cur: &AtomicU64, sample: f64| {
if sample.is_finite() && sample > 0.0 {
let old = cur.load(Ordering::Relaxed) as f64;
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
cur.store((old * 0.75 + sample * 0.25) as u64, Ordering::Relaxed);
}
};
if prefill_ms > 0 && !was_cached(r.prompt_tokens, prefill_ms) {
ewma(
&PREFILL_TOK_S,
r.prompt_tokens as f64 / (prefill_ms as f64 / 1000.0),
);
}
if decode_ms > 0 {
ewma(
&DECODE_TOK_S,
r.eval_tokens as f64 / (decode_ms as f64 / 1000.0),
);
}
ewma(&EXPECT_GEN, r.eval_tokens as f64);
}
pub trait Transport {
fn post(
&self,
url: &str,
body: &str,
timeout: Duration,
) -> Result<Box<dyn BufRead + Send>, String>;
fn timings_are_real(&self) -> bool {
true
}
}
#[derive(Debug, Default, Clone, Copy)]
pub struct Http;
impl Transport for Http {
fn post(
&self,
url: &str,
body: &str,
timeout: Duration,
) -> Result<Box<dyn BufRead + Send>, String> {
let agent: ureq::Agent = ureq::Agent::config_builder()
.timeout_recv_response(Some(timeout))
.build()
.into();
let resp = agent
.post(url)
.content_type("application/json")
.send(body)
.map_err(|e| format!("{url}: {e}"))?;
Ok(Box::new(std::io::BufReader::new(
resp.into_body().into_reader(),
)))
}
}
#[derive(Debug, Clone)]
pub struct Reply {
pub text: String,
pub thinking: String,
pub prompt_tokens: u64,
pub eval_tokens: u64,
pub ms: u128,
}
fn endpoint() -> String {
std::env::var("SHERD_ENDPOINT")
.unwrap_or_else(|_| "http://localhost:11434".into())
}
fn model() -> String {
std::env::var("SHERD_MODEL").unwrap_or_else(|_| "gpt-oss:20b".into())
}
fn num_ctx() -> u64 {
std::env::var("SHERD_NUM_CTX")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(131_072)
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Sampling {
pub temperature: f64,
pub seed: u64,
}
impl Sampling {
pub const DETERMINISTIC: Self = Self {
temperature: 0.0,
seed: 0,
};
#[must_use]
pub fn candidate(k: usize) -> Self {
if k == 0 {
Self::DETERMINISTIC
} else {
Self {
temperature: 0.6,
seed: k as u64,
}
}
}
}
pub fn generate_with(
prompt: &str,
eta: Eta,
on_chunk: &mut dyn FnMut(&str),
) -> Result<Reply, String> {
generate_labelled(prompt, "", eta, on_chunk)
}
const NOTICE_1: f64 = 1.0;
const NOTICE_2: f64 = 2.0;
const WARN: f64 = 5.0;
const ABORT: f64 = 10.0;
pub fn generate_labelled(
prompt: &str,
label: &str,
eta: Eta,
on_chunk: &mut dyn FnMut(&str),
) -> Result<Reply, String> {
generate_via(&Http, prompt, label, Sampling::DETERMINISTIC, eta, on_chunk)
}
pub fn generate_sampled(
prompt: &str,
label: &str,
sampling: Sampling,
eta: Eta,
on_chunk: &mut dyn FnMut(&str),
) -> Result<Reply, String> {
generate_via(&Http, prompt, label, sampling, eta, on_chunk)
}
pub fn stream(
transport: &dyn Transport,
prompt: &str,
label: &str,
sampling: Sampling,
eta: Eta,
on_chunk: &mut dyn FnMut(&str),
) -> Result<Streamed, String> {
let budget = eta.total();
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
let hard = budget.mul_f64(ABORT);
let body = serde_json::json!({
"model": model(),
"prompt": prompt,
"stream": true,
"options": { "num_ctx": num_ctx(), "temperature": sampling.temperature,
"seed": sampling.seed },
});
let started = Instant::now();
let stream = transport.post(
&format!("{}/api/generate", endpoint()),
&body.to_string(),
hard,
)?;
let mut text = String::new();
let mut thinking = String::new();
let mut warned = 0u8;
let mut first_chunk: Option<Instant> = None;
let (mut prompt_tokens, mut eval_tokens) = (0, 0);
for line in stream.lines() {
let line = line.map_err(|e| e.to_string())?;
if line.trim().is_empty() {
continue;
}
let v: serde_json::Value =
serde_json::from_str(&line).map_err(|e| e.to_string())?;
if let Some(th) = v["thinking"].as_str()
&& !th.is_empty()
{
thinking.push_str(th);
if first_chunk.is_none() {
first_chunk = Some(Instant::now())
}
on_chunk("");
}
if let Some(chunk) = v["response"].as_str() {
if first_chunk.is_none() && !chunk.is_empty() {
first_chunk = Some(Instant::now());
}
text.push_str(chunk);
on_chunk(chunk);
}
let over = started.elapsed().as_secs_f64() / budget.as_secs_f64();
let level = if over >= ABORT {
4
} else if over >= WARN {
3
} else if over >= NOTICE_2 {
2
} else if over >= NOTICE_1 {
1
} else {
0
};
if level > warned {
warned = level;
let el = started.elapsed().as_secs_f64();
match level {
1 => eprintln!(
"\n [pace] {el:.0}s: past the {:.0}s estimate, still streaming",
budget.as_secs_f64()
),
2 => eprintln!(
"\n [pace] {el:.0}s: 2x the estimate -- the endpoint is slower than this run assumed"
),
3 => eprintln!(
"\n [pace] WARNING {el:.0}s: {WARN:.0}x the estimate. Hard stop at {:.0}s.",
hard.as_secs_f64()
),
_ => {
let seen = (text.len() + thinking.len()) as u64 / 4;
raise_gen_floor(label, seen);
return Err(format!(
"aborted after {el:.0}s -- 4x the {:.0}s estimate. \
{} output + {} reasoning tokens so far (recorded as a floor \
for `{label}`). Endpoint {} may be overloaded, or \
SHERD_NUM_CTX too large for its memory.",
budget.as_secs_f64(),
text.len() / 4,
thinking.len() / 4,
endpoint()
));
}
}
}
if v["done"].as_bool().unwrap_or(false) {
prompt_tokens = v["prompt_eval_count"].as_u64().unwrap_or(0);
eval_tokens = v["eval_count"].as_u64().unwrap_or(0);
LAST_LOAD_MS.store(
v["load_duration"].as_u64().unwrap_or(0) / 1_000_000,
Ordering::Relaxed,
);
}
}
let reply = Reply {
text,
thinking,
prompt_tokens,
eval_tokens,
ms: started.elapsed().as_millis(),
};
Ok(Streamed {
reply,
started,
first_chunk,
})
}
pub struct Streamed {
pub reply: Reply,
pub started: Instant,
pub first_chunk: Option<Instant>,
}
pub fn prewarm(t: &dyn Transport) {
if !t.timings_are_real() {
return;
}
let eta = Eta {
prefill_s: 60.0,
decode_s: 60.0,
gen_est: 1,
};
let _ = stream(t, "ok", "", Sampling::DETERMINISTIC, eta, &mut |_| {});
let ms = last_load_ms();
if ms > 500 {
eprintln!(
" warm: model loaded from disk in {ms}ms -- not charged to the run"
);
}
}
pub fn generate_via(
transport: &dyn Transport,
prompt: &str,
label: &str,
sampling: Sampling,
eta: Eta,
on_chunk: &mut dyn FnMut(&str),
) -> Result<Reply, String> {
let s = stream(transport, prompt, label, sampling, eta, on_chunk)?;
if transport.timings_are_real() {
learn_from(&s.reply, label, s.started, s.first_chunk);
}
Ok(s.reply)
}
pub fn learn_from(
reply: &Reply,
label: &str,
started: Instant,
first_chunk: Option<Instant>,
) {
let mut st = crate::state::State::load();
learn_from_in(&mut st, reply, label, timing(reply, started, first_chunk));
st.save();
}
#[must_use]
pub fn timing(
reply: &Reply,
started: Instant,
first_chunk: Option<Instant>,
) -> Telemetry {
let load_ms = u128::from(LAST_LOAD_MS.load(Ordering::Relaxed));
let pre_ms = first_chunk
.map_or(reply.ms, |t| (t - started).as_millis())
.saturating_sub(load_ms);
Telemetry {
prompt_tok: reply.prompt_tokens,
prefill_ms: pre_ms,
eval_tok: reply.eval_tokens,
decode_ms: reply.ms.saturating_sub(pre_ms),
cold: load_ms > 500,
}
}
pub fn learn_from_in(
st: &mut crate::state::State,
reply: &Reply,
label: &str,
t: Telemetry,
) {
observe(reply, t.prefill_ms, t.decode_ms);
LAST_CACHED.store(
was_cached(reply.prompt_tokens, t.prefill_ms),
Ordering::Relaxed,
);
record_obs_in(st, label, &t);
save_pace_in(st);
}
pub fn generate(prompt: &str) -> Result<Reply, String> {
generate_with(prompt, predict(0), &mut |_| {})
}
#[must_use]
pub fn rust_block(text: &str) -> String {
let mut best = String::new();
let mut rest = text;
while let Some(open) = rest.find("```") {
let after = &rest[open + 3..];
let start = after.find('\n').map_or(0, |i| i + 1);
let Some(close) = after[start..].find("```") else {
break;
};
let block = &after[start..start + close];
if block.len() > best.len() {
best = block.to_string();
}
rest = &after[start + close..];
}
if best.is_empty() {
text.trim().to_string()
} else {
best.trim().to_string()
}
}
#[cfg(test)]
mod tests {
use super::*;
fn store(tag: &str) -> crate::state::State {
use std::sync::atomic::{AtomicUsize, Ordering};
static N: AtomicUsize = AtomicUsize::new(0);
let n = N.fetch_add(1, Ordering::Relaxed);
crate::state::State::at(
std::env::temp_dir()
.join(format!("sherd-pace-{tag}-{}-{n}", std::process::id())),
)
}
#[test]
fn timing_splits_prefill_from_decode_at_the_first_token() {
let r = Reply {
text: String::new(),
thinking: String::new(),
prompt_tokens: 900,
eval_tokens: 100,
ms: 3_000,
};
let started = Instant::now();
let first = started.checked_add(Duration::from_millis(1_000));
let t = timing(&r, started, first);
assert_eq!(t.prefill_ms, 1_000, "everything before the first token");
assert_eq!(t.decode_ms, 2_000, "and everything after it");
assert_eq!(t.prompt_tok, 900);
assert_eq!(t.eval_tok, 100);
}
#[test]
fn a_reply_with_no_token_at_all_is_all_prefill() {
let r = Reply {
text: String::new(),
thinking: String::new(),
prompt_tokens: 10,
eval_tokens: 0,
ms: 500,
};
let t = timing(&r, Instant::now(), None);
assert_eq!(t.prefill_ms, 500);
assert_eq!(t.decode_ms, 0);
}
#[test]
fn a_telemetry_row_round_trips_its_fields_in_order() {
let t = Telemetry {
prompt_tok: 1,
prefill_ms: 2,
eval_tok: 3,
decode_ms: 4,
cold: true,
};
assert_eq!(t.line("lbl"), "lbl 1 2 3 4 1");
}
#[test]
fn learning_from_a_reply_retains_it_and_persists_the_rates() {
let mut st = store("learn");
let r = Reply {
text: "x".into(),
thinking: String::new(),
prompt_tokens: 900,
eval_tokens: 100,
ms: 2_000,
};
let t = Telemetry {
prompt_tok: 900,
prefill_ms: 1_000,
eval_tok: 100,
decode_ms: 1_000,
cold: false,
};
learn_from_in(&mut st, &r, "1 test", t);
assert!(!st.all("obs").is_empty(), "the observation is retained");
assert!(st.get("pace", "prefill").is_some(), "rates persisted");
}
#[test]
fn the_same_observation_twice_is_retained_once() {
let mut st = store("idem");
let t = Telemetry {
prompt_tok: 7,
prefill_ms: 3,
eval_tok: 2,
decode_ms: 4,
cold: false,
};
record_obs_in(&mut st, "1 test", &t);
let after_one = st.all("obs").len();
record_obs_in(&mut st, "1 test", &t);
assert_eq!(st.all("obs").len(), after_one, "recorded once, not twice");
}
#[test]
fn a_scripted_transport_declares_its_timings_fake() {
assert!(Http.timings_are_real(), "a real endpoint is real");
assert!(
!Canned(String::new()).timings_are_real(),
"a double must declare itself -- B9 is the absence of this"
);
}
struct WarmSpy {
calls: std::cell::Cell<usize>,
}
impl Transport for WarmSpy {
fn post(
&self,
_u: &str,
_b: &str,
_t: Duration,
) -> Result<Box<dyn BufRead + Send>, String> {
self.calls.set(self.calls.get().saturating_add(1));
Ok(Box::new(std::io::Cursor::new(
"{\"response\":\"ok\",\"done\":true}\n".as_bytes().to_vec(),
)))
}
}
#[test]
fn prewarming_makes_exactly_one_discarded_call() {
let t = WarmSpy {
calls: std::cell::Cell::new(0),
};
prewarm(&t);
assert_eq!(t.calls.get(), 1, "one call, and only one");
}
#[test]
fn prewarming_learns_nothing_from_the_call_it_makes() {
let (_f, p) = scratch_state("warm");
let before = crate::state::State::at(&p).all("gen").len();
prewarm(&WarmSpy {
calls: std::cell::Cell::new(0),
});
assert_eq!(
crate::state::State::at(&p).all("gen").len(),
before,
"a warm-up records nothing"
);
}
fn scratch_state(tag: &str) -> ((), std::path::PathBuf) {
use std::sync::atomic::{AtomicUsize, Ordering};
static N: AtomicUsize = AtomicUsize::new(0);
let n = N.fetch_add(1, Ordering::Relaxed);
(
(),
std::env::temp_dir()
.join(format!("sherd-warm-{tag}-{}-{n}", std::process::id())),
)
}
#[test]
fn prewarming_a_double_costs_nothing_and_asks_it_nothing() {
let t = Canned("{\"response\":\"x\",\"done\":true}\n".into());
prewarm(&t);
assert!(drain(&t).is_ok(), "the double's first reply is still there");
}
#[test]
fn a_floor_only_ever_rises() {
let mut st = store("floor");
raise_gen_floor_in(&mut st, "4 repair-1", 500);
assert_eq!(st.get_u64("gen", "4 repair-1"), Some(500));
raise_gen_floor_in(&mut st, "4 repair-1", 200);
assert_eq!(
st.get_u64("gen", "4 repair-1"),
Some(500),
"a LOWER observation must not lower the floor"
);
raise_gen_floor_in(&mut st, "4 repair-1", 900);
assert_eq!(st.get_u64("gen", "4 repair-1"), Some(900), "higher wins");
}
#[test]
fn an_empty_label_or_a_zero_count_records_nothing() {
let mut st = store("guard");
raise_gen_floor_in(&mut st, "", 500);
raise_gen_floor_in(&mut st, "label", 0);
observe_gen_in(&mut st, "", 500);
observe_gen_in(&mut st, "label", 0);
assert_eq!(st.get_u64("gen", ""), None);
assert_eq!(st.get_u64("gen", "label"), None);
}
#[test]
fn the_first_observation_seeds_from_itself_and_later_ones_average() {
let mut st = store("ewma");
observe_gen_in(&mut st, "1 test", 400);
assert_eq!(
st.get_u64("gen", "1 test"),
Some(400),
"the first observation is the estimate"
);
observe_gen_in(&mut st, "1 test", 800);
assert_eq!(st.get_u64("gen", "1 test"), Some(500));
assert!(
st.get_u64("gen", "1 test").is_some_and(|v| v < 800),
"weight 1/4: one call does not swing the next prediction"
);
}
struct Flaky {
fail_times: std::cell::Cell<u32>,
body: String,
}
impl Transport for Flaky {
fn timings_are_real(&self) -> bool {
false
}
fn post(
&self,
url: &str,
_b: &str,
_t: Duration,
) -> Result<Box<dyn BufRead + Send>, String> {
let left = self.fail_times.get();
if left > 0 {
self.fail_times.set(left - 1);
return Err(format!("{url}: simulated transport failure"));
}
Ok(Box::new(std::io::Cursor::new(
self.body.clone().into_bytes(),
)))
}
}
struct Canned(String);
impl Transport for Canned {
fn timings_are_real(&self) -> bool {
false
}
fn post(
&self,
_u: &str,
_b: &str,
_t: Duration,
) -> Result<Box<dyn BufRead + Send>, String> {
Ok(Box::new(std::io::Cursor::new(self.0.clone().into_bytes())))
}
}
fn slow_eta() -> Eta {
Eta {
prefill_s: 600.0,
decode_s: 600.0,
gen_est: 100,
}
}
fn drain(t: &dyn Transport) -> Result<Reply, String> {
stream(
t,
"p",
"test",
Sampling::candidate(0),
slow_eta(),
&mut |_| {},
)
.map(|s| s.reply)
}
#[test]
fn a_streamed_reply_is_assembled_from_its_chunks() {
let body = concat!(
"{\"response\":\"pub fn \",\"done\":false}\n",
"\n",
"{\"response\":\"x() {}\",\"done\":false}\n",
"{\"response\":\"\",\"done\":true,\"prompt_eval_count\":1234,\
\"eval_count\":56,\"load_duration\":0}\n"
);
let r = drain(&Canned(body.into()));
let Ok(r) = r else {
unreachable!("canned stream must parse")
};
assert_eq!(r.text, "pub fn x() {}", "chunks concatenate in order");
assert_eq!(r.prompt_tokens, 1234, "counts come off the final frame");
assert_eq!(r.eval_tokens, 56);
}
#[test]
fn reasoning_is_kept_apart_from_the_answer() {
let body = concat!(
"{\"thinking\":\"let me think\",\"done\":false}\n",
"{\"response\":\"fn a(){}\",\"done\":false}\n",
"{\"response\":\"\",\"done\":true,\"eval_count\":2}\n"
);
let Ok(r) = drain(&Canned(body.into())) else {
unreachable!("canned stream must parse")
};
assert_eq!(r.text, "fn a(){}");
assert_eq!(r.thinking, "let me think");
}
#[test]
fn a_transport_failure_is_an_error_and_never_an_empty_reply() {
let t = Flaky {
fail_times: std::cell::Cell::new(1),
body: String::new(),
};
assert!(drain(&t).is_err(), "a dead transport is an ERROR");
}
#[test]
fn a_malformed_frame_is_an_error_and_not_a_silent_skip() {
let body = "{\"response\":\"a\",\"done\":false}\nnot json at all\n";
assert!(drain(&Canned(body.into())).is_err());
}
#[test]
fn the_longest_fenced_block_wins_and_a_bare_reply_survives() {
assert_eq!(rust_block("no fence here"), "no fence here");
assert_eq!(rust_block("pre\n```rust\nfn a(){}\n```\npost"), "fn a(){}");
let two =
"```\nfn a(){}\n```\ntext\n```rust\nfn long(){ let x = 1; }\n```";
assert_eq!(rust_block(two), "fn long(){ let x = 1; }");
assert_eq!(rust_block("```rust\nfn a(){}"), "```rust\nfn a(){}");
}
#[test]
fn a_transport_can_be_substituted_and_made_to_fail() {
let t = Flaky {
fail_times: std::cell::Cell::new(1),
body: String::new(),
};
assert!(
t.post("u", "b", Duration::from_secs(1)).is_err(),
"first call fails"
);
assert!(
t.post("u", "b", Duration::from_secs(1)).is_ok(),
"then succeeds"
);
}
#[test]
fn generate_via_reads_a_substituted_stream() {
let body = "{\"response\":\"hi\",\"done\":false}\n\
{\"response\":\"\",\"done\":true,\"prompt_eval_count\":7,\"eval_count\":2}\n";
let t = Flaky {
fail_times: std::cell::Cell::new(0),
body: body.into(),
};
let r = generate_via(
&t,
"p",
"test",
Sampling::DETERMINISTIC,
predict(10),
&mut |_| {},
)
.unwrap();
assert_eq!(r.text, "hi");
assert_eq!((r.prompt_tokens, r.eval_tokens), (7, 2));
}
struct Spy(std::cell::RefCell<String>);
impl Transport for Spy {
fn timings_are_real(&self) -> bool {
false
}
fn post(
&self,
_u: &str,
b: &str,
_t: Duration,
) -> Result<Box<dyn BufRead + Send>, String> {
self.0.replace(b.to_string());
Ok(Box::new(std::io::Cursor::new(
"{\"response\":\"x\",\"done\":true}\n".as_bytes().to_vec(),
)))
}
}
fn posted(s: Sampling) -> serde_json::Value {
let spy = Spy(std::cell::RefCell::new(String::new()));
generate_via(&spy, "p", "test", s, predict(10), &mut |_| {}).unwrap();
let body = spy.0.borrow().clone();
serde_json::from_str(&body).unwrap()
}
#[test]
fn sampling_reaches_the_request_and_candidates_differ() {
let zero = posted(Sampling::candidate(0));
assert_eq!(
zero["options"]["temperature"], 0.0,
"candidate 0 must be today's deterministic call, unchanged"
);
let two = posted(Sampling::candidate(2));
assert!(
two["options"]["temperature"].as_f64().unwrap() > 0.0,
"a competing candidate at temperature 0 is the same call twice (V17)"
);
assert_eq!(
two["options"]["seed"], 2,
"diverse but REPRODUCIBLE -- an unseeded failure cannot be re-examined"
);
assert_ne!(zero["options"]["seed"], two["options"]["seed"]);
}
#[test]
fn takes_the_longest_fenced_block() {
let t = "blah\n```rust\nfn a() {}\n```\nmore\n```rust\nfn bb() { todo!() }\n```\n";
assert_eq!(rust_block(t), "fn bb() { todo!() }");
}
#[test]
fn unfenced_text_survives_whole() {
assert_eq!(rust_block(" fn a() {} "), "fn a() {}");
}
#[test]
fn unterminated_fence_does_not_hang_or_panic() {
assert_eq!(rust_block("```rust\nfn a() {}"), "```rust\nfn a() {}");
}
}
#[cfg(test)]
mod pace_tests {
use super::*;
#[test]
fn a_bucket_is_a_size_band_not_a_scalar() {
const BANDS: [(u64, &str); 7] = [
(0, "b0"),
(1_999, "b0"),
(2_000, "b2"),
(7_999, "b2"),
(8_000, "b8"),
(31_999, "b8"),
(32_000, "b32"),
];
for (n, want) in BANDS {
assert_eq!(bucket(n), want, "{n} belongs in {want}");
}
}
#[test]
fn a_cache_hit_is_an_absolute_rate_not_a_multiple_of_the_learned_one() {
assert!(was_cached(10_000, 1_000), "10k tok/s is a hit");
assert!(!was_cached(1_000, 1_000), "1k tok/s is a cold run");
assert!(
!was_cached(3_000, 1_000),
"exactly 3k is NOT over the bound"
);
assert!(!was_cached(0, 500), "no prompt is not a cache hit");
assert!(was_cached(500, 0), "no measurable prefill is a hit");
}
#[test]
fn verbose_is_a_mode_that_can_be_turned_off_again() {
let before = verbose();
set_verbose(true);
assert!(verbose());
set_verbose(false);
assert!(!verbose());
set_verbose(before);
}
#[test]
fn an_eta_is_prefill_plus_decode_and_says_what_it_expects() {
let e = predict(10_000);
assert!(e.prefill_s > 0.0, "prefill must be predicted: {e:?}");
assert!(e.decode_s > 0.0, "decode must be predicted: {e:?}");
assert!(e.total() > std::time::Duration::ZERO);
assert!(e.gen_est > 0, "an eta of zero tokens bounds nothing");
}
#[test]
fn a_bigger_prompt_never_predicts_a_shorter_prefill() {
let small = predict(1_000);
let big = predict(30_000);
assert!(
big.prefill_s >= small.prefill_s,
"{big:?} must not be quicker than {small:?}"
);
}
}