pub mod http;
pub mod ironworks;
pub mod local;
pub mod remote;
use std::collections::{BTreeMap, VecDeque};
use std::fmt;
use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender};
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
use std::time::{Duration, Instant};
pub const HISTORY_CAPACITY: usize = 120;
pub const POLL_TIMEOUT: Duration = Duration::from_millis(1000);
pub const METRICS_NOT_ENABLED: &str =
"engine running, metrics not enabled — start it with --metrics";
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum ProviderKind {
IronWorks,
Ollama,
LlamaCpp,
Vllm,
LmStudio,
MlxLm,
OpenAi,
Anthropic,
}
impl ProviderKind {
pub const ALL: &'static [ProviderKind] = &[
ProviderKind::IronWorks,
ProviderKind::Ollama,
ProviderKind::LlamaCpp,
ProviderKind::Vllm,
ProviderKind::LmStudio,
ProviderKind::MlxLm,
ProviderKind::OpenAi,
ProviderKind::Anthropic,
];
pub fn as_str(self) -> &'static str {
match self {
Self::IronWorks => "ironworks",
Self::Ollama => "ollama",
Self::LlamaCpp => "llamacpp",
Self::Vllm => "vllm",
Self::LmStudio => "lmstudio",
Self::MlxLm => "mlx",
Self::OpenAi => "openai",
Self::Anthropic => "anthropic",
}
}
pub fn display_name(self) -> &'static str {
match self {
Self::IronWorks => "IronWorks",
Self::Ollama => "Ollama",
Self::LlamaCpp => "llama.cpp",
Self::Vllm => "vLLM",
Self::LmStudio => "LM Studio",
Self::MlxLm => "MLX-LM",
Self::OpenAi => "OpenAI",
Self::Anthropic => "Anthropic",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Locality {
Local,
Remote,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SourceError {
NotSupported(String),
Unreachable(String),
Status(u16),
MetricsNotEnabled,
NotJson(String),
Rejected(String),
NotConfigured(String),
}
impl SourceError {
pub fn engine_answered(&self) -> bool {
matches!(
self,
Self::Status(_) | Self::MetricsNotEnabled | Self::NotJson(_) | Self::Rejected(_)
)
}
pub fn summary(&self, endpoint: &str) -> String {
match self {
Self::MetricsNotEnabled => METRICS_NOT_ENABLED.to_string(),
Self::Unreachable(why) => format!("no engine answered at {endpoint}: {why}"),
Self::NotSupported(why) => {
format!("the metrics at {endpoint} cannot be read in this build: {why}")
}
Self::Status(code) => {
format!("{endpoint} answered HTTP {code}, not IronWorks metrics")
}
Self::NotJson(why) => format!("{endpoint} did not return JSON: {why}"),
Self::Rejected(why) => {
format!("{endpoint} did not return IronWorks metrics this build reads: {why}")
}
Self::NotConfigured(hint) => format!("not configured: {hint}"),
}
}
}
impl fmt::Display for SourceError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::MetricsNotEnabled => f.write_str(METRICS_NOT_ENABLED),
Self::NotSupported(why)
| Self::Unreachable(why)
| Self::Rejected(why)
| Self::NotConfigured(why) => f.write_str(why),
Self::Status(code) => write!(f, "HTTP {code}"),
Self::NotJson(why) => write!(f, "the body is not JSON: {why}"),
}
}
}
impl std::error::Error for SourceError {}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Identity {
pub model: Option<String>,
pub backend: Option<String>,
pub version: Option<String>,
pub status: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RatioDef {
pub key: &'static str,
pub numerator: &'static str,
pub denominator: &'static [&'static str],
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct ProviderSample {
pub sampled_at_ns: Option<u64>,
pub uptime_ns: Option<u64>,
pub identity: Identity,
pub counters: BTreeMap<String, u64>,
pub gauges: BTreeMap<String, f64>,
pub cost: Option<remote::CostReading>,
pub ratio_defs: &'static [RatioDef],
pub ironworks: Option<ironworks::Section>,
}
pub const BACKOFF_CAP_LOCAL: Duration = Duration::from_secs(10);
pub trait InferenceSource: Send + Sync {
fn kind(&self) -> ProviderKind;
fn endpoint(&self) -> String;
fn locality(&self) -> Locality;
fn poll(&self, timeout: Duration) -> Result<ProviderSample, SourceError>;
fn poll_timeout(&self) -> Duration {
POLL_TIMEOUT
}
fn min_interval(&self) -> Duration {
Duration::ZERO
}
fn backoff_cap(&self) -> Duration {
BACKOFF_CAP_LOCAL
}
}
pub fn next_wait(base: Duration, cap: Duration, failures: u32) -> Duration {
if failures <= 1 {
return base;
}
let shift = (failures - 1).min(16);
base.saturating_mul(1u32 << shift).min(cap.max(base))
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CompactRow {
pub status: &'static str,
pub detail: String,
}
pub fn compact_row(state: &SourceState) -> CompactRow {
if let (true, Some(sample)) = (state.reachable, state.latest.as_ref()) {
return CompactRow {
status: "running",
detail: sample_detail(sample),
};
}
match state.last_error.as_ref() {
None => CompactRow {
status: "waiting",
detail: "the first poll has not answered yet".into(),
},
Some(failure) => {
let status = match &failure.error {
SourceError::Unreachable(_) => "not running",
SourceError::NotConfigured(_) => "not configured",
_ => "not answering",
};
CompactRow {
status,
detail: local::reason(state.kind, &state.endpoint, &failure.error),
}
}
}
}
fn sample_detail(sample: &ProviderSample) -> String {
let mut parts: Vec<String> = Vec::new();
if let Some(model) = &sample.identity.model {
parts.push(format!("model {model}"));
}
if let Some(version) = &sample.identity.version {
parts.push(format!("version {version}"));
}
if let Some(status) = &sample.identity.status {
parts.push(format!("health {status}"));
}
if let Some(n) = sample.gauges.get("models_loaded") {
parts.push(format!("{} loaded", *n as u64));
}
if let Some(n) = sample.gauges.get("num_requests_running") {
parts.push(format!("{} running", *n as u64));
}
if let Some(n) = sample.gauges.get("num_requests_waiting") {
parts.push(format!("{} waiting", *n as u64));
}
if let Some(n) = sample.gauges.get("requests_processing") {
parts.push(format!("{} processing", *n as u64));
}
if let Some(p) = sample.gauges.get("kv_cache_usage_perc") {
parts.push(format!("KV cache {:.0}%", p * 100.0));
}
if let Some(n) = sample.gauges.get("models_listed") {
parts.push(format!("{} models listed", *n as u64));
}
if let Some(cost) = &sample.cost {
parts.push(match cost {
remote::CostReading::Known {
amount,
currency,
scale,
..
} => format!("month to date {amount} {currency} ({scale})"),
remote::CostReading::Unavailable { reason, .. } => {
format!("cost unavailable: {reason}")
}
});
}
if let Some(n) = sample.counters.get("prompt_tokens_total") {
parts.push(format!("{n} prompt tokens"));
}
if let Some(n) = sample.counters.get("generation_tokens_total") {
parts.push(format!("{n} generated"));
}
if let Some(n) = sample.counters.get("tokens_predicted_total") {
parts.push(format!("{n} generated"));
}
if parts.is_empty() {
"running; the server reported none of the values this tab reads".into()
} else {
parts.join(" · ")
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct Point {
pub at_ns: u64,
pub value: f64,
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Series {
points: VecDeque<Point>,
}
impl Series {
fn push(&mut self, point: Point) {
if self.points.len() == HISTORY_CAPACITY {
self.points.pop_front();
}
self.points.push_back(point);
}
pub fn iter(&self) -> impl Iterator<Item = &Point> {
self.points.iter()
}
pub fn len(&self) -> usize {
self.points.len()
}
pub fn is_empty(&self) -> bool {
self.points.is_empty()
}
pub fn latest(&self) -> Option<&Point> {
self.points.back()
}
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Derived {
pub window_ns: Option<u64>,
pub rates: BTreeMap<String, f64>,
pub ratios: BTreeMap<String, f64>,
}
impl Derived {
pub fn rate(&self, counter: &str) -> Option<f64> {
self.rates.get(counter).copied()
}
pub fn ratio(&self, key: &str) -> Option<f64> {
self.ratios.get(key).copied()
}
}
#[derive(Debug, Default)]
pub struct InferenceSampler {
previous: Option<ProviderSample>,
derived: Derived,
history: BTreeMap<String, Series>,
}
impl InferenceSampler {
pub fn update(&mut self, sample: ProviderSample) {
self.derived = match &self.previous {
Some(prev) if !regressed(prev, &sample) => derive(prev, &sample),
_ => Derived::default(),
};
if let Some(at_ns) = sample.uptime_ns {
let history = &mut self.history;
for (name, value) in &self.derived.rates {
push_point(history, format!("rate:{name}"), at_ns, *value);
}
for (key, value) in &self.derived.ratios {
push_point(history, format!("ratio:{key}"), at_ns, *value);
}
}
self.previous = Some(sample);
}
pub fn derived(&self) -> &Derived {
&self.derived
}
pub fn history(&self) -> &BTreeMap<String, Series> {
&self.history
}
}
fn push_point(history: &mut BTreeMap<String, Series>, key: String, at_ns: u64, value: f64) {
history.entry(key).or_default().push(Point { at_ns, value });
}
fn regressed(prev: &ProviderSample, cur: &ProviderSample) -> bool {
if let (Some(a), Some(b)) = (prev.uptime_ns, cur.uptime_ns) {
if b < a {
return true;
}
}
prev.counters
.iter()
.any(|(name, before)| cur.counters.get(name).is_some_and(|now| now < before))
}
fn derive(prev: &ProviderSample, cur: &ProviderSample) -> Derived {
let window_ns = match (prev.uptime_ns, cur.uptime_ns) {
(Some(a), Some(b)) if b > a => Some(b - a),
_ => None,
};
let mut out = Derived {
window_ns,
..Derived::default()
};
let Some(window) = window_ns else {
return out;
};
let seconds = window as f64 / 1e9;
let delta = |name: &str| -> Option<u64> {
let before = prev.counters.get(name)?;
let now = cur.counters.get(name)?;
Some(now - before)
};
for name in cur.counters.keys() {
if let Some(d) = delta(name) {
out.rates.insert(name.clone(), d as f64 / seconds);
}
}
for def in cur.ratio_defs {
let Some(numerator) = delta(def.numerator) else {
continue;
};
let Some(total) = def
.denominator
.iter()
.try_fold(0u64, |acc, name| delta(name).map(|d| acc + d))
else {
continue;
};
if total > 0 {
out.ratios
.insert(def.key.to_string(), numerator as f64 / total as f64);
}
}
out
}
#[derive(Debug, Clone)]
pub struct SourceState {
pub kind: ProviderKind,
pub locality: Locality,
pub endpoint: String,
pub latest: Option<ProviderSample>,
pub reachable: bool,
pub last_success: Option<Instant>,
pub last_error: Option<Failure>,
pub derived: Derived,
pub history: BTreeMap<String, Series>,
}
#[derive(Debug, Clone)]
pub struct Failure {
pub error: SourceError,
pub at: Instant,
}
impl SourceState {
fn new(source: &dyn InferenceSource) -> Self {
Self {
kind: source.kind(),
locality: source.locality(),
endpoint: source.endpoint(),
latest: None,
reachable: false,
last_success: None,
last_error: None,
derived: Derived::default(),
history: BTreeMap::new(),
}
}
}
pub struct InferencePoller {
slots: Vec<Slot>,
}
struct Slot {
shared: Arc<Mutex<SourceState>>,
_stop: Sender<()>,
}
impl InferencePoller {
pub fn spawn(sources: Vec<Box<dyn InferenceSource>>, interval: Duration) -> Self {
let slots = sources
.into_iter()
.map(|source| {
let shared = Arc::new(Mutex::new(SourceState::new(source.as_ref())));
let (stop_tx, stop_rx) = mpsc::channel();
let thread_shared = Arc::clone(&shared);
let name = format!("inference-{}", source.kind().as_str());
let spawned = std::thread::Builder::new()
.name(name)
.spawn(move || poll_loop(source, interval, thread_shared, stop_rx));
if let Err(e) = spawned {
let mut st = lock(&shared);
st.last_error = Some(Failure {
error: SourceError::Unreachable(format!(
"could not start a poller thread: {e}"
)),
at: Instant::now(),
});
}
Slot {
shared,
_stop: stop_tx,
}
})
.collect();
Self { slots }
}
pub fn len(&self) -> usize {
self.slots.len()
}
pub fn is_empty(&self) -> bool {
self.slots.is_empty()
}
pub fn state(&self, index: usize) -> Option<SourceState> {
self.slots.get(index).map(|s| lock(&s.shared).clone())
}
pub fn states(&self) -> Vec<SourceState> {
self.slots.iter().map(|s| lock(&s.shared).clone()).collect()
}
}
fn poll_loop(
source: Box<dyn InferenceSource>,
interval: Duration,
shared: Arc<Mutex<SourceState>>,
stop: Receiver<()>,
) {
let mut sampler = InferenceSampler::default();
let base = interval.max(source.min_interval());
let cap = source.backoff_cap();
let mut failures: u32 = 0;
loop {
let outcome = source.poll(source.poll_timeout());
let at = Instant::now();
{
let mut st = lock(&shared);
match outcome {
Ok(sample) => {
failures = 0;
sampler.update(sample.clone());
st.latest = Some(sample);
st.derived = sampler.derived().clone();
st.history = sampler.history().clone();
st.reachable = true;
st.last_success = Some(at);
}
Err(error) => {
failures = failures.saturating_add(1);
st.reachable = false;
st.last_error = Some(Failure { error, at });
}
}
}
let wait = next_wait(base, cap, failures.max(1));
match stop.recv_timeout(wait) {
Err(RecvTimeoutError::Timeout) => {}
_ => return,
}
}
}
fn lock<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
m.lock().unwrap_or_else(PoisonError::into_inner)
}
#[cfg(test)]
mod tests {
use super::*;
const TEST_RATIOS: &[RatioDef] = &[
RatioDef {
key: "tier.vram",
numerator: "tier.vram.hits_total",
denominator: &[
"tier.vram.hits_total",
"tier.vram.fills_total",
"tier.vram.bypass_total",
],
},
RatioDef {
key: "tier.ram",
numerator: "tier.ram.hits_total",
denominator: &["tier.ram.hits_total", "tier.ram.misses_total"],
},
];
fn sample(at: Option<u64>, counters: &[(&str, u64)]) -> ProviderSample {
ProviderSample {
uptime_ns: at,
counters: counters.iter().map(|(k, v)| (k.to_string(), *v)).collect(),
ratio_defs: TEST_RATIOS,
..ProviderSample::default()
}
}
const SEC: u64 = 1_000_000_000;
#[test]
fn first_sample_gives_no_rates() {
let mut s = InferenceSampler::default();
s.update(sample(Some(SEC), &[("requests.total", 5)]));
assert!(s.derived().rates.is_empty());
assert!(s.derived().ratios.is_empty());
assert_eq!(s.derived().rate("requests.total"), None);
}
#[test]
fn two_samples_give_rates_and_a_named_denominator_ratio() {
let mut s = InferenceSampler::default();
s.update(sample(
Some(SEC),
&[
("requests.total", 10),
("tier.vram.hits_total", 100),
("tier.vram.fills_total", 10),
("tier.vram.bypass_total", 10),
],
));
s.update(sample(
Some(3 * SEC),
&[
("requests.total", 14),
("tier.vram.hits_total", 130),
("tier.vram.fills_total", 15),
("tier.vram.bypass_total", 15),
],
));
let d = s.derived();
assert_eq!(d.window_ns, Some(2 * SEC));
assert_eq!(d.rate("requests.total"), Some(2.0));
assert_eq!(d.rate("tier.vram.hits_total"), Some(15.0));
assert_eq!(d.ratio("tier.vram"), Some(0.75));
}
#[test]
fn a_ratio_needs_every_denominator_counter() {
let mut s = InferenceSampler::default();
s.update(sample(
Some(SEC),
&[("tier.vram.hits_total", 1), ("tier.vram.fills_total", 1)],
));
s.update(sample(
Some(2 * SEC),
&[("tier.vram.hits_total", 9), ("tier.vram.fills_total", 2)],
));
assert_eq!(s.derived().ratio("tier.vram"), None);
assert_eq!(s.derived().rate("tier.vram.hits_total"), Some(8.0));
}
#[test]
fn a_ratio_is_none_when_its_denominator_gained_nothing() {
let mut s = InferenceSampler::default();
let pair = [("tier.ram.hits_total", 5), ("tier.ram.misses_total", 5)];
s.update(sample(Some(SEC), &pair));
s.update(sample(Some(2 * SEC), &pair));
assert_eq!(s.derived().ratio("tier.ram"), None);
assert_eq!(s.derived().rate("tier.ram.hits_total"), Some(0.0));
}
#[test]
fn counter_going_backwards_resets_the_sampler() {
let mut s = InferenceSampler::default();
s.update(sample(Some(SEC), &[("requests.total", 50)]));
s.update(sample(Some(2 * SEC), &[("requests.total", 60)]));
assert_eq!(s.derived().rate("requests.total"), Some(10.0));
s.update(sample(Some(3 * SEC), &[("requests.total", 2)]));
assert!(s.derived().rates.is_empty());
s.update(sample(Some(4 * SEC), &[("requests.total", 7)]));
assert_eq!(s.derived().rate("requests.total"), Some(5.0));
}
#[test]
fn uptime_going_backwards_resets_the_sampler() {
let mut s = InferenceSampler::default();
s.update(sample(Some(5 * SEC), &[("requests.total", 1)]));
s.update(sample(Some(2 * SEC), &[("requests.total", 9)]));
assert!(s.derived().rates.is_empty());
assert_eq!(s.derived().window_ns, None);
}
#[test]
fn a_wall_clock_jump_does_not_reset_or_skew_the_window() {
let mut first = sample(Some(SEC), &[("requests.total", 10)]);
first.sampled_at_ns = Some(10_000 * SEC);
let mut second = sample(Some(3 * SEC), &[("requests.total", 14)]);
second.sampled_at_ns = Some(6_400 * SEC);
let mut s = InferenceSampler::default();
s.update(first);
s.update(second);
assert_eq!(s.derived().window_ns, Some(2 * SEC));
assert_eq!(s.derived().rate("requests.total"), Some(2.0));
}
#[test]
fn missing_counter_gives_no_rate() {
let mut s = InferenceSampler::default();
s.update(sample(Some(SEC), &[("requests.total", 1)]));
s.update(sample(Some(2 * SEC), &[("tier.nvme.reads_total", 4)]));
assert_eq!(s.derived().rate("requests.total"), None);
assert_eq!(s.derived().rate("tier.nvme.reads_total"), None);
}
#[test]
fn non_advancing_uptime_gives_no_rate() {
let mut s = InferenceSampler::default();
s.update(sample(Some(SEC), &[("requests.total", 1)]));
s.update(sample(Some(SEC), &[("requests.total", 4)]));
assert!(s.derived().rates.is_empty());
assert_eq!(s.derived().window_ns, None);
}
#[test]
fn missing_uptime_gives_no_rate() {
let mut s = InferenceSampler::default();
s.update(sample(None, &[("requests.total", 1)]));
s.update(sample(Some(2 * SEC), &[("requests.total", 4)]));
assert!(s.derived().rates.is_empty());
}
#[test]
fn history_skips_underived_points() {
let mut s = InferenceSampler::default();
let hit = |h: u64, f: u64| {
[
("tier.vram.hits_total", h),
("tier.vram.fills_total", f),
("tier.vram.bypass_total", 0),
]
};
s.update(sample(Some(SEC), &hit(0, 0)));
assert!(s.history().is_empty());
s.update(sample(Some(2 * SEC), &hit(3, 1)));
assert_eq!(s.history()["ratio:tier.vram"].len(), 1);
s.update(sample(Some(2 * SEC), &hit(3, 1)));
assert_eq!(s.history()["ratio:tier.vram"].len(), 1);
s.update(sample(Some(3 * SEC), &hit(3, 1)));
assert_eq!(s.history()["ratio:tier.vram"].len(), 1);
s.update(sample(Some(4 * SEC), &hit(5, 1)));
let series = &s.history()["ratio:tier.vram"];
assert_eq!(series.len(), 2);
assert_eq!(series.latest().map(|p| p.value), Some(1.0));
}
#[test]
fn series_is_bounded() {
let mut s = InferenceSampler::default();
for i in 0..(HISTORY_CAPACITY as u64 + 10) {
s.update(sample(Some((i + 1) * SEC), &[("requests.total", i * 2)]));
}
let series = &s.history()["rate:requests.total"];
assert_eq!(series.len(), HISTORY_CAPACITY);
assert_eq!(series.iter().next().map(|p| p.at_ns), Some(11 * SEC));
}
#[test]
fn source_error_display_names_the_cause() {
assert_eq!(SourceError::Status(500).to_string(), "HTTP 500");
assert_eq!(
SourceError::Unreachable("connection refused".into()).to_string(),
"connection refused"
);
assert_eq!(
SourceError::MetricsNotEnabled.to_string(),
METRICS_NOT_ENABLED
);
}
#[test]
fn only_a_metrics_off_reply_or_a_body_counts_as_an_engine_answering() {
assert!(SourceError::MetricsNotEnabled.engine_answered());
assert!(SourceError::Status(500).engine_answered());
assert!(!SourceError::Unreachable("connection refused".into()).engine_answered());
}
#[test]
fn summaries_tell_the_metrics_off_state_from_no_engine() {
let off = SourceError::MetricsNotEnabled.summary("http://127.0.0.1:8080");
let none =
SourceError::Unreachable("connection refused".into()).summary("http://127.0.0.1:8080");
assert!(
off.starts_with("engine running, metrics not enabled"),
"{off}"
);
assert!(off.contains("--metrics"), "{off}");
assert!(none.starts_with("no engine answered at"), "{none}");
assert_ne!(off, none);
}
}
#[cfg(test)]
pub(crate) mod poller_tests {
use super::*;
use crate::inference::ironworks::IronWorksSource;
use std::io::{Read, Write};
use std::net::TcpListener;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
pub(crate) const A_BODY: &str =
include_str!("../../tests/fixtures/inference/a_nvme_configured.json");
pub(crate) const METRICS_OFF_BODY: &str =
include_str!("../../tests/fixtures/inference/metrics_not_enabled.json");
pub(crate) struct Stub {
port: u16,
reply: Arc<Mutex<(u16, String)>>,
stop: Arc<AtomicBool>,
thread: Option<std::thread::JoinHandle<()>>,
}
impl Stub {
pub(crate) fn start(status: u16, body: &str) -> Stub {
let listener = TcpListener::bind("127.0.0.1:0").expect("bind stub");
listener.set_nonblocking(true).expect("nonblocking");
let port = listener.local_addr().expect("addr").port();
let reply = Arc::new(Mutex::new((status, body.to_string())));
let stop = Arc::new(AtomicBool::new(false));
let (r, s) = (Arc::clone(&reply), Arc::clone(&stop));
let thread = std::thread::spawn(move || {
while !s.load(Ordering::SeqCst) {
match listener.accept() {
Ok((mut conn, _)) => {
let _ = conn.set_nonblocking(false);
let _ = conn.set_read_timeout(Some(Duration::from_secs(2)));
let mut buf = [0u8; 4096];
let _ = conn.read(&mut buf);
let (status, body) = r.lock().expect("reply lock").clone();
let reason = match status {
200 => "OK",
404 => "Not Found",
_ => "Error",
};
let response = format!(
"HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
let _ = conn.write_all(response.as_bytes());
}
Err(_) => std::thread::sleep(Duration::from_millis(5)),
}
}
});
Stub {
port,
reply,
stop,
thread: Some(thread),
}
}
pub(crate) fn url(&self) -> String {
format!("http://127.0.0.1:{}", self.port)
}
#[allow(dead_code)]
pub(crate) fn set_reply(&self, status: u16, body: &str) {
*self.reply.lock().expect("reply lock") = (status, body.to_string());
}
pub(crate) fn stop(mut self) {
self.stop.store(true, Ordering::SeqCst);
if let Some(t) = self.thread.take() {
let _ = t.join();
}
}
}
fn wait_until(limit: Duration, mut cond: impl FnMut() -> bool) -> bool {
let end = Instant::now() + limit;
while Instant::now() < end {
if cond() {
return true;
}
std::thread::sleep(Duration::from_millis(10));
}
cond()
}
fn poller_for(url: &str) -> InferencePoller {
let sources: Vec<Box<dyn InferenceSource>> = vec![Box::new(IronWorksSource::new(url))];
InferencePoller::spawn(sources, Duration::from_millis(50))
}
#[test]
fn poller_reports_a_value_from_the_stub() {
let stub = Stub::start(200, A_BODY);
let poller = poller_for(&stub.url());
let ok = wait_until(Duration::from_secs(10), || {
poller
.state(0)
.is_some_and(|s| s.reachable && s.latest.is_some())
});
assert!(ok, "poller never reported a value");
let st = poller.state(0).expect("state");
assert_eq!(st.kind, ProviderKind::IronWorks);
assert_eq!(st.locality, Locality::Local);
assert!(st.last_success.is_some());
let latest = st.latest.expect("latest sample");
assert_eq!(latest.identity.model.as_deref(), Some("glm-5.3-flash"));
drop(poller);
stub.stop();
}
#[test]
fn poller_reports_a_500_with_its_status() {
let stub = Stub::start(500, "internal failure");
let poller = poller_for(&stub.url());
let ok = wait_until(Duration::from_secs(10), || {
poller.state(0).is_some_and(|s| s.last_error.is_some())
});
assert!(ok, "poller never recorded the error");
let st = poller.state(0).expect("state");
assert!(!st.reachable);
assert!(st.latest.is_none());
let failure = st.last_error.expect("error");
assert_eq!(failure.error, SourceError::Status(500));
drop(poller);
stub.stop();
}
#[test]
fn poller_reports_metrics_not_enabled_on_a_404_with_the_engines_body() {
let stub = Stub::start(404, METRICS_OFF_BODY);
let poller = poller_for(&stub.url());
let ok = wait_until(Duration::from_secs(10), || {
poller.state(0).is_some_and(|s| s.last_error.is_some())
});
assert!(ok, "poller never recorded the error");
let st = poller.state(0).expect("state");
assert!(!st.reachable);
let failure = st.last_error.expect("error");
assert_eq!(failure.error, SourceError::MetricsNotEnabled);
assert!(failure.error.engine_answered());
drop(poller);
stub.stop();
}
#[test]
fn a_404_with_some_other_body_is_not_claimed_as_metrics_off() {
let stub = Stub::start(404, "<html>no such page</html>");
let poller = poller_for(&stub.url());
let ok = wait_until(Duration::from_secs(10), || {
poller.state(0).is_some_and(|s| s.last_error.is_some())
});
assert!(ok, "poller never recorded the error");
let failure = poller.state(0).expect("state").last_error.expect("error");
assert_eq!(failure.error, SourceError::Status(404));
drop(poller);
stub.stop();
}
#[test]
fn poller_reports_unreachable_after_the_stub_stops() {
let stub = Stub::start(200, A_BODY);
let poller = poller_for(&stub.url());
assert!(wait_until(Duration::from_secs(10), || {
poller.state(0).is_some_and(|s| s.reachable)
}));
stub.stop();
let gone = wait_until(Duration::from_secs(10), || {
poller.state(0).is_some_and(|s| {
!s.reachable
&& matches!(
s.last_error.as_ref().map(|f| &f.error),
Some(SourceError::Unreachable(_))
)
})
});
assert!(gone, "poller did not report the engine unreachable");
drop(poller);
}
#[test]
fn dropping_the_poller_ends_its_thread() {
let stub = Stub::start(200, A_BODY);
let poller = poller_for(&stub.url());
assert!(wait_until(Duration::from_secs(10), || {
poller.state(0).is_some_and(|s| s.reachable)
}));
let weak = Arc::downgrade(&poller.slots[0].shared);
drop(poller);
let exited = wait_until(Duration::from_secs(5), || weak.strong_count() == 0);
assert!(exited, "the poller thread kept its state alive after drop");
stub.stop();
}
}