loadmaster_probe/
embed.rs

1//! The embedding check (`probe --embeddings`, and `up --verify` for embedding models):
2//! `/v1/embeddings` returns finite vectors of one length, the same text gives the same
3//! vector every time, and a batch of two texts gives two different vectors.
4
5use serde_json::{Value, json};
6
7use crate::client::Client;
8use crate::run::{FixtureResult, Outcome, PROBE_VERSION, Report, verdict};
9
10const TEXT: &str = "loadmaster checks that this sentence embeds the same way every time.";
11
12/// The vectors in an `/v1/embeddings` response, or the failure class.
13fn vectors(status: u16, body: &Value) -> Result<Vec<Vec<f64>>, String> {
14    if !(200..300).contains(&status) {
15        return Err(format!("embed_http_error {status}"));
16    }
17    let data = body["data"]
18        .as_array()
19        .filter(|d| !d.is_empty())
20        .ok_or("embed_no_vector")?;
21    data.iter()
22        .map(|d| -> Result<Vec<f64>, String> {
23            let v: Vec<f64> = d["embedding"]
24                .as_array()
25                .ok_or("embed_no_vector")?
26                .iter()
27                .map(|x| x.as_f64().ok_or("embed_not_finite"))
28                .collect::<Result<_, _>>()?;
29            if v.is_empty() {
30                Err("embed_no_vector".into())
31            } else if v.iter().any(|x| !x.is_finite()) {
32                Err("embed_not_finite".into())
33            } else {
34                Ok(v)
35            }
36        })
37        .collect()
38}
39
40fn result(id: &str, description: &str, core: bool, runs: Vec<Outcome>) -> FixtureResult {
41    let outcome = runs
42        .iter()
43        .find(|o| !matches!(o, Outcome::Pass))
44        .cloned()
45        .unwrap_or(Outcome::Pass);
46    FixtureResult {
47        id: id.into(),
48        description: description.into(),
49        core,
50        outcome,
51        runs,
52        latency_ms: 0,
53    }
54}
55
56pub async fn probe(c: &Client, model: &str, repeat: u32) -> Report {
57    let detected = crate::detect::detect(c).await;
58    let mut shape = Vec::new();
59    let mut first: Option<Vec<f64>> = None;
60    let mut same = Vec::new();
61    for _ in 0..repeat.max(1) {
62        let (status, body) = c
63            .post("/v1/embeddings", json!({ "model": model, "input": TEXT }))
64            .await;
65        match vectors(status, &body) {
66            Ok(mut v) => {
67                shape.push(Outcome::Pass);
68                let v = v.swap_remove(0);
69                match &first {
70                    None => {
71                        first = Some(v);
72                        same.push(Outcome::Pass);
73                    }
74                    Some(f) => same.push(
75                        if f.len() == v.len() && f.iter().zip(&v).all(|(a, b)| (a - b).abs() < 1e-4)
76                        {
77                            Outcome::Pass
78                        } else {
79                            Outcome::Fail("embed_nondeterministic".into())
80                        },
81                    ),
82                }
83            }
84            Err(class) => {
85                shape.push(Outcome::Fail(class.clone()));
86                same.push(Outcome::Skip("no vector".into()));
87            }
88        }
89    }
90    let (status, body) = c
91        .post(
92            "/v1/embeddings",
93            json!({ "model": model, "input": [TEXT, "A completely different sentence about the sea."] }),
94        )
95        .await;
96    let batch = match vectors(status, &body) {
97        Ok(v) if v.len() == 2 && v[0].len() == v[1].len() && v[0] != v[1] => Outcome::Pass,
98        Ok(v) if v.len() == 2 && v[0] == v[1] => Outcome::Fail("embed_identical".into()),
99        Ok(_) => Outcome::Fail("embed_batch_shape".into()),
100        Err(class) => Outcome::Fail(class),
101    };
102    let results = vec![
103        result(
104            "embed.shape",
105            "one input returns one finite, non-empty vector",
106            true,
107            shape,
108        ),
109        result(
110            "embed.deterministic",
111            "the same input returns the same vector every time",
112            true,
113            same,
114        ),
115        result(
116            "embed.batch",
117            "two inputs return two vectors of the same length that differ",
118            false,
119            vec![batch],
120        ),
121    ];
122    let mut failures: Vec<String> = results
123        .iter()
124        .filter_map(|r| match &r.outcome {
125            Outcome::Fail(c) => Some(c.split(' ').next().unwrap_or(c).to_owned()),
126            _ => None,
127        })
128        .collect();
129    failures.sort();
130    failures.dedup();
131    Report {
132        probe_version: PROBE_VERSION.into(),
133        url: c.base().to_owned(),
134        model: model.into(),
135        template_blake3: None,
136        quant: None,
137        verdict: verdict(&results).into(),
138        detected,
139        results,
140        failures,
141        known_issues: Vec::new(),
142    }
143}
loadmaster_probe/embed.rs

loadmaster_probe/
embed.rs
