hanzo_client/apis/
eval_api.rs

1/*
2 * Hanzo Cloud API
3 *
4 * The Hanzo Cloud API as a customer calls it: every operation under /v1/ except the operator's admin product, relay routes, legacy spellings and capabilities still reached by flag. Tagged by product: the first path segment after /v1/.
5 *
6 * The version of the OpenAPI document: v1
7 * 
8 * Generated by: https://openapi-generator.tech
9 */
10
11
12use reqwest;
13use serde::{Deserialize, Serialize, de::Error as _};
14use crate::{apis::ResponseContent, models};
15use super::{Error, configuration, ContentType};
16
17
18/// struct for typed errors of method [`delete_eval_datasets_by_name`]
19#[derive(Debug, Clone, Serialize, Deserialize)]
20#[serde(untagged)]
21pub enum DeleteEvalDatasetsByNameError {
22    UnknownValue(serde_json::Value),
23}
24
25/// struct for typed errors of method [`get_eval_datasets`]
26#[derive(Debug, Clone, Serialize, Deserialize)]
27#[serde(untagged)]
28pub enum GetEvalDatasetsError {
29    UnknownValue(serde_json::Value),
30}
31
32/// struct for typed errors of method [`get_eval_datasets_by_name`]
33#[derive(Debug, Clone, Serialize, Deserialize)]
34#[serde(untagged)]
35pub enum GetEvalDatasetsByNameError {
36    UnknownValue(serde_json::Value),
37}
38
39/// struct for typed errors of method [`get_eval_datasets_by_name_items`]
40#[derive(Debug, Clone, Serialize, Deserialize)]
41#[serde(untagged)]
42pub enum GetEvalDatasetsByNameItemsError {
43    UnknownValue(serde_json::Value),
44}
45
46/// struct for typed errors of method [`get_eval_evaluators`]
47#[derive(Debug, Clone, Serialize, Deserialize)]
48#[serde(untagged)]
49pub enum GetEvalEvaluatorsError {
50    UnknownValue(serde_json::Value),
51}
52
53/// struct for typed errors of method [`get_eval_metrics`]
54#[derive(Debug, Clone, Serialize, Deserialize)]
55#[serde(untagged)]
56pub enum GetEvalMetricsError {
57    UnknownValue(serde_json::Value),
58}
59
60/// struct for typed errors of method [`get_eval_rubrics`]
61#[derive(Debug, Clone, Serialize, Deserialize)]
62#[serde(untagged)]
63pub enum GetEvalRubricsError {
64    UnknownValue(serde_json::Value),
65}
66
67/// struct for typed errors of method [`get_eval_runs`]
68#[derive(Debug, Clone, Serialize, Deserialize)]
69#[serde(untagged)]
70pub enum GetEvalRunsError {
71    UnknownValue(serde_json::Value),
72}
73
74/// struct for typed errors of method [`get_eval_scores`]
75#[derive(Debug, Clone, Serialize, Deserialize)]
76#[serde(untagged)]
77pub enum GetEvalScoresError {
78    UnknownValue(serde_json::Value),
79}
80
81/// struct for typed errors of method [`get_eval_traces`]
82#[derive(Debug, Clone, Serialize, Deserialize)]
83#[serde(untagged)]
84pub enum GetEvalTracesError {
85    UnknownValue(serde_json::Value),
86}
87
88/// struct for typed errors of method [`post_eval_datasets`]
89#[derive(Debug, Clone, Serialize, Deserialize)]
90#[serde(untagged)]
91pub enum PostEvalDatasetsError {
92    UnknownValue(serde_json::Value),
93}
94
95/// struct for typed errors of method [`post_eval_datasets_by_name_items`]
96#[derive(Debug, Clone, Serialize, Deserialize)]
97#[serde(untagged)]
98pub enum PostEvalDatasetsByNameItemsError {
99    UnknownValue(serde_json::Value),
100}
101
102/// struct for typed errors of method [`post_eval_evaluators`]
103#[derive(Debug, Clone, Serialize, Deserialize)]
104#[serde(untagged)]
105pub enum PostEvalEvaluatorsError {
106    UnknownValue(serde_json::Value),
107}
108
109/// struct for typed errors of method [`post_eval_rubrics`]
110#[derive(Debug, Clone, Serialize, Deserialize)]
111#[serde(untagged)]
112pub enum PostEvalRubricsError {
113    UnknownValue(serde_json::Value),
114}
115
116/// struct for typed errors of method [`post_eval_runs`]
117#[derive(Debug, Clone, Serialize, Deserialize)]
118#[serde(untagged)]
119pub enum PostEvalRunsError {
120    Status502(models::RunSummary),
121    UnknownValue(serde_json::Value),
122}
123
124/// struct for typed errors of method [`post_eval_scores`]
125#[derive(Debug, Clone, Serialize, Deserialize)]
126#[serde(untagged)]
127pub enum PostEvalScoresError {
128    UnknownValue(serde_json::Value),
129}
130
131
132/// Removes the named dataset of the caller's org AND all of its examples, in one transaction.  This is not a detach: the examples are gone with the set, so a dataset cannot be resurrected by re-creating the name. A name this org does not have is 404 — never a silent success — and a name belonging to another tenant is the same 404, because the delete is predicated on the validated org. Requires a validated principal; 403 without one. Runs and scores already recorded against the dataset are telemetry events and are NOT deleted with it.
133pub async fn delete_eval_datasets_by_name(configuration: &configuration::Configuration, name: &str) -> Result<serde_json::Value, Error<DeleteEvalDatasetsByNameError>> {
134    // add a prefix to parameters to efficiently prevent name collisions
135    let p_name = name;
136
137    let uri_str = format!("{}/v1/eval/datasets/{name}", configuration.base_path, name=crate::apis::urlencode(p_name));
138    let mut req_builder = configuration.client.request(reqwest::Method::DELETE, &uri_str);
139
140    if let Some(ref user_agent) = configuration.user_agent {
141        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
142    }
143    if let Some(ref token) = configuration.bearer_access_token {
144        req_builder = req_builder.bearer_auth(token.to_owned());
145    };
146
147    let req = req_builder.build()?;
148    let resp = configuration.client.execute(req).await?;
149
150    let status = resp.status();
151    let content_type = resp
152        .headers()
153        .get("content-type")
154        .and_then(|v| v.to_str().ok())
155        .unwrap_or("application/octet-stream");
156    let content_type = super::ContentType::from(content_type);
157
158    if !status.is_client_error() && !status.is_server_error() {
159        let content = resp.text().await?;
160        match content_type {
161            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
162            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `serde_json::Value`"))),
163            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `serde_json::Value`")))),
164        }
165    } else {
166        let content = resp.text().await?;
167        let entity: Option<DeleteEvalDatasetsByNameError> = serde_json::from_str(&content).ok();
168        Err(Error::ResponseError(ResponseContent { status, content, entity }))
169    }
170}
171
172/// Is the datasets your org has, each with its name, description, metadata and timestamps.  It is the only way to enumerate what an org holds. Requires a validated principal; 403 without one. Every row is filtered on the validated org, so there is no parameter that reaches another tenant's datasets. The item count is NOT populated here — read one dataset to get it.
173pub async fn get_eval_datasets(configuration: &configuration::Configuration, limit: Option<i32>) -> Result<models::DatasetList, Error<GetEvalDatasetsError>> {
174    // add a prefix to parameters to efficiently prevent name collisions
175    let p_limit = limit;
176
177    let uri_str = format!("{}/v1/eval/datasets", configuration.base_path);
178    let mut req_builder = configuration.client.request(reqwest::Method::GET, &uri_str);
179
180    if let Some(ref param_value) = p_limit {
181        req_builder = req_builder.query(&[("limit", &param_value.to_string())]);
182    }
183    if let Some(ref user_agent) = configuration.user_agent {
184        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
185    }
186    if let Some(ref token) = configuration.bearer_access_token {
187        req_builder = req_builder.bearer_auth(token.to_owned());
188    };
189
190    let req = req_builder.build()?;
191    let resp = configuration.client.execute(req).await?;
192
193    let status = resp.status();
194    let content_type = resp
195        .headers()
196        .get("content-type")
197        .and_then(|v| v.to_str().ok())
198        .unwrap_or("application/octet-stream");
199    let content_type = super::ContentType::from(content_type);
200
201    if !status.is_client_error() && !status.is_server_error() {
202        let content = resp.text().await?;
203        match content_type {
204            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
205            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::DatasetList`"))),
206            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::DatasetList`")))),
207        }
208    } else {
209        let content = resp.text().await?;
210        let entity: Option<GetEvalDatasetsError> = serde_json::from_str(&content).ok();
211        Err(Error::ResponseError(ResponseContent { status, content, entity }))
212    }
213}
214
215/// Returns one dataset of the caller's org by name, together with its live item count — the one read that answers how big the set actually is.  A name this org does not have is 404, which is also what another tenant's dataset looks like from here. Requires a validated principal; 403 without one.
216pub async fn get_eval_datasets_by_name(configuration: &configuration::Configuration, name: &str) -> Result<models::DatasetView, Error<GetEvalDatasetsByNameError>> {
217    // add a prefix to parameters to efficiently prevent name collisions
218    let p_name = name;
219
220    let uri_str = format!("{}/v1/eval/datasets/{name}", configuration.base_path, name=crate::apis::urlencode(p_name));
221    let mut req_builder = configuration.client.request(reqwest::Method::GET, &uri_str);
222
223    if let Some(ref user_agent) = configuration.user_agent {
224        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
225    }
226    if let Some(ref token) = configuration.bearer_access_token {
227        req_builder = req_builder.bearer_auth(token.to_owned());
228    };
229
230    let req = req_builder.build()?;
231    let resp = configuration.client.execute(req).await?;
232
233    let status = resp.status();
234    let content_type = resp
235        .headers()
236        .get("content-type")
237        .and_then(|v| v.to_str().ok())
238        .unwrap_or("application/octet-stream");
239    let content_type = super::ContentType::from(content_type);
240
241    if !status.is_client_error() && !status.is_server_error() {
242        let content = resp.text().await?;
243        match content_type {
244            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
245            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::DatasetView`"))),
246            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::DatasetView`")))),
247        }
248    } else {
249        let content = resp.text().await?;
250        let entity: Option<GetEvalDatasetsByNameError> = serde_json::from_str(&content).ok();
251        Err(Error::ResponseError(ResponseContent { status, content, entity }))
252    }
253}
254
255/// Is the examples in one of your datasets — the set is named in the path, because this collection only exists inside one.  Archived examples are included, so the caller sees the whole set rather than only what a run would use. Requires a validated principal; 403 without one, and the read is filtered on the validated org, so naming another tenant's dataset returns nothing rather than its contents.
256pub async fn get_eval_datasets_by_name_items(configuration: &configuration::Configuration, name: &str, limit: Option<i32>) -> Result<models::ItemList, Error<GetEvalDatasetsByNameItemsError>> {
257    // add a prefix to parameters to efficiently prevent name collisions
258    let p_name = name;
259    let p_limit = limit;
260
261    let uri_str = format!("{}/v1/eval/datasets/{name}/items", configuration.base_path, name=crate::apis::urlencode(p_name));
262    let mut req_builder = configuration.client.request(reqwest::Method::GET, &uri_str);
263
264    if let Some(ref param_value) = p_limit {
265        req_builder = req_builder.query(&[("limit", &param_value.to_string())]);
266    }
267    if let Some(ref user_agent) = configuration.user_agent {
268        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
269    }
270    if let Some(ref token) = configuration.bearer_access_token {
271        req_builder = req_builder.bearer_auth(token.to_owned());
272    };
273
274    let req = req_builder.build()?;
275    let resp = configuration.client.execute(req).await?;
276
277    let status = resp.status();
278    let content_type = resp
279        .headers()
280        .get("content-type")
281        .and_then(|v| v.to_str().ok())
282        .unwrap_or("application/octet-stream");
283    let content_type = super::ContentType::from(content_type);
284
285    if !status.is_client_error() && !status.is_server_error() {
286        let content = resp.text().await?;
287        match content_type {
288            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
289            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::ItemList`"))),
290            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::ItemList`")))),
291        }
292    } else {
293        let content = resp.text().await?;
294        let entity: Option<GetEvalDatasetsByNameItemsError> = serde_json::from_str(&content).ok();
295        Err(Error::ResponseError(ResponseContent { status, content, entity }))
296    }
297}
298
299/// Is the judges your org has defined, each with its judge model, criteria and the score name it writes under.  Requires a validated principal; 403 without one, and the listing is filtered on the validated org.
300pub async fn get_eval_evaluators(configuration: &configuration::Configuration, limit: Option<i32>) -> Result<models::EvaluatorList, Error<GetEvalEvaluatorsError>> {
301    // add a prefix to parameters to efficiently prevent name collisions
302    let p_limit = limit;
303
304    let uri_str = format!("{}/v1/eval/evaluators", configuration.base_path);
305    let mut req_builder = configuration.client.request(reqwest::Method::GET, &uri_str);
306
307    if let Some(ref param_value) = p_limit {
308        req_builder = req_builder.query(&[("limit", &param_value.to_string())]);
309    }
310    if let Some(ref user_agent) = configuration.user_agent {
311        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
312    }
313    if let Some(ref token) = configuration.bearer_access_token {
314        req_builder = req_builder.bearer_auth(token.to_owned());
315    };
316
317    let req = req_builder.build()?;
318    let resp = configuration.client.execute(req).await?;
319
320    let status = resp.status();
321    let content_type = resp
322        .headers()
323        .get("content-type")
324        .and_then(|v| v.to_str().ok())
325        .unwrap_or("application/octet-stream");
326    let content_type = super::ContentType::from(content_type);
327
328    if !status.is_client_error() && !status.is_server_error() {
329        let content = resp.text().await?;
330        match content_type {
331            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
332            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::EvaluatorList`"))),
333            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::EvaluatorList`")))),
334        }
335    } else {
336        let content = resp.text().await?;
337        let entity: Option<GetEvalEvaluatorsError> = serde_json::from_str(&content).ok();
338        Err(Error::ResponseError(ResponseContent { status, content, entity }))
339    }
340}
341
342/// Is your org's AI overview board over a window: totals (generations, prompt and completion tokens, cost in cents, errors, success rate, distinct models and users), a gap-filled time series, a per-model breakdown with the long tail folded into \"other\", and latency percentiles read from the GenAI spans.  The window the answer was actually computed over is echoed back, so a client never has to infer it. A platform admin sees the board across ALL orgs; everyone else sees their own.  The board is HONEST-EMPTY where it cannot be computed: with no datastore wired, or under a named project scope the usage ledger does not yet carry, it answers a valid board with zero totals and a flat series rather than a fabricated number or a 500. Requires a validated principal; 403 without one.
343pub async fn get_eval_metrics(configuration: &configuration::Configuration, range: Option<&str>, interval: Option<&str>) -> Result<models::Board, Error<GetEvalMetricsError>> {
344    // add a prefix to parameters to efficiently prevent name collisions
345    let p_range = range;
346    let p_interval = interval;
347
348    let uri_str = format!("{}/v1/eval/metrics", configuration.base_path);
349    let mut req_builder = configuration.client.request(reqwest::Method::GET, &uri_str);
350
351    if let Some(ref param_value) = p_range {
352        req_builder = req_builder.query(&[("range", &param_value.to_string())]);
353    }
354    if let Some(ref param_value) = p_interval {
355        req_builder = req_builder.query(&[("interval", &param_value.to_string())]);
356    }
357    if let Some(ref user_agent) = configuration.user_agent {
358        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
359    }
360    if let Some(ref token) = configuration.bearer_access_token {
361        req_builder = req_builder.bearer_auth(token.to_owned());
362    };
363
364    let req = req_builder.build()?;
365    let resp = configuration.client.execute(req).await?;
366
367    let status = resp.status();
368    let content_type = resp
369        .headers()
370        .get("content-type")
371        .and_then(|v| v.to_str().ok())
372        .unwrap_or("application/octet-stream");
373    let content_type = super::ContentType::from(content_type);
374
375    if !status.is_client_error() && !status.is_server_error() {
376        let content = resp.text().await?;
377        match content_type {
378            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
379            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::Board`"))),
380            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::Board`")))),
381        }
382    } else {
383        let content = resp.text().await?;
384        let entity: Option<GetEvalMetricsError> = serde_json::from_str(&content).ok();
385        Err(Error::ResponseError(ResponseContent { status, content, entity }))
386    }
387}
388
389/// Is the score shapes your org has declared — each name's data type, its numeric bounds and its allowed categories.  Requires a validated principal; 403 without one, and the listing is filtered on the validated org.
390pub async fn get_eval_rubrics(configuration: &configuration::Configuration, limit: Option<i32>) -> Result<models::ScoreConfigList, Error<GetEvalRubricsError>> {
391    // add a prefix to parameters to efficiently prevent name collisions
392    let p_limit = limit;
393
394    let uri_str = format!("{}/v1/eval/rubrics", configuration.base_path);
395    let mut req_builder = configuration.client.request(reqwest::Method::GET, &uri_str);
396
397    if let Some(ref param_value) = p_limit {
398        req_builder = req_builder.query(&[("limit", &param_value.to_string())]);
399    }
400    if let Some(ref user_agent) = configuration.user_agent {
401        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
402    }
403    if let Some(ref token) = configuration.bearer_access_token {
404        req_builder = req_builder.bearer_auth(token.to_owned());
405    };
406
407    let req = req_builder.build()?;
408    let resp = configuration.client.execute(req).await?;
409
410    let status = resp.status();
411    let content_type = resp
412        .headers()
413        .get("content-type")
414        .and_then(|v| v.to_str().ok())
415        .unwrap_or("application/octet-stream");
416    let content_type = super::ContentType::from(content_type);
417
418    if !status.is_client_error() && !status.is_server_error() {
419        let content = resp.text().await?;
420        match content_type {
421            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
422            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::ScoreConfigList`"))),
423            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::ScoreConfigList`")))),
424        }
425    } else {
426        let content = resp.text().await?;
427        let entity: Option<GetEvalRubricsError> = serde_json::from_str(&content).ok();
428        Err(Error::ResponseError(ResponseContent { status, content, entity }))
429    }
430}
431
432/// Is your past runs and how they scored — the dataset and model, the judge model, how many examples were attempted and how many scored, the average score, and when it happened.  Requires a validated principal; 403 without one, and rows are filtered on the validated org. These records come from the metastore rather than the datastore, so they are readable on a deployment with no telemetry wired — but a run's traces and scores are not.
433pub async fn get_eval_runs(configuration: &configuration::Configuration, dataset_name: Option<&str>, limit: Option<i32>) -> Result<models::Runs, Error<GetEvalRunsError>> {
434    // add a prefix to parameters to efficiently prevent name collisions
435    let p_dataset_name = dataset_name;
436    let p_limit = limit;
437
438    let uri_str = format!("{}/v1/eval/runs", configuration.base_path);
439    let mut req_builder = configuration.client.request(reqwest::Method::GET, &uri_str);
440
441    if let Some(ref param_value) = p_dataset_name {
442        req_builder = req_builder.query(&[("datasetName", &param_value.to_string())]);
443    }
444    if let Some(ref param_value) = p_limit {
445        req_builder = req_builder.query(&[("limit", &param_value.to_string())]);
446    }
447    if let Some(ref user_agent) = configuration.user_agent {
448        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
449    }
450    if let Some(ref token) = configuration.bearer_access_token {
451        req_builder = req_builder.bearer_auth(token.to_owned());
452    };
453
454    let req = req_builder.build()?;
455    let resp = configuration.client.execute(req).await?;
456
457    let status = resp.status();
458    let content_type = resp
459        .headers()
460        .get("content-type")
461        .and_then(|v| v.to_str().ok())
462        .unwrap_or("application/octet-stream");
463    let content_type = super::ContentType::from(content_type);
464
465    if !status.is_client_error() && !status.is_server_error() {
466        let content = resp.text().await?;
467        match content_type {
468            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
469            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::Runs`"))),
470            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::Runs`")))),
471        }
472    } else {
473        let content = resp.text().await?;
474        let entity: Option<GetEvalRunsError> = serde_json::from_str(&content).ok();
475        Err(Error::ResponseError(ResponseContent { status, content, entity }))
476    }
477}
478
479/// Is the score events your org has recorded, narrowed by any of name, runName and traceId.  The org is bound as an authoritative predicate on the query, never taken from a header, so a filter can narrow the caller's own scores but can never widen past them. Requires a validated principal; 403 without one. Scores live in the datastore, so a deployment with none wired answers 503 rather than an empty page that would read as \"no scores\".
480pub async fn get_eval_scores(configuration: &configuration::Configuration, name: Option<&str>, run_name: Option<&str>, trace_id: Option<&str>, limit: Option<i32>) -> Result<models::ScoreList, Error<GetEvalScoresError>> {
481    // add a prefix to parameters to efficiently prevent name collisions
482    let p_name = name;
483    let p_run_name = run_name;
484    let p_trace_id = trace_id;
485    let p_limit = limit;
486
487    let uri_str = format!("{}/v1/eval/scores", configuration.base_path);
488    let mut req_builder = configuration.client.request(reqwest::Method::GET, &uri_str);
489
490    if let Some(ref param_value) = p_name {
491        req_builder = req_builder.query(&[("name", &param_value.to_string())]);
492    }
493    if let Some(ref param_value) = p_run_name {
494        req_builder = req_builder.query(&[("runName", &param_value.to_string())]);
495    }
496    if let Some(ref param_value) = p_trace_id {
497        req_builder = req_builder.query(&[("traceId", &param_value.to_string())]);
498    }
499    if let Some(ref param_value) = p_limit {
500        req_builder = req_builder.query(&[("limit", &param_value.to_string())]);
501    }
502    if let Some(ref user_agent) = configuration.user_agent {
503        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
504    }
505    if let Some(ref token) = configuration.bearer_access_token {
506        req_builder = req_builder.bearer_auth(token.to_owned());
507    };
508
509    let req = req_builder.build()?;
510    let resp = configuration.client.execute(req).await?;
511
512    let status = resp.status();
513    let content_type = resp
514        .headers()
515        .get("content-type")
516        .and_then(|v| v.to_str().ok())
517        .unwrap_or("application/octet-stream");
518    let content_type = super::ContentType::from(content_type);
519
520    if !status.is_client_error() && !status.is_server_error() {
521        let content = resp.text().await?;
522        match content_type {
523            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
524            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::ScoreList`"))),
525            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::ScoreList`")))),
526        }
527    } else {
528        let content = resp.text().await?;
529        let entity: Option<GetEvalScoresError> = serde_json::from_str(&content).ok();
530        Err(Error::ResponseError(ResponseContent { status, content, entity }))
531    }
532}
533
534/// Is the traces behind your evaluations — one per model call an evaluation made, carrying its input, output, model and timing — narrowed by any of sessionId, runName and datasetName.  Scoped by org AND by project: the project is the caller's server-minted scope, not a parameter, so it cannot be widened by asking. Requires a validated principal; 403 without one. Traces live in the datastore, so a deployment with none wired answers 503 rather than an empty page.
535pub async fn get_eval_traces(configuration: &configuration::Configuration, session_id: Option<&str>, run_name: Option<&str>, dataset_name: Option<&str>, limit: Option<i32>) -> Result<models::TraceList, Error<GetEvalTracesError>> {
536    // add a prefix to parameters to efficiently prevent name collisions
537    let p_session_id = session_id;
538    let p_run_name = run_name;
539    let p_dataset_name = dataset_name;
540    let p_limit = limit;
541
542    let uri_str = format!("{}/v1/eval/traces", configuration.base_path);
543    let mut req_builder = configuration.client.request(reqwest::Method::GET, &uri_str);
544
545    if let Some(ref param_value) = p_session_id {
546        req_builder = req_builder.query(&[("sessionId", &param_value.to_string())]);
547    }
548    if let Some(ref param_value) = p_run_name {
549        req_builder = req_builder.query(&[("runName", &param_value.to_string())]);
550    }
551    if let Some(ref param_value) = p_dataset_name {
552        req_builder = req_builder.query(&[("datasetName", &param_value.to_string())]);
553    }
554    if let Some(ref param_value) = p_limit {
555        req_builder = req_builder.query(&[("limit", &param_value.to_string())]);
556    }
557    if let Some(ref user_agent) = configuration.user_agent {
558        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
559    }
560    if let Some(ref token) = configuration.bearer_access_token {
561        req_builder = req_builder.bearer_auth(token.to_owned());
562    };
563
564    let req = req_builder.build()?;
565    let resp = configuration.client.execute(req).await?;
566
567    let status = resp.status();
568    let content_type = resp
569        .headers()
570        .get("content-type")
571        .and_then(|v| v.to_str().ok())
572        .unwrap_or("application/octet-stream");
573    let content_type = super::ContentType::from(content_type);
574
575    if !status.is_client_error() && !status.is_server_error() {
576        let content = resp.text().await?;
577        match content_type {
578            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
579            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::TraceList`"))),
580            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::TraceList`")))),
581        }
582    } else {
583        let content = resp.text().await?;
584        let entity: Option<GetEvalTracesError> = serde_json::from_str(&content).ok();
585        Err(Error::ResponseError(ResponseContent { status, content, entity }))
586    }
587}
588
589/// Writes a dataset — the named set of graded examples a run scores a model against — under the caller's org and answers 201 with it.  The NAME is the key, not an id: posting a name the org already has updates that dataset's description and metadata and keeps its original creation time, so this is create-or-edit and never a duplicate. Its items are untouched.  Requires a validated principal; 403 without one. The org comes from the validated owner claim, never from a client X-Org-Id, so a dataset can only ever be written under the caller's own tenant. A description over 64 KiB is 400.
590pub async fn post_eval_datasets(configuration: &configuration::Configuration, dataset_req: models::DatasetReq) -> Result<models::DatasetView, Error<PostEvalDatasetsError>> {
591    // add a prefix to parameters to efficiently prevent name collisions
592    let p_dataset_req = dataset_req;
593
594    let uri_str = format!("{}/v1/eval/datasets", configuration.base_path);
595    let mut req_builder = configuration.client.request(reqwest::Method::POST, &uri_str);
596
597    if let Some(ref user_agent) = configuration.user_agent {
598        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
599    }
600    if let Some(ref token) = configuration.bearer_access_token {
601        req_builder = req_builder.bearer_auth(token.to_owned());
602    };
603    req_builder = req_builder.json(&p_dataset_req);
604
605    let req = req_builder.build()?;
606    let resp = configuration.client.execute(req).await?;
607
608    let status = resp.status();
609    let content_type = resp
610        .headers()
611        .get("content-type")
612        .and_then(|v| v.to_str().ok())
613        .unwrap_or("application/octet-stream");
614    let content_type = super::ContentType::from(content_type);
615
616    if !status.is_client_error() && !status.is_server_error() {
617        let content = resp.text().await?;
618        match content_type {
619            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
620            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::DatasetView`"))),
621            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::DatasetView`")))),
622        }
623    } else {
624        let content = resp.text().await?;
625        let entity: Option<PostEvalDatasetsError> = serde_json::from_str(&content).ok();
626        Err(Error::ResponseError(ResponseContent { status, content, entity }))
627    }
628}
629
630/// Writes one graded example — its input, its expected output, free-form metadata and a status — into the dataset named in the path, and answers 201 with it.  That dataset MUST already exist for this org: an unknown one is 404, never a silent create. Requires a validated principal; 403 without one.
631pub async fn post_eval_datasets_by_name_items(configuration: &configuration::Configuration, name: &str, item_req: models::ItemReq) -> Result<models::ItemView, Error<PostEvalDatasetsByNameItemsError>> {
632    // add a prefix to parameters to efficiently prevent name collisions
633    let p_name = name;
634    let p_item_req = item_req;
635
636    let uri_str = format!("{}/v1/eval/datasets/{name}/items", configuration.base_path, name=crate::apis::urlencode(p_name));
637    let mut req_builder = configuration.client.request(reqwest::Method::POST, &uri_str);
638
639    if let Some(ref user_agent) = configuration.user_agent {
640        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
641    }
642    if let Some(ref token) = configuration.bearer_access_token {
643        req_builder = req_builder.bearer_auth(token.to_owned());
644    };
645    req_builder = req_builder.json(&p_item_req);
646
647    let req = req_builder.build()?;
648    let resp = configuration.client.execute(req).await?;
649
650    let status = resp.status();
651    let content_type = resp
652        .headers()
653        .get("content-type")
654        .and_then(|v| v.to_str().ok())
655        .unwrap_or("application/octet-stream");
656    let content_type = super::ContentType::from(content_type);
657
658    if !status.is_client_error() && !status.is_server_error() {
659        let content = resp.text().await?;
660        match content_type {
661            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
662            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::ItemView`"))),
663            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::ItemView`")))),
664        }
665    } else {
666        let content = resp.text().await?;
667        let entity: Option<PostEvalDatasetsByNameItemsError> = serde_json::from_str(&content).ok();
668        Err(Error::ResponseError(ResponseContent { status, content, entity }))
669    }
670}
671
672/// Saves a reusable judge for the caller's org — the judge model and the written criteria it grades against — and answers 201 with it.  Like a dataset, the NAME is the key: re-posting a name edits that judge rather than adding a second one. Requires a validated principal; 403 without one.
673pub async fn post_eval_evaluators(configuration: &configuration::Configuration, evaluator_req: models::EvaluatorReq) -> Result<models::EvaluatorView, Error<PostEvalEvaluatorsError>> {
674    // add a prefix to parameters to efficiently prevent name collisions
675    let p_evaluator_req = evaluator_req;
676
677    let uri_str = format!("{}/v1/eval/evaluators", configuration.base_path);
678    let mut req_builder = configuration.client.request(reqwest::Method::POST, &uri_str);
679
680    if let Some(ref user_agent) = configuration.user_agent {
681        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
682    }
683    if let Some(ref token) = configuration.bearer_access_token {
684        req_builder = req_builder.bearer_auth(token.to_owned());
685    };
686    req_builder = req_builder.json(&p_evaluator_req);
687
688    let req = req_builder.build()?;
689    let resp = configuration.client.execute(req).await?;
690
691    let status = resp.status();
692    let content_type = resp
693        .headers()
694        .get("content-type")
695        .and_then(|v| v.to_str().ok())
696        .unwrap_or("application/octet-stream");
697    let content_type = super::ContentType::from(content_type);
698
699    if !status.is_client_error() && !status.is_server_error() {
700        let content = resp.text().await?;
701        match content_type {
702            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
703            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::EvaluatorView`"))),
704            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::EvaluatorView`")))),
705        }
706    } else {
707        let content = resp.text().await?;
708        let entity: Option<PostEvalEvaluatorsError> = serde_json::from_str(&content).ok();
709        Err(Error::ResponseError(ResponseContent { status, content, entity }))
710    }
711}
712
713/// Defines the shape of one score name for the caller's org and answers 201 with it.  This is the integrity contract, not documentation: once a rubric exists for a name, every score recorded under that name is checked against it and the rubric's data type is AUTHORITATIVE — a caller cannot claim a different one. Out-of-range values, unlisted labels and non-finite numbers are refused at write time.  A CATEGORICAL rubric with no categories is 400, as is a non-finite bound or a minValue above maxValue. Requires a validated principal; 403 without one.
714pub async fn post_eval_rubrics(configuration: &configuration::Configuration, score_config_req: models::ScoreConfigReq) -> Result<models::ScoreConfigView, Error<PostEvalRubricsError>> {
715    // add a prefix to parameters to efficiently prevent name collisions
716    let p_score_config_req = score_config_req;
717
718    let uri_str = format!("{}/v1/eval/rubrics", configuration.base_path);
719    let mut req_builder = configuration.client.request(reqwest::Method::POST, &uri_str);
720
721    if let Some(ref user_agent) = configuration.user_agent {
722        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
723    }
724    if let Some(ref token) = configuration.bearer_access_token {
725        req_builder = req_builder.bearer_auth(token.to_owned());
726    };
727    req_builder = req_builder.json(&p_score_config_req);
728
729    let req = req_builder.build()?;
730    let resp = configuration.client.execute(req).await?;
731
732    let status = resp.status();
733    let content_type = resp
734        .headers()
735        .get("content-type")
736        .and_then(|v| v.to_str().ok())
737        .unwrap_or("application/octet-stream");
738    let content_type = super::ContentType::from(content_type);
739
740    if !status.is_client_error() && !status.is_server_error() {
741        let content = resp.text().await?;
742        match content_type {
743            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
744            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::ScoreConfigView`"))),
745            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::ScoreConfigView`")))),
746        }
747    } else {
748        let content = resp.text().await?;
749        let entity: Option<PostEvalRubricsError> = serde_json::from_str(&content).ok();
750        Err(Error::ResponseError(ResponseContent { status, content, entity }))
751    }
752}
753
754/// Runs a real evaluation and answers the summary when it is finished — this is synchronous work, not a job id.  For each ACTIVE example in the dataset it calls the model under test, records a trace, calls the LLM-as-judge, and records the judge's score with its reasoning. The answer carries the per-item results (item id, trace id, score, output or error) alongside items, scored and avgScore.  The dataset must belong to the caller's org (404 otherwise) and must have at least one ACTIVE example (422 otherwise).  It runs as YOU: the caller's own Authorization bearer drives the model gateway, so a request without one is 401 rather than a run made anonymously or under a service identity. Only a non-reversible hash of that credential is recorded on the traces.  Bounded and honest about it: an org may have at most 4 runs in flight and the fifth is 429 rather than queued, and the whole run is capped at 10 minutes — examples past the deadline come back with an error instead of a score, and scored counts only real successes. A run where NOTHING scored answers 502, not a 200 that looks like an evaluation. A run must be able to persist what it produces, so a deployment with no datastore wired is 503 up front. Requires a validated principal; 403 without one.
755pub async fn post_eval_runs(configuration: &configuration::Configuration, run_request: models::RunRequest, authorization: Option<&str>) -> Result<models::RunSummary, Error<PostEvalRunsError>> {
756    // add a prefix to parameters to efficiently prevent name collisions
757    let p_run_request = run_request;
758    let p_authorization = authorization;
759
760    let uri_str = format!("{}/v1/eval/runs", configuration.base_path);
761    let mut req_builder = configuration.client.request(reqwest::Method::POST, &uri_str);
762
763    if let Some(ref user_agent) = configuration.user_agent {
764        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
765    }
766    if let Some(param_value) = p_authorization {
767        req_builder = req_builder.header("Authorization", param_value.to_string());
768    }
769    if let Some(ref token) = configuration.bearer_access_token {
770        req_builder = req_builder.bearer_auth(token.to_owned());
771    };
772    req_builder = req_builder.json(&p_run_request);
773
774    let req = req_builder.build()?;
775    let resp = configuration.client.execute(req).await?;
776
777    let status = resp.status();
778    let content_type = resp
779        .headers()
780        .get("content-type")
781        .and_then(|v| v.to_str().ok())
782        .unwrap_or("application/octet-stream");
783    let content_type = super::ContentType::from(content_type);
784
785    if !status.is_client_error() && !status.is_server_error() {
786        let content = resp.text().await?;
787        match content_type {
788            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
789            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::RunSummary`"))),
790            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::RunSummary`")))),
791        }
792    } else {
793        let content = resp.text().await?;
794        let entity: Option<PostEvalRunsError> = serde_json::from_str(&content).ok();
795        Err(Error::ResponseError(ResponseContent { status, content, entity }))
796    }
797}
798
799/// Files one score event for the caller's org and answers 201 with it.  This is how human review and out-of-band graders land beside the automatic ones: name the score, give it a value (or a stringValue for a categorical label), and attach it to a trace, a run, a dataset example, or any combination.  Scores are validated fail-closed. A value must be FINITE — NaN and Inf are 400 — and if the org has declared a rubric for this name, that rubric decides the type and the value must satisfy it: inside the numeric bounds, or one of the allowed categories. A caller cannot override the declared type by sending a different dataType.  A score is TELEMETRY, not metadata, so it needs the datastore: a deployment with none wired answers 503 rather than accepting a score it cannot persist. Requires a validated principal; 403 without one, and the org is stamped from the validated claim rather than read off the body.
800pub async fn post_eval_scores(configuration: &configuration::Configuration, score_req: models::ScoreReq) -> Result<models::ScoreView, Error<PostEvalScoresError>> {
801    // add a prefix to parameters to efficiently prevent name collisions
802    let p_score_req = score_req;
803
804    let uri_str = format!("{}/v1/eval/scores", configuration.base_path);
805    let mut req_builder = configuration.client.request(reqwest::Method::POST, &uri_str);
806
807    if let Some(ref user_agent) = configuration.user_agent {
808        req_builder = req_builder.header(reqwest::header::USER_AGENT, user_agent.clone());
809    }
810    if let Some(ref token) = configuration.bearer_access_token {
811        req_builder = req_builder.bearer_auth(token.to_owned());
812    };
813    req_builder = req_builder.json(&p_score_req);
814
815    let req = req_builder.build()?;
816    let resp = configuration.client.execute(req).await?;
817
818    let status = resp.status();
819    let content_type = resp
820        .headers()
821        .get("content-type")
822        .and_then(|v| v.to_str().ok())
823        .unwrap_or("application/octet-stream");
824    let content_type = super::ContentType::from(content_type);
825
826    if !status.is_client_error() && !status.is_server_error() {
827        let content = resp.text().await?;
828        match content_type {
829            ContentType::Json => serde_json::from_str(&content).map_err(Error::from),
830            ContentType::Text => return Err(Error::from(serde_json::Error::custom("Received `text/plain` content type response that cannot be converted to `models::ScoreView`"))),
831            ContentType::Unsupported(unknown_type) => return Err(Error::from(serde_json::Error::custom(format!("Received `{unknown_type}` content type response that cannot be converted to `models::ScoreView`")))),
832        }
833    } else {
834        let content = resp.text().await?;
835        let entity: Option<PostEvalScoresError> = serde_json::from_str(&content).ok();
836        Err(Error::ResponseError(ResponseContent { status, content, entity }))
837    }
838}
839
hanzo_client/apis/eval_api.rs

hanzo_client/apis/
eval_api.rs
