openai_interface/evals/runs/
mod.rs

1//! Manage the runs of an eval via `/evals/{eval_id}/runs`.
2//!
3//! > ![warn] This module is untested!
4//! > If you encounter any issues, please report them on the repository.
5//!
6//! A run applies an eval's testing criteria to a data source (e.g. a
7//! set of completions) and reports per-item scores.
8//! Submodules: [`create`], [`retrieve`], [`cancel`], [`delete`],
9//! [`output_items`]; the list endpoint lives directly in this module.
10
11pub mod cancel;
12pub mod create;
13pub mod delete;
14pub mod output_items;
15pub mod retrieve;
16
17use url::Url;
18
19use crate::{
20    errors::OapiError,
21    pagination::PaginationQuery,
22    rest::get::{Get, GetNoStream},
23};
24
25/// The lifecycle status of an eval run.
26#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
27#[serde(rename_all = "snake_case")]
28pub enum EvalRunStatus {
29    /// The run is queued.
30    Queued,
31    /// The run is executing.
32    InProgress,
33    /// The run failed.
34    Failed,
35    /// The run was cancelled.
36    Canceled,
37    /// The run finished successfully.
38    Completed,
39}
40
41/// The per-testing-criteria result counts of an eval run.
42#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
43pub struct EvalRunCounts {
44    /// The number of items that passed all criteria.
45    #[serde(default)]
46    pub passed: u64,
47    /// The number of items that failed at least one criterion.
48    #[serde(default)]
49    pub failed: u64,
50    /// The number of items with errors.
51    #[serde(default)]
52    pub errored: u64,
53    /// The total number of items.
54    #[serde(default)]
55    pub total: u64,
56}
57
58/// The per-criterion pass/fail counts of an eval run.
59#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
60pub struct EvalRunPerTestingCriteriaResult {
61    /// The ID of the testing criterion.
62    #[serde(default)]
63    pub testing_criteria: Option<String>,
64    /// The number of items that passed this criterion.
65    #[serde(default)]
66    pub passed: u64,
67    /// The number of items that failed this criterion.
68    #[serde(default)]
69    pub failed: u64,
70}
71
72/// The error of a failed eval run.
73#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
74pub struct EvalRunError {
75    /// An error code identifying the failure mode.
76    #[serde(default)]
77    pub code: Option<String>,
78    /// A human-readable error message.
79    #[serde(default)]
80    pub message: Option<String>,
81}
82
83/// An eval run object.
84#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
85pub struct EvalRun {
86    /// The run ID, e.g. `evalrun_...`.
87    pub id: String,
88    /// The object type, always `eval.run`.
89    #[serde(default)]
90    pub object: Option<String>,
91    /// Unix timestamp (seconds) of when the run was created.
92    #[serde(default)]
93    pub created_at: Option<u64>,
94    /// The ID of the eval this run belongs to.
95    pub eval_id: String,
96    /// The current lifecycle status.
97    pub status: EvalRunStatus,
98    /// The data source of the run, as raw JSON.
99    #[serde(default)]
100    pub data_source: Option<serde_json::Value>,
101    /// The per-criterion results of the run.
102    #[serde(default)]
103    pub result_counts: Option<EvalRunCounts>,
104    /// The per-criterion pass/fail breakdown.
105    #[serde(default)]
106    pub per_testing_criteria_results: Option<Vec<EvalRunPerTestingCriteriaResult>>,
107    /// The error of a failed run.
108    #[serde(default)]
109    pub error: Option<EvalRunError>,
110    /// Arbitrary key-value metadata attached to the run.
111    #[serde(default)]
112    pub metadata: Option<std::collections::HashMap<String, String>>,
113    /// The name of the run.
114    #[serde(default)]
115    pub name: Option<String>,
116    /// The model of the run, if applicable.
117    #[serde(default)]
118    pub model: Option<String>,
119}
120
121crate::impl_from_str!(EvalRun);
122
123/// Lists the runs of an eval.
124#[derive(Debug, Clone, Default)]
125pub struct ListEvalRunsRequest<'a> {
126    /// The ID of the eval whose runs to list, e.g. `eval_...`.
127    pub eval_id: &'a str,
128    /// The standard pagination parameters (`before`, `after`,
129    /// `limit`, `order`).
130    pub pagination: PaginationQuery<'a>,
131    /// Additional query parameters appended verbatim to the URL.
132    pub extra_query: Option<std::collections::HashMap<String, String>>,
133}
134
135impl Get for ListEvalRunsRequest<'_> {
136    /// Builds the URL for the request.
137    ///
138    /// `base_url` should be like <https://api.openai.com/v1>
139    fn build_url(&self, base_url: &str) -> Result<String, OapiError> {
140        let mut url = Url::parse(base_url.trim_end_matches('/')).map_err(OapiError::UrlError)?;
141        url.path_segments_mut()
142            .map_err(|_| OapiError::UrlCannotBeBase(base_url.to_string()))?
143            .push("evals")
144            .push(self.eval_id)
145            .push("runs");
146
147        let mut touched = false;
148        {
149            let mut pairs = url.query_pairs_mut();
150            if self.pagination.any_set() {
151                self.pagination.append_to(&mut pairs);
152                touched = true;
153            }
154            if let Some(extra_query) = &self.extra_query {
155                for (key, value) in extra_query {
156                    pairs.append_pair(key, value);
157                }
158                touched = true;
159            }
160        }
161        if !touched {
162            url.set_query(None);
163        }
164
165        Ok(url.to_string())
166    }
167}
168
169impl GetNoStream for ListEvalRunsRequest<'_> {
170    type Response = ListEvalRunsResponse;
171}
172
173/// The response of listing an eval's runs.
174#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
175pub struct ListEvalRunsResponse {
176    /// The runs on this page.
177    #[serde(default)]
178    pub data: Vec<EvalRun>,
179    /// Whether more runs exist after this page.
180    #[serde(default)]
181    pub has_more: Option<bool>,
182    /// The ID of the first run on the page, for cursor pagination.
183    #[serde(default)]
184    pub first_id: Option<String>,
185    /// The ID of the last run on the page, for cursor pagination.
186    #[serde(default)]
187    pub last_id: Option<String>,
188    /// The object type (`list`), if the provider sends it.
189    #[serde(default)]
190    pub object: Option<String>,
191}
192
193crate::impl_from_str!(ListEvalRunsResponse);
openai_interface/evals/runs/mod.rs

openai_interface/evals/runs/
mod.rs
