openai-interface 0.14.0

//! Manage the runs of an eval via `/evals/{eval_id}/runs`.
//!
//! > ![warn] This module is untested!
//! > If you encounter any issues, please report them on the repository.
//!
//! A run applies an eval's testing criteria to a data source (e.g. a
//! set of completions) and reports per-item scores.
//! Submodules: [`create`], [`retrieve`], [`cancel`], [`delete`],
//! [`output_items`]; the list endpoint lives directly in this module.

pub mod cancel;
pub mod create;
pub mod delete;
pub mod output_items;
pub mod retrieve;

use url::Url;

use crate::{
    errors::OapiError,
    pagination::PaginationQuery,
    rest::get::{Get, GetNoStream},
};

/// The lifecycle status of an eval run.
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum EvalRunStatus {
    /// The run is queued.
    Queued,
    /// The run is executing.
    InProgress,
    /// The run failed.
    Failed,
    /// The run was cancelled.
    Canceled,
    /// The run finished successfully.
    Completed,
}

/// The per-testing-criteria result counts of an eval run.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct EvalRunCounts {
    /// The number of items that passed all criteria.
    #[serde(default)]
    pub passed: u64,
    /// The number of items that failed at least one criterion.
    #[serde(default)]
    pub failed: u64,
    /// The number of items with errors.
    #[serde(default)]
    pub errored: u64,
    /// The total number of items.
    #[serde(default)]
    pub total: u64,
}

/// The per-criterion pass/fail counts of an eval run.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct EvalRunPerTestingCriteriaResult {
    /// The ID of the testing criterion.
    #[serde(default)]
    pub testing_criteria: Option<String>,
    /// The number of items that passed this criterion.
    #[serde(default)]
    pub passed: u64,
    /// The number of items that failed this criterion.
    #[serde(default)]
    pub failed: u64,
}

/// The error of a failed eval run.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct EvalRunError {
    /// An error code identifying the failure mode.
    #[serde(default)]
    pub code: Option<String>,
    /// A human-readable error message.
    #[serde(default)]
    pub message: Option<String>,
}

/// An eval run object.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct EvalRun {
    /// The run ID, e.g. `evalrun_...`.
    pub id: String,
    /// The object type, always `eval.run`.
    #[serde(default)]
    pub object: Option<String>,
    /// Unix timestamp (seconds) of when the run was created.
    #[serde(default)]
    pub created_at: Option<u64>,
    /// The ID of the eval this run belongs to.
    pub eval_id: String,
    /// The current lifecycle status.
    pub status: EvalRunStatus,
    /// The data source of the run, as raw JSON.
    #[serde(default)]
    pub data_source: Option<serde_json::Value>,
    /// The per-criterion results of the run.
    #[serde(default)]
    pub result_counts: Option<EvalRunCounts>,
    /// The per-criterion pass/fail breakdown.
    #[serde(default)]
    pub per_testing_criteria_results: Option<Vec<EvalRunPerTestingCriteriaResult>>,
    /// The error of a failed run.
    #[serde(default)]
    pub error: Option<EvalRunError>,
    /// Arbitrary key-value metadata attached to the run.
    #[serde(default)]
    pub metadata: Option<std::collections::HashMap<String, String>>,
    /// The name of the run.
    #[serde(default)]
    pub name: Option<String>,
    /// The model of the run, if applicable.
    #[serde(default)]
    pub model: Option<String>,
}

crate::impl_from_str!(EvalRun);

/// Lists the runs of an eval.
#[derive(Debug, Clone, Default)]
pub struct ListEvalRunsRequest<'a> {
    /// The ID of the eval whose runs to list, e.g. `eval_...`.
    pub eval_id: &'a str,
    /// The standard pagination parameters (`before`, `after`,
    /// `limit`, `order`).
    pub pagination: PaginationQuery<'a>,
    /// Additional query parameters appended verbatim to the URL.
    pub extra_query: Option<std::collections::HashMap<String, String>>,
}

impl Get for ListEvalRunsRequest<'_> {
    /// Builds the URL for the request.
    ///
    /// `base_url` should be like <https://api.openai.com/v1>
    fn build_url(&self, base_url: &str) -> Result<String, OapiError> {
        let mut url = Url::parse(base_url.trim_end_matches('/')).map_err(OapiError::UrlError)?;
        url.path_segments_mut()
            .map_err(|_| OapiError::UrlCannotBeBase(base_url.to_string()))?
            .push("evals")
            .push(self.eval_id)
            .push("runs");

        let mut touched = false;
        {
            let mut pairs = url.query_pairs_mut();
            if self.pagination.any_set() {
                self.pagination.append_to(&mut pairs);
                touched = true;
            }
            if let Some(extra_query) = &self.extra_query {
                for (key, value) in extra_query {
                    pairs.append_pair(key, value);
                }
                touched = true;
            }
        }
        if !touched {
            url.set_query(None);
        }

        Ok(url.to_string())
    }
}

impl GetNoStream for ListEvalRunsRequest<'_> {
    type Response = ListEvalRunsResponse;
}

/// The response of listing an eval's runs.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct ListEvalRunsResponse {
    /// The runs on this page.
    #[serde(default)]
    pub data: Vec<EvalRun>,
    /// Whether more runs exist after this page.
    #[serde(default)]
    pub has_more: Option<bool>,
    /// The ID of the first run on the page, for cursor pagination.
    #[serde(default)]
    pub first_id: Option<String>,
    /// The ID of the last run on the page, for cursor pagination.
    #[serde(default)]
    pub last_id: Option<String>,
    /// The object type (`list`), if the provider sends it.
    #[serde(default)]
    pub object: Option<String>,
}

crate::impl_from_str!(ListEvalRunsResponse);