async_openai/types/evals/
eval.rs

1use derive_builder::Builder;
2use serde::{Deserialize, Serialize};
3
4use crate::error::OpenAIError;
5use crate::types::chat::{ChatCompletionTool, ImageDetail, InputAudio, ResponseFormat};
6use crate::types::graders::{
7    GraderLabelModel, GraderPython, GraderScoreModel, GraderStringCheck, GraderTextSimilarity,
8};
9use crate::types::responses::{ResponseTextParam, Tool};
10use crate::types::Metadata;
11
12// Re-export commonly used types
13pub use crate::types::responses::{EasyInputMessage, InputTextContent, ReasoningEffort};
14
15/// An Eval object with a data source config and testing criteria.
16/// An Eval represents a task to be done for your LLM integration.
17/// Like:
18/// - Improve the quality of my chatbot
19/// - See how well my chatbot handles customer support
20/// - Check if o4-mini is better at my usecase than gpt-4o
21#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
22pub struct Eval {
23    /// The object type, which is always "eval".
24    pub object: String,
25    /// Unique identifier for the evaluation.
26    pub id: String,
27    /// The name of the evaluation.
28    pub name: String,
29    /// Configuration of data sources used in runs of the evaluation.
30    pub data_source_config: EvalDataSourceConfig,
31    /// A list of testing criteria.
32    pub testing_criteria: Vec<EvalTestingCriterion>,
33    /// The Unix timestamp (in seconds) for when the eval was created.
34    pub created_at: u64,
35    pub metadata: Metadata,
36}
37
38/// Configuration of data sources used in runs of the evaluation.
39#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
40#[serde(tag = "type", rename_all = "snake_case")]
41pub enum EvalDataSourceConfig {
42    /// Custom data source config.
43    Custom(EvalCustomDataSourceConfig),
44    /// Logs data source config.
45    Logs(EvalLogsDataSourceConfig),
46    /// Stored completions data source config (deprecated).
47    StoredCompletions(EvalStoredCompletionsDataSourceConfig),
48}
49
50/// Custom data source config.
51#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
52pub struct EvalCustomDataSourceConfig {
53    /// The type of data source. Always "custom".
54    #[serde(rename = "type")]
55    pub r#type: String,
56    /// The json schema for the run data source items.
57    pub schema: serde_json::Value,
58}
59
60/// Logs data source config.
61#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
62pub struct EvalLogsDataSourceConfig {
63    /// The type of data source. Always "logs".
64    #[serde(rename = "type")]
65    pub r#type: String,
66    /// Metadata filters for the logs data source.
67    #[serde(skip_serializing_if = "Option::is_none")]
68    pub metadata: Option<Metadata>,
69    /// The json schema for the run data source items.
70    pub schema: serde_json::Value,
71}
72
73/// Stored completions data source config (deprecated).
74#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
75pub struct EvalStoredCompletionsDataSourceConfig {
76    /// The type of data source. Always "stored_completions".
77    #[serde(rename = "type")]
78    pub r#type: String,
79    /// Metadata filters for the stored completions data source.
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub metadata: Option<Metadata>,
82    /// The json schema for the run data source items.
83    pub schema: serde_json::Value,
84}
85
86/// A list of testing criteria.
87#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
88#[serde(tag = "type", rename_all = "snake_case")]
89pub enum EvalTestingCriterion {
90    /// Label model grader.
91    LabelModel(EvalGraderLabelModel),
92    /// String check grader.
93    StringCheck(EvalGraderStringCheck),
94    /// Text similarity grader.
95    TextSimilarity(EvalGraderTextSimilarity),
96    /// Python grader.
97    Python(EvalGraderPython),
98    /// Score model grader.
99    ScoreModel(EvalGraderScoreModel),
100}
101
102/// Label model grader.
103#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
104#[serde(transparent)]
105pub struct EvalGraderLabelModel(pub GraderLabelModel);
106
107/// String check grader.
108#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
109#[serde(transparent)]
110pub struct EvalGraderStringCheck(pub GraderStringCheck);
111
112/// Text similarity grader.
113#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
114pub struct EvalGraderTextSimilarity {
115    #[serde(flatten)]
116    pub grader: GraderTextSimilarity,
117    pub pass_threshold: f64,
118}
119
120/// Text similarity metric.
121#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq)]
122#[serde(rename_all = "snake_case")]
123pub enum TextSimilarityMetric {
124    /// Cosine similarity.
125    Cosine,
126    /// Fuzzy match.
127    FuzzyMatch,
128    /// BLEU score.
129    Bleu,
130    /// GLEU score.
131    Gleu,
132    /// METEOR score.
133    Meteor,
134    /// ROUGE-1.
135    Rouge1,
136    /// ROUGE-2.
137    Rouge2,
138    /// ROUGE-3.
139    Rouge3,
140    /// ROUGE-4.
141    Rouge4,
142    /// ROUGE-5.
143    Rouge5,
144    /// ROUGE-L.
145    RougeL,
146}
147
148/// Python grader.
149/// also in openapi spec: GraderPython
150#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
151pub struct EvalGraderPython {
152    #[serde(flatten)]
153    pub grader: GraderPython,
154    pub pass_threshold: Option<f64>,
155}
156
157#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
158pub struct SamplingParams {
159    /// A seed value to initialize the randomness, during sampling.
160    #[serde(skip_serializing_if = "Option::is_none")]
161    pub seed: Option<i32>,
162    /// An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
163    #[serde(skip_serializing_if = "Option::is_none")]
164    pub top_p: Option<f64>,
165    /// A higher temperature increases randomness in the outputs.
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub temperature: Option<f64>,
168    /// The maximum number of tokens the grader model may generate in its response.
169    #[serde(skip_serializing_if = "Option::is_none")]
170    pub max_completion_tokens: Option<i32>,
171    /// Optional reasoning effort parameter.
172    #[serde(skip_serializing_if = "Option::is_none")]
173    pub reasoning_effort: Option<ReasoningEffort>,
174}
175
176/// Score model grader.
177/// also in openapi spec: GraderScoreModel
178#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
179pub struct EvalGraderScoreModel {
180    #[serde(flatten)]
181    pub grader: GraderScoreModel,
182    /// The threshold for the score.
183    pub pass_threshold: Option<f64>,
184}
185
186#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
187pub struct EvalItem {
188    /// The role of the message input. One of `user`, `assistant`, `system`, or
189    /// `developer`.
190    pub role: EvalItemRole,
191    /// Inputs to the model - can contain template strings. Supports text, output text, input images, and
192    /// input audio, either as a single item or an array of items.
193    pub content: EvalItemContent,
194}
195
196/// The role of the message input.
197#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq)]
198#[serde(rename_all = "lowercase")]
199pub enum EvalItemRole {
200    /// User role.
201    User,
202    /// Assistant role.
203    Assistant,
204    /// System role.
205    System,
206    /// Developer role.
207    Developer,
208}
209
210/// Output text from the model.
211#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
212pub struct EvalItemContentOutputText {
213    /// The text output from the model.
214    pub text: String,
215}
216
217/// Input image block used within EvalItem content arrays.
218#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
219pub struct EvalItemInputImage {
220    /// The URL of the image input.
221    pub image_url: String,
222    /// The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`.
223    /// Defaults to `auto`.
224    #[serde(skip_serializing_if = "Option::is_none")]
225    pub detail: Option<ImageDetail>,
226}
227
228/// Inputs to the model - can contain template strings.
229/// Supports text, output text, input images, and input audio, either as a single item or an array of items.
230#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
231#[serde(untagged)]
232pub enum EvalItemContent {
233    /// An array of Input text, Output text, Input image, and Input audio
234    Array(Vec<EvalItemContentItem>),
235    /// A single content item
236    Single(EvalItemContentItem),
237}
238
239/// A single content item: input text, output text, input image, or input audio.
240#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
241#[serde(tag = "type", rename_all = "snake_case")]
242pub enum EvalItemContentItem {
243    /// An input text content object with type field.
244    InputText(InputTextContent),
245    /// An output text from the model.
246    OutputText(EvalItemContentOutputText),
247    /// An image input to the model.
248    InputImage(EvalItemInputImage),
249    /// An audio input to the model.
250    InputAudio(InputAudio),
251    /// A text input to the model (plain string).
252    #[serde(untagged)]
253    Text(String),
254}
255
256/// List of evals.
257#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
258pub struct EvalList {
259    /// The object type, which is always "list".
260    pub object: String,
261    /// An array of eval objects.
262    pub data: Vec<Eval>,
263    /// The identifier of the first eval in the data array.
264    pub first_id: Option<String>,
265    /// The identifier of the last eval in the data array.
266    pub last_id: Option<String>,
267    /// Indicates whether there are more evals available.
268    pub has_more: bool,
269}
270
271#[derive(Debug, Serialize, Clone, Builder, PartialEq, Default)]
272#[builder(name = "CreateEvalRequestArgs")]
273#[builder(pattern = "mutable")]
274#[builder(setter(into, strip_option), default)]
275#[builder(derive(Debug))]
276#[builder(build_fn(error = "OpenAIError"))]
277pub struct CreateEvalRequest {
278    /// The name of the evaluation.
279    pub name: Option<String>,
280    ///The configuration for the data source used for the evaluation runs.
281    /// Dictates the schema of the data used in the evaluation.
282    pub data_source_config: CreateEvalDataSourceConfig,
283    /// A list of graders for all eval runs in this group. Graders can reference variables in the data
284    /// source using double curly braces notation, like `{{item.variable_name}}`. To reference the model's
285    /// output, use the `sample` namespace (ie, `{{sample.output_text}}`).
286    pub testing_criteria: Vec<CreateEvalTestingCriterion>,
287    #[serde(skip_serializing_if = "Option::is_none")]
288    pub metadata: Option<Metadata>,
289}
290
291#[derive(Debug, Serialize, Clone, PartialEq)]
292#[serde(tag = "type", rename_all = "snake_case")]
293pub enum CreateEvalDataSourceConfig {
294    /// A CustomDataSourceConfig object that defines the schema for the data source used for the evaluation
295    /// runs. This schema is used to define the shape of the data that will be:
296    /// - Used to define your testing criteria and
297    /// - What data is required when creating a run
298    Custom(CreateEvalCustomDataSourceConfig),
299    /// A data source config which specifies the metadata property of your logs query.
300    /// This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc.
301    Logs(CreateEvalLogsDataSourceConfig),
302}
303
304impl Default for CreateEvalDataSourceConfig {
305    fn default() -> Self {
306        Self::Custom(CreateEvalCustomDataSourceConfig::default())
307    }
308}
309
310#[derive(Debug, Serialize, Clone, PartialEq, Builder, Default)]
311#[builder(name = "CreateEvalCustomDataSourceConfigArgs")]
312#[builder(pattern = "mutable")]
313#[builder(setter(into, strip_option), default)]
314#[builder(derive(Debug))]
315#[builder(build_fn(error = "OpenAIError"))]
316pub struct CreateEvalCustomDataSourceConfig {
317    /// The json schema for each row in the data source.
318    pub item_schema: serde_json::Value,
319    /// Whether the eval should expect you to populate the sample namespace (ie, by generating responses
320    /// off of your data source).
321    #[serde(skip_serializing_if = "Option::is_none")]
322    pub include_sample_schema: Option<bool>,
323}
324
325/// Logs data source config for creating an eval.
326#[derive(Debug, Serialize, Clone, PartialEq, Builder, Default)]
327#[builder(name = "CreateEvalLogsDataSourceConfigArgs")]
328#[builder(pattern = "mutable")]
329#[builder(setter(into, strip_option), default)]
330#[builder(derive(Debug))]
331#[builder(build_fn(error = "OpenAIError"))]
332pub struct CreateEvalLogsDataSourceConfig {
333    /// Metadata filters for the logs data source.
334    #[serde(skip_serializing_if = "Option::is_none")]
335    pub metadata: Option<Metadata>,
336}
337
338#[derive(Debug, Serialize, Clone, PartialEq)]
339#[serde(tag = "type", rename_all = "snake_case")]
340pub enum CreateEvalTestingCriterion {
341    /// A LabelModelGrader object which uses a model to assign labels to each item
342    /// in the evaluation.
343    LabelModel(CreateEvalLabelModelGrader),
344    /// A StringCheckGrader object that performs a string comparison between input and reference using a
345    /// specified operation.
346    StringCheck(EvalGraderStringCheck),
347    /// Text similarity grader.
348    TextSimilarity(EvalGraderTextSimilarity),
349    /// Python grader.
350    Python(EvalGraderPython),
351    /// Score model grader.
352    ScoreModel(EvalGraderScoreModel),
353}
354
355/// Label model grader for creating an eval.
356#[derive(Debug, Serialize, Clone, PartialEq, Builder, Default)]
357#[builder(name = "CreateEvalLabelModelGraderArgs")]
358#[builder(pattern = "mutable")]
359#[builder(setter(into, strip_option), default)]
360#[builder(derive(Debug))]
361#[builder(build_fn(error = "OpenAIError"))]
362pub struct CreateEvalLabelModelGrader {
363    /// The name of the grader.
364    pub name: String,
365    /// The model to use for the evaluation. Must support structured outputs.
366    pub model: String,
367    /// A list of chat messages forming the prompt or context. May include variable references to the
368    /// `item` namespace, ie `{{item.name}}`.
369    pub input: Vec<CreateEvalItem>,
370    /// The labels to classify to each item in the evaluation.
371    pub labels: Vec<String>,
372    /// The labels that indicate a passing result. Must be a subset of labels.
373    pub passing_labels: Vec<String>,
374}
375
376#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)]
377pub struct SimpleInputMessage {
378    /// The role of the message.
379    pub role: String,
380    /// The content of the message.
381    pub content: String,
382}
383
384/// A chat message that makes up the prompt or context.
385#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
386#[serde(tag = "type", rename_all = "snake_case")]
387pub enum CreateEvalItem {
388    /// A message input to the model with a role indicating instruction following
389    /// hierarchy. Instructions given with the `developer` or `system` role take
390    /// precedence over instructions given with the `user` role. Messages with the
391    /// `assistant` role are presumed to have been generated by the model in previous
392    /// interactions.
393    Message(EvalItem),
394
395    /// SimpleInputMessage
396    #[serde(untagged)]
397    Simple(SimpleInputMessage),
398}
399
400/// Request to update an eval.
401#[derive(Debug, Serialize, Clone, Builder, PartialEq, Default)]
402#[builder(name = "UpdateEvalRequestArgs")]
403#[builder(pattern = "mutable")]
404#[builder(setter(into, strip_option), default)]
405#[builder(derive(Debug))]
406#[builder(build_fn(error = "OpenAIError"))]
407pub struct UpdateEvalRequest {
408    /// Rename the evaluation.
409    #[serde(skip_serializing_if = "Option::is_none")]
410    pub name: Option<String>,
411    /// Metadata attached to the eval.
412    #[serde(skip_serializing_if = "Option::is_none")]
413    pub metadata: Option<Metadata>,
414}
415
416/// Response from deleting an eval.
417#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
418pub struct DeleteEvalResponse {
419    /// The object type, which is always "eval.deleted".
420    pub object: String,
421    /// Whether the eval was deleted.
422    pub deleted: bool,
423    /// The ID of the deleted eval.
424    pub eval_id: String,
425}
426
427// EvalRun types
428
429/// A schema representing an evaluation run.
430#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
431pub struct EvalRun {
432    /// The object type, which is always "eval.run".
433    pub object: String,
434    /// Unique identifier for the evaluation run.
435    pub id: String,
436    /// The identifier of the associated evaluation.
437    pub eval_id: String,
438    /// The status of the evaluation run.
439    pub status: EvalRunStatus,
440    /// The model that is evaluated, if applicable.
441    pub model: Option<String>,
442    /// The name of the evaluation run.
443    pub name: Option<String>,
444    /// Unix timestamp (in seconds) when the evaluation run was created.
445    pub created_at: u64,
446    /// The URL to the rendered evaluation run report on the UI dashboard.
447    pub report_url: String,
448    /// Counters summarizing the outcomes of the evaluation run.
449    pub result_counts: EvalRunResultCounts,
450    /// Usage statistics for each model during the evaluation run.
451    pub per_model_usage: Option<Vec<EvalRunModelUsage>>,
452    /// Results per testing criteria applied during the evaluation run.
453    pub per_testing_criteria_results: Option<Vec<EvalRunTestingCriteriaResult>>,
454    /// Information about the run's data source.
455    pub data_source: EvalRunDataSource,
456    /// Metadata attached to the run.
457    pub metadata: Metadata,
458    /// Error information, if any.
459    pub error: Option<EvalApiError>,
460}
461
462/// Status of an evaluation run.
463#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq)]
464#[serde(rename_all = "snake_case")]
465pub enum EvalRunStatus {
466    /// Queued.
467    Queued,
468    /// In progress.
469    InProgress,
470    /// Completed.
471    Completed,
472    /// Failed.
473    Failed,
474    /// Canceled.
475    Canceled,
476}
477
478/// Counters summarizing the outcomes of the evaluation run.
479#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
480pub struct EvalRunResultCounts {
481    /// Total number of executed output items.
482    pub total: u32,
483    /// Number of output items that resulted in an error.
484    pub errored: u32,
485    /// Number of output items that failed to pass the evaluation.
486    pub failed: u32,
487    /// Number of output items that passed the evaluation.
488    pub passed: u32,
489}
490
491/// Usage statistics for each model during the evaluation run.
492#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
493pub struct EvalRunModelUsage {
494    /// The name of the model.
495    pub model_name: String,
496    /// The number of invocations.
497    pub invocation_count: u32,
498    /// The number of prompt tokens used.
499    pub prompt_tokens: u32,
500    /// The number of completion tokens generated.
501    pub completion_tokens: u32,
502    /// The total number of tokens used.
503    pub total_tokens: u32,
504    /// The number of tokens retrieved from cache.
505    pub cached_tokens: u32,
506}
507
508/// Results per testing criteria applied during the evaluation run.
509#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
510pub struct EvalRunTestingCriteriaResult {
511    /// A description of the testing criteria.
512    pub testing_criteria: String,
513    /// Number of tests passed for this criteria.
514    pub passed: u32,
515    /// Number of tests failed for this criteria.
516    pub failed: u32,
517}
518
519/// Information about the run's data source.
520#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
521#[serde(tag = "type", rename_all = "snake_case")]
522pub enum EvalRunDataSource {
523    /// A JsonlRunDataSource object with that specifies a JSONL file that matches the eval
524    Jsonl(CreateEvalJsonlRunDataSource),
525    /// A CompletionsRunDataSource object describing a model sampling configuration.
526    Completions(CreateEvalCompletionsRunDataSource),
527    /// A ResponsesRunDataSource object describing a model sampling configuration.
528    Responses(CreateEvalResponsesRunDataSource),
529}
530
531/// JSONL run data source.
532#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
533pub struct CreateEvalJsonlRunDataSource {
534    /// Determines what populates the `item` namespace in the data source.
535    pub source: EvalJsonlSource,
536}
537
538/// JSONL source.
539#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
540#[serde(tag = "type", rename_all = "snake_case")]
541pub enum EvalJsonlSource {
542    /// File content source.
543    FileContent(EvalJsonlFileContentSource),
544    /// File ID source.
545    FileId(EvalJsonlFileIdSource),
546}
547
548/// JSONL file content source.
549#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
550pub struct EvalJsonlFileContentSource {
551    /// The content of the jsonl file.
552    pub content: Vec<EvalJsonlContentItem>,
553}
554
555/// JSONL file ID source.
556#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
557pub struct EvalJsonlFileIdSource {
558    /// The identifier of the file.
559    pub id: String,
560}
561
562/// JSONL content item.
563#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
564pub struct EvalJsonlContentItem {
565    /// The item data.
566    pub item: serde_json::Value,
567    /// The sample data, if any.
568    #[serde(skip_serializing_if = "Option::is_none")]
569    pub sample: Option<serde_json::Value>,
570}
571
572/// Completions run data source.
573#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
574pub struct CreateEvalCompletionsRunDataSource {
575    /// Used when sampling from a model. Dictates the structure of the messages passed into the model. Can
576    /// either be a reference to a prebuilt trajectory (ie, `item.input_trajectory`), or a template with
577    /// variable references to the `item` namespace.
578    pub input_messages: EvalInputMessages,
579    /// The sampling parameters for the model.
580    #[serde(skip_serializing_if = "Option::is_none")]
581    pub sampling_params: Option<EvalSamplingParams>,
582    /// The name of the model to use for generating completions (e.g. "o3-mini").
583    pub model: String,
584    /// Determines what populates the `item` namespace in this run's data source.
585    pub source: EvalCompletionsSource,
586}
587
588#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
589pub struct TemplateInputMessages {
590    /// A list of chat messages forming the prompt or context. May include variable references to
591    /// the `item` namespace, ie {{item.name}}.
592    pub template: Vec<CreateEvalItem>,
593}
594
595#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
596pub struct ItemReference {
597    /// A reference to a variable in the `item` namespace. Ie, "item.input_trajectory"
598    pub item_reference: String,
599}
600
601/// Input messages for completions.
602#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
603#[serde(tag = "type", rename_all = "snake_case")]
604pub enum EvalInputMessages {
605    /// Template input messages.
606    Template(TemplateInputMessages),
607    /// Item reference input messages.
608    ItemReference(ItemReference),
609}
610
611/// Sampling parameters for the model.
612#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
613pub struct EvalSamplingParams {
614    /// A seed value to initialize the randomness, during sampling.
615    #[serde(skip_serializing_if = "Option::is_none")]
616    pub seed: Option<i32>,
617    /// An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
618    #[serde(skip_serializing_if = "Option::is_none")]
619    pub top_p: Option<f64>,
620    /// A higher temperature increases randomness in the outputs.
621    #[serde(skip_serializing_if = "Option::is_none")]
622    pub temperature: Option<f64>,
623    /// The maximum number of tokens in the generated output.
624    #[serde(skip_serializing_if = "Option::is_none")]
625    pub max_completion_tokens: Option<i32>,
626    /// Optional reasoning effort parameter.
627    #[serde(skip_serializing_if = "Option::is_none")]
628    pub reasoning_effort: Option<ReasoningEffort>,
629    /// An object specifying the format that the model must output.
630    #[serde(skip_serializing_if = "Option::is_none")]
631    pub response_format: Option<ResponseFormat>,
632    /// A list of tools the model may call.
633    #[serde(skip_serializing_if = "Option::is_none")]
634    pub tools: Option<Vec<ChatCompletionTool>>,
635}
636
637#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
638pub struct EvalResponsesSamplingParams {
639    /// A seed value to initialize the randomness, during sampling.
640    #[serde(skip_serializing_if = "Option::is_none")]
641    pub seed: Option<i32>,
642    /// An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
643    #[serde(skip_serializing_if = "Option::is_none")]
644    pub top_p: Option<f64>,
645    /// A higher temperature increases randomness in the outputs.
646    #[serde(skip_serializing_if = "Option::is_none")]
647    pub temperature: Option<f64>,
648    /// The maximum number of tokens in the generated output.
649    #[serde(skip_serializing_if = "Option::is_none")]
650    pub max_completion_tokens: Option<u32>,
651    /// Optional reasoning effort parameter.
652    #[serde(skip_serializing_if = "Option::is_none")]
653    pub reasoning_effort: Option<ReasoningEffort>,
654    /// An object specifying the format that the model must output.
655    #[serde(skip_serializing_if = "Option::is_none")]
656    pub response_format: Option<ResponseFormat>,
657    /// A list of tools the model may call.
658    #[serde(skip_serializing_if = "Option::is_none")]
659    pub tools: Option<Vec<Tool>>,
660    /// Configuration options for a text response from the model. Can be plain
661    /// text or structured JSON data. Learn more:
662    /// - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
663    /// - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
664    #[serde(skip_serializing_if = "Option::is_none")]
665    pub text: Option<ResponseTextParam>,
666}
667
668/// Completions source.
669#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
670#[serde(tag = "type", rename_all = "snake_case")]
671pub enum EvalCompletionsSource {
672    /// File content source.
673    FileContent(EvalJsonlFileContentSource),
674    /// File ID source.
675    FileId(EvalJsonlFileIdSource),
676    /// Stored completions source.
677    StoredCompletions(EvalStoredCompletionsSource),
678}
679
680/// Stored completions source.
681#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
682pub struct EvalStoredCompletionsSource {
683    /// Metadata filters for the stored completions.
684    #[serde(skip_serializing_if = "Option::is_none")]
685    pub metadata: Option<Metadata>,
686    /// An optional model to filter by.
687    #[serde(skip_serializing_if = "Option::is_none")]
688    pub model: Option<String>,
689    /// An optional Unix timestamp to filter items created after this time.
690    #[serde(skip_serializing_if = "Option::is_none")]
691    pub created_after: Option<u64>,
692    /// An optional Unix timestamp to filter items created before this time.
693    #[serde(skip_serializing_if = "Option::is_none")]
694    pub created_before: Option<u64>,
695    /// An optional maximum number of items to return.
696    #[serde(skip_serializing_if = "Option::is_none")]
697    pub limit: Option<i32>,
698}
699
700/// Responses run data source.
701#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
702pub struct CreateEvalResponsesRunDataSource {
703    /// Used when sampling from a model. Dictates the structure of the messages passed into the model.
704    #[serde(skip_serializing_if = "Option::is_none")]
705    pub input_messages: Option<EvalInputMessages>,
706    /// The sampling parameters for the model.
707    #[serde(skip_serializing_if = "Option::is_none")]
708    pub sampling_params: Option<EvalResponsesSamplingParams>,
709    #[serde(skip_serializing_if = "Option::is_none")]
710    pub model: Option<String>,
711    /// Determines what populates the `item` namespace in this run's data source.
712    pub source: EvalResponsesRunSource,
713}
714
715/// Responses source.
716#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
717#[serde(tag = "type", rename_all = "snake_case")]
718pub enum EvalResponsesRunSource {
719    /// File content source.
720    FileContent(EvalJsonlFileContentSource),
721    /// File ID source.
722    FileId(EvalJsonlFileIdSource),
723    /// A EvalResponsesSource object describing a run data source configuration.
724    Responses(EvalResponsesSource),
725}
726
727/// A EvalResponsesSource object describing a run data source configuration.
728#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
729pub struct EvalResponsesSource {
730    /// Metadata filter for the responses. This is a query parameter used to select responses.
731    #[serde(skip_serializing_if = "Option::is_none")]
732    pub metadata: Option<serde_json::Value>,
733    /// The name of the model to find responses for. This is a query parameter used to select responses.
734    #[serde(skip_serializing_if = "Option::is_none")]
735    pub model: Option<String>,
736    /// Optional string to search the 'instructions' field. This is a query parameter used to select responses.
737    #[serde(skip_serializing_if = "Option::is_none")]
738    pub instructions_search: Option<String>,
739    /// Only include items created after this timestamp (inclusive). This is a query parameter used to select responses.
740    #[serde(skip_serializing_if = "Option::is_none")]
741    pub created_after: Option<u64>,
742    /// Only include items created before this timestamp (inclusive). This is a query parameter used to select responses.
743    #[serde(skip_serializing_if = "Option::is_none")]
744    pub created_before: Option<u64>,
745    /// Optional reasoning effort parameter. This is a query parameter used to select responses.
746    #[serde(skip_serializing_if = "Option::is_none")]
747    pub reasoning_effort: Option<ReasoningEffort>,
748    /// Sampling temperature. This is a query parameter used to select responses.
749    #[serde(skip_serializing_if = "Option::is_none")]
750    pub temperature: Option<f64>,
751    /// Nucleus sampling parameter. This is a query parameter used to select responses.
752    #[serde(skip_serializing_if = "Option::is_none")]
753    pub top_p: Option<f64>,
754    /// List of user identifiers. This is a query parameter used to select responses.
755    #[serde(skip_serializing_if = "Option::is_none")]
756    pub users: Option<Vec<String>>,
757    /// List of tool names. This is a query parameter used to select responses.
758    #[serde(skip_serializing_if = "Option::is_none")]
759    pub tools: Option<Vec<String>>,
760}
761
762/// List of eval runs.
763#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
764pub struct EvalRunList {
765    /// The object type, which is always "list".
766    pub object: String,
767    /// An array of eval run objects.
768    pub data: Vec<EvalRun>,
769    /// The identifier of the first eval run in the data array.
770    pub first_id: Option<String>,
771    /// The identifier of the last eval run in the data array.
772    pub last_id: Option<String>,
773    /// Indicates whether there are more evals available.
774    pub has_more: bool,
775}
776
777/// Request to create an eval run.
778#[derive(Debug, Serialize, Clone, Builder, PartialEq, Default)]
779#[builder(name = "CreateEvalRunRequestArgs")]
780#[builder(pattern = "mutable")]
781#[builder(setter(into, strip_option), default)]
782#[builder(derive(Debug))]
783#[builder(build_fn(error = "OpenAIError"))]
784pub struct CreateEvalRunRequest {
785    /// The name of the run.
786    #[serde(skip_serializing_if = "Option::is_none")]
787    pub name: Option<String>,
788    /// Details about the run's data source.
789    pub data_source: CreateEvalRunDataSource,
790    /// Metadata attached to the run.
791    #[serde(skip_serializing_if = "Option::is_none")]
792    pub metadata: Option<Metadata>,
793}
794
795/// Details about the run's data source.
796#[derive(Debug, Serialize, Clone, PartialEq)]
797#[serde(tag = "type", rename_all = "snake_case")]
798pub enum CreateEvalRunDataSource {
799    /// JSONL data source.
800    Jsonl(CreateEvalJsonlRunDataSource),
801    /// Completions data source.
802    Completions(CreateEvalCompletionsRunDataSource),
803    /// Responses data source.
804    Responses(CreateEvalResponsesRunDataSource),
805}
806
807// Manual Default implementation for Builder compatibility
808impl Default for CreateEvalRunDataSource {
809    fn default() -> Self {
810        todo!()
811    }
812}
813
814/// Response from deleting an eval run.
815#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
816pub struct DeleteEvalRunResponse {
817    /// The object type, which is always "eval.run.deleted".
818    pub object: String,
819    /// Whether the eval run was deleted.
820    pub deleted: bool,
821    /// The ID of the deleted eval run.
822    pub run_id: String,
823}
824
825// EvalRunOutputItem types
826
827/// A schema representing an evaluation run output item.
828#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
829pub struct EvalRunOutputItem {
830    /// The object type, which is always "eval.run.output_item".
831    pub object: String,
832    /// Unique identifier for the evaluation run output item.
833    pub id: String,
834    /// The identifier of the evaluation run associated with this output item.
835    pub run_id: String,
836    /// The identifier of the evaluation group.
837    pub eval_id: String,
838    /// Unix timestamp (in seconds) when the evaluation run was created.
839    pub created_at: u64,
840    /// The status of the evaluation run.
841    pub status: String,
842    /// The identifier for the data source item.
843    pub datasource_item_id: u64,
844    /// Details of the input data source item.
845    pub datasource_item: serde_json::Value,
846    /// A list of grader results for this output item.
847    pub results: Vec<EvalRunOutputItemResult>,
848    /// A sample containing the input and output of the evaluation run.
849    pub sample: EvalRunOutputItemSample,
850}
851
852/// A single grader result for an evaluation run output item.
853#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
854pub struct EvalRunOutputItemResult {
855    /// The name of the grader.
856    pub name: String,
857    /// The numeric score produced by the grader.
858    pub score: f64,
859    /// Whether the grader considered the output a pass.
860    pub passed: bool,
861    /// Optional sample or intermediate data produced by the grader.
862    #[serde(skip_serializing_if = "Option::is_none")]
863    pub sample: Option<serde_json::Value>,
864}
865
866#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
867pub struct SimpleOutputMessage {
868    pub role: String,
869    pub content: String,
870}
871
872/// A sample containing the input and output of the evaluation run.
873#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
874pub struct EvalRunOutputItemSample {
875    /// An array of input messages.
876    pub input: Vec<SimpleInputMessage>,
877    /// An array of output messages.
878    pub output: Vec<SimpleOutputMessage>,
879    /// The reason why the sample generation was finished.
880    pub finish_reason: String,
881    /// The model used for generating the sample.
882    pub model: String,
883    /// Token usage details for the sample.
884    pub usage: EvalRunOutputItemUsage,
885    /// Error information, if any.
886    pub error: Option<EvalApiError>,
887    /// The sampling temperature used.
888    pub temperature: f64,
889    /// The maximum number of tokens allowed for completion.
890    pub max_completion_tokens: i32,
891    /// The top_p value used for sampling.
892    pub top_p: f64,
893    /// The seed used for generating the sample.
894    pub seed: i32,
895}
896
897/// Token usage details for the sample.
898#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
899pub struct EvalRunOutputItemUsage {
900    /// The total number of tokens used.
901    pub total_tokens: i32,
902    /// The number of completion tokens generated.
903    pub completion_tokens: i32,
904    /// The number of prompt tokens used.
905    pub prompt_tokens: i32,
906    /// The number of tokens retrieved from cache.
907    pub cached_tokens: i32,
908}
909
910/// List of eval run output items.
911#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
912pub struct EvalRunOutputItemList {
913    /// The object type, which is always "list".
914    pub object: String,
915    /// An array of eval run output item objects.
916    pub data: Vec<EvalRunOutputItem>,
917    /// The identifier of the first eval run output item in the data array.
918    pub first_id: Option<String>,
919    /// The identifier of the last eval run output item in the data array.
920    pub last_id: Option<String>,
921    /// Indicates whether there are more eval run output items available.
922    pub has_more: bool,
923}
924
925/// An object representing an error response from the Eval API.
926#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
927pub struct EvalApiError {
928    /// The error code.
929    pub code: String,
930    /// The error message.
931    pub message: String,
932}
async_openai/types/evals/eval.rs

async_openai/types/evals/
eval.rs
