outfox_openai/spec/evals/
eval.rs

1use derive_builder::Builder;
2use serde::{Deserialize, Serialize};
3
4use crate::error::OpenAIError;
5use crate::spec::Metadata;
6use crate::spec::chat::{ChatCompletionTool, ImageDetail, InputAudio, ResponseFormat};
7use crate::spec::graders::{
8    GraderLabelModel, GraderPython, GraderScoreModel, GraderStringCheck, GraderTextSimilarity,
9};
10// Re-export commonly used types
11pub use crate::spec::responses::{EasyInputMessage, InputTextContent, ReasoningEffort};
12use crate::spec::responses::{ResponseTextParam, Tool};
13
14/// An Eval object with a data source config and testing criteria.
15/// An Eval represents a task to be done for your LLM integration.
16/// Like:
17/// - Improve the quality of my chatbot
18/// - See how well my chatbot handles customer support
19/// - Check if o4-mini is better at my usecase than gpt-4o
20#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
21pub struct Eval {
22    /// The object type, which is always "eval".
23    pub object: String,
24    /// Unique identifier for the evaluation.
25    pub id: String,
26    /// The name of the evaluation.
27    pub name: String,
28    /// Configuration of data sources used in runs of the evaluation.
29    pub data_source_config: EvalDataSourceConfig,
30    /// A list of testing criteria.
31    pub testing_criteria: Vec<EvalTestingCriterion>,
32    /// The Unix timestamp (in seconds) for when the eval was created.
33    pub created_at: u64,
34    pub metadata: Metadata,
35}
36
37/// Configuration of data sources used in runs of the evaluation.
38#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
39#[serde(tag = "type", rename_all = "snake_case")]
40pub enum EvalDataSourceConfig {
41    /// Custom data source config.
42    Custom(EvalCustomDataSourceConfig),
43    /// Logs data source config.
44    Logs(EvalLogsDataSourceConfig),
45    /// Stored completions data source config (deprecated).
46    #[serde(rename = "stored_completions")]
47    StoredCompletions(EvalStoredCompletionsDataSourceConfig),
48}
49
50/// Custom data source config.
51#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
52pub struct EvalCustomDataSourceConfig {
53    /// The type of data source. Always "custom".
54    #[serde(rename = "type")]
55    pub kind: String,
56    /// The json schema for the run data source items.
57    pub schema: serde_json::Value,
58}
59
60/// Logs data source config.
61#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
62pub struct EvalLogsDataSourceConfig {
63    /// The type of data source. Always "logs".
64    #[serde(rename = "type")]
65    pub kind: String,
66    /// Metadata filters for the logs data source.
67    #[serde(skip_serializing_if = "Option::is_none")]
68    pub metadata: Option<Metadata>,
69    /// The json schema for the run data source items.
70    pub schema: serde_json::Value,
71}
72
73/// Stored completions data source config (deprecated).
74#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
75pub struct EvalStoredCompletionsDataSourceConfig {
76    /// The type of data source. Always "stored_completions".
77    #[serde(rename = "type")]
78    pub kind: String,
79    /// Metadata filters for the stored completions data source.
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub metadata: Option<Metadata>,
82    /// The json schema for the run data source items.
83    pub schema: serde_json::Value,
84}
85
86/// A list of testing criteria.
87#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
88#[serde(tag = "type", rename_all = "snake_case")]
89pub enum EvalTestingCriterion {
90    /// Label model grader.
91    LabelModel(EvalGraderLabelModel),
92    /// String check grader.
93    StringCheck(EvalGraderStringCheck),
94    /// Text similarity grader.
95    TextSimilarity(EvalGraderTextSimilarity),
96    /// Python grader.
97    Python(EvalGraderPython),
98    /// Score model grader.
99    ScoreModel(EvalGraderScoreModel),
100}
101
102/// Label model grader.
103#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
104#[serde(transparent)]
105pub struct EvalGraderLabelModel(pub GraderLabelModel);
106
107/// String check grader.
108#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
109#[serde(transparent)]
110pub struct EvalGraderStringCheck(pub GraderStringCheck);
111
112/// Text similarity grader.
113#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
114pub struct EvalGraderTextSimilarity {
115    #[serde(flatten)]
116    pub grader: GraderTextSimilarity,
117    pub pass_threshold: f64,
118}
119
120/// Text similarity metric.
121#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq)]
122#[serde(rename_all = "snake_case")]
123pub enum TextSimilarityMetric {
124    /// Cosine similarity.
125    Cosine,
126    /// Fuzzy match.
127    FuzzyMatch,
128    /// BLEU score.
129    Bleu,
130    /// GLEU score.
131    Gleu,
132    /// METEOR score.
133    Meteor,
134    /// ROUGE-1.
135    Rouge1,
136    /// ROUGE-2.
137    Rouge2,
138    /// ROUGE-3.
139    Rouge3,
140    /// ROUGE-4.
141    Rouge4,
142    /// ROUGE-5.
143    Rouge5,
144    /// ROUGE-L.
145    RougeL,
146}
147
148/// Python grader.
149/// also in openapi spec: GraderPython
150#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
151pub struct EvalGraderPython {
152    #[serde(flatten)]
153    pub grader: GraderPython,
154    pub pass_threshold: Option<f64>,
155}
156
157#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
158pub struct SamplingParams {
159    /// A seed value to initialize the randomness, during sampling.
160    #[serde(skip_serializing_if = "Option::is_none")]
161    pub seed: Option<i32>,
162    /// An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
163    #[serde(skip_serializing_if = "Option::is_none")]
164    pub top_p: Option<f64>,
165    /// A higher temperature increases randomness in the outputs.
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub temperature: Option<f64>,
168    /// The maximum number of tokens the grader model may generate in its response.
169    #[serde(skip_serializing_if = "Option::is_none")]
170    pub max_completion_tokens: Option<i32>,
171    /// Optional reasoning effort parameter.
172    #[serde(skip_serializing_if = "Option::is_none")]
173    pub reasoning_effort: Option<ReasoningEffort>,
174}
175
176/// Score model grader.
177/// also in openapi spec: GraderScoreModel
178#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
179pub struct EvalGraderScoreModel {
180    #[serde(flatten)]
181    pub grader: GraderScoreModel,
182    /// The threshold for the score.
183    pub pass_threshold: Option<f64>,
184}
185
186#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
187pub struct EvalItem {
188    /// The role of the message input. One of `user`, `assistant`, `system`, or
189    /// `developer`.
190    pub role: EvalItemRole,
191    /// Inputs to the model - can contain template strings. Supports text, output text, input
192    /// images, and input audio, either as a single item or an array of items.
193    pub content: EvalItemContent,
194}
195
196/// The role of the message input.
197#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq)]
198#[serde(rename_all = "lowercase")]
199pub enum EvalItemRole {
200    /// User role.
201    User,
202    /// Assistant role.
203    Assistant,
204    /// System role.
205    System,
206    /// Developer role.
207    Developer,
208}
209
210/// Output text from the model.
211#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
212pub struct EvalItemContentOutputText {
213    /// The text output from the model.
214    pub text: String,
215}
216
217/// Input image block used within EvalItem content arrays.
218#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
219pub struct EvalItemInputImage {
220    /// The URL of the image input.
221    pub image_url: String,
222    /// The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`.
223    /// Defaults to `auto`.
224    #[serde(skip_serializing_if = "Option::is_none")]
225    pub detail: Option<ImageDetail>,
226}
227
228/// Inputs to the model - can contain template strings.
229/// Supports text, output text, input images, and input audio, either as a single item or an array
230/// of items.
231#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
232#[serde(untagged)]
233pub enum EvalItemContent {
234    /// An array of Input text, Output text, Input image, and Input audio
235    Array(Vec<EvalItemContentItem>),
236    /// A single content item
237    Single(EvalItemContentItem),
238}
239
240/// A single content item: input text, output text, input image, or input audio.
241#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
242#[serde(tag = "type", rename_all = "snake_case")]
243pub enum EvalItemContentItem {
244    /// An input text content object with type field.
245    InputText(InputTextContent),
246    /// An output text from the model.
247    OutputText(EvalItemContentOutputText),
248    /// An image input to the model.
249    InputImage(EvalItemInputImage),
250    /// An audio input to the model.
251    InputAudio(InputAudio),
252    /// A text input to the model (plain string).
253    #[serde(untagged)]
254    Text(String),
255}
256
257/// List of evals.
258#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
259pub struct EvalList {
260    /// The object type, which is always "list".
261    pub object: String,
262    /// An array of eval objects.
263    pub data: Vec<Eval>,
264    /// The identifier of the first eval in the data array.
265    pub first_id: Option<String>,
266    /// The identifier of the last eval in the data array.
267    pub last_id: Option<String>,
268    /// Indicates whether there are more evals available.
269    pub has_more: bool,
270}
271
272#[derive(Debug, Serialize, Clone, Builder, PartialEq, Default)]
273#[builder(name = "CreateEvalRequestArgs")]
274#[builder(pattern = "mutable")]
275#[builder(setter(into, strip_option), default)]
276#[builder(derive(Debug))]
277#[builder(build_fn(error = "OpenAIError"))]
278pub struct CreateEvalRequest {
279    /// The name of the evaluation.
280    pub name: Option<String>,
281    /// The configuration for the data source used for the evaluation runs.
282    /// Dictates the schema of the data used in the evaluation.
283    pub data_source_config: CreateEvalDataSourceConfig,
284    /// A list of graders for all eval runs in this group. Graders can reference variables in the
285    /// data source using double curly braces notation, like `{{item.variable_name}}`. To
286    /// reference the model's output, use the `sample` namespace (ie,
287    /// `{{sample.output_text}}`).
288    pub testing_criteria: Vec<CreateEvalTestingCriterion>,
289    #[serde(skip_serializing_if = "Option::is_none")]
290    pub metadata: Option<Metadata>,
291}
292
293#[derive(Debug, Serialize, Clone, PartialEq)]
294#[serde(tag = "type", rename_all = "snake_case")]
295pub enum CreateEvalDataSourceConfig {
296    /// A CustomDataSourceConfig object that defines the schema for the data source used for the
297    /// evaluation runs. This schema is used to define the shape of the data that will be:
298    /// - Used to define your testing criteria and
299    /// - What data is required when creating a run
300    Custom(CreateEvalCustomDataSourceConfig),
301    /// A data source config which specifies the metadata property of your logs query.
302    /// This is usually metadata like `usecase=chatbot` or `prompt-version=v2`, etc.
303    Logs(CreateEvalLogsDataSourceConfig),
304}
305
306impl Default for CreateEvalDataSourceConfig {
307    fn default() -> Self {
308        Self::Custom(CreateEvalCustomDataSourceConfig::default())
309    }
310}
311
312#[derive(Debug, Serialize, Clone, PartialEq, Builder, Default)]
313#[builder(name = "CreateEvalCustomDataSourceConfigArgs")]
314#[builder(pattern = "mutable")]
315#[builder(setter(into, strip_option), default)]
316#[builder(derive(Debug))]
317#[builder(build_fn(error = "OpenAIError"))]
318pub struct CreateEvalCustomDataSourceConfig {
319    /// The json schema for each row in the data source.
320    pub item_schema: serde_json::Value,
321    /// Whether the eval should expect you to populate the sample namespace (ie, by generating
322    /// responses off of your data source).
323    #[serde(skip_serializing_if = "Option::is_none")]
324    pub include_sample_schema: Option<bool>,
325}
326
327/// Logs data source config for creating an eval.
328#[derive(Debug, Serialize, Clone, PartialEq, Builder, Default)]
329#[builder(name = "CreateEvalLogsDataSourceConfigArgs")]
330#[builder(pattern = "mutable")]
331#[builder(setter(into, strip_option), default)]
332#[builder(derive(Debug))]
333#[builder(build_fn(error = "OpenAIError"))]
334pub struct CreateEvalLogsDataSourceConfig {
335    /// Metadata filters for the logs data source.
336    #[serde(skip_serializing_if = "Option::is_none")]
337    pub metadata: Option<Metadata>,
338}
339
340#[derive(Debug, Serialize, Clone, PartialEq)]
341#[serde(tag = "type", rename_all = "snake_case")]
342pub enum CreateEvalTestingCriterion {
343    /// A LabelModelGrader object which uses a model to assign labels to each item
344    /// in the evaluation.
345    LabelModel(CreateEvalLabelModelGrader),
346    /// A StringCheckGrader object that performs a string comparison between input and reference
347    /// using a specified operation.
348    StringCheck(EvalGraderStringCheck),
349    /// Text similarity grader.
350    TextSimilarity(EvalGraderTextSimilarity),
351    /// Python grader.
352    Python(EvalGraderPython),
353    /// Score model grader.
354    ScoreModel(EvalGraderScoreModel),
355}
356
357/// Label model grader for creating an eval.
358#[derive(Debug, Serialize, Clone, PartialEq, Builder, Default)]
359#[builder(name = "CreateEvalLabelModelGraderArgs")]
360#[builder(pattern = "mutable")]
361#[builder(setter(into, strip_option), default)]
362#[builder(derive(Debug))]
363#[builder(build_fn(error = "OpenAIError"))]
364pub struct CreateEvalLabelModelGrader {
365    /// The name of the grader.
366    pub name: String,
367    /// The model to use for the evaluation. Must support structured outputs.
368    pub model: String,
369    /// A list of chat messages forming the prompt or context. May include variable references to
370    /// the `item` namespace, ie `{{item.name}}`.
371    pub input: Vec<CreateEvalItem>,
372    /// The labels to classify to each item in the evaluation.
373    pub labels: Vec<String>,
374    /// The labels that indicate a passing result. Must be a subset of labels.
375    pub passing_labels: Vec<String>,
376}
377
378#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
379pub struct SimpleInputMessage {
380    /// The role of the message.
381    pub role: String,
382    /// The content of the message.
383    pub content: String,
384}
385
386/// A chat message that makes up the prompt or context.
387#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
388#[serde(tag = "type", rename_all = "snake_case")]
389pub enum CreateEvalItem {
390    /// A message input to the model with a role indicating instruction following
391    /// hierarchy. Instructions given with the `developer` or `system` role take
392    /// precedence over instructions given with the `user` role. Messages with the
393    /// `assistant` role are presumed to have been generated by the model in previous
394    /// interactions.
395    Message(EvalItem),
396
397    /// SimpleInputMessage
398    #[serde(untagged)]
399    Simple(SimpleInputMessage),
400}
401
402/// Request to update an eval.
403#[derive(Debug, Serialize, Clone, Builder, PartialEq, Default)]
404#[builder(name = "UpdateEvalRequestArgs")]
405#[builder(pattern = "mutable")]
406#[builder(setter(into, strip_option), default)]
407#[builder(derive(Debug))]
408#[builder(build_fn(error = "OpenAIError"))]
409pub struct UpdateEvalRequest {
410    /// Rename the evaluation.
411    #[serde(skip_serializing_if = "Option::is_none")]
412    pub name: Option<String>,
413    /// Metadata attached to the eval.
414    #[serde(skip_serializing_if = "Option::is_none")]
415    pub metadata: Option<Metadata>,
416}
417
418/// Response from deleting an eval.
419#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
420pub struct DeleteEvalResponse {
421    /// The object type, which is always "eval.deleted".
422    pub object: String,
423    /// Whether the eval was deleted.
424    pub deleted: bool,
425    /// The ID of the deleted eval.
426    pub eval_id: String,
427}
428
429// EvalRun types
430
431/// A schema representing an evaluation run.
432#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
433pub struct EvalRun {
434    /// The object type, which is always "eval.run".
435    pub object: String,
436    /// Unique identifier for the evaluation run.
437    pub id: String,
438    /// The identifier of the associated evaluation.
439    pub eval_id: String,
440    /// The status of the evaluation run.
441    pub status: EvalRunStatus,
442    /// The model that is evaluated, if applicable.
443    pub model: String,
444    /// The name of the evaluation run.
445    pub name: String,
446    /// Unix timestamp (in seconds) when the evaluation run was created.
447    pub created_at: u64,
448    /// The URL to the rendered evaluation run report on the UI dashboard.
449    pub report_url: String,
450    /// Counters summarizing the outcomes of the evaluation run.
451    pub result_counts: EvalRunResultCounts,
452    /// Usage statistics for each model during the evaluation run.
453    pub per_model_usage: Option<Vec<EvalRunModelUsage>>,
454    /// Results per testing criteria applied during the evaluation run.
455    pub per_testing_criteria_results: Option<Vec<EvalRunTestingCriteriaResult>>,
456    /// Information about the run's data source.
457    pub data_source: EvalRunDataSource,
458    /// Metadata attached to the run.
459    pub metadata: Metadata,
460    /// Error information, if any.
461    pub error: Option<EvalApiError>,
462}
463
464/// Status of an evaluation run.
465#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq)]
466#[serde(rename_all = "snake_case")]
467pub enum EvalRunStatus {
468    /// Queued.
469    Queued,
470    /// In progress.
471    InProgress,
472    /// Completed.
473    Completed,
474    /// Failed.
475    Failed,
476    /// Canceled.
477    Canceled,
478}
479
480/// Counters summarizing the outcomes of the evaluation run.
481#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
482pub struct EvalRunResultCounts {
483    /// Total number of executed output items.
484    pub total: u32,
485    /// Number of output items that resulted in an error.
486    pub errored: u32,
487    /// Number of output items that failed to pass the evaluation.
488    pub failed: u32,
489    /// Number of output items that passed the evaluation.
490    pub passed: u32,
491}
492
493/// Usage statistics for each model during the evaluation run.
494#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
495pub struct EvalRunModelUsage {
496    /// The name of the model.
497    pub model_name: String,
498    /// The number of invocations.
499    pub invocation_count: u32,
500    /// The number of prompt tokens used.
501    pub prompt_tokens: u32,
502    /// The number of completion tokens generated.
503    pub completion_tokens: u32,
504    /// The total number of tokens used.
505    pub total_tokens: u32,
506    /// The number of tokens retrieved from cache.
507    pub cached_tokens: u32,
508}
509
510/// Results per testing criteria applied during the evaluation run.
511#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
512pub struct EvalRunTestingCriteriaResult {
513    /// A description of the testing criteria.
514    pub testing_criteria: String,
515    /// Number of tests passed for this criteria.
516    pub passed: u32,
517    /// Number of tests failed for this criteria.
518    pub failed: u32,
519}
520
521/// Information about the run's data source.
522#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
523#[serde(tag = "type", rename_all = "snake_case")]
524pub enum EvalRunDataSource {
525    /// A JsonlRunDataSource object with that specifies a JSONL file that matches the eval
526    Jsonl(CreateEvalJsonlRunDataSource),
527    /// A CompletionsRunDataSource object describing a model sampling configuration.
528    Completions(CreateEvalCompletionsRunDataSource),
529    /// A ResponsesRunDataSource object describing a model sampling configuration.
530    Responses(CreateEvalResponsesRunDataSource),
531}
532
533/// JSONL run data source.
534#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
535pub struct CreateEvalJsonlRunDataSource {
536    /// Determines what populates the `item` namespace in the data source.
537    pub source: EvalJsonlSource,
538}
539
540/// JSONL source.
541#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
542#[serde(tag = "type", rename_all = "snake_case")]
543pub enum EvalJsonlSource {
544    /// File content source.
545    FileContent(EvalJsonlFileContentSource),
546    /// File ID source.
547    FileId(EvalJsonlFileIdSource),
548}
549
550/// JSONL file content source.
551#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
552pub struct EvalJsonlFileContentSource {
553    /// The content of the jsonl file.
554    pub content: Vec<EvalJsonlContentItem>,
555}
556
557/// JSONL file ID source.
558#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
559pub struct EvalJsonlFileIdSource {
560    /// The identifier of the file.
561    pub id: String,
562}
563
564/// JSONL content item.
565#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
566pub struct EvalJsonlContentItem {
567    /// The item data.
568    pub item: serde_json::Value,
569    /// The sample data, if any.
570    #[serde(skip_serializing_if = "Option::is_none")]
571    pub sample: Option<serde_json::Value>,
572}
573
574/// Completions run data source.
575#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
576pub struct CreateEvalCompletionsRunDataSource {
577    /// Used when sampling from a model. Dictates the structure of the messages passed into the
578    /// model. Can either be a reference to a prebuilt trajectory (ie,
579    /// `item.input_trajectory`), or a template with variable references to the `item`
580    /// namespace.
581    pub input_messages: EvalInputMessages,
582    /// The sampling parameters for the model.
583    #[serde(skip_serializing_if = "Option::is_none")]
584    pub sampling_params: Option<EvalSamplingParams>,
585    /// The name of the model to use for generating completions (e.g. "o3-mini").
586    pub model: String,
587    /// Determines what populates the `item` namespace in this run's data source.
588    pub source: EvalCompletionsSource,
589}
590
591#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
592pub struct TemplateInputMessages {
593    /// A list of chat messages forming the prompt or context. May include variable references to
594    /// the `item` namespace, ie {{item.name}}.
595    pub template: Vec<CreateEvalItem>,
596}
597
598#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
599pub struct ItemReference {
600    /// A reference to a variable in the `item` namespace. Ie, "item.input_trajectory"
601    pub item_reference: String,
602}
603
604/// Input messages for completions.
605#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
606#[serde(tag = "type", rename_all = "snake_case")]
607pub enum EvalInputMessages {
608    /// Template input messages.
609    Template(TemplateInputMessages),
610    /// Item reference input messages.
611    ItemReference(ItemReference),
612}
613
614/// Sampling parameters for the model.
615#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
616pub struct EvalSamplingParams {
617    /// A seed value to initialize the randomness, during sampling.
618    #[serde(skip_serializing_if = "Option::is_none")]
619    pub seed: Option<i32>,
620    /// An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
621    #[serde(skip_serializing_if = "Option::is_none")]
622    pub top_p: Option<f64>,
623    /// A higher temperature increases randomness in the outputs.
624    #[serde(skip_serializing_if = "Option::is_none")]
625    pub temperature: Option<f64>,
626    /// The maximum number of tokens in the generated output.
627    #[serde(skip_serializing_if = "Option::is_none")]
628    pub max_completion_tokens: Option<i32>,
629    /// Optional reasoning effort parameter.
630    #[serde(skip_serializing_if = "Option::is_none")]
631    pub reasoning_effort: Option<ReasoningEffort>,
632    /// An object specifying the format that the model must output.
633    #[serde(skip_serializing_if = "Option::is_none")]
634    pub response_format: Option<ResponseFormat>,
635    /// A list of tools the model may call.
636    #[serde(skip_serializing_if = "Option::is_none")]
637    pub tools: Option<Vec<ChatCompletionTool>>,
638}
639
640#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Default)]
641pub struct EvalResponsesSamplingParams {
642    /// A seed value to initialize the randomness, during sampling.
643    #[serde(skip_serializing_if = "Option::is_none")]
644    pub seed: Option<i32>,
645    /// An alternative to temperature for nucleus sampling; 1.0 includes all tokens.
646    #[serde(skip_serializing_if = "Option::is_none")]
647    pub top_p: Option<f64>,
648    /// A higher temperature increases randomness in the outputs.
649    #[serde(skip_serializing_if = "Option::is_none")]
650    pub temperature: Option<f64>,
651    /// The maximum number of tokens in the generated output.
652    #[serde(skip_serializing_if = "Option::is_none")]
653    pub max_completion_tokens: Option<u32>,
654    /// Optional reasoning effort parameter.
655    #[serde(skip_serializing_if = "Option::is_none")]
656    pub reasoning_effort: Option<ReasoningEffort>,
657    /// An object specifying the format that the model must output.
658    #[serde(skip_serializing_if = "Option::is_none")]
659    pub response_format: Option<ResponseFormat>,
660    /// A list of tools the model may call.
661    #[serde(skip_serializing_if = "Option::is_none")]
662    pub tools: Option<Vec<Tool>>,
663    /// Configuration options for a text response from the model. Can be plain
664    /// text or structured JSON data. Learn more:
665    /// - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
666    /// - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
667    #[serde(skip_serializing_if = "Option::is_none")]
668    pub text: Option<ResponseTextParam>,
669}
670
671/// Completions source.
672#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
673#[serde(tag = "type", rename_all = "snake_case")]
674pub enum EvalCompletionsSource {
675    /// File content source.
676    FileContent(EvalJsonlFileContentSource),
677    /// File ID source.
678    FileId(EvalJsonlFileIdSource),
679    /// Stored completions source.
680    StoredCompletions(EvalStoredCompletionsSource),
681}
682
683/// Stored completions source.
684#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
685pub struct EvalStoredCompletionsSource {
686    /// Metadata filters for the stored completions.
687    #[serde(skip_serializing_if = "Option::is_none")]
688    pub metadata: Option<Metadata>,
689    /// An optional model to filter by.
690    #[serde(skip_serializing_if = "Option::is_none")]
691    pub model: Option<String>,
692    /// An optional Unix timestamp to filter items created after this time.
693    #[serde(skip_serializing_if = "Option::is_none")]
694    pub created_after: Option<u64>,
695    /// An optional Unix timestamp to filter items created before this time.
696    #[serde(skip_serializing_if = "Option::is_none")]
697    pub created_before: Option<u64>,
698    /// An optional maximum number of items to return.
699    #[serde(skip_serializing_if = "Option::is_none")]
700    pub limit: Option<i32>,
701}
702
703/// Responses run data source.
704#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
705pub struct CreateEvalResponsesRunDataSource {
706    /// Used when sampling from a model. Dictates the structure of the messages passed into the
707    /// model.
708    #[serde(skip_serializing_if = "Option::is_none")]
709    pub input_messages: Option<EvalInputMessages>,
710    /// The sampling parameters for the model.
711    #[serde(skip_serializing_if = "Option::is_none")]
712    pub sampling_params: Option<EvalResponsesSamplingParams>,
713    #[serde(skip_serializing_if = "Option::is_none")]
714    pub model: Option<String>,
715    /// Determines what populates the `item` namespace in this run's data source.
716    pub source: EvalResponsesRunSource,
717}
718
719/// Responses source.
720#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
721#[serde(tag = "type", rename_all = "snake_case")]
722pub enum EvalResponsesRunSource {
723    /// File content source.
724    FileContent(EvalJsonlFileContentSource),
725    /// File ID source.
726    FileId(EvalJsonlFileIdSource),
727    /// A EvalResponsesSource object describing a run data source configuration.
728    Responses(EvalResponsesSource),
729}
730
731/// A EvalResponsesSource object describing a run data source configuration.
732#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
733pub struct EvalResponsesSource {
734    /// Metadata filter for the responses. This is a query parameter used to select responses.
735    #[serde(skip_serializing_if = "Option::is_none")]
736    pub metadata: Option<serde_json::Value>,
737    /// The name of the model to find responses for. This is a query parameter used to select
738    /// responses.
739    #[serde(skip_serializing_if = "Option::is_none")]
740    pub model: Option<String>,
741    /// Optional string to search the 'instructions' field. This is a query parameter used to
742    /// select responses.
743    #[serde(skip_serializing_if = "Option::is_none")]
744    pub instructions_search: Option<String>,
745    /// Only include items created after this timestamp (inclusive). This is a query parameter used
746    /// to select responses.
747    #[serde(skip_serializing_if = "Option::is_none")]
748    pub created_after: Option<u64>,
749    /// Only include items created before this timestamp (inclusive). This is a query parameter
750    /// used to select responses.
751    #[serde(skip_serializing_if = "Option::is_none")]
752    pub created_before: Option<u64>,
753    /// Optional reasoning effort parameter. This is a query parameter used to select responses.
754    #[serde(skip_serializing_if = "Option::is_none")]
755    pub reasoning_effort: Option<ReasoningEffort>,
756    /// Sampling temperature. This is a query parameter used to select responses.
757    #[serde(skip_serializing_if = "Option::is_none")]
758    pub temperature: Option<f64>,
759    /// Nucleus sampling parameter. This is a query parameter used to select responses.
760    #[serde(skip_serializing_if = "Option::is_none")]
761    pub top_p: Option<f64>,
762    /// List of user identifiers. This is a query parameter used to select responses.
763    #[serde(skip_serializing_if = "Option::is_none")]
764    pub users: Option<Vec<String>>,
765    /// List of tool names. This is a query parameter used to select responses.
766    #[serde(skip_serializing_if = "Option::is_none")]
767    pub tools: Option<Vec<String>>,
768}
769
770/// List of eval runs.
771#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
772pub struct EvalRunList {
773    /// The object type, which is always "list".
774    pub object: String,
775    /// An array of eval run objects.
776    pub data: Vec<EvalRun>,
777    /// The identifier of the first eval run in the data array.
778    pub first_id: Option<String>,
779    /// The identifier of the last eval run in the data array.
780    pub last_id: Option<String>,
781    /// Indicates whether there are more evals available.
782    pub has_more: bool,
783}
784
785/// Request to create an eval run.
786#[derive(Debug, Serialize, Clone, Builder, PartialEq, Default)]
787#[builder(name = "CreateEvalRunRequestArgs")]
788#[builder(pattern = "mutable")]
789#[builder(setter(into, strip_option), default)]
790#[builder(derive(Debug))]
791#[builder(build_fn(error = "OpenAIError"))]
792pub struct CreateEvalRunRequest {
793    /// The name of the run.
794    #[serde(skip_serializing_if = "Option::is_none")]
795    pub name: Option<String>,
796    /// Details about the run's data source.
797    pub data_source: CreateEvalRunDataSource,
798    /// Metadata attached to the run.
799    #[serde(skip_serializing_if = "Option::is_none")]
800    pub metadata: Option<Metadata>,
801}
802
803/// Details about the run's data source.
804#[derive(Debug, Serialize, Clone, PartialEq)]
805#[serde(tag = "type", rename_all = "snake_case")]
806pub enum CreateEvalRunDataSource {
807    /// JSONL data source.
808    Jsonl(CreateEvalJsonlRunDataSource),
809    /// Completions data source.
810    Completions(CreateEvalCompletionsRunDataSource),
811    /// Responses data source.
812    Responses(CreateEvalResponsesRunDataSource),
813}
814
815// Manual Default implementation for Builder compatibility
816impl Default for CreateEvalRunDataSource {
817    fn default() -> Self {
818        todo!()
819    }
820}
821
822/// Response from deleting an eval run.
823#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
824pub struct DeleteEvalRunResponse {
825    /// The object type, which is always "eval.run.deleted".
826    pub object: String,
827    /// Whether the eval run was deleted.
828    pub deleted: bool,
829    /// The ID of the deleted eval run.
830    pub run_id: String,
831}
832
833// EvalRunOutputItem types
834
835/// A schema representing an evaluation run output item.
836#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
837pub struct EvalRunOutputItem {
838    /// The object type, which is always "eval.run.output_item".
839    pub object: String,
840    /// Unique identifier for the evaluation run output item.
841    pub id: String,
842    /// The identifier of the evaluation run associated with this output item.
843    pub run_id: String,
844    /// The identifier of the evaluation group.
845    pub eval_id: String,
846    /// Unix timestamp (in seconds) when the evaluation run was created.
847    pub created_at: u64,
848    /// The status of the evaluation run.
849    pub status: String,
850    /// The identifier for the data source item.
851    pub datasource_item_id: u64,
852    /// Details of the input data source item.
853    pub datasource_item: serde_json::Value,
854    /// A list of grader results for this output item.
855    pub results: Vec<EvalRunOutputItemResult>,
856    /// A sample containing the input and output of the evaluation run.
857    pub sample: EvalRunOutputItemSample,
858}
859
860/// A single grader result for an evaluation run output item.
861#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
862pub struct EvalRunOutputItemResult {
863    /// The name of the grader.
864    pub name: String,
865    /// The numeric score produced by the grader.
866    pub score: f64,
867    /// Whether the grader considered the output a pass.
868    pub passed: bool,
869    /// Optional sample or intermediate data produced by the grader.
870    #[serde(skip_serializing_if = "Option::is_none")]
871    pub sample: Option<serde_json::Value>,
872}
873
874#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
875pub struct SimpleOutputMessage {
876    pub role: String,
877    pub content: String,
878}
879
880/// A sample containing the input and output of the evaluation run.
881#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
882pub struct EvalRunOutputItemSample {
883    /// An array of input messages.
884    pub input: Vec<SimpleInputMessage>,
885    /// An array of output messages.
886    pub output: Vec<SimpleOutputMessage>,
887    /// The reason why the sample generation was finished.
888    pub finish_reason: String,
889    /// The model used for generating the sample.
890    pub model: String,
891    /// Token usage details for the sample.
892    pub usage: EvalRunOutputItemUsage,
893    /// Error information, if any.
894    pub error: Option<EvalApiError>,
895    /// The sampling temperature used.
896    pub temperature: f64,
897    /// The maximum number of tokens allowed for completion.
898    pub max_completion_tokens: i32,
899    /// The top_p value used for sampling.
900    pub top_p: f64,
901    /// The seed used for generating the sample.
902    pub seed: i32,
903}
904
905/// Token usage details for the sample.
906#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
907pub struct EvalRunOutputItemUsage {
908    /// The total number of tokens used.
909    pub total_tokens: i32,
910    /// The number of completion tokens generated.
911    pub completion_tokens: i32,
912    /// The number of prompt tokens used.
913    pub prompt_tokens: i32,
914    /// The number of tokens retrieved from cache.
915    pub cached_tokens: i32,
916}
917
918/// List of eval run output items.
919#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
920pub struct EvalRunOutputItemList {
921    /// The object type, which is always "list".
922    pub object: String,
923    /// An array of eval run output item objects.
924    pub data: Vec<EvalRunOutputItem>,
925    /// The identifier of the first eval run output item in the data array.
926    pub first_id: Option<String>,
927    /// The identifier of the last eval run output item in the data array.
928    pub last_id: Option<String>,
929    /// Indicates whether there are more eval run output items available.
930    pub has_more: bool,
931}
932
933/// An object representing an error response from the Eval API.
934#[derive(Debug, Deserialize, Serialize, Clone, PartialEq)]
935pub struct EvalApiError {
936    /// The error code.
937    pub code: String,
938    /// The error message.
939    pub message: String,
940}
outfox_openai/spec/evals/eval.rs

outfox_openai/spec/evals/
eval.rs
