synth_ai/
eval.rs

1use serde::{Deserialize, Serialize};
2use serde_json::{json, Value};
3
4use crate::client::{AuthStyle, SynthClient};
5use crate::sse::{stream_sse, SseStream};
6use crate::types::{Result, SynthError};
7
8#[derive(Debug, Clone, Serialize, Deserialize)]
9pub struct EvalJobConfig {
10    pub task_app_url: String,
11    #[serde(skip_serializing_if = "Option::is_none")]
12    pub task_app_api_key: Option<String>,
13    #[serde(skip_serializing_if = "Option::is_none")]
14    pub app_id: Option<String>,
15    #[serde(skip_serializing_if = "Option::is_none")]
16    pub env_name: Option<String>,
17    #[serde(default, skip_serializing_if = "Vec::is_empty")]
18    pub seeds: Vec<u64>,
19    #[serde(default)]
20    pub policy: Value,
21    #[serde(skip_serializing_if = "Option::is_none")]
22    pub env_config: Option<Value>,
23    #[serde(skip_serializing_if = "Option::is_none")]
24    pub verifier_config: Option<Value>,
25    #[serde(rename = "max_concurrent", skip_serializing_if = "Option::is_none")]
26    pub max_concurrent: Option<u64>,
27    #[serde(skip_serializing_if = "Option::is_none")]
28    pub timeout: Option<f64>,
29}
30
31impl EvalJobConfig {
32    pub fn new(task_app_url: impl Into<String>) -> Self {
33        Self {
34            task_app_url: task_app_url.into(),
35            task_app_api_key: None,
36            app_id: None,
37            env_name: None,
38            seeds: Vec::new(),
39            policy: json!({}),
40            env_config: None,
41            verifier_config: None,
42            max_concurrent: None,
43            timeout: None,
44        }
45    }
46}
47
48#[derive(Clone)]
49pub struct EvalJob {
50    client: SynthClient,
51    job_id: String,
52}
53
54impl EvalJob {
55    pub fn new(client: SynthClient, job_id: impl Into<String>) -> Self {
56        Self {
57            client,
58            job_id: job_id.into(),
59        }
60    }
61
62    pub fn job_id(&self) -> &str {
63        &self.job_id
64    }
65
66    pub async fn submit(client: SynthClient, config: &EvalJobConfig) -> Result<Self> {
67        let resp = client
68            .post_json("/eval/jobs", config, AuthStyle::Both)
69            .await?;
70        let job_id = resp
71            .get("job_id")
72            .and_then(|v| v.as_str())
73            .ok_or_else(|| SynthError::UnexpectedResponse("missing job_id".to_string()))?;
74        Ok(Self::new(client, job_id))
75    }
76
77    pub async fn status(&self) -> Result<Value> {
78        let path = format!("/eval/jobs/{}", self.job_id);
79        self.client.get_json(&path, AuthStyle::Both).await
80    }
81
82    pub async fn results(&self) -> Result<Value> {
83        let path = format!("/eval/jobs/{}/results", self.job_id);
84        self.client.get_json(&path, AuthStyle::Both).await
85    }
86
87    pub async fn stream_events(&self) -> Result<SseStream> {
88        let path = format!("/eval/jobs/{}/events/stream", self.job_id);
89        let url = self.client.api_base() + &path;
90        let headers = self.client.auth_headers(AuthStyle::Both);
91        stream_sse(self.client.http(), url, headers).await
92    }
93}
synth_ai/eval.rs

synth_ai/
eval.rs
