alef_e2e/
fixture.rs

1//! Fixture loading, validation, and grouping for e2e test generation.
2
3use anyhow::{Context, Result, bail};
4use serde::{Deserialize, Serialize};
5use std::collections::HashMap;
6use std::path::Path;
7
8/// Mock HTTP response for testing HTTP clients.
9#[derive(Debug, Clone, Serialize, Deserialize)]
10pub struct MockResponse {
11    /// HTTP status code.
12    pub status: u16,
13    /// JSON response body (for non-streaming responses).
14    #[serde(default)]
15    pub body: Option<serde_json::Value>,
16    /// SSE stream chunks (for streaming responses).
17    /// Each chunk is a JSON object sent as `data: <chunk>\n\n`.
18    #[serde(default)]
19    pub stream_chunks: Option<Vec<serde_json::Value>>,
20    /// Response headers to apply to the mock response.
21    /// Bridged from `http.expected_response.headers` for consumer-style fixtures.
22    #[serde(default)]
23    pub headers: HashMap<String, String>,
24}
25
26/// Visitor specification for visitor pattern tests.
27#[derive(Debug, Clone, Serialize, Deserialize)]
28pub struct VisitorSpec {
29    /// Map of callback method name to action.
30    pub callbacks: HashMap<String, CallbackAction>,
31}
32
33/// Action a visitor callback should take.
34#[derive(Debug, Clone, Serialize, Deserialize)]
35#[serde(tag = "action")]
36pub enum CallbackAction {
37    /// Return VisitResult::Skip.
38    #[serde(rename = "skip")]
39    Skip,
40    /// Return VisitResult::Continue.
41    #[serde(rename = "continue")]
42    Continue,
43    /// Return VisitResult::PreserveHtml.
44    #[serde(rename = "preserve_html")]
45    PreserveHtml,
46    /// Return VisitResult::Custom with static output.
47    #[serde(rename = "custom")]
48    Custom {
49        /// The static replacement string.
50        output: String,
51    },
52    /// Return VisitResult::Custom with template interpolation.
53    #[serde(rename = "custom_template")]
54    CustomTemplate {
55        /// Template with placeholders like {text}, {href}.
56        template: String,
57    },
58}
59
60/// Environment variable requirements for a smoke/live test fixture.
61#[derive(Debug, Clone, Serialize, Deserialize)]
62pub struct FixtureEnv {
63    /// Name of the env var that holds the API key (e.g. `"OPENAI_API_KEY"`).
64    #[serde(default)]
65    pub api_key_var: Option<String>,
66}
67
68/// A single e2e test fixture.
69#[derive(Debug, Clone, Serialize, Deserialize)]
70pub struct Fixture {
71    /// Unique identifier (used as test function name).
72    pub id: String,
73    /// Optional category (defaults to parent directory name).
74    #[serde(default)]
75    pub category: Option<String>,
76    /// Human-readable description.
77    pub description: String,
78    /// Optional tags for filtering.
79    #[serde(default)]
80    pub tags: Vec<String>,
81    /// Skip directive.
82    #[serde(default)]
83    pub skip: Option<SkipDirective>,
84    /// Environment variable requirements (used by smoke/live tests).
85    #[serde(default)]
86    pub env: Option<FixtureEnv>,
87    /// Named call config to use (references `[e2e.calls.<name>]`).
88    /// When omitted, uses the default `[e2e.call]`.
89    #[serde(default)]
90    pub call: Option<String>,
91    /// Input data passed to the function under test.
92    #[serde(default)]
93    pub input: serde_json::Value,
94    /// Optional mock HTTP response for testing HTTP clients.
95    #[serde(default)]
96    pub mock_response: Option<MockResponse>,
97    /// Optional visitor specification for visitor pattern tests.
98    #[serde(default)]
99    pub visitor: Option<VisitorSpec>,
100    /// List of assertions to check.
101    #[serde(default)]
102    pub assertions: Vec<Assertion>,
103    /// Source file path (populated during loading).
104    #[serde(skip)]
105    pub source: String,
106    /// HTTP server test specification. When present, this fixture tests
107    /// an HTTP handler rather than a function call.
108    #[serde(default)]
109    pub http: Option<HttpFixture>,
110}
111
112/// HTTP server test specification.
113#[derive(Debug, Clone, Serialize, Deserialize)]
114pub struct HttpFixture {
115    /// Handler/route definition.
116    pub handler: HttpHandler,
117    /// The HTTP request to send.
118    pub request: HttpRequest,
119    /// Expected response.
120    pub expected_response: HttpExpectedResponse,
121}
122
123/// Handler/route definition for HTTP server tests.
124#[derive(Debug, Clone, Serialize, Deserialize)]
125pub struct HttpHandler {
126    /// Route pattern (e.g., "/users/{user_id}").
127    pub route: String,
128    /// HTTP method (GET, POST, PUT, etc.).
129    pub method: String,
130    /// JSON Schema for request body validation.
131    #[serde(default)]
132    pub body_schema: Option<serde_json::Value>,
133    /// Parameter schemas by source (path, query, header, cookie).
134    #[serde(default)]
135    pub parameters: HashMap<String, HashMap<String, serde_json::Value>>,
136    /// Middleware configuration.
137    #[serde(default)]
138    pub middleware: Option<HttpMiddleware>,
139}
140
141/// HTTP request to send in a server test.
142#[derive(Debug, Clone, Serialize, Deserialize)]
143pub struct HttpRequest {
144    pub method: String,
145    pub path: String,
146    #[serde(default)]
147    pub headers: HashMap<String, String>,
148    #[serde(default)]
149    pub query_params: HashMap<String, serde_json::Value>,
150    #[serde(default)]
151    pub cookies: HashMap<String, String>,
152    #[serde(default)]
153    pub body: Option<serde_json::Value>,
154    #[serde(default)]
155    pub content_type: Option<String>,
156}
157
158/// Expected HTTP response specification.
159#[derive(Debug, Clone, Serialize, Deserialize)]
160pub struct HttpExpectedResponse {
161    pub status_code: u16,
162    /// Exact body match.
163    #[serde(default)]
164    pub body: Option<serde_json::Value>,
165    /// Partial body match (only check specified fields).
166    #[serde(default)]
167    pub body_partial: Option<serde_json::Value>,
168    /// Header expectations. Special tokens: `<<uuid>>`, `<<present>>`, `<<absent>>`.
169    #[serde(default)]
170    pub headers: HashMap<String, String>,
171    /// Expected validation errors (for 422 responses).
172    #[serde(default)]
173    pub validation_errors: Option<Vec<ValidationErrorExpectation>>,
174}
175
176/// Expected validation error entry.
177#[derive(Debug, Clone, Serialize, Deserialize)]
178pub struct ValidationErrorExpectation {
179    pub loc: Vec<String>,
180    pub msg: String,
181    #[serde(rename = "type")]
182    pub error_type: String,
183}
184
185/// CORS policy configuration for HTTP handler tests.
186#[derive(Debug, Clone, Default, Serialize, Deserialize)]
187pub struct CorsConfig {
188    /// Allowed origins (e.g. `["https://example.com"]`). Empty means deny all.
189    #[serde(default)]
190    pub allow_origins: Vec<String>,
191    /// Allowed HTTP methods (e.g. `["GET", "POST"]`). Empty means deny all.
192    #[serde(default)]
193    pub allow_methods: Vec<String>,
194    /// Allowed request headers (e.g. `["Content-Type"]`). Empty means deny all.
195    #[serde(default)]
196    pub allow_headers: Vec<String>,
197    /// Exposed response headers (e.g. `["X-Total-Count"]`).
198    #[serde(default)]
199    pub expose_headers: Vec<String>,
200    /// `Access-Control-Max-Age` value in seconds.
201    #[serde(default)]
202    pub max_age: Option<u64>,
203    /// Whether to allow credentials.
204    #[serde(default)]
205    pub allow_credentials: bool,
206}
207
208/// A single static file entry for the static-files middleware.
209#[derive(Debug, Clone, Serialize, Deserialize)]
210pub struct StaticFile {
211    /// Relative path within the served directory (e.g. `"hello.txt"`).
212    pub path: String,
213    /// File content (plain text or HTML string).
214    pub content: String,
215}
216
217/// Static-files middleware configuration for HTTP handler tests.
218#[derive(Debug, Clone, Serialize, Deserialize)]
219pub struct StaticFilesConfig {
220    /// URL route prefix (e.g. `"/public"`).
221    pub route_prefix: String,
222    /// Files to write to the temporary directory.
223    #[serde(default)]
224    pub files: Vec<StaticFile>,
225    /// Whether to serve `index.html` for directory requests.
226    #[serde(default)]
227    pub index_file: bool,
228    /// `Cache-Control` header value to apply.
229    #[serde(default)]
230    pub cache_control: Option<String>,
231}
232
233/// Middleware configuration for HTTP handler tests.
234#[derive(Debug, Clone, Default, Serialize, Deserialize)]
235pub struct HttpMiddleware {
236    #[serde(default)]
237    pub jwt_auth: Option<serde_json::Value>,
238    #[serde(default)]
239    pub api_key_auth: Option<serde_json::Value>,
240    #[serde(default)]
241    pub compression: Option<serde_json::Value>,
242    #[serde(default)]
243    pub rate_limit: Option<serde_json::Value>,
244    #[serde(default)]
245    pub request_timeout: Option<serde_json::Value>,
246    #[serde(default)]
247    pub request_id: Option<serde_json::Value>,
248    /// CORS policy to apply via tower-http `CorsLayer`.
249    #[serde(default)]
250    pub cors: Option<CorsConfig>,
251    /// Static-files configuration to serve via tower-http `ServeDir`.
252    #[serde(default)]
253    pub static_files: Option<Vec<StaticFilesConfig>>,
254}
255
256impl Fixture {
257    /// Returns true if this is an HTTP server test fixture.
258    pub fn is_http_test(&self) -> bool {
259        self.http.is_some()
260    }
261
262    /// Returns true if this fixture requires a mock HTTP server.
263    /// This is true when either `mock_response` (liter-llm shape) or
264    /// `http.expected_response` (consumer shape) is present.
265    pub fn needs_mock_server(&self) -> bool {
266        self.mock_response.is_some() || self.http.is_some()
267    }
268
269    /// Returns the effective mock response for this fixture, bridging both schemas:
270    /// - liter-llm shape: `mock_response: { status, body, stream_chunks }`
271    /// - consumer shape: `http.expected_response: { status_code, body, headers }`
272    ///
273    /// Returns `None` if neither schema is present.
274    pub fn as_mock_response(&self) -> Option<MockResponse> {
275        if let Some(mock) = &self.mock_response {
276            return Some(mock.clone());
277        }
278        if let Some(http) = &self.http {
279            return Some(MockResponse {
280                status: http.expected_response.status_code,
281                body: http.expected_response.body.clone(),
282                stream_chunks: None,
283                headers: http.expected_response.headers.clone(),
284            });
285        }
286        None
287    }
288
289    /// Returns true if the mock response uses streaming (SSE).
290    pub fn is_streaming_mock(&self) -> bool {
291        self.mock_response
292            .as_ref()
293            .and_then(|m| m.stream_chunks.as_ref())
294            .map(|c| !c.is_empty())
295            .unwrap_or(false)
296    }
297
298    /// Get the resolved category (explicit or from source directory).
299    pub fn resolved_category(&self) -> String {
300        self.category.clone().unwrap_or_else(|| {
301            Path::new(&self.source)
302                .parent()
303                .and_then(|p| p.file_name())
304                .and_then(|n| n.to_str())
305                .unwrap_or("default")
306                .to_string()
307        })
308    }
309}
310
311/// Skip directive for conditionally excluding fixtures.
312#[derive(Debug, Clone, Serialize, Deserialize)]
313pub struct SkipDirective {
314    /// Languages to skip (empty means skip all).
315    #[serde(default)]
316    pub languages: Vec<String>,
317    /// Human-readable reason for skipping.
318    #[serde(default)]
319    pub reason: Option<String>,
320}
321
322impl SkipDirective {
323    /// Check if this fixture should be skipped for a given language.
324    pub fn should_skip(&self, language: &str) -> bool {
325        self.languages.is_empty() || self.languages.iter().any(|l| l == language)
326    }
327}
328
329/// A single assertion in a fixture.
330#[derive(Debug, Clone, Serialize, Deserialize)]
331pub struct Assertion {
332    /// Assertion type (equals, contains, not_empty, error, etc.).
333    #[serde(rename = "type")]
334    pub assertion_type: String,
335    /// Field path to access on the result (dot-separated).
336    #[serde(default)]
337    pub field: Option<String>,
338    /// Expected value (string, number, bool, or array depending on type).
339    #[serde(default)]
340    pub value: Option<serde_json::Value>,
341    /// Expected values (for contains_all, contains_any).
342    #[serde(default)]
343    pub values: Option<Vec<serde_json::Value>>,
344    /// Method name to call on the result (for method_result assertions).
345    #[serde(default)]
346    pub method: Option<String>,
347    /// Assertion check type for the method result (equals, is_true, is_false, greater_than_or_equal, count_min).
348    #[serde(default)]
349    pub check: Option<String>,
350    /// Arguments to pass to the method call (for method_result assertions).
351    #[serde(default)]
352    pub args: Option<serde_json::Value>,
353}
354
355/// A group of fixtures sharing the same category.
356#[derive(Debug, Clone)]
357pub struct FixtureGroup {
358    pub category: String,
359    pub fixtures: Vec<Fixture>,
360}
361
362/// Load all fixtures from a directory recursively.
363pub fn load_fixtures(dir: &Path) -> Result<Vec<Fixture>> {
364    let mut fixtures = Vec::new();
365    load_fixtures_recursive(dir, dir, &mut fixtures)?;
366
367    // Validate: check for duplicate IDs
368    let mut seen: HashMap<String, String> = HashMap::new();
369    for f in &fixtures {
370        if let Some(prev_source) = seen.get(&f.id) {
371            bail!(
372                "duplicate fixture ID '{}': found in '{}' and '{}'",
373                f.id,
374                prev_source,
375                f.source
376            );
377        }
378        seen.insert(f.id.clone(), f.source.clone());
379    }
380
381    // Sort by (category, id) for deterministic output
382    fixtures.sort_by(|a, b| {
383        let cat_cmp = a.resolved_category().cmp(&b.resolved_category());
384        cat_cmp.then_with(|| a.id.cmp(&b.id))
385    });
386
387    Ok(fixtures)
388}
389
390fn load_fixtures_recursive(base: &Path, dir: &Path, fixtures: &mut Vec<Fixture>) -> Result<()> {
391    let entries =
392        std::fs::read_dir(dir).with_context(|| format!("failed to read fixture directory: {}", dir.display()))?;
393
394    let mut paths: Vec<_> = entries.filter_map(|e| e.ok()).map(|e| e.path()).collect();
395    paths.sort();
396
397    for path in paths {
398        if path.is_dir() {
399            load_fixtures_recursive(base, &path, fixtures)?;
400        } else if path.extension().is_some_and(|ext| ext == "json") {
401            let filename = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
402            // Skip schema files and files starting with _
403            if filename == "schema.json" || filename.starts_with('_') {
404                continue;
405            }
406            let content = std::fs::read_to_string(&path)
407                .with_context(|| format!("failed to read fixture: {}", path.display()))?;
408            let relative = path.strip_prefix(base).unwrap_or(&path).to_string_lossy().to_string();
409
410            // Try parsing as array first, then as single fixture
411            let parsed: Vec<Fixture> = if content.trim_start().starts_with('[') {
412                serde_json::from_str(&content)
413                    .with_context(|| format!("failed to parse fixture array: {}", path.display()))?
414            } else {
415                let single: Fixture = serde_json::from_str(&content)
416                    .with_context(|| format!("failed to parse fixture: {}", path.display()))?;
417                vec![single]
418            };
419
420            for mut fixture in parsed {
421                fixture.source = relative.clone();
422                // Expand template expressions (e.g. `{{ repeat 'x' 10000 times }}`)
423                // in all JSON string values so generators emit the expanded values.
424                expand_json_templates(&mut fixture.input);
425                if let Some(ref mut http) = fixture.http {
426                    for (_, v) in http.request.headers.iter_mut() {
427                        *v = crate::escape::expand_fixture_templates(v);
428                    }
429                    if let Some(ref mut body) = http.request.body {
430                        expand_json_templates(body);
431                    }
432                }
433                fixtures.push(fixture);
434            }
435        }
436    }
437    Ok(())
438}
439
440/// Group fixtures by their resolved category.
441pub fn group_fixtures(fixtures: &[Fixture]) -> Vec<FixtureGroup> {
442    let mut groups: HashMap<String, Vec<Fixture>> = HashMap::new();
443    for f in fixtures {
444        groups.entry(f.resolved_category()).or_default().push(f.clone());
445    }
446    let mut result: Vec<FixtureGroup> = groups
447        .into_iter()
448        .map(|(category, fixtures)| FixtureGroup { category, fixtures })
449        .collect();
450    result.sort_by(|a, b| a.category.cmp(&b.category));
451    result
452}
453
454/// Recursively expand fixture template expressions in all string values of a JSON tree.
455fn expand_json_templates(value: &mut serde_json::Value) {
456    match value {
457        serde_json::Value::String(s) => {
458            let expanded = crate::escape::expand_fixture_templates(s);
459            if expanded != *s {
460                *s = expanded;
461            }
462        }
463        serde_json::Value::Array(arr) => {
464            for item in arr {
465                expand_json_templates(item);
466            }
467        }
468        serde_json::Value::Object(map) => {
469            for (_, v) in map.iter_mut() {
470                expand_json_templates(v);
471            }
472        }
473        _ => {}
474    }
475}
476
477#[cfg(test)]
478mod tests {
479    use super::*;
480
481    #[test]
482    fn test_fixture_with_mock_response() {
483        let json = r#"{
484            "id": "test_chat",
485            "description": "Test chat",
486            "call": "chat",
487            "input": {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]},
488            "mock_response": {
489                "status": 200,
490                "body": {"choices": [{"message": {"content": "hello"}}]}
491            },
492            "assertions": [{"type": "not_error"}]
493        }"#;
494        let fixture: Fixture = serde_json::from_str(json).unwrap();
495        assert!(fixture.needs_mock_server());
496        assert!(!fixture.is_streaming_mock());
497        assert_eq!(fixture.mock_response.unwrap().status, 200);
498    }
499
500    #[test]
501    fn test_fixture_with_streaming_mock_response() {
502        let json = r#"{
503            "id": "test_stream",
504            "description": "Test streaming",
505            "input": {},
506            "mock_response": {
507                "status": 200,
508                "stream_chunks": [{"delta": "hello"}, {"delta": " world"}]
509            },
510            "assertions": []
511        }"#;
512        let fixture: Fixture = serde_json::from_str(json).unwrap();
513        assert!(fixture.needs_mock_server());
514        assert!(fixture.is_streaming_mock());
515    }
516
517    #[test]
518    fn test_fixture_without_mock_response() {
519        let json = r#"{
520            "id": "test_no_mock",
521            "description": "No mock",
522            "input": {},
523            "assertions": []
524        }"#;
525        let fixture: Fixture = serde_json::from_str(json).unwrap();
526        assert!(!fixture.needs_mock_server());
527        assert!(!fixture.is_streaming_mock());
528    }
529}
alef_e2e/fixture.rs

alef_e2e/
fixture.rs
