skill_runtime/generation/
fixtures.rs

1//! Test fixtures for AI generation testing
2//!
3//! Provides realistic tool schemas, mock LLM responses, and test utilities
4//! for comprehensive testing of the generation pipeline.
5
6use std::collections::HashMap;
7use std::pin::Pin;
8use anyhow::Result;
9use async_trait::async_trait;
10use futures_util::Stream;
11
12use crate::skill_md::{ToolDocumentation, ParameterDoc, ParameterType, CodeExample};
13use super::llm_provider::{LlmProvider, LlmResponse, LlmChunk, CompletionRequest};
14
15// =============================================================================
16// Tool Fixtures
17// =============================================================================
18
19/// Create a complex Kubernetes tool fixture
20pub fn kubernetes_apply_tool() -> ToolDocumentation {
21    ToolDocumentation {
22        name: "apply".to_string(),
23        description: "Apply a configuration to a resource by file name or stdin. The resource name must be specified.".to_string(),
24        usage: Some("skill run kubernetes:apply --file=<manifest.yaml> [--namespace=<ns>] [--dry-run]".to_string()),
25        parameters: vec![
26            ParameterDoc {
27                name: "file".to_string(),
28                param_type: ParameterType::String,
29                description: "Path to the file that contains the configuration to apply".to_string(),
30                required: true,
31                default: None,
32                allowed_values: vec![],
33            },
34            ParameterDoc {
35                name: "namespace".to_string(),
36                param_type: ParameterType::String,
37                description: "If present, the namespace scope for this CLI request".to_string(),
38                required: false,
39                default: Some("default".to_string()),
40                allowed_values: vec![],
41            },
42            ParameterDoc {
43                name: "dry-run".to_string(),
44                param_type: ParameterType::Boolean,
45                description: "Preview the object that would be sent without actually sending it".to_string(),
46                required: false,
47                default: None,
48                allowed_values: vec![],
49            },
50            ParameterDoc {
51                name: "output".to_string(),
52                param_type: ParameterType::String,
53                description: "Output format".to_string(),
54                required: false,
55                default: None,
56                allowed_values: vec!["json".to_string(), "yaml".to_string(), "wide".to_string()],
57            },
58            ParameterDoc {
59                name: "wait".to_string(),
60                param_type: ParameterType::Boolean,
61                description: "Wait for resources to be ready".to_string(),
62                required: false,
63                default: None,
64                allowed_values: vec![],
65            },
66            ParameterDoc {
67                name: "timeout".to_string(),
68                param_type: ParameterType::Integer,
69                description: "Timeout in seconds for the operation".to_string(),
70                required: false,
71                default: Some("300".to_string()),
72                allowed_values: vec![],
73            },
74        ],
75        examples: vec![
76            CodeExample {
77                language: Some("bash".to_string()),
78                code: "skill run kubernetes:apply --file=deployment.yaml".to_string(),
79                description: Some("Apply a deployment manifest".to_string()),
80            },
81        ],
82    }
83}
84
85/// Create a simple tool with minimal parameters
86pub fn simple_tool() -> ToolDocumentation {
87    ToolDocumentation {
88        name: "list".to_string(),
89        description: "List all resources of a given type".to_string(),
90        usage: None,
91        parameters: vec![
92            ParameterDoc {
93                name: "type".to_string(),
94                param_type: ParameterType::String,
95                description: "Resource type to list".to_string(),
96                required: true,
97                default: None,
98                allowed_values: vec![],
99            },
100        ],
101        examples: vec![],
102    }
103}
104
105/// Create a tool with enum constraints
106pub fn tool_with_constraints() -> ToolDocumentation {
107    ToolDocumentation {
108        name: "get".to_string(),
109        description: "Display one or many resources".to_string(),
110        usage: None,
111        parameters: vec![
112            ParameterDoc {
113                name: "resource".to_string(),
114                param_type: ParameterType::String,
115                description: "Resource type".to_string(),
116                required: true,
117                default: None,
118                allowed_values: vec![
119                    "pods".to_string(),
120                    "deployments".to_string(),
121                    "services".to_string(),
122                    "configmaps".to_string(),
123                    "secrets".to_string(),
124                ],
125            },
126            ParameterDoc {
127                name: "output".to_string(),
128                param_type: ParameterType::String,
129                description: "Output format".to_string(),
130                required: false,
131                default: None,
132                allowed_values: vec!["json".to_string(), "yaml".to_string(), "wide".to_string()],
133            },
134            ParameterDoc {
135                name: "all-namespaces".to_string(),
136                param_type: ParameterType::Boolean,
137                description: "List across all namespaces".to_string(),
138                required: false,
139                default: None,
140                allowed_values: vec![],
141            },
142        ],
143        examples: vec![],
144    }
145}
146
147/// Create an AWS S3 tool
148pub fn aws_s3_tool() -> ToolDocumentation {
149    ToolDocumentation {
150        name: "s3-copy".to_string(),
151        description: "Copy files between S3 buckets or between local and S3".to_string(),
152        usage: None,
153        parameters: vec![
154            ParameterDoc {
155                name: "source".to_string(),
156                param_type: ParameterType::String,
157                description: "Source path (local path or s3://bucket/key)".to_string(),
158                required: true,
159                default: None,
160                allowed_values: vec![],
161            },
162            ParameterDoc {
163                name: "destination".to_string(),
164                param_type: ParameterType::String,
165                description: "Destination path (local path or s3://bucket/key)".to_string(),
166                required: true,
167                default: None,
168                allowed_values: vec![],
169            },
170            ParameterDoc {
171                name: "recursive".to_string(),
172                param_type: ParameterType::Boolean,
173                description: "Copy recursively".to_string(),
174                required: false,
175                default: None,
176                allowed_values: vec![],
177            },
178            ParameterDoc {
179                name: "region".to_string(),
180                param_type: ParameterType::String,
181                description: "AWS region".to_string(),
182                required: false,
183                default: Some("us-east-1".to_string()),
184                allowed_values: vec![],
185            },
186        ],
187        examples: vec![],
188    }
189}
190
191/// Create a Docker build tool
192pub fn docker_build_tool() -> ToolDocumentation {
193    ToolDocumentation {
194        name: "build".to_string(),
195        description: "Build an image from a Dockerfile".to_string(),
196        usage: None,
197        parameters: vec![
198            ParameterDoc {
199                name: "context".to_string(),
200                param_type: ParameterType::String,
201                description: "Build context directory".to_string(),
202                required: true,
203                default: None,
204                allowed_values: vec![],
205            },
206            ParameterDoc {
207                name: "tag".to_string(),
208                param_type: ParameterType::String,
209                description: "Name and optionally a tag (name:tag)".to_string(),
210                required: false,
211                default: None,
212                allowed_values: vec![],
213            },
214            ParameterDoc {
215                name: "file".to_string(),
216                param_type: ParameterType::String,
217                description: "Name of the Dockerfile".to_string(),
218                required: false,
219                default: Some("Dockerfile".to_string()),
220                allowed_values: vec![],
221            },
222            ParameterDoc {
223                name: "no-cache".to_string(),
224                param_type: ParameterType::Boolean,
225                description: "Do not use cache when building".to_string(),
226                required: false,
227                default: None,
228                allowed_values: vec![],
229            },
230        ],
231        examples: vec![],
232    }
233}
234
235// =============================================================================
236// Mock LLM Responses
237// =============================================================================
238
239/// Get a mock JSON response for a tool
240pub fn mock_response_for_tool(tool_name: &str) -> String {
241    match tool_name {
242        "apply" => r#"[
243            {"command": "skill run kubernetes:apply --file=deployment.yaml", "explanation": "Apply a deployment manifest to the cluster"},
244            {"command": "skill run kubernetes:apply --file=service.yaml --namespace=production", "explanation": "Apply a service in the production namespace"},
245            {"command": "skill run kubernetes:apply --file=configmap.yaml --dry-run=true", "explanation": "Preview applying a configmap without making changes"},
246            {"command": "skill run kubernetes:apply --file=app.yaml --namespace=staging --output=json", "explanation": "Apply manifest to staging and output result as JSON"},
247            {"command": "skill run kubernetes:apply --file=./manifests/full-stack.yaml --wait --timeout=120", "explanation": "Apply and wait up to 120 seconds for resources to be ready"}
248        ]"#.to_string(),
249        "list" => r#"[
250            {"command": "skill run tool:list --type=pods", "explanation": "List all pods"},
251            {"command": "skill run tool:list --type=services", "explanation": "List all services"},
252            {"command": "skill run tool:list --type=deployments", "explanation": "List all deployments"}
253        ]"#.to_string(),
254        "get" => r#"[
255            {"command": "skill run kubernetes:get --resource=pods", "explanation": "Get all pods in the default namespace"},
256            {"command": "skill run kubernetes:get --resource=deployments --output=json", "explanation": "Get deployments as JSON"},
257            {"command": "skill run kubernetes:get --resource=services --all-namespaces", "explanation": "Get services across all namespaces"},
258            {"command": "skill run kubernetes:get --resource=configmaps --output=yaml", "explanation": "Get configmaps in YAML format"},
259            {"command": "skill run kubernetes:get --resource=secrets --all-namespaces --output=wide", "explanation": "Get secrets with extended info"}
260        ]"#.to_string(),
261        "s3-copy" => r#"[
262            {"command": "skill run aws:s3-copy --source=./local-file.txt --destination=s3://my-bucket/file.txt", "explanation": "Upload local file to S3"},
263            {"command": "skill run aws:s3-copy --source=s3://bucket-a/data.json --destination=s3://bucket-b/data.json", "explanation": "Copy file between S3 buckets"},
264            {"command": "skill run aws:s3-copy --source=./data/ --destination=s3://backup-bucket/data/ --recursive", "explanation": "Upload entire directory to S3"},
265            {"command": "skill run aws:s3-copy --source=s3://bucket/file.csv --destination=./downloads/file.csv --region=eu-west-1", "explanation": "Download from EU region bucket"}
266        ]"#.to_string(),
267        "build" => r#"[
268            {"command": "skill run docker:build --context=. --tag=myapp:latest", "explanation": "Build image from current directory"},
269            {"command": "skill run docker:build --context=./app --tag=myapp:v1.0 --file=Dockerfile.prod", "explanation": "Build with custom Dockerfile"},
270            {"command": "skill run docker:build --context=. --tag=test:ci --no-cache", "explanation": "Build without cache for CI"},
271            {"command": "skill run docker:build --context=./backend --tag=api:latest", "explanation": "Build backend API image"}
272        ]"#.to_string(),
273        _ => r#"[
274            {"command": "skill run tool:command --param=value", "explanation": "Example command"}
275        ]"#.to_string(),
276    }
277}
278
279/// Get a mock response with some invalid examples for testing validation
280pub fn mock_response_with_errors(tool_name: &str) -> String {
281    match tool_name {
282        "apply" => r#"[
283            {"command": "skill run kubernetes:apply --file=valid.yaml", "explanation": "Valid example"},
284            {"command": "skill run kubernetes:apply --namespace=prod", "explanation": "Missing required file parameter"},
285            {"command": "skill run kubernetes:apply --file=test.yaml", "explanation": ""},
286            {"command": "skill run kubernetes:apply --file=good.yaml --output=json", "explanation": "Another valid example"}
287        ]"#.to_string(),
288        _ => mock_response_for_tool(tool_name),
289    }
290}
291
292// =============================================================================
293// Mock LLM Provider
294// =============================================================================
295
296/// A deterministic mock LLM provider for testing
297pub struct DeterministicMockProvider {
298    /// Pre-configured responses by tool name
299    responses: HashMap<String, String>,
300    /// Default response when tool not found
301    default_response: String,
302    /// Artificial delay in ms (for latency testing)
303    delay_ms: u64,
304    /// Call counter
305    call_count: std::sync::atomic::AtomicUsize,
306}
307
308impl DeterministicMockProvider {
309    /// Create a new mock provider
310    pub fn new() -> Self {
311        let mut responses = HashMap::new();
312        responses.insert("apply".to_string(), mock_response_for_tool("apply"));
313        responses.insert("list".to_string(), mock_response_for_tool("list"));
314        responses.insert("get".to_string(), mock_response_for_tool("get"));
315        responses.insert("s3-copy".to_string(), mock_response_for_tool("s3-copy"));
316        responses.insert("build".to_string(), mock_response_for_tool("build"));
317
318        Self {
319            responses,
320            default_response: r#"[{"command": "skill run tool --param=value", "explanation": "Generic example"}]"#.to_string(),
321            delay_ms: 0,
322            call_count: std::sync::atomic::AtomicUsize::new(0),
323        }
324    }
325
326    /// Create a mock that returns errors for some tools
327    pub fn with_validation_errors() -> Self {
328        let mut provider = Self::new();
329        provider.responses.insert("apply".to_string(), mock_response_with_errors("apply"));
330        provider
331    }
332
333    /// Set artificial delay for latency testing
334    pub fn with_delay(mut self, delay_ms: u64) -> Self {
335        self.delay_ms = delay_ms;
336        self
337    }
338
339    /// Add a custom response for a tool
340    pub fn with_response(mut self, tool_name: &str, response: &str) -> Self {
341        self.responses.insert(tool_name.to_string(), response.to_string());
342        self
343    }
344
345    /// Get the number of calls made
346    pub fn call_count(&self) -> usize {
347        self.call_count.load(std::sync::atomic::Ordering::SeqCst)
348    }
349
350    /// Extract tool name from prompt
351    fn extract_tool_name(&self, prompt: &str) -> String {
352        // Look for "Name: <tool_name>" in the prompt
353        for line in prompt.lines() {
354            if line.starts_with("- **Name**:") || line.starts_with("Name:") {
355                return line
356                    .split(':')
357                    .nth(1)
358                    .map(|s| s.trim().to_string())
359                    .unwrap_or_default();
360            }
361        }
362        "unknown".to_string()
363    }
364}
365
366impl Default for DeterministicMockProvider {
367    fn default() -> Self {
368        Self::new()
369    }
370}
371
372#[async_trait]
373impl LlmProvider for DeterministicMockProvider {
374    fn name(&self) -> &str {
375        "mock"
376    }
377
378    fn model(&self) -> &str {
379        "deterministic-test"
380    }
381
382    async fn complete(&self, request: &CompletionRequest) -> Result<LlmResponse> {
383        self.call_count.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
384
385        // Apply delay if configured
386        if self.delay_ms > 0 {
387            tokio::time::sleep(tokio::time::Duration::from_millis(self.delay_ms)).await;
388        }
389
390        // Extract tool name from user message
391        let empty = String::new();
392        let user_message = request.messages.iter()
393            .find(|m| m.role == "user")
394            .map(|m| &m.content)
395            .unwrap_or(&empty);
396
397        let tool_name = self.extract_tool_name(user_message);
398
399        let content = self.responses
400            .get(&tool_name)
401            .cloned()
402            .unwrap_or_else(|| self.default_response.clone());
403
404        Ok(LlmResponse {
405            content,
406            model: "deterministic-test".to_string(),
407            usage: None,
408            finish_reason: Some("stop".to_string()),
409        })
410    }
411
412    async fn complete_stream(
413        &self,
414        request: &CompletionRequest,
415    ) -> Result<Pin<Box<dyn Stream<Item = Result<LlmChunk>> + Send>>> {
416        // For simplicity, just return the full response as one chunk
417        let response = self.complete(request).await?;
418
419        let stream = async_stream::stream! {
420            yield Ok(LlmChunk {
421                delta: response.content,
422                is_final: true,
423            });
424        };
425
426        Ok(Box::pin(stream))
427    }
428}
429
430// =============================================================================
431// Failing Mock Provider
432// =============================================================================
433
434/// A mock provider that always fails
435pub struct FailingMockProvider {
436    error_message: String,
437}
438
439impl FailingMockProvider {
440    pub fn new(message: &str) -> Self {
441        Self {
442            error_message: message.to_string(),
443        }
444    }
445}
446
447#[async_trait]
448impl LlmProvider for FailingMockProvider {
449    fn name(&self) -> &str {
450        "failing-mock"
451    }
452
453    fn model(&self) -> &str {
454        "error-test"
455    }
456
457    async fn complete(&self, _request: &CompletionRequest) -> Result<LlmResponse> {
458        anyhow::bail!("{}", self.error_message)
459    }
460
461    async fn complete_stream(
462        &self,
463        _request: &CompletionRequest,
464    ) -> Result<Pin<Box<dyn Stream<Item = Result<LlmChunk>> + Send>>> {
465        anyhow::bail!("{}", self.error_message)
466    }
467}
468
469// =============================================================================
470// Tests
471// =============================================================================
472
473#[cfg(test)]
474mod tests {
475    use super::*;
476
477    #[test]
478    fn test_kubernetes_tool_fixture() {
479        let tool = kubernetes_apply_tool();
480        assert_eq!(tool.name, "apply");
481        assert!(!tool.parameters.is_empty());
482
483        // Check required parameter
484        let file_param = tool.parameters.iter().find(|p| p.name == "file").unwrap();
485        assert!(file_param.required);
486    }
487
488    #[test]
489    fn test_tool_with_constraints() {
490        let tool = tool_with_constraints();
491        let resource_param = tool.parameters.iter().find(|p| p.name == "resource").unwrap();
492        assert!(!resource_param.allowed_values.is_empty());
493        assert!(resource_param.allowed_values.contains(&"pods".to_string()));
494    }
495
496    #[test]
497    fn test_mock_response_parsing() {
498        let response = mock_response_for_tool("apply");
499        let parsed: Vec<serde_json::Value> = serde_json::from_str(&response).unwrap();
500        assert_eq!(parsed.len(), 5);
501
502        for example in &parsed {
503            assert!(example.get("command").is_some());
504            assert!(example.get("explanation").is_some());
505        }
506    }
507
508    #[tokio::test]
509    async fn test_deterministic_mock_provider() {
510        let provider = DeterministicMockProvider::new();
511
512        let request = CompletionRequest::with_system(
513            "You are a CLI expert",
514            "Generate examples for:\n- **Name**: apply\n- **Description**: Apply manifest"
515        );
516
517        let response = provider.complete(&request).await.unwrap();
518        assert!(response.content.contains("deployment.yaml"));
519        assert_eq!(provider.call_count(), 1);
520    }
521
522    #[tokio::test]
523    async fn test_failing_provider() {
524        let provider = FailingMockProvider::new("Test error");
525        let request = CompletionRequest::new("test");
526
527        let result = provider.complete(&request).await;
528        assert!(result.is_err());
529        assert!(result.unwrap_err().to_string().contains("Test error"));
530    }
531}
skill_runtime/generation/fixtures.rs

skill_runtime/generation/
fixtures.rs
