Skip to main content

llm_test_bench_core/plugins/
api.rs

1// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
2// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
3// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
4// option. This file may not be copied, modified, or distributed
5// except according to those terms.
6
7//! Plugin API definitions and traits.
8
9use serde::{Deserialize, Serialize};
10use async_trait::async_trait;
11use anyhow::Result;
12
13/// Plugin input data
14#[derive(Debug, Clone, Serialize, Deserialize)]
15pub struct PluginInput {
16    /// Input data as JSON
17    pub data: serde_json::Value,
18
19    /// Additional context
20    #[serde(default)]
21    pub context: PluginContext,
22}
23
24/// Plugin execution context
25#[derive(Debug, Clone, Default, Serialize, Deserialize)]
26pub struct PluginContext {
27    /// Request ID for tracking
28    pub request_id: Option<String>,
29
30    /// Additional metadata
31    #[serde(default)]
32    pub metadata: std::collections::HashMap<String, String>,
33}
34
35/// Plugin output data
36#[derive(Debug, Clone, Serialize, Deserialize)]
37pub struct PluginOutput {
38    /// Output data as JSON
39    pub data: serde_json::Value,
40
41    /// Execution metadata
42    #[serde(default)]
43    pub metadata: OutputMetadata,
44}
45
46/// Output metadata
47#[derive(Debug, Clone, Default, Serialize, Deserialize)]
48pub struct OutputMetadata {
49    /// Execution time in milliseconds
50    pub execution_time_ms: u64,
51
52    /// Memory used in bytes
53    pub memory_used_bytes: Option<usize>,
54
55    /// Additional metrics
56    #[serde(default)]
57    pub metrics: std::collections::HashMap<String, f64>,
58}
59
60/// Plugin result type
61pub type PluginResult<T> = Result<T, PluginError>;
62
63/// Plugin error (distinct from PluginError in types.rs for API-level errors)
64#[derive(Debug, Clone, Serialize, Deserialize)]
65pub struct PluginError {
66    pub message: String,
67    pub code: String,
68}
69
70impl std::fmt::Display for PluginError {
71    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
72        write!(f, "{}: {}", self.code, self.message)
73    }
74}
75
76impl std::error::Error for PluginError {}
77
78/// Base plugin API trait
79#[async_trait]
80pub trait PluginApi: Send + Sync {
81    /// Get plugin name
82    fn name(&self) -> &str;
83
84    /// Get plugin version
85    fn version(&self) -> &str;
86
87    /// Initialize the plugin
88    async fn initialize(&mut self, config: serde_json::Value) -> Result<()>;
89
90    /// Execute the plugin
91    async fn execute(&self, input: PluginInput) -> Result<PluginOutput>;
92
93    /// Shutdown the plugin
94    async fn shutdown(&mut self) -> Result<()>;
95}
96
97/// Evaluator plugin trait
98#[async_trait]
99pub trait EvaluatorPlugin: PluginApi {
100    /// Evaluate an LLM output
101    async fn evaluate(
102        &self,
103        input: &str,
104        output: &str,
105        expected: Option<&str>,
106    ) -> Result<EvaluationResult>;
107}
108
109/// Evaluation result
110#[derive(Debug, Clone, Serialize, Deserialize)]
111pub struct EvaluationResult {
112    /// Evaluation score (0.0 to 1.0)
113    pub score: f64,
114
115    /// Detailed scores by metric
116    #[serde(default)]
117    pub metrics: std::collections::HashMap<String, f64>,
118
119    /// Explanation of the evaluation
120    pub explanation: Option<String>,
121
122    /// Confidence in the evaluation (0.0 to 1.0)
123    pub confidence: Option<f64>,
124}
125
126/// Provider plugin trait
127#[async_trait]
128pub trait ProviderPlugin: PluginApi {
129    /// Complete a prompt
130    async fn complete(&self, request: CompletionRequest) -> Result<CompletionResponse>;
131
132    /// Get supported models
133    fn supported_models(&self) -> Vec<String>;
134}
135
136/// Completion request
137#[derive(Debug, Clone, Serialize, Deserialize)]
138pub struct CompletionRequest {
139    /// Model identifier
140    pub model: String,
141
142    /// Input prompt or messages
143    pub prompt: String,
144
145    /// Maximum tokens to generate
146    pub max_tokens: Option<usize>,
147
148    /// Temperature (0.0 to 2.0)
149    pub temperature: Option<f32>,
150
151    /// Top-p sampling
152    pub top_p: Option<f32>,
153
154    /// Stop sequences
155    #[serde(default)]
156    pub stop: Vec<String>,
157}
158
159/// Completion response
160#[derive(Debug, Clone, Serialize, Deserialize)]
161pub struct CompletionResponse {
162    /// Generated text
163    pub text: String,
164
165    /// Token usage
166    pub usage: Option<TokenUsage>,
167
168    /// Model used
169    pub model: String,
170
171    /// Finish reason
172    pub finish_reason: Option<String>,
173}
174
175/// Token usage information
176#[derive(Debug, Clone, Serialize, Deserialize)]
177pub struct TokenUsage {
178    pub prompt_tokens: u64,
179    pub completion_tokens: u64,
180    pub total_tokens: u64,
181}
182
183/// Transform plugin trait
184#[async_trait]
185pub trait TransformPlugin: PluginApi {
186    /// Transform input data
187    async fn transform(&self, data: serde_json::Value) -> Result<serde_json::Value>;
188}
189
190/// Filter plugin trait
191#[async_trait]
192pub trait FilterPlugin: PluginApi {
193    /// Filter results based on criteria
194    async fn filter(&self, items: Vec<serde_json::Value>) -> Result<Vec<serde_json::Value>>;
195}
196
197/// WASM plugin interface (exported from WASM module)
198pub mod wasm_interface {
199    use super::*;
200
201    /// Plugin initialization function signature
202    /// extern "C" fn plugin_init(config_ptr: *const u8, config_len: usize) -> i32
203    pub const PLUGIN_INIT: &str = "plugin_init";
204
205    /// Plugin execution function signature
206    /// extern "C" fn plugin_execute(input_ptr: *const u8, input_len: usize, output_ptr: *mut u8, output_len: *mut usize) -> i32
207    pub const PLUGIN_EXECUTE: &str = "plugin_execute";
208
209    /// Plugin shutdown function signature
210    /// extern "C" fn plugin_shutdown() -> i32
211    pub const PLUGIN_SHUTDOWN: &str = "plugin_shutdown";
212
213    /// Get plugin metadata function signature
214    /// extern "C" fn plugin_metadata(output_ptr: *mut u8, output_len: *mut usize) -> i32
215    pub const PLUGIN_METADATA: &str = "plugin_metadata";
216
217    /// Memory allocation function (for host to allocate in plugin)
218    /// extern "C" fn plugin_alloc(size: usize) -> *mut u8
219    pub const PLUGIN_ALLOC: &str = "plugin_alloc";
220
221    /// Memory deallocation function
222    /// extern "C" fn plugin_free(ptr: *mut u8, size: usize)
223    pub const PLUGIN_FREE: &str = "plugin_free";
224
225    /// Result codes
226    pub const RESULT_OK: i32 = 0;
227    pub const RESULT_ERROR: i32 = -1;
228    pub const RESULT_INVALID_INPUT: i32 = -2;
229    pub const RESULT_TIMEOUT: i32 = -3;
230    pub const RESULT_OUT_OF_MEMORY: i32 = -4;
231}
232
233#[cfg(test)]
234mod tests {
235    use super::*;
236
237    #[test]
238    fn test_plugin_input() {
239        let input = PluginInput {
240            data: serde_json::json!({"key": "value"}),
241            context: PluginContext::default(),
242        };
243
244        assert!(input.data.is_object());
245    }
246
247    #[test]
248    fn test_evaluation_result() {
249        let result = EvaluationResult {
250            score: 0.85,
251            metrics: [("accuracy".to_string(), 0.9)].iter().cloned().collect(),
252            explanation: Some("Good result".to_string()),
253            confidence: Some(0.95),
254        };
255
256        assert_eq!(result.score, 0.85);
257        assert_eq!(result.metrics.get("accuracy"), Some(&0.9));
258    }
259
260    #[test]
261    fn test_completion_request() {
262        let request = CompletionRequest {
263            model: "gpt-4".to_string(),
264            prompt: "Hello, world!".to_string(),
265            max_tokens: Some(100),
266            temperature: Some(0.7),
267            top_p: Some(0.9),
268            stop: vec![],
269        };
270
271        assert_eq!(request.model, "gpt-4");
272        assert_eq!(request.max_tokens, Some(100));
273    }
274}