llm_test_bench_core/plugins/
api.rs1use serde::{Deserialize, Serialize};
10use async_trait::async_trait;
11use anyhow::Result;
12
13#[derive(Debug, Clone, Serialize, Deserialize)]
15pub struct PluginInput {
16 pub data: serde_json::Value,
18
19 #[serde(default)]
21 pub context: PluginContext,
22}
23
24#[derive(Debug, Clone, Default, Serialize, Deserialize)]
26pub struct PluginContext {
27 pub request_id: Option<String>,
29
30 #[serde(default)]
32 pub metadata: std::collections::HashMap<String, String>,
33}
34
35#[derive(Debug, Clone, Serialize, Deserialize)]
37pub struct PluginOutput {
38 pub data: serde_json::Value,
40
41 #[serde(default)]
43 pub metadata: OutputMetadata,
44}
45
46#[derive(Debug, Clone, Default, Serialize, Deserialize)]
48pub struct OutputMetadata {
49 pub execution_time_ms: u64,
51
52 pub memory_used_bytes: Option<usize>,
54
55 #[serde(default)]
57 pub metrics: std::collections::HashMap<String, f64>,
58}
59
60pub type PluginResult<T> = Result<T, PluginError>;
62
63#[derive(Debug, Clone, Serialize, Deserialize)]
65pub struct PluginError {
66 pub message: String,
67 pub code: String,
68}
69
70impl std::fmt::Display for PluginError {
71 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
72 write!(f, "{}: {}", self.code, self.message)
73 }
74}
75
76impl std::error::Error for PluginError {}
77
78#[async_trait]
80pub trait PluginApi: Send + Sync {
81 fn name(&self) -> &str;
83
84 fn version(&self) -> &str;
86
87 async fn initialize(&mut self, config: serde_json::Value) -> Result<()>;
89
90 async fn execute(&self, input: PluginInput) -> Result<PluginOutput>;
92
93 async fn shutdown(&mut self) -> Result<()>;
95}
96
97#[async_trait]
99pub trait EvaluatorPlugin: PluginApi {
100 async fn evaluate(
102 &self,
103 input: &str,
104 output: &str,
105 expected: Option<&str>,
106 ) -> Result<EvaluationResult>;
107}
108
109#[derive(Debug, Clone, Serialize, Deserialize)]
111pub struct EvaluationResult {
112 pub score: f64,
114
115 #[serde(default)]
117 pub metrics: std::collections::HashMap<String, f64>,
118
119 pub explanation: Option<String>,
121
122 pub confidence: Option<f64>,
124}
125
126#[async_trait]
128pub trait ProviderPlugin: PluginApi {
129 async fn complete(&self, request: CompletionRequest) -> Result<CompletionResponse>;
131
132 fn supported_models(&self) -> Vec<String>;
134}
135
136#[derive(Debug, Clone, Serialize, Deserialize)]
138pub struct CompletionRequest {
139 pub model: String,
141
142 pub prompt: String,
144
145 pub max_tokens: Option<usize>,
147
148 pub temperature: Option<f32>,
150
151 pub top_p: Option<f32>,
153
154 #[serde(default)]
156 pub stop: Vec<String>,
157}
158
159#[derive(Debug, Clone, Serialize, Deserialize)]
161pub struct CompletionResponse {
162 pub text: String,
164
165 pub usage: Option<TokenUsage>,
167
168 pub model: String,
170
171 pub finish_reason: Option<String>,
173}
174
175#[derive(Debug, Clone, Serialize, Deserialize)]
177pub struct TokenUsage {
178 pub prompt_tokens: u64,
179 pub completion_tokens: u64,
180 pub total_tokens: u64,
181}
182
183#[async_trait]
185pub trait TransformPlugin: PluginApi {
186 async fn transform(&self, data: serde_json::Value) -> Result<serde_json::Value>;
188}
189
190#[async_trait]
192pub trait FilterPlugin: PluginApi {
193 async fn filter(&self, items: Vec<serde_json::Value>) -> Result<Vec<serde_json::Value>>;
195}
196
197pub mod wasm_interface {
199 use super::*;
200
201 pub const PLUGIN_INIT: &str = "plugin_init";
204
205 pub const PLUGIN_EXECUTE: &str = "plugin_execute";
208
209 pub const PLUGIN_SHUTDOWN: &str = "plugin_shutdown";
212
213 pub const PLUGIN_METADATA: &str = "plugin_metadata";
216
217 pub const PLUGIN_ALLOC: &str = "plugin_alloc";
220
221 pub const PLUGIN_FREE: &str = "plugin_free";
224
225 pub const RESULT_OK: i32 = 0;
227 pub const RESULT_ERROR: i32 = -1;
228 pub const RESULT_INVALID_INPUT: i32 = -2;
229 pub const RESULT_TIMEOUT: i32 = -3;
230 pub const RESULT_OUT_OF_MEMORY: i32 = -4;
231}
232
233#[cfg(test)]
234mod tests {
235 use super::*;
236
237 #[test]
238 fn test_plugin_input() {
239 let input = PluginInput {
240 data: serde_json::json!({"key": "value"}),
241 context: PluginContext::default(),
242 };
243
244 assert!(input.data.is_object());
245 }
246
247 #[test]
248 fn test_evaluation_result() {
249 let result = EvaluationResult {
250 score: 0.85,
251 metrics: [("accuracy".to_string(), 0.9)].iter().cloned().collect(),
252 explanation: Some("Good result".to_string()),
253 confidence: Some(0.95),
254 };
255
256 assert_eq!(result.score, 0.85);
257 assert_eq!(result.metrics.get("accuracy"), Some(&0.9));
258 }
259
260 #[test]
261 fn test_completion_request() {
262 let request = CompletionRequest {
263 model: "gpt-4".to_string(),
264 prompt: "Hello, world!".to_string(),
265 max_tokens: Some(100),
266 temperature: Some(0.7),
267 top_p: Some(0.9),
268 stop: vec![],
269 };
270
271 assert_eq!(request.model, "gpt-4");
272 assert_eq!(request.max_tokens, Some(100));
273 }
274}