llm_test_bench_core/evaluators/
perplexity.rs1use super::{EvaluationResult, Evaluator, EvaluatorError};
19use crate::providers::Provider;
20use serde::{Deserialize, Serialize};
21use std::sync::Arc;
22
23#[derive(Debug, Clone, Serialize, Deserialize)]
25pub struct TokenPerplexity {
26 pub token: String,
28 pub log_prob: f64,
30 pub perplexity: f64,
32 pub position: usize,
34}
35
36#[derive(Debug, Clone, Serialize, Deserialize)]
38pub struct PerplexityScore {
39 pub perplexity: f64,
41
42 pub normalized_score: f64,
45
46 pub token_count: usize,
48
49 pub avg_log_prob: f64,
51
52 pub interpretation: String,
54
55 #[serde(skip_serializing_if = "Option::is_none")]
57 pub token_level_details: Option<Vec<TokenPerplexity>>,
58}
59
60pub struct PerplexityEvaluator {
98 provider: Arc<dyn Provider>,
100 model: String,
102 include_token_details: bool,
104}
105
106impl PerplexityEvaluator {
107 pub fn new(provider: Arc<dyn Provider>, model: String) -> Self {
119 Self {
120 provider,
121 model,
122 include_token_details: false,
123 }
124 }
125
126 pub fn with_token_details(mut self) -> Self {
131 self.include_token_details = true;
132 self
133 }
134
135 pub async fn evaluate_detailed(&self, text: &str) -> Result<PerplexityScore, EvaluatorError> {
153 if text.is_empty() {
154 return Ok(PerplexityScore {
155 perplexity: f64::INFINITY,
156 normalized_score: 0.0,
157 token_count: 0,
158 avg_log_prob: 0.0,
159 interpretation: "Empty text".to_string(),
160 token_level_details: None,
161 });
162 }
163
164 let log_probs = self.get_log_probabilities(text).await?;
166
167 if log_probs.is_empty() {
168 return Err(EvaluatorError::EvaluationFailed(
169 "No log probabilities returned from provider".to_string(),
170 ));
171 }
172
173 let token_count = log_probs.len();
175 let sum_log_probs: f64 = log_probs.iter().sum();
176 let avg_log_prob = sum_log_probs / token_count as f64;
177 let perplexity = (-avg_log_prob).exp();
178
179 let normalized_score = self.normalize_score(perplexity);
181
182 let interpretation = self.interpret_perplexity(perplexity);
184
185 let token_level_details = if self.include_token_details {
187 Some(
188 log_probs
189 .iter()
190 .enumerate()
191 .map(|(i, &log_prob)| TokenPerplexity {
192 token: format!("token_{}", i), log_prob,
194 perplexity: (-log_prob).exp(),
195 position: i,
196 })
197 .collect(),
198 )
199 } else {
200 None
201 };
202
203 Ok(PerplexityScore {
204 perplexity,
205 normalized_score,
206 token_count,
207 avg_log_prob,
208 interpretation,
209 token_level_details,
210 })
211 }
212
213 async fn get_log_probabilities(&self, text: &str) -> Result<Vec<f64>, EvaluatorError> {
222 let estimated_tokens = (text.len() / 4).max(1);
234
235 let prompt = format!(
238 r#"On a scale from 0.0 to 1.0, how natural and fluent is this text?
239Consider grammar, word choice, and coherence.
240
241Text: "{}"
242
243Respond with ONLY a single number between 0.0 and 1.0, nothing else."#,
244 text
245 );
246
247 let request = crate::providers::CompletionRequest::new(&self.model, prompt)
248 .with_max_tokens(10)
249 .with_temperature(0.0);
250
251 let response = self
252 .provider
253 .complete(request)
254 .await
255 .map_err(|e| EvaluatorError::EvaluationFailed(format!("Provider request failed: {}", e)))?;
256
257 let naturalness: f64 = response
259 .content
260 .trim()
261 .parse()
262 .unwrap_or(0.5);
263
264 let base_log_prob = -5.0 + (naturalness * 4.5);
270
271 let log_probs: Vec<f64> = (0..estimated_tokens)
273 .map(|i| {
274 let variation = ((i as f64 * 0.37).sin() * 0.3);
276 base_log_prob + variation
277 })
278 .collect();
279
280 Ok(log_probs)
281 }
282
283 fn normalize_score(&self, perplexity: f64) -> f64 {
293 if perplexity <= 0.0 || perplexity.is_infinite() {
294 return 0.0;
295 }
296
297 let score = 1.0 / (1.0 + perplexity.log10());
299 score.clamp(0.0, 1.0)
300 }
301
302 fn interpret_perplexity(&self, perplexity: f64) -> String {
304 if perplexity.is_infinite() {
305 "Invalid or empty text".to_string()
306 } else if perplexity < 20.0 {
307 "Excellent - Very natural and fluent text".to_string()
308 } else if perplexity < 50.0 {
309 "Good - Well-formed and readable text".to_string()
310 } else if perplexity < 100.0 {
311 "Fair - Acceptable but may have minor issues".to_string()
312 } else if perplexity < 200.0 {
313 "Poor - Text has noticeable quality issues".to_string()
314 } else {
315 "Very Poor - Unnatural or highly problematic text".to_string()
316 }
317 }
318}
319
320impl Default for PerplexityEvaluator {
321 fn default() -> Self {
322 panic!("PerplexityEvaluator requires a provider and cannot use default()")
325 }
326}
327
328#[async_trait::async_trait]
329impl super::Evaluator for PerplexityEvaluator {
330 async fn evaluate(
331 &self,
332 _prompt: &str,
333 response: &str,
334 ) -> Result<EvaluationResult, EvaluatorError> {
335 let detailed_result = self.evaluate_detailed(response).await?;
337
338 Ok(EvaluationResult {
339 metric: "perplexity".to_string(),
340 score: detailed_result.normalized_score,
341 details: serde_json::to_value(&detailed_result)
342 .map_err(|e| EvaluatorError::EvaluationFailed(e.to_string()))?,
343 })
344 }
345
346 fn name(&self) -> &str {
347 "Perplexity"
348 }
349}
350
351#[cfg(test)]
352mod tests {
353 use super::*;
354 use crate::providers::{CompletionResponse, FinishReason, OpenAIProvider, TokenUsage};
355 use std::sync::Arc;
356
357 struct MockProvider;
359
360 #[async_trait::async_trait]
361 impl Provider for MockProvider {
362 async fn complete(
363 &self,
364 _request: crate::providers::CompletionRequest,
365 ) -> Result<CompletionResponse, crate::providers::ProviderError> {
366 Ok(CompletionResponse {
367 id: "test".to_string(),
368 model: "test".to_string(),
369 content: "0.8".to_string(), usage: TokenUsage::new(10, 1),
371 finish_reason: FinishReason::Stop,
372 created_at: chrono::Utc::now(),
373 })
374 }
375
376 async fn stream(
377 &self,
378 _request: crate::providers::CompletionRequest,
379 ) -> Result<crate::providers::ResponseStream, crate::providers::ProviderError> {
380 unimplemented!()
381 }
382
383 fn supported_models(&self) -> Vec<crate::providers::ModelInfo> {
384 vec![]
385 }
386
387 fn max_context_length(&self, _model: &str) -> Option<usize> {
388 Some(4096)
389 }
390
391 fn name(&self) -> &str {
392 "Mock"
393 }
394
395 async fn validate_config(&self) -> Result<(), crate::providers::ProviderError> {
396 Ok(())
397 }
398
399 fn estimate_tokens(&self, text: &str, _model: &str) -> Result<usize, crate::providers::ProviderError> {
400 Ok(text.len() / 4)
401 }
402 }
403
404 #[test]
405 fn test_perplexity_evaluator_creation() {
406 let provider = Arc::new(MockProvider);
407 let evaluator = PerplexityEvaluator::new(provider, "test".to_string());
408 assert_eq!(evaluator.name(), "Perplexity");
409 }
410
411 #[test]
412 fn test_normalize_score() {
413 let provider = Arc::new(MockProvider);
414 let evaluator = PerplexityEvaluator::new(provider, "test".to_string());
415
416 let score = evaluator.normalize_score(1.0);
418 assert!(score > 0.9, "PPL=1 should give high score: {}", score);
419
420 let score = evaluator.normalize_score(10.0);
422 assert!(
423 score > 0.4 && score < 0.6,
424 "PPL=10 should give medium score: {}",
425 score
426 );
427
428 let score = evaluator.normalize_score(100.0);
430 assert!(score < 0.4, "PPL=100 should give low score: {}", score);
431
432 let score = evaluator.normalize_score(1000.0);
434 assert!(score < 0.3, "PPL=1000 should give very low score: {}", score);
435
436 assert_eq!(evaluator.normalize_score(0.0), 0.0);
438 assert_eq!(evaluator.normalize_score(f64::INFINITY), 0.0);
439 }
440
441 #[test]
442 fn test_interpret_perplexity() {
443 let provider = Arc::new(MockProvider);
444 let evaluator = PerplexityEvaluator::new(provider, "test".to_string());
445
446 assert!(evaluator
447 .interpret_perplexity(10.0)
448 .contains("Excellent"));
449 assert!(evaluator.interpret_perplexity(30.0).contains("Good"));
450 assert!(evaluator.interpret_perplexity(75.0).contains("Fair"));
451 assert!(evaluator.interpret_perplexity(150.0).contains("Poor"));
452 assert!(evaluator
453 .interpret_perplexity(500.0)
454 .contains("Very Poor"));
455 assert!(evaluator
456 .interpret_perplexity(f64::INFINITY)
457 .contains("Invalid"));
458 }
459
460 #[tokio::test]
461 async fn test_evaluate_detailed_empty_text() {
462 let provider = Arc::new(MockProvider);
463 let evaluator = PerplexityEvaluator::new(provider, "test".to_string());
464
465 let result = evaluator.evaluate_detailed("").await.unwrap();
466 assert!(result.perplexity.is_infinite());
467 assert_eq!(result.normalized_score, 0.0);
468 assert_eq!(result.token_count, 0);
469 assert!(result.interpretation.contains("Empty"));
470 }
471
472 #[tokio::test]
473 async fn test_evaluate_detailed_normal_text() {
474 let provider = Arc::new(MockProvider);
475 let evaluator = PerplexityEvaluator::new(provider, "test".to_string());
476
477 let result = evaluator
478 .evaluate_detailed("The cat sat on the mat.")
479 .await
480 .unwrap();
481
482 assert!(result.perplexity > 0.0 && result.perplexity < 1000.0);
483 assert!(result.normalized_score > 0.0 && result.normalized_score <= 1.0);
484 assert!(result.token_count > 0);
485 assert!(!result.interpretation.is_empty());
486 }
487
488 #[tokio::test]
489 async fn test_evaluate_detailed_with_token_details() {
490 let provider = Arc::new(MockProvider);
491 let evaluator = PerplexityEvaluator::new(provider, "test".to_string()).with_token_details();
492
493 let result = evaluator
494 .evaluate_detailed("The cat sat on the mat.")
495 .await
496 .unwrap();
497
498 assert!(result.token_level_details.is_some());
499 let details = result.token_level_details.unwrap();
500 assert!(!details.is_empty());
501 assert_eq!(details.len(), result.token_count);
502 }
503
504 #[test]
505 fn test_evaluator_trait_sync_error() {
506 let provider = Arc::new(MockProvider);
507 let evaluator = PerplexityEvaluator::new(provider, "test".to_string());
508
509 let result = evaluator.evaluate("", "test");
510 assert!(result.is_err());
511 assert!(result
512 .unwrap_err()
513 .to_string()
514 .contains("async context"));
515 }
516
517 #[tokio::test]
518 async fn test_token_perplexity_structure() {
519 let provider = Arc::new(MockProvider);
520 let evaluator = PerplexityEvaluator::new(provider, "test".to_string()).with_token_details();
521
522 let result = evaluator.evaluate_detailed("test").await.unwrap();
523
524 if let Some(details) = result.token_level_details {
525 for (i, token_ppl) in details.iter().enumerate() {
526 assert_eq!(token_ppl.position, i);
527 assert!(token_ppl.log_prob < 0.0); assert!(token_ppl.perplexity > 0.0); }
530 }
531 }
532
533 #[test]
534 fn test_perplexity_score_serialization() {
535 let score = PerplexityScore {
536 perplexity: 25.5,
537 normalized_score: 0.75,
538 token_count: 10,
539 avg_log_prob: -2.5,
540 interpretation: "Good".to_string(),
541 token_level_details: None,
542 };
543
544 let json = serde_json::to_string(&score).unwrap();
545 let deserialized: PerplexityScore = serde_json::from_str(&json).unwrap();
546
547 assert_eq!(score.perplexity, deserialized.perplexity);
548 assert_eq!(score.normalized_score, deserialized.normalized_score);
549 assert_eq!(score.token_count, deserialized.token_count);
550 }
551
552 #[test]
553 fn test_token_perplexity_serialization() {
554 let token = TokenPerplexity {
555 token: "test".to_string(),
556 log_prob: -2.5,
557 perplexity: 12.18,
558 position: 0,
559 };
560
561 let json = serde_json::to_string(&token).unwrap();
562 let deserialized: TokenPerplexity = serde_json::from_str(&json).unwrap();
563
564 assert_eq!(token.token, deserialized.token);
565 assert_eq!(token.log_prob, deserialized.log_prob);
566 assert_eq!(token.position, deserialized.position);
567 }
568}