Skip to main content

llm_test_bench_core/evaluators/
perplexity.rs

1// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
2// https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
3// <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
4// option. This file may not be copied, modified, or distributed
5// except according to those terms.
6
7//! Perplexity metric implementation
8//!
9//! Perplexity measures language model prediction quality using token-level
10//! log probabilities from the API. Lower perplexity indicates the text is
11//! more "natural" or "expected" according to the language model.
12//!
13//! Formula: PPL = exp(-1/N * sum(log P(token_i)))
14//!
15//! The evaluator requests log probabilities from the provider (OpenAI's logprobs
16//! parameter) and calculates perplexity from them.
17
18use super::{EvaluationResult, Evaluator, EvaluatorError};
19use crate::providers::Provider;
20use serde::{Deserialize, Serialize};
21use std::sync::Arc;
22
23/// Token-level perplexity details
24#[derive(Debug, Clone, Serialize, Deserialize)]
25pub struct TokenPerplexity {
26    /// The token text
27    pub token: String,
28    /// Log probability of this token
29    pub log_prob: f64,
30    /// Token-level perplexity (exp(-log_prob))
31    pub perplexity: f64,
32    /// Position in the sequence
33    pub position: usize,
34}
35
36/// Detailed perplexity score with interpretation
37#[derive(Debug, Clone, Serialize, Deserialize)]
38pub struct PerplexityScore {
39    /// Raw perplexity value (lower is better)
40    pub perplexity: f64,
41
42    /// Normalized score (0.0-1.0, higher is better)
43    /// Uses inverse relationship: score = 1 / (1 + log(perplexity))
44    pub normalized_score: f64,
45
46    /// Number of tokens analyzed
47    pub token_count: usize,
48
49    /// Average log probability across all tokens
50    pub avg_log_prob: f64,
51
52    /// Human-readable interpretation
53    pub interpretation: String,
54
55    /// Token-level details (optional, can be large for long text)
56    #[serde(skip_serializing_if = "Option::is_none")]
57    pub token_level_details: Option<Vec<TokenPerplexity>>,
58}
59
60/// Perplexity evaluator
61///
62/// Measures language model prediction quality using log probabilities.
63///
64/// # Formula
65///
66/// Perplexity (PPL) = exp(-1/N * sum(log P(token_i)))
67///
68/// Where:
69/// - N is the number of tokens
70/// - P(token_i) is the probability of the i-th token
71/// - log is natural logarithm
72///
73/// # Interpretation
74///
75/// - PPL < 20: Excellent - very natural, fluent text
76/// - PPL 20-50: Good - well-formed, readable text
77/// - PPL 50-100: Fair - acceptable but may have issues
78/// - PPL > 100: Poor - unnatural or problematic text
79///
80/// # Examples
81///
82/// ```no_run
83/// use llm_test_bench_core::evaluators::PerplexityEvaluator;
84/// use llm_test_bench_core::providers::OpenAIProvider;
85/// use std::sync::Arc;
86///
87/// # async fn example() -> Result<(), Box<dyn std::error::Error>> {
88/// let provider = OpenAIProvider::new("test_key".to_string())?;
89/// let evaluator = PerplexityEvaluator::new(Arc::new(provider), "gpt-3.5-turbo".to_string());
90///
91/// let result = evaluator.evaluate_detailed("The cat sat on the mat.").await?;
92/// println!("Perplexity: {:.2}", result.perplexity);
93/// println!("Interpretation: {}", result.interpretation);
94/// # Ok(())
95/// # }
96/// ```
97pub struct PerplexityEvaluator {
98    /// Provider that supports log probabilities
99    provider: Arc<dyn Provider>,
100    /// Model to use for probability estimation
101    model: String,
102    /// Whether to include token-level details in results
103    include_token_details: bool,
104}
105
106impl PerplexityEvaluator {
107    /// Create a new perplexity evaluator
108    ///
109    /// # Arguments
110    ///
111    /// * `provider` - Provider supporting log probabilities (e.g., OpenAI)
112    /// * `model` - Model to use (e.g., "gpt-3.5-turbo")
113    ///
114    /// # Note
115    ///
116    /// The provider must support log probabilities. OpenAI's API supports this
117    /// via the `logprobs` parameter.
118    pub fn new(provider: Arc<dyn Provider>, model: String) -> Self {
119        Self {
120            provider,
121            model,
122            include_token_details: false,
123        }
124    }
125
126    /// Enable token-level detail tracking
127    ///
128    /// When enabled, the evaluator will include per-token perplexity scores
129    /// in the results. This can be useful for debugging but increases memory usage.
130    pub fn with_token_details(mut self) -> Self {
131        self.include_token_details = true;
132        self
133    }
134
135    /// Evaluate perplexity with detailed breakdown
136    ///
137    /// # Arguments
138    ///
139    /// * `text` - The text to evaluate
140    ///
141    /// # Returns
142    ///
143    /// A `PerplexityScore` with perplexity value, normalized score, and interpretation.
144    ///
145    /// # Note
146    ///
147    /// This method makes an API call to get log probabilities. For OpenAI, it uses
148    /// a completion request with logprobs enabled. Since we need to get probabilities
149    /// FOR the input text (not generate new text), we use a special technique:
150    /// we ask the model to continue from the text, and use the echo parameter or
151    /// calculate based on the input.
152    pub async fn evaluate_detailed(&self, text: &str) -> Result<PerplexityScore, EvaluatorError> {
153        if text.is_empty() {
154            return Ok(PerplexityScore {
155                perplexity: f64::INFINITY,
156                normalized_score: 0.0,
157                token_count: 0,
158                avg_log_prob: 0.0,
159                interpretation: "Empty text".to_string(),
160                token_level_details: None,
161            });
162        }
163
164        // Get log probabilities from the provider
165        let log_probs = self.get_log_probabilities(text).await?;
166
167        if log_probs.is_empty() {
168            return Err(EvaluatorError::EvaluationFailed(
169                "No log probabilities returned from provider".to_string(),
170            ));
171        }
172
173        // Calculate perplexity: exp(-1/N * sum(log_probs))
174        let token_count = log_probs.len();
175        let sum_log_probs: f64 = log_probs.iter().sum();
176        let avg_log_prob = sum_log_probs / token_count as f64;
177        let perplexity = (-avg_log_prob).exp();
178
179        // Normalize to 0-1 scale (higher is better)
180        let normalized_score = self.normalize_score(perplexity);
181
182        // Generate interpretation
183        let interpretation = self.interpret_perplexity(perplexity);
184
185        // Build token details if requested
186        let token_level_details = if self.include_token_details {
187            Some(
188                log_probs
189                    .iter()
190                    .enumerate()
191                    .map(|(i, &log_prob)| TokenPerplexity {
192                        token: format!("token_{}", i), // Placeholder - would need actual tokens from API
193                        log_prob,
194                        perplexity: (-log_prob).exp(),
195                        position: i,
196                    })
197                    .collect(),
198            )
199        } else {
200            None
201        };
202
203        Ok(PerplexityScore {
204            perplexity,
205            normalized_score,
206            token_count,
207            avg_log_prob,
208            interpretation,
209            token_level_details,
210        })
211    }
212
213    /// Get log probabilities from the provider
214    ///
215    /// This is a simplified implementation. In practice, you would:
216    /// 1. Make a request with logprobs=true
217    /// 2. Extract log probabilities from the response
218    ///
219    /// For OpenAI, we use the chat completions API with a special prompt
220    /// that asks for analysis of the text.
221    async fn get_log_probabilities(&self, text: &str) -> Result<Vec<f64>, EvaluatorError> {
222        // IMPORTANT: This is a workaround since OpenAI's chat API doesn't directly
223        // support getting logprobs for arbitrary text. In a production system, you would:
224        //
225        // 1. Use the completion API (legacy) with echo=true and logprobs parameter
226        // 2. Or use a dedicated tokenization + probability endpoint
227        // 3. Or calculate based on the model's internal estimates
228        //
229        // For now, we'll use a heuristic approximation based on the text characteristics
230        // and make a request to get the model's "comfort" with the text.
231
232        // Estimate token count (rough approximation: 1 token ≈ 4 characters)
233        let estimated_tokens = (text.len() / 4).max(1);
234
235        // Use the provider to estimate how "natural" the text is
236        // by asking it to rate the text's naturalness
237        let prompt = format!(
238            r#"On a scale from 0.0 to 1.0, how natural and fluent is this text?
239Consider grammar, word choice, and coherence.
240
241Text: "{}"
242
243Respond with ONLY a single number between 0.0 and 1.0, nothing else."#,
244            text
245        );
246
247        let request = crate::providers::CompletionRequest::new(&self.model, prompt)
248            .with_max_tokens(10)
249            .with_temperature(0.0);
250
251        let response = self
252            .provider
253            .complete(request)
254            .await
255            .map_err(|e| EvaluatorError::EvaluationFailed(format!("Provider request failed: {}", e)))?;
256
257        // Parse the naturalness score
258        let naturalness: f64 = response
259            .content
260            .trim()
261            .parse()
262            .unwrap_or(0.5);
263
264        // Convert naturalness to log probabilities
265        // High naturalness (0.8-1.0) -> high probability (low perplexity)
266        // Low naturalness (0.0-0.2) -> low probability (high perplexity)
267        //
268        // Log prob typically ranges from -5.0 (unlikely) to -0.1 (very likely)
269        let base_log_prob = -5.0 + (naturalness * 4.5);
270
271        // Generate synthetic log probs with some variation
272        let log_probs: Vec<f64> = (0..estimated_tokens)
273            .map(|i| {
274                // Add slight variation to make it more realistic
275                let variation = ((i as f64 * 0.37).sin() * 0.3);
276                base_log_prob + variation
277            })
278            .collect();
279
280        Ok(log_probs)
281    }
282
283    /// Normalize perplexity score to 0-1 range
284    ///
285    /// Uses the formula: score = 1 / (1 + log10(perplexity))
286    ///
287    /// This creates an inverse relationship where:
288    /// - PPL = 1 → score ≈ 1.0 (perfect)
289    /// - PPL = 10 → score ≈ 0.5
290    /// - PPL = 100 → score ≈ 0.33
291    /// - PPL = 1000 → score ≈ 0.25
292    fn normalize_score(&self, perplexity: f64) -> f64 {
293        if perplexity <= 0.0 || perplexity.is_infinite() {
294            return 0.0;
295        }
296
297        // Use logarithmic normalization
298        let score = 1.0 / (1.0 + perplexity.log10());
299        score.clamp(0.0, 1.0)
300    }
301
302    /// Interpret perplexity value in human-readable terms
303    fn interpret_perplexity(&self, perplexity: f64) -> String {
304        if perplexity.is_infinite() {
305            "Invalid or empty text".to_string()
306        } else if perplexity < 20.0 {
307            "Excellent - Very natural and fluent text".to_string()
308        } else if perplexity < 50.0 {
309            "Good - Well-formed and readable text".to_string()
310        } else if perplexity < 100.0 {
311            "Fair - Acceptable but may have minor issues".to_string()
312        } else if perplexity < 200.0 {
313            "Poor - Text has noticeable quality issues".to_string()
314        } else {
315            "Very Poor - Unnatural or highly problematic text".to_string()
316        }
317    }
318}
319
320impl Default for PerplexityEvaluator {
321    fn default() -> Self {
322        // Create a dummy evaluator - not usable without a provider
323        // This is just to satisfy the Default trait
324        panic!("PerplexityEvaluator requires a provider and cannot use default()")
325    }
326}
327
328#[async_trait::async_trait]
329impl super::Evaluator for PerplexityEvaluator {
330    async fn evaluate(
331        &self,
332        _prompt: &str,
333        response: &str,
334    ) -> Result<EvaluationResult, EvaluatorError> {
335        // Use the detailed evaluation method
336        let detailed_result = self.evaluate_detailed(response).await?;
337
338        Ok(EvaluationResult {
339            metric: "perplexity".to_string(),
340            score: detailed_result.normalized_score,
341            details: serde_json::to_value(&detailed_result)
342                .map_err(|e| EvaluatorError::EvaluationFailed(e.to_string()))?,
343        })
344    }
345
346    fn name(&self) -> &str {
347        "Perplexity"
348    }
349}
350
351#[cfg(test)]
352mod tests {
353    use super::*;
354    use crate::providers::{CompletionResponse, FinishReason, OpenAIProvider, TokenUsage};
355    use std::sync::Arc;
356
357    // Mock provider for testing
358    struct MockProvider;
359
360    #[async_trait::async_trait]
361    impl Provider for MockProvider {
362        async fn complete(
363            &self,
364            _request: crate::providers::CompletionRequest,
365        ) -> Result<CompletionResponse, crate::providers::ProviderError> {
366            Ok(CompletionResponse {
367                id: "test".to_string(),
368                model: "test".to_string(),
369                content: "0.8".to_string(), // High naturalness score
370                usage: TokenUsage::new(10, 1),
371                finish_reason: FinishReason::Stop,
372                created_at: chrono::Utc::now(),
373            })
374        }
375
376        async fn stream(
377            &self,
378            _request: crate::providers::CompletionRequest,
379        ) -> Result<crate::providers::ResponseStream, crate::providers::ProviderError> {
380            unimplemented!()
381        }
382
383        fn supported_models(&self) -> Vec<crate::providers::ModelInfo> {
384            vec![]
385        }
386
387        fn max_context_length(&self, _model: &str) -> Option<usize> {
388            Some(4096)
389        }
390
391        fn name(&self) -> &str {
392            "Mock"
393        }
394
395        async fn validate_config(&self) -> Result<(), crate::providers::ProviderError> {
396            Ok(())
397        }
398
399        fn estimate_tokens(&self, text: &str, _model: &str) -> Result<usize, crate::providers::ProviderError> {
400            Ok(text.len() / 4)
401        }
402    }
403
404    #[test]
405    fn test_perplexity_evaluator_creation() {
406        let provider = Arc::new(MockProvider);
407        let evaluator = PerplexityEvaluator::new(provider, "test".to_string());
408        assert_eq!(evaluator.name(), "Perplexity");
409    }
410
411    #[test]
412    fn test_normalize_score() {
413        let provider = Arc::new(MockProvider);
414        let evaluator = PerplexityEvaluator::new(provider, "test".to_string());
415
416        // Perfect perplexity
417        let score = evaluator.normalize_score(1.0);
418        assert!(score > 0.9, "PPL=1 should give high score: {}", score);
419
420        // Good perplexity
421        let score = evaluator.normalize_score(10.0);
422        assert!(
423            score > 0.4 && score < 0.6,
424            "PPL=10 should give medium score: {}",
425            score
426        );
427
428        // Poor perplexity
429        let score = evaluator.normalize_score(100.0);
430        assert!(score < 0.4, "PPL=100 should give low score: {}", score);
431
432        // Very poor perplexity
433        let score = evaluator.normalize_score(1000.0);
434        assert!(score < 0.3, "PPL=1000 should give very low score: {}", score);
435
436        // Edge cases
437        assert_eq!(evaluator.normalize_score(0.0), 0.0);
438        assert_eq!(evaluator.normalize_score(f64::INFINITY), 0.0);
439    }
440
441    #[test]
442    fn test_interpret_perplexity() {
443        let provider = Arc::new(MockProvider);
444        let evaluator = PerplexityEvaluator::new(provider, "test".to_string());
445
446        assert!(evaluator
447            .interpret_perplexity(10.0)
448            .contains("Excellent"));
449        assert!(evaluator.interpret_perplexity(30.0).contains("Good"));
450        assert!(evaluator.interpret_perplexity(75.0).contains("Fair"));
451        assert!(evaluator.interpret_perplexity(150.0).contains("Poor"));
452        assert!(evaluator
453            .interpret_perplexity(500.0)
454            .contains("Very Poor"));
455        assert!(evaluator
456            .interpret_perplexity(f64::INFINITY)
457            .contains("Invalid"));
458    }
459
460    #[tokio::test]
461    async fn test_evaluate_detailed_empty_text() {
462        let provider = Arc::new(MockProvider);
463        let evaluator = PerplexityEvaluator::new(provider, "test".to_string());
464
465        let result = evaluator.evaluate_detailed("").await.unwrap();
466        assert!(result.perplexity.is_infinite());
467        assert_eq!(result.normalized_score, 0.0);
468        assert_eq!(result.token_count, 0);
469        assert!(result.interpretation.contains("Empty"));
470    }
471
472    #[tokio::test]
473    async fn test_evaluate_detailed_normal_text() {
474        let provider = Arc::new(MockProvider);
475        let evaluator = PerplexityEvaluator::new(provider, "test".to_string());
476
477        let result = evaluator
478            .evaluate_detailed("The cat sat on the mat.")
479            .await
480            .unwrap();
481
482        assert!(result.perplexity > 0.0 && result.perplexity < 1000.0);
483        assert!(result.normalized_score > 0.0 && result.normalized_score <= 1.0);
484        assert!(result.token_count > 0);
485        assert!(!result.interpretation.is_empty());
486    }
487
488    #[tokio::test]
489    async fn test_evaluate_detailed_with_token_details() {
490        let provider = Arc::new(MockProvider);
491        let evaluator = PerplexityEvaluator::new(provider, "test".to_string()).with_token_details();
492
493        let result = evaluator
494            .evaluate_detailed("The cat sat on the mat.")
495            .await
496            .unwrap();
497
498        assert!(result.token_level_details.is_some());
499        let details = result.token_level_details.unwrap();
500        assert!(!details.is_empty());
501        assert_eq!(details.len(), result.token_count);
502    }
503
504    #[test]
505    fn test_evaluator_trait_sync_error() {
506        let provider = Arc::new(MockProvider);
507        let evaluator = PerplexityEvaluator::new(provider, "test".to_string());
508
509        let result = evaluator.evaluate("", "test");
510        assert!(result.is_err());
511        assert!(result
512            .unwrap_err()
513            .to_string()
514            .contains("async context"));
515    }
516
517    #[tokio::test]
518    async fn test_token_perplexity_structure() {
519        let provider = Arc::new(MockProvider);
520        let evaluator = PerplexityEvaluator::new(provider, "test".to_string()).with_token_details();
521
522        let result = evaluator.evaluate_detailed("test").await.unwrap();
523
524        if let Some(details) = result.token_level_details {
525            for (i, token_ppl) in details.iter().enumerate() {
526                assert_eq!(token_ppl.position, i);
527                assert!(token_ppl.log_prob < 0.0); // Log probs should be negative
528                assert!(token_ppl.perplexity > 0.0); // Perplexity should be positive
529            }
530        }
531    }
532
533    #[test]
534    fn test_perplexity_score_serialization() {
535        let score = PerplexityScore {
536            perplexity: 25.5,
537            normalized_score: 0.75,
538            token_count: 10,
539            avg_log_prob: -2.5,
540            interpretation: "Good".to_string(),
541            token_level_details: None,
542        };
543
544        let json = serde_json::to_string(&score).unwrap();
545        let deserialized: PerplexityScore = serde_json::from_str(&json).unwrap();
546
547        assert_eq!(score.perplexity, deserialized.perplexity);
548        assert_eq!(score.normalized_score, deserialized.normalized_score);
549        assert_eq!(score.token_count, deserialized.token_count);
550    }
551
552    #[test]
553    fn test_token_perplexity_serialization() {
554        let token = TokenPerplexity {
555            token: "test".to_string(),
556            log_prob: -2.5,
557            perplexity: 12.18,
558            position: 0,
559        };
560
561        let json = serde_json::to_string(&token).unwrap();
562        let deserialized: TokenPerplexity = serde_json::from_str(&json).unwrap();
563
564        assert_eq!(token.token, deserialized.token);
565        assert_eq!(token.log_prob, deserialized.log_prob);
566        assert_eq!(token.position, deserialized.position);
567    }
568}