anno_eval/eval/
relation.rs

1//! Relation extraction evaluation metrics.
2//!
3//! # Overview
4//!
5//! Relation extraction identifies semantic relationships between entity pairs,
6//! producing (head, relation, tail) triples. This module provides metrics for:
7//!
8//! - **Boundary evaluation (Rel)**: Relation correct if entity boundaries match
9//! - **Strict evaluation (Rel+)**: Relation correct if entities exactly match
10//! - **Per-relation breakdown**: F1 for each relation type
11//!
12//! # Evaluation Protocols
13//!
14//! From joint entity-relation extraction research (arXiv:2502.09247):
15//!
16//! | Mode | Entity Match | Relation Match | Use Case |
17//! |------|--------------|----------------|----------|
18//! | Boundary (Rel) | Span overlap | Type match | Lenient |
19//! | Strict (Rel+) | Exact span + type | Type match | Strict |
20//!
21//! # Research Alignment
22//!
23//! Standard benchmarks include DocRED, TACRED, SciERC, and NYT-style corpora.
24
25use anno::RelationTriple;
26use serde::{Deserialize, Serialize};
27use std::collections::HashMap;
28
29/// Create relation predictions from entity pairs using a conservative heuristic baseline.
30///
31/// This is intended as an evaluation fallback when a backend does not implement
32/// joint relation extraction. It is deliberately simple and low-dependency.
33///
34/// Notes:
35/// - `relation_types` is accepted for call-site compatibility, but is currently not used
36///   for filtering (the heuristic emits a small, fixed label set + `UNKNOWN`).
37/// - Offsets are treated as **character** offsets.
38#[must_use]
39pub fn create_entity_pair_relations(
40    entities: &[anno::Entity],
41    text: &str,
42    relation_types: &[&str],
43) -> Vec<RelationPrediction> {
44    let _ = relation_types;
45
46    let text_char_len = text.chars().count();
47    let max_distance = 200; // catch some cross-sentence relations
48
49    let mut pred_relations = Vec::new();
50
51    // Validate entities first to avoid panics.
52    let valid_entities: Vec<&anno::Entity> = entities
53        .iter()
54        .filter(|e| e.start() < e.end() && e.end() <= text_char_len && e.start() < text_char_len)
55        .collect();
56
57    // Limit to avoid O(n²) explosion with many entities.
58    let max_entities = 50.min(valid_entities.len());
59
60    for i in 0..max_entities {
61        for j in (i + 1)..max_entities {
62            let head = valid_entities[i];
63            let tail = valid_entities[j];
64
65            // Calculate distance using character offsets.
66            let distance = if tail.start() >= head.end() {
67                tail.start() - head.end()
68            } else if head.start() >= tail.end() {
69                head.start() - tail.end()
70            } else {
71                // Overlapping entities - skip.
72                continue;
73            };
74
75            if distance > max_distance {
76                continue;
77            }
78
79            // Extract text between entities using character offsets (not byte offsets).
80            let between_text = if head.end() <= tail.start() {
81                text.chars()
82                    .skip(head.end())
83                    .take(tail.start() - head.end())
84                    .collect::<String>()
85            } else {
86                text.chars()
87                    .skip(tail.end())
88                    .take(head.start() - tail.end())
89                    .collect::<String>()
90            };
91
92            let between_lower = between_text.to_lowercase();
93            let rel_type = if between_lower.contains("founded") || between_lower.contains("founder")
94            {
95                "FOUNDED"
96            } else if between_lower.contains("works for")
97                || between_lower.contains("employee")
98                || between_lower.contains("employed")
99            {
100                "WORKS_FOR"
101            } else if between_lower.contains("located in")
102                || between_lower.contains("based in")
103                || between_lower.contains("headquartered")
104            {
105                "LOCATED_IN"
106            } else if between_lower.contains("born in") || between_lower.contains("native of") {
107                "BORN_IN"
108            } else if between_lower.contains("ceo of")
109                || between_lower.contains("president of")
110                || between_lower.contains("leads")
111            {
112                "HEAD_OF"
113            } else if between_lower.contains("acquired")
114                || between_lower.contains("bought")
115                || between_lower.contains("merged")
116            {
117                "ACQUIRED"
118            } else {
119                // Explicit "UNKNOWN" so it cannot accidentally match a gold label.
120                "UNKNOWN"
121            };
122
123            pred_relations.push(RelationPrediction {
124                head_span: (head.start(), head.end()),
125                head_type: head.entity_type.as_label().to_string(),
126                tail_span: (tail.start(), tail.end()),
127                tail_type: tail.entity_type.as_label().to_string(),
128                relation_type: rel_type.to_string(),
129                confidence: 0.5,
130            });
131        }
132    }
133
134    pred_relations
135}
136
137/// Ground truth relation.
138#[derive(Debug, Clone, Serialize, Deserialize)]
139pub struct RelationGold {
140    /// Head entity span (start, end)
141    pub head_span: (usize, usize),
142    /// Head entity type
143    pub head_type: String,
144    /// Head entity text
145    pub head_text: String,
146    /// Tail entity span (start, end)
147    pub tail_span: (usize, usize),
148    /// Tail entity type
149    pub tail_type: String,
150    /// Tail entity text
151    pub tail_text: String,
152    /// Relation type
153    pub relation_type: String,
154}
155
156impl RelationGold {
157    /// Create a new relation gold standard.
158    pub fn new(
159        head_span: (usize, usize),
160        head_type: impl Into<String>,
161        head_text: impl Into<String>,
162        tail_span: (usize, usize),
163        tail_type: impl Into<String>,
164        tail_text: impl Into<String>,
165        relation_type: impl Into<String>,
166    ) -> Self {
167        Self {
168            head_span,
169            head_type: head_type.into(),
170            head_text: head_text.into(),
171            tail_span,
172            tail_type: tail_type.into(),
173            tail_text: tail_text.into(),
174            relation_type: relation_type.into(),
175        }
176    }
177}
178
179/// Predicted relation with entity information.
180#[derive(Debug, Clone)]
181pub struct RelationPrediction {
182    /// Head entity span (start, end)
183    pub head_span: (usize, usize),
184    /// Head entity type
185    pub head_type: String,
186    /// Tail entity span (start, end)
187    pub tail_span: (usize, usize),
188    /// Tail entity type
189    pub tail_type: String,
190    /// Relation type
191    pub relation_type: String,
192    /// Confidence score
193    pub confidence: f32,
194}
195
196impl RelationPrediction {
197    /// Create from a RelationTriple and entity list.
198    pub fn from_triple_with_entities(
199        triple: &RelationTriple,
200        entities: &[anno::Entity],
201    ) -> Option<Self> {
202        let head = entities.get(triple.head_idx)?;
203        let tail = entities.get(triple.tail_idx)?;
204
205        Some(Self {
206            head_span: (head.start(), head.end()),
207            head_type: head.entity_type.as_label().to_string(),
208            tail_span: (tail.start(), tail.end()),
209            tail_type: tail.entity_type.as_label().to_string(),
210            relation_type: triple.relation_type.clone(),
211            confidence: triple.confidence.value() as f32,
212        })
213    }
214}
215
216/// Metrics for relation extraction evaluation.
217#[derive(Debug, Clone, Serialize, Deserialize)]
218pub struct RelationMetrics {
219    /// Boundary evaluation F1 (Rel): entity boundaries match
220    pub boundary_f1: f64,
221    /// Boundary precision
222    pub boundary_precision: f64,
223    /// Boundary recall
224    pub boundary_recall: f64,
225    /// Strict evaluation F1 (Rel+): exact entity match
226    pub strict_f1: f64,
227    /// Strict precision
228    pub strict_precision: f64,
229    /// Strict recall
230    pub strict_recall: f64,
231    /// Total predicted relations
232    pub num_predicted: usize,
233    /// Total gold relations
234    pub num_gold: usize,
235    /// Boundary matches
236    pub boundary_matches: usize,
237    /// Strict matches
238    pub strict_matches: usize,
239    /// Per-relation-type breakdown
240    pub per_relation: HashMap<String, RelationTypeMetrics>,
241}
242
243/// Per-relation-type metrics.
244#[derive(Debug, Clone, Default, Serialize, Deserialize)]
245pub struct RelationTypeMetrics {
246    /// F1 score for this relation type (boundary mode)
247    pub boundary_f1: f64,
248    /// F1 score for this relation type (strict mode)
249    pub strict_f1: f64,
250    /// Number of gold relations of this type
251    pub gold_count: usize,
252    /// Number of predicted relations of this type
253    pub pred_count: usize,
254    /// Boundary matches
255    pub boundary_matches: usize,
256    /// Strict matches
257    pub strict_matches: usize,
258}
259
260/// Configuration for relation evaluation.
261#[derive(Debug, Clone)]
262pub struct RelationEvalConfig {
263    /// Overlap threshold for boundary matching (default: 0.5)
264    pub overlap_threshold: f64,
265    /// Whether to require entity type match (default: true)
266    pub require_entity_type_match: bool,
267    /// Whether relation direction matters (default: true)
268    pub directed_relations: bool,
269}
270
271impl Default for RelationEvalConfig {
272    fn default() -> Self {
273        Self {
274            overlap_threshold: 0.5,
275            require_entity_type_match: true,
276            directed_relations: true,
277        }
278    }
279}
280
281/// Evaluate relation extraction predictions against gold standard.
282///
283/// # Arguments
284/// * `gold` - Ground truth relations
285/// * `pred` - Predicted relations
286/// * `config` - Evaluation configuration
287///
288/// # Returns
289/// Comprehensive relation extraction metrics
290///
291/// # Example
292///
293/// ```rust
294/// use anno_eval::eval::relation::{evaluate_relations, RelationGold, RelationPrediction, RelationEvalConfig};
295///
296/// let gold = vec![
297///     RelationGold::new(
298///         (0, 10), "PER", "Steve Jobs",
299///         (20, 25), "ORG", "Apple",
300///         "FOUNDED"
301///     ),
302/// ];
303///
304/// let pred = vec![
305///     RelationPrediction {
306///         head_span: (0, 10),
307///         head_type: "PER".to_string(),
308///         tail_span: (20, 25),
309///         tail_type: "ORG".to_string(),
310///         relation_type: "FOUNDED".to_string(),
311///         confidence: 0.9,
312///     },
313/// ];
314///
315/// let metrics = evaluate_relations(&gold, &pred, &RelationEvalConfig::default());
316/// assert!((metrics.strict_f1 - 1.0).abs() < 0.001);
317/// ```
318pub fn evaluate_relations(
319    gold: &[RelationGold],
320    pred: &[RelationPrediction],
321    config: &RelationEvalConfig,
322) -> RelationMetrics {
323    if gold.is_empty() && pred.is_empty() {
324        return RelationMetrics {
325            boundary_f1: 1.0,
326            boundary_precision: 1.0,
327            boundary_recall: 1.0,
328            strict_f1: 1.0,
329            strict_precision: 1.0,
330            strict_recall: 1.0,
331            num_predicted: 0,
332            num_gold: 0,
333            boundary_matches: 0,
334            strict_matches: 0,
335            per_relation: HashMap::new(),
336        };
337    }
338
339    // Track matches
340    let mut gold_matched_boundary = vec![false; gold.len()];
341    let mut gold_matched_strict = vec![false; gold.len()];
342    let mut pred_matched_boundary = vec![false; pred.len()];
343    let mut pred_matched_strict = vec![false; pred.len()];
344
345    // Per-relation tracking: (gold_count, pred_count, boundary_matches, strict_matches)
346    let mut rel_stats: HashMap<String, (usize, usize, usize, usize)> = HashMap::new();
347
348    // Count gold per relation type
349    for g in gold {
350        let entry = rel_stats.entry(g.relation_type.clone()).or_default();
351        entry.0 += 1;
352    }
353
354    // Count pred per relation type
355    for p in pred {
356        let entry = rel_stats.entry(p.relation_type.clone()).or_default();
357        entry.1 += 1;
358    }
359
360    // Strict matching: exact entity spans + relation type
361    for (pi, p) in pred.iter().enumerate() {
362        // Skip predictions that are already matched
363        if pred_matched_strict[pi] {
364            continue;
365        }
366        for (gi, g) in gold.iter().enumerate() {
367            if gold_matched_strict[gi] {
368                continue;
369            }
370
371            // Relation type must match (case-insensitive)
372            if p.relation_type.to_lowercase() != g.relation_type.to_lowercase() {
373                continue;
374            }
375
376            // Check entity type match if required
377            if config.require_entity_type_match
378                && (p.head_type != g.head_type || p.tail_type != g.tail_type)
379            {
380                continue;
381            }
382
383            // Exact span match (or reversed if undirected)
384            let forward_match = p.head_span == g.head_span && p.tail_span == g.tail_span;
385            let reverse_match = !config.directed_relations
386                && p.head_span == g.tail_span
387                && p.tail_span == g.head_span;
388
389            if forward_match || reverse_match {
390                gold_matched_strict[gi] = true;
391                pred_matched_strict[pi] = true;
392
393                let entry = rel_stats.entry(g.relation_type.clone()).or_default();
394                entry.3 += 1;
395                break;
396            }
397        }
398    }
399
400    // Boundary matching: overlapping entity spans + relation type
401    for (pi, p) in pred.iter().enumerate() {
402        // Skip predictions that are already matched
403        if pred_matched_boundary[pi] {
404            continue;
405        }
406        for (gi, g) in gold.iter().enumerate() {
407            if gold_matched_boundary[gi] {
408                continue;
409            }
410
411            // Relation type must match (case-insensitive)
412            if p.relation_type.to_lowercase() != g.relation_type.to_lowercase() {
413                continue;
414            }
415
416            if config.require_entity_type_match
417                && (p.head_type != g.head_type || p.tail_type != g.tail_type)
418            {
419                continue;
420            }
421
422            // Boundary match: overlapping spans
423            let head_overlap = calculate_span_overlap(p.head_span, g.head_span);
424            let tail_overlap = calculate_span_overlap(p.tail_span, g.tail_span);
425
426            let forward_match = head_overlap >= config.overlap_threshold
427                && tail_overlap >= config.overlap_threshold;
428
429            let reverse_match = if !config.directed_relations {
430                let rev_head_overlap = calculate_span_overlap(p.head_span, g.tail_span);
431                let rev_tail_overlap = calculate_span_overlap(p.tail_span, g.head_span);
432                rev_head_overlap >= config.overlap_threshold
433                    && rev_tail_overlap >= config.overlap_threshold
434            } else {
435                false
436            };
437
438            if forward_match || reverse_match {
439                gold_matched_boundary[gi] = true;
440                pred_matched_boundary[pi] = true;
441
442                let entry = rel_stats.entry(g.relation_type.clone()).or_default();
443                entry.2 += 1;
444                break;
445            }
446        }
447    }
448
449    // Calculate metrics
450    let boundary_matches = pred_matched_boundary.iter().filter(|&&m| m).count();
451    let strict_matches = pred_matched_strict.iter().filter(|&&m| m).count();
452
453    let boundary_precision = if !pred.is_empty() {
454        boundary_matches as f64 / pred.len() as f64
455    } else {
456        0.0
457    };
458    let boundary_recall = if !gold.is_empty() {
459        boundary_matches as f64 / gold.len() as f64
460    } else {
461        0.0
462    };
463    let boundary_f1 = f1_score(boundary_precision, boundary_recall);
464
465    let strict_precision = if !pred.is_empty() {
466        strict_matches as f64 / pred.len() as f64
467    } else {
468        0.0
469    };
470    let strict_recall = if !gold.is_empty() {
471        strict_matches as f64 / gold.len() as f64
472    } else {
473        0.0
474    };
475    let strict_f1 = f1_score(strict_precision, strict_recall);
476
477    // Build per-relation metrics
478    let per_relation: HashMap<String, RelationTypeMetrics> = rel_stats
479        .into_iter()
480        .map(|(rel, (gold_count, pred_count, boundary, strict))| {
481            let b_p = if pred_count > 0 {
482                boundary as f64 / pred_count as f64
483            } else {
484                0.0
485            };
486            let b_r = if gold_count > 0 {
487                boundary as f64 / gold_count as f64
488            } else {
489                0.0
490            };
491            let s_p = if pred_count > 0 {
492                strict as f64 / pred_count as f64
493            } else {
494                0.0
495            };
496            let s_r = if gold_count > 0 {
497                strict as f64 / gold_count as f64
498            } else {
499                0.0
500            };
501
502            (
503                rel,
504                RelationTypeMetrics {
505                    boundary_f1: f1_score(b_p, b_r),
506                    strict_f1: f1_score(s_p, s_r),
507                    gold_count,
508                    pred_count,
509                    boundary_matches: boundary,
510                    strict_matches: strict,
511                },
512            )
513        })
514        .collect();
515
516    RelationMetrics {
517        boundary_f1,
518        boundary_precision,
519        boundary_recall,
520        strict_f1,
521        strict_precision,
522        strict_recall,
523        num_predicted: pred.len(),
524        num_gold: gold.len(),
525        boundary_matches,
526        strict_matches,
527        per_relation,
528    }
529}
530
531/// Render relation evaluation as HTML report.
532pub fn render_relation_eval_html(metrics: &RelationMetrics) -> String {
533    let mut html = String::new();
534    html.push_str("<!DOCTYPE html>\n<html><head><title>Relation Extraction Evaluation</title>");
535    html.push_str("<style>body{font-family:monospace;margin:20px;}table{border-collapse:collapse;}th,td{padding:8px;border:1px solid #ddd;}</style>");
536    html.push_str("</head><body>");
537    html.push_str("<h1>Relation Extraction Evaluation</h1>");
538    html.push_str("<h2>Overall Metrics</h2>");
539    html.push_str("<table>");
540    html.push_str("<tr><th>Metric</th><th>Boundary (Rel)</th><th>Strict (Rel+)</th></tr>");
541    html.push_str(&format!(
542        "<tr><td>Precision</td><td>{:.3}</td><td>{:.3}</td></tr>",
543        metrics.boundary_precision, metrics.strict_precision
544    ));
545    html.push_str(&format!(
546        "<tr><td>Recall</td><td>{:.3}</td><td>{:.3}</td></tr>",
547        metrics.boundary_recall, metrics.strict_recall
548    ));
549    html.push_str(&format!(
550        "<tr><td>F1</td><td>{:.3}</td><td>{:.3}</td></tr>",
551        metrics.boundary_f1, metrics.strict_f1
552    ));
553    html.push_str("</table>");
554    html.push_str(&format!(
555        "<p>Gold: {}  Predicted: {}  Boundary matches: {}  Strict matches: {}</p>",
556        metrics.num_gold, metrics.num_predicted, metrics.boundary_matches, metrics.strict_matches
557    ));
558
559    if !metrics.per_relation.is_empty() {
560        html.push_str("<h2>Per-Relation Breakdown</h2>");
561        html.push_str("<table>");
562        html.push_str("<tr><th>Relation Type</th><th>Boundary F1</th><th>Strict F1</th><th>Gold</th><th>Pred</th><th>Boundary Matches</th><th>Strict Matches</th></tr>");
563        let mut rels: Vec<_> = metrics.per_relation.iter().collect();
564        rels.sort_by_key(|b| std::cmp::Reverse(b.1.gold_count));
565        for (rel_type, rel_metrics) in rels {
566            html.push_str(&format!(
567                "<tr><td>{}</td><td>{:.3}</td><td>{:.3}</td><td>{}</td><td>{}</td><td>{}</td><td>{}</td></tr>",
568                rel_type, rel_metrics.boundary_f1, rel_metrics.strict_f1,
569                rel_metrics.gold_count, rel_metrics.pred_count,
570                rel_metrics.boundary_matches, rel_metrics.strict_matches
571            ));
572        }
573        html.push_str("</table>");
574    }
575
576    html.push_str("</body></html>");
577    html
578}
579
580impl RelationMetrics {
581    /// Generate human-readable string representation.
582    pub fn to_string_human(&self, verbose: bool) -> String {
583        let mut out = String::new();
584
585        out.push_str("Relation Extraction Evaluation\n");
586        out.push_str(
587            "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n",
588        );
589        out.push_str(&format!(
590            "Boundary (Rel):  P={:.1}%  R={:.1}%  F1={:.1}%\n",
591            self.boundary_precision * 100.0,
592            self.boundary_recall * 100.0,
593            self.boundary_f1 * 100.0
594        ));
595        out.push_str(&format!(
596            "Strict (Rel+):   P={:.1}%  R={:.1}%  F1={:.1}%\n",
597            self.strict_precision * 100.0,
598            self.strict_recall * 100.0,
599            self.strict_f1 * 100.0
600        ));
601        out.push_str(&format!(
602            "Gold: {}  Predicted: {}  Boundary matches: {}  Strict matches: {}\n",
603            self.num_gold, self.num_predicted, self.boundary_matches, self.strict_matches
604        ));
605
606        if verbose && !self.per_relation.is_empty() {
607            out.push_str("\nPer-Relation Breakdown:\n");
608            let mut rels: Vec<_> = self.per_relation.iter().collect();
609            rels.sort_by_key(|b| std::cmp::Reverse(b.1.gold_count));
610
611            for (rel_type, metrics) in rels {
612                if metrics.gold_count > 0 || metrics.pred_count > 0 {
613                    // Use stored precision/recall from RelationTypeMetrics (already computed)
614                    // Calculate from stored matches to avoid duplication
615                    let boundary_p = if metrics.pred_count > 0 {
616                        metrics.boundary_matches as f64 / metrics.pred_count as f64
617                    } else {
618                        0.0
619                    };
620                    let boundary_r = if metrics.gold_count > 0 {
621                        metrics.boundary_matches as f64 / metrics.gold_count as f64
622                    } else {
623                        0.0
624                    };
625                    let strict_p = if metrics.pred_count > 0 {
626                        metrics.strict_matches as f64 / metrics.pred_count as f64
627                    } else {
628                        0.0
629                    };
630                    let strict_r = if metrics.gold_count > 0 {
631                        metrics.strict_matches as f64 / metrics.gold_count as f64
632                    } else {
633                        0.0
634                    };
635                    // Use stored F1 scores (already computed correctly)
636                    out.push_str(&format!(
637                        "  {:20} Boundary: F1={:.1}% (P={:.1}% R={:.1}%)  Strict: F1={:.1}% (P={:.1}% R={:.1}%)  [gold={} pred={} matches={}/{}]\n",
638                        rel_type,
639                        metrics.boundary_f1 * 100.0,
640                        boundary_p * 100.0,
641                        boundary_r * 100.0,
642                        metrics.strict_f1 * 100.0,
643                        strict_p * 100.0,
644                        strict_r * 100.0,
645                        metrics.gold_count,
646                        metrics.pred_count,
647                        metrics.boundary_matches,
648                        metrics.strict_matches
649                    ));
650                }
651            }
652        }
653
654        out
655    }
656}
657
658impl std::fmt::Display for RelationMetrics {
659    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
660        write!(f, "{}", self.to_string_human(false))
661    }
662}
663
664/// Calculate overlap (IoU) between two spans.
665fn calculate_span_overlap(a: (usize, usize), b: (usize, usize)) -> f64 {
666    let intersection_start = a.0.max(b.0);
667    let intersection_end = a.1.min(b.1);
668
669    if intersection_start >= intersection_end {
670        return 0.0;
671    }
672
673    let intersection = (intersection_end - intersection_start) as f64;
674    let union = ((a.1 - a.0) + (b.1 - b.0) - (intersection_end - intersection_start)) as f64;
675
676    if union == 0.0 {
677        return 1.0;
678    }
679
680    intersection / union
681}
682
683/// Calculate F1 score from precision and recall.
684fn f1_score(precision: f64, recall: f64) -> f64 {
685    if precision + recall > 0.0 {
686        2.0 * precision * recall / (precision + recall)
687    } else {
688        0.0
689    }
690}
691
692#[cfg(test)]
693mod tests {
694    use super::*;
695
696    #[test]
697    fn test_exact_relation_match() {
698        let gold = vec![RelationGold::new(
699            (0, 10),
700            "PER",
701            "Steve Jobs",
702            (20, 25),
703            "ORG",
704            "Apple",
705            "FOUNDED",
706        )];
707        let pred = vec![RelationPrediction {
708            head_span: (0, 10),
709            head_type: "PER".to_string(),
710            tail_span: (20, 25),
711            tail_type: "ORG".to_string(),
712            relation_type: "FOUNDED".to_string(),
713            confidence: 0.9,
714        }];
715
716        let metrics = evaluate_relations(&gold, &pred, &RelationEvalConfig::default());
717        assert!((metrics.strict_f1 - 1.0).abs() < 0.001);
718        assert!((metrics.boundary_f1 - 1.0).abs() < 0.001);
719    }
720
721    #[test]
722    fn test_boundary_match_not_strict() {
723        let gold = vec![RelationGold::new(
724            (0, 10),
725            "PER",
726            "Steve Jobs",
727            (20, 30),
728            "ORG",
729            "Apple Inc",
730            "FOUNDED",
731        )];
732        // Overlapping but not exact
733        let pred = vec![RelationPrediction {
734            head_span: (0, 10),
735            head_type: "PER".to_string(),
736            tail_span: (20, 25), // Different end
737            tail_type: "ORG".to_string(),
738            relation_type: "FOUNDED".to_string(),
739            confidence: 0.9,
740        }];
741
742        let metrics = evaluate_relations(&gold, &pred, &RelationEvalConfig::default());
743        // Strict should fail
744        assert!(metrics.strict_f1 < 1.0);
745        // Boundary should succeed (50% overlap > 0.5 threshold)
746        assert!(metrics.boundary_f1 > 0.0);
747    }
748
749    #[test]
750    fn test_wrong_relation_type() {
751        let gold = vec![RelationGold::new(
752            (0, 10),
753            "PER",
754            "Steve Jobs",
755            (20, 25),
756            "ORG",
757            "Apple",
758            "FOUNDED",
759        )];
760        let pred = vec![RelationPrediction {
761            head_span: (0, 10),
762            head_type: "PER".to_string(),
763            tail_span: (20, 25),
764            tail_type: "ORG".to_string(),
765            relation_type: "WORKS_FOR".to_string(), // Wrong relation
766            confidence: 0.9,
767        }];
768
769        let metrics = evaluate_relations(&gold, &pred, &RelationEvalConfig::default());
770        assert!(metrics.strict_f1 < 0.001);
771    }
772
773    #[test]
774    fn test_undirected_relations() {
775        let gold = vec![RelationGold::new(
776            (0, 10),
777            "PER",
778            "Alice",
779            (20, 25),
780            "PER",
781            "Bob",
782            "SIBLING",
783        )];
784        // Reversed head/tail
785        let pred = vec![RelationPrediction {
786            head_span: (20, 25),
787            head_type: "PER".to_string(),
788            tail_span: (0, 10),
789            tail_type: "PER".to_string(),
790            relation_type: "SIBLING".to_string(),
791            confidence: 0.9,
792        }];
793
794        // Directed: should fail
795        let config_directed = RelationEvalConfig {
796            directed_relations: true,
797            ..Default::default()
798        };
799        let metrics = evaluate_relations(&gold, &pred, &config_directed);
800        assert!(metrics.strict_f1 < 0.001);
801
802        // Undirected: should succeed
803        let config_undirected = RelationEvalConfig {
804            directed_relations: false,
805            ..Default::default()
806        };
807        let metrics = evaluate_relations(&gold, &pred, &config_undirected);
808        assert!((metrics.strict_f1 - 1.0).abs() < 0.001);
809    }
810
811    #[test]
812    fn test_empty_inputs() {
813        let metrics = evaluate_relations(&[], &[], &RelationEvalConfig::default());
814        assert!((metrics.strict_f1 - 1.0).abs() < 0.001);
815    }
816
817    #[test]
818    fn test_per_relation_breakdown() {
819        let gold = vec![
820            RelationGold::new((0, 5), "PER", "A", (10, 15), "ORG", "B", "FOUNDED"),
821            RelationGold::new((20, 25), "PER", "C", (30, 35), "ORG", "D", "WORKS_FOR"),
822        ];
823        let pred = vec![RelationPrediction {
824            head_span: (0, 5),
825            head_type: "PER".to_string(),
826            tail_span: (10, 15),
827            tail_type: "ORG".to_string(),
828            relation_type: "FOUNDED".to_string(),
829            confidence: 0.9,
830        }];
831
832        let metrics = evaluate_relations(&gold, &pred, &RelationEvalConfig::default());
833
834        // Check per-relation breakdown
835        assert!(metrics.per_relation.contains_key("FOUNDED"));
836        assert!(metrics.per_relation.contains_key("WORKS_FOR"));
837
838        let founded = metrics.per_relation.get("FOUNDED").unwrap();
839        assert!((founded.strict_f1 - 1.0).abs() < 0.001); // Perfect for FOUNDED
840
841        let works_for = metrics.per_relation.get("WORKS_FOR").unwrap();
842        assert!(works_for.strict_f1 < 0.001); // No predictions for WORKS_FOR
843    }
844
845    #[test]
846    fn test_span_overlap() {
847        // Exact match
848        assert!((calculate_span_overlap((0, 10), (0, 10)) - 1.0).abs() < 0.001);
849
850        // No overlap
851        assert!(calculate_span_overlap((0, 5), (10, 15)) < 0.001);
852
853        // 50% overlap
854        let overlap = calculate_span_overlap((0, 10), (5, 15));
855        assert!(overlap > 0.3 && overlap < 0.4); // IoU for this case
856    }
857
858    #[test]
859    fn test_relation_type_case_insensitive() {
860        // Relation types should match case-insensitively
861        let gold = vec![RelationGold::new(
862            (0, 10),
863            "PER",
864            "Steve Jobs",
865            (20, 25),
866            "ORG",
867            "Apple",
868            "FOUNDED",
869        )];
870        let pred = vec![RelationPrediction {
871            head_span: (0, 10),
872            head_type: "PER".to_string(),
873            tail_span: (20, 25),
874            tail_type: "ORG".to_string(),
875            relation_type: "founded".to_string(), // lowercase
876            confidence: 0.9,
877        }];
878
879        let metrics = evaluate_relations(&gold, &pred, &RelationEvalConfig::default());
880        assert!(
881            (metrics.strict_f1 - 1.0).abs() < 0.001,
882            "Relation type matching should be case-insensitive"
883        );
884    }
885
886    #[test]
887    fn test_entity_type_match_disabled() {
888        // When require_entity_type_match is false, entity types don't need to match
889        let gold = vec![RelationGold::new(
890            (0, 10),
891            "PER",
892            "Steve Jobs",
893            (20, 25),
894            "ORG",
895            "Apple",
896            "FOUNDED",
897        )];
898        let pred = vec![RelationPrediction {
899            head_span: (0, 10),
900            head_type: "PERSON".to_string(), // Different from PER
901            tail_span: (20, 25),
902            tail_type: "COMPANY".to_string(), // Different from ORG
903            relation_type: "FOUNDED".to_string(),
904            confidence: 0.9,
905        }];
906
907        // With type match required: should fail
908        let config_strict = RelationEvalConfig {
909            require_entity_type_match: true,
910            ..Default::default()
911        };
912        let metrics = evaluate_relations(&gold, &pred, &config_strict);
913        assert!(metrics.strict_f1 < 0.001, "Type mismatch should fail");
914
915        // Without type match required: should succeed
916        let config_lenient = RelationEvalConfig {
917            require_entity_type_match: false,
918            ..Default::default()
919        };
920        let metrics = evaluate_relations(&gold, &pred, &config_lenient);
921        assert!(
922            (metrics.strict_f1 - 1.0).abs() < 0.001,
923            "Without type matching, should succeed"
924        );
925    }
926
927    #[test]
928    fn test_no_gold_all_pred() {
929        // No gold annotations, but predictions exist
930        let gold: Vec<RelationGold> = vec![];
931        let pred = vec![RelationPrediction {
932            head_span: (0, 10),
933            head_type: "PER".to_string(),
934            tail_span: (20, 25),
935            tail_type: "ORG".to_string(),
936            relation_type: "FOUNDED".to_string(),
937            confidence: 0.9,
938        }];
939
940        let metrics = evaluate_relations(&gold, &pred, &RelationEvalConfig::default());
941        // Precision = 0 (no correct), Recall = undefined (no gold) -> 0
942        assert!(metrics.strict_precision < 0.001);
943        assert!(metrics.strict_f1 < 0.001);
944    }
945
946    #[test]
947    fn test_all_gold_no_pred() {
948        // Gold annotations exist, but no predictions
949        let gold = vec![RelationGold::new(
950            (0, 10),
951            "PER",
952            "Steve Jobs",
953            (20, 25),
954            "ORG",
955            "Apple",
956            "FOUNDED",
957        )];
958        let pred: Vec<RelationPrediction> = vec![];
959
960        let metrics = evaluate_relations(&gold, &pred, &RelationEvalConfig::default());
961        // Precision = undefined (no pred) -> 0, Recall = 0 (no correct)
962        assert!(metrics.strict_recall < 0.001);
963        assert!(metrics.strict_f1 < 0.001);
964    }
965
966    #[test]
967    fn test_boundary_vs_strict_matching() {
968        // Test that boundary matching is more lenient than strict
969        let gold = vec![RelationGold::new(
970            (0, 15),
971            "PER",
972            "Steve Jobs Jr.",
973            (25, 35),
974            "ORG",
975            "Apple Inc.",
976            "FOUNDED",
977        )];
978        // Predictions have overlapping but not exact spans
979        let pred = vec![RelationPrediction {
980            head_span: (0, 10), // Partial overlap (10/15 = 66%)
981            head_type: "PER".to_string(),
982            tail_span: (25, 30), // Partial overlap (5/10 = 50%)
983            tail_type: "ORG".to_string(),
984            relation_type: "FOUNDED".to_string(),
985            confidence: 0.9,
986        }];
987
988        let metrics = evaluate_relations(&gold, &pred, &RelationEvalConfig::default());
989
990        // Strict should fail (spans don't match exactly)
991        assert!(
992            metrics.strict_f1 < 0.001,
993            "Strict should fail for partial overlap"
994        );
995        // Boundary should succeed (>= 0.5 overlap threshold)
996        assert!(
997            metrics.boundary_f1 > 0.5,
998            "Boundary should succeed for sufficient overlap"
999        );
1000    }
1001}
anno_eval/eval/relation.rs

anno_eval/eval/
relation.rs
