Skip to main content

anno_eval/eval/
abstract_anaphora.rs

1//! Abstract anaphora evaluation infrastructure.
2//!
3//! # Why This Exists
4//!
5//! Standard coreference resolvers are typically evaluated on nominal entity
6//! coreference. They often fail on abstract anaphora—references to events,
7//! propositions, facts, and situations rather than nominal entities.
8//!
9//! ```text
10//! "The company announced layoffs. This shocked employees."
11//!                                 ^^^^
12//!                                 What does "this" refer to?
13//!                                 → The *announcement* (event)
14//!                                 → The *fact* of layoffs
15//!                                 → The *situation* of job losses
16//!
17//! Standard coreference: ??? (no nominal antecedent found)
18//! ```
19//!
20//! This module provides evaluation infrastructure to measure this gap and
21//! track progress toward systems that handle abstract anaphora.
22//!
23//! The gap is not a minor limitation—it is a different kind of reference
24//! that requires different modeling and evaluation.
25//!
26//! # Theoretical Foundation
27//!
28//! Following Dalrymple, Shieber & Pereira (1991), anaphora resolution can be
29//! framed as solving `P(s₁, ..., sₙ) = s` where P is the property being
30//! predicated. For abstract anaphora, P operates over events/propositions,
31//! not entities.
32//!
33//! # Key Papers
34//!
35//! - Dalrymple et al. (1991): "Ellipsis and Higher-Order Unification" - theoretical foundation
36//! - Kolhatkar & Hirst (2012): "Resolving 'this-issue' anaphors"
37//! - Marasović et al. (2017): LSTM-Siamese model, EMNLP, outperforms on shell nouns
38//! - Moosavi & Strube (2016): LEA metric addresses "mention identification effect"
39//!
40//! # Example
41//!
42//! ```rust
43//! use anno_eval::eval::abstract_anaphora::{
44//!     AbstractAnaphoraDataset, AbstractAnaphoraEvaluator, AnaphoraType
45//! };
46//! use anno_eval::eval::coref_resolver::SimpleCorefResolver;
47//!
48//! let dataset = AbstractAnaphoraDataset::default();
49//! let resolver = SimpleCorefResolver::default();
50//! let evaluator = AbstractAnaphoraEvaluator::new(resolver);
51//!
52//! let results = evaluator.evaluate(&dataset);
53//! println!("Nominal accuracy: {:.1}%", results.nominal_accuracy * 100.0);
54//! println!("Abstract accuracy: {:.1}%", results.abstract_accuracy * 100.0);
55//! ```
56
57use crate::discourse::{classify_shell_noun, ReferentType, ShellNoun, ShellNounClass};
58use crate::eval::coref::{CorefChain, Mention};
59use crate::eval::coref_metrics::{lea_score, CorefScores};
60use crate::eval::coref_resolver::{
61    DiscourseAwareResolver, DiscourseCorefConfig, SimpleCorefResolver,
62};
63use crate::{Entity, EntityCategory, EntityType};
64use serde::{Deserialize, Serialize};
65use std::collections::HashMap;
66
67// =============================================================================
68// Anaphora Types
69// =============================================================================
70
71/// Type of anaphoric reference.
72#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
73pub enum AnaphoraType {
74    /// Standard nominal coreference: "John" → "he"
75    Nominal,
76    /// Event anaphora: "The crash happened" → "This shocked everyone"
77    Event,
78    /// Fact anaphora: "He won" → "This is undeniable"
79    Fact,
80    /// Proposition anaphora: "She might leave" → "This worries me"
81    Proposition,
82    /// Situation anaphora: "Prices rose while wages fell" → "This was unsustainable"
83    Situation,
84}
85
86impl AnaphoraType {
87    /// Is this an abstract (non-nominal) anaphora type?
88    #[must_use]
89    pub const fn is_abstract(&self) -> bool {
90        !matches!(self, AnaphoraType::Nominal)
91    }
92
93    /// Human-readable name.
94    #[must_use]
95    pub const fn as_str(&self) -> &'static str {
96        match self {
97            AnaphoraType::Nominal => "nominal",
98            AnaphoraType::Event => "event",
99            AnaphoraType::Fact => "fact",
100            AnaphoraType::Proposition => "proposition",
101            AnaphoraType::Situation => "situation",
102        }
103    }
104}
105
106// =============================================================================
107// Test Cases
108// =============================================================================
109
110/// A single anaphora test case.
111#[derive(Debug, Clone, Serialize, Deserialize)]
112pub struct AnaphoraTestCase {
113    /// Unique identifier
114    pub id: String,
115    /// Full text containing antecedent and anaphor
116    pub text: String,
117    /// The antecedent span (what the anaphor refers to)
118    pub antecedent: AntecedentSpan,
119    /// The anaphoric expression
120    pub anaphor: AnaphorSpan,
121    /// Type of anaphora
122    pub anaphora_type: AnaphoraType,
123    /// Expected: should resolver link these?
124    pub should_resolve: bool,
125    /// Notes on why this case is interesting
126    pub notes: Option<String>,
127}
128
129/// The antecedent (what is being referred to).
130#[derive(Debug, Clone, Serialize, Deserialize)]
131pub struct AntecedentSpan {
132    /// Text of the antecedent
133    pub text: String,
134    /// Start character offset
135    pub start: usize,
136    /// End character offset
137    pub end: usize,
138    /// For events: the trigger verb/noun
139    pub trigger: Option<String>,
140}
141
142/// The anaphoric expression.
143#[derive(Debug, Clone, Serialize, Deserialize)]
144pub struct AnaphorSpan {
145    /// Text of the anaphor ("this", "that", "it", etc.)
146    pub text: String,
147    /// Start character offset
148    pub start: usize,
149    /// End character offset
150    pub end: usize,
151}
152
153impl AnaphoraTestCase {
154    /// Create a new test case.
155    pub fn new(
156        id: impl Into<String>,
157        text: impl Into<String>,
158        antecedent: AntecedentSpan,
159        anaphor: AnaphorSpan,
160        anaphora_type: AnaphoraType,
161    ) -> Self {
162        Self {
163            id: id.into(),
164            text: text.into(),
165            antecedent,
166            anaphor,
167            anaphora_type,
168            should_resolve: true,
169            notes: None,
170        }
171    }
172
173    /// Add notes to explain the test case.
174    #[must_use]
175    pub fn with_notes(mut self, notes: impl Into<String>) -> Self {
176        self.notes = Some(notes.into());
177        self
178    }
179}
180
181// =============================================================================
182// Dataset
183// =============================================================================
184
185/// Dataset of anaphora test cases.
186#[derive(Debug, Clone, Default, Serialize, Deserialize)]
187pub struct AbstractAnaphoraDataset {
188    /// All test cases
189    pub cases: Vec<AnaphoraTestCase>,
190}
191
192impl AbstractAnaphoraDataset {
193    /// Create a new empty dataset.
194    #[must_use]
195    pub fn new() -> Self {
196        Self { cases: Vec::new() }
197    }
198
199    /// Add a test case.
200    pub fn add(&mut self, case: AnaphoraTestCase) {
201        self.cases.push(case);
202    }
203
204    /// Get cases by type.
205    #[must_use]
206    pub fn by_type(&self, anaphora_type: AnaphoraType) -> Vec<&AnaphoraTestCase> {
207        self.cases
208            .iter()
209            .filter(|c| c.anaphora_type == anaphora_type)
210            .collect()
211    }
212
213    /// Get all nominal (non-abstract) cases.
214    #[must_use]
215    pub fn nominal_cases(&self) -> Vec<&AnaphoraTestCase> {
216        self.by_type(AnaphoraType::Nominal)
217    }
218
219    /// Get all abstract cases.
220    #[must_use]
221    pub fn abstract_cases(&self) -> Vec<&AnaphoraTestCase> {
222        self.cases
223            .iter()
224            .filter(|c| c.anaphora_type.is_abstract())
225            .collect()
226    }
227
228    /// Create a standard evaluation dataset.
229    ///
230    /// Contains a balanced mix of nominal and abstract anaphora cases.
231    #[must_use]
232    pub fn standard() -> Self {
233        let mut dataset = Self::new();
234
235        // =================================================================
236        // NOMINAL COREFERENCE (Baseline - should work)
237        // =================================================================
238
239        dataset.add(
240            AnaphoraTestCase::new(
241                "nom_01",
242                "John Smith went to the store. He bought milk.",
243                AntecedentSpan {
244                    text: "John Smith".to_string(),
245                    start: 0,
246                    end: 10,
247                    trigger: None,
248                },
249                AnaphorSpan {
250                    text: "He".to_string(),
251                    start: 32,
252                    end: 34,
253                },
254                AnaphoraType::Nominal,
255            )
256            .with_notes("Simple pronoun resolution - baseline case"),
257        );
258
259        dataset.add(
260            AnaphoraTestCase::new(
261                "nom_02",
262                "Microsoft announced layoffs. The company cited economic conditions.",
263                AntecedentSpan {
264                    text: "Microsoft".to_string(),
265                    start: 0,
266                    end: 9,
267                    trigger: None,
268                },
269                AnaphorSpan {
270                    text: "The company".to_string(),
271                    start: 29,
272                    end: 40,
273                },
274                AnaphoraType::Nominal,
275            )
276            .with_notes("Definite NP resolution"),
277        );
278
279        dataset.add(
280            AnaphoraTestCase::new(
281                "nom_03",
282                "Dr. Sarah Chen published a paper. She presented it at EMNLP.",
283                AntecedentSpan {
284                    text: "Dr. Sarah Chen".to_string(),
285                    start: 0,
286                    end: 14,
287                    trigger: None,
288                },
289                AnaphorSpan {
290                    text: "She".to_string(),
291                    start: 35,
292                    end: 38,
293                },
294                AnaphoraType::Nominal,
295            )
296            .with_notes("Pronoun with title prefix"),
297        );
298
299        dataset.add(
300            AnaphoraTestCase::new(
301                "nom_04",
302                "The CEO of Nvidia is Jensen Huang. He co-founded the company.",
303                AntecedentSpan {
304                    text: "Jensen Huang".to_string(),
305                    start: 20,
306                    end: 32,
307                    trigger: None,
308                },
309                AnaphorSpan {
310                    text: "He".to_string(),
311                    start: 34,
312                    end: 36,
313                },
314                AnaphoraType::Nominal,
315            )
316            .with_notes("Pronoun binds to proper name, not role description"),
317        );
318
319        dataset.add(
320            AnaphoraTestCase::new(
321                "nom_05",
322                "Apple Inc. reported record earnings. Apple's stock rose 5%.",
323                AntecedentSpan {
324                    text: "Apple Inc.".to_string(),
325                    start: 0,
326                    end: 10,
327                    trigger: None,
328                },
329                AnaphorSpan {
330                    text: "Apple's".to_string(),
331                    start: 37,
332                    end: 44,
333                },
334                AnaphoraType::Nominal,
335            )
336            .with_notes("Possessive form coreference"),
337        );
338
339        // =================================================================
340        // EVENT ANAPHORA (Should fail - no event detection)
341        // =================================================================
342
343        dataset.add(
344            AnaphoraTestCase::new(
345                "event_01",
346                "Russia invaded Ukraine in 2022. This caused a global energy crisis.",
347                AntecedentSpan {
348                    text: "Russia invaded Ukraine in 2022".to_string(),
349                    start: 0,
350                    end: 30,
351                    trigger: Some("invaded".to_string()),
352                },
353                AnaphorSpan {
354                    text: "This".to_string(),
355                    start: 32,
356                    end: 36,
357                },
358                AnaphoraType::Event,
359            )
360            .with_notes(
361                "Classic event anaphora - 'This' refers to invasion EVENT, not Russia or Ukraine",
362            ),
363        );
364
365        dataset.add(
366            AnaphoraTestCase::new(
367                "event_02",
368                "The earthquake struck at dawn. It destroyed thousands of homes.",
369                AntecedentSpan {
370                    text: "The earthquake struck at dawn".to_string(),
371                    start: 0,
372                    end: 29,
373                    trigger: Some("struck".to_string()),
374                },
375                AnaphorSpan {
376                    text: "It".to_string(),
377                    start: 31,
378                    end: 33,
379                },
380                AnaphoraType::Event,
381            )
382            .with_notes("'It' refers to the earthquake event, not just the noun 'earthquake'"),
383        );
384
385        dataset.add(
386            AnaphoraTestCase::new(
387                "event_03",
388                "The merger was announced yesterday. This surprised investors.",
389                AntecedentSpan {
390                    text: "The merger was announced yesterday".to_string(),
391                    start: 0,
392                    end: 34,
393                    trigger: Some("announced".to_string()),
394                },
395                AnaphorSpan {
396                    text: "This".to_string(),
397                    start: 36,
398                    end: 40,
399                },
400                AnaphoraType::Event,
401            )
402            .with_notes("Announcement event, not the merger entity"),
403        );
404
405        dataset.add(
406            AnaphoraTestCase::new(
407                "event_04",
408                "Scientists discovered a new species. This happened in the Amazon.",
409                AntecedentSpan {
410                    text: "Scientists discovered a new species".to_string(),
411                    start: 0,
412                    end: 35,
413                    trigger: Some("discovered".to_string()),
414                },
415                AnaphorSpan {
416                    text: "This".to_string(),
417                    start: 37,
418                    end: 41,
419                },
420                AnaphoraType::Event,
421            )
422            .with_notes("Discovery event"),
423        );
424
425        dataset.add(
426            AnaphoraTestCase::new(
427                "event_05",
428                "The patient underwent surgery. This took six hours.",
429                AntecedentSpan {
430                    text: "The patient underwent surgery".to_string(),
431                    start: 0,
432                    end: 29,
433                    trigger: Some("underwent".to_string()),
434                },
435                AnaphorSpan {
436                    text: "This".to_string(),
437                    start: 31,
438                    end: 35,
439                },
440                AnaphoraType::Event,
441            )
442            .with_notes("Medical procedure event"),
443        );
444
445        // =================================================================
446        // FACT ANAPHORA (Should fail)
447        // =================================================================
448
449        dataset.add(
450            AnaphoraTestCase::new(
451                "fact_01",
452                "The Earth orbits the Sun. This is well established.",
453                AntecedentSpan {
454                    text: "The Earth orbits the Sun".to_string(),
455                    start: 0,
456                    end: 24,
457                    trigger: None,
458                },
459                AnaphorSpan {
460                    text: "This".to_string(),
461                    start: 26,
462                    end: 30,
463                },
464                AnaphoraType::Fact,
465            )
466            .with_notes("'This' refers to the FACT, not Earth or Sun"),
467        );
468
469        dataset.add(
470            AnaphoraTestCase::new(
471                "fact_02",
472                "Water boils at 100 degrees Celsius. This is basic chemistry.",
473                AntecedentSpan {
474                    text: "Water boils at 100 degrees Celsius".to_string(),
475                    start: 0,
476                    end: 34,
477                    trigger: None,
478                },
479                AnaphorSpan {
480                    text: "This".to_string(),
481                    start: 36,
482                    end: 40,
483                },
484                AnaphoraType::Fact,
485            )
486            .with_notes("Scientific fact reference"),
487        );
488
489        dataset.add(
490            AnaphoraTestCase::new(
491                "fact_03",
492                "He lied under oath. This was proven in court.",
493                AntecedentSpan {
494                    text: "He lied under oath".to_string(),
495                    start: 0,
496                    end: 18,
497                    trigger: None,
498                },
499                AnaphorSpan {
500                    text: "This".to_string(),
501                    start: 20,
502                    end: 24,
503                },
504                AnaphoraType::Fact,
505            )
506            .with_notes("Fact about past action"),
507        );
508
509        // =================================================================
510        // PROPOSITION ANAPHORA (Should fail)
511        // =================================================================
512
513        dataset.add(
514            AnaphoraTestCase::new(
515                "prop_01",
516                "She might resign. This worries the board.",
517                AntecedentSpan {
518                    text: "She might resign".to_string(),
519                    start: 0,
520                    end: 16,
521                    trigger: None,
522                },
523                AnaphorSpan {
524                    text: "This".to_string(),
525                    start: 18,
526                    end: 22,
527                },
528                AnaphoraType::Proposition,
529            )
530            .with_notes("'This' refers to the POSSIBILITY of resignation"),
531        );
532
533        dataset.add(
534            AnaphoraTestCase::new(
535                "prop_02",
536                "The company could go bankrupt. This scenario keeps investors awake.",
537                AntecedentSpan {
538                    text: "The company could go bankrupt".to_string(),
539                    start: 0,
540                    end: 29,
541                    trigger: None,
542                },
543                AnaphorSpan {
544                    text: "This scenario".to_string(),
545                    start: 31,
546                    end: 44,
547                },
548                AnaphoraType::Proposition,
549            )
550            .with_notes("Hypothetical proposition"),
551        );
552
553        dataset.add(
554            AnaphoraTestCase::new(
555                "prop_03",
556                "Interest rates may rise again. This possibility concernos economists.",
557                AntecedentSpan {
558                    text: "Interest rates may rise again".to_string(),
559                    start: 0,
560                    end: 29,
561                    trigger: None,
562                },
563                AnaphorSpan {
564                    text: "This possibility".to_string(),
565                    start: 31,
566                    end: 47,
567                },
568                AnaphoraType::Proposition,
569            )
570            .with_notes("Modal proposition"),
571        );
572
573        // =================================================================
574        // SITUATION ANAPHORA (Should fail)
575        // =================================================================
576
577        dataset.add(
578            AnaphoraTestCase::new(
579                "sit_01",
580                "Prices rose while wages fell. This was unsustainable.",
581                AntecedentSpan {
582                    text: "Prices rose while wages fell".to_string(),
583                    start: 0,
584                    end: 28,
585                    trigger: None,
586                },
587                AnaphorSpan {
588                    text: "This".to_string(),
589                    start: 30,
590                    end: 34,
591                },
592                AnaphoraType::Situation,
593            )
594            .with_notes("'This' refers to the combined SITUATION, not prices or wages"),
595        );
596
597        dataset.add(
598            AnaphoraTestCase::new(
599                "sit_02",
600                "Traffic was gridlocked and tempers flared. This chaos lasted hours.",
601                AntecedentSpan {
602                    text: "Traffic was gridlocked and tempers flared".to_string(),
603                    start: 0,
604                    end: 41,
605                    trigger: None,
606                },
607                AnaphorSpan {
608                    text: "This chaos".to_string(),
609                    start: 43,
610                    end: 53,
611                },
612                AnaphoraType::Situation,
613            )
614            .with_notes("Complex situation with multiple aspects"),
615        );
616
617        dataset.add(
618            AnaphoraTestCase::new(
619                "sit_03",
620                "The server crashed, emails were lost, and backups failed. This disaster cost millions.",
621                AntecedentSpan {
622                    text: "The server crashed, emails were lost, and backups failed".to_string(),
623                    start: 0,
624                    end: 56,
625                    trigger: None,
626                },
627                AnaphorSpan {
628                    text: "This disaster".to_string(),
629                    start: 58,
630                    end: 71,
631                },
632                AnaphoraType::Situation,
633            )
634            .with_notes("Multi-clause situation"),
635        );
636
637        dataset
638    }
639
640    /// Extended dataset with ARRAU-style shell noun examples.
641    ///
642    /// ARRAU corpus annotates "discourse deixis" which includes shell nouns
643    /// like "this issue", "the problem", "that decision".
644    #[must_use]
645    pub fn extended() -> Self {
646        let mut dataset = Self::standard();
647
648        // =================================================================
649        // SHELL NOUN CASES (based on Schmid 2000 taxonomy)
650        // =================================================================
651
652        // Factual shell nouns
653        dataset.add(
654            AnaphoraTestCase::new(
655                "shell_fact_01",
656                "The GDP grew by 3%. This fact surprised analysts.",
657                AntecedentSpan {
658                    text: "The GDP grew by 3%".to_string(),
659                    start: 0,
660                    end: 18,
661                    trigger: Some("grew".to_string()),
662                },
663                AnaphorSpan {
664                    text: "This fact".to_string(),
665                    start: 20,
666                    end: 29,
667                },
668                AnaphoraType::Fact,
669            )
670            .with_notes("Shell noun 'fact' - factual class (Schmid 2000)"),
671        );
672
673        dataset.add(
674            AnaphoraTestCase::new(
675                "shell_fact_02",
676                "Prices doubled in one year. The reason was supply chain disruption.",
677                AntecedentSpan {
678                    text: "Prices doubled in one year".to_string(),
679                    start: 0,
680                    end: 26,
681                    trigger: Some("doubled".to_string()),
682                },
683                AnaphorSpan {
684                    text: "The reason".to_string(),
685                    start: 28,
686                    end: 38,
687                },
688                AnaphoraType::Fact,
689            )
690            .with_notes("Shell noun 'reason' - factual class, cataphoric"),
691        );
692
693        // Linguistic shell nouns
694        dataset.add(
695            AnaphoraTestCase::new(
696                "shell_ling_01",
697                "The CEO promised higher wages. This claim was later retracted.",
698                AntecedentSpan {
699                    text: "The CEO promised higher wages".to_string(),
700                    start: 0,
701                    end: 29,
702                    trigger: Some("promised".to_string()),
703                },
704                AnaphorSpan {
705                    text: "This claim".to_string(),
706                    start: 31,
707                    end: 41,
708                },
709                AnaphoraType::Proposition,
710            )
711            .with_notes("Shell noun 'claim' - linguistic class"),
712        );
713
714        dataset.add(
715            AnaphoraTestCase::new(
716                "shell_ling_02",
717                "We should invest in renewables. The argument convinced the board.",
718                AntecedentSpan {
719                    text: "We should invest in renewables".to_string(),
720                    start: 0,
721                    end: 30,
722                    trigger: None,
723                },
724                AnaphorSpan {
725                    text: "The argument".to_string(),
726                    start: 32,
727                    end: 44,
728                },
729                AnaphoraType::Proposition,
730            )
731            .with_notes("Shell noun 'argument' - linguistic class"),
732        );
733
734        // Mental shell nouns
735        dataset.add(
736            AnaphoraTestCase::new(
737                "shell_mental_01",
738                "Automation will replace most jobs. This belief is controversial.",
739                AntecedentSpan {
740                    text: "Automation will replace most jobs".to_string(),
741                    start: 0,
742                    end: 33,
743                    trigger: None,
744                },
745                AnaphorSpan {
746                    text: "This belief".to_string(),
747                    start: 35,
748                    end: 46,
749                },
750                AnaphoraType::Proposition,
751            )
752            .with_notes("Shell noun 'belief' - mental class"),
753        );
754
755        dataset.add(
756            AnaphoraTestCase::new(
757                "shell_mental_02",
758                "The new policy will fail. This view is shared by experts.",
759                AntecedentSpan {
760                    text: "The new policy will fail".to_string(),
761                    start: 0,
762                    end: 24,
763                    trigger: None,
764                },
765                AnaphorSpan {
766                    text: "This view".to_string(),
767                    start: 26,
768                    end: 35,
769                },
770                AnaphoraType::Proposition,
771            )
772            .with_notes("Shell noun 'view' - mental class"),
773        );
774
775        // Modal shell nouns
776        dataset.add(
777            AnaphoraTestCase::new(
778                "shell_modal_01",
779                "The system could crash under load. This possibility concernoed engineers.",
780                AntecedentSpan {
781                    text: "The system could crash under load".to_string(),
782                    start: 0,
783                    end: 33,
784                    trigger: None,
785                },
786                AnaphorSpan {
787                    text: "This possibility".to_string(),
788                    start: 35,
789                    end: 51,
790                },
791                AnaphoraType::Proposition,
792            )
793            .with_notes("Shell noun 'possibility' - modal class"),
794        );
795
796        // Eventive shell nouns
797        dataset.add(
798            AnaphoraTestCase::new(
799                "shell_event_01",
800                "The company laid off 500 workers. This decision shocked employees.",
801                AntecedentSpan {
802                    text: "The company laid off 500 workers".to_string(),
803                    start: 0,
804                    end: 32,
805                    trigger: Some("laid off".to_string()),
806                },
807                AnaphorSpan {
808                    text: "This decision".to_string(),
809                    start: 34,
810                    end: 47,
811                },
812                AnaphoraType::Event,
813            )
814            .with_notes("Shell noun 'decision' - eventive class"),
815        );
816
817        dataset.add(
818            AnaphoraTestCase::new(
819                "shell_event_02",
820                "A meteor struck the desert. The incident was witnessed by campers.",
821                AntecedentSpan {
822                    text: "A meteor struck the desert".to_string(),
823                    start: 0,
824                    end: 26,
825                    trigger: Some("struck".to_string()),
826                },
827                AnaphorSpan {
828                    text: "The incident".to_string(),
829                    start: 28,
830                    end: 40,
831                },
832                AnaphoraType::Event,
833            )
834            .with_notes("Shell noun 'incident' - eventive class"),
835        );
836
837        // Circumstantial shell nouns
838        dataset.add(
839            AnaphoraTestCase::new(
840                "shell_circ_01",
841                "Inflation is rising while wages stagnate. This situation is unsustainable.",
842                AntecedentSpan {
843                    text: "Inflation is rising while wages stagnate".to_string(),
844                    start: 0,
845                    end: 40,
846                    trigger: None,
847                },
848                AnaphorSpan {
849                    text: "This situation".to_string(),
850                    start: 42,
851                    end: 56,
852                },
853                AnaphoraType::Situation,
854            )
855            .with_notes("Shell noun 'situation' - circumstantial class"),
856        );
857
858        dataset.add(
859            AnaphoraTestCase::new(
860                "shell_circ_02",
861                "The code has bugs and the deadline is tomorrow. This problem needs addressing.",
862                AntecedentSpan {
863                    text: "The code has bugs and the deadline is tomorrow".to_string(),
864                    start: 0,
865                    end: 46,
866                    trigger: None,
867                },
868                AnaphorSpan {
869                    text: "This problem".to_string(),
870                    start: 48,
871                    end: 60,
872                },
873                AnaphoraType::Situation,
874            )
875            .with_notes("Shell noun 'problem' - circumstantial class"),
876        );
877
878        // =================================================================
879        // DISCOURSE DISTANCE CASES (testing longer-range anaphora)
880        // =================================================================
881
882        dataset.add(
883            AnaphoraTestCase::new(
884                "dist_01",
885                "The protests began in March. Police deployed tear gas. Several arrests were made. This response drew international criticism.",
886                AntecedentSpan {
887                    text: "Police deployed tear gas. Several arrests were made".to_string(),
888                    start: 29,
889                    end: 80,
890                    trigger: None,
891                },
892                AnaphorSpan {
893                    text: "This response".to_string(),
894                    start: 82,
895                    end: 95,
896                },
897                AnaphoraType::Event,
898            )
899            .with_notes("Multi-sentence antecedent (2 sentences back)"),
900        );
901
902        dataset
903    }
904
905    // =================================================================
906    // Domain-Specific Datasets
907    // =================================================================
908
909    /// Legal domain abstract anaphora dataset.
910    ///
911    /// Legal texts heavily use abstract reference for precedents,
912    /// rulings, statutes, and legal principles.
913    #[must_use]
914    pub fn legal_domain() -> Self {
915        let mut dataset = Self::new();
916
917        dataset.add(
918            AnaphoraTestCase::new(
919                "legal_01",
920                "The court ruled in favor of the plaintiff. This decision sets a precedent.",
921                AntecedentSpan {
922                    text: "The court ruled in favor of the plaintiff".to_string(),
923                    start: 0,
924                    end: 41,
925                    trigger: Some("ruled".to_string()),
926                },
927                AnaphorSpan {
928                    text: "This decision".to_string(),
929                    start: 43,
930                    end: 56,
931                },
932                AnaphoraType::Event,
933            )
934            .with_notes("Court ruling reference"),
935        );
936
937        dataset.add(
938            AnaphoraTestCase::new(
939                "legal_02",
940                "The defendant violated the contract terms. This breach entitles the claimant to damages.",
941                AntecedentSpan {
942                    text: "The defendant violated the contract terms".to_string(),
943                    start: 0,
944                    end: 41,
945                    trigger: Some("violated".to_string()),
946                },
947                AnaphorSpan {
948                    text: "This breach".to_string(),
949                    start: 43,
950                    end: 54,
951                },
952                AnaphoraType::Event,
953            )
954            .with_notes("Legal violation reference"),
955        );
956
957        dataset.add(
958            AnaphoraTestCase::new(
959                "legal_03",
960                "Corporations must disclose material information. Failure to do so constitutes fraud.",
961                AntecedentSpan {
962                    text: "Corporations must disclose material information".to_string(),
963                    start: 0,
964                    end: 47,
965                    trigger: None,
966                },
967                AnaphorSpan {
968                    text: "Failure to do so".to_string(),
969                    start: 49,
970                    end: 65,
971                },
972                AnaphoraType::Fact,
973            )
974            .with_notes("Obligation reference with negation"),
975        );
976
977        dataset.add(
978            AnaphoraTestCase::new(
979                "legal_04",
980                "The statute requires prior notice. This requirement was not met.",
981                AntecedentSpan {
982                    text: "The statute requires prior notice".to_string(),
983                    start: 0,
984                    end: 33,
985                    trigger: Some("requires".to_string()),
986                },
987                AnaphorSpan {
988                    text: "This requirement".to_string(),
989                    start: 35,
990                    end: 51,
991                },
992                AnaphoraType::Fact,
993            )
994            .with_notes("Legal requirement reference"),
995        );
996
997        dataset.add(
998            AnaphoraTestCase::new(
999                "legal_05",
1000                "The witness may have lied. If this is true, perjury charges apply.",
1001                AntecedentSpan {
1002                    text: "The witness may have lied".to_string(),
1003                    start: 0,
1004                    end: 25,
1005                    trigger: Some("lied".to_string()),
1006                },
1007                AnaphorSpan {
1008                    text: "this".to_string(),
1009                    start: 30,
1010                    end: 34,
1011                },
1012                AnaphoraType::Proposition,
1013            )
1014            .with_notes("Modal proposition in legal context"),
1015        );
1016
1017        dataset.add(
1018            AnaphoraTestCase::new(
1019                "legal_06",
1020                "The parties agreed to arbitration. This agreement is binding.",
1021                AntecedentSpan {
1022                    text: "The parties agreed to arbitration".to_string(),
1023                    start: 0,
1024                    end: 33,
1025                    trigger: Some("agreed".to_string()),
1026                },
1027                AnaphorSpan {
1028                    text: "This agreement".to_string(),
1029                    start: 35,
1030                    end: 49,
1031                },
1032                AnaphoraType::Event,
1033            )
1034            .with_notes("Agreement event reference"),
1035        );
1036
1037        dataset.add(
1038            AnaphoraTestCase::new(
1039                "legal_07",
1040                "The prosecution alleged embezzlement. The allegation was later withdrawn.",
1041                AntecedentSpan {
1042                    text: "The prosecution alleged embezzlement".to_string(),
1043                    start: 0,
1044                    end: 36,
1045                    trigger: Some("alleged".to_string()),
1046                },
1047                AnaphorSpan {
1048                    text: "The allegation".to_string(),
1049                    start: 38,
1050                    end: 52,
1051                },
1052                AnaphoraType::Event,
1053            )
1054            .with_notes("Allegation event reference"),
1055        );
1056
1057        dataset.add(
1058            AnaphoraTestCase::new(
1059                "legal_08",
1060                "Evidence was obtained without a warrant. This fact renders it inadmissible.",
1061                AntecedentSpan {
1062                    text: "Evidence was obtained without a warrant".to_string(),
1063                    start: 0,
1064                    end: 39,
1065                    trigger: Some("obtained".to_string()),
1066                },
1067                AnaphorSpan {
1068                    text: "This fact".to_string(),
1069                    start: 41,
1070                    end: 50,
1071                },
1072                AnaphoraType::Fact,
1073            )
1074            .with_notes("Factual shell noun in legal context"),
1075        );
1076
1077        // Nominal case for contrast
1078        dataset.add(
1079            AnaphoraTestCase::new(
1080                "legal_nom_01",
1081                "The defendant hired a lawyer. He filed an appeal.",
1082                AntecedentSpan {
1083                    text: "a lawyer".to_string(),
1084                    start: 21,
1085                    end: 29,
1086                    trigger: None,
1087                },
1088                AnaphorSpan {
1089                    text: "He".to_string(),
1090                    start: 31,
1091                    end: 33,
1092                },
1093                AnaphoraType::Nominal,
1094            )
1095            .with_notes("Standard nominal coreference (lawyer)"),
1096        );
1097
1098        dataset
1099    }
1100
1101    /// Medical/clinical domain abstract anaphora dataset.
1102    ///
1103    /// Medical texts reference diagnoses, procedures, symptoms,
1104    /// and treatment outcomes abstractly.
1105    #[must_use]
1106    pub fn medical_domain() -> Self {
1107        let mut dataset = Self::new();
1108
1109        dataset.add(
1110            AnaphoraTestCase::new(
1111                "med_01",
1112                "The patient presented with chest pain. This symptom suggested cardiac involvement.",
1113                AntecedentSpan {
1114                    text: "The patient presented with chest pain".to_string(),
1115                    start: 0,
1116                    end: 37,
1117                    trigger: Some("presented".to_string()),
1118                },
1119                AnaphorSpan {
1120                    text: "This symptom".to_string(),
1121                    start: 39,
1122                    end: 51,
1123                },
1124                AnaphoraType::Fact,
1125            )
1126            .with_notes("Symptom presentation reference"),
1127        );
1128
1129        dataset.add(
1130            AnaphoraTestCase::new(
1131                "med_02",
1132                "Surgery was performed to remove the tumor. This procedure lasted four hours.",
1133                AntecedentSpan {
1134                    text: "Surgery was performed to remove the tumor".to_string(),
1135                    start: 0,
1136                    end: 41,
1137                    trigger: Some("performed".to_string()),
1138                },
1139                AnaphorSpan {
1140                    text: "This procedure".to_string(),
1141                    start: 43,
1142                    end: 57,
1143                },
1144                AnaphoraType::Event,
1145            )
1146            .with_notes("Surgical procedure reference"),
1147        );
1148
1149        dataset.add(
1150            AnaphoraTestCase::new(
1151                "med_03",
1152                "Blood pressure normalized after treatment. This improvement was sustained.",
1153                AntecedentSpan {
1154                    text: "Blood pressure normalized after treatment".to_string(),
1155                    start: 0,
1156                    end: 41,
1157                    trigger: Some("normalized".to_string()),
1158                },
1159                AnaphorSpan {
1160                    text: "This improvement".to_string(),
1161                    start: 43,
1162                    end: 59,
1163                },
1164                AnaphoraType::Event,
1165            )
1166            .with_notes("Clinical improvement reference"),
1167        );
1168
1169        dataset.add(
1170            AnaphoraTestCase::new(
1171                "med_04",
1172                "The medication may cause drowsiness. This side effect is usually temporary.",
1173                AntecedentSpan {
1174                    text: "The medication may cause drowsiness".to_string(),
1175                    start: 0,
1176                    end: 35,
1177                    trigger: Some("cause".to_string()),
1178                },
1179                AnaphorSpan {
1180                    text: "This side effect".to_string(),
1181                    start: 37,
1182                    end: 53,
1183                },
1184                AnaphoraType::Proposition,
1185            )
1186            .with_notes("Potential side effect reference"),
1187        );
1188
1189        dataset.add(
1190            AnaphoraTestCase::new(
1191                "med_05",
1192                "The patient was diagnosed with diabetes. Managing this condition requires lifestyle changes.",
1193                AntecedentSpan {
1194                    text: "diabetes".to_string(),
1195                    start: 31,
1196                    end: 39,
1197                    trigger: None,
1198                },
1199                AnaphorSpan {
1200                    text: "this condition".to_string(),
1201                    start: 51,
1202                    end: 65,
1203                },
1204                AnaphoraType::Situation,
1205            )
1206            .with_notes("Medical condition reference"),
1207        );
1208
1209        dataset.add(
1210            AnaphoraTestCase::new(
1211                "med_06",
1212                "The biopsy revealed malignant cells. This finding necessitated further testing.",
1213                AntecedentSpan {
1214                    text: "The biopsy revealed malignant cells".to_string(),
1215                    start: 0,
1216                    end: 35,
1217                    trigger: Some("revealed".to_string()),
1218                },
1219                AnaphorSpan {
1220                    text: "This finding".to_string(),
1221                    start: 37,
1222                    end: 49,
1223                },
1224                AnaphoraType::Fact,
1225            )
1226            .with_notes("Diagnostic finding reference"),
1227        );
1228
1229        dataset.add(
1230            AnaphoraTestCase::new(
1231                "med_07",
1232                "The patient's fever spiked overnight. This development concernoed the medical team.",
1233                AntecedentSpan {
1234                    text: "The patient's fever spiked overnight".to_string(),
1235                    start: 0,
1236                    end: 36,
1237                    trigger: Some("spiked".to_string()),
1238                },
1239                AnaphorSpan {
1240                    text: "This development".to_string(),
1241                    start: 38,
1242                    end: 54,
1243                },
1244                AnaphoraType::Event,
1245            )
1246            .with_notes("Clinical event reference"),
1247        );
1248
1249        dataset.add(
1250            AnaphoraTestCase::new(
1251                "med_08",
1252                "Chemotherapy was discontinued due to adverse reactions. This decision was made by the oncologist.",
1253                AntecedentSpan {
1254                    text: "Chemotherapy was discontinued due to adverse reactions".to_string(),
1255                    start: 0,
1256                    end: 54,
1257                    trigger: Some("discontinued".to_string()),
1258                },
1259                AnaphorSpan {
1260                    text: "This decision".to_string(),
1261                    start: 56,
1262                    end: 69,
1263                },
1264                AnaphoraType::Event,
1265            )
1266            .with_notes("Treatment decision reference"),
1267        );
1268
1269        // Nominal case for contrast
1270        dataset.add(
1271            AnaphoraTestCase::new(
1272                "med_nom_01",
1273                "The surgeon consulted a specialist. She recommended immediate intervention.",
1274                AntecedentSpan {
1275                    text: "a specialist".to_string(),
1276                    start: 23,
1277                    end: 35,
1278                    trigger: None,
1279                },
1280                AnaphorSpan {
1281                    text: "She".to_string(),
1282                    start: 37,
1283                    end: 40,
1284                },
1285                AnaphoraType::Nominal,
1286            )
1287            .with_notes("Standard nominal coreference (specialist)"),
1288        );
1289
1290        dataset
1291    }
1292
1293    /// Financial/business domain abstract anaphora dataset.
1294    ///
1295    /// Financial texts reference market events, transactions,
1296    /// decisions, and economic phenomena.
1297    #[must_use]
1298    pub fn financial_domain() -> Self {
1299        let mut dataset = Self::new();
1300
1301        dataset.add(
1302            AnaphoraTestCase::new(
1303                "fin_01",
1304                "The Fed raised interest rates. This move sent shockwaves through markets.",
1305                AntecedentSpan {
1306                    text: "The Fed raised interest rates".to_string(),
1307                    start: 0,
1308                    end: 29,
1309                    trigger: Some("raised".to_string()),
1310                },
1311                AnaphorSpan {
1312                    text: "This move".to_string(),
1313                    start: 31,
1314                    end: 40,
1315                },
1316                AnaphoraType::Event,
1317            )
1318            .with_notes("Policy decision reference"),
1319        );
1320
1321        dataset.add(
1322            AnaphoraTestCase::new(
1323                "fin_02",
1324                "The merger was approved by regulators. This development boosted investor confidence.",
1325                AntecedentSpan {
1326                    text: "The merger was approved by regulators".to_string(),
1327                    start: 0,
1328                    end: 37,
1329                    trigger: Some("approved".to_string()),
1330                },
1331                AnaphorSpan {
1332                    text: "This development".to_string(),
1333                    start: 39,
1334                    end: 55,
1335                },
1336                AnaphoraType::Event,
1337            )
1338            .with_notes("Regulatory approval reference"),
1339        );
1340
1341        dataset.add(
1342            AnaphoraTestCase::new(
1343                "fin_03",
1344                "Quarterly earnings exceeded expectations. This performance led to a stock rally.",
1345                AntecedentSpan {
1346                    text: "Quarterly earnings exceeded expectations".to_string(),
1347                    start: 0,
1348                    end: 40,
1349                    trigger: Some("exceeded".to_string()),
1350                },
1351                AnaphorSpan {
1352                    text: "This performance".to_string(),
1353                    start: 42,
1354                    end: 58,
1355                },
1356                AnaphoraType::Event,
1357            )
1358            .with_notes("Financial performance reference"),
1359        );
1360
1361        dataset.add(
1362            AnaphoraTestCase::new(
1363                "fin_04",
1364                "The company might default on its loans. This risk has alarmed bondholders.",
1365                AntecedentSpan {
1366                    text: "The company might default on its loans".to_string(),
1367                    start: 0,
1368                    end: 38,
1369                    trigger: Some("default".to_string()),
1370                },
1371                AnaphorSpan {
1372                    text: "This risk".to_string(),
1373                    start: 40,
1374                    end: 49,
1375                },
1376                AnaphoraType::Proposition,
1377            )
1378            .with_notes("Financial risk proposition"),
1379        );
1380
1381        dataset.add(
1382            AnaphoraTestCase::new(
1383                "fin_05",
1384                "Supply chain disruptions are causing inflation. This situation could persist for years.",
1385                AntecedentSpan {
1386                    text: "Supply chain disruptions are causing inflation".to_string(),
1387                    start: 0,
1388                    end: 46,
1389                    trigger: Some("causing".to_string()),
1390                },
1391                AnaphorSpan {
1392                    text: "This situation".to_string(),
1393                    start: 48,
1394                    end: 62,
1395                },
1396                AnaphoraType::Situation,
1397            )
1398            .with_notes("Economic situation reference"),
1399        );
1400
1401        dataset.add(
1402            AnaphoraTestCase::new(
1403                "fin_06",
1404                "The CEO announced a stock buyback program. The announcement pushed shares higher.",
1405                AntecedentSpan {
1406                    text: "The CEO announced a stock buyback program".to_string(),
1407                    start: 0,
1408                    end: 41,
1409                    trigger: Some("announced".to_string()),
1410                },
1411                AnaphorSpan {
1412                    text: "The announcement".to_string(),
1413                    start: 43,
1414                    end: 59,
1415                },
1416                AnaphoraType::Event,
1417            )
1418            .with_notes("Corporate announcement reference"),
1419        );
1420
1421        dataset.add(
1422            AnaphoraTestCase::new(
1423                "fin_07",
1424                "Revenue grew by 15% year-over-year. This growth outpaced analyst forecasts.",
1425                AntecedentSpan {
1426                    text: "Revenue grew by 15% year-over-year".to_string(),
1427                    start: 0,
1428                    end: 34,
1429                    trigger: Some("grew".to_string()),
1430                },
1431                AnaphorSpan {
1432                    text: "This growth".to_string(),
1433                    start: 36,
1434                    end: 47,
1435                },
1436                AnaphoraType::Event,
1437            )
1438            .with_notes("Revenue growth event reference"),
1439        );
1440
1441        dataset.add(
1442            AnaphoraTestCase::new(
1443                "fin_08",
1444                "The acquisition was completed yesterday. This transaction creates the largest retailer.",
1445                AntecedentSpan {
1446                    text: "The acquisition was completed yesterday".to_string(),
1447                    start: 0,
1448                    end: 39,
1449                    trigger: Some("completed".to_string()),
1450                },
1451                AnaphorSpan {
1452                    text: "This transaction".to_string(),
1453                    start: 41,
1454                    end: 57,
1455                },
1456                AnaphoraType::Event,
1457            )
1458            .with_notes("Business transaction reference"),
1459        );
1460
1461        // Nominal case for contrast
1462        dataset.add(
1463            AnaphoraTestCase::new(
1464                "fin_nom_01",
1465                "The CFO presented the report. She highlighted key metrics.",
1466                AntecedentSpan {
1467                    text: "The CFO".to_string(),
1468                    start: 0,
1469                    end: 7,
1470                    trigger: None,
1471                },
1472                AnaphorSpan {
1473                    text: "She".to_string(),
1474                    start: 31,
1475                    end: 34,
1476                },
1477                AnaphoraType::Nominal,
1478            )
1479            .with_notes("Standard nominal coreference (CFO)"),
1480        );
1481
1482        dataset
1483    }
1484
1485    /// Scientific/technical domain abstract anaphora dataset.
1486    ///
1487    /// Scientific writing references experiments, observations,
1488    /// hypotheses, and theoretical concepts.
1489    #[must_use]
1490    pub fn scientific_domain() -> Self {
1491        let mut dataset = Self::new();
1492
1493        dataset.add(
1494            AnaphoraTestCase::new(
1495                "sci_01",
1496                "The experiment failed to replicate earlier results. This failure suggests methodological issues.",
1497                AntecedentSpan {
1498                    text: "The experiment failed to replicate earlier results".to_string(),
1499                    start: 0,
1500                    end: 50,
1501                    trigger: Some("failed".to_string()),
1502                },
1503                AnaphorSpan {
1504                    text: "This failure".to_string(),
1505                    start: 52,
1506                    end: 64,
1507                },
1508                AnaphoraType::Event,
1509            )
1510            .with_notes("Experimental failure reference"),
1511        );
1512
1513        dataset.add(
1514            AnaphoraTestCase::new(
1515                "sci_02",
1516                "The data shows a correlation between diet and longevity. This finding aligns with previous studies.",
1517                AntecedentSpan {
1518                    text: "The data shows a correlation between diet and longevity".to_string(),
1519                    start: 0,
1520                    end: 55,
1521                    trigger: Some("shows".to_string()),
1522                },
1523                AnaphorSpan {
1524                    text: "This finding".to_string(),
1525                    start: 57,
1526                    end: 69,
1527                },
1528                AnaphoraType::Fact,
1529            )
1530            .with_notes("Scientific finding reference"),
1531        );
1532
1533        dataset.add(
1534            AnaphoraTestCase::new(
1535                "sci_03",
1536                "Quantum entanglement may enable faster communication. If this is possible, it would revolutionize networking.",
1537                AntecedentSpan {
1538                    text: "Quantum entanglement may enable faster communication".to_string(),
1539                    start: 0,
1540                    end: 52,
1541                    trigger: Some("enable".to_string()),
1542                },
1543                AnaphorSpan {
1544                    text: "this".to_string(),
1545                    start: 57,
1546                    end: 61,
1547                },
1548                AnaphoraType::Proposition,
1549            )
1550            .with_notes("Scientific hypothesis reference"),
1551        );
1552
1553        dataset.add(
1554            AnaphoraTestCase::new(
1555                "sci_04",
1556                "The samples were contaminated during transport. This problem invalidated the study.",
1557                AntecedentSpan {
1558                    text: "The samples were contaminated during transport".to_string(),
1559                    start: 0,
1560                    end: 46,
1561                    trigger: Some("contaminated".to_string()),
1562                },
1563                AnaphorSpan {
1564                    text: "This problem".to_string(),
1565                    start: 48,
1566                    end: 60,
1567                },
1568                AnaphoraType::Event,
1569            )
1570            .with_notes("Experimental problem reference"),
1571        );
1572
1573        dataset.add(
1574            AnaphoraTestCase::new(
1575                "sci_05",
1576                "The protein folded incorrectly under high temperatures. This observation was unexpected.",
1577                AntecedentSpan {
1578                    text: "The protein folded incorrectly under high temperatures".to_string(),
1579                    start: 0,
1580                    end: 54,
1581                    trigger: Some("folded".to_string()),
1582                },
1583                AnaphorSpan {
1584                    text: "This observation".to_string(),
1585                    start: 56,
1586                    end: 72,
1587                },
1588                AnaphoraType::Fact,
1589            )
1590            .with_notes("Observational fact reference"),
1591        );
1592
1593        dataset.add(
1594            AnaphoraTestCase::new(
1595                "sci_06",
1596                "The simulation predicted climate warming. This prediction matched observed data.",
1597                AntecedentSpan {
1598                    text: "The simulation predicted climate warming".to_string(),
1599                    start: 0,
1600                    end: 40,
1601                    trigger: Some("predicted".to_string()),
1602                },
1603                AnaphorSpan {
1604                    text: "This prediction".to_string(),
1605                    start: 42,
1606                    end: 57,
1607                },
1608                AnaphoraType::Fact,
1609            )
1610            .with_notes("Model prediction reference"),
1611        );
1612
1613        dataset.add(
1614            AnaphoraTestCase::new(
1615                "sci_07",
1616                "The theory was disproven by new evidence. Despite this setback, research continues.",
1617                AntecedentSpan {
1618                    text: "The theory was disproven by new evidence".to_string(),
1619                    start: 0,
1620                    end: 40,
1621                    trigger: Some("disproven".to_string()),
1622                },
1623                AnaphorSpan {
1624                    text: "this setback".to_string(),
1625                    start: 50,
1626                    end: 62,
1627                },
1628                AnaphoraType::Event,
1629            )
1630            .with_notes("Scientific setback reference"),
1631        );
1632
1633        dataset.add(
1634            AnaphoraTestCase::new(
1635                "sci_08",
1636                "The algorithm achieved high accuracy. This result was widely discussed.",
1637                AntecedentSpan {
1638                    text: "The algorithm achieved high accuracy".to_string(),
1639                    start: 0,
1640                    end: 35,
1641                    trigger: Some("achieved".to_string()),
1642                },
1643                AnaphorSpan {
1644                    text: "This result".to_string(),
1645                    start: 37,
1646                    end: 48,
1647                },
1648                AnaphoraType::Fact,
1649            )
1650            .with_notes("Experimental result reference"),
1651        );
1652
1653        // Nominal case for contrast
1654        dataset.add(
1655            AnaphoraTestCase::new(
1656                "sci_nom_01",
1657                "The researcher published her findings. She received several awards.",
1658                AntecedentSpan {
1659                    text: "The researcher".to_string(),
1660                    start: 0,
1661                    end: 14,
1662                    trigger: None,
1663                },
1664                AnaphorSpan {
1665                    text: "She".to_string(),
1666                    start: 40,
1667                    end: 43,
1668                },
1669                AnaphoraType::Nominal,
1670            )
1671            .with_notes("Standard nominal coreference (researcher)"),
1672        );
1673
1674        dataset
1675    }
1676
1677    /// News/journalism domain abstract anaphora dataset.
1678    ///
1679    /// News articles reference events, statements, developments,
1680    /// and reactions.
1681    #[must_use]
1682    pub fn news_domain() -> Self {
1683        let mut dataset = Self::new();
1684
1685        dataset.add(
1686            AnaphoraTestCase::new(
1687                "news_01",
1688                "The president signed the bill into law. This action fulfilled a campaign promise.",
1689                AntecedentSpan {
1690                    text: "The president signed the bill into law".to_string(),
1691                    start: 0,
1692                    end: 38,
1693                    trigger: Some("signed".to_string()),
1694                },
1695                AnaphorSpan {
1696                    text: "This action".to_string(),
1697                    start: 40,
1698                    end: 51,
1699                },
1700                AnaphoraType::Event,
1701            )
1702            .with_notes("Political action reference"),
1703        );
1704
1705        dataset.add(
1706            AnaphoraTestCase::new(
1707                "news_02",
1708                "Protests erupted across major cities. This unrest prompted a government response.",
1709                AntecedentSpan {
1710                    text: "Protests erupted across major cities".to_string(),
1711                    start: 0,
1712                    end: 36,
1713                    trigger: Some("erupted".to_string()),
1714                },
1715                AnaphorSpan {
1716                    text: "This unrest".to_string(),
1717                    start: 38,
1718                    end: 49,
1719                },
1720                AnaphoraType::Event,
1721            )
1722            .with_notes("Social unrest reference"),
1723        );
1724
1725        dataset.add(
1726            AnaphoraTestCase::new(
1727                "news_03",
1728                "The minister denied any wrongdoing. This denial contradicted earlier statements.",
1729                AntecedentSpan {
1730                    text: "The minister denied any wrongdoing".to_string(),
1731                    start: 0,
1732                    end: 34,
1733                    trigger: Some("denied".to_string()),
1734                },
1735                AnaphorSpan {
1736                    text: "This denial".to_string(),
1737                    start: 36,
1738                    end: 47,
1739                },
1740                AnaphoraType::Event,
1741            )
1742            .with_notes("Statement/denial reference"),
1743        );
1744
1745        dataset.add(
1746            AnaphoraTestCase::new(
1747                "news_04",
1748                "Peace talks collapsed after three days. The breakdown disappointed international observers.",
1749                AntecedentSpan {
1750                    text: "Peace talks collapsed after three days".to_string(),
1751                    start: 0,
1752                    end: 38,
1753                    trigger: Some("collapsed".to_string()),
1754                },
1755                AnaphorSpan {
1756                    text: "The breakdown".to_string(),
1757                    start: 40,
1758                    end: 53,
1759                },
1760                AnaphoraType::Event,
1761            )
1762            .with_notes("Diplomatic breakdown reference"),
1763        );
1764
1765        dataset.add(
1766            AnaphoraTestCase::new(
1767                "news_05",
1768                "The hurricane devastated coastal towns. This disaster left thousands homeless.",
1769                AntecedentSpan {
1770                    text: "The hurricane devastated coastal towns".to_string(),
1771                    start: 0,
1772                    end: 38,
1773                    trigger: Some("devastated".to_string()),
1774                },
1775                AnaphorSpan {
1776                    text: "This disaster".to_string(),
1777                    start: 40,
1778                    end: 53,
1779                },
1780                AnaphoraType::Event,
1781            )
1782            .with_notes("Natural disaster reference"),
1783        );
1784
1785        dataset.add(
1786            AnaphoraTestCase::new(
1787                "news_06",
1788                "The celebrity apologized publicly. This apology came after widespread backlash.",
1789                AntecedentSpan {
1790                    text: "The celebrity apologized publicly".to_string(),
1791                    start: 0,
1792                    end: 33,
1793                    trigger: Some("apologized".to_string()),
1794                },
1795                AnaphorSpan {
1796                    text: "This apology".to_string(),
1797                    start: 35,
1798                    end: 47,
1799                },
1800                AnaphoraType::Event,
1801            )
1802            .with_notes("Public apology reference"),
1803        );
1804
1805        dataset.add(
1806            AnaphoraTestCase::new(
1807                "news_07",
1808                "The election results were contested. This controversy led to legal challenges.",
1809                AntecedentSpan {
1810                    text: "The election results were contested".to_string(),
1811                    start: 0,
1812                    end: 35,
1813                    trigger: Some("contested".to_string()),
1814                },
1815                AnaphorSpan {
1816                    text: "This controversy".to_string(),
1817                    start: 37,
1818                    end: 53,
1819                },
1820                AnaphoraType::Event,
1821            )
1822            .with_notes("Political controversy reference"),
1823        );
1824
1825        dataset.add(
1826            AnaphoraTestCase::new(
1827                "news_08",
1828                "Unemployment fell to a historic low. This improvement boosted consumer spending.",
1829                AntecedentSpan {
1830                    text: "Unemployment fell to a historic low".to_string(),
1831                    start: 0,
1832                    end: 35,
1833                    trigger: Some("fell".to_string()),
1834                },
1835                AnaphorSpan {
1836                    text: "This improvement".to_string(),
1837                    start: 37,
1838                    end: 53,
1839                },
1840                AnaphoraType::Event,
1841            )
1842            .with_notes("Economic improvement reference"),
1843        );
1844
1845        // Nominal case for contrast
1846        dataset.add(
1847            AnaphoraTestCase::new(
1848                "news_nom_01",
1849                "The mayor addressed the media. He promised immediate action.",
1850                AntecedentSpan {
1851                    text: "The mayor".to_string(),
1852                    start: 0,
1853                    end: 9,
1854                    trigger: None,
1855                },
1856                AnaphorSpan {
1857                    text: "He".to_string(),
1858                    start: 32,
1859                    end: 34,
1860                },
1861                AnaphoraType::Nominal,
1862            )
1863            .with_notes("Standard nominal coreference (mayor)"),
1864        );
1865
1866        dataset
1867    }
1868
1869    /// Complex/challenging abstract anaphora cases.
1870    ///
1871    /// These cases test difficult scenarios: long-distance,
1872    /// cataphoric, ambiguous, and multi-clause antecedents.
1873    #[must_use]
1874    pub fn challenging_cases() -> Self {
1875        let mut dataset = Self::new();
1876
1877        // Long-distance anaphora (multiple sentences back)
1878        dataset.add(
1879            AnaphoraTestCase::new(
1880                "chal_01",
1881                "The company reported strong earnings. Analysts praised the results. Investors celebrated. This success was unexpected.",
1882                AntecedentSpan {
1883                    text: "The company reported strong earnings".to_string(),
1884                    start: 0,
1885                    end: 36,
1886                    trigger: Some("reported".to_string()),
1887                },
1888                AnaphorSpan {
1889                    text: "This success".to_string(),
1890                    start: 91,
1891                    end: 103,
1892                },
1893                AnaphoraType::Event,
1894            )
1895            .with_notes("Long-distance (3 sentences back)"),
1896        );
1897
1898        // Cataphoric reference (anaphor before antecedent)
1899        dataset.add(
1900            AnaphoraTestCase::new(
1901                "chal_02",
1902                "This much is clear: the policy has failed.",
1903                AntecedentSpan {
1904                    text: "the policy has failed".to_string(),
1905                    start: 20,
1906                    end: 41,
1907                    trigger: Some("failed".to_string()),
1908                },
1909                AnaphorSpan {
1910                    text: "This much".to_string(),
1911                    start: 0,
1912                    end: 9,
1913                },
1914                AnaphoraType::Fact,
1915            )
1916            .with_notes("Cataphoric reference"),
1917        );
1918
1919        // Complex multi-clause antecedent
1920        dataset.add(
1921            AnaphoraTestCase::new(
1922                "chal_03",
1923                "Inflation rose while wages stagnated and unemployment increased. This combination created economic hardship.",
1924                AntecedentSpan {
1925                    text: "Inflation rose while wages stagnated and unemployment increased".to_string(),
1926                    start: 0,
1927                    end: 63,
1928                    trigger: None,
1929                },
1930                AnaphorSpan {
1931                    text: "This combination".to_string(),
1932                    start: 65,
1933                    end: 81,
1934                },
1935                AnaphoraType::Situation,
1936            )
1937            .with_notes("Multi-clause conjunction antecedent"),
1938        );
1939
1940        // Embedded clause antecedent
1941        dataset.add(
1942            AnaphoraTestCase::new(
1943                "chal_04",
1944                "The CEO said that layoffs were necessary. This claim angered workers.",
1945                AntecedentSpan {
1946                    text: "layoffs were necessary".to_string(),
1947                    start: 18,
1948                    end: 40,
1949                    trigger: None,
1950                },
1951                AnaphorSpan {
1952                    text: "This claim".to_string(),
1953                    start: 42,
1954                    end: 52,
1955                },
1956                AnaphoraType::Proposition,
1957            )
1958            .with_notes("Embedded clause antecedent"),
1959        );
1960
1961        // Negated antecedent
1962        dataset.add(
1963            AnaphoraTestCase::new(
1964                "chal_05",
1965                "The witness did not appear in court. This absence was noted by the judge.",
1966                AntecedentSpan {
1967                    text: "The witness did not appear in court".to_string(),
1968                    start: 0,
1969                    end: 35,
1970                    trigger: Some("appear".to_string()),
1971                },
1972                AnaphorSpan {
1973                    text: "This absence".to_string(),
1974                    start: 37,
1975                    end: 49,
1976                },
1977                AnaphoraType::Event,
1978            )
1979            .with_notes("Negated event antecedent"),
1980        );
1981
1982        // Disjunction antecedent
1983        dataset.add(
1984            AnaphoraTestCase::new(
1985                "chal_06",
1986                "Either the system crashed or data was corrupted. This problem halted operations.",
1987                AntecedentSpan {
1988                    text: "Either the system crashed or data was corrupted".to_string(),
1989                    start: 0,
1990                    end: 47,
1991                    trigger: None,
1992                },
1993                AnaphorSpan {
1994                    text: "This problem".to_string(),
1995                    start: 49,
1996                    end: 61,
1997                },
1998                AnaphoraType::Situation,
1999            )
2000            .with_notes("Disjunction antecedent"),
2001        );
2002
2003        // Conditional antecedent
2004        dataset.add(
2005            AnaphoraTestCase::new(
2006                "chal_07",
2007                "If interest rates rise, housing prices will fall. This scenario worries homeowners.",
2008                AntecedentSpan {
2009                    text: "If interest rates rise, housing prices will fall".to_string(),
2010                    start: 0,
2011                    end: 48,
2012                    trigger: None,
2013                },
2014                AnaphorSpan {
2015                    text: "This scenario".to_string(),
2016                    start: 50,
2017                    end: 63,
2018                },
2019                AnaphoraType::Proposition,
2020            )
2021            .with_notes("Conditional antecedent"),
2022        );
2023
2024        // Comparator antecedent
2025        dataset.add(
2026            AnaphoraTestCase::new(
2027                "chal_08",
2028                "Profits are higher than last year. This exceeds expectations.",
2029                AntecedentSpan {
2030                    text: "Profits are higher than last year".to_string(),
2031                    start: 0,
2032                    end: 33,
2033                    trigger: None,
2034                },
2035                AnaphorSpan {
2036                    text: "This".to_string(),
2037                    start: 35,
2038                    end: 39,
2039                },
2040                AnaphoraType::Fact,
2041            )
2042            .with_notes("Comparative statement antecedent"),
2043        );
2044
2045        // Question as antecedent
2046        dataset.add(
2047            AnaphoraTestCase::new(
2048                "chal_09",
2049                "Will the company surcerno? This question haunts investors.",
2050                AntecedentSpan {
2051                    text: "Will the company surcerno".to_string(),
2052                    start: 0,
2053                    end: 24,
2054                    trigger: None,
2055                },
2056                AnaphorSpan {
2057                    text: "This question".to_string(),
2058                    start: 27,
2059                    end: 40,
2060                },
2061                AnaphoraType::Proposition,
2062            )
2063            .with_notes("Interrogative clause antecedent"),
2064        );
2065
2066        // Generic statement antecedent
2067        dataset.add(
2068            AnaphoraTestCase::new(
2069                "chal_10",
2070                "Power corrupts. This truth has been known for centuries.",
2071                AntecedentSpan {
2072                    text: "Power corrupts".to_string(),
2073                    start: 0,
2074                    end: 14,
2075                    trigger: Some("corrupts".to_string()),
2076                },
2077                AnaphorSpan {
2078                    text: "This truth".to_string(),
2079                    start: 16,
2080                    end: 26,
2081                },
2082                AnaphoraType::Fact,
2083            )
2084            .with_notes("Generic statement antecedent"),
2085        );
2086
2087        dataset
2088    }
2089
2090    /// Create a comprehensive dataset combining all domains.
2091    ///
2092    /// This is the most complete test set for abstract anaphora.
2093    #[must_use]
2094    pub fn comprehensive() -> Self {
2095        let mut dataset = Self::extended();
2096
2097        // Merge in domain-specific cases
2098        for case in Self::legal_domain().cases {
2099            dataset.add(case);
2100        }
2101        for case in Self::medical_domain().cases {
2102            dataset.add(case);
2103        }
2104        for case in Self::financial_domain().cases {
2105            dataset.add(case);
2106        }
2107        for case in Self::scientific_domain().cases {
2108            dataset.add(case);
2109        }
2110        for case in Self::news_domain().cases {
2111            dataset.add(case);
2112        }
2113        for case in Self::challenging_cases().cases {
2114            dataset.add(case);
2115        }
2116
2117        dataset
2118    }
2119
2120    /// Statistics about the dataset.
2121    #[must_use]
2122    pub fn stats(&self) -> DatasetStats {
2123        let mut by_type: HashMap<AnaphoraType, usize> = HashMap::new();
2124        for case in &self.cases {
2125            *by_type.entry(case.anaphora_type).or_default() += 1;
2126        }
2127
2128        DatasetStats {
2129            total: self.cases.len(),
2130            nominal: by_type.get(&AnaphoraType::Nominal).copied().unwrap_or(0),
2131            event: by_type.get(&AnaphoraType::Event).copied().unwrap_or(0),
2132            fact: by_type.get(&AnaphoraType::Fact).copied().unwrap_or(0),
2133            proposition: by_type
2134                .get(&AnaphoraType::Proposition)
2135                .copied()
2136                .unwrap_or(0),
2137            situation: by_type.get(&AnaphoraType::Situation).copied().unwrap_or(0),
2138        }
2139    }
2140}
2141
2142/// Dataset statistics.
2143#[derive(Debug, Clone, Serialize, Deserialize)]
2144pub struct DatasetStats {
2145    /// Total number of test cases
2146    pub total: usize,
2147    /// Nominal coreference cases
2148    pub nominal: usize,
2149    /// Event anaphora cases
2150    pub event: usize,
2151    /// Fact anaphora cases
2152    pub fact: usize,
2153    /// Proposition anaphora cases
2154    pub proposition: usize,
2155    /// Situation anaphora cases
2156    pub situation: usize,
2157}
2158
2159impl DatasetStats {
2160    /// Total abstract (non-nominal) cases.
2161    #[must_use]
2162    pub fn abstract_total(&self) -> usize {
2163        self.event + self.fact + self.proposition + self.situation
2164    }
2165}
2166
2167/// Analysis of shell noun distribution in a dataset.
2168#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2169pub struct ShellNounAnalysis {
2170    /// Total shell nouns detected
2171    pub total_shell_nouns: usize,
2172    /// Shell nouns by semantic class
2173    pub by_class: HashMap<ShellNounClass, usize>,
2174    /// How many are demonstrative ("this X")
2175    pub demonstrative_count: usize,
2176    /// How many have matching antecedent type for their class
2177    pub type_match_count: usize,
2178}
2179
2180// =============================================================================
2181// Candidate Ranking Metrics (Marasović 2017 style)
2182// =============================================================================
2183
2184/// Candidate ranking evaluation metrics.
2185///
2186/// These metrics match the evaluation approach from:
2187/// - Marasović et al. (2017): "A Mention-Ranking Model for Abstract Anaphora Resolution"
2188/// - Li & Ng (2022): "End-to-End Neural Discourse Deixis Resolution"
2189#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2190pub struct CandidateRankingMetrics {
2191    /// Accuracy@1: proportion where correct candidate ranked first
2192    pub accuracy_at_1: f64,
2193    /// Mean Reciprocal Rank: average of 1/rank of correct candidate
2194    pub mrr: f64,
2195    /// Antecedent containment: proportion where gold in candidate set
2196    pub containment: f64,
2197    /// Average number of candidates per case
2198    pub avg_candidates: f64,
2199    /// Total cases evaluated
2200    pub total_cases: usize,
2201}
2202
2203impl CandidateRankingMetrics {
2204    /// Compute metrics from a list of (gold_rank, num_candidates) tuples.
2205    ///
2206    /// `gold_rank` is 1-indexed (1 = ranked first), 0 = not in candidates
2207    #[must_use]
2208    pub fn from_rankings(rankings: &[(usize, usize)]) -> Self {
2209        if rankings.is_empty() {
2210            return Self::default();
2211        }
2212
2213        let total = rankings.len();
2214        let mut correct_at_1 = 0;
2215        let mut reciprocal_sum = 0.0;
2216        let mut contained = 0;
2217        let mut total_candidates = 0;
2218
2219        for &(gold_rank, num_candidates) in rankings {
2220            total_candidates += num_candidates;
2221
2222            if gold_rank > 0 {
2223                contained += 1;
2224                reciprocal_sum += 1.0 / gold_rank as f64;
2225
2226                if gold_rank == 1 {
2227                    correct_at_1 += 1;
2228                }
2229            }
2230        }
2231
2232        Self {
2233            accuracy_at_1: correct_at_1 as f64 / total as f64,
2234            mrr: reciprocal_sum / total as f64,
2235            containment: contained as f64 / total as f64,
2236            avg_candidates: total_candidates as f64 / total as f64,
2237            total_cases: total,
2238        }
2239    }
2240
2241    /// Summary string.
2242    #[must_use]
2243    pub fn summary(&self) -> String {
2244        format!(
2245            "Candidate Ranking:\n  Accuracy@1: {:.1}%\n  MRR: {:.3}\n  Containment: {:.1}%\n  Avg candidates: {:.1}",
2246            self.accuracy_at_1 * 100.0,
2247            self.mrr,
2248            self.containment * 100.0,
2249            self.avg_candidates
2250        )
2251    }
2252}
2253
2254impl ShellNounAnalysis {
2255    /// Percentage of shell nouns that are demonstrative.
2256    #[must_use]
2257    pub fn demonstrative_ratio(&self) -> f64 {
2258        if self.total_shell_nouns == 0 {
2259            0.0
2260        } else {
2261            self.demonstrative_count as f64 / self.total_shell_nouns as f64
2262        }
2263    }
2264
2265    /// Percentage of shell nouns whose class matches the antecedent type.
2266    #[must_use]
2267    pub fn type_match_ratio(&self) -> f64 {
2268        if self.total_shell_nouns == 0 {
2269            0.0
2270        } else {
2271            self.type_match_count as f64 / self.total_shell_nouns as f64
2272        }
2273    }
2274
2275    /// Summary string.
2276    #[must_use]
2277    pub fn summary(&self) -> String {
2278        let mut s = format!(
2279            "Shell nouns: {} total, {:.0}% demonstrative, {:.0}% type-matched\n",
2280            self.total_shell_nouns,
2281            self.demonstrative_ratio() * 100.0,
2282            self.type_match_ratio() * 100.0
2283        );
2284        for (class, count) in &self.by_class {
2285            s.push_str(&format!("  {}: {}\n", class.as_str(), count));
2286        }
2287        s
2288    }
2289}
2290
2291// =============================================================================
2292// Evaluator
2293// =============================================================================
2294
2295/// Resolver backend for the evaluator.
2296#[derive(Debug, Clone)]
2297pub enum ResolverBackend {
2298    /// Simple rule-based nominal resolver (baseline)
2299    Simple(SimpleCorefResolver),
2300    /// Discourse-aware resolver with event extraction
2301    DiscourseAware,
2302}
2303
2304impl Default for ResolverBackend {
2305    fn default() -> Self {
2306        ResolverBackend::Simple(SimpleCorefResolver::default())
2307    }
2308}
2309
2310/// Evaluator for abstract anaphora resolution.
2311#[derive(Debug, Clone)]
2312pub struct AbstractAnaphoraEvaluator {
2313    resolver: SimpleCorefResolver,
2314    use_discourse: bool,
2315}
2316
2317impl Default for AbstractAnaphoraEvaluator {
2318    fn default() -> Self {
2319        Self::new(SimpleCorefResolver::default())
2320    }
2321}
2322
2323impl AbstractAnaphoraEvaluator {
2324    /// Create with a specific resolver.
2325    #[must_use]
2326    pub fn new(resolver: SimpleCorefResolver) -> Self {
2327        Self {
2328            resolver,
2329            use_discourse: false,
2330        }
2331    }
2332
2333    /// Create an evaluator using discourse-aware resolution.
2334    ///
2335    /// This evaluator will use `DiscourseAwareResolver` with event extraction
2336    /// to attempt resolution of abstract anaphora.
2337    #[must_use]
2338    pub fn discourse_aware() -> Self {
2339        Self {
2340            resolver: SimpleCorefResolver::default(),
2341            use_discourse: true,
2342        }
2343    }
2344
2345    /// Enable or disable discourse-aware resolution.
2346    #[must_use]
2347    pub fn with_discourse(mut self, enable: bool) -> Self {
2348        self.use_discourse = enable;
2349        self
2350    }
2351
2352    /// Evaluate the resolver on a dataset.
2353    #[must_use]
2354    pub fn evaluate(&self, dataset: &AbstractAnaphoraDataset) -> EvaluationResults {
2355        let mut results = EvaluationResults::default();
2356
2357        for case in &dataset.cases {
2358            let result = if self.use_discourse {
2359                self.evaluate_case_discourse(case)
2360            } else {
2361                self.evaluate_case(case)
2362            };
2363            results.case_results.push(result.clone());
2364
2365            if case.anaphora_type == AnaphoraType::Nominal {
2366                results.nominal_total += 1;
2367                if result.resolved_correctly {
2368                    results.nominal_correct += 1;
2369                }
2370            } else {
2371                results.abstract_total += 1;
2372                if result.resolved_correctly {
2373                    results.abstract_correct += 1;
2374                }
2375                // Track by specific type
2376                results
2377                    .by_type
2378                    .entry(case.anaphora_type)
2379                    .or_insert_with(TypeResults::default)
2380                    .add(&result);
2381            }
2382        }
2383
2384        results.compute_accuracy();
2385        results
2386    }
2387
2388    /// Evaluate a single test case using the simple resolver.
2389    fn evaluate_case(&self, case: &AnaphoraTestCase) -> CaseResult {
2390        // Extract entities from the text
2391        // For nominal cases, we create Person/Org entities
2392        // For abstract cases, we also create entities but the resolver won't link events
2393
2394        let entities = self.extract_entities_for_case(case);
2395        let resolved = self.resolver.resolve(&entities);
2396
2397        // Check if antecedent and anaphor got the same canonical_id
2398        let antecedent_id = resolved
2399            .iter()
2400            .find(|e| {
2401                e.start() == case.antecedent.start
2402                    || self.text_matches(&e.text, &case.antecedent.text)
2403            })
2404            .and_then(|e| e.canonical_id.map(|id| id.get()));
2405
2406        let anaphor_id = resolved
2407            .iter()
2408            .find(|e| {
2409                e.start() == case.anaphor.start || self.text_matches(&e.text, &case.anaphor.text)
2410            })
2411            .and_then(|e| e.canonical_id.map(|id| id.get()));
2412
2413        let resolved_correctly = match (antecedent_id, anaphor_id) {
2414            (Some(a), Some(b)) => a == b,
2415            _ => false,
2416        };
2417
2418        CaseResult {
2419            case_id: case.id.clone(),
2420            anaphora_type: case.anaphora_type,
2421            resolved_correctly,
2422            antecedent_found: antecedent_id.is_some(),
2423            anaphor_found: anaphor_id.is_some(),
2424            antecedent_id,
2425            anaphor_id,
2426            failure_reason: if resolved_correctly {
2427                None
2428            } else {
2429                Some(self.diagnose_failure(case, antecedent_id, anaphor_id))
2430            },
2431        }
2432    }
2433
2434    /// Evaluate a single test case using the discourse-aware resolver.
2435    fn evaluate_case_discourse(&self, case: &AnaphoraTestCase) -> CaseResult {
2436        let config = DiscourseCorefConfig::default();
2437        let resolver = DiscourseAwareResolver::new(config, &case.text);
2438
2439        // For abstract cases, check if we can find the event antecedent
2440        if case.anaphora_type.is_abstract() {
2441            // Create an Entity from the anaphor span for the resolver
2442            let anaphor_entity = anno::Entity::new(
2443                &case.anaphor.text,
2444                anno::EntityType::custom("Anaphor", anno::EntityCategory::Misc),
2445                case.anaphor.start,
2446                case.anaphor.end,
2447                1.0,
2448            );
2449
2450            // Try to find the anaphor in the text and get its antecedent
2451            let resolved_correctly =
2452                if let Some(referent) = resolver.find_discourse_antecedent(&anaphor_entity) {
2453                    // Check if the found antecedent overlaps with the expected one
2454                    let spans_overlap = referent.start < case.antecedent.end
2455                        && referent.end > case.antecedent.start;
2456
2457                    // Or check if the referent text contains the trigger
2458                    let trigger_found = case
2459                        .antecedent
2460                        .trigger
2461                        .as_ref()
2462                        .map(|t| referent.text.as_ref().is_some_and(|rt| rt.contains(t)))
2463                        .unwrap_or(false);
2464
2465                    spans_overlap || trigger_found
2466                } else {
2467                    false
2468                };
2469
2470            CaseResult {
2471                case_id: case.id.clone(),
2472                anaphora_type: case.anaphora_type,
2473                resolved_correctly,
2474                antecedent_found: true, // If we're here, there's an antecedent in the case
2475                anaphor_found: true,
2476                antecedent_id: Some(0),
2477                anaphor_id: if resolved_correctly { Some(0) } else { None },
2478                failure_reason: if resolved_correctly {
2479                    None
2480                } else {
2481                    Some("Discourse resolver couldn't find event antecedent".to_string())
2482                },
2483            }
2484        } else {
2485            // For nominal cases, use the standard resolver
2486            self.evaluate_case(case)
2487        }
2488    }
2489
2490    /// Extract entities for evaluation.
2491    ///
2492    /// This simulates what a NER system would produce.
2493    fn extract_entities_for_case(&self, case: &AnaphoraTestCase) -> Vec<Entity> {
2494        let mut entities = Vec::new();
2495
2496        // For nominal cases, create person/org entities
2497        if case.anaphora_type == AnaphoraType::Nominal {
2498            // Antecedent is typically a named entity
2499            entities.push(Entity::new(
2500                &case.antecedent.text,
2501                self.infer_entity_type(&case.antecedent.text),
2502                case.antecedent.start,
2503                case.antecedent.end,
2504                0.9,
2505            ));
2506
2507            // Anaphor might be a pronoun or definite NP
2508            entities.push(Entity::new(
2509                &case.anaphor.text,
2510                self.infer_entity_type(&case.anaphor.text),
2511                case.anaphor.start,
2512                case.anaphor.end,
2513                0.85,
2514            ));
2515        } else {
2516            // For abstract anaphora, we still need to create some entities
2517            // The resolver will try to resolve, but should fail because
2518            // the antecedent is an EVENT, not an entity
2519
2520            // Extract any named entities from the antecedent text
2521            let antecedent_entities =
2522                self.extract_named_entities(&case.antecedent.text, case.antecedent.start);
2523            entities.extend(antecedent_entities);
2524
2525            // Add the anaphor (This/That/It)
2526            entities.push(Entity::new(
2527                &case.anaphor.text,
2528                EntityType::custom("abstract_anaphor", EntityCategory::Misc),
2529                case.anaphor.start,
2530                case.anaphor.end,
2531                0.8,
2532            ));
2533        }
2534
2535        entities
2536    }
2537
2538    /// Extract named entities from text (simple heuristic).
2539    ///
2540    /// `offset` is a **character offset** into the original document.
2541    fn extract_named_entities(&self, text: &str, offset: usize) -> Vec<Entity> {
2542        let mut entities = Vec::new();
2543
2544        // Simple capitalized word detection (Unicode-safe offsets).
2545        let mut prev_is_ws = true;
2546        let mut char_idx = 0usize;
2547        for (byte_idx, c) in text.char_indices() {
2548            if c.is_uppercase() && (byte_idx == 0 || prev_is_ws) {
2549                // Find end of word by scanning forward from this byte index.
2550                let mut end_byte = text.len();
2551                let mut end_char_idx = char_idx;
2552                for (j, cc) in text[byte_idx..].char_indices() {
2553                    if j == 0 {
2554                        continue;
2555                    }
2556                    if cc.is_whitespace() || cc == '.' || cc == ',' {
2557                        end_byte = byte_idx + j;
2558                        // Number of chars from start to delimiter gives end char idx.
2559                        end_char_idx = char_idx + text[byte_idx..end_byte].chars().count();
2560                        break;
2561                    }
2562                }
2563                if end_byte == text.len() {
2564                    end_char_idx = char_idx + text[byte_idx..].chars().count();
2565                }
2566
2567                let word = &text[byte_idx..end_byte];
2568                if word.chars().count() > 1 && !self.is_sentence_starter(word, char_idx) {
2569                    entities.push(Entity::new(
2570                        word,
2571                        self.infer_entity_type(word),
2572                        offset + char_idx,
2573                        offset + end_char_idx,
2574                        0.7,
2575                    ));
2576                }
2577            }
2578            prev_is_ws = c.is_whitespace();
2579            char_idx += 1;
2580        }
2581
2582        entities
2583    }
2584
2585    /// Check if word is just a sentence starter.
2586    fn is_sentence_starter(&self, word: &str, pos: usize) -> bool {
2587        pos == 0
2588            && matches!(
2589                word.to_lowercase().as_str(),
2590                "the" | "a" | "an" | "this" | "that" | "it" | "he" | "she" | "they"
2591            )
2592    }
2593
2594    /// Infer entity type from text.
2595    fn infer_entity_type(&self, text: &str) -> EntityType {
2596        let lower = text.to_lowercase();
2597
2598        // Pronouns
2599        if matches!(
2600            lower.as_str(),
2601            "he" | "him" | "his" | "she" | "her" | "hers" | "they" | "them" | "their"
2602        ) {
2603            return EntityType::Person;
2604        }
2605
2606        // Definite NPs for organizations
2607        if lower.starts_with("the company")
2608            || lower.starts_with("the firm")
2609            || lower.starts_with("the organization")
2610        {
2611            return EntityType::Organization;
2612        }
2613
2614        // Common org suffixes
2615        if text.ends_with("Inc.") || text.ends_with("Corp.") || text.ends_with("LLC") {
2616            return EntityType::Organization;
2617        }
2618
2619        // Title prefixes suggest person
2620        if text.starts_with("Dr.")
2621            || text.starts_with("Mr.")
2622            || text.starts_with("Ms.")
2623            || text.starts_with("Prof.")
2624        {
2625            return EntityType::Person;
2626        }
2627
2628        // Default to person for proper nouns
2629        if text.chars().next().is_some_and(|c| c.is_uppercase()) {
2630            return EntityType::Person;
2631        }
2632
2633        EntityType::custom("unknown", EntityCategory::Misc)
2634    }
2635
2636    /// Check if texts match (case-insensitive, ignoring punctuation).
2637    fn text_matches(&self, a: &str, b: &str) -> bool {
2638        let normalize = |s: &str| {
2639            s.to_lowercase()
2640                .chars()
2641                .filter(|c| c.is_alphanumeric() || c.is_whitespace())
2642                .collect::<String>()
2643        };
2644        normalize(a) == normalize(b)
2645    }
2646
2647    /// Detect if the anaphor is a shell noun phrase (e.g., "this problem").
2648    ///
2649    /// Shell nouns are abstract nouns that encapsulate propositions.
2650    /// Recognizing them is a first step toward abstract anaphora resolution.
2651    #[must_use]
2652    pub fn detect_shell_noun(&self, anaphor_text: &str) -> Option<ShellNoun> {
2653        let words: Vec<&str> = anaphor_text.split_whitespace().collect();
2654
2655        // Check for "this/that/the + noun" patterns
2656        if words.len() >= 2 {
2657            let determiner = words[0].to_lowercase();
2658            if matches!(
2659                determiner.as_str(),
2660                "this" | "that" | "the" | "these" | "those"
2661            ) {
2662                // Safe: words.len() >= 2 ensures last() returns Some
2663                let noun = words
2664                    .last()
2665                    .expect("words has at least 2 elements")
2666                    .to_lowercase();
2667                // Remove trailing punctuation
2668                let noun = noun.trim_matches(|c: char| !c.is_alphanumeric());
2669
2670                if let Some(class) = classify_shell_noun(noun) {
2671                    return Some(
2672                        ShellNoun::new(noun, class)
2673                            .with_determiner(&determiner)
2674                            .with_full_text(anaphor_text),
2675                    );
2676                }
2677            }
2678        }
2679
2680        // Check single-word shell nouns
2681        if words.len() == 1 {
2682            let noun = words[0].to_lowercase();
2683            let noun = noun.trim_matches(|c: char| !c.is_alphanumeric());
2684            if let Some(class) = classify_shell_noun(noun) {
2685                return Some(ShellNoun::new(noun, class).with_full_text(anaphor_text));
2686            }
2687        }
2688
2689        None
2690    }
2691
2692    /// Analyze shell noun distribution in a dataset.
2693    #[must_use]
2694    pub fn analyze_shell_nouns(&self, dataset: &AbstractAnaphoraDataset) -> ShellNounAnalysis {
2695        let mut analysis = ShellNounAnalysis::default();
2696
2697        for case in &dataset.cases {
2698            if let Some(shell) = self.detect_shell_noun(&case.anaphor.text) {
2699                analysis.total_shell_nouns += 1;
2700                *analysis.by_class.entry(shell.class).or_default() += 1;
2701
2702                if shell.is_demonstrative() {
2703                    analysis.demonstrative_count += 1;
2704                }
2705
2706                // Check if shell noun class matches anaphora type
2707                let expected_types = shell.typical_antecedent_types();
2708                let actual_type: ReferentType = match case.anaphora_type {
2709                    AnaphoraType::Nominal => ReferentType::Nominal,
2710                    AnaphoraType::Event => ReferentType::Event,
2711                    AnaphoraType::Fact => ReferentType::Fact,
2712                    AnaphoraType::Proposition => ReferentType::Proposition,
2713                    AnaphoraType::Situation => ReferentType::Situation,
2714                };
2715
2716                if expected_types.contains(&actual_type) {
2717                    analysis.type_match_count += 1;
2718                }
2719            }
2720        }
2721
2722        analysis
2723    }
2724
2725    /// Diagnose why resolution failed.
2726    fn diagnose_failure(
2727        &self,
2728        case: &AnaphoraTestCase,
2729        antecedent_id: Option<u64>,
2730        anaphor_id: Option<u64>,
2731    ) -> String {
2732        // Check for shell noun
2733        let shell_info = if let Some(shell) = self.detect_shell_noun(&case.anaphor.text) {
2734            format!(" [shell noun: {} ({})]", shell.lemma, shell.class.as_str())
2735        } else {
2736            String::new()
2737        };
2738
2739        if case.anaphora_type.is_abstract() {
2740            return format!(
2741                "Abstract anaphora ({}) - resolver cannot detect event/proposition antecedents{}",
2742                case.anaphora_type.as_str(),
2743                shell_info
2744            );
2745        }
2746
2747        match (antecedent_id, anaphor_id) {
2748            (None, None) => "Neither antecedent nor anaphor was assigned a cluster".to_string(),
2749            (None, Some(_)) => "Antecedent was not assigned a cluster".to_string(),
2750            (Some(_), None) => "Anaphor was not assigned a cluster".to_string(),
2751            (Some(a), Some(b)) => format!("Assigned to different clusters: {} vs {}", a, b),
2752        }
2753    }
2754}
2755
2756// =============================================================================
2757// Results
2758// =============================================================================
2759
2760/// Results for a single test case.
2761#[derive(Debug, Clone, Serialize, Deserialize)]
2762pub struct CaseResult {
2763    /// Test case identifier
2764    pub case_id: String,
2765    /// Type of anaphora being tested
2766    pub anaphora_type: AnaphoraType,
2767    /// Whether resolution was correct
2768    pub resolved_correctly: bool,
2769    /// Whether antecedent was found/assigned a cluster
2770    pub antecedent_found: bool,
2771    /// Whether anaphor was found/assigned a cluster
2772    pub anaphor_found: bool,
2773    /// Cluster ID assigned to antecedent
2774    pub antecedent_id: Option<u64>,
2775    /// Cluster ID assigned to anaphor
2776    pub anaphor_id: Option<u64>,
2777    /// Explanation of failure if resolution failed
2778    pub failure_reason: Option<String>,
2779}
2780
2781/// Results aggregated by anaphora type.
2782#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2783pub struct TypeResults {
2784    /// Total cases of this type
2785    pub total: usize,
2786    /// Correctly resolved cases
2787    pub correct: usize,
2788}
2789
2790impl TypeResults {
2791    fn add(&mut self, result: &CaseResult) {
2792        self.total += 1;
2793        if result.resolved_correctly {
2794            self.correct += 1;
2795        }
2796    }
2797
2798    /// Accuracy for this type.
2799    #[must_use]
2800    pub fn accuracy(&self) -> f64 {
2801        if self.total == 0 {
2802            0.0
2803        } else {
2804            self.correct as f64 / self.total as f64
2805        }
2806    }
2807}
2808
2809/// Full evaluation results.
2810#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2811pub struct EvaluationResults {
2812    /// Individual case results
2813    pub case_results: Vec<CaseResult>,
2814    /// Total nominal coreference cases
2815    pub nominal_total: usize,
2816    /// Correctly resolved nominal cases
2817    pub nominal_correct: usize,
2818    /// Nominal accuracy (correct/total)
2819    pub nominal_accuracy: f64,
2820    /// Total abstract anaphora cases
2821    pub abstract_total: usize,
2822    /// Correctly resolved abstract cases
2823    pub abstract_correct: usize,
2824    /// Abstract accuracy (correct/total)
2825    pub abstract_accuracy: f64,
2826    /// Results by abstract type (event, fact, etc.)
2827    pub by_type: HashMap<AnaphoraType, TypeResults>,
2828}
2829
2830/// LEA (Link-based Entity-Aware) analysis split by anaphora type.
2831///
2832/// LEA is the recommended metric (Moosavi & Strube 2016) because it:
2833/// - Is link-based (like MUC) so it measures resolution quality
2834/// - Is entity-aware (like B³) so it weights by entity importance
2835/// - Avoids the "mention identification effect" that inflates B³/CEAF
2836#[derive(Debug, Clone, Default)]
2837pub struct LeaAnalysis {
2838    /// LEA scores for nominal coreference cases
2839    pub nominal: CorefScores,
2840    /// LEA scores for abstract anaphora cases
2841    pub abstract_anaphora: CorefScores,
2842}
2843
2844impl LeaAnalysis {
2845    /// The F1 gap between nominal and abstract LEA.
2846    #[must_use]
2847    pub fn f1_gap(&self) -> f64 {
2848        self.nominal.f1 - self.abstract_anaphora.f1
2849    }
2850
2851    /// Summary string.
2852    #[must_use]
2853    pub fn summary(&self) -> String {
2854        format!(
2855            "LEA Analysis:\n  Nominal:  P={:.1}% R={:.1}% F1={:.1}%\n  Abstract: P={:.1}% R={:.1}% F1={:.1}%\n  Gap: {:.1}pp",
2856            self.nominal.precision * 100.0,
2857            self.nominal.recall * 100.0,
2858            self.nominal.f1 * 100.0,
2859            self.abstract_anaphora.precision * 100.0,
2860            self.abstract_anaphora.recall * 100.0,
2861            self.abstract_anaphora.f1 * 100.0,
2862            self.f1_gap() * 100.0
2863        )
2864    }
2865}
2866
2867impl EvaluationResults {
2868    fn compute_accuracy(&mut self) {
2869        self.nominal_accuracy = if self.nominal_total == 0 {
2870            0.0
2871        } else {
2872            self.nominal_correct as f64 / self.nominal_total as f64
2873        };
2874
2875        self.abstract_accuracy = if self.abstract_total == 0 {
2876            0.0
2877        } else {
2878            self.abstract_correct as f64 / self.abstract_total as f64
2879        };
2880    }
2881
2882    /// The gap between nominal and abstract accuracy.
2883    #[must_use]
2884    pub fn accuracy_gap(&self) -> f64 {
2885        self.nominal_accuracy - self.abstract_accuracy
2886    }
2887
2888    /// Compute LEA scores from the case results.
2889    ///
2890    /// Converts our case-level results to coreference chains and
2891    /// computes the LEA metric (Moosavi & Strube, 2016).
2892    ///
2893    /// Returns LEA scores separately for nominal and abstract cases.
2894    ///
2895    /// Note: LEA requires the same mention set in both gold and predicted.
2896    /// When prediction is incorrect, we split the gold chain into singletons.
2897    #[must_use]
2898    pub fn compute_lea_scores(&self, dataset: &AbstractAnaphoraDataset) -> LeaAnalysis {
2899        let mut nominal_gold = Vec::new();
2900        let mut nominal_pred = Vec::new();
2901        let mut abstract_gold = Vec::new();
2902        let mut abstract_pred = Vec::new();
2903
2904        for (case, result) in dataset.cases.iter().zip(self.case_results.iter()) {
2905            let antecedent_mention = Mention::new(
2906                &case.antecedent.text,
2907                case.antecedent.start,
2908                case.antecedent.end,
2909            );
2910            let anaphor_mention =
2911                Mention::new(&case.anaphor.text, case.anaphor.start, case.anaphor.end);
2912
2913            // Create gold chain: antecedent + anaphor should corefer
2914            let gold_chain =
2915                CorefChain::new(vec![antecedent_mention.clone(), anaphor_mention.clone()]);
2916
2917            // Create predicted chains based on resolution result
2918            // LEA requires same mention set, so we always include both mentions
2919            let pred_chains: Vec<CorefChain> = if result.resolved_correctly {
2920                // Correct: both in same chain
2921                vec![CorefChain::new(vec![
2922                    antecedent_mention.clone(),
2923                    anaphor_mention.clone(),
2924                ])]
2925            } else {
2926                // Incorrect: each mention in its own singleton chain
2927                vec![
2928                    CorefChain::new(vec![antecedent_mention.clone()]),
2929                    CorefChain::new(vec![anaphor_mention.clone()]),
2930                ]
2931            };
2932
2933            if case.anaphora_type == AnaphoraType::Nominal {
2934                nominal_gold.push(gold_chain);
2935                nominal_pred.extend(pred_chains);
2936            } else {
2937                abstract_gold.push(gold_chain);
2938                abstract_pred.extend(pred_chains);
2939            }
2940        }
2941
2942        let nominal_lea = lea_score(&nominal_pred, &nominal_gold);
2943        let abstract_lea = lea_score(&abstract_pred, &abstract_gold);
2944
2945        LeaAnalysis {
2946            nominal: CorefScores::new(nominal_lea.0, nominal_lea.1),
2947            abstract_anaphora: CorefScores::new(abstract_lea.0, abstract_lea.1),
2948        }
2949    }
2950
2951    /// Generate a summary report.
2952    #[must_use]
2953    pub fn summary(&self) -> String {
2954        let mut s = String::new();
2955
2956        s.push_str("=== Abstract Anaphora Evaluation Results ===\n\n");
2957
2958        s.push_str(&format!(
2959            "Nominal Coreference: {}/{} ({:.1}%)\n",
2960            self.nominal_correct,
2961            self.nominal_total,
2962            self.nominal_accuracy * 100.0
2963        ));
2964
2965        s.push_str(&format!(
2966            "Abstract Anaphora:   {}/{} ({:.1}%)\n",
2967            self.abstract_correct,
2968            self.abstract_total,
2969            self.abstract_accuracy * 100.0
2970        ));
2971
2972        s.push_str(&format!(
2973            "\nAccuracy Gap: {:.1} percentage points\n",
2974            self.accuracy_gap() * 100.0
2975        ));
2976
2977        s.push_str("\n--- By Abstract Type ---\n");
2978        for (atype, results) in &self.by_type {
2979            s.push_str(&format!(
2980                "  {}: {}/{} ({:.1}%)\n",
2981                atype.as_str(),
2982                results.correct,
2983                results.total,
2984                results.accuracy() * 100.0
2985            ));
2986        }
2987
2988        s.push_str("\n--- Failure Analysis ---\n");
2989        let failures: Vec<_> = self
2990            .case_results
2991            .iter()
2992            .filter(|r| !r.resolved_correctly)
2993            .collect();
2994        for result in failures.iter().take(10) {
2995            s.push_str(&format!(
2996                "  [{}] {}: {}\n",
2997                result.case_id,
2998                result.anaphora_type.as_str(),
2999                result.failure_reason.as_deref().unwrap_or("unknown")
3000            ));
3001        }
3002        if failures.len() > 10 {
3003            s.push_str(&format!(
3004                "  ... and {} more failures\n",
3005                failures.len() - 10
3006            ));
3007        }
3008
3009        s
3010    }
3011
3012    /// Generate HTML report.
3013    #[must_use]
3014    pub fn to_html(&self, dataset: &AbstractAnaphoraDataset) -> String {
3015        let mut html = String::new();
3016
3017        html.push_str(
3018            r#"<!DOCTYPE html>
3019<html lang="en">
3020<head>
3021    <meta charset="UTF-8">
3022    <meta name="viewport" content="width=device-width, initial-scale=1.0">
3023    <title>Abstract Anaphora Evaluation</title>
3024    <style>
3025        :root {
3026            --bg: #0d1117;
3027            --fg: #c9d1d9;
3028            --accent: #58a6ff;
3029            --success: #3fb950;
3030            --failure: #f85149;
3031            --warning: #d29922;
3032            --border: #30363d;
3033            --card-bg: #161b22;
3034        }
3035        body {
3036            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
3037            background: var(--bg);
3038            color: var(--fg);
3039            margin: 0;
3040            padding: 2rem;
3041            line-height: 1.6;
3042        }
3043        h1, h2, h3 { color: var(--accent); margin-top: 2rem; }
3044        h1 { font-size: 2rem; border-bottom: 1px solid var(--border); padding-bottom: 0.5rem; }
3045        .summary-cards {
3046            display: grid;
3047            grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
3048            gap: 1rem;
3049            margin: 1.5rem 0;
3050        }
3051        .card {
3052            background: var(--card-bg);
3053            border: 1px solid var(--border);
3054            border-radius: 8px;
3055            padding: 1.5rem;
3056            text-align: center;
3057        }
3058        .card-value {
3059            font-size: 2.5rem;
3060            font-weight: bold;
3061        }
3062        .card-label { color: #8b949e; margin-top: 0.5rem; }
3063        .success { color: var(--success); }
3064        .failure { color: var(--failure); }
3065        .warning { color: var(--warning); }
3066        table {
3067            width: 100%;
3068            border-collapse: collapse;
3069            margin: 1rem 0;
3070        }
3071        th, td {
3072            border: 1px solid var(--border);
3073            padding: 0.75rem;
3074            text-align: left;
3075        }
3076        th { background: var(--card-bg); color: var(--accent); }
3077        tr:nth-child(even) { background: rgba(255,255,255,0.02); }
3078        .badge {
3079            display: inline-block;
3080            padding: 0.25rem 0.5rem;
3081            border-radius: 4px;
3082            font-size: 0.85rem;
3083            font-weight: 500;
3084        }
3085        .badge-success { background: rgba(63,185,80,0.2); color: var(--success); }
3086        .badge-failure { background: rgba(248,81,73,0.2); color: var(--failure); }
3087        .badge-nominal { background: rgba(88,166,255,0.2); color: var(--accent); }
3088        .badge-abstract { background: rgba(210,153,34,0.2); color: var(--warning); }
3089        .case-text {
3090            font-family: 'SF Mono', Monaco, monospace;
3091            background: var(--card-bg);
3092            padding: 0.75rem;
3093            border-radius: 4px;
3094            margin: 0.5rem 0;
3095            font-size: 0.9rem;
3096        }
3097        .antecedent { background: rgba(63,185,80,0.3); padding: 2px 4px; border-radius: 2px; }
3098        .anaphor { background: rgba(248,81,73,0.3); padding: 2px 4px; border-radius: 2px; }
3099        .conclusion {
3100            background: var(--card-bg);
3101            border-left: 4px solid var(--failure);
3102            padding: 1rem 1.5rem;
3103            margin: 2rem 0;
3104        }
3105        .chart-bar {
3106            height: 24px;
3107            background: var(--border);
3108            border-radius: 4px;
3109            overflow: hidden;
3110            margin: 0.5rem 0;
3111        }
3112        .chart-fill {
3113            height: 100%;
3114            transition: width 0.3s;
3115        }
3116    </style>
3117</head>
3118<body>
3119    <h1>Abstract Anaphora Evaluation Report</h1>
3120    <p>Demonstrating the gap between nominal coreference and abstract anaphora resolution.</p>
3121"#,
3122        );
3123
3124        // Summary cards
3125        html.push_str(
3126            r#"
3127    <div class="summary-cards">
3128        <div class="card">
3129            <div class="card-value success">"#,
3130        );
3131        html.push_str(&format!("{:.0}%", self.nominal_accuracy * 100.0));
3132        html.push_str(
3133            r#"</div>
3134            <div class="card-label">Nominal Accuracy</div>
3135        </div>
3136        <div class="card">
3137            <div class="card-value failure">"#,
3138        );
3139        html.push_str(&format!("{:.0}%", self.abstract_accuracy * 100.0));
3140        html.push_str(
3141            r#"</div>
3142            <div class="card-label">Abstract Accuracy</div>
3143        </div>
3144        <div class="card">
3145            <div class="card-value warning">"#,
3146        );
3147        html.push_str(&format!("{:.0}pp", self.accuracy_gap() * 100.0));
3148        html.push_str(
3149            r#"</div>
3150            <div class="card-label">Performance Gap</div>
3151        </div>
3152        <div class="card">
3153            <div class="card-value">"#,
3154        );
3155        html.push_str(&format!("{}", self.case_results.len()));
3156        html.push_str(
3157            r#"</div>
3158            <div class="card-label">Test Cases</div>
3159        </div>
3160    </div>
3161"#,
3162        );
3163
3164        // Accuracy by type
3165        html.push_str(
3166            r#"
3167    <h2>Accuracy by Anaphora Type</h2>
3168    <table>
3169        <tr>
3170            <th>Type</th>
3171            <th>Correct</th>
3172            <th>Total</th>
3173            <th>Accuracy</th>
3174            <th>Visual</th>
3175        </tr>
3176        <tr>
3177            <td><span class="badge badge-nominal">Nominal</span></td>
3178            <td>"#,
3179        );
3180        html.push_str(&format!("{}", self.nominal_correct));
3181        html.push_str("</td><td>");
3182        html.push_str(&format!("{}", self.nominal_total));
3183        html.push_str("</td><td class=\"success\">");
3184        html.push_str(&format!("{:.1}%", self.nominal_accuracy * 100.0));
3185        html.push_str(
3186            r#"</td>
3187            <td><div class="chart-bar"><div class="chart-fill" style="width: "#,
3188        );
3189        html.push_str(&format!("{}%", (self.nominal_accuracy * 100.0) as u32));
3190        html.push_str(
3191            r#"; background: var(--success);"></div></div></td>
3192        </tr>"#,
3193        );
3194
3195        for (atype, results) in &self.by_type {
3196            html.push_str(&format!(r#"
3197        <tr>
3198            <td><span class="badge badge-abstract">{}</span></td>
3199            <td>{}</td>
3200            <td>{}</td>
3201            <td class="failure">{:.1}%</td>
3202            <td><div class="chart-bar"><div class="chart-fill" style="width: {}%; background: var(--failure);"></div></div></td>
3203        </tr>"#,
3204                atype.as_str(),
3205                results.correct,
3206                results.total,
3207                results.accuracy() * 100.0,
3208                (results.accuracy() * 100.0) as u32
3209            ));
3210        }
3211
3212        html.push_str("</table>");
3213
3214        // Conclusion
3215        html.push_str(
3216            r#"
3217    <div class="conclusion">
3218        <h3 style="margin-top: 0;">Conclusion</h3>
3219        <p>The current <code>SimpleCorefResolver</code> achieves <strong class="success">"#,
3220        );
3221        html.push_str(&format!("{:.0}%", self.nominal_accuracy * 100.0));
3222        html.push_str(
3223            r#"</strong> accuracy on nominal coreference but
3224        <strong class="failure">"#,
3225        );
3226        html.push_str(&format!("{:.0}%", self.abstract_accuracy * 100.0));
3227        html.push_str(
3228            r#"</strong> on abstract anaphora.</p>
3229        <p>This "#,
3230        );
3231        html.push_str(&format!("{:.0}", self.accuracy_gap() * 100.0));
3232        html.push_str(r#" percentage point gap demonstrates that:</p>
3233        <ul>
3234            <li>The resolver has <strong>no mechanism</strong> to detect event/proposition antecedents</li>
3235            <li>Abstract pronouns ("this", "that") are linked to the nearest <em>entity</em>, not to events</li>
3236            <li>Solving this requires event extraction + discourse structure modeling</li>
3237        </ul>
3238    </div>
3239"#);
3240
3241        // Detailed case results
3242        html.push_str(
3243            r#"
3244    <h2>Detailed Results</h2>
3245    <table>
3246        <tr>
3247            <th>ID</th>
3248            <th>Type</th>
3249            <th>Result</th>
3250            <th>Text (highlighted)</th>
3251            <th>Failure Reason</th>
3252        </tr>"#,
3253        );
3254
3255        for (case, result) in dataset.cases.iter().zip(self.case_results.iter()) {
3256            let badge_class = if result.resolved_correctly {
3257                "badge-success"
3258            } else {
3259                "badge-failure"
3260            };
3261            let result_text = if result.resolved_correctly {
3262                "PASS"
3263            } else {
3264                "FAIL"
3265            };
3266
3267            // Highlight text
3268            let mut highlighted = case.text.clone();
3269            // Insert spans (do anaphor first if it comes after antecedent)
3270            if case.anaphor.start > case.antecedent.end {
3271                highlighted.insert_str(case.anaphor.end, "</span>");
3272                highlighted.insert_str(case.anaphor.start, "<span class=\"anaphor\">");
3273                highlighted.insert_str(case.antecedent.end, "</span>");
3274                highlighted.insert_str(case.antecedent.start, "<span class=\"antecedent\">");
3275            }
3276
3277            html.push_str(&format!(
3278                r#"
3279        <tr>
3280            <td>{}</td>
3281            <td><span class="badge {}">{}</span></td>
3282            <td><span class="badge {}">{}</span></td>
3283            <td class="case-text">{}</td>
3284            <td>{}</td>
3285        </tr>"#,
3286                result.case_id,
3287                if result.anaphora_type.is_abstract() {
3288                    "badge-abstract"
3289                } else {
3290                    "badge-nominal"
3291                },
3292                result.anaphora_type.as_str(),
3293                badge_class,
3294                result_text,
3295                highlighted,
3296                result.failure_reason.as_deref().unwrap_or("-")
3297            ));
3298        }
3299
3300        html.push_str("</table>");
3301
3302        // Footer
3303        html.push_str(r#"
3304    <footer style="margin-top: 3rem; padding-top: 1rem; border-top: 1px solid var(--border); color: #8b949e; font-size: 0.9rem;">
3305        <p>Generated by <code>anno_eval::eval::abstract_anaphora</code></p>
3306        <p>See <code>docs/</code> for repo-local notes and entry points.</p>
3307    </footer>
3308</body>
3309</html>"#);
3310
3311        html
3312    }
3313}
3314
3315// =============================================================================
3316// Tests
3317// =============================================================================
3318
3319#[cfg(test)]
3320mod tests {
3321    use super::*;
3322
3323    #[test]
3324    fn test_dataset_creation() {
3325        let dataset = AbstractAnaphoraDataset::standard();
3326        let stats = dataset.stats();
3327
3328        assert!(stats.total > 0, "Dataset should have cases");
3329        assert!(stats.nominal > 0, "Should have nominal cases");
3330        assert!(stats.abstract_total() > 0, "Should have abstract cases");
3331
3332        println!("Dataset stats: {:?}", stats);
3333    }
3334
3335    #[test]
3336    fn test_anaphora_types() {
3337        assert!(!AnaphoraType::Nominal.is_abstract());
3338        assert!(AnaphoraType::Event.is_abstract());
3339        assert!(AnaphoraType::Fact.is_abstract());
3340        assert!(AnaphoraType::Proposition.is_abstract());
3341        assert!(AnaphoraType::Situation.is_abstract());
3342    }
3343
3344    #[test]
3345    fn test_evaluation_runs() {
3346        let dataset = AbstractAnaphoraDataset::standard();
3347        let evaluator = AbstractAnaphoraEvaluator::default();
3348        let results = evaluator.evaluate(&dataset);
3349
3350        println!("{}", results.summary());
3351
3352        // Nominal should do better than abstract
3353        // (We expect nominal to be decent, abstract to be near 0)
3354        assert!(
3355            results.nominal_accuracy >= results.abstract_accuracy,
3356            "Nominal accuracy ({:.1}%) should be >= abstract ({:.1}%)",
3357            results.nominal_accuracy * 100.0,
3358            results.abstract_accuracy * 100.0
3359        );
3360    }
3361
3362    #[test]
3363    fn test_accuracy_gap_exists() {
3364        let dataset = AbstractAnaphoraDataset::standard();
3365        let evaluator = AbstractAnaphoraEvaluator::default();
3366        let results = evaluator.evaluate(&dataset);
3367
3368        // The gap should be substantial (this is the point of the research)
3369        let gap = results.accuracy_gap();
3370        println!("Accuracy gap: {:.1} percentage points", gap * 100.0);
3371
3372        // We expect at least some gap - if nominal works at all
3373        // and abstract doesn't, gap should be positive
3374        if results.nominal_accuracy > 0.0 {
3375            assert!(
3376                gap > 0.0,
3377                "Expected positive accuracy gap, got {:.1}pp",
3378                gap * 100.0
3379            );
3380        }
3381    }
3382
3383    #[test]
3384    fn test_html_generation() {
3385        let dataset = AbstractAnaphoraDataset::standard();
3386        let evaluator = AbstractAnaphoraEvaluator::default();
3387        let results = evaluator.evaluate(&dataset);
3388
3389        let html = results.to_html(&dataset);
3390        assert!(html.contains("Abstract Anaphora Evaluation"));
3391        assert!(html.contains("Nominal Accuracy"));
3392        assert!(html.contains("Abstract Accuracy"));
3393    }
3394
3395    // =================================================================
3396    // Domain-Specific Dataset Tests
3397    // =================================================================
3398
3399    #[test]
3400    fn test_legal_domain_dataset() {
3401        let dataset = AbstractAnaphoraDataset::legal_domain();
3402        let stats = dataset.stats();
3403
3404        assert!(
3405            stats.total >= 8,
3406            "Legal domain should have at least 8 cases"
3407        );
3408        assert!(stats.abstract_total() >= 7, "Most should be abstract");
3409
3410        // Should have at least one nominal for contrast
3411        assert!(stats.nominal >= 1, "Should include nominal baseline case");
3412    }
3413
3414    #[test]
3415    fn test_medical_domain_dataset() {
3416        let dataset = AbstractAnaphoraDataset::medical_domain();
3417        let stats = dataset.stats();
3418
3419        assert!(
3420            stats.total >= 8,
3421            "Medical domain should have at least 8 cases"
3422        );
3423        assert!(
3424            stats.event >= 3,
3425            "Medical should have event cases (procedures)"
3426        );
3427    }
3428
3429    #[test]
3430    fn test_financial_domain_dataset() {
3431        let dataset = AbstractAnaphoraDataset::financial_domain();
3432        let stats = dataset.stats();
3433
3434        assert!(
3435            stats.total >= 8,
3436            "Financial domain should have at least 8 cases"
3437        );
3438        assert!(
3439            stats.event >= 4,
3440            "Financial should have event cases (transactions)"
3441        );
3442    }
3443
3444    #[test]
3445    fn test_scientific_domain_dataset() {
3446        let dataset = AbstractAnaphoraDataset::scientific_domain();
3447        let stats = dataset.stats();
3448
3449        assert!(
3450            stats.total >= 8,
3451            "Scientific domain should have at least 8 cases"
3452        );
3453        assert!(
3454            stats.fact >= 3,
3455            "Scientific should have fact cases (findings)"
3456        );
3457    }
3458
3459    #[test]
3460    fn test_news_domain_dataset() {
3461        let dataset = AbstractAnaphoraDataset::news_domain();
3462        let stats = dataset.stats();
3463
3464        assert!(stats.total >= 8, "News domain should have at least 8 cases");
3465        assert!(stats.event >= 5, "News should have many event cases");
3466    }
3467
3468    #[test]
3469    fn test_challenging_cases_dataset() {
3470        let dataset = AbstractAnaphoraDataset::challenging_cases();
3471        let stats = dataset.stats();
3472
3473        assert!(
3474            stats.total >= 10,
3475            "Challenging cases should have at least 10 cases"
3476        );
3477
3478        // These are specifically hard cases, so no nominal baselines
3479        assert!(
3480            stats.abstract_total() == stats.total,
3481            "All challenging cases should be abstract"
3482        );
3483    }
3484
3485    #[test]
3486    fn test_comprehensive_dataset() {
3487        let dataset = AbstractAnaphoraDataset::comprehensive();
3488        let stats = dataset.stats();
3489
3490        // Comprehensive should include all other datasets
3491        let extended_stats = AbstractAnaphoraDataset::extended().stats();
3492        let legal_count = AbstractAnaphoraDataset::legal_domain().stats().total;
3493        let medical_count = AbstractAnaphoraDataset::medical_domain().stats().total;
3494        let financial_count = AbstractAnaphoraDataset::financial_domain().stats().total;
3495        let scientific_count = AbstractAnaphoraDataset::scientific_domain().stats().total;
3496        let news_count = AbstractAnaphoraDataset::news_domain().stats().total;
3497        let challenging_count = AbstractAnaphoraDataset::challenging_cases().stats().total;
3498
3499        let expected_min = extended_stats.total
3500            + legal_count
3501            + medical_count
3502            + financial_count
3503            + scientific_count
3504            + news_count
3505            + challenging_count;
3506
3507        assert!(
3508            stats.total >= expected_min,
3509            "Comprehensive should have at least {} cases, got {}",
3510            expected_min,
3511            stats.total
3512        );
3513
3514        println!("Comprehensive dataset stats:");
3515        println!("  Total: {}", stats.total);
3516        println!("  Nominal: {}", stats.nominal);
3517        println!("  Event: {}", stats.event);
3518        println!("  Fact: {}", stats.fact);
3519        println!("  Proposition: {}", stats.proposition);
3520        println!("  Situation: {}", stats.situation);
3521    }
3522
3523    #[test]
3524    fn test_domain_dataset_evaluation() {
3525        // Run evaluation on each domain dataset
3526        let evaluator = AbstractAnaphoraEvaluator::default();
3527
3528        let domains = [
3529            ("Legal", AbstractAnaphoraDataset::legal_domain()),
3530            ("Medical", AbstractAnaphoraDataset::medical_domain()),
3531            ("Financial", AbstractAnaphoraDataset::financial_domain()),
3532            ("Scientific", AbstractAnaphoraDataset::scientific_domain()),
3533            ("News", AbstractAnaphoraDataset::news_domain()),
3534        ];
3535
3536        for (name, dataset) in domains {
3537            let results = evaluator.evaluate(&dataset);
3538            println!(
3539                "{} domain: {:.1}% abstract accuracy",
3540                name,
3541                results.abstract_accuracy * 100.0
3542            );
3543
3544            // Simple resolver should struggle with all domains
3545            assert!(
3546                results.abstract_accuracy < 0.5,
3547                "{} domain: Simple resolver shouldn't exceed 50% on abstract cases",
3548                name
3549            );
3550        }
3551    }
3552}