1use crate::discourse::{classify_shell_noun, ReferentType, ShellNoun, ShellNounClass};
58use crate::eval::coref::{CorefChain, Mention};
59use crate::eval::coref_metrics::{lea_score, CorefScores};
60use crate::eval::coref_resolver::{
61 DiscourseAwareResolver, DiscourseCorefConfig, SimpleCorefResolver,
62};
63use crate::{Entity, EntityCategory, EntityType};
64use serde::{Deserialize, Serialize};
65use std::collections::HashMap;
66
67#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
73pub enum AnaphoraType {
74 Nominal,
76 Event,
78 Fact,
80 Proposition,
82 Situation,
84}
85
86impl AnaphoraType {
87 #[must_use]
89 pub const fn is_abstract(&self) -> bool {
90 !matches!(self, AnaphoraType::Nominal)
91 }
92
93 #[must_use]
95 pub const fn as_str(&self) -> &'static str {
96 match self {
97 AnaphoraType::Nominal => "nominal",
98 AnaphoraType::Event => "event",
99 AnaphoraType::Fact => "fact",
100 AnaphoraType::Proposition => "proposition",
101 AnaphoraType::Situation => "situation",
102 }
103 }
104}
105
106#[derive(Debug, Clone, Serialize, Deserialize)]
112pub struct AnaphoraTestCase {
113 pub id: String,
115 pub text: String,
117 pub antecedent: AntecedentSpan,
119 pub anaphor: AnaphorSpan,
121 pub anaphora_type: AnaphoraType,
123 pub should_resolve: bool,
125 pub notes: Option<String>,
127}
128
129#[derive(Debug, Clone, Serialize, Deserialize)]
131pub struct AntecedentSpan {
132 pub text: String,
134 pub start: usize,
136 pub end: usize,
138 pub trigger: Option<String>,
140}
141
142#[derive(Debug, Clone, Serialize, Deserialize)]
144pub struct AnaphorSpan {
145 pub text: String,
147 pub start: usize,
149 pub end: usize,
151}
152
153impl AnaphoraTestCase {
154 pub fn new(
156 id: impl Into<String>,
157 text: impl Into<String>,
158 antecedent: AntecedentSpan,
159 anaphor: AnaphorSpan,
160 anaphora_type: AnaphoraType,
161 ) -> Self {
162 Self {
163 id: id.into(),
164 text: text.into(),
165 antecedent,
166 anaphor,
167 anaphora_type,
168 should_resolve: true,
169 notes: None,
170 }
171 }
172
173 #[must_use]
175 pub fn with_notes(mut self, notes: impl Into<String>) -> Self {
176 self.notes = Some(notes.into());
177 self
178 }
179}
180
181#[derive(Debug, Clone, Default, Serialize, Deserialize)]
187pub struct AbstractAnaphoraDataset {
188 pub cases: Vec<AnaphoraTestCase>,
190}
191
192impl AbstractAnaphoraDataset {
193 #[must_use]
195 pub fn new() -> Self {
196 Self { cases: Vec::new() }
197 }
198
199 pub fn add(&mut self, case: AnaphoraTestCase) {
201 self.cases.push(case);
202 }
203
204 #[must_use]
206 pub fn by_type(&self, anaphora_type: AnaphoraType) -> Vec<&AnaphoraTestCase> {
207 self.cases
208 .iter()
209 .filter(|c| c.anaphora_type == anaphora_type)
210 .collect()
211 }
212
213 #[must_use]
215 pub fn nominal_cases(&self) -> Vec<&AnaphoraTestCase> {
216 self.by_type(AnaphoraType::Nominal)
217 }
218
219 #[must_use]
221 pub fn abstract_cases(&self) -> Vec<&AnaphoraTestCase> {
222 self.cases
223 .iter()
224 .filter(|c| c.anaphora_type.is_abstract())
225 .collect()
226 }
227
228 #[must_use]
232 pub fn standard() -> Self {
233 let mut dataset = Self::new();
234
235 dataset.add(
240 AnaphoraTestCase::new(
241 "nom_01",
242 "John Smith went to the store. He bought milk.",
243 AntecedentSpan {
244 text: "John Smith".to_string(),
245 start: 0,
246 end: 10,
247 trigger: None,
248 },
249 AnaphorSpan {
250 text: "He".to_string(),
251 start: 32,
252 end: 34,
253 },
254 AnaphoraType::Nominal,
255 )
256 .with_notes("Simple pronoun resolution - baseline case"),
257 );
258
259 dataset.add(
260 AnaphoraTestCase::new(
261 "nom_02",
262 "Microsoft announced layoffs. The company cited economic conditions.",
263 AntecedentSpan {
264 text: "Microsoft".to_string(),
265 start: 0,
266 end: 9,
267 trigger: None,
268 },
269 AnaphorSpan {
270 text: "The company".to_string(),
271 start: 29,
272 end: 40,
273 },
274 AnaphoraType::Nominal,
275 )
276 .with_notes("Definite NP resolution"),
277 );
278
279 dataset.add(
280 AnaphoraTestCase::new(
281 "nom_03",
282 "Dr. Sarah Chen published a paper. She presented it at EMNLP.",
283 AntecedentSpan {
284 text: "Dr. Sarah Chen".to_string(),
285 start: 0,
286 end: 14,
287 trigger: None,
288 },
289 AnaphorSpan {
290 text: "She".to_string(),
291 start: 35,
292 end: 38,
293 },
294 AnaphoraType::Nominal,
295 )
296 .with_notes("Pronoun with title prefix"),
297 );
298
299 dataset.add(
300 AnaphoraTestCase::new(
301 "nom_04",
302 "The CEO of Nvidia is Jensen Huang. He co-founded the company.",
303 AntecedentSpan {
304 text: "Jensen Huang".to_string(),
305 start: 20,
306 end: 32,
307 trigger: None,
308 },
309 AnaphorSpan {
310 text: "He".to_string(),
311 start: 34,
312 end: 36,
313 },
314 AnaphoraType::Nominal,
315 )
316 .with_notes("Pronoun binds to proper name, not role description"),
317 );
318
319 dataset.add(
320 AnaphoraTestCase::new(
321 "nom_05",
322 "Apple Inc. reported record earnings. Apple's stock rose 5%.",
323 AntecedentSpan {
324 text: "Apple Inc.".to_string(),
325 start: 0,
326 end: 10,
327 trigger: None,
328 },
329 AnaphorSpan {
330 text: "Apple's".to_string(),
331 start: 37,
332 end: 44,
333 },
334 AnaphoraType::Nominal,
335 )
336 .with_notes("Possessive form coreference"),
337 );
338
339 dataset.add(
344 AnaphoraTestCase::new(
345 "event_01",
346 "Russia invaded Ukraine in 2022. This caused a global energy crisis.",
347 AntecedentSpan {
348 text: "Russia invaded Ukraine in 2022".to_string(),
349 start: 0,
350 end: 30,
351 trigger: Some("invaded".to_string()),
352 },
353 AnaphorSpan {
354 text: "This".to_string(),
355 start: 32,
356 end: 36,
357 },
358 AnaphoraType::Event,
359 )
360 .with_notes(
361 "Classic event anaphora - 'This' refers to invasion EVENT, not Russia or Ukraine",
362 ),
363 );
364
365 dataset.add(
366 AnaphoraTestCase::new(
367 "event_02",
368 "The earthquake struck at dawn. It destroyed thousands of homes.",
369 AntecedentSpan {
370 text: "The earthquake struck at dawn".to_string(),
371 start: 0,
372 end: 29,
373 trigger: Some("struck".to_string()),
374 },
375 AnaphorSpan {
376 text: "It".to_string(),
377 start: 31,
378 end: 33,
379 },
380 AnaphoraType::Event,
381 )
382 .with_notes("'It' refers to the earthquake event, not just the noun 'earthquake'"),
383 );
384
385 dataset.add(
386 AnaphoraTestCase::new(
387 "event_03",
388 "The merger was announced yesterday. This surprised investors.",
389 AntecedentSpan {
390 text: "The merger was announced yesterday".to_string(),
391 start: 0,
392 end: 34,
393 trigger: Some("announced".to_string()),
394 },
395 AnaphorSpan {
396 text: "This".to_string(),
397 start: 36,
398 end: 40,
399 },
400 AnaphoraType::Event,
401 )
402 .with_notes("Announcement event, not the merger entity"),
403 );
404
405 dataset.add(
406 AnaphoraTestCase::new(
407 "event_04",
408 "Scientists discovered a new species. This happened in the Amazon.",
409 AntecedentSpan {
410 text: "Scientists discovered a new species".to_string(),
411 start: 0,
412 end: 35,
413 trigger: Some("discovered".to_string()),
414 },
415 AnaphorSpan {
416 text: "This".to_string(),
417 start: 37,
418 end: 41,
419 },
420 AnaphoraType::Event,
421 )
422 .with_notes("Discovery event"),
423 );
424
425 dataset.add(
426 AnaphoraTestCase::new(
427 "event_05",
428 "The patient underwent surgery. This took six hours.",
429 AntecedentSpan {
430 text: "The patient underwent surgery".to_string(),
431 start: 0,
432 end: 29,
433 trigger: Some("underwent".to_string()),
434 },
435 AnaphorSpan {
436 text: "This".to_string(),
437 start: 31,
438 end: 35,
439 },
440 AnaphoraType::Event,
441 )
442 .with_notes("Medical procedure event"),
443 );
444
445 dataset.add(
450 AnaphoraTestCase::new(
451 "fact_01",
452 "The Earth orbits the Sun. This is well established.",
453 AntecedentSpan {
454 text: "The Earth orbits the Sun".to_string(),
455 start: 0,
456 end: 24,
457 trigger: None,
458 },
459 AnaphorSpan {
460 text: "This".to_string(),
461 start: 26,
462 end: 30,
463 },
464 AnaphoraType::Fact,
465 )
466 .with_notes("'This' refers to the FACT, not Earth or Sun"),
467 );
468
469 dataset.add(
470 AnaphoraTestCase::new(
471 "fact_02",
472 "Water boils at 100 degrees Celsius. This is basic chemistry.",
473 AntecedentSpan {
474 text: "Water boils at 100 degrees Celsius".to_string(),
475 start: 0,
476 end: 34,
477 trigger: None,
478 },
479 AnaphorSpan {
480 text: "This".to_string(),
481 start: 36,
482 end: 40,
483 },
484 AnaphoraType::Fact,
485 )
486 .with_notes("Scientific fact reference"),
487 );
488
489 dataset.add(
490 AnaphoraTestCase::new(
491 "fact_03",
492 "He lied under oath. This was proven in court.",
493 AntecedentSpan {
494 text: "He lied under oath".to_string(),
495 start: 0,
496 end: 18,
497 trigger: None,
498 },
499 AnaphorSpan {
500 text: "This".to_string(),
501 start: 20,
502 end: 24,
503 },
504 AnaphoraType::Fact,
505 )
506 .with_notes("Fact about past action"),
507 );
508
509 dataset.add(
514 AnaphoraTestCase::new(
515 "prop_01",
516 "She might resign. This worries the board.",
517 AntecedentSpan {
518 text: "She might resign".to_string(),
519 start: 0,
520 end: 16,
521 trigger: None,
522 },
523 AnaphorSpan {
524 text: "This".to_string(),
525 start: 18,
526 end: 22,
527 },
528 AnaphoraType::Proposition,
529 )
530 .with_notes("'This' refers to the POSSIBILITY of resignation"),
531 );
532
533 dataset.add(
534 AnaphoraTestCase::new(
535 "prop_02",
536 "The company could go bankrupt. This scenario keeps investors awake.",
537 AntecedentSpan {
538 text: "The company could go bankrupt".to_string(),
539 start: 0,
540 end: 29,
541 trigger: None,
542 },
543 AnaphorSpan {
544 text: "This scenario".to_string(),
545 start: 31,
546 end: 44,
547 },
548 AnaphoraType::Proposition,
549 )
550 .with_notes("Hypothetical proposition"),
551 );
552
553 dataset.add(
554 AnaphoraTestCase::new(
555 "prop_03",
556 "Interest rates may rise again. This possibility concernos economists.",
557 AntecedentSpan {
558 text: "Interest rates may rise again".to_string(),
559 start: 0,
560 end: 29,
561 trigger: None,
562 },
563 AnaphorSpan {
564 text: "This possibility".to_string(),
565 start: 31,
566 end: 47,
567 },
568 AnaphoraType::Proposition,
569 )
570 .with_notes("Modal proposition"),
571 );
572
573 dataset.add(
578 AnaphoraTestCase::new(
579 "sit_01",
580 "Prices rose while wages fell. This was unsustainable.",
581 AntecedentSpan {
582 text: "Prices rose while wages fell".to_string(),
583 start: 0,
584 end: 28,
585 trigger: None,
586 },
587 AnaphorSpan {
588 text: "This".to_string(),
589 start: 30,
590 end: 34,
591 },
592 AnaphoraType::Situation,
593 )
594 .with_notes("'This' refers to the combined SITUATION, not prices or wages"),
595 );
596
597 dataset.add(
598 AnaphoraTestCase::new(
599 "sit_02",
600 "Traffic was gridlocked and tempers flared. This chaos lasted hours.",
601 AntecedentSpan {
602 text: "Traffic was gridlocked and tempers flared".to_string(),
603 start: 0,
604 end: 41,
605 trigger: None,
606 },
607 AnaphorSpan {
608 text: "This chaos".to_string(),
609 start: 43,
610 end: 53,
611 },
612 AnaphoraType::Situation,
613 )
614 .with_notes("Complex situation with multiple aspects"),
615 );
616
617 dataset.add(
618 AnaphoraTestCase::new(
619 "sit_03",
620 "The server crashed, emails were lost, and backups failed. This disaster cost millions.",
621 AntecedentSpan {
622 text: "The server crashed, emails were lost, and backups failed".to_string(),
623 start: 0,
624 end: 56,
625 trigger: None,
626 },
627 AnaphorSpan {
628 text: "This disaster".to_string(),
629 start: 58,
630 end: 71,
631 },
632 AnaphoraType::Situation,
633 )
634 .with_notes("Multi-clause situation"),
635 );
636
637 dataset
638 }
639
640 #[must_use]
645 pub fn extended() -> Self {
646 let mut dataset = Self::standard();
647
648 dataset.add(
654 AnaphoraTestCase::new(
655 "shell_fact_01",
656 "The GDP grew by 3%. This fact surprised analysts.",
657 AntecedentSpan {
658 text: "The GDP grew by 3%".to_string(),
659 start: 0,
660 end: 18,
661 trigger: Some("grew".to_string()),
662 },
663 AnaphorSpan {
664 text: "This fact".to_string(),
665 start: 20,
666 end: 29,
667 },
668 AnaphoraType::Fact,
669 )
670 .with_notes("Shell noun 'fact' - factual class (Schmid 2000)"),
671 );
672
673 dataset.add(
674 AnaphoraTestCase::new(
675 "shell_fact_02",
676 "Prices doubled in one year. The reason was supply chain disruption.",
677 AntecedentSpan {
678 text: "Prices doubled in one year".to_string(),
679 start: 0,
680 end: 26,
681 trigger: Some("doubled".to_string()),
682 },
683 AnaphorSpan {
684 text: "The reason".to_string(),
685 start: 28,
686 end: 38,
687 },
688 AnaphoraType::Fact,
689 )
690 .with_notes("Shell noun 'reason' - factual class, cataphoric"),
691 );
692
693 dataset.add(
695 AnaphoraTestCase::new(
696 "shell_ling_01",
697 "The CEO promised higher wages. This claim was later retracted.",
698 AntecedentSpan {
699 text: "The CEO promised higher wages".to_string(),
700 start: 0,
701 end: 29,
702 trigger: Some("promised".to_string()),
703 },
704 AnaphorSpan {
705 text: "This claim".to_string(),
706 start: 31,
707 end: 41,
708 },
709 AnaphoraType::Proposition,
710 )
711 .with_notes("Shell noun 'claim' - linguistic class"),
712 );
713
714 dataset.add(
715 AnaphoraTestCase::new(
716 "shell_ling_02",
717 "We should invest in renewables. The argument convinced the board.",
718 AntecedentSpan {
719 text: "We should invest in renewables".to_string(),
720 start: 0,
721 end: 30,
722 trigger: None,
723 },
724 AnaphorSpan {
725 text: "The argument".to_string(),
726 start: 32,
727 end: 44,
728 },
729 AnaphoraType::Proposition,
730 )
731 .with_notes("Shell noun 'argument' - linguistic class"),
732 );
733
734 dataset.add(
736 AnaphoraTestCase::new(
737 "shell_mental_01",
738 "Automation will replace most jobs. This belief is controversial.",
739 AntecedentSpan {
740 text: "Automation will replace most jobs".to_string(),
741 start: 0,
742 end: 33,
743 trigger: None,
744 },
745 AnaphorSpan {
746 text: "This belief".to_string(),
747 start: 35,
748 end: 46,
749 },
750 AnaphoraType::Proposition,
751 )
752 .with_notes("Shell noun 'belief' - mental class"),
753 );
754
755 dataset.add(
756 AnaphoraTestCase::new(
757 "shell_mental_02",
758 "The new policy will fail. This view is shared by experts.",
759 AntecedentSpan {
760 text: "The new policy will fail".to_string(),
761 start: 0,
762 end: 24,
763 trigger: None,
764 },
765 AnaphorSpan {
766 text: "This view".to_string(),
767 start: 26,
768 end: 35,
769 },
770 AnaphoraType::Proposition,
771 )
772 .with_notes("Shell noun 'view' - mental class"),
773 );
774
775 dataset.add(
777 AnaphoraTestCase::new(
778 "shell_modal_01",
779 "The system could crash under load. This possibility concernoed engineers.",
780 AntecedentSpan {
781 text: "The system could crash under load".to_string(),
782 start: 0,
783 end: 33,
784 trigger: None,
785 },
786 AnaphorSpan {
787 text: "This possibility".to_string(),
788 start: 35,
789 end: 51,
790 },
791 AnaphoraType::Proposition,
792 )
793 .with_notes("Shell noun 'possibility' - modal class"),
794 );
795
796 dataset.add(
798 AnaphoraTestCase::new(
799 "shell_event_01",
800 "The company laid off 500 workers. This decision shocked employees.",
801 AntecedentSpan {
802 text: "The company laid off 500 workers".to_string(),
803 start: 0,
804 end: 32,
805 trigger: Some("laid off".to_string()),
806 },
807 AnaphorSpan {
808 text: "This decision".to_string(),
809 start: 34,
810 end: 47,
811 },
812 AnaphoraType::Event,
813 )
814 .with_notes("Shell noun 'decision' - eventive class"),
815 );
816
817 dataset.add(
818 AnaphoraTestCase::new(
819 "shell_event_02",
820 "A meteor struck the desert. The incident was witnessed by campers.",
821 AntecedentSpan {
822 text: "A meteor struck the desert".to_string(),
823 start: 0,
824 end: 26,
825 trigger: Some("struck".to_string()),
826 },
827 AnaphorSpan {
828 text: "The incident".to_string(),
829 start: 28,
830 end: 40,
831 },
832 AnaphoraType::Event,
833 )
834 .with_notes("Shell noun 'incident' - eventive class"),
835 );
836
837 dataset.add(
839 AnaphoraTestCase::new(
840 "shell_circ_01",
841 "Inflation is rising while wages stagnate. This situation is unsustainable.",
842 AntecedentSpan {
843 text: "Inflation is rising while wages stagnate".to_string(),
844 start: 0,
845 end: 40,
846 trigger: None,
847 },
848 AnaphorSpan {
849 text: "This situation".to_string(),
850 start: 42,
851 end: 56,
852 },
853 AnaphoraType::Situation,
854 )
855 .with_notes("Shell noun 'situation' - circumstantial class"),
856 );
857
858 dataset.add(
859 AnaphoraTestCase::new(
860 "shell_circ_02",
861 "The code has bugs and the deadline is tomorrow. This problem needs addressing.",
862 AntecedentSpan {
863 text: "The code has bugs and the deadline is tomorrow".to_string(),
864 start: 0,
865 end: 46,
866 trigger: None,
867 },
868 AnaphorSpan {
869 text: "This problem".to_string(),
870 start: 48,
871 end: 60,
872 },
873 AnaphoraType::Situation,
874 )
875 .with_notes("Shell noun 'problem' - circumstantial class"),
876 );
877
878 dataset.add(
883 AnaphoraTestCase::new(
884 "dist_01",
885 "The protests began in March. Police deployed tear gas. Several arrests were made. This response drew international criticism.",
886 AntecedentSpan {
887 text: "Police deployed tear gas. Several arrests were made".to_string(),
888 start: 29,
889 end: 80,
890 trigger: None,
891 },
892 AnaphorSpan {
893 text: "This response".to_string(),
894 start: 82,
895 end: 95,
896 },
897 AnaphoraType::Event,
898 )
899 .with_notes("Multi-sentence antecedent (2 sentences back)"),
900 );
901
902 dataset
903 }
904
905 #[must_use]
914 pub fn legal_domain() -> Self {
915 let mut dataset = Self::new();
916
917 dataset.add(
918 AnaphoraTestCase::new(
919 "legal_01",
920 "The court ruled in favor of the plaintiff. This decision sets a precedent.",
921 AntecedentSpan {
922 text: "The court ruled in favor of the plaintiff".to_string(),
923 start: 0,
924 end: 41,
925 trigger: Some("ruled".to_string()),
926 },
927 AnaphorSpan {
928 text: "This decision".to_string(),
929 start: 43,
930 end: 56,
931 },
932 AnaphoraType::Event,
933 )
934 .with_notes("Court ruling reference"),
935 );
936
937 dataset.add(
938 AnaphoraTestCase::new(
939 "legal_02",
940 "The defendant violated the contract terms. This breach entitles the claimant to damages.",
941 AntecedentSpan {
942 text: "The defendant violated the contract terms".to_string(),
943 start: 0,
944 end: 41,
945 trigger: Some("violated".to_string()),
946 },
947 AnaphorSpan {
948 text: "This breach".to_string(),
949 start: 43,
950 end: 54,
951 },
952 AnaphoraType::Event,
953 )
954 .with_notes("Legal violation reference"),
955 );
956
957 dataset.add(
958 AnaphoraTestCase::new(
959 "legal_03",
960 "Corporations must disclose material information. Failure to do so constitutes fraud.",
961 AntecedentSpan {
962 text: "Corporations must disclose material information".to_string(),
963 start: 0,
964 end: 47,
965 trigger: None,
966 },
967 AnaphorSpan {
968 text: "Failure to do so".to_string(),
969 start: 49,
970 end: 65,
971 },
972 AnaphoraType::Fact,
973 )
974 .with_notes("Obligation reference with negation"),
975 );
976
977 dataset.add(
978 AnaphoraTestCase::new(
979 "legal_04",
980 "The statute requires prior notice. This requirement was not met.",
981 AntecedentSpan {
982 text: "The statute requires prior notice".to_string(),
983 start: 0,
984 end: 33,
985 trigger: Some("requires".to_string()),
986 },
987 AnaphorSpan {
988 text: "This requirement".to_string(),
989 start: 35,
990 end: 51,
991 },
992 AnaphoraType::Fact,
993 )
994 .with_notes("Legal requirement reference"),
995 );
996
997 dataset.add(
998 AnaphoraTestCase::new(
999 "legal_05",
1000 "The witness may have lied. If this is true, perjury charges apply.",
1001 AntecedentSpan {
1002 text: "The witness may have lied".to_string(),
1003 start: 0,
1004 end: 25,
1005 trigger: Some("lied".to_string()),
1006 },
1007 AnaphorSpan {
1008 text: "this".to_string(),
1009 start: 30,
1010 end: 34,
1011 },
1012 AnaphoraType::Proposition,
1013 )
1014 .with_notes("Modal proposition in legal context"),
1015 );
1016
1017 dataset.add(
1018 AnaphoraTestCase::new(
1019 "legal_06",
1020 "The parties agreed to arbitration. This agreement is binding.",
1021 AntecedentSpan {
1022 text: "The parties agreed to arbitration".to_string(),
1023 start: 0,
1024 end: 33,
1025 trigger: Some("agreed".to_string()),
1026 },
1027 AnaphorSpan {
1028 text: "This agreement".to_string(),
1029 start: 35,
1030 end: 49,
1031 },
1032 AnaphoraType::Event,
1033 )
1034 .with_notes("Agreement event reference"),
1035 );
1036
1037 dataset.add(
1038 AnaphoraTestCase::new(
1039 "legal_07",
1040 "The prosecution alleged embezzlement. The allegation was later withdrawn.",
1041 AntecedentSpan {
1042 text: "The prosecution alleged embezzlement".to_string(),
1043 start: 0,
1044 end: 36,
1045 trigger: Some("alleged".to_string()),
1046 },
1047 AnaphorSpan {
1048 text: "The allegation".to_string(),
1049 start: 38,
1050 end: 52,
1051 },
1052 AnaphoraType::Event,
1053 )
1054 .with_notes("Allegation event reference"),
1055 );
1056
1057 dataset.add(
1058 AnaphoraTestCase::new(
1059 "legal_08",
1060 "Evidence was obtained without a warrant. This fact renders it inadmissible.",
1061 AntecedentSpan {
1062 text: "Evidence was obtained without a warrant".to_string(),
1063 start: 0,
1064 end: 39,
1065 trigger: Some("obtained".to_string()),
1066 },
1067 AnaphorSpan {
1068 text: "This fact".to_string(),
1069 start: 41,
1070 end: 50,
1071 },
1072 AnaphoraType::Fact,
1073 )
1074 .with_notes("Factual shell noun in legal context"),
1075 );
1076
1077 dataset.add(
1079 AnaphoraTestCase::new(
1080 "legal_nom_01",
1081 "The defendant hired a lawyer. He filed an appeal.",
1082 AntecedentSpan {
1083 text: "a lawyer".to_string(),
1084 start: 21,
1085 end: 29,
1086 trigger: None,
1087 },
1088 AnaphorSpan {
1089 text: "He".to_string(),
1090 start: 31,
1091 end: 33,
1092 },
1093 AnaphoraType::Nominal,
1094 )
1095 .with_notes("Standard nominal coreference (lawyer)"),
1096 );
1097
1098 dataset
1099 }
1100
1101 #[must_use]
1106 pub fn medical_domain() -> Self {
1107 let mut dataset = Self::new();
1108
1109 dataset.add(
1110 AnaphoraTestCase::new(
1111 "med_01",
1112 "The patient presented with chest pain. This symptom suggested cardiac involvement.",
1113 AntecedentSpan {
1114 text: "The patient presented with chest pain".to_string(),
1115 start: 0,
1116 end: 37,
1117 trigger: Some("presented".to_string()),
1118 },
1119 AnaphorSpan {
1120 text: "This symptom".to_string(),
1121 start: 39,
1122 end: 51,
1123 },
1124 AnaphoraType::Fact,
1125 )
1126 .with_notes("Symptom presentation reference"),
1127 );
1128
1129 dataset.add(
1130 AnaphoraTestCase::new(
1131 "med_02",
1132 "Surgery was performed to remove the tumor. This procedure lasted four hours.",
1133 AntecedentSpan {
1134 text: "Surgery was performed to remove the tumor".to_string(),
1135 start: 0,
1136 end: 41,
1137 trigger: Some("performed".to_string()),
1138 },
1139 AnaphorSpan {
1140 text: "This procedure".to_string(),
1141 start: 43,
1142 end: 57,
1143 },
1144 AnaphoraType::Event,
1145 )
1146 .with_notes("Surgical procedure reference"),
1147 );
1148
1149 dataset.add(
1150 AnaphoraTestCase::new(
1151 "med_03",
1152 "Blood pressure normalized after treatment. This improvement was sustained.",
1153 AntecedentSpan {
1154 text: "Blood pressure normalized after treatment".to_string(),
1155 start: 0,
1156 end: 41,
1157 trigger: Some("normalized".to_string()),
1158 },
1159 AnaphorSpan {
1160 text: "This improvement".to_string(),
1161 start: 43,
1162 end: 59,
1163 },
1164 AnaphoraType::Event,
1165 )
1166 .with_notes("Clinical improvement reference"),
1167 );
1168
1169 dataset.add(
1170 AnaphoraTestCase::new(
1171 "med_04",
1172 "The medication may cause drowsiness. This side effect is usually temporary.",
1173 AntecedentSpan {
1174 text: "The medication may cause drowsiness".to_string(),
1175 start: 0,
1176 end: 35,
1177 trigger: Some("cause".to_string()),
1178 },
1179 AnaphorSpan {
1180 text: "This side effect".to_string(),
1181 start: 37,
1182 end: 53,
1183 },
1184 AnaphoraType::Proposition,
1185 )
1186 .with_notes("Potential side effect reference"),
1187 );
1188
1189 dataset.add(
1190 AnaphoraTestCase::new(
1191 "med_05",
1192 "The patient was diagnosed with diabetes. Managing this condition requires lifestyle changes.",
1193 AntecedentSpan {
1194 text: "diabetes".to_string(),
1195 start: 31,
1196 end: 39,
1197 trigger: None,
1198 },
1199 AnaphorSpan {
1200 text: "this condition".to_string(),
1201 start: 51,
1202 end: 65,
1203 },
1204 AnaphoraType::Situation,
1205 )
1206 .with_notes("Medical condition reference"),
1207 );
1208
1209 dataset.add(
1210 AnaphoraTestCase::new(
1211 "med_06",
1212 "The biopsy revealed malignant cells. This finding necessitated further testing.",
1213 AntecedentSpan {
1214 text: "The biopsy revealed malignant cells".to_string(),
1215 start: 0,
1216 end: 35,
1217 trigger: Some("revealed".to_string()),
1218 },
1219 AnaphorSpan {
1220 text: "This finding".to_string(),
1221 start: 37,
1222 end: 49,
1223 },
1224 AnaphoraType::Fact,
1225 )
1226 .with_notes("Diagnostic finding reference"),
1227 );
1228
1229 dataset.add(
1230 AnaphoraTestCase::new(
1231 "med_07",
1232 "The patient's fever spiked overnight. This development concernoed the medical team.",
1233 AntecedentSpan {
1234 text: "The patient's fever spiked overnight".to_string(),
1235 start: 0,
1236 end: 36,
1237 trigger: Some("spiked".to_string()),
1238 },
1239 AnaphorSpan {
1240 text: "This development".to_string(),
1241 start: 38,
1242 end: 54,
1243 },
1244 AnaphoraType::Event,
1245 )
1246 .with_notes("Clinical event reference"),
1247 );
1248
1249 dataset.add(
1250 AnaphoraTestCase::new(
1251 "med_08",
1252 "Chemotherapy was discontinued due to adverse reactions. This decision was made by the oncologist.",
1253 AntecedentSpan {
1254 text: "Chemotherapy was discontinued due to adverse reactions".to_string(),
1255 start: 0,
1256 end: 54,
1257 trigger: Some("discontinued".to_string()),
1258 },
1259 AnaphorSpan {
1260 text: "This decision".to_string(),
1261 start: 56,
1262 end: 69,
1263 },
1264 AnaphoraType::Event,
1265 )
1266 .with_notes("Treatment decision reference"),
1267 );
1268
1269 dataset.add(
1271 AnaphoraTestCase::new(
1272 "med_nom_01",
1273 "The surgeon consulted a specialist. She recommended immediate intervention.",
1274 AntecedentSpan {
1275 text: "a specialist".to_string(),
1276 start: 23,
1277 end: 35,
1278 trigger: None,
1279 },
1280 AnaphorSpan {
1281 text: "She".to_string(),
1282 start: 37,
1283 end: 40,
1284 },
1285 AnaphoraType::Nominal,
1286 )
1287 .with_notes("Standard nominal coreference (specialist)"),
1288 );
1289
1290 dataset
1291 }
1292
1293 #[must_use]
1298 pub fn financial_domain() -> Self {
1299 let mut dataset = Self::new();
1300
1301 dataset.add(
1302 AnaphoraTestCase::new(
1303 "fin_01",
1304 "The Fed raised interest rates. This move sent shockwaves through markets.",
1305 AntecedentSpan {
1306 text: "The Fed raised interest rates".to_string(),
1307 start: 0,
1308 end: 29,
1309 trigger: Some("raised".to_string()),
1310 },
1311 AnaphorSpan {
1312 text: "This move".to_string(),
1313 start: 31,
1314 end: 40,
1315 },
1316 AnaphoraType::Event,
1317 )
1318 .with_notes("Policy decision reference"),
1319 );
1320
1321 dataset.add(
1322 AnaphoraTestCase::new(
1323 "fin_02",
1324 "The merger was approved by regulators. This development boosted investor confidence.",
1325 AntecedentSpan {
1326 text: "The merger was approved by regulators".to_string(),
1327 start: 0,
1328 end: 37,
1329 trigger: Some("approved".to_string()),
1330 },
1331 AnaphorSpan {
1332 text: "This development".to_string(),
1333 start: 39,
1334 end: 55,
1335 },
1336 AnaphoraType::Event,
1337 )
1338 .with_notes("Regulatory approval reference"),
1339 );
1340
1341 dataset.add(
1342 AnaphoraTestCase::new(
1343 "fin_03",
1344 "Quarterly earnings exceeded expectations. This performance led to a stock rally.",
1345 AntecedentSpan {
1346 text: "Quarterly earnings exceeded expectations".to_string(),
1347 start: 0,
1348 end: 40,
1349 trigger: Some("exceeded".to_string()),
1350 },
1351 AnaphorSpan {
1352 text: "This performance".to_string(),
1353 start: 42,
1354 end: 58,
1355 },
1356 AnaphoraType::Event,
1357 )
1358 .with_notes("Financial performance reference"),
1359 );
1360
1361 dataset.add(
1362 AnaphoraTestCase::new(
1363 "fin_04",
1364 "The company might default on its loans. This risk has alarmed bondholders.",
1365 AntecedentSpan {
1366 text: "The company might default on its loans".to_string(),
1367 start: 0,
1368 end: 38,
1369 trigger: Some("default".to_string()),
1370 },
1371 AnaphorSpan {
1372 text: "This risk".to_string(),
1373 start: 40,
1374 end: 49,
1375 },
1376 AnaphoraType::Proposition,
1377 )
1378 .with_notes("Financial risk proposition"),
1379 );
1380
1381 dataset.add(
1382 AnaphoraTestCase::new(
1383 "fin_05",
1384 "Supply chain disruptions are causing inflation. This situation could persist for years.",
1385 AntecedentSpan {
1386 text: "Supply chain disruptions are causing inflation".to_string(),
1387 start: 0,
1388 end: 46,
1389 trigger: Some("causing".to_string()),
1390 },
1391 AnaphorSpan {
1392 text: "This situation".to_string(),
1393 start: 48,
1394 end: 62,
1395 },
1396 AnaphoraType::Situation,
1397 )
1398 .with_notes("Economic situation reference"),
1399 );
1400
1401 dataset.add(
1402 AnaphoraTestCase::new(
1403 "fin_06",
1404 "The CEO announced a stock buyback program. The announcement pushed shares higher.",
1405 AntecedentSpan {
1406 text: "The CEO announced a stock buyback program".to_string(),
1407 start: 0,
1408 end: 41,
1409 trigger: Some("announced".to_string()),
1410 },
1411 AnaphorSpan {
1412 text: "The announcement".to_string(),
1413 start: 43,
1414 end: 59,
1415 },
1416 AnaphoraType::Event,
1417 )
1418 .with_notes("Corporate announcement reference"),
1419 );
1420
1421 dataset.add(
1422 AnaphoraTestCase::new(
1423 "fin_07",
1424 "Revenue grew by 15% year-over-year. This growth outpaced analyst forecasts.",
1425 AntecedentSpan {
1426 text: "Revenue grew by 15% year-over-year".to_string(),
1427 start: 0,
1428 end: 34,
1429 trigger: Some("grew".to_string()),
1430 },
1431 AnaphorSpan {
1432 text: "This growth".to_string(),
1433 start: 36,
1434 end: 47,
1435 },
1436 AnaphoraType::Event,
1437 )
1438 .with_notes("Revenue growth event reference"),
1439 );
1440
1441 dataset.add(
1442 AnaphoraTestCase::new(
1443 "fin_08",
1444 "The acquisition was completed yesterday. This transaction creates the largest retailer.",
1445 AntecedentSpan {
1446 text: "The acquisition was completed yesterday".to_string(),
1447 start: 0,
1448 end: 39,
1449 trigger: Some("completed".to_string()),
1450 },
1451 AnaphorSpan {
1452 text: "This transaction".to_string(),
1453 start: 41,
1454 end: 57,
1455 },
1456 AnaphoraType::Event,
1457 )
1458 .with_notes("Business transaction reference"),
1459 );
1460
1461 dataset.add(
1463 AnaphoraTestCase::new(
1464 "fin_nom_01",
1465 "The CFO presented the report. She highlighted key metrics.",
1466 AntecedentSpan {
1467 text: "The CFO".to_string(),
1468 start: 0,
1469 end: 7,
1470 trigger: None,
1471 },
1472 AnaphorSpan {
1473 text: "She".to_string(),
1474 start: 31,
1475 end: 34,
1476 },
1477 AnaphoraType::Nominal,
1478 )
1479 .with_notes("Standard nominal coreference (CFO)"),
1480 );
1481
1482 dataset
1483 }
1484
1485 #[must_use]
1490 pub fn scientific_domain() -> Self {
1491 let mut dataset = Self::new();
1492
1493 dataset.add(
1494 AnaphoraTestCase::new(
1495 "sci_01",
1496 "The experiment failed to replicate earlier results. This failure suggests methodological issues.",
1497 AntecedentSpan {
1498 text: "The experiment failed to replicate earlier results".to_string(),
1499 start: 0,
1500 end: 50,
1501 trigger: Some("failed".to_string()),
1502 },
1503 AnaphorSpan {
1504 text: "This failure".to_string(),
1505 start: 52,
1506 end: 64,
1507 },
1508 AnaphoraType::Event,
1509 )
1510 .with_notes("Experimental failure reference"),
1511 );
1512
1513 dataset.add(
1514 AnaphoraTestCase::new(
1515 "sci_02",
1516 "The data shows a correlation between diet and longevity. This finding aligns with previous studies.",
1517 AntecedentSpan {
1518 text: "The data shows a correlation between diet and longevity".to_string(),
1519 start: 0,
1520 end: 55,
1521 trigger: Some("shows".to_string()),
1522 },
1523 AnaphorSpan {
1524 text: "This finding".to_string(),
1525 start: 57,
1526 end: 69,
1527 },
1528 AnaphoraType::Fact,
1529 )
1530 .with_notes("Scientific finding reference"),
1531 );
1532
1533 dataset.add(
1534 AnaphoraTestCase::new(
1535 "sci_03",
1536 "Quantum entanglement may enable faster communication. If this is possible, it would revolutionize networking.",
1537 AntecedentSpan {
1538 text: "Quantum entanglement may enable faster communication".to_string(),
1539 start: 0,
1540 end: 52,
1541 trigger: Some("enable".to_string()),
1542 },
1543 AnaphorSpan {
1544 text: "this".to_string(),
1545 start: 57,
1546 end: 61,
1547 },
1548 AnaphoraType::Proposition,
1549 )
1550 .with_notes("Scientific hypothesis reference"),
1551 );
1552
1553 dataset.add(
1554 AnaphoraTestCase::new(
1555 "sci_04",
1556 "The samples were contaminated during transport. This problem invalidated the study.",
1557 AntecedentSpan {
1558 text: "The samples were contaminated during transport".to_string(),
1559 start: 0,
1560 end: 46,
1561 trigger: Some("contaminated".to_string()),
1562 },
1563 AnaphorSpan {
1564 text: "This problem".to_string(),
1565 start: 48,
1566 end: 60,
1567 },
1568 AnaphoraType::Event,
1569 )
1570 .with_notes("Experimental problem reference"),
1571 );
1572
1573 dataset.add(
1574 AnaphoraTestCase::new(
1575 "sci_05",
1576 "The protein folded incorrectly under high temperatures. This observation was unexpected.",
1577 AntecedentSpan {
1578 text: "The protein folded incorrectly under high temperatures".to_string(),
1579 start: 0,
1580 end: 54,
1581 trigger: Some("folded".to_string()),
1582 },
1583 AnaphorSpan {
1584 text: "This observation".to_string(),
1585 start: 56,
1586 end: 72,
1587 },
1588 AnaphoraType::Fact,
1589 )
1590 .with_notes("Observational fact reference"),
1591 );
1592
1593 dataset.add(
1594 AnaphoraTestCase::new(
1595 "sci_06",
1596 "The simulation predicted climate warming. This prediction matched observed data.",
1597 AntecedentSpan {
1598 text: "The simulation predicted climate warming".to_string(),
1599 start: 0,
1600 end: 40,
1601 trigger: Some("predicted".to_string()),
1602 },
1603 AnaphorSpan {
1604 text: "This prediction".to_string(),
1605 start: 42,
1606 end: 57,
1607 },
1608 AnaphoraType::Fact,
1609 )
1610 .with_notes("Model prediction reference"),
1611 );
1612
1613 dataset.add(
1614 AnaphoraTestCase::new(
1615 "sci_07",
1616 "The theory was disproven by new evidence. Despite this setback, research continues.",
1617 AntecedentSpan {
1618 text: "The theory was disproven by new evidence".to_string(),
1619 start: 0,
1620 end: 40,
1621 trigger: Some("disproven".to_string()),
1622 },
1623 AnaphorSpan {
1624 text: "this setback".to_string(),
1625 start: 50,
1626 end: 62,
1627 },
1628 AnaphoraType::Event,
1629 )
1630 .with_notes("Scientific setback reference"),
1631 );
1632
1633 dataset.add(
1634 AnaphoraTestCase::new(
1635 "sci_08",
1636 "The algorithm achieved high accuracy. This result was widely discussed.",
1637 AntecedentSpan {
1638 text: "The algorithm achieved high accuracy".to_string(),
1639 start: 0,
1640 end: 35,
1641 trigger: Some("achieved".to_string()),
1642 },
1643 AnaphorSpan {
1644 text: "This result".to_string(),
1645 start: 37,
1646 end: 48,
1647 },
1648 AnaphoraType::Fact,
1649 )
1650 .with_notes("Experimental result reference"),
1651 );
1652
1653 dataset.add(
1655 AnaphoraTestCase::new(
1656 "sci_nom_01",
1657 "The researcher published her findings. She received several awards.",
1658 AntecedentSpan {
1659 text: "The researcher".to_string(),
1660 start: 0,
1661 end: 14,
1662 trigger: None,
1663 },
1664 AnaphorSpan {
1665 text: "She".to_string(),
1666 start: 40,
1667 end: 43,
1668 },
1669 AnaphoraType::Nominal,
1670 )
1671 .with_notes("Standard nominal coreference (researcher)"),
1672 );
1673
1674 dataset
1675 }
1676
1677 #[must_use]
1682 pub fn news_domain() -> Self {
1683 let mut dataset = Self::new();
1684
1685 dataset.add(
1686 AnaphoraTestCase::new(
1687 "news_01",
1688 "The president signed the bill into law. This action fulfilled a campaign promise.",
1689 AntecedentSpan {
1690 text: "The president signed the bill into law".to_string(),
1691 start: 0,
1692 end: 38,
1693 trigger: Some("signed".to_string()),
1694 },
1695 AnaphorSpan {
1696 text: "This action".to_string(),
1697 start: 40,
1698 end: 51,
1699 },
1700 AnaphoraType::Event,
1701 )
1702 .with_notes("Political action reference"),
1703 );
1704
1705 dataset.add(
1706 AnaphoraTestCase::new(
1707 "news_02",
1708 "Protests erupted across major cities. This unrest prompted a government response.",
1709 AntecedentSpan {
1710 text: "Protests erupted across major cities".to_string(),
1711 start: 0,
1712 end: 36,
1713 trigger: Some("erupted".to_string()),
1714 },
1715 AnaphorSpan {
1716 text: "This unrest".to_string(),
1717 start: 38,
1718 end: 49,
1719 },
1720 AnaphoraType::Event,
1721 )
1722 .with_notes("Social unrest reference"),
1723 );
1724
1725 dataset.add(
1726 AnaphoraTestCase::new(
1727 "news_03",
1728 "The minister denied any wrongdoing. This denial contradicted earlier statements.",
1729 AntecedentSpan {
1730 text: "The minister denied any wrongdoing".to_string(),
1731 start: 0,
1732 end: 34,
1733 trigger: Some("denied".to_string()),
1734 },
1735 AnaphorSpan {
1736 text: "This denial".to_string(),
1737 start: 36,
1738 end: 47,
1739 },
1740 AnaphoraType::Event,
1741 )
1742 .with_notes("Statement/denial reference"),
1743 );
1744
1745 dataset.add(
1746 AnaphoraTestCase::new(
1747 "news_04",
1748 "Peace talks collapsed after three days. The breakdown disappointed international observers.",
1749 AntecedentSpan {
1750 text: "Peace talks collapsed after three days".to_string(),
1751 start: 0,
1752 end: 38,
1753 trigger: Some("collapsed".to_string()),
1754 },
1755 AnaphorSpan {
1756 text: "The breakdown".to_string(),
1757 start: 40,
1758 end: 53,
1759 },
1760 AnaphoraType::Event,
1761 )
1762 .with_notes("Diplomatic breakdown reference"),
1763 );
1764
1765 dataset.add(
1766 AnaphoraTestCase::new(
1767 "news_05",
1768 "The hurricane devastated coastal towns. This disaster left thousands homeless.",
1769 AntecedentSpan {
1770 text: "The hurricane devastated coastal towns".to_string(),
1771 start: 0,
1772 end: 38,
1773 trigger: Some("devastated".to_string()),
1774 },
1775 AnaphorSpan {
1776 text: "This disaster".to_string(),
1777 start: 40,
1778 end: 53,
1779 },
1780 AnaphoraType::Event,
1781 )
1782 .with_notes("Natural disaster reference"),
1783 );
1784
1785 dataset.add(
1786 AnaphoraTestCase::new(
1787 "news_06",
1788 "The celebrity apologized publicly. This apology came after widespread backlash.",
1789 AntecedentSpan {
1790 text: "The celebrity apologized publicly".to_string(),
1791 start: 0,
1792 end: 33,
1793 trigger: Some("apologized".to_string()),
1794 },
1795 AnaphorSpan {
1796 text: "This apology".to_string(),
1797 start: 35,
1798 end: 47,
1799 },
1800 AnaphoraType::Event,
1801 )
1802 .with_notes("Public apology reference"),
1803 );
1804
1805 dataset.add(
1806 AnaphoraTestCase::new(
1807 "news_07",
1808 "The election results were contested. This controversy led to legal challenges.",
1809 AntecedentSpan {
1810 text: "The election results were contested".to_string(),
1811 start: 0,
1812 end: 35,
1813 trigger: Some("contested".to_string()),
1814 },
1815 AnaphorSpan {
1816 text: "This controversy".to_string(),
1817 start: 37,
1818 end: 53,
1819 },
1820 AnaphoraType::Event,
1821 )
1822 .with_notes("Political controversy reference"),
1823 );
1824
1825 dataset.add(
1826 AnaphoraTestCase::new(
1827 "news_08",
1828 "Unemployment fell to a historic low. This improvement boosted consumer spending.",
1829 AntecedentSpan {
1830 text: "Unemployment fell to a historic low".to_string(),
1831 start: 0,
1832 end: 35,
1833 trigger: Some("fell".to_string()),
1834 },
1835 AnaphorSpan {
1836 text: "This improvement".to_string(),
1837 start: 37,
1838 end: 53,
1839 },
1840 AnaphoraType::Event,
1841 )
1842 .with_notes("Economic improvement reference"),
1843 );
1844
1845 dataset.add(
1847 AnaphoraTestCase::new(
1848 "news_nom_01",
1849 "The mayor addressed the media. He promised immediate action.",
1850 AntecedentSpan {
1851 text: "The mayor".to_string(),
1852 start: 0,
1853 end: 9,
1854 trigger: None,
1855 },
1856 AnaphorSpan {
1857 text: "He".to_string(),
1858 start: 32,
1859 end: 34,
1860 },
1861 AnaphoraType::Nominal,
1862 )
1863 .with_notes("Standard nominal coreference (mayor)"),
1864 );
1865
1866 dataset
1867 }
1868
1869 #[must_use]
1874 pub fn challenging_cases() -> Self {
1875 let mut dataset = Self::new();
1876
1877 dataset.add(
1879 AnaphoraTestCase::new(
1880 "chal_01",
1881 "The company reported strong earnings. Analysts praised the results. Investors celebrated. This success was unexpected.",
1882 AntecedentSpan {
1883 text: "The company reported strong earnings".to_string(),
1884 start: 0,
1885 end: 36,
1886 trigger: Some("reported".to_string()),
1887 },
1888 AnaphorSpan {
1889 text: "This success".to_string(),
1890 start: 91,
1891 end: 103,
1892 },
1893 AnaphoraType::Event,
1894 )
1895 .with_notes("Long-distance (3 sentences back)"),
1896 );
1897
1898 dataset.add(
1900 AnaphoraTestCase::new(
1901 "chal_02",
1902 "This much is clear: the policy has failed.",
1903 AntecedentSpan {
1904 text: "the policy has failed".to_string(),
1905 start: 20,
1906 end: 41,
1907 trigger: Some("failed".to_string()),
1908 },
1909 AnaphorSpan {
1910 text: "This much".to_string(),
1911 start: 0,
1912 end: 9,
1913 },
1914 AnaphoraType::Fact,
1915 )
1916 .with_notes("Cataphoric reference"),
1917 );
1918
1919 dataset.add(
1921 AnaphoraTestCase::new(
1922 "chal_03",
1923 "Inflation rose while wages stagnated and unemployment increased. This combination created economic hardship.",
1924 AntecedentSpan {
1925 text: "Inflation rose while wages stagnated and unemployment increased".to_string(),
1926 start: 0,
1927 end: 63,
1928 trigger: None,
1929 },
1930 AnaphorSpan {
1931 text: "This combination".to_string(),
1932 start: 65,
1933 end: 81,
1934 },
1935 AnaphoraType::Situation,
1936 )
1937 .with_notes("Multi-clause conjunction antecedent"),
1938 );
1939
1940 dataset.add(
1942 AnaphoraTestCase::new(
1943 "chal_04",
1944 "The CEO said that layoffs were necessary. This claim angered workers.",
1945 AntecedentSpan {
1946 text: "layoffs were necessary".to_string(),
1947 start: 18,
1948 end: 40,
1949 trigger: None,
1950 },
1951 AnaphorSpan {
1952 text: "This claim".to_string(),
1953 start: 42,
1954 end: 52,
1955 },
1956 AnaphoraType::Proposition,
1957 )
1958 .with_notes("Embedded clause antecedent"),
1959 );
1960
1961 dataset.add(
1963 AnaphoraTestCase::new(
1964 "chal_05",
1965 "The witness did not appear in court. This absence was noted by the judge.",
1966 AntecedentSpan {
1967 text: "The witness did not appear in court".to_string(),
1968 start: 0,
1969 end: 35,
1970 trigger: Some("appear".to_string()),
1971 },
1972 AnaphorSpan {
1973 text: "This absence".to_string(),
1974 start: 37,
1975 end: 49,
1976 },
1977 AnaphoraType::Event,
1978 )
1979 .with_notes("Negated event antecedent"),
1980 );
1981
1982 dataset.add(
1984 AnaphoraTestCase::new(
1985 "chal_06",
1986 "Either the system crashed or data was corrupted. This problem halted operations.",
1987 AntecedentSpan {
1988 text: "Either the system crashed or data was corrupted".to_string(),
1989 start: 0,
1990 end: 47,
1991 trigger: None,
1992 },
1993 AnaphorSpan {
1994 text: "This problem".to_string(),
1995 start: 49,
1996 end: 61,
1997 },
1998 AnaphoraType::Situation,
1999 )
2000 .with_notes("Disjunction antecedent"),
2001 );
2002
2003 dataset.add(
2005 AnaphoraTestCase::new(
2006 "chal_07",
2007 "If interest rates rise, housing prices will fall. This scenario worries homeowners.",
2008 AntecedentSpan {
2009 text: "If interest rates rise, housing prices will fall".to_string(),
2010 start: 0,
2011 end: 48,
2012 trigger: None,
2013 },
2014 AnaphorSpan {
2015 text: "This scenario".to_string(),
2016 start: 50,
2017 end: 63,
2018 },
2019 AnaphoraType::Proposition,
2020 )
2021 .with_notes("Conditional antecedent"),
2022 );
2023
2024 dataset.add(
2026 AnaphoraTestCase::new(
2027 "chal_08",
2028 "Profits are higher than last year. This exceeds expectations.",
2029 AntecedentSpan {
2030 text: "Profits are higher than last year".to_string(),
2031 start: 0,
2032 end: 33,
2033 trigger: None,
2034 },
2035 AnaphorSpan {
2036 text: "This".to_string(),
2037 start: 35,
2038 end: 39,
2039 },
2040 AnaphoraType::Fact,
2041 )
2042 .with_notes("Comparative statement antecedent"),
2043 );
2044
2045 dataset.add(
2047 AnaphoraTestCase::new(
2048 "chal_09",
2049 "Will the company surcerno? This question haunts investors.",
2050 AntecedentSpan {
2051 text: "Will the company surcerno".to_string(),
2052 start: 0,
2053 end: 24,
2054 trigger: None,
2055 },
2056 AnaphorSpan {
2057 text: "This question".to_string(),
2058 start: 27,
2059 end: 40,
2060 },
2061 AnaphoraType::Proposition,
2062 )
2063 .with_notes("Interrogative clause antecedent"),
2064 );
2065
2066 dataset.add(
2068 AnaphoraTestCase::new(
2069 "chal_10",
2070 "Power corrupts. This truth has been known for centuries.",
2071 AntecedentSpan {
2072 text: "Power corrupts".to_string(),
2073 start: 0,
2074 end: 14,
2075 trigger: Some("corrupts".to_string()),
2076 },
2077 AnaphorSpan {
2078 text: "This truth".to_string(),
2079 start: 16,
2080 end: 26,
2081 },
2082 AnaphoraType::Fact,
2083 )
2084 .with_notes("Generic statement antecedent"),
2085 );
2086
2087 dataset
2088 }
2089
2090 #[must_use]
2094 pub fn comprehensive() -> Self {
2095 let mut dataset = Self::extended();
2096
2097 for case in Self::legal_domain().cases {
2099 dataset.add(case);
2100 }
2101 for case in Self::medical_domain().cases {
2102 dataset.add(case);
2103 }
2104 for case in Self::financial_domain().cases {
2105 dataset.add(case);
2106 }
2107 for case in Self::scientific_domain().cases {
2108 dataset.add(case);
2109 }
2110 for case in Self::news_domain().cases {
2111 dataset.add(case);
2112 }
2113 for case in Self::challenging_cases().cases {
2114 dataset.add(case);
2115 }
2116
2117 dataset
2118 }
2119
2120 #[must_use]
2122 pub fn stats(&self) -> DatasetStats {
2123 let mut by_type: HashMap<AnaphoraType, usize> = HashMap::new();
2124 for case in &self.cases {
2125 *by_type.entry(case.anaphora_type).or_default() += 1;
2126 }
2127
2128 DatasetStats {
2129 total: self.cases.len(),
2130 nominal: by_type.get(&AnaphoraType::Nominal).copied().unwrap_or(0),
2131 event: by_type.get(&AnaphoraType::Event).copied().unwrap_or(0),
2132 fact: by_type.get(&AnaphoraType::Fact).copied().unwrap_or(0),
2133 proposition: by_type
2134 .get(&AnaphoraType::Proposition)
2135 .copied()
2136 .unwrap_or(0),
2137 situation: by_type.get(&AnaphoraType::Situation).copied().unwrap_or(0),
2138 }
2139 }
2140}
2141
2142#[derive(Debug, Clone, Serialize, Deserialize)]
2144pub struct DatasetStats {
2145 pub total: usize,
2147 pub nominal: usize,
2149 pub event: usize,
2151 pub fact: usize,
2153 pub proposition: usize,
2155 pub situation: usize,
2157}
2158
2159impl DatasetStats {
2160 #[must_use]
2162 pub fn abstract_total(&self) -> usize {
2163 self.event + self.fact + self.proposition + self.situation
2164 }
2165}
2166
2167#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2169pub struct ShellNounAnalysis {
2170 pub total_shell_nouns: usize,
2172 pub by_class: HashMap<ShellNounClass, usize>,
2174 pub demonstrative_count: usize,
2176 pub type_match_count: usize,
2178}
2179
2180#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2190pub struct CandidateRankingMetrics {
2191 pub accuracy_at_1: f64,
2193 pub mrr: f64,
2195 pub containment: f64,
2197 pub avg_candidates: f64,
2199 pub total_cases: usize,
2201}
2202
2203impl CandidateRankingMetrics {
2204 #[must_use]
2208 pub fn from_rankings(rankings: &[(usize, usize)]) -> Self {
2209 if rankings.is_empty() {
2210 return Self::default();
2211 }
2212
2213 let total = rankings.len();
2214 let mut correct_at_1 = 0;
2215 let mut reciprocal_sum = 0.0;
2216 let mut contained = 0;
2217 let mut total_candidates = 0;
2218
2219 for &(gold_rank, num_candidates) in rankings {
2220 total_candidates += num_candidates;
2221
2222 if gold_rank > 0 {
2223 contained += 1;
2224 reciprocal_sum += 1.0 / gold_rank as f64;
2225
2226 if gold_rank == 1 {
2227 correct_at_1 += 1;
2228 }
2229 }
2230 }
2231
2232 Self {
2233 accuracy_at_1: correct_at_1 as f64 / total as f64,
2234 mrr: reciprocal_sum / total as f64,
2235 containment: contained as f64 / total as f64,
2236 avg_candidates: total_candidates as f64 / total as f64,
2237 total_cases: total,
2238 }
2239 }
2240
2241 #[must_use]
2243 pub fn summary(&self) -> String {
2244 format!(
2245 "Candidate Ranking:\n Accuracy@1: {:.1}%\n MRR: {:.3}\n Containment: {:.1}%\n Avg candidates: {:.1}",
2246 self.accuracy_at_1 * 100.0,
2247 self.mrr,
2248 self.containment * 100.0,
2249 self.avg_candidates
2250 )
2251 }
2252}
2253
2254impl ShellNounAnalysis {
2255 #[must_use]
2257 pub fn demonstrative_ratio(&self) -> f64 {
2258 if self.total_shell_nouns == 0 {
2259 0.0
2260 } else {
2261 self.demonstrative_count as f64 / self.total_shell_nouns as f64
2262 }
2263 }
2264
2265 #[must_use]
2267 pub fn type_match_ratio(&self) -> f64 {
2268 if self.total_shell_nouns == 0 {
2269 0.0
2270 } else {
2271 self.type_match_count as f64 / self.total_shell_nouns as f64
2272 }
2273 }
2274
2275 #[must_use]
2277 pub fn summary(&self) -> String {
2278 let mut s = format!(
2279 "Shell nouns: {} total, {:.0}% demonstrative, {:.0}% type-matched\n",
2280 self.total_shell_nouns,
2281 self.demonstrative_ratio() * 100.0,
2282 self.type_match_ratio() * 100.0
2283 );
2284 for (class, count) in &self.by_class {
2285 s.push_str(&format!(" {}: {}\n", class.as_str(), count));
2286 }
2287 s
2288 }
2289}
2290
2291#[derive(Debug, Clone)]
2297pub enum ResolverBackend {
2298 Simple(SimpleCorefResolver),
2300 DiscourseAware,
2302}
2303
2304impl Default for ResolverBackend {
2305 fn default() -> Self {
2306 ResolverBackend::Simple(SimpleCorefResolver::default())
2307 }
2308}
2309
2310#[derive(Debug, Clone)]
2312pub struct AbstractAnaphoraEvaluator {
2313 resolver: SimpleCorefResolver,
2314 use_discourse: bool,
2315}
2316
2317impl Default for AbstractAnaphoraEvaluator {
2318 fn default() -> Self {
2319 Self::new(SimpleCorefResolver::default())
2320 }
2321}
2322
2323impl AbstractAnaphoraEvaluator {
2324 #[must_use]
2326 pub fn new(resolver: SimpleCorefResolver) -> Self {
2327 Self {
2328 resolver,
2329 use_discourse: false,
2330 }
2331 }
2332
2333 #[must_use]
2338 pub fn discourse_aware() -> Self {
2339 Self {
2340 resolver: SimpleCorefResolver::default(),
2341 use_discourse: true,
2342 }
2343 }
2344
2345 #[must_use]
2347 pub fn with_discourse(mut self, enable: bool) -> Self {
2348 self.use_discourse = enable;
2349 self
2350 }
2351
2352 #[must_use]
2354 pub fn evaluate(&self, dataset: &AbstractAnaphoraDataset) -> EvaluationResults {
2355 let mut results = EvaluationResults::default();
2356
2357 for case in &dataset.cases {
2358 let result = if self.use_discourse {
2359 self.evaluate_case_discourse(case)
2360 } else {
2361 self.evaluate_case(case)
2362 };
2363 results.case_results.push(result.clone());
2364
2365 if case.anaphora_type == AnaphoraType::Nominal {
2366 results.nominal_total += 1;
2367 if result.resolved_correctly {
2368 results.nominal_correct += 1;
2369 }
2370 } else {
2371 results.abstract_total += 1;
2372 if result.resolved_correctly {
2373 results.abstract_correct += 1;
2374 }
2375 results
2377 .by_type
2378 .entry(case.anaphora_type)
2379 .or_insert_with(TypeResults::default)
2380 .add(&result);
2381 }
2382 }
2383
2384 results.compute_accuracy();
2385 results
2386 }
2387
2388 fn evaluate_case(&self, case: &AnaphoraTestCase) -> CaseResult {
2390 let entities = self.extract_entities_for_case(case);
2395 let resolved = self.resolver.resolve(&entities);
2396
2397 let antecedent_id = resolved
2399 .iter()
2400 .find(|e| {
2401 e.start() == case.antecedent.start
2402 || self.text_matches(&e.text, &case.antecedent.text)
2403 })
2404 .and_then(|e| e.canonical_id.map(|id| id.get()));
2405
2406 let anaphor_id = resolved
2407 .iter()
2408 .find(|e| {
2409 e.start() == case.anaphor.start || self.text_matches(&e.text, &case.anaphor.text)
2410 })
2411 .and_then(|e| e.canonical_id.map(|id| id.get()));
2412
2413 let resolved_correctly = match (antecedent_id, anaphor_id) {
2414 (Some(a), Some(b)) => a == b,
2415 _ => false,
2416 };
2417
2418 CaseResult {
2419 case_id: case.id.clone(),
2420 anaphora_type: case.anaphora_type,
2421 resolved_correctly,
2422 antecedent_found: antecedent_id.is_some(),
2423 anaphor_found: anaphor_id.is_some(),
2424 antecedent_id,
2425 anaphor_id,
2426 failure_reason: if resolved_correctly {
2427 None
2428 } else {
2429 Some(self.diagnose_failure(case, antecedent_id, anaphor_id))
2430 },
2431 }
2432 }
2433
2434 fn evaluate_case_discourse(&self, case: &AnaphoraTestCase) -> CaseResult {
2436 let config = DiscourseCorefConfig::default();
2437 let resolver = DiscourseAwareResolver::new(config, &case.text);
2438
2439 if case.anaphora_type.is_abstract() {
2441 let anaphor_entity = anno::Entity::new(
2443 &case.anaphor.text,
2444 anno::EntityType::custom("Anaphor", anno::EntityCategory::Misc),
2445 case.anaphor.start,
2446 case.anaphor.end,
2447 1.0,
2448 );
2449
2450 let resolved_correctly =
2452 if let Some(referent) = resolver.find_discourse_antecedent(&anaphor_entity) {
2453 let spans_overlap = referent.start < case.antecedent.end
2455 && referent.end > case.antecedent.start;
2456
2457 let trigger_found = case
2459 .antecedent
2460 .trigger
2461 .as_ref()
2462 .map(|t| referent.text.as_ref().is_some_and(|rt| rt.contains(t)))
2463 .unwrap_or(false);
2464
2465 spans_overlap || trigger_found
2466 } else {
2467 false
2468 };
2469
2470 CaseResult {
2471 case_id: case.id.clone(),
2472 anaphora_type: case.anaphora_type,
2473 resolved_correctly,
2474 antecedent_found: true, anaphor_found: true,
2476 antecedent_id: Some(0),
2477 anaphor_id: if resolved_correctly { Some(0) } else { None },
2478 failure_reason: if resolved_correctly {
2479 None
2480 } else {
2481 Some("Discourse resolver couldn't find event antecedent".to_string())
2482 },
2483 }
2484 } else {
2485 self.evaluate_case(case)
2487 }
2488 }
2489
2490 fn extract_entities_for_case(&self, case: &AnaphoraTestCase) -> Vec<Entity> {
2494 let mut entities = Vec::new();
2495
2496 if case.anaphora_type == AnaphoraType::Nominal {
2498 entities.push(Entity::new(
2500 &case.antecedent.text,
2501 self.infer_entity_type(&case.antecedent.text),
2502 case.antecedent.start,
2503 case.antecedent.end,
2504 0.9,
2505 ));
2506
2507 entities.push(Entity::new(
2509 &case.anaphor.text,
2510 self.infer_entity_type(&case.anaphor.text),
2511 case.anaphor.start,
2512 case.anaphor.end,
2513 0.85,
2514 ));
2515 } else {
2516 let antecedent_entities =
2522 self.extract_named_entities(&case.antecedent.text, case.antecedent.start);
2523 entities.extend(antecedent_entities);
2524
2525 entities.push(Entity::new(
2527 &case.anaphor.text,
2528 EntityType::custom("abstract_anaphor", EntityCategory::Misc),
2529 case.anaphor.start,
2530 case.anaphor.end,
2531 0.8,
2532 ));
2533 }
2534
2535 entities
2536 }
2537
2538 fn extract_named_entities(&self, text: &str, offset: usize) -> Vec<Entity> {
2542 let mut entities = Vec::new();
2543
2544 let mut prev_is_ws = true;
2546 let mut char_idx = 0usize;
2547 for (byte_idx, c) in text.char_indices() {
2548 if c.is_uppercase() && (byte_idx == 0 || prev_is_ws) {
2549 let mut end_byte = text.len();
2551 let mut end_char_idx = char_idx;
2552 for (j, cc) in text[byte_idx..].char_indices() {
2553 if j == 0 {
2554 continue;
2555 }
2556 if cc.is_whitespace() || cc == '.' || cc == ',' {
2557 end_byte = byte_idx + j;
2558 end_char_idx = char_idx + text[byte_idx..end_byte].chars().count();
2560 break;
2561 }
2562 }
2563 if end_byte == text.len() {
2564 end_char_idx = char_idx + text[byte_idx..].chars().count();
2565 }
2566
2567 let word = &text[byte_idx..end_byte];
2568 if word.chars().count() > 1 && !self.is_sentence_starter(word, char_idx) {
2569 entities.push(Entity::new(
2570 word,
2571 self.infer_entity_type(word),
2572 offset + char_idx,
2573 offset + end_char_idx,
2574 0.7,
2575 ));
2576 }
2577 }
2578 prev_is_ws = c.is_whitespace();
2579 char_idx += 1;
2580 }
2581
2582 entities
2583 }
2584
2585 fn is_sentence_starter(&self, word: &str, pos: usize) -> bool {
2587 pos == 0
2588 && matches!(
2589 word.to_lowercase().as_str(),
2590 "the" | "a" | "an" | "this" | "that" | "it" | "he" | "she" | "they"
2591 )
2592 }
2593
2594 fn infer_entity_type(&self, text: &str) -> EntityType {
2596 let lower = text.to_lowercase();
2597
2598 if matches!(
2600 lower.as_str(),
2601 "he" | "him" | "his" | "she" | "her" | "hers" | "they" | "them" | "their"
2602 ) {
2603 return EntityType::Person;
2604 }
2605
2606 if lower.starts_with("the company")
2608 || lower.starts_with("the firm")
2609 || lower.starts_with("the organization")
2610 {
2611 return EntityType::Organization;
2612 }
2613
2614 if text.ends_with("Inc.") || text.ends_with("Corp.") || text.ends_with("LLC") {
2616 return EntityType::Organization;
2617 }
2618
2619 if text.starts_with("Dr.")
2621 || text.starts_with("Mr.")
2622 || text.starts_with("Ms.")
2623 || text.starts_with("Prof.")
2624 {
2625 return EntityType::Person;
2626 }
2627
2628 if text.chars().next().is_some_and(|c| c.is_uppercase()) {
2630 return EntityType::Person;
2631 }
2632
2633 EntityType::custom("unknown", EntityCategory::Misc)
2634 }
2635
2636 fn text_matches(&self, a: &str, b: &str) -> bool {
2638 let normalize = |s: &str| {
2639 s.to_lowercase()
2640 .chars()
2641 .filter(|c| c.is_alphanumeric() || c.is_whitespace())
2642 .collect::<String>()
2643 };
2644 normalize(a) == normalize(b)
2645 }
2646
2647 #[must_use]
2652 pub fn detect_shell_noun(&self, anaphor_text: &str) -> Option<ShellNoun> {
2653 let words: Vec<&str> = anaphor_text.split_whitespace().collect();
2654
2655 if words.len() >= 2 {
2657 let determiner = words[0].to_lowercase();
2658 if matches!(
2659 determiner.as_str(),
2660 "this" | "that" | "the" | "these" | "those"
2661 ) {
2662 let noun = words
2664 .last()
2665 .expect("words has at least 2 elements")
2666 .to_lowercase();
2667 let noun = noun.trim_matches(|c: char| !c.is_alphanumeric());
2669
2670 if let Some(class) = classify_shell_noun(noun) {
2671 return Some(
2672 ShellNoun::new(noun, class)
2673 .with_determiner(&determiner)
2674 .with_full_text(anaphor_text),
2675 );
2676 }
2677 }
2678 }
2679
2680 if words.len() == 1 {
2682 let noun = words[0].to_lowercase();
2683 let noun = noun.trim_matches(|c: char| !c.is_alphanumeric());
2684 if let Some(class) = classify_shell_noun(noun) {
2685 return Some(ShellNoun::new(noun, class).with_full_text(anaphor_text));
2686 }
2687 }
2688
2689 None
2690 }
2691
2692 #[must_use]
2694 pub fn analyze_shell_nouns(&self, dataset: &AbstractAnaphoraDataset) -> ShellNounAnalysis {
2695 let mut analysis = ShellNounAnalysis::default();
2696
2697 for case in &dataset.cases {
2698 if let Some(shell) = self.detect_shell_noun(&case.anaphor.text) {
2699 analysis.total_shell_nouns += 1;
2700 *analysis.by_class.entry(shell.class).or_default() += 1;
2701
2702 if shell.is_demonstrative() {
2703 analysis.demonstrative_count += 1;
2704 }
2705
2706 let expected_types = shell.typical_antecedent_types();
2708 let actual_type: ReferentType = match case.anaphora_type {
2709 AnaphoraType::Nominal => ReferentType::Nominal,
2710 AnaphoraType::Event => ReferentType::Event,
2711 AnaphoraType::Fact => ReferentType::Fact,
2712 AnaphoraType::Proposition => ReferentType::Proposition,
2713 AnaphoraType::Situation => ReferentType::Situation,
2714 };
2715
2716 if expected_types.contains(&actual_type) {
2717 analysis.type_match_count += 1;
2718 }
2719 }
2720 }
2721
2722 analysis
2723 }
2724
2725 fn diagnose_failure(
2727 &self,
2728 case: &AnaphoraTestCase,
2729 antecedent_id: Option<u64>,
2730 anaphor_id: Option<u64>,
2731 ) -> String {
2732 let shell_info = if let Some(shell) = self.detect_shell_noun(&case.anaphor.text) {
2734 format!(" [shell noun: {} ({})]", shell.lemma, shell.class.as_str())
2735 } else {
2736 String::new()
2737 };
2738
2739 if case.anaphora_type.is_abstract() {
2740 return format!(
2741 "Abstract anaphora ({}) - resolver cannot detect event/proposition antecedents{}",
2742 case.anaphora_type.as_str(),
2743 shell_info
2744 );
2745 }
2746
2747 match (antecedent_id, anaphor_id) {
2748 (None, None) => "Neither antecedent nor anaphor was assigned a cluster".to_string(),
2749 (None, Some(_)) => "Antecedent was not assigned a cluster".to_string(),
2750 (Some(_), None) => "Anaphor was not assigned a cluster".to_string(),
2751 (Some(a), Some(b)) => format!("Assigned to different clusters: {} vs {}", a, b),
2752 }
2753 }
2754}
2755
2756#[derive(Debug, Clone, Serialize, Deserialize)]
2762pub struct CaseResult {
2763 pub case_id: String,
2765 pub anaphora_type: AnaphoraType,
2767 pub resolved_correctly: bool,
2769 pub antecedent_found: bool,
2771 pub anaphor_found: bool,
2773 pub antecedent_id: Option<u64>,
2775 pub anaphor_id: Option<u64>,
2777 pub failure_reason: Option<String>,
2779}
2780
2781#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2783pub struct TypeResults {
2784 pub total: usize,
2786 pub correct: usize,
2788}
2789
2790impl TypeResults {
2791 fn add(&mut self, result: &CaseResult) {
2792 self.total += 1;
2793 if result.resolved_correctly {
2794 self.correct += 1;
2795 }
2796 }
2797
2798 #[must_use]
2800 pub fn accuracy(&self) -> f64 {
2801 if self.total == 0 {
2802 0.0
2803 } else {
2804 self.correct as f64 / self.total as f64
2805 }
2806 }
2807}
2808
2809#[derive(Debug, Clone, Default, Serialize, Deserialize)]
2811pub struct EvaluationResults {
2812 pub case_results: Vec<CaseResult>,
2814 pub nominal_total: usize,
2816 pub nominal_correct: usize,
2818 pub nominal_accuracy: f64,
2820 pub abstract_total: usize,
2822 pub abstract_correct: usize,
2824 pub abstract_accuracy: f64,
2826 pub by_type: HashMap<AnaphoraType, TypeResults>,
2828}
2829
2830#[derive(Debug, Clone, Default)]
2837pub struct LeaAnalysis {
2838 pub nominal: CorefScores,
2840 pub abstract_anaphora: CorefScores,
2842}
2843
2844impl LeaAnalysis {
2845 #[must_use]
2847 pub fn f1_gap(&self) -> f64 {
2848 self.nominal.f1 - self.abstract_anaphora.f1
2849 }
2850
2851 #[must_use]
2853 pub fn summary(&self) -> String {
2854 format!(
2855 "LEA Analysis:\n Nominal: P={:.1}% R={:.1}% F1={:.1}%\n Abstract: P={:.1}% R={:.1}% F1={:.1}%\n Gap: {:.1}pp",
2856 self.nominal.precision * 100.0,
2857 self.nominal.recall * 100.0,
2858 self.nominal.f1 * 100.0,
2859 self.abstract_anaphora.precision * 100.0,
2860 self.abstract_anaphora.recall * 100.0,
2861 self.abstract_anaphora.f1 * 100.0,
2862 self.f1_gap() * 100.0
2863 )
2864 }
2865}
2866
2867impl EvaluationResults {
2868 fn compute_accuracy(&mut self) {
2869 self.nominal_accuracy = if self.nominal_total == 0 {
2870 0.0
2871 } else {
2872 self.nominal_correct as f64 / self.nominal_total as f64
2873 };
2874
2875 self.abstract_accuracy = if self.abstract_total == 0 {
2876 0.0
2877 } else {
2878 self.abstract_correct as f64 / self.abstract_total as f64
2879 };
2880 }
2881
2882 #[must_use]
2884 pub fn accuracy_gap(&self) -> f64 {
2885 self.nominal_accuracy - self.abstract_accuracy
2886 }
2887
2888 #[must_use]
2898 pub fn compute_lea_scores(&self, dataset: &AbstractAnaphoraDataset) -> LeaAnalysis {
2899 let mut nominal_gold = Vec::new();
2900 let mut nominal_pred = Vec::new();
2901 let mut abstract_gold = Vec::new();
2902 let mut abstract_pred = Vec::new();
2903
2904 for (case, result) in dataset.cases.iter().zip(self.case_results.iter()) {
2905 let antecedent_mention = Mention::new(
2906 &case.antecedent.text,
2907 case.antecedent.start,
2908 case.antecedent.end,
2909 );
2910 let anaphor_mention =
2911 Mention::new(&case.anaphor.text, case.anaphor.start, case.anaphor.end);
2912
2913 let gold_chain =
2915 CorefChain::new(vec![antecedent_mention.clone(), anaphor_mention.clone()]);
2916
2917 let pred_chains: Vec<CorefChain> = if result.resolved_correctly {
2920 vec![CorefChain::new(vec![
2922 antecedent_mention.clone(),
2923 anaphor_mention.clone(),
2924 ])]
2925 } else {
2926 vec![
2928 CorefChain::new(vec![antecedent_mention.clone()]),
2929 CorefChain::new(vec![anaphor_mention.clone()]),
2930 ]
2931 };
2932
2933 if case.anaphora_type == AnaphoraType::Nominal {
2934 nominal_gold.push(gold_chain);
2935 nominal_pred.extend(pred_chains);
2936 } else {
2937 abstract_gold.push(gold_chain);
2938 abstract_pred.extend(pred_chains);
2939 }
2940 }
2941
2942 let nominal_lea = lea_score(&nominal_pred, &nominal_gold);
2943 let abstract_lea = lea_score(&abstract_pred, &abstract_gold);
2944
2945 LeaAnalysis {
2946 nominal: CorefScores::new(nominal_lea.0, nominal_lea.1),
2947 abstract_anaphora: CorefScores::new(abstract_lea.0, abstract_lea.1),
2948 }
2949 }
2950
2951 #[must_use]
2953 pub fn summary(&self) -> String {
2954 let mut s = String::new();
2955
2956 s.push_str("=== Abstract Anaphora Evaluation Results ===\n\n");
2957
2958 s.push_str(&format!(
2959 "Nominal Coreference: {}/{} ({:.1}%)\n",
2960 self.nominal_correct,
2961 self.nominal_total,
2962 self.nominal_accuracy * 100.0
2963 ));
2964
2965 s.push_str(&format!(
2966 "Abstract Anaphora: {}/{} ({:.1}%)\n",
2967 self.abstract_correct,
2968 self.abstract_total,
2969 self.abstract_accuracy * 100.0
2970 ));
2971
2972 s.push_str(&format!(
2973 "\nAccuracy Gap: {:.1} percentage points\n",
2974 self.accuracy_gap() * 100.0
2975 ));
2976
2977 s.push_str("\n--- By Abstract Type ---\n");
2978 for (atype, results) in &self.by_type {
2979 s.push_str(&format!(
2980 " {}: {}/{} ({:.1}%)\n",
2981 atype.as_str(),
2982 results.correct,
2983 results.total,
2984 results.accuracy() * 100.0
2985 ));
2986 }
2987
2988 s.push_str("\n--- Failure Analysis ---\n");
2989 let failures: Vec<_> = self
2990 .case_results
2991 .iter()
2992 .filter(|r| !r.resolved_correctly)
2993 .collect();
2994 for result in failures.iter().take(10) {
2995 s.push_str(&format!(
2996 " [{}] {}: {}\n",
2997 result.case_id,
2998 result.anaphora_type.as_str(),
2999 result.failure_reason.as_deref().unwrap_or("unknown")
3000 ));
3001 }
3002 if failures.len() > 10 {
3003 s.push_str(&format!(
3004 " ... and {} more failures\n",
3005 failures.len() - 10
3006 ));
3007 }
3008
3009 s
3010 }
3011
3012 #[must_use]
3014 pub fn to_html(&self, dataset: &AbstractAnaphoraDataset) -> String {
3015 let mut html = String::new();
3016
3017 html.push_str(
3018 r#"<!DOCTYPE html>
3019<html lang="en">
3020<head>
3021 <meta charset="UTF-8">
3022 <meta name="viewport" content="width=device-width, initial-scale=1.0">
3023 <title>Abstract Anaphora Evaluation</title>
3024 <style>
3025 :root {
3026 --bg: #0d1117;
3027 --fg: #c9d1d9;
3028 --accent: #58a6ff;
3029 --success: #3fb950;
3030 --failure: #f85149;
3031 --warning: #d29922;
3032 --border: #30363d;
3033 --card-bg: #161b22;
3034 }
3035 body {
3036 font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
3037 background: var(--bg);
3038 color: var(--fg);
3039 margin: 0;
3040 padding: 2rem;
3041 line-height: 1.6;
3042 }
3043 h1, h2, h3 { color: var(--accent); margin-top: 2rem; }
3044 h1 { font-size: 2rem; border-bottom: 1px solid var(--border); padding-bottom: 0.5rem; }
3045 .summary-cards {
3046 display: grid;
3047 grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
3048 gap: 1rem;
3049 margin: 1.5rem 0;
3050 }
3051 .card {
3052 background: var(--card-bg);
3053 border: 1px solid var(--border);
3054 border-radius: 8px;
3055 padding: 1.5rem;
3056 text-align: center;
3057 }
3058 .card-value {
3059 font-size: 2.5rem;
3060 font-weight: bold;
3061 }
3062 .card-label { color: #8b949e; margin-top: 0.5rem; }
3063 .success { color: var(--success); }
3064 .failure { color: var(--failure); }
3065 .warning { color: var(--warning); }
3066 table {
3067 width: 100%;
3068 border-collapse: collapse;
3069 margin: 1rem 0;
3070 }
3071 th, td {
3072 border: 1px solid var(--border);
3073 padding: 0.75rem;
3074 text-align: left;
3075 }
3076 th { background: var(--card-bg); color: var(--accent); }
3077 tr:nth-child(even) { background: rgba(255,255,255,0.02); }
3078 .badge {
3079 display: inline-block;
3080 padding: 0.25rem 0.5rem;
3081 border-radius: 4px;
3082 font-size: 0.85rem;
3083 font-weight: 500;
3084 }
3085 .badge-success { background: rgba(63,185,80,0.2); color: var(--success); }
3086 .badge-failure { background: rgba(248,81,73,0.2); color: var(--failure); }
3087 .badge-nominal { background: rgba(88,166,255,0.2); color: var(--accent); }
3088 .badge-abstract { background: rgba(210,153,34,0.2); color: var(--warning); }
3089 .case-text {
3090 font-family: 'SF Mono', Monaco, monospace;
3091 background: var(--card-bg);
3092 padding: 0.75rem;
3093 border-radius: 4px;
3094 margin: 0.5rem 0;
3095 font-size: 0.9rem;
3096 }
3097 .antecedent { background: rgba(63,185,80,0.3); padding: 2px 4px; border-radius: 2px; }
3098 .anaphor { background: rgba(248,81,73,0.3); padding: 2px 4px; border-radius: 2px; }
3099 .conclusion {
3100 background: var(--card-bg);
3101 border-left: 4px solid var(--failure);
3102 padding: 1rem 1.5rem;
3103 margin: 2rem 0;
3104 }
3105 .chart-bar {
3106 height: 24px;
3107 background: var(--border);
3108 border-radius: 4px;
3109 overflow: hidden;
3110 margin: 0.5rem 0;
3111 }
3112 .chart-fill {
3113 height: 100%;
3114 transition: width 0.3s;
3115 }
3116 </style>
3117</head>
3118<body>
3119 <h1>Abstract Anaphora Evaluation Report</h1>
3120 <p>Demonstrating the gap between nominal coreference and abstract anaphora resolution.</p>
3121"#,
3122 );
3123
3124 html.push_str(
3126 r#"
3127 <div class="summary-cards">
3128 <div class="card">
3129 <div class="card-value success">"#,
3130 );
3131 html.push_str(&format!("{:.0}%", self.nominal_accuracy * 100.0));
3132 html.push_str(
3133 r#"</div>
3134 <div class="card-label">Nominal Accuracy</div>
3135 </div>
3136 <div class="card">
3137 <div class="card-value failure">"#,
3138 );
3139 html.push_str(&format!("{:.0}%", self.abstract_accuracy * 100.0));
3140 html.push_str(
3141 r#"</div>
3142 <div class="card-label">Abstract Accuracy</div>
3143 </div>
3144 <div class="card">
3145 <div class="card-value warning">"#,
3146 );
3147 html.push_str(&format!("{:.0}pp", self.accuracy_gap() * 100.0));
3148 html.push_str(
3149 r#"</div>
3150 <div class="card-label">Performance Gap</div>
3151 </div>
3152 <div class="card">
3153 <div class="card-value">"#,
3154 );
3155 html.push_str(&format!("{}", self.case_results.len()));
3156 html.push_str(
3157 r#"</div>
3158 <div class="card-label">Test Cases</div>
3159 </div>
3160 </div>
3161"#,
3162 );
3163
3164 html.push_str(
3166 r#"
3167 <h2>Accuracy by Anaphora Type</h2>
3168 <table>
3169 <tr>
3170 <th>Type</th>
3171 <th>Correct</th>
3172 <th>Total</th>
3173 <th>Accuracy</th>
3174 <th>Visual</th>
3175 </tr>
3176 <tr>
3177 <td><span class="badge badge-nominal">Nominal</span></td>
3178 <td>"#,
3179 );
3180 html.push_str(&format!("{}", self.nominal_correct));
3181 html.push_str("</td><td>");
3182 html.push_str(&format!("{}", self.nominal_total));
3183 html.push_str("</td><td class=\"success\">");
3184 html.push_str(&format!("{:.1}%", self.nominal_accuracy * 100.0));
3185 html.push_str(
3186 r#"</td>
3187 <td><div class="chart-bar"><div class="chart-fill" style="width: "#,
3188 );
3189 html.push_str(&format!("{}%", (self.nominal_accuracy * 100.0) as u32));
3190 html.push_str(
3191 r#"; background: var(--success);"></div></div></td>
3192 </tr>"#,
3193 );
3194
3195 for (atype, results) in &self.by_type {
3196 html.push_str(&format!(r#"
3197 <tr>
3198 <td><span class="badge badge-abstract">{}</span></td>
3199 <td>{}</td>
3200 <td>{}</td>
3201 <td class="failure">{:.1}%</td>
3202 <td><div class="chart-bar"><div class="chart-fill" style="width: {}%; background: var(--failure);"></div></div></td>
3203 </tr>"#,
3204 atype.as_str(),
3205 results.correct,
3206 results.total,
3207 results.accuracy() * 100.0,
3208 (results.accuracy() * 100.0) as u32
3209 ));
3210 }
3211
3212 html.push_str("</table>");
3213
3214 html.push_str(
3216 r#"
3217 <div class="conclusion">
3218 <h3 style="margin-top: 0;">Conclusion</h3>
3219 <p>The current <code>SimpleCorefResolver</code> achieves <strong class="success">"#,
3220 );
3221 html.push_str(&format!("{:.0}%", self.nominal_accuracy * 100.0));
3222 html.push_str(
3223 r#"</strong> accuracy on nominal coreference but
3224 <strong class="failure">"#,
3225 );
3226 html.push_str(&format!("{:.0}%", self.abstract_accuracy * 100.0));
3227 html.push_str(
3228 r#"</strong> on abstract anaphora.</p>
3229 <p>This "#,
3230 );
3231 html.push_str(&format!("{:.0}", self.accuracy_gap() * 100.0));
3232 html.push_str(r#" percentage point gap demonstrates that:</p>
3233 <ul>
3234 <li>The resolver has <strong>no mechanism</strong> to detect event/proposition antecedents</li>
3235 <li>Abstract pronouns ("this", "that") are linked to the nearest <em>entity</em>, not to events</li>
3236 <li>Solving this requires event extraction + discourse structure modeling</li>
3237 </ul>
3238 </div>
3239"#);
3240
3241 html.push_str(
3243 r#"
3244 <h2>Detailed Results</h2>
3245 <table>
3246 <tr>
3247 <th>ID</th>
3248 <th>Type</th>
3249 <th>Result</th>
3250 <th>Text (highlighted)</th>
3251 <th>Failure Reason</th>
3252 </tr>"#,
3253 );
3254
3255 for (case, result) in dataset.cases.iter().zip(self.case_results.iter()) {
3256 let badge_class = if result.resolved_correctly {
3257 "badge-success"
3258 } else {
3259 "badge-failure"
3260 };
3261 let result_text = if result.resolved_correctly {
3262 "PASS"
3263 } else {
3264 "FAIL"
3265 };
3266
3267 let mut highlighted = case.text.clone();
3269 if case.anaphor.start > case.antecedent.end {
3271 highlighted.insert_str(case.anaphor.end, "</span>");
3272 highlighted.insert_str(case.anaphor.start, "<span class=\"anaphor\">");
3273 highlighted.insert_str(case.antecedent.end, "</span>");
3274 highlighted.insert_str(case.antecedent.start, "<span class=\"antecedent\">");
3275 }
3276
3277 html.push_str(&format!(
3278 r#"
3279 <tr>
3280 <td>{}</td>
3281 <td><span class="badge {}">{}</span></td>
3282 <td><span class="badge {}">{}</span></td>
3283 <td class="case-text">{}</td>
3284 <td>{}</td>
3285 </tr>"#,
3286 result.case_id,
3287 if result.anaphora_type.is_abstract() {
3288 "badge-abstract"
3289 } else {
3290 "badge-nominal"
3291 },
3292 result.anaphora_type.as_str(),
3293 badge_class,
3294 result_text,
3295 highlighted,
3296 result.failure_reason.as_deref().unwrap_or("-")
3297 ));
3298 }
3299
3300 html.push_str("</table>");
3301
3302 html.push_str(r#"
3304 <footer style="margin-top: 3rem; padding-top: 1rem; border-top: 1px solid var(--border); color: #8b949e; font-size: 0.9rem;">
3305 <p>Generated by <code>anno_eval::eval::abstract_anaphora</code></p>
3306 <p>See <code>docs/</code> for repo-local notes and entry points.</p>
3307 </footer>
3308</body>
3309</html>"#);
3310
3311 html
3312 }
3313}
3314
3315#[cfg(test)]
3320mod tests {
3321 use super::*;
3322
3323 #[test]
3324 fn test_dataset_creation() {
3325 let dataset = AbstractAnaphoraDataset::standard();
3326 let stats = dataset.stats();
3327
3328 assert!(stats.total > 0, "Dataset should have cases");
3329 assert!(stats.nominal > 0, "Should have nominal cases");
3330 assert!(stats.abstract_total() > 0, "Should have abstract cases");
3331
3332 println!("Dataset stats: {:?}", stats);
3333 }
3334
3335 #[test]
3336 fn test_anaphora_types() {
3337 assert!(!AnaphoraType::Nominal.is_abstract());
3338 assert!(AnaphoraType::Event.is_abstract());
3339 assert!(AnaphoraType::Fact.is_abstract());
3340 assert!(AnaphoraType::Proposition.is_abstract());
3341 assert!(AnaphoraType::Situation.is_abstract());
3342 }
3343
3344 #[test]
3345 fn test_evaluation_runs() {
3346 let dataset = AbstractAnaphoraDataset::standard();
3347 let evaluator = AbstractAnaphoraEvaluator::default();
3348 let results = evaluator.evaluate(&dataset);
3349
3350 println!("{}", results.summary());
3351
3352 assert!(
3355 results.nominal_accuracy >= results.abstract_accuracy,
3356 "Nominal accuracy ({:.1}%) should be >= abstract ({:.1}%)",
3357 results.nominal_accuracy * 100.0,
3358 results.abstract_accuracy * 100.0
3359 );
3360 }
3361
3362 #[test]
3363 fn test_accuracy_gap_exists() {
3364 let dataset = AbstractAnaphoraDataset::standard();
3365 let evaluator = AbstractAnaphoraEvaluator::default();
3366 let results = evaluator.evaluate(&dataset);
3367
3368 let gap = results.accuracy_gap();
3370 println!("Accuracy gap: {:.1} percentage points", gap * 100.0);
3371
3372 if results.nominal_accuracy > 0.0 {
3375 assert!(
3376 gap > 0.0,
3377 "Expected positive accuracy gap, got {:.1}pp",
3378 gap * 100.0
3379 );
3380 }
3381 }
3382
3383 #[test]
3384 fn test_html_generation() {
3385 let dataset = AbstractAnaphoraDataset::standard();
3386 let evaluator = AbstractAnaphoraEvaluator::default();
3387 let results = evaluator.evaluate(&dataset);
3388
3389 let html = results.to_html(&dataset);
3390 assert!(html.contains("Abstract Anaphora Evaluation"));
3391 assert!(html.contains("Nominal Accuracy"));
3392 assert!(html.contains("Abstract Accuracy"));
3393 }
3394
3395 #[test]
3400 fn test_legal_domain_dataset() {
3401 let dataset = AbstractAnaphoraDataset::legal_domain();
3402 let stats = dataset.stats();
3403
3404 assert!(
3405 stats.total >= 8,
3406 "Legal domain should have at least 8 cases"
3407 );
3408 assert!(stats.abstract_total() >= 7, "Most should be abstract");
3409
3410 assert!(stats.nominal >= 1, "Should include nominal baseline case");
3412 }
3413
3414 #[test]
3415 fn test_medical_domain_dataset() {
3416 let dataset = AbstractAnaphoraDataset::medical_domain();
3417 let stats = dataset.stats();
3418
3419 assert!(
3420 stats.total >= 8,
3421 "Medical domain should have at least 8 cases"
3422 );
3423 assert!(
3424 stats.event >= 3,
3425 "Medical should have event cases (procedures)"
3426 );
3427 }
3428
3429 #[test]
3430 fn test_financial_domain_dataset() {
3431 let dataset = AbstractAnaphoraDataset::financial_domain();
3432 let stats = dataset.stats();
3433
3434 assert!(
3435 stats.total >= 8,
3436 "Financial domain should have at least 8 cases"
3437 );
3438 assert!(
3439 stats.event >= 4,
3440 "Financial should have event cases (transactions)"
3441 );
3442 }
3443
3444 #[test]
3445 fn test_scientific_domain_dataset() {
3446 let dataset = AbstractAnaphoraDataset::scientific_domain();
3447 let stats = dataset.stats();
3448
3449 assert!(
3450 stats.total >= 8,
3451 "Scientific domain should have at least 8 cases"
3452 );
3453 assert!(
3454 stats.fact >= 3,
3455 "Scientific should have fact cases (findings)"
3456 );
3457 }
3458
3459 #[test]
3460 fn test_news_domain_dataset() {
3461 let dataset = AbstractAnaphoraDataset::news_domain();
3462 let stats = dataset.stats();
3463
3464 assert!(stats.total >= 8, "News domain should have at least 8 cases");
3465 assert!(stats.event >= 5, "News should have many event cases");
3466 }
3467
3468 #[test]
3469 fn test_challenging_cases_dataset() {
3470 let dataset = AbstractAnaphoraDataset::challenging_cases();
3471 let stats = dataset.stats();
3472
3473 assert!(
3474 stats.total >= 10,
3475 "Challenging cases should have at least 10 cases"
3476 );
3477
3478 assert!(
3480 stats.abstract_total() == stats.total,
3481 "All challenging cases should be abstract"
3482 );
3483 }
3484
3485 #[test]
3486 fn test_comprehensive_dataset() {
3487 let dataset = AbstractAnaphoraDataset::comprehensive();
3488 let stats = dataset.stats();
3489
3490 let extended_stats = AbstractAnaphoraDataset::extended().stats();
3492 let legal_count = AbstractAnaphoraDataset::legal_domain().stats().total;
3493 let medical_count = AbstractAnaphoraDataset::medical_domain().stats().total;
3494 let financial_count = AbstractAnaphoraDataset::financial_domain().stats().total;
3495 let scientific_count = AbstractAnaphoraDataset::scientific_domain().stats().total;
3496 let news_count = AbstractAnaphoraDataset::news_domain().stats().total;
3497 let challenging_count = AbstractAnaphoraDataset::challenging_cases().stats().total;
3498
3499 let expected_min = extended_stats.total
3500 + legal_count
3501 + medical_count
3502 + financial_count
3503 + scientific_count
3504 + news_count
3505 + challenging_count;
3506
3507 assert!(
3508 stats.total >= expected_min,
3509 "Comprehensive should have at least {} cases, got {}",
3510 expected_min,
3511 stats.total
3512 );
3513
3514 println!("Comprehensive dataset stats:");
3515 println!(" Total: {}", stats.total);
3516 println!(" Nominal: {}", stats.nominal);
3517 println!(" Event: {}", stats.event);
3518 println!(" Fact: {}", stats.fact);
3519 println!(" Proposition: {}", stats.proposition);
3520 println!(" Situation: {}", stats.situation);
3521 }
3522
3523 #[test]
3524 fn test_domain_dataset_evaluation() {
3525 let evaluator = AbstractAnaphoraEvaluator::default();
3527
3528 let domains = [
3529 ("Legal", AbstractAnaphoraDataset::legal_domain()),
3530 ("Medical", AbstractAnaphoraDataset::medical_domain()),
3531 ("Financial", AbstractAnaphoraDataset::financial_domain()),
3532 ("Scientific", AbstractAnaphoraDataset::scientific_domain()),
3533 ("News", AbstractAnaphoraDataset::news_domain()),
3534 ];
3535
3536 for (name, dataset) in domains {
3537 let results = evaluator.evaluate(&dataset);
3538 println!(
3539 "{} domain: {:.1}% abstract accuracy",
3540 name,
3541 results.abstract_accuracy * 100.0
3542 );
3543
3544 assert!(
3546 results.abstract_accuracy < 0.5,
3547 "{} domain: Simple resolver shouldn't exceed 50% on abstract cases",
3548 name
3549 );
3550 }
3551 }
3552}