optirs_core/research/
peer_review.rs

1// Peer review tools and anonymous review systems
2//
3// This module provides tools for managing peer review processes,
4// including anonymous reviews, reviewer assignment, and review quality assessment.
5
6use crate::error::{OptimError, Result};
7use chrono::{DateTime, Utc};
8use serde::{Deserialize, Serialize};
9use std::collections::HashMap;
10
11/// Peer review system manager
12#[derive(Debug, Clone, Serialize, Deserialize)]
13pub struct PeerReviewSystem {
14    /// Review sessions
15    pub sessions: HashMap<String, ReviewSession>,
16    /// Reviewer pool
17    pub reviewers: HashMap<String, Reviewer>,
18    /// Review assignments
19    pub assignments: Vec<ReviewAssignment>,
20    /// Review quality metrics
21    pub quality_metrics: Vec<ReviewQualityMetric>,
22}
23
24/// Review session for a paper or project
25#[derive(Debug, Clone, Serialize, Deserialize)]
26pub struct ReviewSession {
27    /// Session ID
28    pub id: String,
29    /// Paper/project ID
30    pub submission_id: String,
31    /// Review type
32    pub review_type: ReviewType,
33    /// Session status
34    pub status: ReviewSessionStatus,
35    /// Review criteria
36    pub criteria: Vec<ReviewCriterion>,
37    /// Deadline
38    pub deadline: DateTime<Utc>,
39    /// Reviews collected
40    pub reviews: Vec<PeerReview>,
41    /// Meta-review
42    pub meta_review: Option<MetaReview>,
43    /// Discussion thread
44    pub discussion: Vec<ReviewDiscussion>,
45}
46
47/// Types of peer review
48#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
49pub enum ReviewType {
50    /// Single-blind review
51    SingleBlind,
52    /// Double-blind review
53    DoubleBlind,
54    /// Open review
55    Open,
56    /// Post-publication review
57    PostPublication,
58    /// Internal review
59    Internal,
60}
61
62/// Review session status
63#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
64pub enum ReviewSessionStatus {
65    /// Waiting for reviewers
66    WaitingForReviewers,
67    /// Reviews in progress
68    InProgress,
69    /// Reviews complete
70    ReviewsComplete,
71    /// Meta-review in progress
72    MetaReviewInProgress,
73    /// Session complete
74    Complete,
75    /// Session cancelled
76    Cancelled,
77}
78
79/// Review criterion
80#[derive(Debug, Clone, Serialize, Deserialize)]
81pub struct ReviewCriterion {
82    /// Criterion name
83    pub name: String,
84    /// Description
85    pub description: String,
86    /// Score range
87    pub score_range: (f64, f64),
88    /// Weight in overall score
89    pub weight: f64,
90    /// Required for review
91    pub required: bool,
92}
93
94/// Individual peer review
95#[derive(Debug, Clone, Serialize, Deserialize)]
96pub struct PeerReview {
97    /// Review ID
98    pub id: String,
99    /// Anonymous reviewer ID
100    pub reviewer_id: String,
101    /// Overall recommendation
102    pub recommendation: ReviewRecommendation,
103    /// Scores per criterion
104    pub criterion_scores: HashMap<String, f64>,
105    /// Overall score
106    pub overall_score: f64,
107    /// Confidence level
108    pub confidence: f64,
109    /// Written review
110    pub written_review: WrittenReview,
111    /// Review status
112    pub status: ReviewStatus,
113    /// Submission timestamp
114    pub submitted_at: Option<DateTime<Utc>>,
115    /// Time spent on review (minutes)
116    pub time_spent_minutes: Option<u32>,
117}
118
119/// Review recommendations
120#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
121pub enum ReviewRecommendation {
122    /// Strong accept
123    StrongAccept,
124    /// Accept
125    Accept,
126    /// Weak accept
127    WeakAccept,
128    /// Borderline accept
129    BorderlineAccept,
130    /// Borderline reject
131    BorderlineReject,
132    /// Weak reject
133    WeakReject,
134    /// Reject
135    Reject,
136    /// Strong reject
137    StrongReject,
138}
139
140/// Written review components
141#[derive(Debug, Clone, Serialize, Deserialize)]
142pub struct WrittenReview {
143    /// Summary
144    pub summary: String,
145    /// Strengths
146    pub strengths: Vec<String>,
147    /// Weaknesses
148    pub weaknesses: Vec<String>,
149    /// Detailed comments
150    pub detailed_comments: String,
151    /// Questions for authors
152    pub questions: Vec<String>,
153    /// Minor issues
154    pub minor_issues: Vec<String>,
155    /// Suggestions for improvement
156    pub suggestions: Vec<String>,
157    /// Comments for committee only
158    pub committee_comments: Option<String>,
159}
160
161/// Review status
162#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
163pub enum ReviewStatus {
164    /// Assigned but not started
165    Assigned,
166    /// In progress
167    InProgress,
168    /// Draft completed
169    Draft,
170    /// Submitted
171    Submitted,
172    /// Revision requested
173    RevisionRequested,
174    /// Declined
175    Declined,
176    /// Overdue
177    Overdue,
178}
179
180/// Meta-review (review of reviews)
181#[derive(Debug, Clone, Serialize, Deserialize)]
182pub struct MetaReview {
183    /// Meta-reviewer ID
184    pub meta_reviewer_id: String,
185    /// Summary of individual reviews
186    pub review_summary: String,
187    /// Final recommendation
188    pub final_recommendation: ReviewRecommendation,
189    /// Justification
190    pub justification: String,
191    /// Review quality assessment
192    pub review_quality: Vec<ReviewQualityAssessment>,
193    /// Areas of agreement
194    pub areas_of_agreement: Vec<String>,
195    /// Areas of disagreement
196    pub areas_of_disagreement: Vec<String>,
197    /// Decision rationale
198    pub decision_rationale: String,
199}
200
201/// Assessment of individual review quality
202#[derive(Debug, Clone, Serialize, Deserialize)]
203pub struct ReviewQualityAssessment {
204    /// Review ID
205    pub review_id: String,
206    /// Quality dimensions
207    pub quality_scores: HashMap<String, f64>,
208    /// Overall quality score
209    pub overall_quality: f64,
210    /// Helpfulness to authors
211    pub helpfulness: f64,
212    /// Comments on review quality
213    pub comments: String,
214}
215
216/// Review discussion thread
217#[derive(Debug, Clone, Serialize, Deserialize)]
218pub struct ReviewDiscussion {
219    /// Post ID
220    pub id: String,
221    /// Author (anonymous)
222    pub author: String,
223    /// Post content
224    pub content: String,
225    /// Reply to post ID
226    pub reply_to: Option<String>,
227    /// Post timestamp
228    pub posted_at: DateTime<Utc>,
229    /// Post type
230    pub post_type: DiscussionPostType,
231}
232
233/// Types of discussion posts
234#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
235pub enum DiscussionPostType {
236    /// Question
237    Question,
238    /// Answer
239    Answer,
240    /// Clarification
241    Clarification,
242    /// Disagreement
243    Disagreement,
244    /// Consensus
245    Consensus,
246    /// Moderator message
247    ModeratorMessage,
248}
249
250/// Reviewer information
251#[derive(Debug, Clone, Serialize, Deserialize)]
252pub struct Reviewer {
253    /// Anonymous reviewer ID
254    pub id: String,
255    /// Expertise areas
256    pub expertise_areas: Vec<String>,
257    /// Experience level
258    pub experience_level: ExperienceLevel,
259    /// Review history
260    pub review_history: ReviewerHistory,
261    /// Availability
262    pub availability: ReviewerAvailability,
263    /// Quality metrics
264    pub quality_metrics: ReviewerQualityMetrics,
265    /// Preferences
266    pub preferences: ReviewerPreferences,
267}
268
269/// Reviewer experience levels
270#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
271pub enum ExperienceLevel {
272    /// Expert reviewer
273    Expert,
274    /// Senior reviewer
275    Senior,
276    /// Experienced reviewer
277    Experienced,
278    /// Junior reviewer
279    Junior,
280    /// Novice reviewer
281    Novice,
282}
283
284/// Reviewer history and statistics
285#[derive(Debug, Clone, Serialize, Deserialize)]
286pub struct ReviewerHistory {
287    /// Total reviews completed
288    pub total_reviews: usize,
289    /// Reviews in last 12 months
290    pub reviews_last_year: usize,
291    /// Average review time (days)
292    pub avg_review_time_days: f64,
293    /// On-time submission rate
294    pub on_time_rate: f64,
295    /// Average review quality score
296    pub avg_quality_score: f64,
297    /// Review acceptance rate
298    pub review_acceptance_rate: f64,
299}
300
301/// Reviewer availability
302#[derive(Debug, Clone, Serialize, Deserialize)]
303pub struct ReviewerAvailability {
304    /// Currently available
305    pub available: bool,
306    /// Maximum reviews per month
307    pub max_reviews_per_month: u32,
308    /// Current review load
309    pub current_load: u32,
310    /// Unavailable periods
311    pub unavailable_periods: Vec<(DateTime<Utc>, DateTime<Utc>)>,
312    /// Preferred review types
313    pub preferred_types: Vec<ReviewType>,
314}
315
316/// Reviewer quality metrics
317#[derive(Debug, Clone, Serialize, Deserialize)]
318pub struct ReviewerQualityMetrics {
319    /// Thoroughness score
320    pub thoroughness: f64,
321    /// Constructiveness score
322    pub constructiveness: f64,
323    /// Timeliness score
324    pub timeliness: f64,
325    /// Expertise match score
326    pub expertise_match: f64,
327    /// Overall reviewer score
328    pub overall_score: f64,
329}
330
331/// Reviewer preferences
332#[derive(Debug, Clone, Serialize, Deserialize)]
333pub struct ReviewerPreferences {
334    /// Preferred paper types
335    pub preferred_paper_types: Vec<String>,
336    /// Avoid paper types
337    pub avoid_paper_types: Vec<String>,
338    /// Maximum review length preference
339    pub max_review_length: Option<u32>,
340    /// Anonymous review preference
341    pub anonymous_preference: bool,
342    /// Notification preferences
343    pub notification_preferences: NotificationPreferences,
344}
345
346/// Notification preferences for reviewers
347#[derive(Debug, Clone, Serialize, Deserialize)]
348pub struct NotificationPreferences {
349    /// Email notifications
350    pub email: bool,
351    /// Reminder frequency (days)
352    pub reminder_frequency: u32,
353    /// Deadline notifications
354    pub deadline_notifications: bool,
355    /// Discussion notifications
356    pub discussion_notifications: bool,
357}
358
359/// Review assignment
360#[derive(Debug, Clone, Serialize, Deserialize)]
361pub struct ReviewAssignment {
362    /// Assignment ID
363    pub id: String,
364    /// Session ID
365    pub session_id: String,
366    /// Reviewer ID
367    pub reviewer_id: String,
368    /// Assignment date
369    pub assigned_at: DateTime<Utc>,
370    /// Due date
371    pub due_date: DateTime<Utc>,
372    /// Assignment status
373    pub status: AssignmentStatus,
374    /// Assignment method
375    pub assignment_method: AssignmentMethod,
376    /// Expertise match score
377    pub expertise_match: f64,
378}
379
380/// Assignment status
381#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
382pub enum AssignmentStatus {
383    /// Pending acceptance
384    Pending,
385    /// Accepted
386    Accepted,
387    /// Declined
388    Declined,
389    /// Completed
390    Completed,
391    /// Overdue
392    Overdue,
393    /// Cancelled
394    Cancelled,
395}
396
397/// Assignment methods
398#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
399pub enum AssignmentMethod {
400    /// Manual assignment
401    Manual,
402    /// Automatic based on expertise
403    AutomaticExpertise,
404    /// Automatic load balancing
405    AutomaticLoadBalancing,
406    /// Hybrid assignment
407    Hybrid,
408    /// Self-assignment
409    SelfAssignment,
410}
411
412/// Review quality metric
413#[derive(Debug, Clone, Serialize, Deserialize)]
414pub struct ReviewQualityMetric {
415    /// Metric name
416    pub name: String,
417    /// Description
418    pub description: String,
419    /// Value range
420    pub value_range: (f64, f64),
421    /// Higher is better
422    pub higher_is_better: bool,
423    /// Calculation method
424    pub calculation_method: String,
425}
426
427impl Default for PeerReviewSystem {
428    fn default() -> Self {
429        Self::new()
430    }
431}
432
433impl PeerReviewSystem {
434    /// Create a new peer review system
435    pub fn new() -> Self {
436        Self {
437            sessions: HashMap::new(),
438            reviewers: HashMap::new(),
439            assignments: Vec::new(),
440            quality_metrics: Self::create_default_quality_metrics(),
441        }
442    }
443
444    /// Create a new review session
445    pub fn create_review_session(
446        &mut self,
447        submission_id: &str,
448        review_type: ReviewType,
449        criteria: Vec<ReviewCriterion>,
450        deadline: DateTime<Utc>,
451    ) -> String {
452        let session_id = uuid::Uuid::new_v4().to_string();
453        let session = ReviewSession {
454            id: session_id.clone(),
455            submission_id: submission_id.to_string(),
456            review_type,
457            status: ReviewSessionStatus::WaitingForReviewers,
458            criteria,
459            deadline,
460            reviews: Vec::new(),
461            meta_review: None,
462            discussion: Vec::new(),
463        };
464
465        self.sessions.insert(session_id.clone(), session);
466        session_id
467    }
468
469    /// Assign reviewers to a session.
470    ///
471    /// Skips any reviewer who is unknown or who already has an active
472    /// (pending/accepted/completed) assignment for this session -- without
473    /// this check, calling this twice with overlapping reviewer lists (or
474    /// passing a duplicate ID) created two assignments for the same
475    /// (session, reviewer) pair, which corrupted the reviews-complete count
476    /// in [`Self::submit_review`] and double-counted reviewer workload.
477    pub fn assign_reviewers(
478        &mut self,
479        session_id: &str,
480        reviewer_ids: &[String],
481        assignment_method: AssignmentMethod,
482    ) -> Result<Vec<String>> {
483        if !self.sessions.contains_key(session_id) {
484            return Err(OptimError::InvalidConfig("Session not found".to_string()));
485        }
486
487        let mut assignment_ids = Vec::new();
488        let now = Utc::now();
489        let deadline = self
490            .sessions
491            .get(session_id)
492            .ok_or_else(|| {
493                OptimError::InvalidState(format!("session '{session_id}' vanished during lookup"))
494            })?
495            .deadline;
496
497        for reviewer_id in reviewer_ids {
498            if !self.reviewers.contains_key(reviewer_id) {
499                continue; // Skip unknown reviewers
500            }
501
502            let already_assigned = self
503                .assignments
504                .iter()
505                .any(|a| a.session_id == session_id && a.reviewer_id == *reviewer_id);
506            if already_assigned {
507                continue;
508            }
509
510            let assignment_id = uuid::Uuid::new_v4().to_string();
511            let assignment = ReviewAssignment {
512                id: assignment_id.clone(),
513                session_id: session_id.to_string(),
514                reviewer_id: reviewer_id.clone(),
515                assigned_at: now,
516                due_date: deadline,
517                status: AssignmentStatus::Pending,
518                assignment_method: assignment_method.clone(),
519                expertise_match: self.calculate_expertise_match(reviewer_id, session_id),
520            };
521
522            self.assignments.push(assignment);
523            assignment_ids.push(assignment_id);
524
525            // Track load on the reviewer record itself: this is what
526            // `get_available_reviewers` consults to avoid overloading a
527            // reviewer, so it must reflect assignments as they happen.
528            if let Some(reviewer) = self.reviewers.get_mut(reviewer_id) {
529                reviewer.availability.current_load += 1;
530            }
531        }
532
533        // Update session status
534        if let Some(session) = self.sessions.get_mut(session_id) {
535            session.status = ReviewSessionStatus::InProgress;
536        }
537
538        Ok(assignment_ids)
539    }
540
541    /// Submit a peer review.
542    ///
543    /// Rejects a second submission from the same reviewer for the same
544    /// session: without this check, a duplicate call inflated
545    /// `session.reviews` past the number of distinct assigned reviewers,
546    /// which both skewed the meta-review consensus (the same opinion
547    /// counted twice) and could prevent `reviews.len() == total_assignments`
548    /// from ever matching (so the session would never reach
549    /// `ReviewsComplete`).
550    pub fn submit_review(
551        &mut self,
552        session_id: &str,
553        reviewer_id: &str,
554        review: PeerReview,
555    ) -> Result<()> {
556        let session = self
557            .sessions
558            .get_mut(session_id)
559            .ok_or_else(|| OptimError::InvalidConfig("Session not found".to_string()))?;
560
561        // Verify reviewer is assigned
562        let assignment = self
563            .assignments
564            .iter_mut()
565            .find(|a| a.session_id == session_id && a.reviewer_id == reviewer_id)
566            .ok_or_else(|| {
567                OptimError::InvalidConfig("Reviewer not assigned to this session".to_string())
568            })?;
569
570        if assignment.status == AssignmentStatus::Completed {
571            return Err(OptimError::InvalidConfig(format!(
572                "reviewer '{reviewer_id}' already submitted a review for session '{session_id}'"
573            )));
574        }
575
576        // Update assignment status
577        assignment.status = AssignmentStatus::Completed;
578
579        // Add review to session
580        session.reviews.push(review);
581
582        // Check if all reviews are complete
583        let total_assignments = self
584            .assignments
585            .iter()
586            .filter(|a| a.session_id == session_id)
587            .count();
588
589        if session.reviews.len() == total_assignments {
590            session.status = ReviewSessionStatus::ReviewsComplete;
591        }
592
593        // A completed review is no longer part of the reviewer's *active*
594        // load (mirrors `calculate_reviewer_workload`, which only counts
595        // Pending/Accepted assignments), freeing capacity for new
596        // assignments now that this one is done.
597        if let Some(reviewer) = self.reviewers.get_mut(reviewer_id) {
598            reviewer.availability.current_load =
599                reviewer.availability.current_load.saturating_sub(1);
600        }
601
602        Ok(())
603    }
604
605    /// Generate meta-review
606    pub fn generate_meta_review(&mut self, session_id: &str, meta_reviewer_id: &str) -> Result<()> {
607        // First, check session status and create meta review with immutable access
608        let meta_review = {
609            let session = self
610                .sessions
611                .get(session_id)
612                .ok_or_else(|| OptimError::InvalidConfig("Session not found".to_string()))?;
613
614            if session.status != ReviewSessionStatus::ReviewsComplete {
615                return Err(OptimError::InvalidConfig(
616                    "Not all reviews are complete".to_string(),
617                ));
618            }
619
620            self.create_meta_review(session, meta_reviewer_id)
621        };
622
623        // Now update the session with mutable access.
624        let session = self.sessions.get_mut(session_id).ok_or_else(|| {
625            OptimError::InvalidState(format!("session '{session_id}' vanished during lookup"))
626        })?;
627        session.meta_review = Some(meta_review);
628        session.status = ReviewSessionStatus::Complete;
629
630        Ok(())
631    }
632
633    /// Calculate reviewer workload
634    pub fn calculate_reviewer_workload(&self, reviewer_id: &str) -> u32 {
635        self.assignments
636            .iter()
637            .filter(|a| {
638                a.reviewer_id == reviewer_id
639                    && matches!(
640                        a.status,
641                        AssignmentStatus::Pending | AssignmentStatus::Accepted
642                    )
643            })
644            .count() as u32
645    }
646
647    /// Get available reviewers for expertise area
648    pub fn get_available_reviewers(&self, expertise_area: &str) -> Vec<&Reviewer> {
649        self.reviewers
650            .values()
651            .filter(|r| {
652                r.availability.available
653                    && r.expertise_areas
654                        .iter()
655                        .any(|area| area.to_lowercase().contains(&expertise_area.to_lowercase()))
656                    && r.availability.current_load < r.availability.max_reviews_per_month
657            })
658            .collect()
659    }
660
661    /// Calculate review quality score
662    pub fn calculate_review_quality(&self, review: &PeerReview) -> f64 {
663        let mut quality_score = 0.0;
664        let mut total_weight = 0.0;
665
666        // Length and detail assessment
667        let review_length = review.written_review.detailed_comments.len()
668            + review
669                .written_review
670                .strengths
671                .iter()
672                .map(|s| s.len())
673                .sum::<usize>()
674            + review
675                .written_review
676                .weaknesses
677                .iter()
678                .map(|s| s.len())
679                .sum::<usize>();
680
681        // `ln_1p` (ln(1+x)) rather than `ln(x)`: a review with zero
682        // measured length (empty comments/strengths/weaknesses) is a
683        // realistic, valid input, and `ln(0) == -inf` would otherwise poison
684        // the entire quality score to `-inf` (see regression test below).
685        let length_score = ((review_length as f64).ln_1p() / 10.0).clamp(0.0, 1.0);
686        quality_score += length_score * 0.3;
687        total_weight += 0.3;
688
689        // Number of specific points
690        let specific_points = review.written_review.strengths.len()
691            + review.written_review.weaknesses.len()
692            + review.written_review.suggestions.len();
693
694        let specificity_score = (specific_points as f64 / 10.0).min(1.0);
695        quality_score += specificity_score * 0.4;
696        total_weight += 0.4;
697
698        // Confidence level
699        quality_score += review.confidence * 0.3;
700        total_weight += 0.3;
701
702        quality_score / total_weight
703    }
704
705    /// How well a reviewer's declared expertise covers a session's criteria, in
706    /// `[0, 1]`.
707    ///
708    /// The score is the fraction of the session's review criteria whose name or
709    /// description mentions one of the reviewer's expertise areas
710    /// (case-insensitive substring match). A reviewer with no declared expertise
711    /// scores `0.0`, and a session with no criteria yields `0.5` -- there is
712    /// nothing to match against, so neither a good nor a bad match can be
713    /// claimed.
714    ///
715    /// Until 0.3.2 this ignored `sessionid` entirely and returned the constant
716    /// `0.8` for any reviewer with a non-empty `expertise_areas` list and `0.5`
717    /// otherwise, so assignment ranked a cryptographer and a numerical analyst
718    /// identically on an optimization paper.
719    fn calculate_expertise_match(&self, reviewer_id: &str, sessionid: &str) -> f64 {
720        let Some(reviewer) = self.reviewers.get(reviewer_id) else {
721            return 0.0;
722        };
723        if reviewer.expertise_areas.is_empty() {
724            return 0.0;
725        }
726        let Some(session) = self.sessions.get(sessionid) else {
727            return 0.5;
728        };
729        if session.criteria.is_empty() {
730            return 0.5;
731        }
732
733        let areas: Vec<String> = reviewer
734            .expertise_areas
735            .iter()
736            .map(|area| area.to_ascii_lowercase())
737            .filter(|area| !area.is_empty())
738            .collect();
739        if areas.is_empty() {
740            return 0.0;
741        }
742
743        let matched = session
744            .criteria
745            .iter()
746            .filter(|criterion| {
747                let haystack =
748                    format!("{} {}", criterion.name, criterion.description).to_ascii_lowercase();
749                areas.iter().any(|area| haystack.contains(area))
750            })
751            .count();
752        matched as f64 / session.criteria.len() as f64
753    }
754
755    fn create_meta_review(&self, session: &ReviewSession, meta_reviewer_id: &str) -> MetaReview {
756        let review_summary = format!("Meta-review of {} reviews", session.reviews.len());
757
758        // Calculate consensus
759        let recommendations: Vec<_> = session.reviews.iter().map(|r| &r.recommendation).collect();
760
761        let final_recommendation = self.determine_consensus_recommendation(&recommendations);
762
763        // Assess review quality
764        let review_quality: Vec<_> = session
765            .reviews
766            .iter()
767            .map(|review| {
768                let quality_score = self.calculate_review_quality(review);
769                ReviewQualityAssessment {
770                    review_id: review.id.clone(),
771                    quality_scores: HashMap::new(),
772                    overall_quality: quality_score,
773                    helpfulness: quality_score * 0.9, // Simplified
774                    comments: if quality_score > 0.7 {
775                        "High quality review".to_string()
776                    } else {
777                        "Review could be more detailed".to_string()
778                    },
779                }
780            })
781            .collect();
782
783        MetaReview {
784            meta_reviewer_id: meta_reviewer_id.to_string(),
785            review_summary,
786            final_recommendation,
787            justification: "Based on consensus of reviewer recommendations".to_string(),
788            review_quality,
789            areas_of_agreement: vec!["Technical quality assessment".to_string()],
790            areas_of_disagreement: vec!["Significance of contribution".to_string()],
791            decision_rationale: "Decision based on majority reviewer consensus".to_string(),
792        }
793    }
794
795    /// Deterministic consensus: the median of the reviewers' ordinal ranks.
796    ///
797    /// A plain "most common recommendation" vote is not well-defined when
798    /// there is a tie (e.g. two `Accept` and two `Reject`): breaking the tie
799    /// by iterating a `HashMap` makes the outcome depend on hash iteration
800    /// order, so the same set of reviews could yield a different consensus
801    /// recommendation on different runs. The median is always well-defined,
802    /// deterministic, and (unlike the mode) robust to a single outlier
803    /// review.
804    fn determine_consensus_recommendation(
805        &self,
806        recommendations: &[&ReviewRecommendation],
807    ) -> ReviewRecommendation {
808        if recommendations.is_empty() {
809            return ReviewRecommendation::BorderlineReject;
810        }
811
812        let mut ranks: Vec<u8> = recommendations
813            .iter()
814            .map(|rec| Self::recommendation_rank(rec))
815            .collect();
816        ranks.sort_unstable();
817
818        let mid = ranks.len() / 2;
819        let median_rank = if ranks.len().is_multiple_of(2) {
820            // Even count: average the two middle ranks, rounding toward the
821            // more critical (reject) side on an exact half -- an "err on
822            // the side of caution" convention that is itself deterministic.
823            let lower = u16::from(ranks[mid - 1]);
824            let upper = u16::from(ranks[mid]);
825            (lower + upper).div_ceil(2) as u8
826        } else {
827            ranks[mid]
828        };
829
830        Self::recommendation_from_rank(median_rank)
831    }
832
833    /// Ordinal rank of a recommendation from most (0) to least (7)
834    /// favorable, used to compute a deterministic median consensus.
835    fn recommendation_rank(rec: &ReviewRecommendation) -> u8 {
836        match rec {
837            ReviewRecommendation::StrongAccept => 0,
838            ReviewRecommendation::Accept => 1,
839            ReviewRecommendation::WeakAccept => 2,
840            ReviewRecommendation::BorderlineAccept => 3,
841            ReviewRecommendation::BorderlineReject => 4,
842            ReviewRecommendation::WeakReject => 5,
843            ReviewRecommendation::Reject => 6,
844            ReviewRecommendation::StrongReject => 7,
845        }
846    }
847
848    /// Inverse of [`Self::recommendation_rank`].
849    fn recommendation_from_rank(rank: u8) -> ReviewRecommendation {
850        match rank {
851            0 => ReviewRecommendation::StrongAccept,
852            1 => ReviewRecommendation::Accept,
853            2 => ReviewRecommendation::WeakAccept,
854            3 => ReviewRecommendation::BorderlineAccept,
855            4 => ReviewRecommendation::BorderlineReject,
856            5 => ReviewRecommendation::WeakReject,
857            6 => ReviewRecommendation::Reject,
858            _ => ReviewRecommendation::StrongReject,
859        }
860    }
861
862    fn create_default_quality_metrics() -> Vec<ReviewQualityMetric> {
863        vec![
864            ReviewQualityMetric {
865                name: "Thoroughness".to_string(),
866                description: "How comprehensive and detailed the review is".to_string(),
867                value_range: (0.0, 1.0),
868                higher_is_better: true,
869                calculation_method: "Based on review length and number of specific points"
870                    .to_string(),
871            },
872            ReviewQualityMetric {
873                name: "Constructiveness".to_string(),
874                description: "How helpful the review is for improving the work".to_string(),
875                value_range: (0.0, 1.0),
876                higher_is_better: true,
877                calculation_method: "Based on number of suggestions and actionable feedback"
878                    .to_string(),
879            },
880            ReviewQualityMetric {
881                name: "Timeliness".to_string(),
882                description: "How promptly the review was submitted".to_string(),
883                value_range: (0.0, 1.0),
884                higher_is_better: true,
885                calculation_method: "Based on submission time relative to deadline".to_string(),
886            },
887        ]
888    }
889}
890
891impl Default for ReviewerHistory {
892    fn default() -> Self {
893        Self {
894            total_reviews: 0,
895            reviews_last_year: 0,
896            avg_review_time_days: 14.0,
897            on_time_rate: 1.0,
898            avg_quality_score: 0.7,
899            review_acceptance_rate: 0.9,
900        }
901    }
902}
903
904impl Default for ReviewerAvailability {
905    fn default() -> Self {
906        Self {
907            available: true,
908            max_reviews_per_month: 5,
909            current_load: 0,
910            unavailable_periods: Vec::new(),
911            preferred_types: vec![ReviewType::DoubleBlind],
912        }
913    }
914}
915
916impl Default for ReviewerQualityMetrics {
917    fn default() -> Self {
918        Self {
919            thoroughness: 0.7,
920            constructiveness: 0.7,
921            timeliness: 0.8,
922            expertise_match: 0.7,
923            overall_score: 0.7,
924        }
925    }
926}
927
928impl Default for NotificationPreferences {
929    fn default() -> Self {
930        Self {
931            email: true,
932            reminder_frequency: 7,
933            deadline_notifications: true,
934            discussion_notifications: false,
935        }
936    }
937}
938
939#[cfg(test)]
940mod tests {
941    use super::*;
942
943    #[test]
944    fn test_peer_review_system_creation() {
945        let system = PeerReviewSystem::new();
946        assert!(system.sessions.is_empty());
947        assert!(system.reviewers.is_empty());
948        assert!(!system.quality_metrics.is_empty());
949    }
950
951    #[test]
952    fn test_create_review_session() {
953        let mut system = PeerReviewSystem::new();
954
955        let criteria = vec![ReviewCriterion {
956            name: "Technical Quality".to_string(),
957            description: "Assessment of technical merit".to_string(),
958            score_range: (1.0, 5.0),
959            weight: 0.4,
960            required: true,
961        }];
962
963        let deadline = Utc::now() + chrono::Duration::days(14);
964        let session_id =
965            system.create_review_session("paper123", ReviewType::DoubleBlind, criteria, deadline);
966
967        assert!(system.sessions.contains_key(&session_id));
968        let session = &system.sessions[&session_id];
969        assert_eq!(session.submission_id, "paper123");
970        assert_eq!(session.review_type, ReviewType::DoubleBlind);
971    }
972
973    #[test]
974    fn test_reviewer_workload_calculation() {
975        let mut system = PeerReviewSystem::new();
976
977        // Add some assignments
978        system.assignments.push(ReviewAssignment {
979            id: "assign1".to_string(),
980            session_id: "session1".to_string(),
981            reviewer_id: "reviewer1".to_string(),
982            assigned_at: Utc::now(),
983            due_date: Utc::now() + chrono::Duration::days(14),
984            status: AssignmentStatus::Pending,
985            assignment_method: AssignmentMethod::Manual,
986            expertise_match: 0.8,
987        });
988
989        let workload = system.calculate_reviewer_workload("reviewer1");
990        assert_eq!(workload, 1);
991
992        let workload = system.calculate_reviewer_workload("reviewer2");
993        assert_eq!(workload, 0);
994    }
995
996    fn make_reviewer(id: &str) -> Reviewer {
997        Reviewer {
998            id: id.to_string(),
999            expertise_areas: vec!["optimization".to_string()],
1000            experience_level: ExperienceLevel::Senior,
1001            review_history: ReviewerHistory::default(),
1002            availability: ReviewerAvailability::default(),
1003            quality_metrics: ReviewerQualityMetrics::default(),
1004            preferences: ReviewerPreferences {
1005                preferred_paper_types: Vec::new(),
1006                avoid_paper_types: Vec::new(),
1007                max_review_length: None,
1008                anonymous_preference: true,
1009                notification_preferences: NotificationPreferences::default(),
1010            },
1011        }
1012    }
1013
1014    fn make_review(reviewer_id: &str, recommendation: ReviewRecommendation) -> PeerReview {
1015        PeerReview {
1016            id: uuid::Uuid::new_v4().to_string(),
1017            reviewer_id: reviewer_id.to_string(),
1018            recommendation,
1019            criterion_scores: HashMap::new(),
1020            overall_score: 3.0,
1021            confidence: 0.5,
1022            written_review: WrittenReview {
1023                summary: String::new(),
1024                strengths: Vec::new(),
1025                weaknesses: Vec::new(),
1026                detailed_comments: String::new(),
1027                questions: Vec::new(),
1028                minor_issues: Vec::new(),
1029                suggestions: Vec::new(),
1030                committee_comments: None,
1031            },
1032            status: ReviewStatus::Submitted,
1033            submitted_at: Some(Utc::now()),
1034            time_spent_minutes: Some(30),
1035        }
1036    }
1037
1038    // Regression test for F73: `ln(0)` is `-inf`, and a review with no
1039    // written content at all (a realistic, valid input -- e.g. a
1040    // placeholder or a reviewer who only filled in scores) has
1041    // `review_length == 0`, which used to poison the entire quality score
1042    // to `-inf` instead of a valid score in `[0, 1]`.
1043    #[test]
1044    fn test_calculate_review_quality_handles_empty_review() {
1045        let system = PeerReviewSystem::new();
1046        let review = make_review("reviewer1", ReviewRecommendation::BorderlineAccept);
1047
1048        let quality = system.calculate_review_quality(&review);
1049
1050        assert!(
1051            quality.is_finite(),
1052            "quality score must be finite, got {quality}"
1053        );
1054        assert!(
1055            (0.0..=1.0).contains(&quality),
1056            "quality score must be in [0, 1], got {quality}"
1057        );
1058    }
1059
1060    // Regression test for F74: consensus used to be "most frequent
1061    // recommendation, ties broken by HashMap iteration order" -- so a tied
1062    // vote could yield a different result on different runs for the exact
1063    // same input. It must now be the deterministic median, independent of
1064    // the order recommendations are supplied in.
1065    #[test]
1066    fn test_determine_consensus_recommendation_is_deterministic_median() {
1067        let system = PeerReviewSystem::new();
1068
1069        // Tied 1-1 vote between Accept and Reject: the median of ranks
1070        // [1, 6] is rank 4 (BorderlineReject), not an order-dependent pick
1071        // of either tied recommendation.
1072        let accept = ReviewRecommendation::Accept;
1073        let reject = ReviewRecommendation::Reject;
1074        let order_a = system.determine_consensus_recommendation(&[&accept, &reject]);
1075        let order_b = system.determine_consensus_recommendation(&[&reject, &accept]);
1076        assert_eq!(order_a, ReviewRecommendation::BorderlineReject);
1077        assert_eq!(order_a, order_b, "consensus must not depend on input order");
1078
1079        // A single outlier must not dominate the median the way it would a
1080        // naive average: [StrongAccept, Accept, Reject] medians to Accept.
1081        let strong_accept = ReviewRecommendation::StrongAccept;
1082        let median = system.determine_consensus_recommendation(&[&reject, &strong_accept, &accept]);
1083        assert_eq!(median, ReviewRecommendation::Accept);
1084    }
1085
1086    // Regression test for F75: `assign_reviewers` never updated
1087    // `reviewer.availability.current_load`, so the load-based capacity
1088    // check in `get_available_reviewers` never actually reflected real
1089    // assignments; and neither `assign_reviewers` nor `submit_review`
1090    // guarded against the same reviewer being attached to / submitting for
1091    // one session twice.
1092    #[test]
1093    fn test_assign_reviewers_tracks_load_and_rejects_duplicates() {
1094        let mut system = PeerReviewSystem::new();
1095        system
1096            .reviewers
1097            .insert("reviewer1".to_string(), make_reviewer("reviewer1"));
1098
1099        let deadline = Utc::now() + chrono::Duration::days(14);
1100        let session_id =
1101            system.create_review_session("paper1", ReviewType::DoubleBlind, vec![], deadline);
1102
1103        // Duplicate ID within a single call must only produce one assignment.
1104        let ids = system
1105            .assign_reviewers(
1106                &session_id,
1107                &["reviewer1".to_string(), "reviewer1".to_string()],
1108                AssignmentMethod::Manual,
1109            )
1110            .expect("assignment should succeed");
1111        assert_eq!(ids.len(), 1);
1112        assert_eq!(system.reviewers["reviewer1"].availability.current_load, 1);
1113
1114        // A second call for the same (session, reviewer) must be a no-op.
1115        let ids_again = system
1116            .assign_reviewers(
1117                &session_id,
1118                &["reviewer1".to_string()],
1119                AssignmentMethod::Manual,
1120            )
1121            .expect("assignment should succeed");
1122        assert!(ids_again.is_empty());
1123        assert_eq!(system.reviewers["reviewer1"].availability.current_load, 1);
1124        assert_eq!(system.calculate_reviewer_workload("reviewer1"), 1);
1125
1126        // Submitting frees up the reviewer's active load...
1127        let review = make_review("reviewer1", ReviewRecommendation::Accept);
1128        system
1129            .submit_review(&session_id, "reviewer1", review)
1130            .expect("first submission should succeed");
1131        assert_eq!(system.reviewers["reviewer1"].availability.current_load, 0);
1132
1133        // ...but a second submission for the same session must be rejected.
1134        let duplicate_review = make_review("reviewer1", ReviewRecommendation::Reject);
1135        let result = system.submit_review(&session_id, "reviewer1", duplicate_review);
1136        assert!(result.is_err(), "duplicate submission must be rejected");
1137        assert_eq!(
1138            system.sessions[&session_id].reviews.len(),
1139            1,
1140            "duplicate submission must not be recorded"
1141        );
1142    }
1143}
optirs_core/research/peer_review.rs

optirs_core/research/
peer_review.rs
