Skip to main content

rusty_eval/
feedback.rs

1//! Deterministic human-feedback operations for evaluation and learning loops.
2//!
3//! The queue deliberately owns no clock, database, or worker runtime. Callers
4//! supply timestamps, persist the serializable queue, and decide who may act as
5//! a reviewer or adjudicator. That keeps replay and tests deterministic while
6//! still providing the state-machine guarantees needed by a distributed review
7//! service: leases expire, reviewers cannot vote twice, disagreements require
8//! adjudication, and only resolved feedback can become an evaluation case.
9
10use std::collections::{BTreeMap, BTreeSet};
11
12use serde::{Deserialize, Serialize};
13use serde_json::Value;
14
15use crate::dataset::{EvalCase, Expectation, StatePredicate};
16use crate::error::{EvalError, Result};
17
18/// The feedback queue format version this build loads and writes.
19pub const FEEDBACK_FORMAT_VERSION: u64 = 1;
20
21/// A stable reference to the recorded run that produced an annotation task.
22#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
23pub struct TraceRef {
24    /// Stable run identifier in the Flight Recorder.
25    pub run_id: String,
26    /// Optional conversation or thread identifier.
27    #[serde(default, skip_serializing_if = "Option::is_none")]
28    pub thread_id: Option<String>,
29    /// Optional event sequence at which the candidate output was captured.
30    #[serde(default, skip_serializing_if = "Option::is_none")]
31    pub event_seq: Option<u64>,
32}
33
34impl TraceRef {
35    /// Reference a run without a narrower thread or event location.
36    pub fn run(run_id: impl Into<String>) -> Self {
37        Self {
38            run_id: run_id.into(),
39            thread_id: None,
40            event_seq: None,
41        }
42    }
43}
44
45/// One candidate output presented to a reviewer.
46#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
47pub struct ReviewCandidate {
48    /// Stable identifier within the task.
49    pub id: String,
50    /// Arbitrary structured output shown to the reviewer.
51    pub output: Value,
52}
53
54/// One scored dimension in a review rubric.
55#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
56pub struct RubricCriterion {
57    /// Stable criterion identifier used as the score-map key.
58    pub id: String,
59    /// Human-readable guidance for the reviewer.
60    pub description: String,
61}
62
63/// A versioned rubric attached to an annotation task.
64#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
65pub struct ReviewRubric {
66    /// Stable rubric name.
67    pub name: String,
68    /// Caller-managed rubric version.
69    pub version: String,
70    /// Required criteria. Every submitted review must score every criterion.
71    pub criteria: Vec<RubricCriterion>,
72}
73
74/// The actionable result of a review.
75#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
76#[serde(tag = "kind", rename_all = "snake_case")]
77pub enum ReviewDecision {
78    /// Prefer one of the task's candidates.
79    Select { candidate_id: String },
80    /// Supply a corrected output when neither candidate is acceptable.
81    Correct { output: Value },
82    /// Reject the example so it cannot be promoted into a dataset.
83    Reject {
84        #[serde(default, skip_serializing_if = "Option::is_none")]
85        reason: Option<String>,
86    },
87}
88
89/// A review submitted while holding a live task lease.
90#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
91pub struct ReviewSubmission {
92    /// Selection, correction, or rejection.
93    pub decision: ReviewDecision,
94    /// Criterion id to normalized score in the inclusive range `[0, 1]`.
95    pub scores: BTreeMap<String, f64>,
96    /// Optional reviewer rationale.
97    #[serde(default, skip_serializing_if = "Option::is_none")]
98    pub comment: Option<String>,
99}
100
101/// An immutable accepted review with provenance.
102#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
103pub struct StoredReview {
104    /// Reviewer identity supplied by the caller.
105    pub reviewer: String,
106    /// Caller-supplied submission timestamp.
107    pub submitted_at_ms: u64,
108    /// The validated submission.
109    pub submission: ReviewSubmission,
110}
111
112/// How a final decision was reached.
113#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
114#[serde(tag = "kind", rename_all = "snake_case")]
115pub enum ResolutionAuthority {
116    /// All required reviewers independently made the same decision.
117    Consensus,
118    /// A named adjudicator resolved disagreement.
119    Adjudicator { id: String },
120}
121
122/// The final, promotion-eligible result of an annotation task.
123#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
124pub struct ReviewResolution {
125    /// Final decision.
126    pub decision: ReviewDecision,
127    /// Consensus or explicit adjudication.
128    pub authority: ResolutionAuthority,
129    /// Caller-supplied resolution timestamp.
130    pub resolved_at_ms: u64,
131}
132
133/// Current task state at a caller-supplied point in time.
134#[derive(Debug, Clone, Copy, PartialEq, Eq)]
135pub enum AnnotationStatus {
136    /// No accepted reviews or live leases.
137    Open,
138    /// At least one review or live lease exists; more reviews are required.
139    InReview,
140    /// The required reviews disagree and an adjudicator must decide.
141    NeedsAdjudication,
142    /// A consensus or adjudicated resolution exists.
143    Resolved,
144}
145
146/// Proof that a reviewer currently owns a task reservation.
147#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
148pub struct ReviewLease {
149    /// Claimed annotation task.
150    pub task_id: String,
151    /// Reviewer holding the lease.
152    pub reviewer: String,
153    /// Monotonic occurrence id that prevents a stale worker from using a
154    /// later lease issued to the same reviewer.
155    pub lease_id: u64,
156    /// Caller-supplied time at which this occurrence was issued.
157    pub issued_at_ms: u64,
158    /// Exclusive lease expiry; a lease is expired when `now_ms >=` this value.
159    pub expires_at_ms: u64,
160}
161
162#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
163struct LeaseRecord {
164    lease_id: u64,
165    issued_at_ms: u64,
166    expires_at_ms: u64,
167}
168
169/// One pairwise annotation task and its review state.
170#[derive(Debug, Clone, PartialEq, Serialize)]
171pub struct AnnotationTask {
172    id: String,
173    source: TraceRef,
174    input: Value,
175    candidates: Vec<ReviewCandidate>,
176    rubric: ReviewRubric,
177    priority: i32,
178    created_at_ms: u64,
179    required_reviews: usize,
180    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
181    leases: BTreeMap<String, LeaseRecord>,
182    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
183    reviews: BTreeMap<String, StoredReview>,
184    #[serde(default, skip_serializing_if = "Option::is_none")]
185    resolution: Option<ReviewResolution>,
186}
187
188impl AnnotationTask {
189    /// Create a pairwise annotation task.
190    #[allow(clippy::too_many_arguments)]
191    pub fn new(
192        id: impl Into<String>,
193        source: TraceRef,
194        input: Value,
195        candidates: Vec<ReviewCandidate>,
196        rubric: ReviewRubric,
197        priority: i32,
198        created_at_ms: u64,
199        required_reviews: usize,
200    ) -> Result<Self> {
201        let task = Self {
202            id: id.into(),
203            source,
204            input,
205            candidates,
206            rubric,
207            priority,
208            created_at_ms,
209            required_reviews,
210            leases: BTreeMap::new(),
211            reviews: BTreeMap::new(),
212            resolution: None,
213        };
214        task.validate_definition()?;
215        Ok(task)
216    }
217
218    /// Stable task identifier.
219    pub fn id(&self) -> &str {
220        &self.id
221    }
222
223    /// Recorded-run provenance.
224    pub fn source(&self) -> &TraceRef {
225        &self.source
226    }
227
228    /// Input that produced the candidates.
229    pub fn input(&self) -> &Value {
230        &self.input
231    }
232
233    /// The two candidates under comparison.
234    pub fn candidates(&self) -> &[ReviewCandidate] {
235        &self.candidates
236    }
237
238    /// Versioned scoring rubric.
239    pub fn rubric(&self) -> &ReviewRubric {
240        &self.rubric
241    }
242
243    /// Scheduling priority; larger values are claimed first.
244    pub fn priority(&self) -> i32 {
245        self.priority
246    }
247
248    /// Task creation timestamp.
249    pub fn created_at_ms(&self) -> u64 {
250        self.created_at_ms
251    }
252
253    /// Number of independent reviews required before resolution.
254    pub fn required_reviews(&self) -> usize {
255        self.required_reviews
256    }
257
258    /// Accepted reviews, keyed by reviewer identity.
259    pub fn reviews(&self) -> &BTreeMap<String, StoredReview> {
260        &self.reviews
261    }
262
263    /// Final resolution, when one exists.
264    pub fn resolution(&self) -> Option<&ReviewResolution> {
265        self.resolution.as_ref()
266    }
267
268    /// Compute status without mutating or consulting a wall clock.
269    pub fn status_at(&self, now_ms: u64) -> AnnotationStatus {
270        if self.resolution.is_some() {
271            return AnnotationStatus::Resolved;
272        }
273        if self.reviews.len() >= self.required_reviews {
274            return AnnotationStatus::NeedsAdjudication;
275        }
276        if !self.reviews.is_empty()
277            || self
278                .leases
279                .values()
280                .any(|lease| lease.expires_at_ms > now_ms)
281        {
282            AnnotationStatus::InReview
283        } else {
284            AnnotationStatus::Open
285        }
286    }
287
288    /// Convert a resolved selection or correction into a versioned dataset case.
289    pub fn promote(
290        &self,
291        case_id: impl Into<String>,
292        output_pointer: impl Into<String>,
293    ) -> Result<EvalCase> {
294        let case_id = case_id.into();
295        require_non_empty("promoted case id", &case_id)?;
296        let output_pointer = output_pointer.into();
297        validate_json_pointer(&output_pointer)?;
298        let resolution = self
299            .resolution
300            .as_ref()
301            .ok_or_else(|| EvalError::Feedback(format!("task `{}` is not resolved", self.id)))?;
302        let expected = match &resolution.decision {
303            ReviewDecision::Select { candidate_id } => self
304                .candidates
305                .iter()
306                .find(|candidate| candidate.id == *candidate_id)
307                .map(|candidate| candidate.output.clone())
308                .ok_or_else(|| {
309                    EvalError::Feedback(format!(
310                        "task `{}` resolution selects unknown candidate `{candidate_id}`",
311                        self.id
312                    ))
313                })?,
314            ReviewDecision::Correct { output } => output.clone(),
315            ReviewDecision::Reject { .. } => {
316                return Err(EvalError::Feedback(format!(
317                    "task `{}` was rejected and cannot be promoted",
318                    self.id
319                )));
320            }
321        };
322        let authority = match &resolution.authority {
323            ResolutionAuthority::Consensus => "consensus".to_owned(),
324            ResolutionAuthority::Adjudicator { id } => format!("adjudicated:{id}"),
325        };
326
327        Ok(EvalCase {
328            id: case_id,
329            input: self.input.clone(),
330            expect: Expectation {
331                state: vec![StatePredicate {
332                    pointer: output_pointer,
333                    expected,
334                }],
335                ..Expectation::default()
336            },
337            tags: vec![
338                "human-feedback".to_owned(),
339                format!("feedback-task:{}", self.id),
340                format!("source-run:{}", self.source.run_id),
341                authority,
342            ],
343        })
344    }
345
346    fn validate_definition(&self) -> Result<()> {
347        require_non_empty("task id", &self.id)?;
348        require_non_empty("source run id", &self.source.run_id)?;
349        if let Some(thread_id) = &self.source.thread_id {
350            require_non_empty("source thread id", thread_id)?;
351        }
352        if self.candidates.len() != 2 {
353            return feedback_error(format!(
354                "task `{}` must contain exactly two candidates",
355                self.id
356            ));
357        }
358        let mut candidate_ids = BTreeSet::new();
359        for candidate in &self.candidates {
360            require_non_empty("candidate id", &candidate.id)?;
361            if !candidate_ids.insert(candidate.id.as_str()) {
362                return feedback_error(format!(
363                    "task `{}` has duplicate candidate id `{}`",
364                    self.id, candidate.id
365                ));
366            }
367        }
368        require_non_empty("rubric name", &self.rubric.name)?;
369        require_non_empty("rubric version", &self.rubric.version)?;
370        if self.rubric.criteria.is_empty() {
371            return feedback_error(format!(
372                "task `{}` rubric must contain at least one criterion",
373                self.id
374            ));
375        }
376        let mut criterion_ids = BTreeSet::new();
377        for criterion in &self.rubric.criteria {
378            require_non_empty("rubric criterion id", &criterion.id)?;
379            require_non_empty("rubric criterion description", &criterion.description)?;
380            if !criterion_ids.insert(criterion.id.as_str()) {
381                return feedback_error(format!(
382                    "task `{}` has duplicate rubric criterion `{}`",
383                    self.id, criterion.id
384                ));
385            }
386        }
387        if self.required_reviews == 0 {
388            return feedback_error(format!(
389                "task `{}` must require at least one review",
390                self.id
391            ));
392        }
393        Ok(())
394    }
395
396    fn validate_submission(&self, submission: &ReviewSubmission) -> Result<()> {
397        self.validate_decision(&submission.decision)?;
398        let expected: BTreeSet<_> = self
399            .rubric
400            .criteria
401            .iter()
402            .map(|criterion| criterion.id.as_str())
403            .collect();
404        let actual: BTreeSet<_> = submission.scores.keys().map(String::as_str).collect();
405        if actual != expected {
406            return feedback_error(format!(
407                "task `{}` review scores must exactly match rubric criteria",
408                self.id
409            ));
410        }
411        for (criterion, score) in &submission.scores {
412            if !score.is_finite() || !(0.0..=1.0).contains(score) {
413                return feedback_error(format!(
414                    "task `{}` score for `{criterion}` must be finite and between 0 and 1",
415                    self.id
416                ));
417            }
418        }
419        Ok(())
420    }
421
422    fn validate_decision(&self, decision: &ReviewDecision) -> Result<()> {
423        if let ReviewDecision::Select { candidate_id } = decision {
424            if !self
425                .candidates
426                .iter()
427                .any(|candidate| candidate.id == *candidate_id)
428            {
429                return feedback_error(format!(
430                    "task `{}` has no candidate `{candidate_id}`",
431                    self.id
432                ));
433            }
434        }
435        Ok(())
436    }
437
438    fn validate_state(&self) -> Result<()> {
439        self.validate_definition()?;
440        if self.reviews.len() > self.required_reviews {
441            return feedback_error(format!(
442                "task `{}` contains more reviews than required",
443                self.id
444            ));
445        }
446        for (reviewer, stored) in &self.reviews {
447            require_non_empty("reviewer id", reviewer)?;
448            if stored.reviewer != *reviewer {
449                return feedback_error(format!(
450                    "task `{}` review key does not match reviewer",
451                    self.id
452                ));
453            }
454            if self.leases.contains_key(reviewer) {
455                return feedback_error(format!(
456                    "task `{}` reviewer `{reviewer}` has both a lease and review",
457                    self.id
458                ));
459            }
460            if stored.submitted_at_ms < self.created_at_ms {
461                return feedback_error(format!(
462                    "task `{}` review by `{reviewer}` predates task creation",
463                    self.id
464                ));
465            }
466            self.validate_submission(&stored.submission)?;
467        }
468        for (reviewer, lease) in &self.leases {
469            require_non_empty("reviewer id", reviewer)?;
470            if lease.issued_at_ms < self.created_at_ms {
471                return feedback_error(format!(
472                    "task `{}` lease for `{reviewer}` predates task creation",
473                    self.id
474                ));
475            }
476            if lease.expires_at_ms <= lease.issued_at_ms {
477                return feedback_error(format!(
478                    "task `{}` lease for `{reviewer}` has an invalid time range",
479                    self.id
480                ));
481            }
482        }
483        if let Some(resolution) = &self.resolution {
484            if self.reviews.len() < self.required_reviews {
485                return feedback_error(format!(
486                    "task `{}` resolved before collecting required reviews",
487                    self.id
488                ));
489            }
490            if self
491                .reviews
492                .values()
493                .any(|review| review.submitted_at_ms > resolution.resolved_at_ms)
494            {
495                return feedback_error(format!(
496                    "task `{}` resolution predates a submitted review",
497                    self.id
498                ));
499            }
500            if !self.leases.is_empty() {
501                return feedback_error(format!(
502                    "task `{}` is resolved but still contains reviewer leases",
503                    self.id
504                ));
505            }
506            self.validate_decision(&resolution.decision)?;
507            let mut review_decisions = self
508                .reviews
509                .values()
510                .map(|review| &review.submission.decision);
511            let first_review_decision = review_decisions.next().expect("required reviews exist");
512            let unanimous = review_decisions.all(|decision| decision == first_review_decision);
513            match &resolution.authority {
514                ResolutionAuthority::Consensus => {
515                    if !unanimous || *first_review_decision != resolution.decision {
516                        return feedback_error(format!(
517                            "task `{}` consensus resolution does not match every review",
518                            self.id
519                        ));
520                    }
521                }
522                ResolutionAuthority::Adjudicator { id } => {
523                    require_non_empty("adjudicator id", id)?;
524                    if unanimous {
525                        return feedback_error(format!(
526                            "task `{}` uses adjudication despite unanimous reviews",
527                            self.id
528                        ));
529                    }
530                }
531            }
532        } else {
533            if self.reviews.len() + self.leases.len() > self.required_reviews {
534                return feedback_error(format!(
535                    "task `{}` has more reviews and leases than required",
536                    self.id
537                ));
538            }
539            if self.reviews.len() == self.required_reviews {
540                let mut decisions = self
541                    .reviews
542                    .values()
543                    .map(|review| &review.submission.decision);
544                if let Some(first) = decisions.next() {
545                    if decisions.all(|decision| decision == first) {
546                        return feedback_error(format!(
547                            "task `{}` has review consensus but no resolution",
548                            self.id
549                        ));
550                    }
551                }
552            }
553        }
554        Ok(())
555    }
556}
557
558/// A named, versioned collection of annotation tasks.
559#[derive(Debug, Clone, PartialEq, Serialize)]
560pub struct AnnotationQueue {
561    format_version: u64,
562    name: String,
563    tasks: BTreeMap<String, AnnotationTask>,
564    next_lease_id: u64,
565}
566
567impl AnnotationQueue {
568    /// Create an empty queue.
569    pub fn new(name: impl Into<String>) -> Result<Self> {
570        let name = name.into();
571        require_non_empty("queue name", &name)?;
572        Ok(Self {
573            format_version: FEEDBACK_FORMAT_VERSION,
574            name,
575            tasks: BTreeMap::new(),
576            next_lease_id: 1,
577        })
578    }
579
580    /// Queue name.
581    pub fn name(&self) -> &str {
582        &self.name
583    }
584
585    /// All tasks in stable id order.
586    pub fn tasks(&self) -> &BTreeMap<String, AnnotationTask> {
587        &self.tasks
588    }
589
590    /// Find a task by id.
591    pub fn task(&self, task_id: &str) -> Option<&AnnotationTask> {
592        self.tasks.get(task_id)
593    }
594
595    /// Add a validated task without overwriting an existing id.
596    pub fn enqueue(&mut self, task: AnnotationTask) -> Result<()> {
597        task.validate_state()?;
598        if self.tasks.contains_key(task.id()) {
599            return feedback_error(format!("duplicate annotation task `{}`", task.id()));
600        }
601        self.tasks.insert(task.id.clone(), task);
602        Ok(())
603    }
604
605    /// Claim the highest-priority eligible task.
606    ///
607    /// Ordering is deterministic: priority descending, creation time
608    /// ascending, then task id ascending. Calling claim again while the same
609    /// reviewer owns a live lease returns that lease idempotently.
610    pub fn claim(
611        &mut self,
612        reviewer: impl Into<String>,
613        now_ms: u64,
614        lease_ms: u64,
615    ) -> Result<Option<ReviewLease>> {
616        let reviewer = reviewer.into();
617        require_non_empty("reviewer id", &reviewer)?;
618        if lease_ms == 0 {
619            return feedback_error("lease duration must be greater than zero");
620        }
621        self.prune_expired_leases(now_ms);
622        if let Some((task_id, lease)) = self.tasks.iter().find_map(|(task_id, task)| {
623            task.leases
624                .get(&reviewer)
625                .map(|lease| (task_id.clone(), lease.clone()))
626        }) {
627            return Ok(Some(ReviewLease {
628                task_id,
629                reviewer,
630                lease_id: lease.lease_id,
631                issued_at_ms: lease.issued_at_ms,
632                expires_at_ms: lease.expires_at_ms,
633            }));
634        }
635
636        let task_id = self
637            .tasks
638            .values()
639            .filter(|task| {
640                task.created_at_ms <= now_ms
641                    && task.resolution.is_none()
642                    && !task.reviews.contains_key(&reviewer)
643                    && task.reviews.len() < task.required_reviews
644                    && task.reviews.len() + task.leases.len() < task.required_reviews
645            })
646            .min_by(|left, right| {
647                right
648                    .priority
649                    .cmp(&left.priority)
650                    .then_with(|| left.created_at_ms.cmp(&right.created_at_ms))
651                    .then_with(|| left.id.cmp(&right.id))
652            })
653            .map(|task| task.id.clone());
654
655        let Some(task_id) = task_id else {
656            return Ok(None);
657        };
658        let expires_at_ms = now_ms.checked_add(lease_ms).ok_or_else(|| {
659            EvalError::Feedback("lease expiry overflows u64 milliseconds".to_owned())
660        })?;
661        let lease_id = self.next_lease_id;
662        self.next_lease_id = self.next_lease_id.checked_add(1).ok_or_else(|| {
663            EvalError::Feedback("lease occurrence id space is exhausted".to_owned())
664        })?;
665        self.tasks
666            .get_mut(&task_id)
667            .expect("selected task remains present")
668            .leases
669            .insert(
670                reviewer.clone(),
671                LeaseRecord {
672                    lease_id,
673                    issued_at_ms: now_ms,
674                    expires_at_ms,
675                },
676            );
677        Ok(Some(ReviewLease {
678            task_id,
679            reviewer,
680            lease_id,
681            issued_at_ms: now_ms,
682            expires_at_ms,
683        }))
684    }
685
686    /// Submit a review under a live lease.
687    pub fn submit(
688        &mut self,
689        lease: &ReviewLease,
690        now_ms: u64,
691        submission: ReviewSubmission,
692    ) -> Result<AnnotationStatus> {
693        require_non_empty("reviewer id", &lease.reviewer)?;
694        let task_id = &lease.task_id;
695        let reviewer = &lease.reviewer;
696        let task = self
697            .tasks
698            .get_mut(task_id)
699            .ok_or_else(|| EvalError::Feedback(format!("unknown annotation task `{task_id}`")))?;
700        if task.resolution.is_some() {
701            return feedback_error(format!("task `{task_id}` is already resolved"));
702        }
703        if task.reviews.contains_key(reviewer) {
704            return feedback_error(format!(
705                "reviewer `{reviewer}` already reviewed task `{task_id}`"
706            ));
707        }
708        if task.reviews.len() >= task.required_reviews {
709            return feedback_error(format!(
710                "task `{task_id}` already collected all required reviews"
711            ));
712        }
713        let active = task.leases.get(reviewer).cloned().ok_or_else(|| {
714            EvalError::Feedback(format!(
715                "reviewer `{reviewer}` does not hold a lease for task `{task_id}`"
716            ))
717        })?;
718        if active.lease_id != lease.lease_id
719            || active.issued_at_ms != lease.issued_at_ms
720            || active.expires_at_ms != lease.expires_at_ms
721        {
722            return feedback_error(format!(
723                "lease {} for task `{task_id}` is stale",
724                lease.lease_id
725            ));
726        }
727        if now_ms >= active.expires_at_ms {
728            task.leases.remove(reviewer);
729            return feedback_error(format!(
730                "reviewer `{reviewer}` lease for task `{task_id}` expired"
731            ));
732        }
733        if now_ms < active.issued_at_ms || now_ms < task.created_at_ms {
734            return feedback_error(format!(
735                "review timestamp for task `{task_id}` predates its lease or task creation"
736            ));
737        }
738        if task
739            .reviews
740            .values()
741            .any(|review| review.submitted_at_ms > now_ms)
742        {
743            return feedback_error(format!(
744                "review timestamp for task `{task_id}` predates an accepted review"
745            ));
746        }
747        task.validate_submission(&submission)?;
748        task.leases.remove(reviewer);
749        task.reviews.insert(
750            reviewer.to_owned(),
751            StoredReview {
752                reviewer: reviewer.to_owned(),
753                submitted_at_ms: now_ms,
754                submission,
755            },
756        );
757
758        if task.reviews.len() == task.required_reviews {
759            let mut decisions = task
760                .reviews
761                .values()
762                .map(|review| &review.submission.decision);
763            let first = decisions.next().expect("at least one review").clone();
764            if decisions.all(|decision| *decision == first) {
765                task.resolution = Some(ReviewResolution {
766                    decision: first,
767                    authority: ResolutionAuthority::Consensus,
768                    resolved_at_ms: now_ms,
769                });
770                task.leases.clear();
771            }
772        }
773        Ok(task.status_at(now_ms))
774    }
775
776    /// Resolve a disagreement after all required reviews have arrived.
777    pub fn adjudicate(
778        &mut self,
779        task_id: &str,
780        adjudicator: impl Into<String>,
781        now_ms: u64,
782        decision: ReviewDecision,
783    ) -> Result<()> {
784        let adjudicator = adjudicator.into();
785        require_non_empty("adjudicator id", &adjudicator)?;
786        let task = self
787            .tasks
788            .get_mut(task_id)
789            .ok_or_else(|| EvalError::Feedback(format!("unknown annotation task `{task_id}`")))?;
790        if task.resolution.is_some() {
791            return feedback_error(format!("task `{task_id}` is already resolved"));
792        }
793        if task.reviews.len() < task.required_reviews {
794            return feedback_error(format!(
795                "task `{task_id}` has not collected all required reviews"
796            ));
797        }
798        if task
799            .reviews
800            .values()
801            .any(|review| review.submitted_at_ms > now_ms)
802        {
803            return feedback_error(format!(
804                "adjudication for task `{task_id}` predates a submitted review"
805            ));
806        }
807        task.validate_decision(&decision)?;
808        task.resolution = Some(ReviewResolution {
809            decision,
810            authority: ResolutionAuthority::Adjudicator { id: adjudicator },
811            resolved_at_ms: now_ms,
812        });
813        task.leases.clear();
814        Ok(())
815    }
816
817    /// Serialize the queue with stable map ordering.
818    pub fn to_json(&self) -> Result<String> {
819        Ok(serde_json::to_string_pretty(self)?)
820    }
821
822    /// Load a queue, rejecting unsupported versions and invalid state.
823    pub fn from_json(text: &str) -> Result<Self> {
824        let header: FeedbackVersionHeader = serde_json::from_str(text)?;
825        if header.format_version != FEEDBACK_FORMAT_VERSION {
826            return Err(EvalError::UnsupportedFeedbackVersion {
827                found: header.format_version,
828                supported: FEEDBACK_FORMAT_VERSION,
829            });
830        }
831        let wire: AnnotationQueueWire = serde_json::from_str(text)?;
832        let queue = wire.into_queue()?;
833        require_non_empty("queue name", &queue.name)?;
834        let mut lease_ids = BTreeSet::new();
835        let mut leased_reviewers = BTreeSet::new();
836        for (task_id, task) in &queue.tasks {
837            if task_id != task.id() {
838                return feedback_error(format!(
839                    "annotation task map key `{task_id}` does not match task id `{}`",
840                    task.id()
841                ));
842            }
843            task.validate_state()?;
844            for lease in task.leases.values() {
845                if lease.lease_id == 0 || lease.lease_id >= queue.next_lease_id {
846                    return feedback_error(format!(
847                        "task `{task_id}` contains invalid lease occurrence {}",
848                        lease.lease_id
849                    ));
850                }
851                if !lease_ids.insert(lease.lease_id) {
852                    return feedback_error(format!(
853                        "duplicate lease occurrence {}",
854                        lease.lease_id
855                    ));
856                }
857            }
858            for reviewer in task.leases.keys() {
859                if !leased_reviewers.insert(reviewer.as_str()) {
860                    return feedback_error(format!(
861                        "reviewer `{reviewer}` holds leases for multiple tasks"
862                    ));
863                }
864            }
865        }
866        Ok(queue)
867    }
868
869    fn prune_expired_leases(&mut self, now_ms: u64) {
870        for task in self.tasks.values_mut() {
871            task.leases.retain(|_, lease| lease.expires_at_ms > now_ms);
872        }
873    }
874}
875
876#[derive(Deserialize)]
877struct FeedbackVersionHeader {
878    format_version: u64,
879}
880
881#[derive(Deserialize)]
882struct AnnotationTaskWire {
883    id: String,
884    source: TraceRef,
885    input: Value,
886    candidates: Vec<ReviewCandidate>,
887    rubric: ReviewRubric,
888    priority: i32,
889    created_at_ms: u64,
890    required_reviews: usize,
891    #[serde(default)]
892    leases: BTreeMap<String, LeaseRecord>,
893    #[serde(default)]
894    reviews: BTreeMap<String, StoredReview>,
895    #[serde(default)]
896    resolution: Option<ReviewResolution>,
897}
898
899impl AnnotationTaskWire {
900    fn into_task(self) -> AnnotationTask {
901        AnnotationTask {
902            id: self.id,
903            source: self.source,
904            input: self.input,
905            candidates: self.candidates,
906            rubric: self.rubric,
907            priority: self.priority,
908            created_at_ms: self.created_at_ms,
909            required_reviews: self.required_reviews,
910            leases: self.leases,
911            reviews: self.reviews,
912            resolution: self.resolution,
913        }
914    }
915}
916
917#[derive(Deserialize)]
918struct AnnotationQueueWire {
919    format_version: u64,
920    name: String,
921    tasks: BTreeMap<String, AnnotationTaskWire>,
922    next_lease_id: u64,
923}
924
925impl AnnotationQueueWire {
926    fn into_queue(self) -> Result<AnnotationQueue> {
927        if self.next_lease_id == 0 {
928            return feedback_error("next lease occurrence id must be greater than zero");
929        }
930        Ok(AnnotationQueue {
931            format_version: self.format_version,
932            name: self.name,
933            tasks: self
934                .tasks
935                .into_iter()
936                .map(|(id, task)| (id, task.into_task()))
937                .collect(),
938            next_lease_id: self.next_lease_id,
939        })
940    }
941}
942
943fn require_non_empty(label: &str, value: &str) -> Result<()> {
944    if value.trim().is_empty() {
945        feedback_error(format!("{label} must not be empty"))
946    } else {
947        Ok(())
948    }
949}
950
951fn validate_json_pointer(pointer: &str) -> Result<()> {
952    if !pointer.is_empty() && !pointer.starts_with('/') {
953        return feedback_error("output pointer must be an RFC 6901 JSON pointer");
954    }
955    for segment in pointer.split('/').skip(1) {
956        let mut chars = segment.chars();
957        while let Some(character) = chars.next() {
958            if character == '~' && !matches!(chars.next(), Some('0' | '1')) {
959                return feedback_error("output pointer contains an invalid RFC 6901 escape");
960            }
961        }
962    }
963    Ok(())
964}
965
966fn feedback_error<T>(message: impl Into<String>) -> Result<T> {
967    Err(EvalError::Feedback(message.into()))
968}