1use std::collections::{BTreeMap, BTreeSet};
11
12use serde::{Deserialize, Serialize};
13use serde_json::Value;
14
15use crate::dataset::{EvalCase, Expectation, StatePredicate};
16use crate::error::{EvalError, Result};
17
18pub const FEEDBACK_FORMAT_VERSION: u64 = 1;
20
21#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
23pub struct TraceRef {
24 pub run_id: String,
26 #[serde(default, skip_serializing_if = "Option::is_none")]
28 pub thread_id: Option<String>,
29 #[serde(default, skip_serializing_if = "Option::is_none")]
31 pub event_seq: Option<u64>,
32}
33
34impl TraceRef {
35 pub fn run(run_id: impl Into<String>) -> Self {
37 Self {
38 run_id: run_id.into(),
39 thread_id: None,
40 event_seq: None,
41 }
42 }
43}
44
45#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
47pub struct ReviewCandidate {
48 pub id: String,
50 pub output: Value,
52}
53
54#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
56pub struct RubricCriterion {
57 pub id: String,
59 pub description: String,
61}
62
63#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
65pub struct ReviewRubric {
66 pub name: String,
68 pub version: String,
70 pub criteria: Vec<RubricCriterion>,
72}
73
74#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
76#[serde(tag = "kind", rename_all = "snake_case")]
77pub enum ReviewDecision {
78 Select { candidate_id: String },
80 Correct { output: Value },
82 Reject {
84 #[serde(default, skip_serializing_if = "Option::is_none")]
85 reason: Option<String>,
86 },
87}
88
89#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
91pub struct ReviewSubmission {
92 pub decision: ReviewDecision,
94 pub scores: BTreeMap<String, f64>,
96 #[serde(default, skip_serializing_if = "Option::is_none")]
98 pub comment: Option<String>,
99}
100
101#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
103pub struct StoredReview {
104 pub reviewer: String,
106 pub submitted_at_ms: u64,
108 pub submission: ReviewSubmission,
110}
111
112#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
114#[serde(tag = "kind", rename_all = "snake_case")]
115pub enum ResolutionAuthority {
116 Consensus,
118 Adjudicator { id: String },
120}
121
122#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
124pub struct ReviewResolution {
125 pub decision: ReviewDecision,
127 pub authority: ResolutionAuthority,
129 pub resolved_at_ms: u64,
131}
132
133#[derive(Debug, Clone, Copy, PartialEq, Eq)]
135pub enum AnnotationStatus {
136 Open,
138 InReview,
140 NeedsAdjudication,
142 Resolved,
144}
145
146#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
148pub struct ReviewLease {
149 pub task_id: String,
151 pub reviewer: String,
153 pub lease_id: u64,
156 pub issued_at_ms: u64,
158 pub expires_at_ms: u64,
160}
161
162#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
163struct LeaseRecord {
164 lease_id: u64,
165 issued_at_ms: u64,
166 expires_at_ms: u64,
167}
168
169#[derive(Debug, Clone, PartialEq, Serialize)]
171pub struct AnnotationTask {
172 id: String,
173 source: TraceRef,
174 input: Value,
175 candidates: Vec<ReviewCandidate>,
176 rubric: ReviewRubric,
177 priority: i32,
178 created_at_ms: u64,
179 required_reviews: usize,
180 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
181 leases: BTreeMap<String, LeaseRecord>,
182 #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
183 reviews: BTreeMap<String, StoredReview>,
184 #[serde(default, skip_serializing_if = "Option::is_none")]
185 resolution: Option<ReviewResolution>,
186}
187
188impl AnnotationTask {
189 #[allow(clippy::too_many_arguments)]
191 pub fn new(
192 id: impl Into<String>,
193 source: TraceRef,
194 input: Value,
195 candidates: Vec<ReviewCandidate>,
196 rubric: ReviewRubric,
197 priority: i32,
198 created_at_ms: u64,
199 required_reviews: usize,
200 ) -> Result<Self> {
201 let task = Self {
202 id: id.into(),
203 source,
204 input,
205 candidates,
206 rubric,
207 priority,
208 created_at_ms,
209 required_reviews,
210 leases: BTreeMap::new(),
211 reviews: BTreeMap::new(),
212 resolution: None,
213 };
214 task.validate_definition()?;
215 Ok(task)
216 }
217
218 pub fn id(&self) -> &str {
220 &self.id
221 }
222
223 pub fn source(&self) -> &TraceRef {
225 &self.source
226 }
227
228 pub fn input(&self) -> &Value {
230 &self.input
231 }
232
233 pub fn candidates(&self) -> &[ReviewCandidate] {
235 &self.candidates
236 }
237
238 pub fn rubric(&self) -> &ReviewRubric {
240 &self.rubric
241 }
242
243 pub fn priority(&self) -> i32 {
245 self.priority
246 }
247
248 pub fn created_at_ms(&self) -> u64 {
250 self.created_at_ms
251 }
252
253 pub fn required_reviews(&self) -> usize {
255 self.required_reviews
256 }
257
258 pub fn reviews(&self) -> &BTreeMap<String, StoredReview> {
260 &self.reviews
261 }
262
263 pub fn resolution(&self) -> Option<&ReviewResolution> {
265 self.resolution.as_ref()
266 }
267
268 pub fn status_at(&self, now_ms: u64) -> AnnotationStatus {
270 if self.resolution.is_some() {
271 return AnnotationStatus::Resolved;
272 }
273 if self.reviews.len() >= self.required_reviews {
274 return AnnotationStatus::NeedsAdjudication;
275 }
276 if !self.reviews.is_empty()
277 || self
278 .leases
279 .values()
280 .any(|lease| lease.expires_at_ms > now_ms)
281 {
282 AnnotationStatus::InReview
283 } else {
284 AnnotationStatus::Open
285 }
286 }
287
288 pub fn promote(
290 &self,
291 case_id: impl Into<String>,
292 output_pointer: impl Into<String>,
293 ) -> Result<EvalCase> {
294 let case_id = case_id.into();
295 require_non_empty("promoted case id", &case_id)?;
296 let output_pointer = output_pointer.into();
297 validate_json_pointer(&output_pointer)?;
298 let resolution = self
299 .resolution
300 .as_ref()
301 .ok_or_else(|| EvalError::Feedback(format!("task `{}` is not resolved", self.id)))?;
302 let expected = match &resolution.decision {
303 ReviewDecision::Select { candidate_id } => self
304 .candidates
305 .iter()
306 .find(|candidate| candidate.id == *candidate_id)
307 .map(|candidate| candidate.output.clone())
308 .ok_or_else(|| {
309 EvalError::Feedback(format!(
310 "task `{}` resolution selects unknown candidate `{candidate_id}`",
311 self.id
312 ))
313 })?,
314 ReviewDecision::Correct { output } => output.clone(),
315 ReviewDecision::Reject { .. } => {
316 return Err(EvalError::Feedback(format!(
317 "task `{}` was rejected and cannot be promoted",
318 self.id
319 )));
320 }
321 };
322 let authority = match &resolution.authority {
323 ResolutionAuthority::Consensus => "consensus".to_owned(),
324 ResolutionAuthority::Adjudicator { id } => format!("adjudicated:{id}"),
325 };
326
327 Ok(EvalCase {
328 id: case_id,
329 input: self.input.clone(),
330 expect: Expectation {
331 state: vec![StatePredicate {
332 pointer: output_pointer,
333 expected,
334 }],
335 ..Expectation::default()
336 },
337 tags: vec![
338 "human-feedback".to_owned(),
339 format!("feedback-task:{}", self.id),
340 format!("source-run:{}", self.source.run_id),
341 authority,
342 ],
343 })
344 }
345
346 fn validate_definition(&self) -> Result<()> {
347 require_non_empty("task id", &self.id)?;
348 require_non_empty("source run id", &self.source.run_id)?;
349 if let Some(thread_id) = &self.source.thread_id {
350 require_non_empty("source thread id", thread_id)?;
351 }
352 if self.candidates.len() != 2 {
353 return feedback_error(format!(
354 "task `{}` must contain exactly two candidates",
355 self.id
356 ));
357 }
358 let mut candidate_ids = BTreeSet::new();
359 for candidate in &self.candidates {
360 require_non_empty("candidate id", &candidate.id)?;
361 if !candidate_ids.insert(candidate.id.as_str()) {
362 return feedback_error(format!(
363 "task `{}` has duplicate candidate id `{}`",
364 self.id, candidate.id
365 ));
366 }
367 }
368 require_non_empty("rubric name", &self.rubric.name)?;
369 require_non_empty("rubric version", &self.rubric.version)?;
370 if self.rubric.criteria.is_empty() {
371 return feedback_error(format!(
372 "task `{}` rubric must contain at least one criterion",
373 self.id
374 ));
375 }
376 let mut criterion_ids = BTreeSet::new();
377 for criterion in &self.rubric.criteria {
378 require_non_empty("rubric criterion id", &criterion.id)?;
379 require_non_empty("rubric criterion description", &criterion.description)?;
380 if !criterion_ids.insert(criterion.id.as_str()) {
381 return feedback_error(format!(
382 "task `{}` has duplicate rubric criterion `{}`",
383 self.id, criterion.id
384 ));
385 }
386 }
387 if self.required_reviews == 0 {
388 return feedback_error(format!(
389 "task `{}` must require at least one review",
390 self.id
391 ));
392 }
393 Ok(())
394 }
395
396 fn validate_submission(&self, submission: &ReviewSubmission) -> Result<()> {
397 self.validate_decision(&submission.decision)?;
398 let expected: BTreeSet<_> = self
399 .rubric
400 .criteria
401 .iter()
402 .map(|criterion| criterion.id.as_str())
403 .collect();
404 let actual: BTreeSet<_> = submission.scores.keys().map(String::as_str).collect();
405 if actual != expected {
406 return feedback_error(format!(
407 "task `{}` review scores must exactly match rubric criteria",
408 self.id
409 ));
410 }
411 for (criterion, score) in &submission.scores {
412 if !score.is_finite() || !(0.0..=1.0).contains(score) {
413 return feedback_error(format!(
414 "task `{}` score for `{criterion}` must be finite and between 0 and 1",
415 self.id
416 ));
417 }
418 }
419 Ok(())
420 }
421
422 fn validate_decision(&self, decision: &ReviewDecision) -> Result<()> {
423 if let ReviewDecision::Select { candidate_id } = decision {
424 if !self
425 .candidates
426 .iter()
427 .any(|candidate| candidate.id == *candidate_id)
428 {
429 return feedback_error(format!(
430 "task `{}` has no candidate `{candidate_id}`",
431 self.id
432 ));
433 }
434 }
435 Ok(())
436 }
437
438 fn validate_state(&self) -> Result<()> {
439 self.validate_definition()?;
440 if self.reviews.len() > self.required_reviews {
441 return feedback_error(format!(
442 "task `{}` contains more reviews than required",
443 self.id
444 ));
445 }
446 for (reviewer, stored) in &self.reviews {
447 require_non_empty("reviewer id", reviewer)?;
448 if stored.reviewer != *reviewer {
449 return feedback_error(format!(
450 "task `{}` review key does not match reviewer",
451 self.id
452 ));
453 }
454 if self.leases.contains_key(reviewer) {
455 return feedback_error(format!(
456 "task `{}` reviewer `{reviewer}` has both a lease and review",
457 self.id
458 ));
459 }
460 if stored.submitted_at_ms < self.created_at_ms {
461 return feedback_error(format!(
462 "task `{}` review by `{reviewer}` predates task creation",
463 self.id
464 ));
465 }
466 self.validate_submission(&stored.submission)?;
467 }
468 for (reviewer, lease) in &self.leases {
469 require_non_empty("reviewer id", reviewer)?;
470 if lease.issued_at_ms < self.created_at_ms {
471 return feedback_error(format!(
472 "task `{}` lease for `{reviewer}` predates task creation",
473 self.id
474 ));
475 }
476 if lease.expires_at_ms <= lease.issued_at_ms {
477 return feedback_error(format!(
478 "task `{}` lease for `{reviewer}` has an invalid time range",
479 self.id
480 ));
481 }
482 }
483 if let Some(resolution) = &self.resolution {
484 if self.reviews.len() < self.required_reviews {
485 return feedback_error(format!(
486 "task `{}` resolved before collecting required reviews",
487 self.id
488 ));
489 }
490 if self
491 .reviews
492 .values()
493 .any(|review| review.submitted_at_ms > resolution.resolved_at_ms)
494 {
495 return feedback_error(format!(
496 "task `{}` resolution predates a submitted review",
497 self.id
498 ));
499 }
500 if !self.leases.is_empty() {
501 return feedback_error(format!(
502 "task `{}` is resolved but still contains reviewer leases",
503 self.id
504 ));
505 }
506 self.validate_decision(&resolution.decision)?;
507 let mut review_decisions = self
508 .reviews
509 .values()
510 .map(|review| &review.submission.decision);
511 let first_review_decision = review_decisions.next().expect("required reviews exist");
512 let unanimous = review_decisions.all(|decision| decision == first_review_decision);
513 match &resolution.authority {
514 ResolutionAuthority::Consensus => {
515 if !unanimous || *first_review_decision != resolution.decision {
516 return feedback_error(format!(
517 "task `{}` consensus resolution does not match every review",
518 self.id
519 ));
520 }
521 }
522 ResolutionAuthority::Adjudicator { id } => {
523 require_non_empty("adjudicator id", id)?;
524 if unanimous {
525 return feedback_error(format!(
526 "task `{}` uses adjudication despite unanimous reviews",
527 self.id
528 ));
529 }
530 }
531 }
532 } else {
533 if self.reviews.len() + self.leases.len() > self.required_reviews {
534 return feedback_error(format!(
535 "task `{}` has more reviews and leases than required",
536 self.id
537 ));
538 }
539 if self.reviews.len() == self.required_reviews {
540 let mut decisions = self
541 .reviews
542 .values()
543 .map(|review| &review.submission.decision);
544 if let Some(first) = decisions.next() {
545 if decisions.all(|decision| decision == first) {
546 return feedback_error(format!(
547 "task `{}` has review consensus but no resolution",
548 self.id
549 ));
550 }
551 }
552 }
553 }
554 Ok(())
555 }
556}
557
558#[derive(Debug, Clone, PartialEq, Serialize)]
560pub struct AnnotationQueue {
561 format_version: u64,
562 name: String,
563 tasks: BTreeMap<String, AnnotationTask>,
564 next_lease_id: u64,
565}
566
567impl AnnotationQueue {
568 pub fn new(name: impl Into<String>) -> Result<Self> {
570 let name = name.into();
571 require_non_empty("queue name", &name)?;
572 Ok(Self {
573 format_version: FEEDBACK_FORMAT_VERSION,
574 name,
575 tasks: BTreeMap::new(),
576 next_lease_id: 1,
577 })
578 }
579
580 pub fn name(&self) -> &str {
582 &self.name
583 }
584
585 pub fn tasks(&self) -> &BTreeMap<String, AnnotationTask> {
587 &self.tasks
588 }
589
590 pub fn task(&self, task_id: &str) -> Option<&AnnotationTask> {
592 self.tasks.get(task_id)
593 }
594
595 pub fn enqueue(&mut self, task: AnnotationTask) -> Result<()> {
597 task.validate_state()?;
598 if self.tasks.contains_key(task.id()) {
599 return feedback_error(format!("duplicate annotation task `{}`", task.id()));
600 }
601 self.tasks.insert(task.id.clone(), task);
602 Ok(())
603 }
604
605 pub fn claim(
611 &mut self,
612 reviewer: impl Into<String>,
613 now_ms: u64,
614 lease_ms: u64,
615 ) -> Result<Option<ReviewLease>> {
616 let reviewer = reviewer.into();
617 require_non_empty("reviewer id", &reviewer)?;
618 if lease_ms == 0 {
619 return feedback_error("lease duration must be greater than zero");
620 }
621 self.prune_expired_leases(now_ms);
622 if let Some((task_id, lease)) = self.tasks.iter().find_map(|(task_id, task)| {
623 task.leases
624 .get(&reviewer)
625 .map(|lease| (task_id.clone(), lease.clone()))
626 }) {
627 return Ok(Some(ReviewLease {
628 task_id,
629 reviewer,
630 lease_id: lease.lease_id,
631 issued_at_ms: lease.issued_at_ms,
632 expires_at_ms: lease.expires_at_ms,
633 }));
634 }
635
636 let task_id = self
637 .tasks
638 .values()
639 .filter(|task| {
640 task.created_at_ms <= now_ms
641 && task.resolution.is_none()
642 && !task.reviews.contains_key(&reviewer)
643 && task.reviews.len() < task.required_reviews
644 && task.reviews.len() + task.leases.len() < task.required_reviews
645 })
646 .min_by(|left, right| {
647 right
648 .priority
649 .cmp(&left.priority)
650 .then_with(|| left.created_at_ms.cmp(&right.created_at_ms))
651 .then_with(|| left.id.cmp(&right.id))
652 })
653 .map(|task| task.id.clone());
654
655 let Some(task_id) = task_id else {
656 return Ok(None);
657 };
658 let expires_at_ms = now_ms.checked_add(lease_ms).ok_or_else(|| {
659 EvalError::Feedback("lease expiry overflows u64 milliseconds".to_owned())
660 })?;
661 let lease_id = self.next_lease_id;
662 self.next_lease_id = self.next_lease_id.checked_add(1).ok_or_else(|| {
663 EvalError::Feedback("lease occurrence id space is exhausted".to_owned())
664 })?;
665 self.tasks
666 .get_mut(&task_id)
667 .expect("selected task remains present")
668 .leases
669 .insert(
670 reviewer.clone(),
671 LeaseRecord {
672 lease_id,
673 issued_at_ms: now_ms,
674 expires_at_ms,
675 },
676 );
677 Ok(Some(ReviewLease {
678 task_id,
679 reviewer,
680 lease_id,
681 issued_at_ms: now_ms,
682 expires_at_ms,
683 }))
684 }
685
686 pub fn submit(
688 &mut self,
689 lease: &ReviewLease,
690 now_ms: u64,
691 submission: ReviewSubmission,
692 ) -> Result<AnnotationStatus> {
693 require_non_empty("reviewer id", &lease.reviewer)?;
694 let task_id = &lease.task_id;
695 let reviewer = &lease.reviewer;
696 let task = self
697 .tasks
698 .get_mut(task_id)
699 .ok_or_else(|| EvalError::Feedback(format!("unknown annotation task `{task_id}`")))?;
700 if task.resolution.is_some() {
701 return feedback_error(format!("task `{task_id}` is already resolved"));
702 }
703 if task.reviews.contains_key(reviewer) {
704 return feedback_error(format!(
705 "reviewer `{reviewer}` already reviewed task `{task_id}`"
706 ));
707 }
708 if task.reviews.len() >= task.required_reviews {
709 return feedback_error(format!(
710 "task `{task_id}` already collected all required reviews"
711 ));
712 }
713 let active = task.leases.get(reviewer).cloned().ok_or_else(|| {
714 EvalError::Feedback(format!(
715 "reviewer `{reviewer}` does not hold a lease for task `{task_id}`"
716 ))
717 })?;
718 if active.lease_id != lease.lease_id
719 || active.issued_at_ms != lease.issued_at_ms
720 || active.expires_at_ms != lease.expires_at_ms
721 {
722 return feedback_error(format!(
723 "lease {} for task `{task_id}` is stale",
724 lease.lease_id
725 ));
726 }
727 if now_ms >= active.expires_at_ms {
728 task.leases.remove(reviewer);
729 return feedback_error(format!(
730 "reviewer `{reviewer}` lease for task `{task_id}` expired"
731 ));
732 }
733 if now_ms < active.issued_at_ms || now_ms < task.created_at_ms {
734 return feedback_error(format!(
735 "review timestamp for task `{task_id}` predates its lease or task creation"
736 ));
737 }
738 if task
739 .reviews
740 .values()
741 .any(|review| review.submitted_at_ms > now_ms)
742 {
743 return feedback_error(format!(
744 "review timestamp for task `{task_id}` predates an accepted review"
745 ));
746 }
747 task.validate_submission(&submission)?;
748 task.leases.remove(reviewer);
749 task.reviews.insert(
750 reviewer.to_owned(),
751 StoredReview {
752 reviewer: reviewer.to_owned(),
753 submitted_at_ms: now_ms,
754 submission,
755 },
756 );
757
758 if task.reviews.len() == task.required_reviews {
759 let mut decisions = task
760 .reviews
761 .values()
762 .map(|review| &review.submission.decision);
763 let first = decisions.next().expect("at least one review").clone();
764 if decisions.all(|decision| *decision == first) {
765 task.resolution = Some(ReviewResolution {
766 decision: first,
767 authority: ResolutionAuthority::Consensus,
768 resolved_at_ms: now_ms,
769 });
770 task.leases.clear();
771 }
772 }
773 Ok(task.status_at(now_ms))
774 }
775
776 pub fn adjudicate(
778 &mut self,
779 task_id: &str,
780 adjudicator: impl Into<String>,
781 now_ms: u64,
782 decision: ReviewDecision,
783 ) -> Result<()> {
784 let adjudicator = adjudicator.into();
785 require_non_empty("adjudicator id", &adjudicator)?;
786 let task = self
787 .tasks
788 .get_mut(task_id)
789 .ok_or_else(|| EvalError::Feedback(format!("unknown annotation task `{task_id}`")))?;
790 if task.resolution.is_some() {
791 return feedback_error(format!("task `{task_id}` is already resolved"));
792 }
793 if task.reviews.len() < task.required_reviews {
794 return feedback_error(format!(
795 "task `{task_id}` has not collected all required reviews"
796 ));
797 }
798 if task
799 .reviews
800 .values()
801 .any(|review| review.submitted_at_ms > now_ms)
802 {
803 return feedback_error(format!(
804 "adjudication for task `{task_id}` predates a submitted review"
805 ));
806 }
807 task.validate_decision(&decision)?;
808 task.resolution = Some(ReviewResolution {
809 decision,
810 authority: ResolutionAuthority::Adjudicator { id: adjudicator },
811 resolved_at_ms: now_ms,
812 });
813 task.leases.clear();
814 Ok(())
815 }
816
817 pub fn to_json(&self) -> Result<String> {
819 Ok(serde_json::to_string_pretty(self)?)
820 }
821
822 pub fn from_json(text: &str) -> Result<Self> {
824 let header: FeedbackVersionHeader = serde_json::from_str(text)?;
825 if header.format_version != FEEDBACK_FORMAT_VERSION {
826 return Err(EvalError::UnsupportedFeedbackVersion {
827 found: header.format_version,
828 supported: FEEDBACK_FORMAT_VERSION,
829 });
830 }
831 let wire: AnnotationQueueWire = serde_json::from_str(text)?;
832 let queue = wire.into_queue()?;
833 require_non_empty("queue name", &queue.name)?;
834 let mut lease_ids = BTreeSet::new();
835 let mut leased_reviewers = BTreeSet::new();
836 for (task_id, task) in &queue.tasks {
837 if task_id != task.id() {
838 return feedback_error(format!(
839 "annotation task map key `{task_id}` does not match task id `{}`",
840 task.id()
841 ));
842 }
843 task.validate_state()?;
844 for lease in task.leases.values() {
845 if lease.lease_id == 0 || lease.lease_id >= queue.next_lease_id {
846 return feedback_error(format!(
847 "task `{task_id}` contains invalid lease occurrence {}",
848 lease.lease_id
849 ));
850 }
851 if !lease_ids.insert(lease.lease_id) {
852 return feedback_error(format!(
853 "duplicate lease occurrence {}",
854 lease.lease_id
855 ));
856 }
857 }
858 for reviewer in task.leases.keys() {
859 if !leased_reviewers.insert(reviewer.as_str()) {
860 return feedback_error(format!(
861 "reviewer `{reviewer}` holds leases for multiple tasks"
862 ));
863 }
864 }
865 }
866 Ok(queue)
867 }
868
869 fn prune_expired_leases(&mut self, now_ms: u64) {
870 for task in self.tasks.values_mut() {
871 task.leases.retain(|_, lease| lease.expires_at_ms > now_ms);
872 }
873 }
874}
875
876#[derive(Deserialize)]
877struct FeedbackVersionHeader {
878 format_version: u64,
879}
880
881#[derive(Deserialize)]
882struct AnnotationTaskWire {
883 id: String,
884 source: TraceRef,
885 input: Value,
886 candidates: Vec<ReviewCandidate>,
887 rubric: ReviewRubric,
888 priority: i32,
889 created_at_ms: u64,
890 required_reviews: usize,
891 #[serde(default)]
892 leases: BTreeMap<String, LeaseRecord>,
893 #[serde(default)]
894 reviews: BTreeMap<String, StoredReview>,
895 #[serde(default)]
896 resolution: Option<ReviewResolution>,
897}
898
899impl AnnotationTaskWire {
900 fn into_task(self) -> AnnotationTask {
901 AnnotationTask {
902 id: self.id,
903 source: self.source,
904 input: self.input,
905 candidates: self.candidates,
906 rubric: self.rubric,
907 priority: self.priority,
908 created_at_ms: self.created_at_ms,
909 required_reviews: self.required_reviews,
910 leases: self.leases,
911 reviews: self.reviews,
912 resolution: self.resolution,
913 }
914 }
915}
916
917#[derive(Deserialize)]
918struct AnnotationQueueWire {
919 format_version: u64,
920 name: String,
921 tasks: BTreeMap<String, AnnotationTaskWire>,
922 next_lease_id: u64,
923}
924
925impl AnnotationQueueWire {
926 fn into_queue(self) -> Result<AnnotationQueue> {
927 if self.next_lease_id == 0 {
928 return feedback_error("next lease occurrence id must be greater than zero");
929 }
930 Ok(AnnotationQueue {
931 format_version: self.format_version,
932 name: self.name,
933 tasks: self
934 .tasks
935 .into_iter()
936 .map(|(id, task)| (id, task.into_task()))
937 .collect(),
938 next_lease_id: self.next_lease_id,
939 })
940 }
941}
942
943fn require_non_empty(label: &str, value: &str) -> Result<()> {
944 if value.trim().is_empty() {
945 feedback_error(format!("{label} must not be empty"))
946 } else {
947 Ok(())
948 }
949}
950
951fn validate_json_pointer(pointer: &str) -> Result<()> {
952 if !pointer.is_empty() && !pointer.starts_with('/') {
953 return feedback_error("output pointer must be an RFC 6901 JSON pointer");
954 }
955 for segment in pointer.split('/').skip(1) {
956 let mut chars = segment.chars();
957 while let Some(character) = chars.next() {
958 if character == '~' && !matches!(chars.next(), Some('0' | '1')) {
959 return feedback_error("output pointer contains an invalid RFC 6901 escape");
960 }
961 }
962 }
963 Ok(())
964}
965
966fn feedback_error<T>(message: impl Into<String>) -> Result<T> {
967 Err(EvalError::Feedback(message.into()))
968}