agentics_domain/models/challenge/
bundle.rs

1//! Challenge bundle contracts and public bundle projections.
2
3use serde::{Deserialize, Serialize};
4
5use crate::models::localization::LocalizedText;
6use crate::models::names::{ChallengeKeyword, ChallengeName, TargetName};
7use crate::models::paths::BundleRelativePath;
8
9use super::datasets::{DatasetsSpec, PublicDatasetsSpec};
10use super::execution::{ChallengeExecutionSpec, PublicChallengeExecutionSpec};
11use super::metrics::MetricSchemaSpec;
12use super::serde_helpers::{required_nullable, required_nullable_schema};
13use super::targets::ChallengeTargetSpec;
14
15/// Minimum public keywords that a challenge must declare.
16pub const MIN_CHALLENGE_KEYWORDS: usize = 1;
17
18/// Maximum public keywords that a challenge may declare.
19pub const MAX_CHALLENGE_KEYWORDS: usize = 6;
20
21/// Parsed `spec.json` contract for a challenge bundle.
22#[derive(Debug, Clone, Serialize, Deserialize, garde::Validate, schemars::JsonSchema)]
23#[garde(allow_unvalidated)]
24#[serde(deny_unknown_fields)]
25pub struct ChallengeBundleSpec {
26    pub schema_version: i32,
27    pub challenge_name: ChallengeName,
28    pub challenge_title: String,
29    /// Localized summary used in compact challenge catalog surfaces.
30    pub summary: LocalizedText,
31    /// Required public keywords used by catalog search and filtering.
32    #[garde(length(min = MIN_CHALLENGE_KEYWORDS, max = MAX_CHALLENGE_KEYWORDS))]
33    #[schemars(length(min = 1, max = 6))]
34    pub keywords: Vec<ChallengeKeyword>,
35    pub solution: SolutionSpec,
36    pub targets: Vec<ChallengeTargetSpec>,
37    pub starts_at: String,
38    #[serde(deserialize_with = "required_nullable")]
39    #[schemars(required, schema_with = "required_nullable_schema::<String>")]
40    pub closes_at: Option<String>,
41    pub eligibility: ChallengeEligibilitySpec,
42    #[serde(deserialize_with = "required_nullable")]
43    #[schemars(required, schema_with = "required_nullable_schema::<i64>")]
44    pub validation_submission_limit: Option<i64>,
45    #[serde(deserialize_with = "required_nullable")]
46    #[schemars(required, schema_with = "required_nullable_schema::<i64>")]
47    pub official_submission_limit: Option<i64>,
48    pub visibility: ChallengeVisibilitySpec,
49    pub solution_publication: ChallengeSolutionPublicationPolicy,
50    pub execution: ChallengeExecutionSpec,
51    pub datasets: DatasetsSpec,
52    /// Metric definitions and ranking metadata used to interpret evaluator output.
53    pub metric_schema: MetricSchemaSpec,
54}
55
56impl ChallengeBundleSpec {
57    /// Look up one target declared by this challenge.
58    pub fn target(&self, target: &TargetName) -> Option<&ChallengeTargetSpec> {
59        self.targets
60            .iter()
61            .find(|candidate| &candidate.name == target)
62    }
63
64    /// Return the only target name when a challenge is unambiguous.
65    pub fn sole_target(&self) -> Option<&TargetName> {
66        match self.targets.as_slice() {
67            [target] => Some(&target.name),
68            _ => None,
69        }
70    }
71
72    /// Return whether official runner diagnostics may contain private benchmark material.
73    pub fn official_evaluation_may_expose_private_material(&self) -> bool {
74        if self.datasets.private_benchmark_enabled || self.execution.has_official_evaluation_setup()
75        {
76            return true;
77        }
78
79        match &self.execution {
80            ChallengeExecutionSpec::SeparatedEvaluator(spec) => {
81                spec.official_runs.as_ref().is_none_or(|path| {
82                    !path
83                        .as_path()
84                        .starts_with(self.datasets.public_dir.as_path())
85                })
86            }
87            ChallengeExecutionSpec::PipedStdio(spec) => {
88                spec.official_session.as_ref().is_none_or(|path| {
89                    !path
90                        .as_path()
91                        .starts_with(self.datasets.public_dir.as_path())
92                })
93            }
94            ChallengeExecutionSpec::CoexecutedBenchmark(_) => false,
95        }
96    }
97}
98
99/// Public projection of a challenge contract safe for unauthenticated clients.
100#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
101#[serde(deny_unknown_fields)]
102pub struct PublicChallengeBundleSpec {
103    pub schema_version: i32,
104    pub challenge_name: ChallengeName,
105    pub challenge_title: String,
106    /// Localized summary used in compact challenge catalog surfaces.
107    pub summary: LocalizedText,
108    /// Required public keywords used by catalog search and filtering.
109    #[schemars(length(min = 1, max = 6))]
110    pub keywords: Vec<ChallengeKeyword>,
111    pub solution: SolutionSpec,
112    pub targets: Vec<ChallengeTargetSpec>,
113    pub starts_at: String,
114    #[serde(default, skip_serializing_if = "Option::is_none")]
115    pub closes_at: Option<String>,
116    pub eligibility: ChallengeEligibilitySpec,
117    #[serde(default, skip_serializing_if = "Option::is_none")]
118    pub validation_submission_limit: Option<i64>,
119    #[serde(default, skip_serializing_if = "Option::is_none")]
120    pub official_submission_limit: Option<i64>,
121    pub visibility: ChallengeVisibilitySpec,
122    pub solution_publication: ChallengeSolutionPublicationPolicy,
123    pub execution: PublicChallengeExecutionSpec,
124    pub datasets: PublicDatasetsSpec,
125    /// Metric definitions and ranking metadata used to interpret evaluator output.
126    #[schemars(required)]
127    pub metric_schema: MetricSchemaSpec,
128}
129
130impl PublicChallengeBundleSpec {
131    /// Look up one public target declared by this challenge.
132    pub fn target(&self, target: &TargetName) -> Option<&ChallengeTargetSpec> {
133        self.targets
134            .iter()
135            .find(|candidate| &candidate.name == target)
136    }
137
138    /// Return the only target name when a public challenge is unambiguous.
139    pub fn sole_target(&self) -> Option<&TargetName> {
140        match self.targets.as_slice() {
141            [target] => Some(&target.name),
142            _ => None,
143        }
144    }
145}
146
147impl From<ChallengeBundleSpec> for PublicChallengeBundleSpec {
148    /// Remove private benchmark locator metadata from a full challenge contract.
149    fn from(spec: ChallengeBundleSpec) -> Self {
150        Self {
151            schema_version: spec.schema_version,
152            challenge_name: spec.challenge_name,
153            challenge_title: spec.challenge_title,
154            summary: spec.summary,
155            keywords: spec.keywords,
156            solution: spec.solution,
157            targets: spec.targets,
158            starts_at: spec.starts_at,
159            closes_at: spec.closes_at,
160            eligibility: spec.eligibility,
161            validation_submission_limit: spec.validation_submission_limit,
162            official_submission_limit: spec.official_submission_limit,
163            visibility: spec.visibility,
164            solution_publication: spec.solution_publication,
165            execution: spec.execution.into(),
166            datasets: PublicDatasetsSpec {
167                public_dir: spec.datasets.public_dir,
168                public_policy: spec.datasets.public_policy,
169                private_benchmark_policy: spec.datasets.private_benchmark_policy,
170                private_benchmark_enabled: spec.datasets.private_benchmark_enabled,
171            },
172            metric_schema: spec.metric_schema,
173        }
174    }
175}
176
177/// Eligibility policy for a challenge.
178#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
179#[serde(deny_unknown_fields)]
180pub struct ChallengeEligibilitySpec {
181    #[serde(rename = "type")]
182    pub eligibility_type: ChallengeEligibilityType,
183}
184
185/// Stable eligibility policy names.
186#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
187#[serde(rename_all = "snake_case")]
188pub enum ChallengeEligibilityType {
189    Open,
190    PrivateShortlist,
191}
192
193/// Visibility policy for challenge result surfaces.
194#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
195#[serde(deny_unknown_fields)]
196pub struct ChallengeVisibilitySpec {
197    pub leaderboard: ChallengeVisibility,
198    pub score_distribution: ChallengeVisibility,
199    pub result_detail: ChallengeResultDetailVisibility,
200}
201
202/// Visibility for public aggregate surfaces.
203#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
204#[serde(rename_all = "snake_case")]
205pub enum ChallengeVisibility {
206    PublicLive,
207    PublicAfterClose,
208    Hidden,
209}
210
211/// Visibility for solution submission details.
212#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
213#[serde(rename_all = "snake_case")]
214pub enum ChallengeResultDetailVisibility {
215    SubmitterLivePublicLive,
216    SubmitterLivePublicAfterClose,
217    SubmitterOnly,
218}
219
220/// Policy controlling when solution artifacts may become public.
221#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
222#[serde(rename_all = "snake_case")]
223pub enum ChallengeSolutionPublicationPolicy {
224    Private,
225    Public,
226    PublicAfterClose,
227}
228
229/// Local solution format constraints declared by a bundle.
230#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
231#[serde(deny_unknown_fields)]
232pub struct SolutionSpec {
233    pub protocol: String,
234    pub manifest_file: BundleRelativePath,
235}
agentics_domain/models/challenge/bundle.rs

agentics_domain/models/challenge/
bundle.rs
