a3s_code_core/research/
reproducibility.rs

1//! Typed reproducibility manifest for one research Run.
2
3use super::{
4    digest, validate_digest_field, validate_id, ResearchContractError, ResearchProvenanceReceiptV1,
5    ResearchReproducibilityV1, ResearchRunV1, RESEARCH_MAX_DIGESTS,
6};
7use serde::{Deserialize, Serialize};
8
9pub const RESEARCH_REPRODUCIBILITY_MANIFEST_SCHEMA_V1: &str =
10    "a3s.code.reproducibility-manifest.v1";
11const RESEARCH_REPRODUCIBILITY_MANIFEST_DIGEST_DOMAIN: &str =
12    "a3s.code.reproducibility-manifest.identity.v1";
13
14/// Digests and non-secret parameters that make one research Run reproducible.
15///
16/// The manifest never stores prompts, credentials, or raw provider payloads.
17/// Hosts retain those behind digests. Code only fences identity drift against
18/// the admitted Run and optional provenance receipts.
19#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
20#[serde(rename_all = "camelCase", deny_unknown_fields)]
21pub struct ResearchReproducibilityManifestV1 {
22    pub schema: String,
23    pub manifest_id: String,
24    pub project_id: String,
25    pub run_id: String,
26    pub provider_id: String,
27    pub model_id: String,
28    #[serde(default, skip_serializing_if = "Option::is_none")]
29    pub model_revision_digest: Option<String>,
30    pub environment_lock_digest: String,
31    pub code_digest: String,
32    pub workflow_digest: String,
33    pub parameter_digest: String,
34    #[serde(default, skip_serializing_if = "Option::is_none")]
35    pub random_seed: Option<u64>,
36    pub tolerance_digests: Vec<String>,
37    pub output_artifact_digests: Vec<String>,
38    pub manifest_digest: String,
39}
40
41impl ResearchReproducibilityManifestV1 {
42    #[allow(clippy::too_many_arguments)]
43    pub fn new_for_run(
44        manifest_id: impl Into<String>,
45        run: &ResearchRunV1,
46        model_revision_digest: Option<String>,
47        environment_lock_digest: impl Into<String>,
48        code_digest: impl Into<String>,
49        workflow_digest: impl Into<String>,
50        parameter_digest: impl Into<String>,
51        mut tolerance_digests: Vec<String>,
52        mut output_artifact_digests: Vec<String>,
53    ) -> Result<Self, ResearchContractError> {
54        tolerance_digests.sort();
55        tolerance_digests.dedup();
56        output_artifact_digests.sort();
57        output_artifact_digests.dedup();
58        let mut manifest = Self {
59            schema: RESEARCH_REPRODUCIBILITY_MANIFEST_SCHEMA_V1.to_owned(),
60            manifest_id: manifest_id.into(),
61            project_id: run.project_id.clone(),
62            run_id: run.run_id.clone(),
63            provider_id: run.provider_id.clone(),
64            model_id: run.model_id.clone(),
65            model_revision_digest,
66            environment_lock_digest: environment_lock_digest.into(),
67            code_digest: code_digest.into(),
68            workflow_digest: workflow_digest.into(),
69            parameter_digest: parameter_digest.into(),
70            random_seed: run.random_seed,
71            tolerance_digests,
72            output_artifact_digests,
73            manifest_digest: String::new(),
74        };
75        manifest.validate_without_digest()?;
76        manifest.validate_against_run(run)?;
77        manifest.manifest_digest = manifest.expected_digest()?;
78        Ok(manifest)
79    }
80
81    pub fn validate_for_run(&self, run: &ResearchRunV1) -> Result<(), ResearchContractError> {
82        self.validate()?;
83        self.validate_against_run(run)
84    }
85
86    /// Require every output artifact in this manifest to appear in the supplied
87    /// provenance receipts for the same project/Run, with matching environment
88    /// and workflow digests.
89    pub fn validate_against_provenance(
90        &self,
91        receipts: &[ResearchProvenanceReceiptV1],
92    ) -> Result<(), ResearchContractError> {
93        self.validate()?;
94        if self.output_artifact_digests.is_empty() {
95            return Err(ResearchContractError::InvalidField("outputArtifactDigests"));
96        }
97        let mut seen = std::collections::BTreeSet::new();
98        for receipt in receipts {
99            receipt.validate()?;
100            if receipt.project_id != self.project_id {
101                return Err(ResearchContractError::InvalidField("receipt.projectId"));
102            }
103            if receipt.run_id != self.run_id {
104                return Err(ResearchContractError::InvalidField("receipt.runId"));
105            }
106            if receipt.environment_digest != self.environment_lock_digest {
107                return Err(ResearchContractError::InvalidField("environmentLockDigest"));
108            }
109            if receipt.workflow_digest != self.workflow_digest {
110                return Err(ResearchContractError::InvalidField("workflowDigest"));
111            }
112            if receipt.code_digest != self.code_digest {
113                return Err(ResearchContractError::InvalidField("codeDigest"));
114            }
115            if receipt.provider_id != self.provider_id {
116                return Err(ResearchContractError::InvalidField("providerId"));
117            }
118            if receipt.random_seed != self.random_seed {
119                return Err(ResearchContractError::InvalidField("randomSeed"));
120            }
121            if !self
122                .output_artifact_digests
123                .iter()
124                .any(|digest| digest == &receipt.artifact_digest)
125            {
126                return Err(ResearchContractError::InvalidField("artifactDigest"));
127            }
128            if !seen.insert(receipt.artifact_digest.as_str()) {
129                return Err(ResearchContractError::InvalidField("artifactDigest"));
130            }
131        }
132        for output in &self.output_artifact_digests {
133            if !seen.contains(output.as_str()) {
134                return Err(ResearchContractError::InvalidField("outputArtifactDigests"));
135            }
136        }
137        Ok(())
138    }
139
140    pub fn validate(&self) -> Result<(), ResearchContractError> {
141        self.validate_without_digest()?;
142        validate_digest_field("manifestDigest", &self.manifest_digest)?;
143        if self.manifest_digest != self.expected_digest()? {
144            return Err(ResearchContractError::DigestMismatch("manifestDigest"));
145        }
146        Ok(())
147    }
148
149    pub fn from_slice(bytes: &[u8]) -> Result<Self, ResearchContractError> {
150        let manifest: Self = super::decode_json_slice(bytes)?;
151        manifest.validate()?;
152        Ok(manifest)
153    }
154
155    pub fn to_vec(&self) -> Result<Vec<u8>, ResearchContractError> {
156        self.validate()?;
157        super::encode_json(self)
158    }
159
160    fn validate_against_run(&self, run: &ResearchRunV1) -> Result<(), ResearchContractError> {
161        if self.project_id != run.project_id {
162            return Err(ResearchContractError::InvalidField("projectId"));
163        }
164        if self.run_id != run.run_id {
165            return Err(ResearchContractError::InvalidField("runId"));
166        }
167        if self.provider_id != run.provider_id {
168            return Err(ResearchContractError::InvalidField("providerId"));
169        }
170        if self.model_id != run.model_id {
171            return Err(ResearchContractError::InvalidField("modelId"));
172        }
173        if self.random_seed != run.random_seed {
174            return Err(ResearchContractError::InvalidField("randomSeed"));
175        }
176        if matches!(
177            run.reproducibility,
178            ResearchReproducibilityV1::Deterministic
179        ) && self.random_seed.is_none()
180        {
181            return Err(ResearchContractError::InvalidField("randomSeed"));
182        }
183        Ok(())
184    }
185
186    fn validate_without_digest(&self) -> Result<(), ResearchContractError> {
187        if self.schema != RESEARCH_REPRODUCIBILITY_MANIFEST_SCHEMA_V1 {
188            return Err(ResearchContractError::UnsupportedSchema);
189        }
190        validate_id("manifestId", &self.manifest_id)?;
191        validate_id("projectId", &self.project_id)?;
192        validate_id("runId", &self.run_id)?;
193        validate_id("providerId", &self.provider_id)?;
194        validate_id("modelId", &self.model_id)?;
195        if let Some(model_revision_digest) = &self.model_revision_digest {
196            validate_digest_field("modelRevisionDigest", model_revision_digest)?;
197        }
198        validate_digest_field("environmentLockDigest", &self.environment_lock_digest)?;
199        validate_digest_field("codeDigest", &self.code_digest)?;
200        validate_digest_field("workflowDigest", &self.workflow_digest)?;
201        validate_digest_field("parameterDigest", &self.parameter_digest)?;
202        if self.tolerance_digests.len() > RESEARCH_MAX_DIGESTS {
203            return Err(ResearchContractError::InvalidField("toleranceDigests"));
204        }
205        if self.output_artifact_digests.len() > RESEARCH_MAX_DIGESTS {
206            return Err(ResearchContractError::InvalidField("outputArtifactDigests"));
207        }
208        validate_sorted_unique_digests("toleranceDigests", &self.tolerance_digests)?;
209        validate_sorted_unique_digests("outputArtifactDigests", &self.output_artifact_digests)?;
210        Ok(())
211    }
212
213    fn expected_digest(&self) -> Result<String, ResearchContractError> {
214        #[derive(Serialize)]
215        struct Identity<'a> {
216            schema: &'a str,
217            manifest_id: &'a str,
218            project_id: &'a str,
219            run_id: &'a str,
220            provider_id: &'a str,
221            model_id: &'a str,
222            model_revision_digest: Option<&'a str>,
223            environment_lock_digest: &'a str,
224            code_digest: &'a str,
225            workflow_digest: &'a str,
226            parameter_digest: &'a str,
227            random_seed: Option<u64>,
228            tolerance_digests: &'a [String],
229            output_artifact_digests: &'a [String],
230        }
231        digest(
232            RESEARCH_REPRODUCIBILITY_MANIFEST_DIGEST_DOMAIN,
233            &Identity {
234                schema: &self.schema,
235                manifest_id: &self.manifest_id,
236                project_id: &self.project_id,
237                run_id: &self.run_id,
238                provider_id: &self.provider_id,
239                model_id: &self.model_id,
240                model_revision_digest: self.model_revision_digest.as_deref(),
241                environment_lock_digest: &self.environment_lock_digest,
242                code_digest: &self.code_digest,
243                workflow_digest: &self.workflow_digest,
244                parameter_digest: &self.parameter_digest,
245                random_seed: self.random_seed,
246                tolerance_digests: &self.tolerance_digests,
247                output_artifact_digests: &self.output_artifact_digests,
248            },
249        )
250    }
251}
252
253fn validate_sorted_unique_digests(
254    field: &'static str,
255    digests: &[String],
256) -> Result<(), ResearchContractError> {
257    for pair in digests.windows(2) {
258        if pair[0] >= pair[1] {
259            return Err(ResearchContractError::InvalidField(field));
260        }
261    }
262    for value in digests {
263        validate_digest_field(field, value)?;
264    }
265    Ok(())
266}
267
268#[cfg(test)]
269mod tests {
270    use super::*;
271    use crate::capability::{
272        CapabilityCeiling, CapabilityContribution, CapabilityDescriptor,
273        CapabilityExecutionCeiling, CapabilityKind, CapabilitySet, CapabilitySource,
274        CodeCatalogGeneration, GovernanceCapabilityCeiling, RunCapabilityBindingV1, Sha256Digest,
275        WorkspaceCapabilityCeiling,
276    };
277    use crate::research::{ResearchArtifactKindV1, ResearchRunStatusV1};
278
279    fn digest(ch: char) -> String {
280        format!("sha256:{}", ch.to_string().repeat(64))
281    }
282
283    fn binding() -> RunCapabilityBindingV1 {
284        let source =
285            CapabilitySource::builtin("test", Sha256Digest::new(digest('c')).unwrap()).unwrap();
286        let descriptor = CapabilityDescriptor::new(
287            &source,
288            CapabilityKind::Tool,
289            "tool",
290            "tool",
291            Sha256Digest::new(digest('d')).unwrap(),
292            [],
293        )
294        .unwrap();
295        let contribution = CapabilityContribution::new(source, [descriptor]).unwrap();
296        let set = CapabilitySet::from_contributions(CodeCatalogGeneration::new(1), [contribution])
297            .unwrap();
298        let ceiling = CapabilityCeiling::all(
299            &set,
300            WorkspaceCapabilityCeiling::default(),
301            GovernanceCapabilityCeiling::default(),
302            CapabilityExecutionCeiling::new(1, 1, None, None, None).unwrap(),
303        )
304        .unwrap();
305        RunCapabilityBindingV1::from_set_and_ceiling(&set, &ceiling).unwrap()
306    }
307
308    fn admitted_run() -> ResearchRunV1 {
309        let mut run = ResearchRunV1::new(
310            "run-1",
311            "project-1",
312            1,
313            digest('1'),
314            digest('2'),
315            binding(),
316            "provider-1",
317            "model-1",
318            ResearchReproducibilityV1::Deterministic,
319            Some(7),
320        )
321        .unwrap();
322        run.transition_to(ResearchRunStatusV1::Admitted).unwrap();
323        run
324    }
325
326    #[test]
327    fn manifest_binds_run_and_provenance_outputs() {
328        let run = admitted_run();
329        let manifest = ResearchReproducibilityManifestV1::new_for_run(
330            "manifest-1",
331            &run,
332            Some(digest('3')),
333            digest('4'),
334            digest('5'),
335            digest('6'),
336            digest('7'),
337            vec![digest('9'), digest('8')],
338            vec![digest('a'), digest('b')],
339        )
340        .unwrap();
341        assert_eq!(manifest.random_seed, Some(7));
342        assert_eq!(manifest.tolerance_digests, vec![digest('8'), digest('9')]);
343        let receipt_a = ResearchProvenanceReceiptV1::new(
344            "project-1",
345            1,
346            "run-1",
347            "artifact-a",
348            ResearchArtifactKindV1::Figure,
349            digest('a'),
350            vec![digest('2')],
351            digest('6'),
352            digest('5'),
353            digest('4'),
354            "provider-1",
355            Some(digest('3')),
356            Some(7),
357            None,
358        )
359        .unwrap();
360        let receipt_b = ResearchProvenanceReceiptV1::new(
361            "project-1",
362            1,
363            "run-1",
364            "artifact-b",
365            ResearchArtifactKindV1::Report,
366            digest('b'),
367            vec![digest('2')],
368            digest('6'),
369            digest('5'),
370            digest('4'),
371            "provider-1",
372            Some(digest('3')),
373            Some(7),
374            None,
375        )
376        .unwrap();
377        manifest
378            .validate_against_provenance(&[receipt_a, receipt_b])
379            .unwrap();
380        let encoded = manifest.to_vec().unwrap();
381        assert_eq!(
382            ResearchReproducibilityManifestV1::from_slice(&encoded).unwrap(),
383            manifest
384        );
385    }
386
387    #[test]
388    fn seed_and_output_drift_fail_closed() {
389        let run = admitted_run();
390        let manifest = ResearchReproducibilityManifestV1::new_for_run(
391            "manifest-1",
392            &run,
393            None,
394            digest('4'),
395            digest('5'),
396            digest('6'),
397            digest('7'),
398            Vec::new(),
399            vec![digest('a')],
400        )
401        .unwrap();
402        let mut drifted = run.clone();
403        drifted.random_seed = Some(8);
404        assert!(matches!(
405            manifest.validate_for_run(&drifted),
406            Err(ResearchContractError::InvalidField("randomSeed"))
407        ));
408        let foreign = ResearchProvenanceReceiptV1::new(
409            "project-1",
410            1,
411            "run-1",
412            "artifact-x",
413            ResearchArtifactKindV1::Figure,
414            digest('c'),
415            vec![digest('2')],
416            digest('6'),
417            digest('5'),
418            digest('4'),
419            "provider-1",
420            None,
421            Some(7),
422            None,
423        )
424        .unwrap();
425        assert!(matches!(
426            manifest.validate_against_provenance(&[foreign]),
427            Err(ResearchContractError::InvalidField("artifactDigest"))
428        ));
429    }
430}
a3s_code_core/research/reproducibility.rs

a3s_code_core/research/
reproducibility.rs
