systemprompt_runtime/optimization/
mod.rs

1//! Cross-domain source verification and evaluation attestation. The managed
2//! domain owns content; evaluation owns measurements; this layer binds them.
3//!
4//! Copyright (c) systemprompt.io — Business Source License 1.1.
5//! See <https://systemprompt.io> for licensing details.
6
7use serde::Serialize;
8use systemprompt_evaluation::campaigns::CampaignPolicy;
9use systemprompt_evaluation::campaigns::comparison::ComparisonDecision;
10use systemprompt_evaluation::campaigns::report::{self, CampaignReport};
11use systemprompt_evaluation::repository::experiments::EvaluationRepositories;
12use systemprompt_identifiers::{EvalCampaignId, EvalExperimentId, ResourceRevisionId, UserId};
13use systemprompt_marketplace::managed::evaluation::EvaluationAttestation;
14use systemprompt_marketplace::managed::{AssetDigest, ManagedRepository};
15mod candidate;
16mod capture;
17mod diagnostics;
18pub mod git_sources;
19pub mod holdout;
20mod holdout_partition;
21pub mod inventory;
22mod iteration;
23
24#[derive(Debug, thiserror::Error)]
25pub enum OptimizationError {
26    #[error(transparent)]
27    Evaluation(#[from] systemprompt_evaluation::EvaluationError),
28    #[error(transparent)]
29    Managed(#[from] systemprompt_marketplace::managed::ManagedError),
30    #[error(transparent)]
31    Bundle(#[from] systemprompt_models::managed::RevisionBundleError),
32    #[error(transparent)]
33    Json(#[from] serde_json::Error),
34    #[error("Source verification failed: {0}")]
35    Source(String),
36}
37
38#[derive(Debug, Clone)]
39pub struct SkillOptimizationOrchestrator {
40    managed: ManagedRepository,
41    evaluations: EvaluationRepositories,
42}
43
44/// The facts an evaluation attestation commits to, hashed as canonical JSON
45/// (RFC 8785) so the digest does not depend on field order.
46#[derive(Debug, Serialize)]
47pub struct EvaluationEvidence<'a> {
48    pub policy: &'a CampaignPolicy,
49    pub experiment_id: &'a EvalExperimentId,
50    pub baseline_bundle_digest: &'a str,
51    pub candidate_bundle_digest: &'a str,
52    pub development: &'a ComparisonDecision,
53    pub holdout: &'a ComparisonDecision,
54}
55
56impl EvaluationEvidence<'_> {
57    pub fn digest(&self) -> Result<AssetDigest, OptimizationError> {
58        Ok(AssetDigest::of(&serde_jcs::to_vec(self)?))
59    }
60}
61
62impl<'a> From<&'a CampaignReport> for EvaluationEvidence<'a> {
63    fn from(report: &'a CampaignReport) -> Self {
64        Self {
65            policy: &report.campaign.policy,
66            experiment_id: &report.experiment_id,
67            baseline_bundle_digest: &report.baseline_bundle_digest,
68            candidate_bundle_digest: &report.candidate_bundle_digest,
69            development: &report.development,
70            holdout: &report.holdout,
71        }
72    }
73}
74
75#[derive(Debug, Clone, serde::Deserialize, schemars::JsonSchema)]
76#[serde(deny_unknown_fields)]
77pub struct SourceAcceptance {
78    pub campaign_id: EvalCampaignId,
79    pub experiment_id: EvalExperimentId,
80    pub evaluated_revision_id: ResourceRevisionId,
81    pub committed_revision_id: ResourceRevisionId,
82    pub source_commit: String,
83}
84
85impl SkillOptimizationOrchestrator {
86    pub const fn new(managed: ManagedRepository, evaluations: EvaluationRepositories) -> Self {
87        Self {
88            managed,
89            evaluations,
90        }
91    }
92
93    pub(super) async fn report_inner(
94        &self,
95        owner: &UserId,
96        campaign: &EvalCampaignId,
97        experiment: &EvalExperimentId,
98    ) -> Result<CampaignReport, OptimizationError> {
99        let report = report::build(
100            &self.evaluations,
101            &self.evaluations.revisions,
102            owner,
103            campaign,
104            experiment,
105        )
106        .await?;
107        let baseline = self
108            .managed
109            .get_revision_bundle(owner, &report.campaign.policy.baseline_revision_id)
110            .await?;
111        if baseline.digest()?.as_str() != report.baseline_bundle_digest {
112            return Err(OptimizationError::Source(
113                "Experiment baseline differs from the campaign baseline".to_owned(),
114            ));
115        }
116        Ok(report)
117    }
118
119    pub async fn accept_source(
120        &self,
121        owner: &UserId,
122        actor: &UserId,
123        input: &SourceAcceptance,
124    ) -> Result<EvaluationAttestation, OptimizationError> {
125        let report = self
126            .report(owner, &input.campaign_id, &input.experiment_id)
127            .await?;
128        if !report.eligible_for_publication {
129            return Err(OptimizationError::Source(
130                "Experiment does not establish an eligible improvement".to_owned(),
131            ));
132        }
133        let resource = &report.campaign.policy.resource_id;
134        for revision in [&input.evaluated_revision_id, &input.committed_revision_id] {
135            if self.managed.revision_resource(owner, revision).await? != *resource {
136                return Err(OptimizationError::Source(
137                    "Source revisions must belong to the campaign resource".to_owned(),
138                ));
139            }
140        }
141        let evaluated = self
142            .managed
143            .get_revision_bundle(owner, &input.evaluated_revision_id)
144            .await?;
145        let committed = self
146            .managed
147            .get_revision_bundle(owner, &input.committed_revision_id)
148            .await?;
149        if evaluated.digest()?.as_str() != report.candidate_bundle_digest
150            || evaluated.content_digest()? != committed.content_digest()?
151        {
152            return Err(OptimizationError::Source("Committed content or dependencies differ from the evaluated candidate; reevaluation required".to_owned()));
153        }
154        self.managed
155            .require_verified_git_content(owner, &input.committed_revision_id, &input.source_commit)
156            .await?;
157        let source_commit = input.source_commit.clone();
158        let evidence = EvaluationAttestation {
159            resource_id: resource.clone(),
160            revision_id: input.committed_revision_id.clone(),
161            bundle_digest: committed.digest()?,
162            experiment_id: input.experiment_id.clone(),
163            campaign_id: input.campaign_id.clone(),
164            evidence_digest: EvaluationEvidence::from(&report).digest()?,
165            source_commit,
166        };
167        self.managed
168            .attest_evaluation(owner, actor, &evidence)
169            .await?;
170        Ok(evidence)
171    }
172}
systemprompt_runtime/optimization/mod.rs

systemprompt_runtime/optimization/
mod.rs
