systemprompt_runtime/optimization/
holdout.rs

1//! Guided independent confirmation retains its matrix before explicit approval.
2//!
3//! Copyright (c) systemprompt.io — Business Source License 1.1.
4//! See <https://systemprompt.io> for licensing details.
5use super::diagnostics::DiagnosticContext;
6use super::{OptimizationError, SkillOptimizationOrchestrator};
7use serde::{Deserialize, Serialize};
8use systemprompt_evaluation::campaigns::diagnostics::{DiagnosticCode, DiagnosticStage};
9use systemprompt_evaluation::campaigns::holdout::HoldoutProposal;
10use systemprompt_evaluation::experiments::ExperimentSpec;
11use systemprompt_evaluation::experiments::records::ExperimentStatus;
12use systemprompt_evaluation::repository::experiments::{CampaignAvailability, CampaignExperiment};
13use systemprompt_identifiers::{
14    EvalCampaignId, EvalExperimentId, EvalHoldoutProposalId, EvalRevisionId, UserId,
15};
16
17/// Select a completed development run and a separately authored holdout
18/// dataset.
19#[derive(Debug, Clone, Deserialize, schemars::JsonSchema)]
20#[serde(deny_unknown_fields)]
21pub struct PrepareHoldout {
22    pub development_experiment_id: EvalExperimentId,
23    pub holdout_dataset_id: EvalRevisionId,
24    pub idempotency_key: String,
25}
26/// Explicit confirmation binds human approval to the retained matrix digest.
27#[derive(Debug, Clone, Deserialize, schemars::JsonSchema)]
28#[serde(deny_unknown_fields)]
29pub struct ConfirmHoldout {
30    pub spec_digest: String,
31    pub confirm_independent_holdout: bool,
32}
33/// The proposal a confirmation targets, and who is approving it.
34#[derive(Debug, Clone, Copy)]
35pub struct HoldoutConfirmationTarget<'a> {
36    pub actor: &'a UserId,
37    pub campaign: &'a EvalCampaignId,
38    pub id: &'a EvalHoldoutProposalId,
39}
40/// Reviewable proposal and current actual execution admission, without reserved
41/// spend.
42#[derive(Debug, Clone, Serialize, schemars::JsonSchema)]
43pub struct HoldoutReview {
44    pub proposal: HoldoutProposal,
45    pub execution_availability: CampaignAvailability,
46}
47impl SkillOptimizationOrchestrator {
48    pub async fn prepare_holdout(
49        &self,
50        owner: &UserId,
51        actor: &UserId,
52        campaign: &EvalCampaignId,
53        input: &PrepareHoldout,
54    ) -> Result<HoldoutReview, OptimizationError> {
55        let result = self.prepare_holdout_inner(owner, campaign, input).await;
56        let ctx = DiagnosticContext {
57            owner,
58            actor,
59            campaign,
60            key: &input.idempotency_key,
61            stage: DiagnosticStage::Holdout,
62        };
63        match &result {
64            Err(error) => self.retain_failure(&ctx, error).await?,
65            Ok(review) if !review.execution_availability.admitted => {
66                self.blocked(&ctx, DiagnosticCode::UnsupportedCapability)
67                    .await?;
68            },
69            Ok(_) => {},
70        }
71        result
72    }
73    async fn prepare_holdout_inner(
74        &self,
75        owner: &UserId,
76        campaign: &EvalCampaignId,
77        input: &PrepareHoldout,
78    ) -> Result<HoldoutReview, OptimizationError> {
79        let report = self
80            .report(owner, campaign, &input.development_experiment_id)
81            .await?;
82        if !report.development.eligible {
83            return Err(OptimizationError::Source("Complete a qualifying development comparison before independent holdout preparation".to_owned()));
84        }
85        let mut spec = self
86            .completed_development_spec(owner, &input.development_experiment_id)
87            .await?;
88        let cases = self
89            .partition_cases(owner, &spec.cases, &input.holdout_dataset_id)
90            .await?;
91        let counts = (
92            i32::try_from(cases.development.len()).unwrap_or(i32::MAX),
93            i32::try_from(cases.holdout.len()).unwrap_or(i32::MAX),
94        );
95        let minimum = i32::try_from(report.campaign.policy.minimum_pairs).unwrap_or(i32::MAX);
96        if counts.0 < minimum || counts.1 < minimum {
97            return Err(OptimizationError::Source(
98                "Both partitions must meet the unchanged campaign minimum paired-case threshold"
99                    .to_owned(),
100            ));
101        }
102        self.freeze_holdout_spec(owner, &mut spec, cases).await?;
103        let proposal = self
104            .evaluations
105            .campaigns
106            .propose_holdout(
107                owner,
108                systemprompt_evaluation::campaigns::holdout::HoldoutProposalRequest {
109                    campaign,
110                    development: &input.development_experiment_id,
111                    key: &input.idempotency_key,
112                    spec: &spec,
113                    counts,
114                },
115            )
116            .await?;
117        Ok(HoldoutReview {
118            execution_availability: self
119                .evaluations
120                .experiments
121                .execution_availability(&proposal.spec),
122            proposal,
123        })
124    }
125    async fn completed_development_spec(
126        &self,
127        owner: &UserId,
128        experiment: &EvalExperimentId,
129    ) -> Result<ExperimentSpec, OptimizationError> {
130        let detail = self.evaluations.experiments.get(owner, experiment).await?;
131        if detail.experiment.status != ExperimentStatus::Completed
132            || detail.experiment.accounting.reserved != 0
133            || detail.experiment.accounting.frozen
134        {
135            return Err(OptimizationError::Source(
136                "Development execution and accounting must be complete".to_owned(),
137            ));
138        }
139        Ok(detail.experiment.spec)
140    }
141    pub async fn confirm_holdout(
142        &self,
143        owner: &UserId,
144        target: &HoldoutConfirmationTarget<'_>,
145        input: &ConfirmHoldout,
146    ) -> Result<HoldoutProposal, OptimizationError> {
147        let ctx = DiagnosticContext {
148            owner,
149            actor: target.actor,
150            campaign: target.campaign,
151            key: target.id.as_str(),
152            stage: DiagnosticStage::Holdout,
153        };
154        let result = self.confirm_holdout_inner(&ctx, target.id, input).await;
155        if let Err(error) = &result {
156            self.retain_failure(&ctx, error).await?;
157        }
158        result
159    }
160    async fn confirm_holdout_inner(
161        &self,
162        ctx: &DiagnosticContext<'_>,
163        id: &EvalHoldoutProposalId,
164        input: &ConfirmHoldout,
165    ) -> Result<HoldoutProposal, OptimizationError> {
166        if !input.confirm_independent_holdout {
167            self.blocked(ctx, DiagnosticCode::InvalidInput).await?;
168            return Err(OptimizationError::Source(
169                "Explicit independent holdout confirmation is required".to_owned(),
170            ));
171        }
172        let proposal = self
173            .evaluations
174            .campaigns
175            .confirm_holdout(
176                ctx.owner,
177                systemprompt_evaluation::campaigns::holdout::HoldoutConfirmation {
178                    actor: ctx.actor,
179                    campaign: ctx.campaign,
180                    id,
181                    digest: &input.spec_digest,
182                },
183            )
184            .await?;
185        if proposal.experiment_id.is_some() {
186            return Ok(proposal);
187        }
188        let experiment = self
189            .launch(
190                ctx.owner,
191                ctx.actor,
192                &CampaignExperiment {
193                    campaign_id: ctx.campaign.clone(),
194                    idempotency_key: format!("holdout:{}", proposal.id),
195                    spec: proposal.spec,
196                },
197            )
198            .await?;
199        Ok(self
200            .evaluations
201            .campaigns
202            .attach_holdout_run(ctx.owner, ctx.campaign, id, &experiment)
203            .await?)
204    }
205}
systemprompt_runtime/optimization/holdout.rs

systemprompt_runtime/optimization/
holdout.rs
