systemprompt_runtime/optimization/
diagnostics.rs

1//! Retain safe actionable failures independently of HTTP and scheduler
2//! lifetimes.
3//!
4//! Copyright (c) systemprompt.io — Business Source License 1.1.
5//! See <https://systemprompt.io> for licensing details.
6use super::{OptimizationError, SkillOptimizationOrchestrator};
7use systemprompt_evaluation::campaigns::diagnostics::{DiagnosticCode, DiagnosticStage};
8use systemprompt_evaluation::campaigns::report::CampaignReport;
9use systemprompt_evaluation::repository::experiments::CampaignExperiment;
10use systemprompt_identifiers::{EvalCampaignId, EvalExperimentId, UserId};
11
12#[derive(Debug, Clone, Copy)]
13pub(super) struct DiagnosticContext<'a> {
14    pub owner: &'a UserId,
15    pub actor: &'a UserId,
16    pub campaign: &'a EvalCampaignId,
17    pub key: &'a str,
18    pub stage: DiagnosticStage,
19}
20
21impl SkillOptimizationOrchestrator {
22    pub(super) async fn blocked(
23        &self,
24        ctx: &DiagnosticContext<'_>,
25        code: DiagnosticCode,
26    ) -> Result<(), OptimizationError> {
27        let operation = format!(
28            "{}:{}",
29            ctx.campaign,
30            systemprompt_evaluation::experiments::content_digest(&ctx.key)?
31        );
32        self.evaluations
33            .campaigns
34            .record_diagnostic(
35                ctx.owner,
36                systemprompt_evaluation::campaigns::diagnostics::DiagnosticRecord {
37                    actor: ctx.actor,
38                    campaign: Some(ctx.campaign),
39                    operation: &operation,
40                    stage: ctx.stage,
41                    code,
42                },
43            )
44            .await?;
45        Ok(())
46    }
47    async fn resolve_blocked(
48        &self,
49        owner: &UserId,
50        campaign: &EvalCampaignId,
51        key: &str,
52    ) -> Result<(), OptimizationError> {
53        let operation = format!(
54            "{}:{}",
55            campaign,
56            systemprompt_evaluation::experiments::content_digest(&key)?
57        );
58        self.evaluations
59            .campaigns
60            .resolve_diagnostics(owner, campaign, &operation)
61            .await?;
62        Ok(())
63    }
64    pub(super) async fn retain_failure(
65        &self,
66        ctx: &DiagnosticContext<'_>,
67        error: &OptimizationError,
68    ) -> Result<(), OptimizationError> {
69        let code = match error {
70            OptimizationError::Evaluation(error) => DiagnosticCode::from_error(error),
71            _ => DiagnosticCode::InvalidInput,
72        };
73        self.blocked(ctx, code).await
74    }
75    pub async fn launch(
76        &self,
77        owner: &UserId,
78        actor: &UserId,
79        input: &CampaignExperiment,
80    ) -> Result<EvalExperimentId, OptimizationError> {
81        let result = self.launch_inner(owner, actor, input).await;
82        if let Err(error) = &result {
83            let ctx = DiagnosticContext {
84                owner,
85                actor,
86                campaign: &input.campaign_id,
87                key: &input.idempotency_key,
88                stage: DiagnosticStage::Launch,
89            };
90            self.retain_failure(&ctx, error).await?;
91        } else {
92            self.resolve_blocked(owner, &input.campaign_id, &input.idempotency_key)
93                .await?;
94        }
95        result
96    }
97    pub async fn advance(
98        &self,
99        owner: &UserId,
100        actor: &UserId,
101        campaign: &EvalCampaignId,
102    ) -> Result<Option<EvalExperimentId>, OptimizationError> {
103        let result = self.advance_inner(owner, actor, campaign).await;
104        if let Err(error) = &result {
105            let ctx = DiagnosticContext {
106                owner,
107                actor,
108                campaign,
109                key: "automatic",
110                stage: DiagnosticStage::AutomaticFollowup,
111            };
112            self.retain_failure(&ctx, error).await?;
113        } else if matches!(&result, Ok(Some(_))) {
114            self.resolve_blocked(owner, campaign, "automatic").await?;
115        }
116        result
117    }
118    pub async fn report(
119        &self,
120        owner: &UserId,
121        campaign: &EvalCampaignId,
122        experiment: &EvalExperimentId,
123    ) -> Result<CampaignReport, OptimizationError> {
124        let result = self.report_inner(owner, campaign, experiment).await;
125        if let Err(error) = &result {
126            let ctx = DiagnosticContext {
127                owner,
128                actor: owner,
129                campaign,
130                key: experiment.as_str(),
131                stage: DiagnosticStage::Report,
132            };
133            self.retain_failure(&ctx, error).await?;
134        } else {
135            self.resolve_blocked(owner, campaign, experiment.as_str())
136                .await?;
137        }
138        result
139    }
140}
systemprompt_runtime/optimization/diagnostics.rs

systemprompt_runtime/optimization/
diagnostics.rs
