systemprompt_models/feedback/
analytics.rs

1//! Normalised invocation facts: the fact key, consumer and resource
2//! attribution, and the spend a consumer recorded for one skill invocation.
3//!
4//! Copyright (c) systemprompt.io — Business Source License 1.1.
5//! See <https://systemprompt.io> for licensing details.
6
7use chrono::{DateTime, Utc};
8use serde::{Deserialize, Serialize};
9use systemprompt_identifiers::{
10    AnalyticsChangeId, AnalyticsFactId, DeviceId, ManagedResourceId, MarketplaceId,
11    NativeSessionId, PluginId, ResourceInvocationId, ResourceRevisionId, UserId,
12};
13
14use super::EvaluatorClient;
15
16#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
17#[serde(rename_all = "snake_case")]
18pub enum AnalyticsFactKind {
19    Invocation,
20    Request,
21    Assessment,
22    ResourceAssociation,
23    Artifact,
24}
25
26#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
27#[serde(deny_unknown_fields)]
28pub struct AnalyticsFactKey {
29    pub kind: AnalyticsFactKind,
30    pub source: String,
31    pub id: AnalyticsFactId,
32}
33
34#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
35#[serde(tag = "status", rename_all = "snake_case")]
36pub enum InvocationConsumerIdentity {
37    HistoricalUnknown,
38    Authenticated {
39        consumer_id: UserId,
40        device_id: DeviceId,
41        host: EvaluatorClient,
42        session_id: NativeSessionId,
43    },
44}
45
46#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
47#[serde(tag = "status", rename_all = "snake_case")]
48pub enum InvocationResourceAttribution {
49    Unknown,
50    Verified {
51        resource_id: ManagedResourceId,
52        revision_id: ResourceRevisionId,
53    },
54}
55
56/// The skill an invocation named, as the hook reported it, and the services
57/// source that shipped it.
58///
59/// Distinct from [`InvocationResourceAttribution`]: that is the revision
60/// proof (a managed resource this device verifiably installed), this is the
61/// identity every invocation has whether or not the skill is a managed
62/// resource. `skill` is the hook's `<plugin>:<skill>` string verbatim;
63/// `source` is `base` or `bundle:<name>` and `source_hash` is that source's
64/// content hash at the time the fact was normalised, so a figure keyed on
65/// the skill can also say which published tree it ran from.
66#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
67#[serde(deny_unknown_fields)]
68pub struct InvocationSkillIdentity {
69    pub plugin_id: PluginId,
70    pub skill: String,
71    #[serde(default, skip_serializing_if = "Option::is_none")]
72    pub marketplace_id: Option<MarketplaceId>,
73    #[serde(default, skip_serializing_if = "Option::is_none")]
74    pub source: Option<String>,
75    #[serde(default, skip_serializing_if = "Option::is_none")]
76    pub source_hash: Option<String>,
77}
78
79#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
80#[serde(deny_unknown_fields)]
81pub struct NormalizedInvocationFact {
82    pub invocation_id: ResourceInvocationId,
83    pub occurred_at: DateTime<Utc>,
84    pub consumer: InvocationConsumerIdentity,
85    pub attribution: InvocationResourceAttribution,
86    #[serde(default, skip_serializing_if = "Option::is_none")]
87    pub skill: Option<InvocationSkillIdentity>,
88    pub succeeded: bool,
89    pub latency_micros: Option<u64>,
90}
91
92#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
93#[serde(tag = "status", rename_all = "snake_case")]
94pub enum RecordedSpend {
95    Known {
96        currency: String,
97        amount_micros: u64,
98    },
99    UnknownPricing,
100}
101
102#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
103#[serde(deny_unknown_fields)]
104pub struct NormalizedRequestFact {
105    pub request_key: AnalyticsFactKey,
106    pub occurred_at: DateTime<Utc>,
107    pub consumer: InvocationConsumerIdentity,
108    pub succeeded: bool,
109    pub spend: RecordedSpend,
110    pub input_tokens: Option<u64>,
111    pub output_tokens: Option<u64>,
112    pub latency_micros: Option<u64>,
113}
114
115#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
116#[serde(tag = "status", rename_all = "snake_case")]
117pub enum AssessmentOutcome {
118    Scored { score_millionths: i64 },
119    Failed,
120    Unavailable,
121}
122
123#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
124#[serde(deny_unknown_fields)]
125pub struct AssessmentConversationKey {
126    pub source: String,
127    pub id: AnalyticsFactId,
128}
129
130#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
131#[serde(deny_unknown_fields)]
132pub struct NormalizedAssessmentFact {
133    pub conversation_key: AssessmentConversationKey,
134    pub assessment_key: AnalyticsFactKey,
135    pub invocation_key: AnalyticsFactKey,
136    pub occurred_at: DateTime<Utc>,
137    pub outcome: AssessmentOutcome,
138}
139
140#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
141#[serde(deny_unknown_fields)]
142pub struct NormalizedResourceAssociationFact {
143    pub association_key: AnalyticsFactKey,
144    pub invocation_key: AnalyticsFactKey,
145    pub request_key: AnalyticsFactKey,
146    pub occurred_at: DateTime<Utc>,
147    pub attribution: InvocationResourceAttribution,
148}
149
150/// One tool result stored as a typed artifact.
151///
152/// It is joined to the invocation and request it belongs to where those are
153/// known. `source` is the vantage point the platform saw the result from and
154/// `correlation` says whether it was joined by an exact key or inferred.
155#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
156#[serde(deny_unknown_fields)]
157pub struct NormalizedArtifactFact {
158    pub artifact_key: AnalyticsFactKey,
159    pub execution_id: String,
160    #[serde(default, skip_serializing_if = "Option::is_none")]
161    pub invocation_key: Option<AnalyticsFactKey>,
162    #[serde(default, skip_serializing_if = "Option::is_none")]
163    pub request_key: Option<AnalyticsFactKey>,
164    pub occurred_at: DateTime<Utc>,
165    pub consumer: InvocationConsumerIdentity,
166    #[serde(default, skip_serializing_if = "Option::is_none")]
167    pub skill: Option<InvocationSkillIdentity>,
168    pub tool_name: String,
169    #[serde(default, skip_serializing_if = "Option::is_none")]
170    pub server_name: Option<String>,
171    pub artifact_type: String,
172    pub source: crate::mcp::ExecutionSource,
173    pub correlation: crate::mcp::Correlation,
174    pub is_structured: bool,
175    pub has_ui_resource: bool,
176    pub succeeded: bool,
177    pub payload_bytes: Option<u64>,
178    pub findings: u64,
179}
180
181#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
182#[serde(tag = "kind", content = "value", rename_all = "snake_case")]
183pub enum NormalizedAnalyticsFact {
184    Invocation(NormalizedInvocationFact),
185    Request(NormalizedRequestFact),
186    Assessment(NormalizedAssessmentFact),
187    ResourceAssociation(NormalizedResourceAssociationFact),
188    Artifact(Box<NormalizedArtifactFact>),
189}
190
191impl NormalizedAnalyticsFact {
192    pub const fn kind(&self) -> AnalyticsFactKind {
193        match self {
194            Self::Invocation(_) => AnalyticsFactKind::Invocation,
195            Self::Request(_) => AnalyticsFactKind::Request,
196            Self::Assessment(_) => AnalyticsFactKind::Assessment,
197            Self::ResourceAssociation(_) => AnalyticsFactKind::ResourceAssociation,
198            Self::Artifact(_) => AnalyticsFactKind::Artifact,
199        }
200    }
201}
202
203#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
204#[serde(tag = "operation", rename_all = "snake_case")]
205#[expect(
206    clippy::large_enum_variant,
207    reason = "wire contract: a tombstone carries nothing by definition and the fact is matched by value"
208)]
209pub enum AnalyticsChangeOperation {
210    Replace { fact: NormalizedAnalyticsFact },
211    Tombstone,
212}
213
214#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
215#[serde(deny_unknown_fields)]
216pub struct AnalyticsChange {
217    pub change_id: AnalyticsChangeId,
218    pub key: AnalyticsFactKey,
219    pub revision: u64,
220    pub occurred_at: DateTime<Utc>,
221    pub recorded_at: DateTime<Utc>,
222    pub operation: AnalyticsChangeOperation,
223}
224
225impl AnalyticsChange {
226    pub fn validate(&self) -> Result<(), super::FeedbackContractError> {
227        if self.revision == 0 || self.key.source.is_empty() || self.key.source.len() > 128 {
228            return Err(super::FeedbackContractError::Bounds);
229        }
230        if let AnalyticsChangeOperation::Replace { fact } = &self.operation
231            && fact.kind() != self.key.kind
232        {
233            return Err(super::FeedbackContractError::IncompleteManifest);
234        }
235        Ok(())
236    }
237}
systemprompt_models/feedback/analytics.rs

systemprompt_models/feedback/
analytics.rs
