systemprompt_models/feedback/
analytics.rs

1//! Normalised invocation facts: the fact key, consumer and resource
2//! attribution, and the spend a consumer recorded for one skill invocation.
3//!
4//! Copyright (c) systemprompt.io — Business Source License 1.1.
5//! See <https://systemprompt.io> for licensing details.
6
7use chrono::{DateTime, Utc};
8use serde::{Deserialize, Serialize};
9use systemprompt_identifiers::{
10    AnalyticsChangeId, AnalyticsFactId, DeviceId, ManagedResourceId, MarketplaceId,
11    NativeSessionId, PluginId, ResourceInvocationId, ResourceRevisionId, UserId,
12};
13
14use super::EvaluatorClient;
15
16#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
17#[serde(rename_all = "snake_case")]
18pub enum AnalyticsFactKind {
19    Invocation,
20    Request,
21    Assessment,
22    ResourceAssociation,
23    Artifact,
24}
25
26#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
27#[serde(deny_unknown_fields)]
28pub struct AnalyticsFactKey {
29    pub kind: AnalyticsFactKind,
30    pub source: String,
31    pub id: AnalyticsFactId,
32}
33
34#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
35#[serde(tag = "status", rename_all = "snake_case")]
36pub enum InvocationConsumerIdentity {
37    HistoricalUnknown,
38    Authenticated {
39        consumer_id: UserId,
40        device_id: DeviceId,
41        host: EvaluatorClient,
42        session_id: NativeSessionId,
43    },
44}
45
46#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
47#[serde(tag = "status", rename_all = "snake_case")]
48pub enum InvocationResourceAttribution {
49    Unknown,
50    Verified {
51        resource_id: ManagedResourceId,
52        revision_id: ResourceRevisionId,
53    },
54}
55
56/// The skill an invocation named, as the hook reported it, and the services
57/// source that shipped it.
58///
59/// Distinct from [`InvocationResourceAttribution`]: that is the revision
60/// proof (a managed resource this device verifiably installed), this is the
61/// identity every invocation has whether or not the skill is a managed
62/// resource. `skill` is the hook's `<plugin>:<skill>` string verbatim;
63/// `source` is `base` or `bundle:<name>` and `source_hash` is that source's
64/// content hash at the time the fact was normalised, so a figure keyed on
65/// the skill can also say which published tree it ran from.
66/// `marketplace_hash` is the content hash of the marketplace version the
67/// plugin was served from, so a figure can be pinned to one published
68/// marketplace even after the source moves on.
69#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
70#[serde(deny_unknown_fields)]
71pub struct InvocationSkillIdentity {
72    pub plugin_id: PluginId,
73    pub skill: String,
74    #[serde(default, skip_serializing_if = "Option::is_none")]
75    pub marketplace_id: Option<MarketplaceId>,
76    #[serde(default, skip_serializing_if = "Option::is_none")]
77    pub source: Option<String>,
78    #[serde(default, skip_serializing_if = "Option::is_none")]
79    pub source_hash: Option<String>,
80    #[serde(default, skip_serializing_if = "Option::is_none")]
81    pub marketplace_hash: Option<String>,
82}
83
84#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
85#[serde(deny_unknown_fields)]
86pub struct NormalizedInvocationFact {
87    pub invocation_id: ResourceInvocationId,
88    pub occurred_at: DateTime<Utc>,
89    pub consumer: InvocationConsumerIdentity,
90    pub attribution: InvocationResourceAttribution,
91    #[serde(default, skip_serializing_if = "Option::is_none")]
92    pub skill: Option<InvocationSkillIdentity>,
93    pub succeeded: bool,
94    pub latency_micros: Option<u64>,
95}
96
97#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
98#[serde(tag = "status", rename_all = "snake_case")]
99pub enum RecordedSpend {
100    Known {
101        currency: String,
102        amount_micros: u64,
103    },
104    UnknownPricing,
105}
106
107#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
108#[serde(deny_unknown_fields)]
109pub struct NormalizedRequestFact {
110    pub request_key: AnalyticsFactKey,
111    pub occurred_at: DateTime<Utc>,
112    pub consumer: InvocationConsumerIdentity,
113    pub succeeded: bool,
114    pub spend: RecordedSpend,
115    pub input_tokens: Option<u64>,
116    pub output_tokens: Option<u64>,
117    pub latency_micros: Option<u64>,
118}
119
120#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
121#[serde(tag = "status", rename_all = "snake_case")]
122pub enum AssessmentOutcome {
123    Scored { score_millionths: i64 },
124    Failed,
125    Unavailable,
126}
127
128#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
129#[serde(deny_unknown_fields)]
130pub struct AssessmentConversationKey {
131    pub source: String,
132    pub id: AnalyticsFactId,
133}
134
135#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
136#[serde(deny_unknown_fields)]
137pub struct NormalizedAssessmentFact {
138    pub conversation_key: AssessmentConversationKey,
139    pub assessment_key: AnalyticsFactKey,
140    pub invocation_key: AnalyticsFactKey,
141    pub occurred_at: DateTime<Utc>,
142    pub outcome: AssessmentOutcome,
143}
144
145#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
146#[serde(deny_unknown_fields)]
147pub struct NormalizedResourceAssociationFact {
148    pub association_key: AnalyticsFactKey,
149    pub invocation_key: AnalyticsFactKey,
150    pub request_key: AnalyticsFactKey,
151    pub occurred_at: DateTime<Utc>,
152    pub attribution: InvocationResourceAttribution,
153}
154
155/// One tool result stored as a typed artifact.
156///
157/// It is joined to the invocation and request it belongs to where those are
158/// known. `source` is the vantage point the platform saw the result from and
159/// `correlation` says whether it was joined by an exact key or inferred.
160#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
161#[serde(deny_unknown_fields)]
162pub struct NormalizedArtifactFact {
163    pub artifact_key: AnalyticsFactKey,
164    pub execution_id: String,
165    #[serde(default, skip_serializing_if = "Option::is_none")]
166    pub invocation_key: Option<AnalyticsFactKey>,
167    #[serde(default, skip_serializing_if = "Option::is_none")]
168    pub request_key: Option<AnalyticsFactKey>,
169    pub occurred_at: DateTime<Utc>,
170    pub consumer: InvocationConsumerIdentity,
171    #[serde(default, skip_serializing_if = "Option::is_none")]
172    pub skill: Option<InvocationSkillIdentity>,
173    pub tool_name: String,
174    #[serde(default, skip_serializing_if = "Option::is_none")]
175    pub server_name: Option<String>,
176    pub artifact_type: String,
177    pub source: crate::mcp::ExecutionSource,
178    pub correlation: crate::mcp::Correlation,
179    pub is_structured: bool,
180    pub has_ui_resource: bool,
181    pub succeeded: bool,
182    pub payload_bytes: Option<u64>,
183    pub findings: u64,
184}
185
186#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
187#[serde(tag = "kind", content = "value", rename_all = "snake_case")]
188pub enum NormalizedAnalyticsFact {
189    Invocation(NormalizedInvocationFact),
190    Request(NormalizedRequestFact),
191    Assessment(NormalizedAssessmentFact),
192    ResourceAssociation(NormalizedResourceAssociationFact),
193    Artifact(Box<NormalizedArtifactFact>),
194}
195
196impl NormalizedAnalyticsFact {
197    pub const fn kind(&self) -> AnalyticsFactKind {
198        match self {
199            Self::Invocation(_) => AnalyticsFactKind::Invocation,
200            Self::Request(_) => AnalyticsFactKind::Request,
201            Self::Assessment(_) => AnalyticsFactKind::Assessment,
202            Self::ResourceAssociation(_) => AnalyticsFactKind::ResourceAssociation,
203            Self::Artifact(_) => AnalyticsFactKind::Artifact,
204        }
205    }
206}
207
208#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
209#[serde(tag = "operation", rename_all = "snake_case")]
210#[expect(
211    clippy::large_enum_variant,
212    reason = "wire contract: a tombstone carries nothing by definition and the fact is matched by value"
213)]
214pub enum AnalyticsChangeOperation {
215    Replace { fact: NormalizedAnalyticsFact },
216    Tombstone,
217}
218
219#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
220#[serde(deny_unknown_fields)]
221pub struct AnalyticsChange {
222    pub change_id: AnalyticsChangeId,
223    pub key: AnalyticsFactKey,
224    pub revision: u64,
225    pub occurred_at: DateTime<Utc>,
226    pub recorded_at: DateTime<Utc>,
227    pub operation: AnalyticsChangeOperation,
228}
229
230impl AnalyticsChange {
231    pub fn validate(&self) -> Result<(), super::FeedbackContractError> {
232        if self.revision == 0 || self.key.source.is_empty() || self.key.source.len() > 128 {
233            return Err(super::FeedbackContractError::Bounds);
234        }
235        if let AnalyticsChangeOperation::Replace { fact } = &self.operation
236            && fact.kind() != self.key.kind
237        {
238            return Err(super::FeedbackContractError::IncompleteManifest);
239        }
240        Ok(())
241    }
242}
systemprompt_models/feedback/analytics.rs

systemprompt_models/feedback/
analytics.rs
