artifactize_core/query/
summary.rs

1use std::{
2    collections::{BTreeMap, BTreeSet},
3    time::Duration,
4};
5
6use serde::Serialize;
7use serde_json::{Value, json};
8use time::OffsetDateTime;
9
10use crate::store::{ExecutionOptions, Request, RequestView, RunView};
11
12/// How a request got a result that another request's execution produced.
13#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
14#[serde(rename_all = "lowercase")]
15pub enum SourceKind {
16    /// A completed local record of the reuse key.
17    Cache,
18    /// The live execution of the reuse key that the request waited for.
19    Joined,
20    /// A record from the remote review store.
21    Remote,
22}
23
24/// The request whose execution produced a reused result.
25#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
26#[serde(rename_all = "camelCase")]
27pub struct Source<'a> {
28    pub run_id: &'a crate::types::RunId,
29    pub request_id: &'a crate::types::RequestId,
30    pub kind: SourceKind,
31}
32
33/// Where a request's result came from: `None` when the request executed itself or has no
34/// result from an execution yet. The one rule behind `source`, `summary.reused`, the text
35/// marker and the monitor.
36pub fn source(request: &Request) -> Option<Source<'_>> {
37    let provenance = request
38        .provenance
39        .as_ref()
40        .filter(|source| source.request_id != request.id)?;
41    let kind = if request.origin.is_some() {
42        SourceKind::Remote
43    } else if request.joined {
44        SourceKind::Joined
45    } else {
46        SourceKind::Cache
47    };
48    Some(Source {
49        run_id: &provenance.run_id,
50        request_id: &provenance.request_id,
51        kind,
52    })
53}
54
55/// The result came from another request's execution, possibly in this Run.
56pub fn reused(request: &Request) -> bool {
57    source(request).is_some()
58}
59
60/// A saved request as output: `source` stands for the saved `joined` flag.
61fn with_source(value: &mut Value, request: &Request) {
62    let object = value.as_object_mut().expect("a request is an object");
63    object.remove("joined");
64    object.insert("source".into(), json!(source(request)));
65    // The reference in the text form `session show` and `session send` take.
66    if let (Some(reference), Some(Value::Object(session))) =
67        (&request.session, object.get_mut("session"))
68    {
69        session.insert("ref".into(), json!(reference.to_string()));
70    }
71}
72
73#[derive(Debug, Default, Serialize)]
74#[serde(rename_all = "camelCase")]
75pub struct UsageTotals {
76    pub attempts: u64,
77    pub usage_state: UsageState,
78    pub reported_attempts: u64,
79    pub unreported_attempts: u64,
80    pub usage: BTreeMap<String, u64>,
81}
82#[derive(Debug, Default, Serialize)]
83#[serde(rename_all = "lowercase")]
84pub enum UsageState {
85    #[default]
86    None,
87    Unreported,
88    Reported,
89    Partial,
90}
91#[derive(Debug, Default, Serialize)]
92pub struct Kinds {
93    pub total: u64,
94    pub runtime: u64,
95    pub agent: u64,
96    pub human: u64,
97}
98impl Kinds {
99    fn add(&mut self, kind: crate::config::ProfileKind) {
100        self.total += 1;
101        match kind {
102            crate::config::ProfileKind::Runtime => self.runtime += 1,
103            crate::config::ProfileKind::Agent => self.agent += 1,
104            crate::config::ProfileKind::Human => self.human += 1,
105        }
106    }
107}
108#[derive(Debug, Default, Serialize)]
109#[serde(rename_all = "camelCase")]
110pub struct Reuses {
111    #[serde(flatten)]
112    pub kinds: Kinds,
113    pub other_profile: u64,
114}
115#[derive(Debug, Serialize)]
116#[serde(rename_all = "camelCase")]
117pub struct RequestSummary<'a> {
118    pub status: crate::types::RequestStatus,
119    #[serde(
120        rename = "wallMs",
121        serialize_with = "crate::config::validation::milliseconds::serialize"
122    )]
123    pub wall_time: Option<Duration>,
124    pub executor_starts: u64,
125    #[serde(flatten)]
126    pub usage: UsageTotals,
127    pub execution_source: Option<&'a crate::store::Provenance>,
128}
129#[derive(Debug, Serialize)]
130#[serde(rename_all = "camelCase")]
131pub struct RunSummary {
132    pub counts: BTreeMap<crate::types::RequestStatus, u64>,
133    pub executed: Kinds,
134    pub reused: Reuses,
135    #[serde(
136        rename = "wallMs",
137        serialize_with = "crate::config::validation::milliseconds::serialize"
138    )]
139    pub wall_time: Option<Duration>,
140    pub executor_starts: u64,
141    #[serde(flatten)]
142    pub usage: UsageTotals,
143}
144
145pub fn request_output(view: &RequestView, now: OffsetDateTime) -> Value {
146    let mut value = json!(view);
147    let request = &view.request;
148    with_source(&mut value, request);
149    let summary = request_summary(view, now);
150    value["summary"] = json!(summary);
151    value
152}
153
154pub fn run_output(view: &RunView, now: OffsetDateTime) -> Value {
155    let mut value = json!(view);
156    let requests = value["requests"]
157        .as_array_mut()
158        .expect("serialized requests are an array");
159    for (output, request) in requests.iter_mut().zip(&view.requests) {
160        with_source(output, request);
161    }
162    let (summary, saved) = run_summary(view, now);
163    value["usage"] = json!({"spent":summary.usage.usage,"saved":saved});
164    value["summary"] = json!(summary);
165    value
166}
167
168/// A short name for the profile that produced a request's result: its variant, or its kind
169/// with the Agent backend, model and reasoning.
170pub fn profile_name(profile: &crate::config::StoredProfile, options: &ExecutionOptions) -> String {
171    if let Some(variant) = &options.variant {
172        return variant.to_string();
173    }
174    match profile.kind() {
175        crate::config::ProfileKind::Agent => [
176            options.backend.as_ref().map(|backend| backend.as_str()),
177            options.model.as_deref(),
178            options.reasoning.map(crate::config::Reasoning::as_str),
179        ]
180        .into_iter()
181        .flatten()
182        .collect::<Vec<_>>()
183        .join(" "),
184        kind => kind.name().to_owned(),
185    }
186}
187
188fn local_execution(request: &Request) -> bool {
189    request
190        .provenance
191        .as_ref()
192        .is_some_and(|source| source.request_id == request.id)
193}
194
195fn wall_time(
196    start: crate::types::Timestamp,
197    end: Option<crate::types::Timestamp>,
198    now: OffsetDateTime,
199) -> Option<Duration> {
200    let end = end.map_or(now, crate::types::Timestamp::time);
201    Some(
202        (end - start.time())
203            .max(time::Duration::ZERO)
204            .unsigned_abs(),
205    )
206}
207
208fn usage_totals<'a>(requests: impl IntoIterator<Item = &'a Request>) -> UsageTotals {
209    let (mut attempts, mut reported, mut unreported) = (0, 0, 0);
210    let mut totals = BTreeMap::<String, u64>::new();
211    for request in requests {
212        if let Some(entries) = request.usage.as_ref() {
213            if entries.is_empty() && request.started_at.is_some() {
214                unreported += 1;
215            }
216            for attempt in entries {
217                attempts += 1;
218                if add_usage(&mut totals, attempt) {
219                    reported += 1;
220                } else {
221                    unreported += 1;
222                }
223            }
224        } else if request.execution_id.is_some()
225            && request.profile.kind() != crate::config::ProfileKind::Agent
226        {
227            attempts += 1;
228            unreported += 1;
229        }
230    }
231    UsageTotals {
232        attempts,
233        usage_state: usage_state(reported, unreported),
234        reported_attempts: reported,
235        unreported_attempts: unreported,
236        usage: totals,
237    }
238}
239
240/// Adds one attempt's reported counters; false when it reported none.
241fn add_usage(totals: &mut BTreeMap<String, u64>, attempt: &crate::llm::Attempt) -> bool {
242    for (key, value) in attempt.usage.iter() {
243        let total = totals.entry(key.clone()).or_default();
244        *total = total.saturating_add(*value);
245    }
246    !attempt.usage.is_empty()
247}
248
249fn usage_state(reported: u64, unreported: u64) -> UsageState {
250    match (reported, unreported) {
251        (0, 0) => UsageState::None,
252        (0, _) => UsageState::Unreported,
253        (_, 0) => UsageState::Reported,
254        _ => UsageState::Partial,
255    }
256}
257
258pub fn request_summary(view: &RequestView, now: OffsetDateTime) -> RequestSummary<'_> {
259    let request = &view.request;
260    RequestSummary {
261        status: request.status(),
262        wall_time: wall_time(request.created_at, request.completed_at(), now),
263        executor_starts: u64::from(
264            local_execution(request)
265                && request.started_at.is_some()
266                && request.profile.kind() != crate::config::ProfileKind::Human,
267        ),
268        usage: usage_totals((!reused(request)).then_some(request)),
269        execution_source: request.provenance.as_ref(),
270    }
271}
272
273pub fn run_summary(view: &RunView, now: OffsetDateTime) -> (RunSummary, BTreeMap<String, u64>) {
274    let mut counts = BTreeMap::new();
275    let mut seen = BTreeSet::new();
276    let local = view.requests.iter().filter(|request| {
277        local_execution(request)
278            && request
279                .execution_id
280                .as_ref()
281                .is_some_and(|id| seen.insert(id))
282    });
283    let usage = usage_totals(local);
284    let (mut executed, mut reuses, mut saved) =
285        (Kinds::default(), Reuses::default(), BTreeMap::new());
286    for request in &view.requests {
287        *counts.entry(request.status()).or_default() += 1;
288        if reused(request) {
289            for attempt in request.reused_usage.iter().flatten() {
290                add_usage(&mut saved, attempt);
291            }
292            reuses.other_profile += u64::from(request.profile != request.requested_profile);
293            reuses.kinds.add(request.profile.kind());
294        } else if local_execution(request) {
295            executed.add(request.profile.kind());
296        }
297    }
298    let summary = RunSummary {
299        counts,
300        executed,
301        reused: reuses,
302        wall_time: wall_time(view.run.created_at, view.run.completed_at(), now),
303        executor_starts: view.run.executions_started,
304        usage,
305    };
306    (summary, saved)
307}
308
309impl std::fmt::Display for UsageState {
310    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
311        formatter.write_str(match self {
312            Self::None => "none",
313            Self::Unreported => "unreported",
314            Self::Reported => "reported",
315            Self::Partial => "partial",
316        })
317    }
318}
319
320#[cfg(test)]
321mod tests {
322    use super::*;
323    use serde_json::json;
324    fn view() -> RunView {
325        serde_json::from_value(json!({
326            "id":"run-legacy",
327            "repoPath":"repo",
328            "stateDir":"state",
329            "status":"RUNNING",
330            "createdAt":"2026-01-01T00:00:00Z",
331            "completedAt":null,
332            "selection":{"kind":"all"},
333            "validation":null,
334            "error":null,
335            "requests":[
336                {
337                    "id":"run-legacy-1",
338                    "runId":"run-legacy",
339                    "evalId":"app/check",
340                    "target":"app",
341                    "title":"Check",
342                    "profile":{"kind":"runtime","command":"fixture","args":[]},
343                    "requestedProfile":{
344                        "kind":"runtime",
345                        "command":"fixture",
346                        "args":[],
347                        "timeoutMs":null,
348                    },
349                    "evalDefHash":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
350                    "executionId":"execution-source-1",
351                    "provenance":{
352                        "repoPath":"source",
353                        "runId":"run-source",
354                        "requestId":"run-source-1",
355                        "evalId":"app/check",
356                        "evalDefHash":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
357                        "completedAt":"2026-01-01T00:00:01Z",
358                    },
359                    "usage":null,
360                    "reusedUsage":[
361                        {
362                            "turn":1,
363                            "attempt":1,
364                            "usage":{"inputTokens":17,"vendorDetail":{"cache":true}},
365                        },
366                    ],
367                    "payload":{},
368                    "references":{},
369                    "deps":[],
370                    "status":"GREEN",
371                    "createdAt":"2026-01-01T00:00:00Z",
372                    "startedAt":null,
373                    "completedAt":"2026-01-01T00:00:01Z",
374                    "cwd":"repo",
375                    "runDir":null,
376                    "argv":null,
377                    "child":null,
378                    "result":{"verdict":"GREEN"},
379                    "error":null,
380                    "errorCode":null,
381                    "blockedReason":null,
382                },
383            ],
384        }))
385        .unwrap()
386    }
387    #[test]
388    fn supplied_time_is_deterministic_and_profile_omission_is_not_null() {
389        let now = "2026-01-01T00:00:05Z"
390            .parse::<crate::types::Timestamp>()
391            .unwrap()
392            .time();
393        let view = view();
394        let output = run_output(&view, now);
395        assert_eq!(output["summary"]["wallMs"], 5000);
396        assert_eq!(output["summary"]["reused"]["otherProfile"], 1);
397        assert_eq!(output["summary"]["reused"]["runtime"], 1);
398        assert_eq!(output["summary"]["attempts"], 0);
399        assert_eq!(output["usage"]["saved"], json!({"inputTokens":17}));
400        assert_eq!(run_output(&view, now), output);
401        assert_eq!(
402            wall_time("2026-01-01T00:00:06Z".parse().unwrap(), None, now),
403            Some(Duration::ZERO)
404        );
405        let request = RequestView {
406            request: view.requests[0].clone(),
407            claim: None,
408            execution: None,
409            definition: None,
410        };
411        assert_eq!(request_output(&request, now)["summary"]["wallMs"], 1000);
412        assert_eq!(
413            serde_json::to_value(&request.request.profile).unwrap(),
414            json!({"kind":"runtime","command":"fixture","args":[]})
415        );
416        assert_eq!(
417            serde_json::to_value(&request.request.requested_profile).unwrap(),
418            json!({"kind":"runtime","command":"fixture","args":[],"timeoutMs":null})
419        );
420    }
421}
artifactize_core/query/summary.rs

artifactize_core/query/
summary.rs
