artifactize_core/query/
summary.rs

1use std::{
2    collections::{BTreeMap, BTreeSet},
3    time::Duration,
4};
5
6use serde::Serialize;
7use serde_json::{Value, json};
8use time::OffsetDateTime;
9
10use crate::store::{ExecutionOptions, Request, RequestView, RunView};
11
12/// How a request got a result that another request's execution produced.
13#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
14#[serde(rename_all = "lowercase")]
15pub enum SourceKind {
16    /// A completed local record of the reuse key.
17    Cache,
18    /// The live execution of the reuse key that the request waited for.
19    Joined,
20    /// A record from the remote review store.
21    Remote,
22    /// Current graph evidence, without an execution or reusable record.
23    Derived,
24}
25
26/// The request whose execution produced a reused result.
27#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
28#[serde(rename_all = "camelCase")]
29pub struct Source<'a> {
30    pub run_id: &'a crate::types::RunId,
31    pub request_id: &'a crate::types::RequestId,
32    pub kind: SourceKind,
33}
34
35/// Where a request's result came from: `None` when the request executed itself or has no
36/// result from an execution yet. The one rule behind `source`, `summary.reused`, the text
37/// marker and the monitor.
38pub fn source(request: &Request) -> Option<Source<'_>> {
39    if request.profile.kind() == crate::config::ProfileKind::Dependency {
40        return Some(Source {
41            run_id: &request.run_id,
42            request_id: &request.id,
43            kind: SourceKind::Derived,
44        });
45    }
46    let provenance = request
47        .provenance
48        .as_ref()
49        .filter(|source| source.request_id != request.id)?;
50    let kind = if request.origin.is_some() {
51        SourceKind::Remote
52    } else if request.joined {
53        SourceKind::Joined
54    } else {
55        SourceKind::Cache
56    };
57    Some(Source {
58        run_id: &provenance.run_id,
59        request_id: &provenance.request_id,
60        kind,
61    })
62}
63
64/// The result came from another request's execution, possibly in this Run.
65pub fn reused(request: &Request) -> bool {
66    source(request).is_some_and(|source| source.kind != SourceKind::Derived)
67}
68
69/// A saved request as output: `source` stands for the saved `joined` flag.
70fn with_source(value: &mut Value, request: &Request) {
71    let object = value.as_object_mut().expect("a request is an object");
72    object.remove("joined");
73    object.insert("source".into(), json!(source(request)));
74    // The reference in the text form `session show` and `session send` take.
75    if let (Some(reference), Some(Value::Object(session))) =
76        (&request.session, object.get_mut("session"))
77    {
78        session.insert("ref".into(), json!(reference.to_string()));
79    }
80}
81
82#[derive(Debug, Default, Serialize)]
83#[serde(rename_all = "camelCase")]
84pub struct UsageTotals {
85    pub attempts: u64,
86    pub usage_state: UsageState,
87    pub reported_attempts: u64,
88    pub unreported_attempts: u64,
89    pub usage: BTreeMap<String, u64>,
90}
91#[derive(Debug, Default, Serialize)]
92#[serde(rename_all = "lowercase")]
93pub enum UsageState {
94    #[default]
95    None,
96    Unreported,
97    Reported,
98    Partial,
99}
100#[derive(Debug, Default, Serialize)]
101pub struct Kinds {
102    pub total: u64,
103    pub runtime: u64,
104    pub agent: u64,
105    pub human: u64,
106}
107impl Kinds {
108    fn add(&mut self, kind: crate::config::ProfileKind) {
109        if kind == crate::config::ProfileKind::Dependency {
110            return;
111        }
112        self.total += 1;
113        match kind {
114            crate::config::ProfileKind::Runtime => self.runtime += 1,
115            crate::config::ProfileKind::Agent => self.agent += 1,
116            crate::config::ProfileKind::Human => self.human += 1,
117            crate::config::ProfileKind::Dependency => unreachable!("dependency Evals are derived"),
118        }
119    }
120}
121#[derive(Debug, Default, Serialize)]
122#[serde(rename_all = "camelCase")]
123pub struct Reuses {
124    #[serde(flatten)]
125    pub kinds: Kinds,
126    pub other_profile: u64,
127}
128#[derive(Debug, Serialize)]
129#[serde(rename_all = "camelCase")]
130pub struct RequestSummary<'a> {
131    pub status: crate::types::RequestStatus,
132    #[serde(
133        rename = "wallMs",
134        serialize_with = "crate::config::validation::milliseconds::serialize"
135    )]
136    pub wall_time: Option<Duration>,
137    pub executor_starts: u64,
138    #[serde(flatten)]
139    pub usage: UsageTotals,
140    pub execution_source: Option<&'a crate::store::Provenance>,
141}
142#[derive(Debug, Serialize)]
143#[serde(rename_all = "camelCase")]
144pub struct RunSummary {
145    pub derived: u64,
146    pub counts: BTreeMap<crate::types::RequestStatus, u64>,
147    pub executed: Kinds,
148    pub reused: Reuses,
149    #[serde(
150        rename = "wallMs",
151        serialize_with = "crate::config::validation::milliseconds::serialize"
152    )]
153    pub wall_time: Option<Duration>,
154    pub executor_starts: u64,
155    #[serde(flatten)]
156    pub usage: UsageTotals,
157}
158
159pub fn request_output(view: &RequestView, now: OffsetDateTime) -> Value {
160    let mut value = json!(view);
161    let request = &view.request;
162    with_source(&mut value, request);
163    let summary = request_summary(view, now);
164    value["summary"] = json!(summary);
165    value
166}
167
168pub fn run_output(view: &RunView, now: OffsetDateTime) -> Value {
169    let mut value = json!(view);
170    let requests = value["requests"]
171        .as_array_mut()
172        .expect("serialized requests are an array");
173    for (output, request) in requests.iter_mut().zip(&view.requests) {
174        with_source(output, request);
175    }
176    let (summary, saved) = run_summary(view, now);
177    value["usage"] = json!({"spent":summary.usage.usage,"saved":saved});
178    value["summary"] = json!(summary);
179    value
180}
181
182/// A short name for the profile that produced a request's result: its variant, or its kind
183/// with the Agent backend, model and reasoning.
184pub fn profile_name(profile: &crate::config::StoredProfile, options: &ExecutionOptions) -> String {
185    if let Some(variant) = &options.variant {
186        return variant.to_string();
187    }
188    match profile.kind() {
189        crate::config::ProfileKind::Agent => [
190            options.backend.as_ref().map(|backend| backend.as_str()),
191            options.model.as_deref(),
192            options.reasoning.map(crate::config::Reasoning::as_str),
193        ]
194        .into_iter()
195        .flatten()
196        .collect::<Vec<_>>()
197        .join(" "),
198        kind => kind.name().to_owned(),
199    }
200}
201
202fn local_execution(request: &Request) -> bool {
203    request
204        .provenance
205        .as_ref()
206        .is_some_and(|source| source.request_id == request.id)
207}
208
209fn wall_time(
210    start: crate::types::Timestamp,
211    end: Option<crate::types::Timestamp>,
212    now: OffsetDateTime,
213) -> Option<Duration> {
214    let end = end.map_or(now, crate::types::Timestamp::time);
215    Some(
216        (end - start.time())
217            .max(time::Duration::ZERO)
218            .unsigned_abs(),
219    )
220}
221
222fn usage_totals<'a>(requests: impl IntoIterator<Item = &'a Request>) -> UsageTotals {
223    let (mut attempts, mut reported, mut unreported) = (0, 0, 0);
224    let mut totals = BTreeMap::<String, u64>::new();
225    for request in requests {
226        if let Some(entries) = request.usage.as_ref() {
227            if entries.is_empty() && request.started_at.is_some() {
228                unreported += 1;
229            }
230            for attempt in entries {
231                attempts += 1;
232                if add_usage(&mut totals, attempt) {
233                    reported += 1;
234                } else {
235                    unreported += 1;
236                }
237            }
238        } else if request.execution_id.is_some()
239            && request.profile.kind() != crate::config::ProfileKind::Agent
240        {
241            attempts += 1;
242            unreported += 1;
243        }
244    }
245    UsageTotals {
246        attempts,
247        usage_state: usage_state(reported, unreported),
248        reported_attempts: reported,
249        unreported_attempts: unreported,
250        usage: totals,
251    }
252}
253
254/// Adds one attempt's reported counters; false when it reported none.
255fn add_usage(totals: &mut BTreeMap<String, u64>, attempt: &crate::llm::Attempt) -> bool {
256    for (key, value) in attempt.usage.iter() {
257        let total = totals.entry(key.clone()).or_default();
258        *total = total.saturating_add(*value);
259    }
260    !attempt.usage.is_empty()
261}
262
263fn usage_state(reported: u64, unreported: u64) -> UsageState {
264    match (reported, unreported) {
265        (0, 0) => UsageState::None,
266        (0, _) => UsageState::Unreported,
267        (_, 0) => UsageState::Reported,
268        _ => UsageState::Partial,
269    }
270}
271
272pub fn request_summary(view: &RequestView, now: OffsetDateTime) -> RequestSummary<'_> {
273    let request = &view.request;
274    RequestSummary {
275        status: request.status(),
276        wall_time: wall_time(request.created_at, request.completed_at(), now),
277        executor_starts: u64::from(
278            local_execution(request)
279                && request.started_at.is_some()
280                && request.profile.kind() != crate::config::ProfileKind::Human,
281        ),
282        usage: usage_totals((!reused(request)).then_some(request)),
283        execution_source: request.provenance.as_ref(),
284    }
285}
286
287pub fn run_summary(view: &RunView, now: OffsetDateTime) -> (RunSummary, BTreeMap<String, u64>) {
288    let mut counts = BTreeMap::new();
289    let mut seen = BTreeSet::new();
290    let local = view.requests.iter().filter(|request| {
291        local_execution(request)
292            && request
293                .execution_id
294                .as_ref()
295                .is_some_and(|id| seen.insert(id))
296    });
297    let usage = usage_totals(local);
298    let (mut executed, mut reuses, mut saved) =
299        (Kinds::default(), Reuses::default(), BTreeMap::new());
300    for request in &view.requests {
301        *counts.entry(request.status()).or_default() += 1;
302        if reused(request) {
303            for attempt in request.reused_usage.iter().flatten() {
304                add_usage(&mut saved, attempt);
305            }
306            reuses.other_profile += u64::from(request.profile != request.requested_profile);
307            reuses.kinds.add(request.profile.kind());
308        } else if local_execution(request) {
309            executed.add(request.profile.kind());
310        }
311    }
312    let summary = RunSummary {
313        derived: view
314            .requests
315            .iter()
316            .filter(|request| request.profile.kind() == crate::config::ProfileKind::Dependency)
317            .count() as u64,
318        counts,
319        executed,
320        reused: reuses,
321        wall_time: wall_time(view.run.created_at, view.run.completed_at(), now),
322        executor_starts: view.run.executions_started,
323        usage,
324    };
325    (summary, saved)
326}
327
328impl std::fmt::Display for UsageState {
329    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
330        formatter.write_str(match self {
331            Self::None => "none",
332            Self::Unreported => "unreported",
333            Self::Reported => "reported",
334            Self::Partial => "partial",
335        })
336    }
337}
338
339#[cfg(test)]
340mod tests {
341    use super::*;
342    use serde_json::json;
343    fn view() -> RunView {
344        serde_json::from_value(json!({
345            "id":"run-legacy",
346            "repoPath":"repo",
347            "stateDir":"state",
348            "status":"RUNNING",
349            "createdAt":"2026-01-01T00:00:00Z",
350            "completedAt":null,
351            "selection":{"kind":"all"},
352            "validation":null,
353            "error":null,
354            "requests":[
355                {
356                    "id":"run-legacy-1",
357                    "runId":"run-legacy",
358                    "evalId":"app/check",
359                    "target":"app",
360                    "title":"Check",
361                    "profile":{"kind":"runtime","command":"fixture","args":[]},
362                    "requestedProfile":{
363                        "kind":"runtime",
364                        "command":"fixture",
365                        "args":[],
366                        "timeoutMs":null,
367                    },
368                    "evalDefHash":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
369                    "executionId":"execution-source-1",
370                    "provenance":{
371                        "repoPath":"source",
372                        "runId":"run-source",
373                        "requestId":"run-source-1",
374                        "evalId":"app/check",
375                        "evalDefHash":"dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd",
376                        "completedAt":"2026-01-01T00:00:01Z",
377                    },
378                    "usage":null,
379                    "reusedUsage":[
380                        {
381                            "turn":1,
382                            "attempt":1,
383                            "usage":{"inputTokens":17,"vendorDetail":{"cache":true}},
384                        },
385                    ],
386                    "payload":{},
387                    "references":{},
388                    "deps":[],
389                    "status":"GREEN",
390                    "createdAt":"2026-01-01T00:00:00Z",
391                    "startedAt":null,
392                    "completedAt":"2026-01-01T00:00:01Z",
393                    "cwd":"repo",
394                    "runDir":null,
395                    "argv":null,
396                    "child":null,
397                    "result":{"verdict":"GREEN"},
398                    "error":null,
399                    "errorCode":null,
400                    "blockedReason":null,
401                },
402            ],
403        }))
404        .unwrap()
405    }
406    #[test]
407    fn supplied_time_is_deterministic_and_profile_omission_is_not_null() {
408        let now = "2026-01-01T00:00:05Z"
409            .parse::<crate::types::Timestamp>()
410            .unwrap()
411            .time();
412        let view = view();
413        let output = run_output(&view, now);
414        assert_eq!(output["summary"]["wallMs"], 5000);
415        assert_eq!(output["summary"]["reused"]["otherProfile"], 1);
416        assert_eq!(output["summary"]["reused"]["runtime"], 1);
417        assert_eq!(output["summary"]["attempts"], 0);
418        assert_eq!(output["usage"]["saved"], json!({"inputTokens":17}));
419        assert_eq!(run_output(&view, now), output);
420        assert_eq!(
421            wall_time("2026-01-01T00:00:06Z".parse().unwrap(), None, now),
422            Some(Duration::ZERO)
423        );
424        let request = RequestView {
425            request: view.requests[0].clone(),
426            claim: None,
427            execution: None,
428            definition: None,
429        };
430        assert_eq!(request_output(&request, now)["summary"]["wallMs"], 1000);
431        assert_eq!(
432            serde_json::to_value(&request.request.profile).unwrap(),
433            json!({"kind":"runtime","command":"fixture","args":[]})
434        );
435        assert_eq!(
436            serde_json::to_value(&request.request.requested_profile).unwrap(),
437            json!({"kind":"runtime","command":"fixture","args":[],"timeoutMs":null})
438        );
439    }
440}
artifactize_core/query/summary.rs

artifactize_core/query/
summary.rs
