Skip to main content

ic_db/store/query/aggregate/jobs/
grouped.rs

1//! Retained grouped aggregates with bounded complete finalization.
2use super::{
3    super::super::Plan,
4    super::{
5        Accumulator, Completed, GroupCount, GroupOrder, GroupValue, Groups, Query, Reducer,
6        decode_group_count, decode_group_value, numeric::Operation,
7    },
8    AggregateJobs, GroupPage, JobHandle, JobStep, field_binding, pages, prepare, start,
9};
10use crate::{Entity, Error, Timestamp, Value};
11use ic_storage::Memory;
12use sha2::{Digest, Sha256};
13
14pub(super) struct GroupJob<'a> {
15    plan: Plan<'a>,
16    groups: Groups,
17    binding: [u8; 32],
18}
19
20impl<E: Entity, M: Memory> Query<'_, E, M> {
21    /// Retain grouped row counts without scanning input. K is the same exact key
22    /// record as synchronous `group_count`. Group predicates, order and output
23    /// row limit bind the job; work/output-byte allowances may change on resume.
24    /// Keep at most 32 groups and the complete job frame within 16 KiB,
25    /// including progress and encoded key records. No spill or partial output.
26    /// # Errors
27    /// Refuses group/query admission, invalid expiry or occupied job capacity.
28    /// # Panics
29    /// Supplier mutation failures trap the enclosing message.
30    pub fn start_group_count<K: Value>(
31        self,
32        jobs: &mut AggregateJobs<M>,
33        now: Timestamp,
34        expires: Timestamp,
35    ) -> Result<JobHandle, Error> {
36        self.start_group_count_job::<K>(jobs, now, expires, None)
37    }
38
39    /// Retain present-field counts per exact key record. Any top-level stored field
40    /// is accepted; empty values count. Groups containing only absent values remain
41    /// with count zero. Uses `start_group_count`'s bounds and selection binding.
42    /// # Errors
43    /// Refuses unknown fields and ordinary grouped-count admission failures.
44    /// # Panics
45    /// Supplier mutation failures trap the enclosing message.
46    pub fn start_group_count_field<K: Value>(
47        self,
48        field: &str,
49        jobs: &mut AggregateJobs<M>,
50        now: Timestamp,
51        expires: Timestamp,
52    ) -> Result<JobHandle, Error> {
53        self.start_group_count_job::<K>(jobs, now, expires, Some(field))
54    }
55
56    fn start_group_count_job<K: Value>(
57        self,
58        jobs: &mut AggregateJobs<M>,
59        now: Timestamp,
60        expires: Timestamp,
61        field: Option<&str>,
62    ) -> Result<JobHandle, Error> {
63        if self.request.groups.page.is_some() {
64            return Err(Error::InvalidQuery);
65        }
66        let plan = self.aggregate_plan(Some(K::definition()), true)?;
67        self.start_groups(jobs, now, expires, plan, self.count_reducer(field)?, field)
68    }
69
70    /// Advance grouped counts with scalar jobs' revision/retry/work/expiry contract.
71    /// Empty input completes as an empty vector; absent keys form their own group.
72    /// Finalize all input before filtering, ordering and selecting completed groups.
73    /// Use `group_page` to page completed output; otherwise all selected groups must fit.
74    /// # Errors
75    /// Refuses mismatched queries/keys/selection, cardinality or the 16 KiB job
76    /// frame. Group/output bounds apply before saving; a refusal preserves prior
77    /// progress. `limit` never limits input or relaxes retained-state bounds.
78    /// # Panics
79    /// Supplier mutation failures trap the enclosing message.
80    pub fn resume_group_count<K: Value>(
81        self,
82        jobs: &mut AggregateJobs<M>,
83        handle: &JobHandle,
84        sequence: u64,
85        now: Timestamp,
86    ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
87        self.resume_group_count_job::<K>(jobs, handle, sequence, now, None)
88    }
89
90    /// Advance grouped present-field counts with `resume_group_count`'s lifecycle.
91    /// Finish all input before selection; zero-count groups are retained. The field
92    /// binds the job, including required fields whose count equals the row count.
93    /// # Errors
94    /// Refuses changed fields, row-count jobs and ordinary grouped-count failures.
95    /// # Panics
96    /// Supplier mutation failures trap the enclosing message.
97    pub fn resume_group_count_field<K: Value>(
98        self,
99        field: &str,
100        jobs: &mut AggregateJobs<M>,
101        handle: &JobHandle,
102        sequence: u64,
103        now: Timestamp,
104    ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
105        self.resume_group_count_job::<K>(jobs, handle, sequence, now, Some(field))
106    }
107
108    fn resume_group_count_job<K: Value>(
109        self,
110        jobs: &mut AggregateJobs<M>,
111        handle: &JobHandle,
112        sequence: u64,
113        now: Timestamp,
114        field: Option<&str>,
115    ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
116        let plan = self.aggregate_plan(Some(K::definition()), true)?;
117        let job = self.retained_groups(plan, self.count_reducer(field)?, field)?;
118        self.resume_groups(jobs, handle, sequence, now, job, decode_group_count)
119    }
120
121    /// Retain grouped integer/Decimal sums with `group_sum`'s exact result types.
122    /// Uses grouped counts' 32-group/16 KiB frame, revision and expiry bounds.
123    /// Key/result types, operation and completed-group selection bind the job.
124    /// # Errors
125    /// Refuses invalid query/result shapes, expiry or exhausted job capacity.
126    /// # Panics
127    /// Supplier mutation failures trap the enclosing message.
128    pub fn start_group_sum<K: Value, R: Value>(
129        self,
130        jobs: &mut AggregateJobs<M>,
131        now: Timestamp,
132        expires: Timestamp,
133    ) -> Result<JobHandle, Error> {
134        self.start_group_numeric::<K, R>(jobs, now, expires, Operation::Sum)
135    }
136
137    /// Advance grouped sums; all-absent groups have `None`, present zero is a value.
138    /// Finish every group before filtering/order/limit. Overflow in an excluded
139    /// group still refuses. Output and frame refusals preserve the saved checkpoint.
140    /// Returns a GroupPage; use `group_page` to page the completed selected groups.
141    /// # Errors
142    /// Grouped-count resume failures and `group_sum` arithmetic/result errors apply.
143    /// # Panics
144    /// Supplier mutation failures trap the enclosing message.
145    pub fn resume_group_sum<K: Value, R: Value>(
146        self,
147        jobs: &mut AggregateJobs<M>,
148        handle: &JobHandle,
149        sequence: u64,
150        now: Timestamp,
151    ) -> Result<JobStep<GroupPage<GroupValue<K, R>>>, Error> {
152        self.resume_group_numeric::<K, R>(jobs, handle, sequence, now, Operation::Sum)
153    }
154
155    /// Retain grouped averages, saving exact totals and each group's present count.
156    /// Round once at completion to R's Decimal scale, nearest with ties to even.
157    /// Bounds and query binding match `start_group_sum`; no precision fallback.
158    /// # Errors
159    /// Refuses invalid average shapes, expiry or exhausted job capacity.
160    /// # Panics
161    /// Supplier mutation failures trap the enclosing message.
162    pub fn start_group_avg<K: Value, R: Value>(
163        self,
164        jobs: &mut AggregateJobs<M>,
165        now: Timestamp,
166        expires: Timestamp,
167    ) -> Result<JobHandle, Error> {
168        self.start_group_numeric::<K, R>(jobs, now, expires, Operation::Average)
169    }
170
171    /// Advance grouped averages with `resume_group_sum`'s complete-input contract.
172    /// Selection compares completed rounded values. Absent operands do not enter
173    /// the denominator; all-absent groups remain `None`. Never average chunk means.
174    /// # Errors
175    /// Grouped-count resume failures and `group_avg` arithmetic/result errors apply.
176    /// # Panics
177    /// Supplier mutation failures trap the enclosing message.
178    pub fn resume_group_avg<K: Value, R: Value>(
179        self,
180        jobs: &mut AggregateJobs<M>,
181        handle: &JobHandle,
182        sequence: u64,
183        now: Timestamp,
184    ) -> Result<JobStep<GroupPage<GroupValue<K, R>>>, Error> {
185        self.resume_group_numeric::<K, R>(jobs, handle, sequence, now, Operation::Average)
186    }
187
188    fn start_group_numeric<K: Value, R: Value>(
189        self,
190        jobs: &mut AggregateJobs<M>,
191        now: Timestamp,
192        expires: Timestamp,
193        operation: Operation,
194    ) -> Result<JobHandle, Error> {
195        let admitted = self.numeric_plan::<R>(Some(K::definition()), operation)?;
196        self.start_group_value(jobs, now, expires, admitted)
197    }
198
199    pub(super) fn start_group_value(
200        self,
201        jobs: &mut AggregateJobs<M>,
202        now: Timestamp,
203        expires: Timestamp,
204        admitted: (Plan<'_>, Reducer),
205    ) -> Result<JobHandle, Error> {
206        if self.request.groups.page.is_some() {
207            return Err(Error::InvalidQuery);
208        }
209        let (plan, reducer) = admitted;
210        self.start_groups(jobs, now, expires, plan, reducer, None)
211    }
212
213    pub(super) fn start_groups(
214        &self,
215        jobs: &mut AggregateJobs<M>,
216        now: Timestamp,
217        expires: Timestamp,
218        plan: Plan<'_>,
219        reducer: Reducer,
220        field: Option<&str>,
221    ) -> Result<JobHandle, Error> {
222        let mut job = self.retained_groups(plan, reducer, field)?;
223        start(
224            jobs,
225            job.binding,
226            self.db.revision_by_id(E::SCHEMA.id)?,
227            now,
228            expires,
229            &Accumulator::Groups(&mut job.groups),
230        )
231    }
232
233    fn resume_group_numeric<K: Value, R: Value>(
234        self,
235        jobs: &mut AggregateJobs<M>,
236        handle: &JobHandle,
237        sequence: u64,
238        now: Timestamp,
239        operation: Operation,
240    ) -> Result<JobStep<GroupPage<GroupValue<K, R>>>, Error> {
241        let admitted = self.numeric_plan::<R>(Some(K::definition()), operation)?;
242        self.resume_group_value(jobs, handle, sequence, now, admitted)
243    }
244
245    pub(super) fn resume_group_value<K: Value, R: Value>(
246        self,
247        jobs: &mut AggregateJobs<M>,
248        handle: &JobHandle,
249        sequence: u64,
250        now: Timestamp,
251        admitted: (Plan<'_>, Reducer),
252    ) -> Result<JobStep<GroupPage<GroupValue<K, R>>>, Error> {
253        let (plan, reducer) = admitted;
254        let job = self.retained_groups(plan, reducer, None)?;
255        self.resume_groups(jobs, handle, sequence, now, job, decode_group_value)
256    }
257
258    pub(super) fn resume_groups<T>(
259        self,
260        jobs: &mut AggregateJobs<M>,
261        handle: &JobHandle,
262        sequence: u64,
263        now: Timestamp,
264        job: GroupJob<'_>,
265        decode: fn(&[u8], Completed) -> Result<T, Error>,
266    ) -> Result<JobStep<GroupPage<T>>, Error> {
267        let GroupJob {
268            plan,
269            mut groups,
270            binding,
271        } = job;
272        let mut record = jobs.load(handle)?;
273        let page = self.request.groups.page;
274        pages::admit(page, handle, &record)?;
275        let (rows, bytes) = (self.request.rows, self.request.bytes);
276        prepare(
277            self.db,
278            E::SCHEMA,
279            self.request,
280            &plan,
281            Accumulator::Groups(&mut groups),
282            &mut record,
283            binding,
284            sequence,
285            now,
286        )?
287        .finish_and_save(jobs, handle, &record, bytes, || {
288            pages::group_page(groups, rows, bytes, page, handle, decode)
289        })
290    }
291
292    pub(super) fn retained_groups<'p>(
293        &self,
294        plan: Plan<'p>,
295        reducer: Reducer,
296        field: Option<&str>,
297    ) -> Result<GroupJob<'p>, Error> {
298        let groups = self.admit_groups(&plan, reducer)?;
299        let binding = field_binding(group_binding(&plan, &groups, self.request.rows)?, field);
300        Ok(GroupJob {
301            plan,
302            groups,
303            binding,
304        })
305    }
306}
307
308fn group_binding(plan: &Plan<'_>, groups: &Groups, rows: usize) -> Result<[u8; 32], Error> {
309    let mut hash = Sha256::new();
310    match &groups.initial {
311        Reducer::Numeric(_) => {
312            hash.update(b"ic-db:group-numeric-job:v1\0");
313            hash.update(super::binding(plan, &groups.initial)?);
314        }
315        Reducer::Extremum { .. } => {
316            hash.update(b"ic-db:group-extremum-job:v1\0");
317            hash.update(super::binding(plan, &groups.initial)?);
318        }
319        Reducer::Distinct { mask, .. } => {
320            hash.update(b"ic-db:group-distinct-count-job:v1\0");
321            hash.update(mask.to_le_bytes());
322        }
323        Reducer::Count { .. } => hash.update(b"ic-db:group-count-job:v1\0"),
324    }
325    hash.update(plan.binding);
326    hash.update([
327        rows as u8,
328        match groups.selection.order {
329            GroupOrder::Key => 0,
330            GroupOrder::AggregateAscending => 1,
331            GroupOrder::AggregateDescending => 2,
332            GroupOrder::KeyDescending => 3,
333        },
334    ]);
335    hash.update([groups.selection.predicates.len() as u8]);
336    for (comparison, operand, _) in &groups.selection.predicates {
337        hash.update([*comparison as u8]);
338        // Frame admitted operands so binding never relies on implicit widths.
339        let bytes = operand.as_deref().unwrap_or_default();
340        hash.update((bytes.len() as u32).to_le_bytes());
341        hash.update(bytes);
342    }
343    // Preserve conjunction-only bindings; add identity only for actual OR clauses.
344    let clauses = groups
345        .selection
346        .predicates
347        .iter()
348        .enumerate()
349        .fold(0_u8, |mask, (i, (_, _, or_next))| {
350            mask | (u8::from(*or_next) << i)
351        });
352    if clauses != 0 {
353        hash.update(b"group-or-clauses\0");
354        hash.update([clauses]);
355    }
356    Ok(hash.finalize().into())
357}