Skip to main content

ic_db/store/query/aggregate/jobs/
grouped.rs

1//! Retained grouped aggregates with bounded complete finalization.
2use super::{
3    super::super::Plan,
4    super::{
5        Accumulator, GroupCount, GroupOrder, GroupValue, Groups, Query, Reducer, numeric::Operation,
6    },
7    AggregateJobs, GroupPage, JobHandle, JobStep, field_binding, pages, prepare, start,
8};
9use crate::{Entity, Error, Timestamp, Value};
10use ic_storage::Memory;
11use sha2::{Digest, Sha256};
12
13impl<E: Entity, M: Memory> Query<'_, E, M> {
14    /// Retain grouped row counts without scanning input. K is the same exact key
15    /// record as synchronous `group_count`. Group predicates, order and output
16    /// row limit bind the job; work/output-byte allowances may change on resume.
17    /// Keep at most 32 groups and the complete job frame within 16 KiB,
18    /// including progress and encoded key records. No spill or partial output.
19    /// # Errors
20    /// Refuses group/query admission, invalid expiry or occupied job capacity.
21    /// # Panics
22    /// Supplier mutation failures trap the enclosing message.
23    pub fn start_group_count<K: Value>(
24        self,
25        jobs: &mut AggregateJobs<M>,
26        now: Timestamp,
27        expires: Timestamp,
28    ) -> Result<JobHandle, Error> {
29        self.start_group_count_job::<K>(jobs, now, expires, None)
30    }
31
32    /// Retain present-field counts per exact key record. Any top-level stored field
33    /// is accepted; empty values count. Groups containing only absent values remain
34    /// with count zero. Uses `start_group_count`'s bounds and selection binding.
35    /// # Errors
36    /// Refuses unknown fields and ordinary grouped-count admission failures.
37    /// # Panics
38    /// Supplier mutation failures trap the enclosing message.
39    pub fn start_group_count_field<K: Value>(
40        self,
41        field: &str,
42        jobs: &mut AggregateJobs<M>,
43        now: Timestamp,
44        expires: Timestamp,
45    ) -> Result<JobHandle, Error> {
46        self.start_group_count_job::<K>(jobs, now, expires, Some(field))
47    }
48
49    fn start_group_count_job<K: Value>(
50        self,
51        jobs: &mut AggregateJobs<M>,
52        now: Timestamp,
53        expires: Timestamp,
54        field: Option<&str>,
55    ) -> Result<JobHandle, Error> {
56        if self.request.groups.page.is_some() {
57            return Err(Error::InvalidQuery);
58        }
59        let plan = self.aggregate_plan(Some(K::definition()), true)?;
60        let groups = self.admit_groups(&plan, self.count_reducer(field)?)?;
61        start(
62            jobs,
63            field_binding(group_binding(&plan, &groups, self.request.rows)?, field),
64            self.db.revision_by_id(E::SCHEMA.id)?,
65            now,
66            expires,
67            vec![0],
68        )
69    }
70
71    /// Advance grouped counts with scalar jobs' revision/retry/work/expiry contract.
72    /// Empty input completes as an empty vector; absent keys form their own group.
73    /// Finalize all input before filtering, ordering and selecting completed groups.
74    /// Use `group_page` to page completed output; otherwise all selected groups must fit.
75    /// # Errors
76    /// Refuses mismatched queries/keys/selection, cardinality or the 16 KiB job
77    /// frame. Group/output bounds apply before saving; a refusal preserves prior
78    /// progress. `limit` never limits input or relaxes retained-state bounds.
79    /// # Panics
80    /// Supplier mutation failures trap the enclosing message.
81    pub fn resume_group_count<K: Value>(
82        self,
83        jobs: &mut AggregateJobs<M>,
84        handle: &JobHandle,
85        sequence: u64,
86        now: Timestamp,
87    ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
88        self.resume_group_count_job::<K>(jobs, handle, sequence, now, None)
89    }
90
91    /// Advance grouped present-field counts with `resume_group_count`'s lifecycle.
92    /// Finish all input before selection; zero-count groups are retained. The field
93    /// binds the job, including required fields whose count equals the row count.
94    /// # Errors
95    /// Refuses changed fields, row-count jobs and ordinary grouped-count failures.
96    /// # Panics
97    /// Supplier mutation failures trap the enclosing message.
98    pub fn resume_group_count_field<K: Value>(
99        self,
100        field: &str,
101        jobs: &mut AggregateJobs<M>,
102        handle: &JobHandle,
103        sequence: u64,
104        now: Timestamp,
105    ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
106        self.resume_group_count_job::<K>(jobs, handle, sequence, now, Some(field))
107    }
108
109    fn resume_group_count_job<K: Value>(
110        self,
111        jobs: &mut AggregateJobs<M>,
112        handle: &JobHandle,
113        sequence: u64,
114        now: Timestamp,
115        field: Option<&str>,
116    ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
117        let plan = self.aggregate_plan(Some(K::definition()), true)?;
118        let mut groups = self.admit_groups(&plan, self.count_reducer(field)?)?;
119        let expected = field_binding(group_binding(&plan, &groups, self.request.rows)?, field);
120        let mut record = jobs.load(handle)?;
121        let page = self.request.groups.page;
122        pages::admit(page, handle, &record)?;
123        let (rows, bytes) = (self.request.rows, self.request.bytes);
124        let step = prepare(
125            self.db,
126            E::SCHEMA,
127            self.request,
128            &plan,
129            Accumulator::Groups(&mut groups),
130            &mut record,
131            expected,
132            sequence,
133            now,
134        )?
135        .finish(bytes, || {
136            pages::count_page(groups, rows, bytes, page, handle)
137        })?;
138        jobs.save(handle, &record)?;
139        Ok(step)
140    }
141
142    /// Retain grouped integer/Decimal sums with `group_sum`'s exact result types.
143    /// Uses grouped counts' 32-group/16 KiB frame, revision and expiry bounds.
144    /// Key/result types, operation and completed-group selection bind the job.
145    /// # Errors
146    /// Refuses invalid query/result shapes, expiry or exhausted job capacity.
147    /// # Panics
148    /// Supplier mutation failures trap the enclosing message.
149    pub fn start_group_sum<K: Value, R: Value>(
150        self,
151        jobs: &mut AggregateJobs<M>,
152        now: Timestamp,
153        expires: Timestamp,
154    ) -> Result<JobHandle, Error> {
155        self.start_group_numeric::<K, R>(jobs, now, expires, Operation::Sum)
156    }
157
158    /// Advance grouped sums; all-absent groups have `None`, present zero is a value.
159    /// Finish every group before filtering/order/limit. Overflow in an excluded
160    /// group still refuses. Output and frame refusals preserve the saved checkpoint.
161    /// Returns a GroupPage; use `group_page` to page the completed selected groups.
162    /// # Errors
163    /// Grouped-count resume failures and `group_sum` arithmetic/result errors apply.
164    /// # Panics
165    /// Supplier mutation failures trap the enclosing message.
166    pub fn resume_group_sum<K: Value, R: Value>(
167        self,
168        jobs: &mut AggregateJobs<M>,
169        handle: &JobHandle,
170        sequence: u64,
171        now: Timestamp,
172    ) -> Result<JobStep<GroupPage<GroupValue<K, R>>>, Error> {
173        self.resume_group_numeric::<K, R>(jobs, handle, sequence, now, Operation::Sum)
174    }
175
176    /// Retain grouped averages, saving exact totals and each group's present count.
177    /// Round once at completion to R's Decimal scale, nearest with ties to even.
178    /// Bounds and query binding match `start_group_sum`; no precision fallback.
179    /// # Errors
180    /// Refuses invalid average shapes, expiry or exhausted job capacity.
181    /// # Panics
182    /// Supplier mutation failures trap the enclosing message.
183    pub fn start_group_avg<K: Value, R: Value>(
184        self,
185        jobs: &mut AggregateJobs<M>,
186        now: Timestamp,
187        expires: Timestamp,
188    ) -> Result<JobHandle, Error> {
189        self.start_group_numeric::<K, R>(jobs, now, expires, Operation::Average)
190    }
191
192    /// Advance grouped averages with `resume_group_sum`'s complete-input contract.
193    /// Selection compares completed rounded values. Absent operands do not enter
194    /// the denominator; all-absent groups remain `None`. Never average chunk means.
195    /// # Errors
196    /// Grouped-count resume failures and `group_avg` arithmetic/result errors apply.
197    /// # Panics
198    /// Supplier mutation failures trap the enclosing message.
199    pub fn resume_group_avg<K: Value, R: Value>(
200        self,
201        jobs: &mut AggregateJobs<M>,
202        handle: &JobHandle,
203        sequence: u64,
204        now: Timestamp,
205    ) -> Result<JobStep<GroupPage<GroupValue<K, R>>>, Error> {
206        self.resume_group_numeric::<K, R>(jobs, handle, sequence, now, Operation::Average)
207    }
208
209    fn start_group_numeric<K: Value, R: Value>(
210        self,
211        jobs: &mut AggregateJobs<M>,
212        now: Timestamp,
213        expires: Timestamp,
214        operation: Operation,
215    ) -> Result<JobHandle, Error> {
216        let admitted = self.numeric_plan::<R>(Some(K::definition()), operation)?;
217        self.start_group_value(jobs, now, expires, admitted)
218    }
219
220    pub(super) fn start_group_value(
221        self,
222        jobs: &mut AggregateJobs<M>,
223        now: Timestamp,
224        expires: Timestamp,
225        admitted: (Plan<'_>, Reducer),
226    ) -> Result<JobHandle, Error> {
227        if self.request.groups.page.is_some() {
228            return Err(Error::InvalidQuery);
229        }
230        let (plan, reducer) = admitted;
231        let groups = self.admit_groups(&plan, reducer)?;
232        start(
233            jobs,
234            group_binding(&plan, &groups, self.request.rows)?,
235            self.db.revision_by_id(E::SCHEMA.id)?,
236            now,
237            expires,
238            vec![0],
239        )
240    }
241
242    fn resume_group_numeric<K: Value, R: Value>(
243        self,
244        jobs: &mut AggregateJobs<M>,
245        handle: &JobHandle,
246        sequence: u64,
247        now: Timestamp,
248        operation: Operation,
249    ) -> Result<JobStep<GroupPage<GroupValue<K, R>>>, Error> {
250        let admitted = self.numeric_plan::<R>(Some(K::definition()), operation)?;
251        self.resume_group_value(jobs, handle, sequence, now, admitted)
252    }
253
254    pub(super) fn resume_group_value<K: Value, R: Value>(
255        self,
256        jobs: &mut AggregateJobs<M>,
257        handle: &JobHandle,
258        sequence: u64,
259        now: Timestamp,
260        admitted: (Plan<'_>, Reducer),
261    ) -> Result<JobStep<GroupPage<GroupValue<K, R>>>, Error> {
262        let (plan, reducer) = admitted;
263        let mut groups = self.admit_groups(&plan, reducer)?;
264        let expected = group_binding(&plan, &groups, self.request.rows)?;
265        let mut record = jobs.load(handle)?;
266        let page = self.request.groups.page;
267        pages::admit(page, handle, &record)?;
268        let (rows, bytes) = (self.request.rows, self.request.bytes);
269        let step = prepare(
270            self.db,
271            E::SCHEMA,
272            self.request,
273            &plan,
274            Accumulator::Groups(&mut groups),
275            &mut record,
276            expected,
277            sequence,
278            now,
279        )?
280        .finish(bytes, || {
281            pages::value_page(groups, rows, bytes, page, handle)
282        })?;
283        jobs.save(handle, &record)?;
284        Ok(step)
285    }
286}
287pub(super) fn group_binding(
288    plan: &Plan<'_>,
289    groups: &Groups,
290    rows: usize,
291) -> Result<[u8; 32], Error> {
292    let mut hash = Sha256::new();
293    if matches!(groups.initial, Reducer::Numeric(_)) {
294        hash.update(b"ic-db:group-numeric-job:v1\0");
295        // Reuse the scalar numeric identity for operand, operation and result scale.
296        hash.update(super::binding(plan, &groups.initial)?);
297    } else if matches!(groups.initial, Reducer::Extremum { .. }) {
298        hash.update(b"ic-db:group-extremum-job:v1\0");
299        hash.update(super::binding(plan, &groups.initial)?);
300    } else if let Reducer::Distinct { mask, .. } = groups.initial {
301        hash.update(b"ic-db:group-distinct-count-job:v1\0");
302        hash.update(mask.to_le_bytes());
303    } else {
304        // Preserve existing grouped-count identities and payloads.
305        hash.update(b"ic-db:group-count-job:v1\0");
306    }
307    hash.update(plan.binding);
308    hash.update([
309        rows as u8,
310        match groups.selection.order {
311            GroupOrder::Key => 0,
312            GroupOrder::AggregateAscending => 1,
313            GroupOrder::AggregateDescending => 2,
314            GroupOrder::KeyDescending => 3,
315        },
316    ]);
317    hash.update([groups.selection.predicates.len() as u8]);
318    for (comparison, operand, _) in &groups.selection.predicates {
319        hash.update([*comparison as u8]);
320        // Frame admitted operands so binding never relies on implicit widths.
321        let bytes = operand.as_deref().unwrap_or_default();
322        hash.update((bytes.len() as u32).to_le_bytes());
323        hash.update(bytes);
324    }
325    // Preserve conjunction-only bindings; add identity only for actual OR clauses.
326    let clauses = groups
327        .selection
328        .predicates
329        .iter()
330        .enumerate()
331        .fold(0_u8, |mask, (i, (_, _, or_next))| {
332            mask | (u8::from(*or_next) << i)
333        });
334    if clauses != 0 {
335        hash.update(b"group-or-clauses\0");
336        hash.update([clauses]);
337    }
338    Ok(hash.finalize().into())
339}