Skip to main content

ic_db/store/query/aggregate/jobs/
grouped.rs

1//! Retained grouped aggregates with bounded complete finalization.
2use super::super::super::Plan;
3use super::super::{
4    Accumulator, GroupCount, GroupOrder, GroupValue, Groups, Query, Reducer, numeric::Operation,
5};
6use super::{AggregateJobs, GroupPage, JobHandle, JobStep, field_binding, pages, prepare, start};
7use crate::{Entity, Error, Timestamp, Value};
8use ic_storage::Memory;
9use sha2::{Digest, Sha256};
10
11impl<E: Entity, M: Memory> Query<'_, E, M> {
12    /// Retain grouped row counts without scanning input. K is the same exact key
13    /// record as synchronous `group_count`. Group predicates, order and output
14    /// row limit bind the job; work/output-byte allowances may change on resume.
15    /// Keep at most 32 groups and the complete job frame within 16 KiB,
16    /// including progress and encoded key records. No spill or partial output.
17    /// # Errors
18    /// Refuses group/query admission, invalid expiry or occupied job capacity.
19    /// # Panics
20    /// Supplier mutation failures trap the enclosing message.
21    pub fn start_group_count<K: Value>(
22        self,
23        jobs: &mut AggregateJobs<impl Memory>,
24        now: Timestamp,
25        expires: Timestamp,
26    ) -> Result<JobHandle, Error> {
27        self.start_group_count_job::<K>(jobs, now, expires, None)
28    }
29
30    /// Retain present-field counts per exact key record. Any top-level stored field
31    /// is accepted; empty values count. Groups containing only absent values remain
32    /// with count zero. Uses `start_group_count`'s bounds and selection binding.
33    /// # Errors
34    /// Refuses unknown fields and ordinary grouped-count admission failures.
35    /// # Panics
36    /// Supplier mutation failures trap the enclosing message.
37    pub fn start_group_count_field<K: Value>(
38        self,
39        field: &str,
40        jobs: &mut AggregateJobs<impl Memory>,
41        now: Timestamp,
42        expires: Timestamp,
43    ) -> Result<JobHandle, Error> {
44        self.start_group_count_job::<K>(jobs, now, expires, Some(field))
45    }
46
47    fn start_group_count_job<K: Value>(
48        self,
49        jobs: &mut AggregateJobs<impl Memory>,
50        now: Timestamp,
51        expires: Timestamp,
52        field: Option<&str>,
53    ) -> Result<JobHandle, Error> {
54        if self.request.groups.page.is_some() {
55            return Err(Error::InvalidQuery);
56        }
57        let plan = self.aggregate_plan(Some(K::definition()), true)?;
58        let groups = self.admit_groups(&plan, self.count_reducer(field)?)?;
59        start(
60            jobs,
61            field_binding(group_binding(&plan, &groups, self.request.rows)?, field),
62            self.db.revision_by_id(E::ID)?,
63            now,
64            expires,
65            vec![0],
66        )
67    }
68
69    /// Advance grouped counts with scalar jobs' revision/retry/work/expiry contract.
70    /// Empty input completes as an empty vector; absent keys form their own group.
71    /// Finalize all input before filtering, ordering and selecting completed groups.
72    /// Use `group_page` to page completed output; otherwise all selected groups must fit.
73    /// # Errors
74    /// Refuses mismatched queries/keys/selection, cardinality or the 16 KiB job
75    /// frame. Group/output bounds apply before saving; a refusal preserves prior
76    /// progress. `limit` never limits input or relaxes retained-state bounds.
77    /// # Panics
78    /// Supplier mutation failures trap the enclosing message.
79    pub fn resume_group_count<K: Value>(
80        self,
81        jobs: &mut AggregateJobs<impl Memory>,
82        handle: &JobHandle,
83        sequence: u64,
84        now: Timestamp,
85    ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
86        self.resume_group_count_job::<K>(jobs, handle, sequence, now, None)
87    }
88
89    /// Advance grouped present-field counts with `resume_group_count`'s lifecycle.
90    /// Finish all input before selection; zero-count groups are retained. The field
91    /// binds the job, including required fields whose count equals the row count.
92    /// # Errors
93    /// Refuses changed fields, row-count jobs and ordinary grouped-count failures.
94    /// # Panics
95    /// Supplier mutation failures trap the enclosing message.
96    pub fn resume_group_count_field<K: Value>(
97        self,
98        field: &str,
99        jobs: &mut AggregateJobs<impl Memory>,
100        handle: &JobHandle,
101        sequence: u64,
102        now: Timestamp,
103    ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
104        self.resume_group_count_job::<K>(jobs, handle, sequence, now, Some(field))
105    }
106
107    fn resume_group_count_job<K: Value>(
108        self,
109        jobs: &mut AggregateJobs<impl Memory>,
110        handle: &JobHandle,
111        sequence: u64,
112        now: Timestamp,
113        field: Option<&str>,
114    ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
115        let plan = self.aggregate_plan(Some(K::definition()), true)?;
116        let mut groups = self.admit_groups(&plan, self.count_reducer(field)?)?;
117        let expected = field_binding(group_binding(&plan, &groups, self.request.rows)?, field);
118        let mut record = jobs.load(handle)?;
119        let page = self.request.groups.page;
120        pages::admit(page, handle, &record)?;
121        let (rows, bytes) = (self.request.rows, self.request.bytes);
122        let step = prepare(
123            self.db,
124            E::SCHEMA,
125            self.request,
126            &plan,
127            Accumulator::Groups(&mut groups),
128            &mut record,
129            expected,
130            sequence,
131            now,
132        )?
133        .finish(bytes, || {
134            pages::count_page(groups, rows, bytes, page, handle)
135        })?;
136        jobs.save(handle, &record)?;
137        Ok(step)
138    }
139
140    /// Retain grouped integer/Decimal sums with `group_sum`'s exact result types.
141    /// Uses grouped counts' 32-group/16 KiB frame, revision and expiry bounds.
142    /// Key/result types, operation and completed-group selection bind the job.
143    /// # Errors
144    /// Refuses invalid query/result shapes, expiry or exhausted job capacity.
145    /// # Panics
146    /// Supplier mutation failures trap the enclosing message.
147    pub fn start_group_sum<K: Value, R: Value>(
148        self,
149        jobs: &mut AggregateJobs<impl Memory>,
150        now: Timestamp,
151        expires: Timestamp,
152    ) -> Result<JobHandle, Error> {
153        self.start_group_numeric::<K, R>(jobs, now, expires, Operation::Sum)
154    }
155
156    /// Advance grouped sums; all-absent groups have `None`, present zero is a value.
157    /// Finish every group before filtering/order/limit. Overflow in an excluded
158    /// group still refuses. Output and frame refusals preserve the saved checkpoint.
159    /// Returns a GroupPage; use `group_page` to page the completed selected groups.
160    /// # Errors
161    /// Grouped-count resume failures and `group_sum` arithmetic/result errors apply.
162    /// # Panics
163    /// Supplier mutation failures trap the enclosing message.
164    pub fn resume_group_sum<K: Value, R: Value>(
165        self,
166        jobs: &mut AggregateJobs<impl Memory>,
167        handle: &JobHandle,
168        sequence: u64,
169        now: Timestamp,
170    ) -> Result<JobStep<GroupPage<GroupValue<K, R>>>, Error> {
171        self.resume_group_numeric::<K, R>(jobs, handle, sequence, now, Operation::Sum)
172    }
173
174    /// Retain grouped averages, saving exact totals and each group's present count.
175    /// Round once at completion to R's Decimal scale, nearest with ties to even.
176    /// Bounds and query binding match `start_group_sum`; no precision fallback.
177    /// # Errors
178    /// Refuses invalid average shapes, expiry or exhausted job capacity.
179    /// # Panics
180    /// Supplier mutation failures trap the enclosing message.
181    pub fn start_group_avg<K: Value, R: Value>(
182        self,
183        jobs: &mut AggregateJobs<impl Memory>,
184        now: Timestamp,
185        expires: Timestamp,
186    ) -> Result<JobHandle, Error> {
187        self.start_group_numeric::<K, R>(jobs, now, expires, Operation::Average)
188    }
189
190    /// Advance grouped averages with `resume_group_sum`'s complete-input contract.
191    /// Selection compares completed rounded values. Absent operands do not enter
192    /// the denominator; all-absent groups remain `None`. Never average chunk means.
193    /// # Errors
194    /// Grouped-count resume failures and `group_avg` arithmetic/result errors apply.
195    /// # Panics
196    /// Supplier mutation failures trap the enclosing message.
197    pub fn resume_group_avg<K: Value, R: Value>(
198        self,
199        jobs: &mut AggregateJobs<impl Memory>,
200        handle: &JobHandle,
201        sequence: u64,
202        now: Timestamp,
203    ) -> Result<JobStep<GroupPage<GroupValue<K, R>>>, Error> {
204        self.resume_group_numeric::<K, R>(jobs, handle, sequence, now, Operation::Average)
205    }
206
207    fn start_group_numeric<K: Value, R: Value>(
208        self,
209        jobs: &mut AggregateJobs<impl Memory>,
210        now: Timestamp,
211        expires: Timestamp,
212        operation: Operation,
213    ) -> Result<JobHandle, Error> {
214        let admitted = self.numeric_plan::<R>(Some(K::definition()), operation)?;
215        self.start_group_value(jobs, now, expires, admitted)
216    }
217
218    pub(super) fn start_group_value(
219        self,
220        jobs: &mut AggregateJobs<impl Memory>,
221        now: Timestamp,
222        expires: Timestamp,
223        admitted: (Plan<'_>, Reducer),
224    ) -> Result<JobHandle, Error> {
225        if self.request.groups.page.is_some() {
226            return Err(Error::InvalidQuery);
227        }
228        let (plan, reducer) = admitted;
229        let groups = self.admit_groups(&plan, reducer)?;
230        start(
231            jobs,
232            group_binding(&plan, &groups, self.request.rows)?,
233            self.db.revision_by_id(E::ID)?,
234            now,
235            expires,
236            vec![0],
237        )
238    }
239
240    fn resume_group_numeric<K: Value, R: Value>(
241        self,
242        jobs: &mut AggregateJobs<impl Memory>,
243        handle: &JobHandle,
244        sequence: u64,
245        now: Timestamp,
246        operation: Operation,
247    ) -> Result<JobStep<GroupPage<GroupValue<K, R>>>, Error> {
248        let admitted = self.numeric_plan::<R>(Some(K::definition()), operation)?;
249        self.resume_group_value(jobs, handle, sequence, now, admitted)
250    }
251
252    pub(super) fn resume_group_value<K: Value, R: Value>(
253        self,
254        jobs: &mut AggregateJobs<impl Memory>,
255        handle: &JobHandle,
256        sequence: u64,
257        now: Timestamp,
258        admitted: (Plan<'_>, Reducer),
259    ) -> Result<JobStep<GroupPage<GroupValue<K, R>>>, Error> {
260        let (plan, reducer) = admitted;
261        let mut groups = self.admit_groups(&plan, reducer)?;
262        let expected = group_binding(&plan, &groups, self.request.rows)?;
263        let mut record = jobs.load(handle)?;
264        let page = self.request.groups.page;
265        pages::admit(page, handle, &record)?;
266        let (rows, bytes) = (self.request.rows, self.request.bytes);
267        let step = prepare(
268            self.db,
269            E::SCHEMA,
270            self.request,
271            &plan,
272            Accumulator::Groups(&mut groups),
273            &mut record,
274            expected,
275            sequence,
276            now,
277        )?
278        .finish(bytes, || {
279            pages::value_page(groups, rows, bytes, page, handle)
280        })?;
281        jobs.save(handle, &record)?;
282        Ok(step)
283    }
284}
285pub(super) fn group_binding(
286    plan: &Plan<'_>,
287    groups: &Groups,
288    rows: usize,
289) -> Result<[u8; 32], Error> {
290    let mut hash = Sha256::new();
291    if matches!(groups.initial, Reducer::Numeric(_)) {
292        hash.update(b"ic-db:group-numeric-job:v1\0");
293        // Reuse the scalar numeric identity for operand, operation and result scale.
294        hash.update(super::binding(plan, &groups.initial)?);
295    } else if matches!(groups.initial, Reducer::Extremum { .. }) {
296        hash.update(b"ic-db:group-extremum-job:v1\0");
297        hash.update(super::binding(plan, &groups.initial)?);
298    } else if let Reducer::Distinct { mask, .. } = groups.initial {
299        hash.update(b"ic-db:group-distinct-count-job:v1\0");
300        hash.update(mask.to_le_bytes());
301    } else {
302        // Preserve existing grouped-count identities and payloads.
303        hash.update(b"ic-db:group-count-job:v1\0");
304    }
305    hash.update(plan.binding);
306    hash.update([
307        rows as u8,
308        match groups.selection.order {
309            GroupOrder::Key => 0,
310            GroupOrder::AggregateAscending => 1,
311            GroupOrder::AggregateDescending => 2,
312            GroupOrder::KeyDescending => 3,
313        },
314    ]);
315    hash.update([groups.selection.predicates.len() as u8]);
316    for (comparison, operand, _) in &groups.selection.predicates {
317        hash.update([*comparison as u8]);
318        // Frame admitted operands so binding never relies on implicit widths.
319        let bytes = operand.as_deref().unwrap_or_default();
320        hash.update((bytes.len() as u32).to_le_bytes());
321        hash.update(bytes);
322    }
323    // Preserve conjunction-only bindings; add identity only for actual OR clauses.
324    let clauses = groups
325        .selection
326        .predicates
327        .iter()
328        .enumerate()
329        .fold(0_u8, |mask, (i, (_, _, or_next))| {
330            mask | (u8::from(*or_next) << i)
331        });
332    if clauses != 0 {
333        hash.update(b"group-or-clauses\0");
334        hash.update([clauses]);
335    }
336    Ok(hash.finalize().into())
337}