ic_db/store/query/aggregate/jobs/distinct.rs
1//! Retained tuple DISTINCT counts using the existing exact key-set reducer.
2use super::super::{Accumulator, GroupCount, Query};
3use super::{
4 AggregateJobs, GroupPage, JobHandle, JobStep, grouped::group_binding, pages, prepare, start,
5};
6use crate::{Entity, Error, Timestamp, Value};
7use ic_storage::Memory;
8use sha2::{Digest, Sha256};
9
10impl<E: Entity, M: Memory> Query<'_, E, M> {
11 /// Retain an exact count of projected K tuples, including an absent tuple.
12 /// Uses synchronous `count_distinct` admission and equality without an index.
13 /// Retain at most 32 distinct keys inside the 16 KiB whole-job frame;
14 /// no spill. Query/projection/algorithm bind the job; work/byte limits may vary.
15 /// # Errors
16 /// Refuses unsupported projections, group modifiers, expiry and capacity.
17 /// # Panics
18 /// Supplier mutation failures trap the enclosing message.
19 pub fn start_count_distinct<K: Value>(
20 self,
21 jobs: &mut AggregateJobs<impl Memory>,
22 now: Timestamp,
23 expires: Timestamp,
24 ) -> Result<JobHandle, Error> {
25 let (plan, _) = self.distinct_plan::<K>()?;
26 if self.request.bytes < 8 {
27 return Err(Error::RowTooLarge);
28 }
29 start(
30 jobs,
31 binding(plan.binding),
32 self.db.revision_by_id(E::ID)?,
33 now,
34 expires,
35 vec![0],
36 )
37 }
38
39 /// Advance DISTINCT count under scalar jobs' work/retry/revision/expiry rules.
40 /// Duplicates across steps count once; empty input returns zero, all-absent
41 /// tuples count once. Finish all input before returning a count. `limit`
42 /// never truncates input or relaxes key-set bounds.
43 /// # Errors
44 /// Refuses mismatched queries/projections, malformed saved keys, cardinality,
45 /// output or whole-frame bounds without changing the previous checkpoint.
46 /// # Panics
47 /// Supplier mutation failures trap the enclosing message.
48 pub fn resume_count_distinct<K: Value>(
49 self,
50 jobs: &mut AggregateJobs<impl Memory>,
51 handle: &JobHandle,
52 sequence: u64,
53 now: Timestamp,
54 ) -> Result<JobStep<u64>, Error> {
55 let (plan, mut reducer) = self.distinct_plan::<K>()?;
56 let mut record = jobs.load(handle)?;
57 let bytes = self.request.bytes;
58 let step = prepare(
59 self.db,
60 E::SCHEMA,
61 self.request,
62 &plan,
63 Accumulator::Scalar(&mut reducer),
64 &mut record,
65 binding(plan.binding),
66 sequence,
67 now,
68 )?
69 .finish(bytes, || Ok((reducer.finish()?.into_count()?, 8)))?;
70 jobs.save(handle, &record)?;
71 Ok(step)
72 }
73
74 /// Retain distinct R tuples separately in each K group, including absent tuples.
75 /// Projections may overlap. Uses synchronous `group_count_distinct` admission.
76 /// Keep at most 32 groups and 32 distinct keys per group inside one 16 KiB
77 /// whole-job frame. Key/operand projections, group selection and limit bind
78 /// the job. Work/output-byte limits may vary; no automatic spill.
79 /// # Errors
80 /// Refuses unsupported projections, expiry or occupied job capacity.
81 /// # Panics
82 /// Supplier mutation failures trap the enclosing message.
83 pub fn start_group_count_distinct<K: Value, R: Value>(
84 self,
85 jobs: &mut AggregateJobs<impl Memory>,
86 now: Timestamp,
87 expires: Timestamp,
88 ) -> Result<JobHandle, Error> {
89 if self.request.groups.page.is_some() {
90 return Err(Error::InvalidQuery);
91 }
92 let (plan, reducer) = self.group_distinct_plan::<K, R>()?;
93 let groups = self.admit_groups(&plan, reducer)?;
94 start(
95 jobs,
96 group_binding(&plan, &groups, self.request.rows)?,
97 self.db.revision_by_id(E::ID)?,
98 now,
99 expires,
100 vec![0],
101 )
102 }
103
104 /// Advance grouped DISTINCT counts with grouped jobs' revision/retry/expiry rules.
105 /// Finish all input before filtering, ordering or limiting groups. Empty input
106 /// returns an empty vector; an all-absent tuple contributes one to its group.
107 /// Returns a GroupPage; use `group_page` to page completed selected groups.
108 /// # Errors
109 /// Refuses mismatched bindings, malformed saved sets, group/key cardinality,
110 /// output or frame bounds. Refusal preserves the previous checkpoint; output
111 /// filters and limits cannot conceal oversized state.
112 /// # Panics
113 /// Supplier mutation failures trap the enclosing message.
114 pub fn resume_group_count_distinct<K: Value, R: Value>(
115 self,
116 jobs: &mut AggregateJobs<impl Memory>,
117 handle: &JobHandle,
118 sequence: u64,
119 now: Timestamp,
120 ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
121 let (plan, reducer) = self.group_distinct_plan::<K, R>()?;
122 let mut groups = self.admit_groups(&plan, reducer)?;
123 let expected = group_binding(&plan, &groups, self.request.rows)?;
124 let mut record = jobs.load(handle)?;
125 let page = self.request.groups.page;
126 pages::admit(page, handle, &record)?;
127 let (rows, bytes) = (self.request.rows, self.request.bytes);
128 let step = prepare(
129 self.db,
130 E::SCHEMA,
131 self.request,
132 &plan,
133 Accumulator::Groups(&mut groups),
134 &mut record,
135 expected,
136 sequence,
137 now,
138 )?
139 .finish(bytes, || {
140 pages::count_page(groups, rows, bytes, page, handle)
141 })?;
142 jobs.save(handle, &record)?;
143 Ok(step)
144 }
145}
146
147fn binding(query: [u8; 32]) -> [u8; 32] {
148 let mut hash = Sha256::new();
149 hash.update(b"ic-db:distinct-count-job:v1\0");
150 hash.update(query);
151 hash.finalize().into()
152}