ic_db/store/query/aggregate/jobs/distinct.rs
1//! Retained tuple DISTINCT counts using the existing exact key-set reducer.
2use super::{
3 super::{Accumulator, GroupCount, Query},
4 AggregateJobs, GroupPage, JobHandle, JobStep,
5 grouped::group_binding,
6 pages, prepare, start,
7};
8use crate::{Entity, Error, Timestamp, Value};
9use ic_storage::Memory;
10use sha2::{Digest, Sha256};
11
12impl<E: Entity, M: Memory> Query<'_, E, M> {
13 /// Retain an exact count of projected K tuples, including an absent tuple.
14 /// Uses synchronous `count_distinct` admission and equality without an index.
15 /// Retain at most 32 distinct keys inside the 16 KiB whole-job frame;
16 /// no spill. Query/projection/algorithm bind the job; work/byte limits may vary.
17 /// # Errors
18 /// Refuses unsupported projections, group modifiers, expiry and capacity.
19 /// # Panics
20 /// Supplier mutation failures trap the enclosing message.
21 pub fn start_count_distinct<K: Value>(
22 self,
23 jobs: &mut AggregateJobs<M>,
24 now: Timestamp,
25 expires: Timestamp,
26 ) -> Result<JobHandle, Error> {
27 let (plan, _) = self.distinct_plan::<K>()?;
28 if self.request.bytes < 8 {
29 return Err(Error::RowTooLarge);
30 }
31 start(
32 jobs,
33 binding(plan.binding),
34 self.db.revision_by_id(E::SCHEMA.id)?,
35 now,
36 expires,
37 vec![0],
38 )
39 }
40
41 /// Advance DISTINCT count under scalar jobs' work/retry/revision/expiry rules.
42 /// Duplicates across steps count once; empty input returns zero, all-absent
43 /// tuples count once. Finish all input before returning a count. `limit`
44 /// never truncates input or relaxes key-set bounds.
45 /// # Errors
46 /// Refuses mismatched queries/projections, malformed saved keys, cardinality,
47 /// output or whole-frame bounds without changing the previous checkpoint.
48 /// # Panics
49 /// Supplier mutation failures trap the enclosing message.
50 pub fn resume_count_distinct<K: Value>(
51 self,
52 jobs: &mut AggregateJobs<M>,
53 handle: &JobHandle,
54 sequence: u64,
55 now: Timestamp,
56 ) -> Result<JobStep<u64>, Error> {
57 let (plan, mut reducer) = self.distinct_plan::<K>()?;
58 let mut record = jobs.load(handle)?;
59 let bytes = self.request.bytes;
60 let step = prepare(
61 self.db,
62 E::SCHEMA,
63 self.request,
64 &plan,
65 Accumulator::Scalar(&mut reducer),
66 &mut record,
67 binding(plan.binding),
68 sequence,
69 now,
70 )?
71 .finish(bytes, || Ok((reducer.finish()?.into_count()?, 8)))?;
72 jobs.save(handle, &record)?;
73 Ok(step)
74 }
75
76 /// Retain distinct R tuples separately in each K group, including absent tuples.
77 /// Projections may overlap. Uses synchronous `group_count_distinct` admission.
78 /// Keep at most 32 groups and 32 distinct keys per group inside one 16 KiB
79 /// whole-job frame. Key/operand projections, group selection and limit bind
80 /// the job. Work/output-byte limits may vary; no automatic spill.
81 /// # Errors
82 /// Refuses unsupported projections, expiry or occupied job capacity.
83 /// # Panics
84 /// Supplier mutation failures trap the enclosing message.
85 pub fn start_group_count_distinct<K: Value, R: Value>(
86 self,
87 jobs: &mut AggregateJobs<M>,
88 now: Timestamp,
89 expires: Timestamp,
90 ) -> Result<JobHandle, Error> {
91 if self.request.groups.page.is_some() {
92 return Err(Error::InvalidQuery);
93 }
94 let (plan, reducer) = self.group_distinct_plan::<K, R>()?;
95 let groups = self.admit_groups(&plan, reducer)?;
96 start(
97 jobs,
98 group_binding(&plan, &groups, self.request.rows)?,
99 self.db.revision_by_id(E::SCHEMA.id)?,
100 now,
101 expires,
102 vec![0],
103 )
104 }
105
106 /// Advance grouped DISTINCT counts with grouped jobs' revision/retry/expiry rules.
107 /// Finish all input before filtering, ordering or limiting groups. Empty input
108 /// returns an empty vector; an all-absent tuple contributes one to its group.
109 /// Returns a GroupPage; use `group_page` to page completed selected groups.
110 /// # Errors
111 /// Refuses mismatched bindings, malformed saved sets, group/key cardinality,
112 /// output or frame bounds. Refusal preserves the previous checkpoint; output
113 /// filters and limits cannot conceal oversized state.
114 /// # Panics
115 /// Supplier mutation failures trap the enclosing message.
116 pub fn resume_group_count_distinct<K: Value, R: Value>(
117 self,
118 jobs: &mut AggregateJobs<M>,
119 handle: &JobHandle,
120 sequence: u64,
121 now: Timestamp,
122 ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
123 let (plan, reducer) = self.group_distinct_plan::<K, R>()?;
124 let mut groups = self.admit_groups(&plan, reducer)?;
125 let expected = group_binding(&plan, &groups, self.request.rows)?;
126 let mut record = jobs.load(handle)?;
127 let page = self.request.groups.page;
128 pages::admit(page, handle, &record)?;
129 let (rows, bytes) = (self.request.rows, self.request.bytes);
130 let step = prepare(
131 self.db,
132 E::SCHEMA,
133 self.request,
134 &plan,
135 Accumulator::Groups(&mut groups),
136 &mut record,
137 expected,
138 sequence,
139 now,
140 )?
141 .finish(bytes, || {
142 pages::count_page(groups, rows, bytes, page, handle)
143 })?;
144 jobs.save(handle, &record)?;
145 Ok(step)
146 }
147}
148
149fn binding(query: [u8; 32]) -> [u8; 32] {
150 let mut hash = Sha256::new();
151 hash.update(b"ic-db:distinct-count-job:v1\0");
152 hash.update(query);
153 hash.finalize().into()
154}