ic_db/store/query/aggregate/jobs/distinct.rs
1//! Retained tuple DISTINCT counts using the existing exact key-set reducer.
2use super::{
3 super::{Accumulator, GroupCount, Query, decode_group_count},
4 AggregateJobs, GroupPage, JobHandle, JobStep, prepare, start,
5};
6use crate::{Entity, Error, Timestamp, Value};
7use ic_storage::Memory;
8use sha2::{Digest, Sha256};
9
10impl<E: Entity, M: Memory> Query<'_, E, M> {
11 /// Retain an exact count of projected K tuples, including an absent tuple.
12 /// Uses synchronous `count_distinct` admission and equality without an index.
13 /// Retain at most 32 distinct keys inside the 16 KiB whole-job frame;
14 /// no spill. Query/projection/algorithm bind the job; work/byte limits may vary.
15 /// # Errors
16 /// Refuses unsupported projections, group modifiers, expiry and capacity.
17 /// # Panics
18 /// Supplier mutation failures trap the enclosing message.
19 pub fn start_count_distinct<K: Value>(
20 self,
21 jobs: &mut AggregateJobs<M>,
22 now: Timestamp,
23 expires: Timestamp,
24 ) -> Result<JobHandle, Error> {
25 let (plan, mut reducer) = self.distinct_plan::<K>()?;
26 if self.request.bytes < 8 {
27 return Err(Error::RowTooLarge);
28 }
29 start(
30 jobs,
31 binding(plan.binding),
32 self.db.revision_by_id(E::SCHEMA.id)?,
33 now,
34 expires,
35 &Accumulator::Scalar(&mut reducer),
36 )
37 }
38
39 /// Advance DISTINCT count under scalar jobs' work/retry/revision/expiry rules.
40 /// Duplicates across steps count once; empty input returns zero, all-absent
41 /// tuples count once. Finish all input before returning a count. `limit`
42 /// never truncates input or relaxes key-set bounds.
43 /// # Errors
44 /// Refuses mismatched queries/projections, malformed saved keys, cardinality,
45 /// output or whole-frame bounds without changing the previous checkpoint.
46 /// # Panics
47 /// Supplier mutation failures trap the enclosing message.
48 pub fn resume_count_distinct<K: Value>(
49 self,
50 jobs: &mut AggregateJobs<M>,
51 handle: &JobHandle,
52 sequence: u64,
53 now: Timestamp,
54 ) -> Result<JobStep<u64>, Error> {
55 let (plan, mut reducer) = self.distinct_plan::<K>()?;
56 let mut record = jobs.load(handle)?;
57 let bytes = self.request.bytes;
58 prepare(
59 self.db,
60 E::SCHEMA,
61 self.request,
62 &plan,
63 Accumulator::Scalar(&mut reducer),
64 &mut record,
65 binding(plan.binding),
66 sequence,
67 now,
68 )?
69 .finish_and_save(jobs, handle, &record, bytes, || {
70 Ok((reducer.finish()?.into_count()?, 8))
71 })
72 }
73
74 /// Retain distinct R tuples separately in each K group, including absent tuples.
75 /// Projections may overlap. Uses synchronous `group_count_distinct` admission.
76 /// Keep at most 32 groups and 32 distinct keys per group inside one 16 KiB
77 /// whole-job frame. Key/operand projections, group selection and limit bind
78 /// the job. Work/output-byte limits may vary; no automatic spill.
79 /// # Errors
80 /// Refuses unsupported projections, expiry or occupied job capacity.
81 /// # Panics
82 /// Supplier mutation failures trap the enclosing message.
83 pub fn start_group_count_distinct<K: Value, R: Value>(
84 self,
85 jobs: &mut AggregateJobs<M>,
86 now: Timestamp,
87 expires: Timestamp,
88 ) -> Result<JobHandle, Error> {
89 if self.request.groups.page.is_some() {
90 return Err(Error::InvalidQuery);
91 }
92 let (plan, reducer) = self.group_distinct_plan::<K, R>()?;
93 self.start_groups(jobs, now, expires, plan, reducer, None)
94 }
95
96 /// Advance grouped DISTINCT counts with grouped jobs' revision/retry/expiry rules.
97 /// Finish all input before filtering, ordering or limiting groups. Empty input
98 /// returns an empty vector; an all-absent tuple contributes one to its group.
99 /// Returns a GroupPage; use `group_page` to page completed selected groups.
100 /// # Errors
101 /// Refuses mismatched bindings, malformed saved sets, group/key cardinality,
102 /// output or frame bounds. Refusal preserves the previous checkpoint; output
103 /// filters and limits cannot conceal oversized state.
104 /// # Panics
105 /// Supplier mutation failures trap the enclosing message.
106 pub fn resume_group_count_distinct<K: Value, R: Value>(
107 self,
108 jobs: &mut AggregateJobs<M>,
109 handle: &JobHandle,
110 sequence: u64,
111 now: Timestamp,
112 ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
113 let (plan, reducer) = self.group_distinct_plan::<K, R>()?;
114 let job = self.retained_groups(plan, reducer, None)?;
115 self.resume_groups(jobs, handle, sequence, now, job, decode_group_count)
116 }
117}
118
119fn binding(query: [u8; 32]) -> [u8; 32] {
120 let mut hash = Sha256::new();
121 hash.update(b"ic-db:distinct-count-job:v1\0");
122 hash.update(query);
123 hash.finalize().into()
124}