Skip to main content

ic_db/store/query/aggregate/jobs/
distinct.rs

1//! Retained tuple DISTINCT counts using the existing exact key-set reducer.
2use super::{
3    super::{Accumulator, GroupCount, Query},
4    AggregateJobs, GroupPage, JobHandle, JobStep,
5    grouped::group_binding,
6    pages, prepare, start,
7};
8use crate::{Entity, Error, Timestamp, Value};
9use ic_storage::Memory;
10use sha2::{Digest, Sha256};
11
12impl<E: Entity, M: Memory> Query<'_, E, M> {
13    /// Retain an exact count of projected K tuples, including an absent tuple.
14    /// Uses synchronous `count_distinct` admission and equality without an index.
15    /// Retain at most 32 distinct keys inside the 16 KiB whole-job frame;
16    /// no spill. Query/projection/algorithm bind the job; work/byte limits may vary.
17    /// # Errors
18    /// Refuses unsupported projections, group modifiers, expiry and capacity.
19    /// # Panics
20    /// Supplier mutation failures trap the enclosing message.
21    pub fn start_count_distinct<K: Value>(
22        self,
23        jobs: &mut AggregateJobs<M>,
24        now: Timestamp,
25        expires: Timestamp,
26    ) -> Result<JobHandle, Error> {
27        let (plan, _) = self.distinct_plan::<K>()?;
28        if self.request.bytes < 8 {
29            return Err(Error::RowTooLarge);
30        }
31        start(
32            jobs,
33            binding(plan.binding),
34            self.db.revision_by_id(E::ID)?,
35            now,
36            expires,
37            vec![0],
38        )
39    }
40
41    /// Advance DISTINCT count under scalar jobs' work/retry/revision/expiry rules.
42    /// Duplicates across steps count once; empty input returns zero, all-absent
43    /// tuples count once. Finish all input before returning a count. `limit`
44    /// never truncates input or relaxes key-set bounds.
45    /// # Errors
46    /// Refuses mismatched queries/projections, malformed saved keys, cardinality,
47    /// output or whole-frame bounds without changing the previous checkpoint.
48    /// # Panics
49    /// Supplier mutation failures trap the enclosing message.
50    pub fn resume_count_distinct<K: Value>(
51        self,
52        jobs: &mut AggregateJobs<M>,
53        handle: &JobHandle,
54        sequence: u64,
55        now: Timestamp,
56    ) -> Result<JobStep<u64>, Error> {
57        let (plan, mut reducer) = self.distinct_plan::<K>()?;
58        let mut record = jobs.load(handle)?;
59        let bytes = self.request.bytes;
60        let step = prepare(
61            self.db,
62            E::SCHEMA,
63            self.request,
64            &plan,
65            Accumulator::Scalar(&mut reducer),
66            &mut record,
67            binding(plan.binding),
68            sequence,
69            now,
70        )?
71        .finish(bytes, || Ok((reducer.finish()?.into_count()?, 8)))?;
72        jobs.save(handle, &record)?;
73        Ok(step)
74    }
75
76    /// Retain distinct R tuples separately in each K group, including absent tuples.
77    /// Projections may overlap. Uses synchronous `group_count_distinct` admission.
78    /// Keep at most 32 groups and 32 distinct keys per group inside one 16 KiB
79    /// whole-job frame. Key/operand projections, group selection and limit bind
80    /// the job. Work/output-byte limits may vary; no automatic spill.
81    /// # Errors
82    /// Refuses unsupported projections, expiry or occupied job capacity.
83    /// # Panics
84    /// Supplier mutation failures trap the enclosing message.
85    pub fn start_group_count_distinct<K: Value, R: Value>(
86        self,
87        jobs: &mut AggregateJobs<M>,
88        now: Timestamp,
89        expires: Timestamp,
90    ) -> Result<JobHandle, Error> {
91        if self.request.groups.page.is_some() {
92            return Err(Error::InvalidQuery);
93        }
94        let (plan, reducer) = self.group_distinct_plan::<K, R>()?;
95        let groups = self.admit_groups(&plan, reducer)?;
96        start(
97            jobs,
98            group_binding(&plan, &groups, self.request.rows)?,
99            self.db.revision_by_id(E::ID)?,
100            now,
101            expires,
102            vec![0],
103        )
104    }
105
106    /// Advance grouped DISTINCT counts with grouped jobs' revision/retry/expiry rules.
107    /// Finish all input before filtering, ordering or limiting groups. Empty input
108    /// returns an empty vector; an all-absent tuple contributes one to its group.
109    /// Returns a GroupPage; use `group_page` to page completed selected groups.
110    /// # Errors
111    /// Refuses mismatched bindings, malformed saved sets, group/key cardinality,
112    /// output or frame bounds. Refusal preserves the previous checkpoint; output
113    /// filters and limits cannot conceal oversized state.
114    /// # Panics
115    /// Supplier mutation failures trap the enclosing message.
116    pub fn resume_group_count_distinct<K: Value, R: Value>(
117        self,
118        jobs: &mut AggregateJobs<M>,
119        handle: &JobHandle,
120        sequence: u64,
121        now: Timestamp,
122    ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
123        let (plan, reducer) = self.group_distinct_plan::<K, R>()?;
124        let mut groups = self.admit_groups(&plan, reducer)?;
125        let expected = group_binding(&plan, &groups, self.request.rows)?;
126        let mut record = jobs.load(handle)?;
127        let page = self.request.groups.page;
128        pages::admit(page, handle, &record)?;
129        let (rows, bytes) = (self.request.rows, self.request.bytes);
130        let step = prepare(
131            self.db,
132            E::SCHEMA,
133            self.request,
134            &plan,
135            Accumulator::Groups(&mut groups),
136            &mut record,
137            expected,
138            sequence,
139            now,
140        )?
141        .finish(bytes, || {
142            pages::count_page(groups, rows, bytes, page, handle)
143        })?;
144        jobs.save(handle, &record)?;
145        Ok(step)
146    }
147}
148
149fn binding(query: [u8; 32]) -> [u8; 32] {
150    let mut hash = Sha256::new();
151    hash.update(b"ic-db:distinct-count-job:v1\0");
152    hash.update(query);
153    hash.finalize().into()
154}