Skip to main content

ic_db/store/query/aggregate/jobs/
distinct.rs

1//! Retained tuple DISTINCT counts using the existing exact key-set reducer.
2use super::super::{Accumulator, GroupCount, Query};
3use super::{
4    AggregateJobs, GroupPage, JobHandle, JobStep, grouped::group_binding, pages, prepare, start,
5};
6use crate::{Entity, Error, Timestamp, Value};
7use ic_storage::Memory;
8use sha2::{Digest, Sha256};
9
10impl<E: Entity, M: Memory> Query<'_, E, M> {
11    /// Retain an exact count of projected K tuples, including an absent tuple.
12    /// Uses synchronous `count_distinct` admission and equality without an index.
13    /// Retain at most 32 distinct keys inside the 16 KiB whole-job frame;
14    /// no spill. Query/projection/algorithm bind the job; work/byte limits may vary.
15    /// # Errors
16    /// Refuses unsupported projections, group modifiers, expiry and capacity.
17    /// # Panics
18    /// Supplier mutation failures trap the enclosing message.
19    pub fn start_count_distinct<K: Value>(
20        self,
21        jobs: &mut AggregateJobs<impl Memory>,
22        now: Timestamp,
23        expires: Timestamp,
24    ) -> Result<JobHandle, Error> {
25        let (plan, _) = self.distinct_plan::<K>()?;
26        if self.request.bytes < 8 {
27            return Err(Error::RowTooLarge);
28        }
29        start(
30            jobs,
31            binding(plan.binding),
32            self.db.revision_by_id(E::ID)?,
33            now,
34            expires,
35            vec![0],
36        )
37    }
38
39    /// Advance DISTINCT count under scalar jobs' work/retry/revision/expiry rules.
40    /// Duplicates across steps count once; empty input returns zero, all-absent
41    /// tuples count once. Finish all input before returning a count. `limit`
42    /// never truncates input or relaxes key-set bounds.
43    /// # Errors
44    /// Refuses mismatched queries/projections, malformed saved keys, cardinality,
45    /// output or whole-frame bounds without changing the previous checkpoint.
46    /// # Panics
47    /// Supplier mutation failures trap the enclosing message.
48    pub fn resume_count_distinct<K: Value>(
49        self,
50        jobs: &mut AggregateJobs<impl Memory>,
51        handle: &JobHandle,
52        sequence: u64,
53        now: Timestamp,
54    ) -> Result<JobStep<u64>, Error> {
55        let (plan, mut reducer) = self.distinct_plan::<K>()?;
56        let mut record = jobs.load(handle)?;
57        let bytes = self.request.bytes;
58        let step = prepare(
59            self.db,
60            E::SCHEMA,
61            self.request,
62            &plan,
63            Accumulator::Scalar(&mut reducer),
64            &mut record,
65            binding(plan.binding),
66            sequence,
67            now,
68        )?
69        .finish(bytes, || Ok((reducer.finish()?.into_count()?, 8)))?;
70        jobs.save(handle, &record)?;
71        Ok(step)
72    }
73
74    /// Retain distinct R tuples separately in each K group, including absent tuples.
75    /// Projections may overlap. Uses synchronous `group_count_distinct` admission.
76    /// Keep at most 32 groups and 32 distinct keys per group inside one 16 KiB
77    /// whole-job frame. Key/operand projections, group selection and limit bind
78    /// the job. Work/output-byte limits may vary; no automatic spill.
79    /// # Errors
80    /// Refuses unsupported projections, expiry or occupied job capacity.
81    /// # Panics
82    /// Supplier mutation failures trap the enclosing message.
83    pub fn start_group_count_distinct<K: Value, R: Value>(
84        self,
85        jobs: &mut AggregateJobs<impl Memory>,
86        now: Timestamp,
87        expires: Timestamp,
88    ) -> Result<JobHandle, Error> {
89        if self.request.groups.page.is_some() {
90            return Err(Error::InvalidQuery);
91        }
92        let (plan, reducer) = self.group_distinct_plan::<K, R>()?;
93        let groups = self.admit_groups(&plan, reducer)?;
94        start(
95            jobs,
96            group_binding(&plan, &groups, self.request.rows)?,
97            self.db.revision_by_id(E::ID)?,
98            now,
99            expires,
100            vec![0],
101        )
102    }
103
104    /// Advance grouped DISTINCT counts with grouped jobs' revision/retry/expiry rules.
105    /// Finish all input before filtering, ordering or limiting groups. Empty input
106    /// returns an empty vector; an all-absent tuple contributes one to its group.
107    /// Returns a GroupPage; use `group_page` to page completed selected groups.
108    /// # Errors
109    /// Refuses mismatched bindings, malformed saved sets, group/key cardinality,
110    /// output or frame bounds. Refusal preserves the previous checkpoint; output
111    /// filters and limits cannot conceal oversized state.
112    /// # Panics
113    /// Supplier mutation failures trap the enclosing message.
114    pub fn resume_group_count_distinct<K: Value, R: Value>(
115        self,
116        jobs: &mut AggregateJobs<impl Memory>,
117        handle: &JobHandle,
118        sequence: u64,
119        now: Timestamp,
120    ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
121        let (plan, reducer) = self.group_distinct_plan::<K, R>()?;
122        let mut groups = self.admit_groups(&plan, reducer)?;
123        let expected = group_binding(&plan, &groups, self.request.rows)?;
124        let mut record = jobs.load(handle)?;
125        let page = self.request.groups.page;
126        pages::admit(page, handle, &record)?;
127        let (rows, bytes) = (self.request.rows, self.request.bytes);
128        let step = prepare(
129            self.db,
130            E::SCHEMA,
131            self.request,
132            &plan,
133            Accumulator::Groups(&mut groups),
134            &mut record,
135            expected,
136            sequence,
137            now,
138        )?
139        .finish(bytes, || {
140            pages::count_page(groups, rows, bytes, page, handle)
141        })?;
142        jobs.save(handle, &record)?;
143        Ok(step)
144    }
145}
146
147fn binding(query: [u8; 32]) -> [u8; 32] {
148    let mut hash = Sha256::new();
149    hash.update(b"ic-db:distinct-count-job:v1\0");
150    hash.update(query);
151    hash.finalize().into()
152}