Skip to main content

ic_db/store/query/aggregate/jobs/
distinct.rs

1//! Retained tuple DISTINCT counts using the existing exact key-set reducer.
2use super::{
3    super::{Accumulator, GroupCount, Query, decode_group_count},
4    AggregateJobs, GroupPage, JobHandle, JobStep, prepare, start,
5};
6use crate::{Entity, Error, Timestamp, Value};
7use ic_storage::Memory;
8use sha2::{Digest, Sha256};
9
10impl<E: Entity, M: Memory> Query<'_, E, M> {
11    /// Retain an exact count of projected K tuples, including an absent tuple.
12    /// Uses synchronous `count_distinct` admission and equality without an index.
13    /// Retain at most 32 distinct keys inside the 16 KiB whole-job frame;
14    /// no spill. Query/projection/algorithm bind the job; work/byte limits may vary.
15    /// # Errors
16    /// Refuses unsupported projections, group modifiers, expiry and capacity.
17    /// # Panics
18    /// Supplier mutation failures trap the enclosing message.
19    pub fn start_count_distinct<K: Value>(
20        self,
21        jobs: &mut AggregateJobs<M>,
22        now: Timestamp,
23        expires: Timestamp,
24    ) -> Result<JobHandle, Error> {
25        let (plan, mut reducer) = self.distinct_plan::<K>()?;
26        if self.request.bytes < 8 {
27            return Err(Error::RowTooLarge);
28        }
29        start(
30            jobs,
31            binding(plan.binding),
32            self.db.revision_by_id(E::SCHEMA.id)?,
33            now,
34            expires,
35            &Accumulator::Scalar(&mut reducer),
36        )
37    }
38
39    /// Advance DISTINCT count under scalar jobs' work/retry/revision/expiry rules.
40    /// Duplicates across steps count once; empty input returns zero, all-absent
41    /// tuples count once. Finish all input before returning a count. `limit`
42    /// never truncates input or relaxes key-set bounds.
43    /// # Errors
44    /// Refuses mismatched queries/projections, malformed saved keys, cardinality,
45    /// output or whole-frame bounds without changing the previous checkpoint.
46    /// # Panics
47    /// Supplier mutation failures trap the enclosing message.
48    pub fn resume_count_distinct<K: Value>(
49        self,
50        jobs: &mut AggregateJobs<M>,
51        handle: &JobHandle,
52        sequence: u64,
53        now: Timestamp,
54    ) -> Result<JobStep<u64>, Error> {
55        let (plan, mut reducer) = self.distinct_plan::<K>()?;
56        let mut record = jobs.load(handle)?;
57        let bytes = self.request.bytes;
58        prepare(
59            self.db,
60            E::SCHEMA,
61            self.request,
62            &plan,
63            Accumulator::Scalar(&mut reducer),
64            &mut record,
65            binding(plan.binding),
66            sequence,
67            now,
68        )?
69        .finish_and_save(jobs, handle, &record, bytes, || {
70            Ok((reducer.finish()?.into_count()?, 8))
71        })
72    }
73
74    /// Retain distinct R tuples separately in each K group, including absent tuples.
75    /// Projections may overlap. Uses synchronous `group_count_distinct` admission.
76    /// Keep at most 32 groups and 32 distinct keys per group inside one 16 KiB
77    /// whole-job frame. Key/operand projections, group selection and limit bind
78    /// the job. Work/output-byte limits may vary; no automatic spill.
79    /// # Errors
80    /// Refuses unsupported projections, expiry or occupied job capacity.
81    /// # Panics
82    /// Supplier mutation failures trap the enclosing message.
83    pub fn start_group_count_distinct<K: Value, R: Value>(
84        self,
85        jobs: &mut AggregateJobs<M>,
86        now: Timestamp,
87        expires: Timestamp,
88    ) -> Result<JobHandle, Error> {
89        if self.request.groups.page.is_some() {
90            return Err(Error::InvalidQuery);
91        }
92        let (plan, reducer) = self.group_distinct_plan::<K, R>()?;
93        self.start_groups(jobs, now, expires, plan, reducer, None)
94    }
95
96    /// Advance grouped DISTINCT counts with grouped jobs' revision/retry/expiry rules.
97    /// Finish all input before filtering, ordering or limiting groups. Empty input
98    /// returns an empty vector; an all-absent tuple contributes one to its group.
99    /// Returns a GroupPage; use `group_page` to page completed selected groups.
100    /// # Errors
101    /// Refuses mismatched bindings, malformed saved sets, group/key cardinality,
102    /// output or frame bounds. Refusal preserves the previous checkpoint; output
103    /// filters and limits cannot conceal oversized state.
104    /// # Panics
105    /// Supplier mutation failures trap the enclosing message.
106    pub fn resume_group_count_distinct<K: Value, R: Value>(
107        self,
108        jobs: &mut AggregateJobs<M>,
109        handle: &JobHandle,
110        sequence: u64,
111        now: Timestamp,
112    ) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
113        let (plan, reducer) = self.group_distinct_plan::<K, R>()?;
114        let job = self.retained_groups(plan, reducer, None)?;
115        self.resume_groups(jobs, handle, sequence, now, job, decode_group_count)
116    }
117}
118
119fn binding(query: [u8; 32]) -> [u8; 32] {
120    let mut hash = Sha256::new();
121    hash.update(b"ic-db:distinct-count-job:v1\0");
122    hash.update(query);
123    hash.finalize().into()
124}