1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
//! Retained tuple DISTINCT counts using the existing exact key-set reducer.
use super::{
super::{Accumulator, GroupCount, Query, decode_group_count},
AggregateJobs, GroupPage, JobHandle, JobStep, prepare, start,
};
use crate::{Entity, Error, Timestamp, Value};
use ic_storage::Memory;
use sha2::{Digest, Sha256};
impl<E: Entity, M: Memory> Query<'_, E, M> {
/// Retain an exact count of projected K tuples, including an absent tuple.
/// Uses synchronous `count_distinct` admission and equality without an index.
/// Retain at most 32 distinct keys inside the 16 KiB whole-job frame;
/// no spill. Query/projection/algorithm bind the job; work/byte limits may vary.
/// # Errors
/// Refuses unsupported projections, group modifiers, expiry and capacity.
/// # Panics
/// Supplier mutation failures trap the enclosing message.
pub fn start_count_distinct<K: Value>(
self,
jobs: &mut AggregateJobs<M>,
now: Timestamp,
expires: Timestamp,
) -> Result<JobHandle, Error> {
let (plan, mut reducer) = self.distinct_plan::<K>()?;
if self.request.bytes < 8 {
return Err(Error::RowTooLarge);
}
start(
jobs,
binding(plan.binding),
self.db.revision_by_id(E::SCHEMA.id)?,
now,
expires,
&Accumulator::Scalar(&mut reducer),
)
}
/// Advance DISTINCT count under scalar jobs' work/retry/revision/expiry rules.
/// Duplicates across steps count once; empty input returns zero, all-absent
/// tuples count once. Finish all input before returning a count. `limit`
/// never truncates input or relaxes key-set bounds.
/// # Errors
/// Refuses mismatched queries/projections, malformed saved keys, cardinality,
/// output or whole-frame bounds without changing the previous checkpoint.
/// # Panics
/// Supplier mutation failures trap the enclosing message.
pub fn resume_count_distinct<K: Value>(
self,
jobs: &mut AggregateJobs<M>,
handle: &JobHandle,
sequence: u64,
now: Timestamp,
) -> Result<JobStep<u64>, Error> {
let (plan, mut reducer) = self.distinct_plan::<K>()?;
let mut record = jobs.load(handle)?;
let bytes = self.request.bytes;
prepare(
self.db,
E::SCHEMA,
self.request,
&plan,
Accumulator::Scalar(&mut reducer),
&mut record,
binding(plan.binding),
sequence,
now,
)?
.finish_and_save(jobs, handle, &record, bytes, || {
Ok((reducer.finish()?.into_count()?, 8))
})
}
/// Retain distinct R tuples separately in each K group, including absent tuples.
/// Projections may overlap. Uses synchronous `group_count_distinct` admission.
/// Keep at most 32 groups and 32 distinct keys per group inside one 16 KiB
/// whole-job frame. Key/operand projections, group selection and limit bind
/// the job. Work/output-byte limits may vary; no automatic spill.
/// # Errors
/// Refuses unsupported projections, expiry or occupied job capacity.
/// # Panics
/// Supplier mutation failures trap the enclosing message.
pub fn start_group_count_distinct<K: Value, R: Value>(
self,
jobs: &mut AggregateJobs<M>,
now: Timestamp,
expires: Timestamp,
) -> Result<JobHandle, Error> {
if self.request.groups.page.is_some() {
return Err(Error::InvalidQuery);
}
let (plan, reducer) = self.group_distinct_plan::<K, R>()?;
self.start_groups(jobs, now, expires, plan, reducer, None)
}
/// Advance grouped DISTINCT counts with grouped jobs' revision/retry/expiry rules.
/// Finish all input before filtering, ordering or limiting groups. Empty input
/// returns an empty vector; an all-absent tuple contributes one to its group.
/// Returns a GroupPage; use `group_page` to page completed selected groups.
/// # Errors
/// Refuses mismatched bindings, malformed saved sets, group/key cardinality,
/// output or frame bounds. Refusal preserves the previous checkpoint; output
/// filters and limits cannot conceal oversized state.
/// # Panics
/// Supplier mutation failures trap the enclosing message.
pub fn resume_group_count_distinct<K: Value, R: Value>(
self,
jobs: &mut AggregateJobs<M>,
handle: &JobHandle,
sequence: u64,
now: Timestamp,
) -> Result<JobStep<GroupPage<GroupCount<K>>>, Error> {
let (plan, reducer) = self.group_distinct_plan::<K, R>()?;
let job = self.retained_groups(plan, reducer, None)?;
self.resume_groups(jobs, handle, sequence, now, job, decode_group_count)
}
}
fn binding(query: [u8; 32]) -> [u8; 32] {
let mut hash = Sha256::new();
hash.update(b"ic-db:distinct-count-job:v1\0");
hash.update(query);
hash.finalize().into()
}