ic_db/store/query/aggregate/jobs/
storage.rs1use super::{JobHandle, JobOutcome, MAX_CANDIDATES};
3use crate::{
4 Error, QueryCursor, Timestamp,
5 codec::{Decoder, Encoder},
6};
7use ic_storage::{Memory, RuntimeMemory, adapter::StableBTreeMap};
8
9pub(super) const SLOTS: u8 = 16;
10pub(super) const MAX_JOB_BYTES: usize = 16 * 1024;
11const LEGACY_MAX_JOB_BYTES: usize = 4096;
12const MAGIC: &[u8; 8] = b"ICDBAJ02";
13const LEGACY_MAGIC: &[u8; 8] = b"ICDBAJ01";
14
15pub struct AggregateJobs<M: Memory> {
21 map: StableBTreeMap<u8, Vec<u8>, M>,
22 incarnation: [u8; 16],
23 legacy: bool,
24}
25impl<M: Memory> AggregateJobs<M> {
26 pub fn create(memory: RuntimeMemory<M>, incarnation: [u8; 16]) -> Result<Self, Error> {
32 if incarnation == [0; 16] {
33 return Err(Error::InvalidJob);
34 }
35 if memory.size() != 0 {
36 return Err(Error::Exists);
37 }
38 let mut map = StableBTreeMap::create(memory);
39 map.insert(0, header(incarnation, 0));
40 Ok(Self {
41 map,
42 incarnation,
43 legacy: false,
44 })
45 }
46
47 pub fn open(memory: RuntimeMemory<M>) -> Result<Self, Error> {
55 if memory.size() == 0 {
56 return Err(Error::Corrupt);
57 }
58 let map: StableBTreeMap<u8, Vec<u8>, M> = StableBTreeMap::load(memory);
59 let bytes = map.get(&0).ok_or(Error::Corrupt)?;
60 let (incarnation, _, legacy) = read_header(&bytes)?;
61 if map.len() > u64::from(SLOTS) + 1
62 || map
63 .range(..)
64 .next_back()
65 .is_some_and(|entry| *entry.key() > SLOTS)
66 {
67 return Err(Error::Corrupt);
68 }
69 Ok(Self {
70 map,
71 incarnation,
72 legacy,
73 })
74 }
75
76 pub fn cancel(&mut self, handle: &JobHandle) -> Result<(), Error> {
84 self.with_record(handle, |_| ())?;
85 self.map.remove(&handle.slot);
86 Ok(())
87 }
88
89 pub fn cleanup_expired(&mut self, now: Timestamp, limit: usize) -> Result<usize, Error> {
97 if !(1..=usize::from(SLOTS)).contains(&limit) {
98 return Err(Error::Limit);
99 }
100 let mut expired = Vec::new();
101 for slot in 1..=SLOTS {
102 if let Some(bytes) = self.map.get(&slot) {
103 let record = self.decode(&bytes)?;
104 if record.expires <= now && expired.len() < limit {
105 expired.push(slot);
106 }
107 }
108 }
109 for slot in &expired {
110 self.map.remove(slot);
111 }
112 Ok(expired.len())
113 }
114
115 pub(super) fn insert(&mut self, mut record: Record) -> Result<JobHandle, Error> {
116 let slot = (1..=SLOTS)
117 .find(|slot| !self.map.contains_key(slot))
118 .ok_or(Error::JobCapacity)?;
119 let (incarnation, generation, _) = read_header(&self.map.get(&0).ok_or(Error::Corrupt)?)?;
120 if incarnation != self.incarnation {
121 return Err(Error::Corrupt);
122 }
123 let generation = generation.checked_add(1).ok_or(Error::Limit)?;
124 record.generation = generation;
125 let bytes = record.encode()?;
126 self.map.insert(0, header(incarnation, generation));
128 self.map.insert(slot, bytes);
129 self.legacy = false;
130 Ok(JobHandle {
131 incarnation,
132 slot,
133 generation,
134 })
135 }
136
137 pub(super) fn load(&self, handle: &JobHandle) -> Result<Record, Error> {
138 self.with_record(handle, |record| record.into_owned())
139 }
140
141 fn with_record<R>(
144 &self,
145 handle: &JobHandle,
146 inspect: impl FnOnce(Record<&[u8]>) -> R,
147 ) -> Result<R, Error> {
148 if handle.incarnation != self.incarnation {
149 return Err(Error::InvalidJob);
150 }
151 let bytes = self.map.get(&handle.slot).ok_or(Error::InvalidJob)?;
152 let record = self.decode(&bytes)?;
153 if record.generation != handle.generation {
154 return Err(Error::InvalidJob);
155 }
156 Ok(inspect(record))
157 }
158
159 pub(super) fn save(&mut self, handle: &JobHandle, record: &Record) -> Result<(), Error> {
160 let bytes = record.encode()?;
161 if self.legacy {
164 let (incarnation, generation, _) =
165 read_header(&self.map.get(&0).ok_or(Error::Corrupt)?)?;
166 self.map.insert(0, header(incarnation, generation));
167 }
168 self.map.insert(handle.slot, bytes);
169 self.legacy = false;
170 Ok(())
171 }
172
173 fn decode<'a>(&self, bytes: &'a [u8]) -> Result<Record<&'a [u8]>, Error> {
174 if self.legacy && bytes.len() > LEGACY_MAX_JOB_BYTES {
175 return Err(Error::Corrupt);
176 }
177 Record::decode(bytes)
178 }
179}
180fn header(incarnation: [u8; 16], generation: u64) -> Vec<u8> {
181 let mut bytes = MAGIC.to_vec();
182 bytes.extend(crate::codec::ROW_CODEC_VERSION.to_le_bytes());
183 bytes.extend(incarnation);
184 bytes.extend(generation.to_le_bytes());
185 bytes
186}
187
188fn read_header(bytes: &[u8]) -> Result<([u8; 16], u64, bool), Error> {
189 if bytes.len() != 36 {
190 return Err(Error::Corrupt);
191 }
192 if (&bytes[..8] != MAGIC && &bytes[..8] != LEGACY_MAGIC)
193 || bytes[8..12] != crate::codec::ROW_CODEC_VERSION.to_le_bytes()
194 {
195 return Err(Error::InvalidJob);
196 }
197 let incarnation = bytes[12..28].try_into().map_err(|_| Error::Corrupt)?;
198 if incarnation == [0; 16] {
199 return Err(Error::Corrupt);
200 }
201 Ok((
202 incarnation,
203 u64::from_le_bytes(bytes[28..].try_into().map_err(|_| Error::Corrupt)?),
204 &bytes[..8] == LEGACY_MAGIC,
205 ))
206}
207
208pub(super) struct Record<Bytes = Vec<u8>> {
211 pub generation: u64,
212 pub binding: [u8; 32],
213 pub revision: u64,
214 pub expires: Timestamp,
215 pub observed: Timestamp,
216 pub sequence: u64,
217 pub examined: u64,
218 pub input_bytes: u64,
219 pub phase: JobOutcome<()>,
220 pub cursor: Bytes,
221 pub count: u64,
222 pub state: Bytes,
224}
225impl Record {
226 fn encode(&self) -> Result<Vec<u8>, Error> {
227 let mut e = Encoder::with_byte_limit(MAX_JOB_BYTES);
228 e.nat64(self.generation)?;
229 e.bytes(&self.binding)?;
230 e.nat64(self.revision)?;
231 e.timestamp(self.expires)?;
232 e.timestamp(self.observed)?;
233 e.nat64(self.sequence)?;
234 e.nat64(self.examined)?;
235 e.nat64(self.input_bytes)?;
236 e.nat8(match self.phase {
237 JobOutcome::Running => 0,
238 JobOutcome::Complete(()) => 1,
239 JobOutcome::Invalidated => 2,
240 JobOutcome::Expired => 3,
241 })?;
242 e.bytes(&self.cursor)?;
243 e.nat64(self.count)?;
244 e.bytes(&self.state)?;
245 Ok(e.finish())
246 }
247}
248impl<'a> Record<&'a [u8]> {
249 fn decode(bytes: &'a [u8]) -> Result<Self, Error> {
250 let mut d = Decoder::with_byte_limit(bytes, MAX_JOB_BYTES)?;
251 let record = Self {
252 generation: d.nat64()?,
253 binding: d.blob_ref(32)?.try_into().map_err(|_| Error::Corrupt)?,
254 revision: d.nat64()?,
255 expires: d.timestamp()?,
256 observed: d.timestamp()?,
257 sequence: d.nat64()?,
258 examined: d.nat64()?,
259 input_bytes: d.nat64()?,
260 phase: match d.nat8()? {
261 0 => JobOutcome::Running,
262 1 => JobOutcome::Complete(()),
263 2 => JobOutcome::Invalidated,
264 3 => JobOutcome::Expired,
265 _ => return Err(Error::Corrupt),
266 },
267 cursor: d.blob_ref(QueryCursor::MAX_ENCODED_BYTES)?,
268 count: d.nat64()?,
269 state: d.blob_ref(MAX_JOB_BYTES)?,
270 };
271 d.finish()?;
272 if record.generation == 0
273 || record.examined > MAX_CANDIDATES
274 || record.count > record.examined
275 || record.sequence > record.examined + 2
276 || record.input_bytes > record.examined * super::super::super::CANDIDATE_BYTES as u64
277 || (matches!(record.phase, JobOutcome::Complete(())) && !record.cursor.is_empty())
278 || (matches!(record.phase, JobOutcome::Running)
279 && (record.observed >= record.expires
280 || (record.examined > 0 && record.cursor.is_empty())))
281 {
282 return Err(Error::Corrupt);
283 }
284 Ok(record)
285 }
286
287 fn into_owned(self) -> Record {
288 Record {
289 generation: self.generation,
290 binding: self.binding,
291 revision: self.revision,
292 expires: self.expires,
293 observed: self.observed,
294 sequence: self.sequence,
295 examined: self.examined,
296 input_bytes: self.input_bytes,
297 phase: self.phase,
298 cursor: self.cursor.to_vec(),
299 count: self.count,
300 state: self.state.to_vec(),
301 }
302 }
303}