Skip to main content

ic_db/store/query/aggregate/jobs/
storage.rs

1//! Fixed slots in one host-owned allocation; bounded frames use the owned codec.
2use super::{JobHandle, JobOutcome, MAX_CANDIDATES};
3use crate::{
4    Error, QueryCursor, Timestamp,
5    codec::{Decoder, Encoder},
6};
7use ic_storage::{Memory, RuntimeMemory, adapter::StableBTreeMap};
8
9pub(super) const SLOTS: u8 = 16;
10pub(super) const MAX_JOB_BYTES: usize = 16 * 1024;
11const LEGACY_MAX_JOB_BYTES: usize = 4096;
12const MAGIC: &[u8; 8] = b"ICDBAJ02";
13const LEGACY_MAGIC: &[u8; 8] = b"ICDBAJ01";
14
15/// Durable bounded aggregates in a host-governed allocation.
16/// Holds at most 16 jobs, each at most 16 KiB encoded bytes. Creation requires a
17/// fresh nonzero incarnation. Handles identify jobs; they do not authorize access.
18/// All mutation must run synchronously within one IC message, without awaits.
19/// Supplier failures trap; a later message trap rolls these changes back too.
20pub struct AggregateJobs<M: Memory> {
21    map: StableBTreeMap<u8, Vec<u8>, M>,
22    incarnation: [u8; 16],
23    legacy: bool,
24}
25impl<M: Memory> AggregateJobs<M> {
26    /// Create in an empty allocation, with a host-supplied fresh incarnation.
27    /// # Errors
28    /// Refuses a zero incarnation or a nonempty/unsupported supplier allocation.
29    /// # Panics
30    /// Supplier failures after mutation begins must trap the enclosing message.
31    pub fn create(memory: RuntimeMemory<M>, incarnation: [u8; 16]) -> Result<Self, Error> {
32        if incarnation == [0; 16] {
33            return Err(Error::InvalidJob);
34        }
35        if memory.size() != 0 {
36            return Err(Error::Exists);
37        }
38        let mut map = StableBTreeMap::create(memory);
39        map.insert(0, header(incarnation, 0));
40        Ok(Self {
41            map,
42            incarnation,
43            legacy: false,
44        })
45    }
46
47    /// Reopen a supported job-store format without modifying it. Version-one
48    /// jobs remain readable; the first successful insert/save atomically admits
49    /// version two and its larger frame bound. Older executables then refuse.
50    /// # Errors
51    /// Refuses missing/corrupt headers and unsupported formats intact.
52    /// # Panics
53    /// Supplier corruption traps without repair.
54    pub fn open(memory: RuntimeMemory<M>) -> Result<Self, Error> {
55        if memory.size() == 0 {
56            return Err(Error::Corrupt);
57        }
58        let map: StableBTreeMap<u8, Vec<u8>, M> = StableBTreeMap::load(memory);
59        let bytes = map.get(&0).ok_or(Error::Corrupt)?;
60        let (incarnation, _, legacy) = read_header(&bytes)?;
61        if map.len() > u64::from(SLOTS) + 1
62            || map
63                .range(..)
64                .next_back()
65                .is_some_and(|entry| *entry.key() > SLOTS)
66        {
67            return Err(Error::Corrupt);
68        }
69        Ok(Self {
70            map,
71            incarnation,
72            legacy,
73        })
74    }
75
76    /// Cancel one job, immediately releasing its logical slot. Repeated/stale
77    /// cancellation refuses; it cannot delete a subsequent occupant of that slot.
78    /// # Errors
79    /// Refuses a foreign, missing, stale or malformed handle/job frame. Cancellation
80    /// need not interpret a known frame's query-specific accumulator payload.
81    /// # Panics
82    /// Supplier mutation failures trap the enclosing message.
83    pub fn cancel(&mut self, handle: &JobHandle) -> Result<(), Error> {
84        self.with_record(handle, |_| ())?;
85        self.map.remove(&handle.slot);
86        Ok(())
87    }
88
89    /// Examine at most 16 slots and remove at most `limit` expired jobs (1..=16).
90    /// Time must be captured by the host; the database reads no clock. This does
91    /// not shrink stable memory or promise physical reclamation by the supplier.
92    /// # Errors
93    /// Refuses invalid limits or malformed records before removing any job.
94    /// # Panics
95    /// Supplier mutation failures trap the enclosing message.
96    pub fn cleanup_expired(&mut self, now: Timestamp, limit: usize) -> Result<usize, Error> {
97        if !(1..=usize::from(SLOTS)).contains(&limit) {
98            return Err(Error::Limit);
99        }
100        let mut expired = Vec::new();
101        for slot in 1..=SLOTS {
102            if let Some(bytes) = self.map.get(&slot) {
103                let record = self.decode(&bytes)?;
104                if record.expires <= now && expired.len() < limit {
105                    expired.push(slot);
106                }
107            }
108        }
109        for slot in &expired {
110            self.map.remove(slot);
111        }
112        Ok(expired.len())
113    }
114
115    pub(super) fn insert(&mut self, mut record: Record) -> Result<JobHandle, Error> {
116        let slot = (1..=SLOTS)
117            .find(|slot| !self.map.contains_key(slot))
118            .ok_or(Error::JobCapacity)?;
119        let (incarnation, generation, _) = read_header(&self.map.get(&0).ok_or(Error::Corrupt)?)?;
120        if incarnation != self.incarnation {
121            return Err(Error::Corrupt);
122        }
123        let generation = generation.checked_add(1).ok_or(Error::Limit)?;
124        record.generation = generation;
125        let bytes = record.encode()?;
126        // No recoverable refusal after the first persistent write.
127        self.map.insert(0, header(incarnation, generation));
128        self.map.insert(slot, bytes);
129        self.legacy = false;
130        Ok(JobHandle {
131            incarnation,
132            slot,
133            generation,
134        })
135    }
136
137    pub(super) fn load(&self, handle: &JobHandle) -> Result<Record, Error> {
138        self.with_record(handle, |record| record.into_owned())
139    }
140
141    // One identity/framing admission for owned resume and borrowed cancellation.
142    // Only resume needs owned cursor/accumulator bytes after the supplier buffer dies.
143    fn with_record<R>(
144        &self,
145        handle: &JobHandle,
146        inspect: impl FnOnce(Record<&[u8]>) -> R,
147    ) -> Result<R, Error> {
148        if handle.incarnation != self.incarnation {
149            return Err(Error::InvalidJob);
150        }
151        let bytes = self.map.get(&handle.slot).ok_or(Error::InvalidJob)?;
152        let record = self.decode(&bytes)?;
153        if record.generation != handle.generation {
154            return Err(Error::InvalidJob);
155        }
156        Ok(inspect(record))
157    }
158
159    pub(super) fn save(&mut self, handle: &JobHandle, record: &Record) -> Result<(), Error> {
160        let bytes = record.encode()?;
161        // Prepare every refusal before publishing the wider bound. Both writes
162        // share the enclosing message's rollback boundary, just like insertion.
163        if self.legacy {
164            let (incarnation, generation, _) =
165                read_header(&self.map.get(&0).ok_or(Error::Corrupt)?)?;
166            self.map.insert(0, header(incarnation, generation));
167        }
168        self.map.insert(handle.slot, bytes);
169        self.legacy = false;
170        Ok(())
171    }
172
173    fn decode<'a>(&self, bytes: &'a [u8]) -> Result<Record<&'a [u8]>, Error> {
174        if self.legacy && bytes.len() > LEGACY_MAX_JOB_BYTES {
175            return Err(Error::Corrupt);
176        }
177        Record::decode(bytes)
178    }
179}
180fn header(incarnation: [u8; 16], generation: u64) -> Vec<u8> {
181    let mut bytes = MAGIC.to_vec();
182    bytes.extend(crate::codec::ROW_CODEC_VERSION.to_le_bytes());
183    bytes.extend(incarnation);
184    bytes.extend(generation.to_le_bytes());
185    bytes
186}
187
188fn read_header(bytes: &[u8]) -> Result<([u8; 16], u64, bool), Error> {
189    if bytes.len() != 36 {
190        return Err(Error::Corrupt);
191    }
192    if (&bytes[..8] != MAGIC && &bytes[..8] != LEGACY_MAGIC)
193        || bytes[8..12] != crate::codec::ROW_CODEC_VERSION.to_le_bytes()
194    {
195        return Err(Error::InvalidJob);
196    }
197    let incarnation = bytes[12..28].try_into().map_err(|_| Error::Corrupt)?;
198    if incarnation == [0; 16] {
199        return Err(Error::Corrupt);
200    }
201    Ok((
202        incarnation,
203        u64::from_le_bytes(bytes[28..].try_into().map_err(|_| Error::Corrupt)?),
204        &bytes[..8] == LEGACY_MAGIC,
205    ))
206}
207
208// The byte boundary has two representations: a validated borrowed frame for
209// inspection, and owned cursor/state for advancing a job. Execution stays shared.
210pub(super) struct Record<Bytes = Vec<u8>> {
211    pub generation: u64,
212    pub binding: [u8; 32],
213    pub revision: u64,
214    pub expires: Timestamp,
215    pub observed: Timestamp,
216    pub sequence: u64,
217    pub examined: u64,
218    pub input_bytes: u64,
219    pub phase: JobOutcome<()>,
220    pub cursor: Bytes,
221    pub count: u64,
222    // Interpretation belongs to the query-bound accumulator, not the job store.
223    pub state: Bytes,
224}
225impl Record {
226    fn encode(&self) -> Result<Vec<u8>, Error> {
227        let mut e = Encoder::with_byte_limit(MAX_JOB_BYTES);
228        e.nat64(self.generation)?;
229        e.bytes(&self.binding)?;
230        e.nat64(self.revision)?;
231        e.timestamp(self.expires)?;
232        e.timestamp(self.observed)?;
233        e.nat64(self.sequence)?;
234        e.nat64(self.examined)?;
235        e.nat64(self.input_bytes)?;
236        e.nat8(match self.phase {
237            JobOutcome::Running => 0,
238            JobOutcome::Complete(()) => 1,
239            JobOutcome::Invalidated => 2,
240            JobOutcome::Expired => 3,
241        })?;
242        e.bytes(&self.cursor)?;
243        e.nat64(self.count)?;
244        e.bytes(&self.state)?;
245        Ok(e.finish())
246    }
247}
248impl<'a> Record<&'a [u8]> {
249    fn decode(bytes: &'a [u8]) -> Result<Self, Error> {
250        let mut d = Decoder::with_byte_limit(bytes, MAX_JOB_BYTES)?;
251        let record = Self {
252            generation: d.nat64()?,
253            binding: d.blob_ref(32)?.try_into().map_err(|_| Error::Corrupt)?,
254            revision: d.nat64()?,
255            expires: d.timestamp()?,
256            observed: d.timestamp()?,
257            sequence: d.nat64()?,
258            examined: d.nat64()?,
259            input_bytes: d.nat64()?,
260            phase: match d.nat8()? {
261                0 => JobOutcome::Running,
262                1 => JobOutcome::Complete(()),
263                2 => JobOutcome::Invalidated,
264                3 => JobOutcome::Expired,
265                _ => return Err(Error::Corrupt),
266            },
267            cursor: d.blob_ref(QueryCursor::MAX_ENCODED_BYTES)?,
268            count: d.nat64()?,
269            state: d.blob_ref(MAX_JOB_BYTES)?,
270        };
271        d.finish()?;
272        if record.generation == 0
273            || record.examined > MAX_CANDIDATES
274            || record.count > record.examined
275            || record.sequence > record.examined + 2
276            || record.input_bytes > record.examined * super::super::super::CANDIDATE_BYTES as u64
277            || (matches!(record.phase, JobOutcome::Complete(())) && !record.cursor.is_empty())
278            || (matches!(record.phase, JobOutcome::Running)
279                && (record.observed >= record.expires
280                    || (record.examined > 0 && record.cursor.is_empty())))
281        {
282            return Err(Error::Corrupt);
283        }
284        Ok(record)
285    }
286
287    fn into_owned(self) -> Record {
288        Record {
289            generation: self.generation,
290            binding: self.binding,
291            revision: self.revision,
292            expires: self.expires,
293            observed: self.observed,
294            sequence: self.sequence,
295            examined: self.examined,
296            input_bytes: self.input_bytes,
297            phase: self.phase,
298            cursor: self.cursor.to_vec(),
299            count: self.count,
300            state: self.state.to_vec(),
301        }
302    }
303}