Skip to main content

ic_db/store/query/aggregate/jobs/
storage.rs

1//! Fixed slots in one host-owned allocation; bounded frames use the owned codec.
2use super::{JobHandle, JobOutcome, MAX_CANDIDATES};
3use crate::{
4    Error, QueryCursor, Timestamp,
5    codec::{Decoder, Encoder},
6};
7use ic_storage::{DefaultMemoryImpl, Memory, RuntimeMemory, adapter::StableBTreeMap};
8
9pub(super) const SLOTS: u8 = 16;
10pub(super) const MAX_JOB_BYTES: usize = 16 * 1024;
11const LEGACY_MAX_JOB_BYTES: usize = 4096;
12const MAGIC: &[u8; 8] = b"ICDBAJ02";
13const LEGACY_MAGIC: &[u8; 8] = b"ICDBAJ01";
14
15/// Durable bounded aggregates in a host-governed allocation.
16/// Holds at most 16 jobs, each at most 16 KiB encoded bytes. Creation requires a
17/// fresh nonzero incarnation. Handles identify jobs; they do not authorize access.
18/// All mutation must run synchronously within one IC message, without awaits.
19/// Supplier failures trap; a later message trap rolls these changes back too.
20pub struct AggregateJobs<M: Memory = DefaultMemoryImpl> {
21    map: StableBTreeMap<u8, Vec<u8>, M>,
22    incarnation: [u8; 16],
23    legacy: bool,
24}
25impl<M: Memory> AggregateJobs<M> {
26    /// Create in an empty allocation, with a host-supplied fresh incarnation.
27    /// # Errors
28    /// Refuses a zero incarnation or a nonempty/unsupported supplier allocation.
29    /// # Panics
30    /// Supplier failures after mutation begins must trap the enclosing message.
31    pub fn create(memory: RuntimeMemory<M>, incarnation: [u8; 16]) -> Result<Self, Error> {
32        if incarnation == [0; 16] {
33            return Err(Error::InvalidJob);
34        }
35        if memory.size() != 0 {
36            return Err(Error::Exists);
37        }
38        let mut map = StableBTreeMap::create(memory);
39        map.insert(0, header(incarnation, 0));
40        Ok(Self {
41            map,
42            incarnation,
43            legacy: false,
44        })
45    }
46
47    /// Reopen a supported job-store format without modifying it. Version-one
48    /// jobs remain readable; the first successful insert/save atomically admits
49    /// version two and its larger frame bound. Older executables then refuse.
50    /// # Errors
51    /// Refuses missing/corrupt headers and unsupported formats intact.
52    /// # Panics
53    /// Supplier corruption traps without repair.
54    pub fn open(memory: RuntimeMemory<M>) -> Result<Self, Error> {
55        if memory.size() == 0 {
56            return Err(Error::Corrupt);
57        }
58        let map: StableBTreeMap<u8, Vec<u8>, M> = StableBTreeMap::load(memory);
59        let bytes = map.get(&0).ok_or(Error::Corrupt)?;
60        let (incarnation, _, legacy) = read_header(&bytes)?;
61        if map.len() > u64::from(SLOTS) + 1
62            || map
63                .range(..)
64                .next_back()
65                .is_some_and(|entry| *entry.key() > SLOTS)
66        {
67            return Err(Error::Corrupt);
68        }
69        Ok(Self {
70            map,
71            incarnation,
72            legacy,
73        })
74    }
75
76    /// Cancel one job, immediately releasing its logical slot. Repeated/stale
77    /// cancellation refuses; it cannot delete a subsequent occupant of that slot.
78    /// # Errors
79    /// Refuses a foreign, missing, stale or malformed handle/job frame. Cancellation
80    /// need not interpret a known frame's query-specific accumulator payload.
81    /// # Panics
82    /// Supplier mutation failures trap the enclosing message.
83    pub fn cancel(&mut self, handle: &JobHandle) -> Result<(), Error> {
84        self.with_record(handle, |_| ())?;
85        self.map.remove(&handle.slot);
86        Ok(())
87    }
88
89    /// Examine at most 16 slots and remove at most `limit` expired jobs (1..=16).
90    /// Time must be captured by the host; the database reads no clock. This does
91    /// not shrink stable memory or promise physical reclamation by the supplier.
92    /// # Errors
93    /// Refuses invalid limits or malformed records before removing any job.
94    /// # Panics
95    /// Supplier mutation failures trap the enclosing message.
96    pub fn cleanup_expired(&mut self, now: Timestamp, limit: usize) -> Result<usize, Error> {
97        if !(1..=usize::from(SLOTS)).contains(&limit) {
98            return Err(Error::Limit);
99        }
100        let mut expired = Vec::new();
101        for slot in 1..=SLOTS {
102            if let Some(bytes) = self.map.get(&slot) {
103                let record = self.decode(&bytes)?;
104                if record.expires <= now && expired.len() < limit {
105                    expired.push(slot);
106                }
107            }
108        }
109        for slot in &expired {
110            self.map.remove(slot);
111        }
112        Ok(expired.len())
113    }
114
115    pub(super) fn insert(&mut self, mut record: Record) -> Result<JobHandle, Error> {
116        let slot = (1..=SLOTS)
117            .find(|slot| !self.map.contains_key(slot))
118            .ok_or(Error::JobCapacity)?;
119        let (incarnation, generation, _) = read_header(&self.map.get(&0).ok_or(Error::Corrupt)?)?;
120        if incarnation != self.incarnation {
121            return Err(Error::Corrupt);
122        }
123        let generation = generation.checked_add(1).ok_or(Error::Limit)?;
124        record.generation = generation;
125        let bytes = record.encode()?;
126        // No recoverable refusal after the first persistent write.
127        self.map.insert(0, header(incarnation, generation));
128        self.map.insert(slot, bytes);
129        self.legacy = false;
130        Ok(JobHandle {
131            incarnation,
132            slot,
133            generation,
134        })
135    }
136
137    pub(super) fn load(&self, handle: &JobHandle) -> Result<Record, Error> {
138        self.with_record(handle, |record| record.into_owned())
139    }
140
141    // One identity/framing admission for owned resume and borrowed cancellation.
142    // Only resume needs owned cursor/accumulator bytes after the supplier buffer dies.
143    fn with_record<R>(
144        &self,
145        handle: &JobHandle,
146        inspect: impl FnOnce(Record<&[u8]>) -> R,
147    ) -> Result<R, Error> {
148        if handle.incarnation != self.incarnation {
149            return Err(Error::InvalidJob);
150        }
151        let bytes = self.map.get(&handle.slot).ok_or(Error::InvalidJob)?;
152        let record = self.decode(&bytes)?;
153        if record.generation != handle.generation {
154            return Err(Error::InvalidJob);
155        }
156        Ok(inspect(record))
157    }
158
159    pub(super) fn save(&mut self, handle: &JobHandle, record: &Record) -> Result<(), Error> {
160        let bytes = record.encode()?;
161        // Prepare every refusal before publishing the wider bound. Both writes
162        // share the enclosing message's rollback boundary, just like insertion.
163        if self.legacy {
164            let (incarnation, generation, _) =
165                read_header(&self.map.get(&0).ok_or(Error::Corrupt)?)?;
166            self.map.insert(0, header(incarnation, generation));
167        }
168        self.map.insert(handle.slot, bytes);
169        self.legacy = false;
170        Ok(())
171    }
172
173    fn decode<'a>(&self, bytes: &'a [u8]) -> Result<Record<&'a [u8]>, Error> {
174        if self.legacy && bytes.len() > LEGACY_MAX_JOB_BYTES {
175            return Err(Error::Corrupt);
176        }
177        Record::decode(bytes)
178    }
179}
180fn header(incarnation: [u8; 16], generation: u64) -> Vec<u8> {
181    let mut bytes = MAGIC.to_vec();
182    bytes.extend(crate::codec::ROW_CODEC_VERSION.to_le_bytes());
183    bytes.extend(incarnation);
184    bytes.extend(generation.to_le_bytes());
185    bytes
186}
187fn read_header(bytes: &[u8]) -> Result<([u8; 16], u64, bool), Error> {
188    if bytes.len() != 36 {
189        return Err(Error::Corrupt);
190    }
191    if (&bytes[..8] != MAGIC && &bytes[..8] != LEGACY_MAGIC)
192        || bytes[8..12] != crate::codec::ROW_CODEC_VERSION.to_le_bytes()
193    {
194        return Err(Error::InvalidJob);
195    }
196    let incarnation = bytes[12..28].try_into().map_err(|_| Error::Corrupt)?;
197    if incarnation == [0; 16] {
198        return Err(Error::Corrupt);
199    }
200    Ok((
201        incarnation,
202        u64::from_le_bytes(bytes[28..].try_into().map_err(|_| Error::Corrupt)?),
203        &bytes[..8] == LEGACY_MAGIC,
204    ))
205}
206
207// The byte boundary has two representations: a validated borrowed frame for
208// inspection, and owned cursor/state for advancing a job. Execution stays shared.
209pub(super) struct Record<Bytes = Vec<u8>> {
210    pub generation: u64,
211    pub binding: [u8; 32],
212    pub revision: u64,
213    pub expires: Timestamp,
214    pub observed: Timestamp,
215    pub sequence: u64,
216    pub examined: u64,
217    pub input_bytes: u64,
218    pub phase: JobOutcome<()>,
219    pub cursor: Bytes,
220    pub count: u64,
221    // Interpretation belongs to the query-bound accumulator, not the job store.
222    pub state: Bytes,
223}
224impl Record {
225    fn encode(&self) -> Result<Vec<u8>, Error> {
226        let mut e = Encoder::with_byte_limit(MAX_JOB_BYTES);
227        e.nat64(self.generation)?;
228        e.bytes(&self.binding)?;
229        e.nat64(self.revision)?;
230        e.timestamp(self.expires)?;
231        e.timestamp(self.observed)?;
232        e.nat64(self.sequence)?;
233        e.nat64(self.examined)?;
234        e.nat64(self.input_bytes)?;
235        e.nat8(match self.phase {
236            JobOutcome::Running => 0,
237            JobOutcome::Complete(()) => 1,
238            JobOutcome::Invalidated => 2,
239            JobOutcome::Expired => 3,
240        })?;
241        e.bytes(&self.cursor)?;
242        e.nat64(self.count)?;
243        e.bytes(&self.state)?;
244        Ok(e.finish())
245    }
246}
247impl<'a> Record<&'a [u8]> {
248    fn decode(bytes: &'a [u8]) -> Result<Self, Error> {
249        let mut d = Decoder::with_byte_limit(bytes, MAX_JOB_BYTES)?;
250        let record = Self {
251            generation: d.nat64()?,
252            binding: d.blob_ref(32)?.try_into().map_err(|_| Error::Corrupt)?,
253            revision: d.nat64()?,
254            expires: d.timestamp()?,
255            observed: d.timestamp()?,
256            sequence: d.nat64()?,
257            examined: d.nat64()?,
258            input_bytes: d.nat64()?,
259            phase: match d.nat8()? {
260                0 => JobOutcome::Running,
261                1 => JobOutcome::Complete(()),
262                2 => JobOutcome::Invalidated,
263                3 => JobOutcome::Expired,
264                _ => return Err(Error::Corrupt),
265            },
266            cursor: d.blob_ref(QueryCursor::MAX_ENCODED_BYTES)?,
267            count: d.nat64()?,
268            state: d.blob_ref(MAX_JOB_BYTES)?,
269        };
270        d.finish()?;
271        if record.generation == 0
272            || record.examined > MAX_CANDIDATES
273            || record.count > record.examined
274            || record.sequence > record.examined + 2
275            || record.input_bytes > record.examined * super::super::super::CANDIDATE_BYTES as u64
276            || (matches!(record.phase, JobOutcome::Complete(())) && !record.cursor.is_empty())
277            || (matches!(record.phase, JobOutcome::Running)
278                && (record.observed >= record.expires
279                    || (record.examined > 0 && record.cursor.is_empty())))
280        {
281            return Err(Error::Corrupt);
282        }
283        Ok(record)
284    }
285
286    fn into_owned(self) -> Record {
287        Record {
288            generation: self.generation,
289            binding: self.binding,
290            revision: self.revision,
291            expires: self.expires,
292            observed: self.observed,
293            sequence: self.sequence,
294            examined: self.examined,
295            input_bytes: self.input_bytes,
296            phase: self.phase,
297            cursor: self.cursor.to_vec(),
298            count: self.count,
299            state: self.state.to_vec(),
300        }
301    }
302}