Skip to main content

fast_cache/storage/embedded_store/
session_slots.rs

1use hashbrown::{HashMap as HashBrownMap, HashTable};
2use std::mem;
3use std::sync::atomic::{AtomicUsize, Ordering};
4
5use crate::config::EvictionPolicy;
6use crate::storage::flat_map::EvictionRank;
7use crate::storage::{Bytes, hash_key};
8
9#[derive(Debug)]
10pub(super) struct SessionSlotEntry {
11    pub(super) hash: u64,
12    pub(super) key: Box<[u8]>,
13    pub(super) value: SessionSlotValue,
14    pub(super) access: SessionAccessMeta,
15}
16
17impl SessionSlotEntry {
18    #[inline(always)]
19    pub(super) fn matches(&self, hash: u64, key: &[u8]) -> bool {
20        self.hash == hash && self.key.as_ref() == key
21    }
22}
23
24#[derive(Debug)]
25pub(super) enum SessionSlotValue {
26    Owned(Box<[u8]>),
27    Packed { offset: u32, len: u32 },
28}
29
30impl SessionSlotValue {
31    #[inline(always)]
32    pub(super) fn len(&self) -> usize {
33        match self {
34            Self::Owned(bytes) => bytes.len(),
35            Self::Packed { len, .. } => *len as usize,
36        }
37    }
38}
39
40#[derive(Debug, Clone, Copy, Default)]
41pub(super) struct SessionAccessMeta {
42    last_touch: u64,
43    frequency: u32,
44}
45
46#[cfg(feature = "embedded")]
47#[derive(Debug)]
48pub(super) struct SessionPackedViewMeta {
49    pub(super) buffer: bytes::Bytes,
50    pub(super) offsets: Vec<usize>,
51    pub(super) lengths: Vec<usize>,
52    pub(super) hit_count: usize,
53    pub(super) total_bytes: usize,
54}
55
56impl SessionAccessMeta {
57    #[inline(always)]
58    pub(super) fn record_access(&mut self, tick: u64) {
59        self.last_touch = tick;
60        self.frequency = self.frequency.saturating_add(1).max(1);
61    }
62
63    #[inline(always)]
64    pub(super) fn rank(&self, policy: EvictionPolicy) -> EvictionRank {
65        match policy {
66            EvictionPolicy::None => EvictionRank {
67                primary: u64::MAX,
68                secondary: u64::MAX,
69            },
70            EvictionPolicy::Lru => EvictionRank {
71                primary: self.last_touch,
72                secondary: 0,
73            },
74            EvictionPolicy::Lfu => EvictionRank {
75                primary: self.frequency as u64,
76                secondary: self.last_touch,
77            },
78        }
79    }
80}
81
82#[derive(Debug, Default)]
83pub(super) struct SessionSlotSlab {
84    pub(super) entries: HashTable<SessionSlotEntry>,
85    pub(super) packed_values: Vec<u8>,
86    pub(super) stored_bytes: usize,
87}
88
89#[derive(Debug, Default)]
90pub(crate) struct SessionSlotMap {
91    sessions: HashBrownMap<Bytes, SessionSlotSlab, xxhash_rust::xxh3::Xxh3DefaultBuilder>,
92    active_readers: AtomicUsize,
93    retired_values: Vec<Box<[u8]>>,
94    retired_slabs: Vec<SessionSlotSlab>,
95    stored_bytes: usize,
96    track_access: bool,
97    sample_reads: bool,
98    access_clock: u64,
99    read_sample_counter: u64,
100    evictions: u64,
101}
102
103/// Packed write buffer for replacing one session's chunk set.
104///
105/// A packed write stores all values for a session in one slab, reducing
106/// allocator work for KV-cache style batches.
107#[derive(Debug)]
108pub struct PackedSessionWrite {
109    pub(super) session_prefix: Bytes,
110    pub(super) slab: SessionSlotSlab,
111}
112
113impl PackedSessionWrite {
114    /// Creates an empty packed session write with preallocated capacities.
115    pub fn with_capacity(
116        session_prefix: Bytes,
117        item_capacity: usize,
118        value_bytes_capacity: usize,
119    ) -> Self {
120        Self {
121            session_prefix,
122            slab: SessionSlotSlab {
123                entries: HashTable::with_capacity(item_capacity),
124                packed_values: Vec::with_capacity(value_bytes_capacity),
125                stored_bytes: 0,
126            },
127        }
128    }
129
130    /// Builds a packed session write from owned key/value pairs.
131    pub fn from_owned_items(session_prefix: Bytes, items: Vec<(Bytes, Bytes)>) -> Self {
132        let total_value_bytes = items.iter().map(|(_, value)| value.len()).sum::<usize>();
133        let mut packed = Self::with_capacity(session_prefix, items.len(), total_value_bytes);
134        for (key, value) in items {
135            packed.push_owned_record(key, value);
136        }
137        packed
138    }
139
140    /// Returns the number of records in the packed write.
141    #[inline(always)]
142    pub fn item_count(&self) -> usize {
143        self.slab.entries.len()
144    }
145
146    /// Returns the number of value bytes stored in the packed write.
147    #[inline(always)]
148    pub fn stored_bytes(&self) -> usize {
149        self.slab.stored_bytes
150    }
151
152    /// Returns the session prefix this write replaces.
153    #[inline(always)]
154    pub fn session_prefix(&self) -> &[u8] {
155        &self.session_prefix
156    }
157
158    /// Returns the length of the contiguous value buffer.
159    #[inline(always)]
160    pub fn value_buffer_len(&self) -> usize {
161        self.slab.packed_values.len()
162    }
163
164    /// Returns the mutable contiguous value buffer for custom packing.
165    #[inline(always)]
166    pub fn value_buffer_mut(&mut self) -> &mut Vec<u8> {
167        &mut self.slab.packed_values
168    }
169
170    /// Appends an owned key/value record to the packed write.
171    pub fn push_owned_record(&mut self, key: Bytes, value: Bytes) {
172        let tick = self.slab.entries.len() as u64 + 1;
173        self.slab
174            .set_packed_hashed(hash_key(&key), key, value, tick);
175    }
176
177    /// Appends a key whose value already lives in the packed value buffer.
178    pub fn push_prepacked_record(&mut self, key: Bytes, offset: usize, len: usize) {
179        let tick = self.slab.entries.len() as u64 + 1;
180        self.slab
181            .set_prepacked_hashed(hash_key(&key), key, offset, len, tick);
182    }
183
184    /// Returns cloned key/value records for inspection or persistence.
185    pub fn cloned_records(&self) -> Vec<(Bytes, Bytes)> {
186        self.slab
187            .entries
188            .iter()
189            .map(|entry| {
190                (
191                    entry.key.to_vec(),
192                    self.slab.entry_value_slice(entry).to_vec(),
193                )
194            })
195            .collect()
196    }
197
198    pub(super) fn into_parts(self) -> (Bytes, SessionSlotSlab) {
199        (self.session_prefix, self.slab)
200    }
201}
202
203impl SessionSlotSlab {
204    #[inline(always)]
205    pub(super) fn entry_value_slice<'a>(&'a self, entry: &'a SessionSlotEntry) -> &'a [u8] {
206        match &entry.value {
207            SessionSlotValue::Owned(bytes) => bytes.as_ref(),
208            SessionSlotValue::Packed { offset, len } => {
209                let offset = *offset as usize;
210                let len = *len as usize;
211                &self.packed_values[offset..offset + len]
212            }
213        }
214    }
215
216    pub(super) fn set_packed_hashed(&mut self, hash: u64, key: Bytes, value: Bytes, tick: u64) {
217        let key_ref = key.as_slice();
218        let key_len = key.len();
219        let value_len = value.len();
220
221        match self.entries.entry(
222            hash,
223            |entry| entry.matches(hash, key_ref),
224            |entry| entry.hash,
225        ) {
226            hashbrown::hash_table::Entry::Occupied(mut occupied) => {
227                let entry = occupied.get_mut();
228                let previous_value_len = entry.value.len();
229                let offset = self.packed_values.len();
230                self.packed_values.extend_from_slice(&value);
231                entry.value = SessionSlotValue::Packed {
232                    offset: offset as u32,
233                    len: value_len as u32,
234                };
235                entry.access.record_access(tick);
236                self.stored_bytes = self
237                    .stored_bytes
238                    .saturating_sub(previous_value_len)
239                    .saturating_add(value_len);
240            }
241            hashbrown::hash_table::Entry::Vacant(vacant) => {
242                let offset = self.packed_values.len();
243                self.packed_values.extend_from_slice(&value);
244                vacant.insert(SessionSlotEntry {
245                    hash,
246                    key: key.into_boxed_slice(),
247                    value: SessionSlotValue::Packed {
248                        offset: offset as u32,
249                        len: value_len as u32,
250                    },
251                    access: SessionAccessMeta {
252                        last_touch: tick,
253                        frequency: 1,
254                    },
255                });
256                self.stored_bytes = self
257                    .stored_bytes
258                    .saturating_add(key_len)
259                    .saturating_add(value_len);
260            }
261        }
262    }
263
264    pub(super) fn set_prepacked_hashed(
265        &mut self,
266        hash: u64,
267        key: Bytes,
268        offset: usize,
269        len: usize,
270        tick: u64,
271    ) {
272        let key_ref = key.as_slice();
273        let key_len = key.len();
274        let value_ref = SessionSlotValue::Packed {
275            offset: offset as u32,
276            len: len as u32,
277        };
278
279        match self.entries.entry(
280            hash,
281            |entry| entry.matches(hash, key_ref),
282            |entry| entry.hash,
283        ) {
284            hashbrown::hash_table::Entry::Occupied(mut occupied) => {
285                let entry = occupied.get_mut();
286                let previous_value_len = entry.value.len();
287                entry.value = value_ref;
288                entry.access.record_access(tick);
289                self.stored_bytes = self
290                    .stored_bytes
291                    .saturating_sub(previous_value_len)
292                    .saturating_add(len);
293            }
294            hashbrown::hash_table::Entry::Vacant(vacant) => {
295                vacant.insert(SessionSlotEntry {
296                    hash,
297                    key: key.into_boxed_slice(),
298                    value: value_ref,
299                    access: SessionAccessMeta {
300                        last_touch: tick,
301                        frequency: 1,
302                    },
303                });
304                self.stored_bytes = self
305                    .stored_bytes
306                    .saturating_add(key_len)
307                    .saturating_add(len);
308            }
309        }
310    }
311}
312
313impl SessionSlotMap {
314    #[inline(always)]
315    pub(super) fn is_empty(&self) -> bool {
316        self.sessions.is_empty()
317    }
318
319    #[inline(always)]
320    pub(super) fn len(&self) -> usize {
321        self.sessions.values().map(|slab| slab.entries.len()).sum()
322    }
323
324    #[inline(always)]
325    pub(super) fn retire_value(&mut self, value: Box<[u8]>) {
326        if self.has_active_readers() {
327            self.retired_values.push(value);
328        }
329    }
330
331    #[inline(always)]
332    pub(super) fn retire_slab(&mut self, slab: SessionSlotSlab) {
333        if self.has_active_readers() {
334            self.retired_slabs.push(slab);
335        }
336    }
337
338    #[inline(always)]
339    pub(super) fn has_active_readers(&self) -> bool {
340        self.active_readers.load(Ordering::Acquire) > 0
341    }
342
343    #[inline(always)]
344    pub(super) fn reclaim_retired_if_quiescent(&mut self) {
345        if !self.has_active_readers() {
346            if !self.retired_values.is_empty() {
347                self.retired_values.clear();
348            }
349            if !self.retired_slabs.is_empty() {
350                self.retired_slabs.clear();
351            }
352        }
353    }
354
355    #[inline(always)]
356    pub(super) fn stored_bytes(&self) -> usize {
357        self.stored_bytes
358    }
359
360    #[inline(always)]
361    pub(super) fn configure_access_tracking(&mut self, enabled: bool) {
362        self.track_access = enabled;
363    }
364
365    #[inline(always)]
366    pub(super) fn configure_read_sampling(&mut self, enabled: bool) {
367        self.sample_reads = enabled;
368    }
369
370    #[inline(always)]
371    pub(super) fn has_session(&self, session_prefix: &[u8]) -> bool {
372        self.sessions.contains_key(session_prefix)
373    }
374
375    #[inline(always)]
376    pub(super) fn get_ref_hashed_shared(
377        &self,
378        session_prefix: &[u8],
379        hash: u64,
380        key: &[u8],
381    ) -> Option<&[u8]> {
382        self.get_ref_hashed_shared_prehashed(hash_key(session_prefix), session_prefix, hash, key)
383    }
384
385    #[inline(always)]
386    pub(super) fn get_ref_hashed_shared_prehashed(
387        &self,
388        _session_hash: u64,
389        session_prefix: &[u8],
390        hash: u64,
391        key: &[u8],
392    ) -> Option<&[u8]> {
393        self.sessions
394            .raw_entry()
395            .from_key(session_prefix)
396            .and_then(|(_, slab)| {
397                slab.entries
398                    .find(hash, |entry| entry.matches(hash, key))
399                    .map(|entry| slab.entry_value_slice(entry))
400            })
401    }
402
403    #[inline(always)]
404    pub(super) fn get_ref_hashed(
405        &mut self,
406        session_prefix: &[u8],
407        hash: u64,
408        key: &[u8],
409    ) -> Option<&[u8]> {
410        if self.should_sample_read() {
411            let tick = self.next_access_tick();
412            self.sessions.get_mut(session_prefix).and_then(|slab| {
413                let SessionSlotSlab {
414                    entries,
415                    packed_values,
416                    ..
417                } = slab;
418                let entry = entries.find_mut(hash, |entry| entry.matches(hash, key))?;
419                entry.access.record_access(tick);
420                Some(match &entry.value {
421                    SessionSlotValue::Owned(bytes) => bytes.as_ref(),
422                    SessionSlotValue::Packed { offset, len } => {
423                        let offset = *offset as usize;
424                        let len = *len as usize;
425                        &packed_values[offset..offset + len]
426                    }
427                })
428            })
429        } else {
430            self.sessions.get(session_prefix).and_then(|slab| {
431                slab.entries
432                    .find(hash, |entry| entry.matches(hash, key))
433                    .map(|entry| slab.entry_value_slice(entry))
434            })
435        }
436    }
437
438    #[cfg(feature = "embedded")]
439    #[inline(always)]
440    pub(super) fn get_ref_hashed_local(
441        &mut self,
442        session_prefix: &[u8],
443        hash: u64,
444        key: &[u8],
445    ) -> Option<&[u8]> {
446        if self.should_sample_read() {
447            let tick = self.next_access_tick();
448            self.sessions.get_mut(session_prefix).and_then(|slab| {
449                let SessionSlotSlab {
450                    entries,
451                    packed_values,
452                    ..
453                } = slab;
454                let entry = entries.find_mut(hash, |entry| entry.matches(hash, key))?;
455                entry.access.record_access(tick);
456                Some(match &entry.value {
457                    SessionSlotValue::Owned(bytes) => bytes.as_ref(),
458                    SessionSlotValue::Packed { offset, len } => {
459                        let offset = *offset as usize;
460                        let len = *len as usize;
461                        &packed_values[offset..offset + len]
462                    }
463                })
464            })
465        } else {
466            self.sessions.get(session_prefix).and_then(|slab| {
467                slab.entries
468                    .find(hash, |entry| entry.matches(hash, key))
469                    .map(|entry| slab.entry_value_slice(entry))
470            })
471        }
472    }
473
474    #[cfg(feature = "embedded")]
475    pub(super) fn get_packed_view_hashed_local(
476        &mut self,
477        session_prefix: &[u8],
478        keys: &[Bytes],
479        key_hashes: &[u64],
480    ) -> Option<SessionPackedViewMeta> {
481        if keys.len() != key_hashes.len() {
482            return None;
483        }
484
485        if self.should_sample_read() {
486            let tick = self.next_access_tick();
487            let slab = self.sessions.get_mut(session_prefix)?;
488            let mut offsets = Vec::with_capacity(keys.len());
489            let mut lengths = Vec::with_capacity(keys.len());
490            let mut hit_count = 0usize;
491            let mut total_bytes = 0usize;
492            for (key, key_hash) in keys.iter().zip(key_hashes.iter().copied()) {
493                let Some(entry) = slab
494                    .entries
495                    .find_mut(key_hash, |entry| entry.matches(key_hash, key))
496                else {
497                    offsets.push(usize::MAX);
498                    lengths.push(0);
499                    continue;
500                };
501                entry.access.record_access(tick);
502                let SessionSlotValue::Packed { offset, len } = entry.value else {
503                    return None;
504                };
505                let offset = offset as usize;
506                let len = len as usize;
507                offsets.push(offset);
508                lengths.push(len);
509                hit_count = hit_count.saturating_add(1);
510                total_bytes = total_bytes.saturating_add(len);
511            }
512            Some(SessionPackedViewMeta {
513                buffer: bytes::Bytes::copy_from_slice(&slab.packed_values),
514                offsets,
515                lengths,
516                hit_count,
517                total_bytes,
518            })
519        } else {
520            let slab = self.sessions.get(session_prefix)?;
521            let mut offsets = Vec::with_capacity(keys.len());
522            let mut lengths = Vec::with_capacity(keys.len());
523            let mut hit_count = 0usize;
524            let mut total_bytes = 0usize;
525            for (key, key_hash) in keys.iter().zip(key_hashes.iter().copied()) {
526                let Some(entry) = slab
527                    .entries
528                    .find(key_hash, |entry| entry.matches(key_hash, key))
529                else {
530                    offsets.push(usize::MAX);
531                    lengths.push(0);
532                    continue;
533                };
534                let SessionSlotValue::Packed { offset, len } = entry.value else {
535                    return None;
536                };
537                let offset = offset as usize;
538                let len = len as usize;
539                offsets.push(offset);
540                lengths.push(len);
541                hit_count = hit_count.saturating_add(1);
542                total_bytes = total_bytes.saturating_add(len);
543            }
544            Some(SessionPackedViewMeta {
545                buffer: bytes::Bytes::copy_from_slice(&slab.packed_values),
546                offsets,
547                lengths,
548                hit_count,
549                total_bytes,
550            })
551        }
552    }
553
554    pub(super) fn set_slice_hashed(
555        &mut self,
556        session_prefix: &[u8],
557        hash: u64,
558        key: &[u8],
559        value: &[u8],
560    ) {
561        self.set_slice_hashed_prehashed(hash_key(session_prefix), session_prefix, hash, key, value);
562    }
563
564    pub(super) fn set_slice_hashed_prehashed(
565        &mut self,
566        _session_hash: u64,
567        session_prefix: &[u8],
568        hash: u64,
569        key: &[u8],
570        value: &[u8],
571    ) {
572        self.reclaim_retired_if_quiescent();
573        let has_active_readers = self.has_active_readers();
574        let tick = if self.track_access {
575            self.next_access_tick()
576        } else {
577            0
578        };
579        let slab = match self.sessions.raw_entry_mut().from_key(session_prefix) {
580            hashbrown::hash_map::RawEntryMut::Occupied(occupied) => occupied.into_mut(),
581            hashbrown::hash_map::RawEntryMut::Vacant(vacant) => {
582                vacant
583                    .insert(session_prefix.to_vec(), SessionSlotSlab::default())
584                    .1
585            }
586        };
587
588        match slab
589            .entries
590            .entry(hash, |entry| entry.matches(hash, key), |entry| entry.hash)
591        {
592            hashbrown::hash_table::Entry::Occupied(mut occupied) => {
593                let mut retired_value = None;
594                let entry = occupied.get_mut();
595                let previous_value_len = entry.value.len();
596                if !has_active_readers
597                    && matches!(&entry.value, SessionSlotValue::Owned(existing) if existing.len() == value.len())
598                {
599                    if let SessionSlotValue::Owned(existing) = &mut entry.value {
600                        existing.copy_from_slice(value);
601                    }
602                } else {
603                    let old = mem::replace(
604                        &mut entry.value,
605                        SessionSlotValue::Owned(value.to_vec().into_boxed_slice()),
606                    );
607                    if let SessionSlotValue::Owned(old_value) = old {
608                        retired_value = Some(old_value);
609                    }
610                }
611                entry.access.record_access(tick);
612                slab.stored_bytes = slab
613                    .stored_bytes
614                    .saturating_sub(previous_value_len)
615                    .saturating_add(entry.value.len());
616                self.stored_bytes = self
617                    .stored_bytes
618                    .saturating_sub(previous_value_len)
619                    .saturating_add(entry.value.len());
620                if let Some(old_value) = retired_value {
621                    self.retire_value(old_value);
622                }
623            }
624            hashbrown::hash_table::Entry::Vacant(vacant) => {
625                let key_len = key.len();
626                let value_len = value.len();
627                vacant.insert(SessionSlotEntry {
628                    hash,
629                    key: key.to_vec().into_boxed_slice(),
630                    value: SessionSlotValue::Owned(value.to_vec().into_boxed_slice()),
631                    access: SessionAccessMeta {
632                        last_touch: tick,
633                        frequency: 1,
634                    },
635                });
636                slab.stored_bytes = slab
637                    .stored_bytes
638                    .saturating_add(key_len)
639                    .saturating_add(value_len);
640                self.stored_bytes = self
641                    .stored_bytes
642                    .saturating_add(key_len)
643                    .saturating_add(value_len);
644            }
645        }
646    }
647
648    pub(super) fn delete_hashed(&mut self, session_prefix: &[u8], hash: u64, key: &[u8]) -> bool {
649        self.reclaim_retired_if_quiescent();
650        let mut removed_value = None;
651        let deleted = match self.sessions.entry(session_prefix.to_vec()) {
652            hashbrown::hash_map::Entry::Occupied(mut session) => {
653                let (value, remove_session) = {
654                    let slab = session.get_mut();
655                    let Some(entry) = slab
656                        .entries
657                        .find_entry(hash, |entry| entry.matches(hash, key))
658                        .ok()
659                    else {
660                        return false;
661                    };
662
663                    let (removed, _) = entry.remove();
664                    slab.stored_bytes = slab
665                        .stored_bytes
666                        .saturating_sub(removed.key.len().saturating_add(removed.value.len()));
667                    self.stored_bytes = self
668                        .stored_bytes
669                        .saturating_sub(removed.key.len().saturating_add(removed.value.len()));
670                    (removed.value, slab.entries.is_empty())
671                };
672
673                removed_value = Some(value);
674                if remove_session {
675                    session.remove();
676                }
677                true
678            }
679            hashbrown::hash_map::Entry::Vacant(_) => false,
680        };
681        if let Some(SessionSlotValue::Owned(value)) = removed_value {
682            self.retire_value(value);
683        }
684        deleted
685    }
686
687    #[cfg(feature = "embedded")]
688    pub(super) fn delete_hashed_local(
689        &mut self,
690        session_prefix: &[u8],
691        hash: u64,
692        key: &[u8],
693    ) -> bool {
694        self.reclaim_retired_if_quiescent();
695        let mut removed_value = None;
696        let mut removed_slab = None;
697        let deleted = match self.sessions.entry(session_prefix.to_vec()) {
698            hashbrown::hash_map::Entry::Occupied(mut session) => {
699                let remove_session = {
700                    let slab = session.get_mut();
701                    let Some(entry) = slab
702                        .entries
703                        .find_entry(hash, |entry| entry.matches(hash, key))
704                        .ok()
705                    else {
706                        return false;
707                    };
708
709                    let removed_key_len = entry.get().key.len();
710                    let removed_value_len = entry.get().value.len();
711                    let (removed, _) = entry.remove();
712                    removed_value = Some(removed.value);
713                    slab.stored_bytes = slab
714                        .stored_bytes
715                        .saturating_sub(removed_key_len.saturating_add(removed_value_len));
716                    self.stored_bytes = self
717                        .stored_bytes
718                        .saturating_sub(removed_key_len.saturating_add(removed_value_len));
719                    slab.entries.is_empty()
720                };
721
722                if remove_session {
723                    removed_slab = Some(session.remove());
724                }
725                true
726            }
727            hashbrown::hash_map::Entry::Vacant(_) => false,
728        };
729        if let Some(SessionSlotValue::Owned(value)) = removed_value {
730            self.retire_value(value);
731        }
732        if let Some(slab) = removed_slab {
733            self.retire_slab(slab);
734        }
735        deleted
736    }
737
738    pub(super) fn replace_session_slab(&mut self, packed: PackedSessionWrite) {
739        self.reclaim_retired_if_quiescent();
740        let (session_prefix, slab) = packed.into_parts();
741        let new_bytes = slab.stored_bytes;
742        let previous = self.sessions.insert(session_prefix, slab);
743        if let Some(previous) = previous {
744            self.stored_bytes = self
745                .stored_bytes
746                .saturating_sub(previous.stored_bytes)
747                .saturating_add(new_bytes);
748            self.retire_slab(previous);
749        } else {
750            self.stored_bytes = self.stored_bytes.saturating_add(new_bytes);
751        }
752    }
753
754    #[cfg(feature = "embedded")]
755    pub(super) fn replace_session_slab_local(&mut self, packed: PackedSessionWrite) {
756        self.reclaim_retired_if_quiescent();
757        let (session_prefix, slab) = packed.into_parts();
758        let new_bytes = slab.stored_bytes;
759        let previous = self.sessions.insert(session_prefix, slab);
760        if let Some(previous) = previous {
761            self.stored_bytes = self
762                .stored_bytes
763                .saturating_sub(previous.stored_bytes)
764                .saturating_add(new_bytes);
765            self.retire_slab(previous);
766        } else {
767            self.stored_bytes = self.stored_bytes.saturating_add(new_bytes);
768        }
769    }
770
771    #[inline(always)]
772    pub(super) fn next_access_tick(&mut self) -> u64 {
773        self.access_clock = self.access_clock.saturating_add(1);
774        self.access_clock
775    }
776
777    #[inline(always)]
778    pub(super) fn should_sample_read(&mut self) -> bool {
779        const READ_TOUCH_SAMPLE_MASK: u64 = 1023;
780        if !self.track_access || !self.sample_reads {
781            return false;
782        }
783        self.read_sample_counter = self.read_sample_counter.saturating_add(1);
784        (self.read_sample_counter & READ_TOUCH_SAMPLE_MASK) == 0
785    }
786
787    pub(super) fn eviction_candidate(
788        &self,
789        policy: EvictionPolicy,
790    ) -> Option<(EvictionRank, Bytes, u64, Bytes)> {
791        if policy == EvictionPolicy::None {
792            return None;
793        }
794
795        self.sessions
796            .iter()
797            .flat_map(|(session_prefix, slab)| {
798                slab.entries.iter().map(|entry| {
799                    (
800                        entry.access.rank(policy),
801                        session_prefix.clone(),
802                        entry.hash,
803                        entry.key.as_ref().to_vec(),
804                    )
805                })
806            })
807            .min_by_key(|(rank, _, _, _)| *rank)
808    }
809
810    pub(super) fn evict_with_policy(&mut self, policy: EvictionPolicy) -> bool {
811        let Some((_rank, session_prefix, hash, key)) = self.eviction_candidate(policy) else {
812            return false;
813        };
814        let deleted = self.delete_hashed(&session_prefix, hash, &key);
815        if deleted {
816            self.evictions = self.evictions.saturating_add(1);
817        }
818        deleted
819    }
820}