mnemosyne-local 0.6.0

Thread-local allocation engine for Mnemosyne
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

use core::ptr::NonNull;
use mnemosyne_core::policy::AllocPolicy;
use mnemosyne_core::types::{Block, Page, Segment, try_pop_bump_block};

/// Allocates one block from a page-local free list or from that page's lazy
/// bump range. Returns `None` when the page has no local free block and no
/// uninitialized block remaining.
///
/// # Safety
///
/// The caller must own `page` through the current thread allocator.
#[inline(always)]
pub(crate) unsafe fn try_allocate_page_local<P: AllocPolicy>(
    page: *mut Page,
) -> Option<NonNull<Block>> {
    // SAFETY: forwarded.
    unsafe {
        try_allocate_page_local_dynamic(
            page,
            P::ENABLE_FREE_LIST_ENCRYPTION,
            P::RANDOMIZE_ALLOCATION,
        )
    }
}

/// Non-generic SSOT for `try_allocate_page_local`.
///
/// # Safety
///
/// Same contract as `try_allocate_page_local`.
#[inline(always)]
pub(crate) unsafe fn try_allocate_page_local_dynamic(
    page: *mut Page,
    enable_encryption: bool,
    randomize: bool,
) -> Option<NonNull<Block>> {
    // SAFETY: caller guarantees `page` identifies a live page it owns.
    unsafe {
        if (*page).free.is_none()
            && (*page).secondary_free.is_none()
            && (*page).initialized_blocks as usize >= (*page).max_blocks()
        {
            return None;
        }
        let block = if let Some(block) = try_pop_bump_block(page) {
            block
        } else if (*page).free.is_some() || (*page).secondary_free.is_some() {
            Page::pop_block_dynamic(page, enable_encryption, randomize)
        } else {
            return None;
        };
        if (*page).alloc_count == 0 {
            // SAFETY: `page` is a live, exclusively-owned page; `increment_alloc_count`
            // derives the parent segment and index internally.
            Page::increment_alloc_count(page);
        } else {
            (*page).alloc_count = ((*page).alloc_count as usize + 1) as u32;
        }
        Some(block)
    }
}

/// Reclaims any pending cross-thread frees on `page` and pops one block.
///
/// # Safety
///
/// Same contract as `Page::reclaim_thread_free_in_segment`.
#[inline(always)]
pub(crate) unsafe fn try_reclaim_and_allocate<P: AllocPolicy>(
    page: *mut Page,
    reclaim_sink: &mut usize,
) -> Option<NonNull<Block>> {
    // SAFETY: forwarded.
    unsafe {
        try_reclaim_and_allocate_dynamic(
            page,
            reclaim_sink,
            P::ENABLE_FREE_LIST_ENCRYPTION,
            P::RANDOMIZE_ALLOCATION,
        )
    }
}

/// Non-generic SSOT for `try_reclaim_and_allocate`.
///
/// # Safety
///
/// Same contract as `try_reclaim_and_allocate`.
#[inline(always)]
pub(crate) unsafe fn try_reclaim_and_allocate_dynamic(
    page: *mut Page,
    reclaim_sink: &mut usize,
    enable_encryption: bool,
    randomize: bool,
) -> Option<NonNull<Block>> {
    // SAFETY: caller guarantees `page` identifies a live page it owns.
    let (segment, page_index) = unsafe {
        if (*page).thread_free.is_empty() {
            return None;
        }
        (Page::parent_segment_of(page), (*page).index_in_segment())
    };

    // SAFETY: `parent_segment`/`index_in_segment` name this page's parent header.
    let encrypted = unsafe { Segment::free_list_encrypted(segment) };
    let randomized = randomize && encrypted;
    let reclaimed =
        unsafe { Page::reclaim_thread_free_in_segment(segment, page_index, encrypted, randomized) };
    if reclaimed == 0 {
        return None;
    }
    *reclaim_sink += reclaimed;
    // SAFETY: a nonzero reclaim count guarantees the drained chain is now
    // linked onto the page's local free list.
    let block = unsafe { try_allocate_page_local_dynamic(page, enable_encryption, randomize) }
        .expect("invariant: reclaimed remote frees populate the page-local free list");
    Some(block)
}