use super::layout::vm_layout_constants::BYTES_IN_CHUNK;
use crate::policy::space::required_chunks;
use crate::util::address::Address;
use crate::util::conversions::*;
use std::sync::{Mutex, MutexGuard};
use crate::util::alloc::embedded_meta_data::*;
use crate::util::heap::layout::vm_layout_constants::LOG_BYTES_IN_CHUNK;
use crate::util::heap::pageresource::CommonPageResource;
use crate::util::OpaquePointer;
use super::layout::map::Map;
use super::layout::Mmapper;
use super::PageResource;
use crate::util::heap::layout::heap_layout::VMMap;
use crate::vm::ActivePlan;
use crate::vm::VMBinding;
use libc::{c_void, memset};
pub struct MonotonePageResource<VM: VMBinding> {
common: CommonPageResource<VM>,
meta_data_pages_per_region: usize,
sync: Mutex<MonotonePageResourceSync>,
}
struct MonotonePageResourceSync {
cursor: Address,
sentinel: Address,
current_chunk: Address,
conditional: MonotonePageResourceConditional,
}
pub enum MonotonePageResourceConditional {
Contiguous {
start: Address,
zeroing_cursor: Address,
zeroing_sentinel: Address,
},
Discontiguous,
}
impl<VM: VMBinding> PageResource<VM> for MonotonePageResource<VM> {
fn common(&self) -> &CommonPageResource<VM> {
&self.common
}
fn common_mut(&mut self) -> &mut CommonPageResource<VM> {
&mut self.common
}
fn reserve_pages(&self, pages: usize) -> usize {
self.common().reserve(pages);
pages
}
fn alloc_pages(
&self,
reserved_pages: usize,
immut_required_pages: usize,
zeroed: bool,
tls: OpaquePointer,
) -> Address {
debug!(
"In MonotonePageResource, reserved_pages = {}, required_pages = {}",
reserved_pages, immut_required_pages
);
let mut required_pages = immut_required_pages;
let mut new_chunk = false;
let mut sync = self.sync.lock().unwrap();
let mut rtn = sync.cursor;
debug!(
"cursor = {}, sentinel = {}, current_chunk = {}",
sync.cursor, sync.sentinel, sync.current_chunk
);
if cfg!(debug = "true") {
if sync.current_chunk > sync.cursor
|| (chunk_align_down(sync.cursor) != sync.current_chunk
&& chunk_align_down(sync.cursor) != sync.current_chunk + BYTES_IN_CHUNK)
{
self.log_chunk_fields("MonotonePageResource.alloc_pages:fail");
}
assert!(sync.current_chunk <= sync.cursor);
assert!(
sync.cursor.is_zero()
|| chunk_align_down(sync.cursor) == sync.current_chunk
|| chunk_align_down(sync.cursor) == (sync.current_chunk + BYTES_IN_CHUNK)
);
}
if self.meta_data_pages_per_region != 0 {
let region_start = Self::get_region_start(sync.cursor + pages_to_bytes(required_pages));
let region_delta = region_start.get_offset(sync.cursor);
if region_delta >= 0 {
required_pages +=
bytes_to_pages(region_delta as usize) + self.meta_data_pages_per_region;
rtn = region_start + pages_to_bytes(self.meta_data_pages_per_region);
}
}
let bytes = pages_to_bytes(required_pages);
debug!("bytes={}", bytes);
let mut tmp = sync.cursor + bytes;
debug!("tmp={:?}", tmp);
if !self.common().contiguous && tmp > sync.sentinel {
let required_chunks = required_chunks(required_pages);
sync.current_chunk = unsafe {
self.common()
.space
.unwrap()
.grow_discontiguous_space(required_chunks)
}; sync.cursor = sync.current_chunk;
sync.sentinel = sync.cursor
+ if sync.current_chunk.is_zero() {
0
} else {
required_chunks << LOG_BYTES_IN_CHUNK
};
rtn = sync.cursor;
tmp = sync.cursor + bytes;
new_chunk = true;
}
debug_assert!(rtn >= sync.cursor && rtn < sync.cursor + bytes);
if tmp > sync.sentinel {
unsafe { Address::zero() }
} else {
let old = sync.cursor;
sync.cursor = tmp;
debug!("update cursor = {}", tmp);
if self.common().contiguous && chunk_align_down(sync.cursor) != sync.current_chunk {
sync.current_chunk = chunk_align_down(sync.cursor);
}
self.commit_pages(reserved_pages, required_pages, tls);
self.common()
.space
.unwrap()
.grow_space(old, bytes, new_chunk);
self.common().space.unwrap().common().mmapper.ensure_mapped(
old,
required_pages,
VM::VMActivePlan::global().global_side_metadata_specs(),
self.common().space.unwrap().local_side_metadata_specs(),
);
if zeroed {
unsafe {
memset(old.to_mut_ptr() as *mut c_void, 0, bytes);
}
}
rtn
}
}
fn adjust_for_metadata(&self, pages: usize) -> usize {
pages
+ ((pages + PAGES_IN_REGION - 1) >> LOG_PAGES_IN_REGION)
* self.meta_data_pages_per_region
}
}
impl<VM: VMBinding> MonotonePageResource<VM> {
pub fn new_contiguous(
start: Address,
bytes: usize,
meta_data_pages_per_region: usize,
_vm_map: &'static VMMap,
) -> Self {
let sentinel = start + bytes;
MonotonePageResource {
common: CommonPageResource::new(true, cfg!(target_pointer_width = "64")),
meta_data_pages_per_region,
sync: Mutex::new(MonotonePageResourceSync {
cursor: start,
current_chunk: chunk_align_down(start),
sentinel,
conditional: MonotonePageResourceConditional::Contiguous {
start,
zeroing_cursor: sentinel,
zeroing_sentinel: start,
},
}),
}
}
pub fn new_discontiguous(meta_data_pages_per_region: usize, _vm_map: &'static VMMap) -> Self {
MonotonePageResource {
common: CommonPageResource::new(false, true),
meta_data_pages_per_region,
sync: Mutex::new(MonotonePageResourceSync {
cursor: unsafe { Address::zero() },
current_chunk: unsafe { Address::zero() },
sentinel: unsafe { Address::zero() },
conditional: MonotonePageResourceConditional::Discontiguous,
}),
}
}
fn log_chunk_fields(&self, site: &str) {
let sync = self.sync.lock().unwrap();
debug!(
"[{}]{}: cursor={}, current_chunk={}, delta={}",
self.common().space.unwrap().common().name,
site,
sync.cursor,
sync.current_chunk,
sync.cursor - sync.current_chunk
);
}
fn get_region_start(addr: Address) -> Address {
addr.align_down(BYTES_IN_REGION)
}
pub unsafe fn reset(&self) {
let mut guard = self.sync.lock().unwrap();
self.common().reset_reserved();
self.common().reset_committed();
self.release_pages(&mut guard);
drop(guard);
}
#[inline]
unsafe fn release_pages(&self, guard: &mut MutexGuard<MonotonePageResourceSync>) {
if self.common().contiguous {
guard.cursor = match guard.conditional {
MonotonePageResourceConditional::Contiguous { start: _start, .. } => _start,
_ => unreachable!(),
};
} else if !guard.cursor.is_zero() {
let bytes = guard.cursor - guard.current_chunk;
self.release_pages_extent(guard.current_chunk, bytes);
while self.move_to_next_chunk(guard) {
let bytes = guard.cursor - guard.current_chunk;
self.release_pages_extent(guard.current_chunk, bytes);
}
guard.current_chunk = Address::zero();
guard.sentinel = Address::zero();
guard.cursor = Address::zero();
self.common().space.as_ref().unwrap().release_all_chunks();
}
}
fn release_pages_extent(&self, _first: Address, bytes: usize) {
let pages = crate::util::conversions::bytes_to_pages(bytes);
debug_assert!(bytes == crate::util::conversions::pages_to_bytes(pages));
}
fn move_to_next_chunk(&self, guard: &mut MutexGuard<MonotonePageResourceSync>) -> bool {
guard.current_chunk = self
.vm_map()
.get_next_contiguous_region(guard.current_chunk);
if guard.current_chunk.is_zero() {
false
} else {
guard.cursor = guard.current_chunk
+ self
.vm_map()
.get_contiguous_region_size(guard.current_chunk);
true
}
}
}