use {
crate::storable_accounts::StorableAccounts,
log::*,
memmap2::MmapMut,
serde::{Deserialize, Serialize},
solana_sdk::{
account::{Account, AccountSharedData, ReadableAccount},
clock::{Epoch, Slot},
hash::Hash,
pubkey::Pubkey,
},
std::{
borrow::Borrow,
convert::TryFrom,
fs::{remove_file, OpenOptions},
io::{self, Seek, SeekFrom, Write},
marker::PhantomData,
mem,
path::{Path, PathBuf},
sync::{
atomic::{AtomicU64, AtomicUsize, Ordering},
Mutex,
},
},
};
pub mod test_utils;
pub const ALIGN_BOUNDARY_OFFSET: usize = mem::size_of::<u64>();
macro_rules! u64_align {
($addr: expr) => {
($addr + (ALIGN_BOUNDARY_OFFSET - 1)) & !(ALIGN_BOUNDARY_OFFSET - 1)
};
}
pub const STORE_META_OVERHEAD: usize = 136;
pub fn aligned_stored_size(data_len: usize) -> usize {
u64_align!(STORE_META_OVERHEAD + data_len)
}
pub const MAXIMUM_APPEND_VEC_FILE_SIZE: u64 = 16 * 1024 * 1024 * 1024;
pub type StoredMetaWriteVersion = u64;
pub struct StorableAccountsWithHashesAndWriteVersions<
'a: 'b,
'b,
T: ReadableAccount + Sync + 'b,
U: StorableAccounts<'a, T>,
V: Borrow<Hash>,
> {
accounts: &'b U,
hashes_and_write_versions: Option<(Vec<V>, Vec<StoredMetaWriteVersion>)>,
_phantom: PhantomData<&'a T>,
}
impl<'a: 'b, 'b, T: ReadableAccount + Sync + 'b, U: StorableAccounts<'a, T>, V: Borrow<Hash>>
StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>
{
pub fn new(accounts: &'b U) -> Self {
assert!(accounts.has_hash_and_write_version());
Self {
accounts,
hashes_and_write_versions: None,
_phantom: PhantomData,
}
}
pub fn new_with_hashes_and_write_versions(
accounts: &'b U,
hashes: Vec<V>,
write_versions: Vec<StoredMetaWriteVersion>,
) -> Self {
assert!(!accounts.has_hash_and_write_version());
assert_eq!(accounts.len(), hashes.len());
assert_eq!(write_versions.len(), hashes.len());
Self {
accounts,
hashes_and_write_versions: Some((hashes, write_versions)),
_phantom: PhantomData,
}
}
pub fn hash(&self, index: usize) -> &Hash {
if self.accounts.has_hash_and_write_version() {
self.accounts.hash(index)
} else {
self.hashes_and_write_versions.as_ref().unwrap().0[index].borrow()
}
}
pub fn write_version(&self, index: usize) -> u64 {
if self.accounts.has_hash_and_write_version() {
self.accounts.write_version(index)
} else {
self.hashes_and_write_versions.as_ref().unwrap().1[index]
}
}
pub fn account(&self, index: usize) -> Option<&T> {
self.accounts.account_default_if_zero_lamport(index)
}
pub fn pubkey(&self, index: usize) -> &Pubkey {
self.accounts.pubkey(index)
}
pub fn len(&self) -> usize {
self.accounts.len()
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
#[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)]
pub struct StoredMeta {
pub write_version_obsolete: StoredMetaWriteVersion,
pub pubkey: Pubkey,
pub data_len: u64,
}
#[derive(Serialize, Deserialize, Clone, Debug, Default, Eq, PartialEq)]
pub struct AccountMeta {
pub lamports: u64,
pub owner: Pubkey,
pub executable: bool,
pub rent_epoch: Epoch,
}
impl<'a, T: ReadableAccount> From<&'a T> for AccountMeta {
fn from(account: &'a T) -> Self {
Self {
lamports: account.lamports(),
owner: *account.owner(),
executable: account.executable(),
rent_epoch: account.rent_epoch(),
}
}
}
impl<'a, T: ReadableAccount> From<Option<&'a T>> for AccountMeta {
fn from(account: Option<&'a T>) -> Self {
match account {
Some(account) => AccountMeta::from(account),
None => AccountMeta::default(),
}
}
}
#[derive(Serialize, Deserialize)]
pub struct StoredAccountMetaClone<'a> {
pub meta: StoredMeta,
pub account_meta: AccountMeta,
pub data: &'a [u8],
pub offset: usize,
pub stored_size: usize,
pub hash: Hash,
}
#[derive(PartialEq, Eq, Debug)]
pub struct StoredAccountMeta<'a> {
pub meta: &'a StoredMeta,
pub account_meta: &'a AccountMeta,
pub data: &'a [u8],
pub offset: usize,
pub stored_size: usize,
pub hash: &'a Hash,
}
impl<'a> StoredAccountMeta<'a> {
pub fn clone_account(&self) -> AccountSharedData {
AccountSharedData::from(Account {
lamports: self.account_meta.lamports,
owner: self.account_meta.owner,
executable: self.account_meta.executable,
rent_epoch: self.account_meta.rent_epoch,
data: self.data.to_vec(),
})
}
pub fn pubkey(&self) -> &Pubkey {
&self.meta.pubkey
}
fn sanitize(&self) -> bool {
self.sanitize_executable() && self.sanitize_lamports()
}
fn sanitize_executable(&self) -> bool {
self.ref_executable_byte() & !1 == 0
}
fn sanitize_lamports(&self) -> bool {
self.account_meta.lamports != 0 || self.clone_account() == AccountSharedData::default()
}
fn ref_executable_byte(&self) -> &u8 {
let executable_bool: &bool = &self.account_meta.executable;
let executable_byte: &u8 = unsafe { &*(executable_bool as *const bool as *const u8) };
executable_byte
}
}
pub struct AppendVecAccountsIter<'a> {
append_vec: &'a AppendVec,
offset: usize,
}
impl<'a> AppendVecAccountsIter<'a> {
pub fn new(append_vec: &'a AppendVec) -> Self {
Self {
append_vec,
offset: 0,
}
}
}
impl<'a> Iterator for AppendVecAccountsIter<'a> {
type Item = StoredAccountMeta<'a>;
fn next(&mut self) -> Option<Self::Item> {
if let Some((account, next_offset)) = self.append_vec.get_account(self.offset) {
self.offset = next_offset;
Some(account)
} else {
None
}
}
}
#[derive(Debug, AbiExample)]
pub struct AppendVec {
path: PathBuf,
map: MmapMut,
append_lock: Mutex<()>,
current_len: AtomicUsize,
file_size: u64,
remove_on_drop: bool,
}
lazy_static! {
pub static ref APPEND_VEC_MMAPPED_FILES_OPEN: AtomicU64 = AtomicU64::default();
}
impl Drop for AppendVec {
fn drop(&mut self) {
if self.remove_on_drop {
APPEND_VEC_MMAPPED_FILES_OPEN.fetch_sub(1, Ordering::Relaxed);
if let Err(_e) = remove_file(&self.path) {
inc_new_counter_info!("append_vec_drop_fail", 1);
}
}
}
}
impl AppendVec {
pub fn new(file: &Path, create: bool, size: usize) -> Self {
let initial_len = 0;
AppendVec::sanitize_len_and_size(initial_len, size).unwrap();
if create {
let _ignored = remove_file(file);
}
let mut data = OpenOptions::new()
.read(true)
.write(true)
.create(create)
.open(file)
.map_err(|e| {
panic!(
"Unable to {} data file {} in current dir({:?}): {:?}",
if create { "create" } else { "open" },
file.display(),
std::env::current_dir(),
e
);
})
.unwrap();
data.seek(SeekFrom::Start((size - 1) as u64)).unwrap();
data.write_all(&[0]).unwrap();
data.rewind().unwrap();
data.flush().unwrap();
let map = unsafe { MmapMut::map_mut(&data) };
let map = map.unwrap_or_else(|e| {
error!(
"Failed to map the data file (size: {}): {}.\n
Please increase sysctl vm.max_map_count or equivalent for your platform.",
size, e
);
std::process::exit(1);
});
APPEND_VEC_MMAPPED_FILES_OPEN.fetch_add(1, Ordering::Relaxed);
AppendVec {
path: file.to_path_buf(),
map,
append_lock: Mutex::new(()),
current_len: AtomicUsize::new(initial_len),
file_size: size as u64,
remove_on_drop: true,
}
}
pub fn set_no_remove_on_drop(&mut self) {
self.remove_on_drop = false;
}
fn sanitize_len_and_size(current_len: usize, file_size: usize) -> io::Result<()> {
if file_size == 0 {
Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!("too small file size {file_size} for AppendVec"),
))
} else if usize::try_from(MAXIMUM_APPEND_VEC_FILE_SIZE)
.map(|max| file_size > max)
.unwrap_or(true)
{
Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!("too large file size {file_size} for AppendVec"),
))
} else if current_len > file_size {
Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!("current_len is larger than file size ({file_size})"),
))
} else {
Ok(())
}
}
pub fn flush(&self) -> io::Result<()> {
self.map.flush()
}
pub fn reset(&self) {
let _lock = self.append_lock.lock().unwrap();
self.current_len.store(0, Ordering::Release);
}
pub fn remaining_bytes(&self) -> u64 {
(self.capacity()).saturating_sub(self.len() as u64)
}
pub fn len(&self) -> usize {
self.current_len.load(Ordering::Acquire)
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn capacity(&self) -> u64 {
self.file_size
}
pub fn file_name(slot: Slot, id: impl std::fmt::Display) -> String {
format!("{slot}.{id}")
}
pub fn new_from_file<P: AsRef<Path>>(path: P, current_len: usize) -> io::Result<(Self, usize)> {
let new = Self::new_from_file_unchecked(&path, current_len)?;
let (sanitized, num_accounts) = new.sanitize_layout_and_length();
if !sanitized {
let err_msg = format!(
"incorrect layout/length/data in the appendvec at path {}",
path.as_ref().display()
);
return Err(std::io::Error::new(std::io::ErrorKind::Other, err_msg));
}
Ok((new, num_accounts))
}
pub fn new_from_file_unchecked<P: AsRef<Path>>(
path: P,
current_len: usize,
) -> io::Result<Self> {
let file_size = std::fs::metadata(&path)?.len();
Self::sanitize_len_and_size(current_len, file_size as usize)?;
let data = OpenOptions::new()
.read(true)
.write(true)
.create(false)
.open(&path)?;
let map = unsafe {
let result = MmapMut::map_mut(&data);
if result.is_err() {
info!("memory map error: {:?}. This may be because vm.max_map_count is not set correctly.", result);
}
result?
};
APPEND_VEC_MMAPPED_FILES_OPEN.fetch_add(1, Ordering::Relaxed);
Ok(AppendVec {
path: path.as_ref().to_path_buf(),
map,
append_lock: Mutex::new(()),
current_len: AtomicUsize::new(current_len),
file_size,
remove_on_drop: true,
})
}
fn sanitize_layout_and_length(&self) -> (bool, usize) {
let mut offset = 0;
let mut num_accounts = 0;
while let Some((account, next_offset)) = self.get_account(offset) {
if !account.sanitize() {
return (false, num_accounts);
}
offset = next_offset;
num_accounts += 1;
}
let aligned_current_len = u64_align!(self.current_len.load(Ordering::Acquire));
(offset == aligned_current_len, num_accounts)
}
fn get_slice(&self, offset: usize, size: usize) -> Option<(&[u8], usize)> {
let (next, overflow) = offset.overflowing_add(size);
if overflow || next > self.len() {
return None;
}
let data = &self.map[offset..next];
let next = u64_align!(next);
Some((
unsafe { std::slice::from_raw_parts(data.as_ptr() as *const u8, size) },
next,
))
}
fn append_ptr(&self, offset: &mut usize, src: *const u8, len: usize) {
let pos = u64_align!(*offset);
let data = &self.map[pos..(pos + len)];
unsafe {
let dst = data.as_ptr() as *mut u8;
std::ptr::copy(src, dst, len);
};
*offset = pos + len;
}
fn append_ptrs_locked(&self, offset: &mut usize, vals: &[(*const u8, usize)]) -> Option<usize> {
let mut end = *offset;
for val in vals {
end = u64_align!(end);
end += val.1;
}
if (self.file_size as usize) < end {
return None;
}
let pos = u64_align!(*offset);
for val in vals {
self.append_ptr(offset, val.0, val.1)
}
self.current_len.store(*offset, Ordering::Release);
Some(pos)
}
fn get_type<'a, T>(&self, offset: usize) -> Option<(&'a T, usize)> {
let (data, next) = self.get_slice(offset, mem::size_of::<T>())?;
let ptr: *const T = data.as_ptr() as *const T;
Some((unsafe { &*ptr }, next))
}
pub fn get_account<'a>(&'a self, offset: usize) -> Option<(StoredAccountMeta<'a>, usize)> {
let (meta, next): (&'a StoredMeta, _) = self.get_type(offset)?;
let (account_meta, next): (&'a AccountMeta, _) = self.get_type(next)?;
let (hash, next): (&'a Hash, _) = self.get_type(next)?;
let (data, next) = self.get_slice(next, meta.data_len as usize)?;
let stored_size = next - offset;
Some((
StoredAccountMeta {
meta,
account_meta,
data,
offset,
stored_size,
hash,
},
next,
))
}
#[cfg(test)]
pub fn get_account_test(&self, offset: usize) -> Option<(StoredMeta, AccountSharedData)> {
let (stored_account, _) = self.get_account(offset)?;
let meta = stored_account.meta.clone();
Some((meta, stored_account.clone_account()))
}
pub fn get_path(&self) -> PathBuf {
self.path.clone()
}
pub fn account_iter(&self) -> AppendVecAccountsIter {
AppendVecAccountsIter::new(self)
}
pub fn accounts(&self, mut offset: usize) -> Vec<StoredAccountMeta> {
let mut accounts = vec![];
while let Some((account, next)) = self.get_account(offset) {
accounts.push(account);
offset = next;
}
accounts
}
pub fn append_accounts<
'a,
'b,
T: ReadableAccount + Sync,
U: StorableAccounts<'a, T>,
V: Borrow<Hash>,
>(
&self,
accounts: &StorableAccountsWithHashesAndWriteVersions<'a, 'b, T, U, V>,
skip: usize,
) -> Option<Vec<usize>> {
let _lock = self.append_lock.lock().unwrap();
let mut offset = self.len();
let mut rv = Vec::with_capacity(accounts.accounts.len());
let len = accounts.accounts.len();
for i in skip..len {
let account = accounts.account(i);
let account_meta = account
.map(|account| AccountMeta {
lamports: account.lamports(),
owner: *account.owner(),
rent_epoch: account.rent_epoch(),
executable: account.executable(),
})
.unwrap_or_default();
let stored_meta = StoredMeta {
pubkey: *accounts.pubkey(i),
data_len: account
.map(|account| account.data().len())
.unwrap_or_default() as u64,
write_version_obsolete: accounts.write_version(i),
};
let meta_ptr = &stored_meta as *const StoredMeta;
let account_meta_ptr = &account_meta as *const AccountMeta;
let data_len = stored_meta.data_len as usize;
let data_ptr = account
.map(|account| account.data())
.unwrap_or_default()
.as_ptr();
let hash_ptr = accounts.hash(i).as_ref().as_ptr();
let ptrs = [
(meta_ptr as *const u8, mem::size_of::<StoredMeta>()),
(account_meta_ptr as *const u8, mem::size_of::<AccountMeta>()),
(hash_ptr as *const u8, mem::size_of::<Hash>()),
(data_ptr, data_len),
];
if let Some(res) = self.append_ptrs_locked(&mut offset, &ptrs) {
rv.push(res)
} else {
break;
}
}
if rv.is_empty() {
None
} else {
rv.push(u64_align!(offset));
Some(rv)
}
}
}
#[cfg(test)]
pub mod tests {
use {
super::{test_utils::*, *},
crate::accounts_db::INCLUDE_SLOT_IN_HASH_TESTS,
assert_matches::assert_matches,
rand::{thread_rng, Rng},
solana_sdk::{
account::{accounts_equal, WritableAccount},
timing::duration_as_ms,
},
std::time::Instant,
};
impl AppendVec {
pub(crate) fn set_current_len_for_tests(&self, len: usize) {
self.current_len.store(len, Ordering::Release);
}
fn append_account_test(&self, data: &(StoredMeta, AccountSharedData)) -> Option<usize> {
let slot_ignored = Slot::MAX;
let accounts = [(&data.0.pubkey, &data.1)];
let slice = &accounts[..];
let account_data = (slot_ignored, slice);
let hash = Hash::default();
let storable_accounts =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&account_data,
vec![&hash],
vec![data.0.write_version_obsolete],
);
self.append_accounts(&storable_accounts, 0)
.map(|res| res[0])
}
}
impl<'a> StoredAccountMeta<'a> {
#[allow(clippy::cast_ref_to_mut)]
fn set_data_len_unsafe(&self, new_data_len: u64) {
unsafe {
*(&self.meta.data_len as *const u64 as *mut u64) = new_data_len;
}
}
fn get_executable_byte(&self) -> u8 {
let executable_bool: bool = self.account_meta.executable;
let executable_byte: u8 = unsafe { std::mem::transmute::<bool, u8>(executable_bool) };
executable_byte
}
#[allow(clippy::cast_ref_to_mut)]
fn set_executable_as_byte(&self, new_executable_byte: u8) {
unsafe {
*(&self.account_meta.executable as *const bool as *mut u8) = new_executable_byte;
}
}
}
static_assertions::const_assert_eq!(
STORE_META_OVERHEAD,
std::mem::size_of::<StoredMeta>()
+ std::mem::size_of::<AccountMeta>()
+ std::mem::size_of::<Hash>()
);
static_assertions::assert_eq_align!(u64, StoredMeta, AccountMeta);
#[test]
#[should_panic(expected = "assertion failed: accounts.has_hash_and_write_version()")]
fn test_storable_accounts_with_hashes_and_write_versions_new() {
let account = AccountSharedData::default();
let slot = 0 as Slot;
let pubkey = Pubkey::default();
StorableAccountsWithHashesAndWriteVersions::<'_, '_, _, _, &Hash>::new(&(
slot,
&[(&pubkey, &account)][..],
INCLUDE_SLOT_IN_HASH_TESTS,
));
}
fn test_mismatch(correct_hashes: bool, correct_write_versions: bool) {
let account = AccountSharedData::default();
let slot = 0 as Slot;
let pubkey = Pubkey::default();
let mut hashes = Vec::default();
if correct_hashes {
hashes.push(Hash::default());
}
let mut write_versions = Vec::default();
if correct_write_versions {
write_versions.push(0);
}
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&(slot, &[(&pubkey, &account)][..], INCLUDE_SLOT_IN_HASH_TESTS),
hashes,
write_versions,
);
}
#[test]
#[should_panic(expected = "assertion failed:")]
fn test_storable_accounts_with_hashes_and_write_versions_new2() {
test_mismatch(false, false);
}
#[test]
#[should_panic(expected = "assertion failed:")]
fn test_storable_accounts_with_hashes_and_write_versions_new3() {
test_mismatch(false, true);
}
#[test]
#[should_panic(expected = "assertion failed:")]
fn test_storable_accounts_with_hashes_and_write_versions_new4() {
test_mismatch(true, false);
}
#[test]
fn test_storable_accounts_with_hashes_and_write_versions_empty() {
let account = AccountSharedData::default();
let slot = 0 as Slot;
let pubkeys = vec![Pubkey::default()];
let hashes = Vec::<Hash>::default();
let write_versions = Vec::default();
let mut accounts = vec![(&pubkeys[0], &account)];
accounts.clear();
let accounts2 = (slot, &accounts[..], INCLUDE_SLOT_IN_HASH_TESTS);
let storable =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&accounts2,
hashes,
write_versions,
);
assert_eq!(storable.len(), 0);
assert!(storable.is_empty());
}
#[test]
fn test_storable_accounts_with_hashes_and_write_versions_hash_and_write_version() {
let account = AccountSharedData::default();
let slot = 0 as Slot;
let pubkeys = vec![Pubkey::new(&[5; 32]), Pubkey::new(&[6; 32])];
let hashes = vec![Hash::new(&[3; 32]), Hash::new(&[4; 32])];
let write_versions = vec![42, 43];
let accounts = vec![(&pubkeys[0], &account), (&pubkeys[1], &account)];
let accounts2 = (slot, &accounts[..], INCLUDE_SLOT_IN_HASH_TESTS);
let storable =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&accounts2,
hashes.clone(),
write_versions.clone(),
);
assert_eq!(storable.len(), pubkeys.len());
assert!(!storable.is_empty());
(0..2).for_each(|i| {
assert_eq!(storable.hash(i), &hashes[i]);
assert_eq!(&storable.write_version(i), &write_versions[i]);
assert_eq!(storable.pubkey(i), &pubkeys[i]);
});
}
#[test]
fn test_storable_accounts_with_hashes_and_write_versions_default() {
let account = Account {
data: vec![0],
..Account::default()
}
.to_account_shared_data();
let slot = 0 as Slot;
let pubkey = Pubkey::default();
let hashes = vec![Hash::default()];
let write_versions = vec![0];
let accounts = vec![(&pubkey, &account)];
let accounts2 = (slot, &accounts[..], INCLUDE_SLOT_IN_HASH_TESTS);
let storable =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&accounts2,
hashes.clone(),
write_versions.clone(),
);
let get_account = storable.account(0);
assert!(get_account.is_none());
let account = Account {
lamports: 1,
data: vec![0],
..Account::default()
}
.to_account_shared_data();
let accounts = vec![(&pubkey, &account)];
let accounts2 = (slot, &accounts[..], INCLUDE_SLOT_IN_HASH_TESTS);
let storable =
StorableAccountsWithHashesAndWriteVersions::new_with_hashes_and_write_versions(
&accounts2,
hashes,
write_versions,
);
let get_account = storable.account(0);
assert!(accounts_equal(&account, get_account.unwrap()));
}
#[test]
fn test_account_meta_default() {
let def1 = AccountMeta::default();
let def2 = AccountMeta::from(&Account::default());
assert_eq!(&def1, &def2);
let def2 = AccountMeta::from(&AccountSharedData::default());
assert_eq!(&def1, &def2);
let def2 = AccountMeta::from(Some(&AccountSharedData::default()));
assert_eq!(&def1, &def2);
let none: Option<&AccountSharedData> = None;
let def2 = AccountMeta::from(none);
assert_eq!(&def1, &def2);
}
#[test]
fn test_account_meta_non_default() {
let def1 = AccountMeta {
lamports: 1,
owner: Pubkey::new_unique(),
executable: true,
rent_epoch: 3,
};
let def2_account = Account {
lamports: def1.lamports,
owner: def1.owner,
executable: def1.executable,
rent_epoch: def1.rent_epoch,
data: Vec::new(),
};
let def2 = AccountMeta::from(&def2_account);
assert_eq!(&def1, &def2);
let def2 = AccountMeta::from(&AccountSharedData::from(def2_account.clone()));
assert_eq!(&def1, &def2);
let def2 = AccountMeta::from(Some(&AccountSharedData::from(def2_account)));
assert_eq!(&def1, &def2);
}
#[test]
#[should_panic(expected = "too small file size 0 for AppendVec")]
fn test_append_vec_new_bad_size() {
let path = get_append_vec_path("test_append_vec_new_bad_size");
let _av = AppendVec::new(&path.path, true, 0);
}
#[test]
fn test_append_vec_new_from_file_bad_size() {
let file = get_append_vec_path("test_append_vec_new_from_file_bad_size");
let path = &file.path;
let _data = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.open(path)
.expect("create a test file for mmap");
let result = AppendVec::new_from_file(path, 0);
assert_matches!(result, Err(ref message) if message.to_string() == *"too small file size 0 for AppendVec");
}
#[test]
fn test_append_vec_sanitize_len_and_size_too_small() {
const LEN: usize = 0;
const SIZE: usize = 0;
let result = AppendVec::sanitize_len_and_size(LEN, SIZE);
assert_matches!(result, Err(ref message) if message.to_string() == *"too small file size 0 for AppendVec");
}
#[test]
fn test_append_vec_sanitize_len_and_size_maximum() {
const LEN: usize = 0;
const SIZE: usize = 16 * 1024 * 1024 * 1024;
let result = AppendVec::sanitize_len_and_size(LEN, SIZE);
assert_matches!(result, Ok(_));
}
#[test]
fn test_append_vec_sanitize_len_and_size_too_large() {
const LEN: usize = 0;
const SIZE: usize = 16 * 1024 * 1024 * 1024 + 1;
let result = AppendVec::sanitize_len_and_size(LEN, SIZE);
assert_matches!(result, Err(ref message) if message.to_string() == *"too large file size 17179869185 for AppendVec");
}
#[test]
fn test_append_vec_sanitize_len_and_size_full_and_same_as_current_len() {
const LEN: usize = 1024 * 1024;
const SIZE: usize = 1024 * 1024;
let result = AppendVec::sanitize_len_and_size(LEN, SIZE);
assert_matches!(result, Ok(_));
}
#[test]
fn test_append_vec_sanitize_len_and_size_larger_current_len() {
const LEN: usize = 1024 * 1024 + 1;
const SIZE: usize = 1024 * 1024;
let result = AppendVec::sanitize_len_and_size(LEN, SIZE);
assert_matches!(result, Err(ref message) if message.to_string() == *"current_len is larger than file size (1048576)");
}
#[test]
fn test_append_vec_one() {
let path = get_append_vec_path("test_append");
let av = AppendVec::new(&path.path, true, 1024 * 1024);
let account = create_test_account(0);
let index = av.append_account_test(&account).unwrap();
assert_eq!(av.get_account_test(index).unwrap(), account);
}
#[test]
fn test_remaining_bytes() {
let path = get_append_vec_path("test_append");
let sz = 1024 * 1024;
let sz64 = sz as u64;
let av = AppendVec::new(&path.path, true, sz);
assert_eq!(av.capacity(), sz64);
assert_eq!(av.remaining_bytes(), sz64);
let account = create_test_account(0);
av.append_account_test(&account).unwrap();
assert_eq!(av.capacity(), sz64);
assert_eq!(av.remaining_bytes(), sz64 - (STORE_META_OVERHEAD as u64));
}
#[test]
fn test_append_vec_data() {
let path = get_append_vec_path("test_append_data");
let av = AppendVec::new(&path.path, true, 1024 * 1024);
let account = create_test_account(5);
let index = av.append_account_test(&account).unwrap();
assert_eq!(av.get_account_test(index).unwrap(), account);
let account1 = create_test_account(6);
let index1 = av.append_account_test(&account1).unwrap();
assert_eq!(av.get_account_test(index).unwrap(), account);
assert_eq!(av.get_account_test(index1).unwrap(), account1);
}
#[test]
fn test_append_vec_append_many() {
let path = get_append_vec_path("test_append_many");
let av = AppendVec::new(&path.path, true, 1024 * 1024);
let size = 1000;
let mut indexes = vec![];
let now = Instant::now();
for sample in 0..size {
let account = create_test_account(sample);
let pos = av.append_account_test(&account).unwrap();
assert_eq!(av.get_account_test(pos).unwrap(), account);
indexes.push(pos)
}
trace!("append time: {} ms", duration_as_ms(&now.elapsed()),);
let now = Instant::now();
for _ in 0..size {
let sample = thread_rng().gen_range(0, indexes.len());
let account = create_test_account(sample);
assert_eq!(av.get_account_test(indexes[sample]).unwrap(), account);
}
trace!("random read time: {} ms", duration_as_ms(&now.elapsed()),);
let now = Instant::now();
assert_eq!(indexes.len(), size);
assert_eq!(indexes[0], 0);
let mut accounts = av.accounts(indexes[0]);
assert_eq!(accounts.len(), size);
for (sample, v) in accounts.iter_mut().enumerate() {
let account = create_test_account(sample);
let recovered = v.clone_account();
assert_eq!(recovered, account.1)
}
trace!(
"sequential read time: {} ms",
duration_as_ms(&now.elapsed()),
);
}
#[test]
fn test_new_from_file_crafted_zero_lamport_account() {
let file = get_append_vec_path("test_append_bytes");
let path = &file.path;
let accounts_len = 139;
{
let append_vec_data = [
0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 192, 118, 150, 1, 185, 209, 118,
82, 154, 222, 172, 202, 110, 26, 218, 140, 143, 96, 61, 43, 212, 73, 203, 7, 190,
88, 80, 222, 110, 114, 67, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 98, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
];
let f = std::fs::File::create(path).unwrap();
let mut writer = std::io::BufWriter::new(f);
writer.write_all(append_vec_data.as_slice()).unwrap();
}
let result = AppendVec::new_from_file(path, accounts_len);
assert_matches!(result, Err(ref message) if message.to_string().starts_with("incorrect layout/length/data"));
}
#[test]
fn test_new_from_file_crafted_data_len() {
let file = get_append_vec_path("test_new_from_file_crafted_data_len");
let path = &file.path;
let mut av = AppendVec::new(path, true, 1024 * 1024);
av.set_no_remove_on_drop();
let crafted_data_len = 1;
av.append_account_test(&create_test_account(10)).unwrap();
let accounts = av.accounts(0);
let account = accounts.first().unwrap();
account.set_data_len_unsafe(crafted_data_len);
assert_eq!(account.meta.data_len, crafted_data_len);
let accounts = av.accounts(0);
let account = accounts.first().unwrap();
assert_eq!(account.meta.data_len, crafted_data_len);
av.flush().unwrap();
let accounts_len = av.len();
drop(av);
let result = AppendVec::new_from_file(path, accounts_len);
assert_matches!(result, Err(ref message) if message.to_string().starts_with("incorrect layout/length/data"));
}
#[test]
fn test_new_from_file_too_large_data_len() {
let file = get_append_vec_path("test_new_from_file_too_large_data_len");
let path = &file.path;
let mut av = AppendVec::new(path, true, 1024 * 1024);
av.set_no_remove_on_drop();
let too_large_data_len = u64::max_value();
av.append_account_test(&create_test_account(10)).unwrap();
let accounts = av.accounts(0);
let account = accounts.first().unwrap();
account.set_data_len_unsafe(too_large_data_len);
assert_eq!(account.meta.data_len, too_large_data_len);
let accounts = av.accounts(0);
assert_matches!(accounts.first(), None);
av.flush().unwrap();
let accounts_len = av.len();
drop(av);
let result = AppendVec::new_from_file(path, accounts_len);
assert_matches!(result, Err(ref message) if message.to_string().starts_with("incorrect layout/length/data"));
}
#[test]
fn test_new_from_file_crafted_executable() {
let file = get_append_vec_path("test_new_from_crafted_executable");
let path = &file.path;
let mut av = AppendVec::new(path, true, 1024 * 1024);
av.set_no_remove_on_drop();
av.append_account_test(&create_test_account(10)).unwrap();
{
let mut executable_account = create_test_account(10);
executable_account.1.set_executable(true);
av.append_account_test(&executable_account).unwrap();
}
let accounts = av.accounts(0);
assert_eq!(*accounts[0].ref_executable_byte(), 0);
assert_eq!(*accounts[1].ref_executable_byte(), 1);
let account = &accounts[0];
let crafted_executable = u8::max_value() - 1;
account.set_executable_as_byte(crafted_executable);
let accounts = av.accounts(0);
let account = accounts.first().unwrap();
assert!(!account.sanitize_executable());
{
let executable_bool: &bool = &account.account_meta.executable;
assert!(!*executable_bool);
#[cfg(not(target_arch = "aarch64"))]
{
const FALSE: bool = false; if *executable_bool == FALSE {
panic!("This didn't occur if this test passed.");
}
}
assert_eq!(*account.ref_executable_byte(), crafted_executable);
}
{
let executable_bool: bool = account.account_meta.executable;
assert!(!executable_bool);
assert_eq!(account.get_executable_byte(), 0); }
av.flush().unwrap();
let accounts_len = av.len();
drop(av);
let result = AppendVec::new_from_file(path, accounts_len);
assert_matches!(result, Err(ref message) if message.to_string().starts_with("incorrect layout/length/data"));
}
}