use std::{borrow::Cow, collections::HashMap, sync::Arc};
use rand::{prelude::SliceRandom, Rng};
use semver::Version;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use smallvec::smallvec;
use casper_types::ExecutionResult;
use super::{Config, Storage};
use crate::{
effect::{
requests::{StateStoreRequest, StorageRequest},
Multiple,
},
testing::{ComponentHarness, TestRng},
types::{Block, BlockHash, Deploy, DeployHash, DeployMetadata},
utils::WithDir,
Chainspec,
};
fn storage_fixture(harness: &mut ComponentHarness<()>) -> Storage {
const MIB: usize = 1024 * 1024;
let cfg = Config {
path: harness.tmp.path().join("storage"),
max_block_store_size: 50 * MIB,
max_deploy_store_size: 50 * MIB,
max_deploy_metadata_store_size: 50 * MIB,
max_state_store_size: 50 * MIB,
};
Storage::new(&WithDir::new(harness.tmp.path(), cfg)).expect(
"could not create storage component
fixture",
)
}
fn random_block_at_height(rng: &mut TestRng, height: u64) -> Box<Block> {
let mut block = Box::new(Block::random(rng));
block.set_height(height);
block
}
fn get_block_at_height(
harness: &mut ComponentHarness<()>,
storage: &mut Storage,
height: u64,
) -> Option<Block> {
let response = harness.send_request(storage, |responder| {
StorageRequest::GetBlockAtHeight { height, responder }.into()
});
assert!(harness.is_idle());
response
}
fn get_block(
harness: &mut ComponentHarness<()>,
storage: &mut Storage,
block_hash: BlockHash,
) -> Option<Block> {
let response = harness.send_request(storage, move |responder| {
StorageRequest::GetBlock {
block_hash,
responder,
}
.into()
});
assert!(harness.is_idle());
response
}
fn get_chainspec(
harness: &mut ComponentHarness<()>,
storage: &mut Storage,
version: Version,
) -> Option<Arc<Chainspec>> {
let response = harness.send_request(storage, move |responder| {
StorageRequest::GetChainspec { version, responder }.into()
});
assert!(harness.is_idle());
response
}
fn get_deploys(
harness: &mut ComponentHarness<()>,
storage: &mut Storage,
deploy_hashes: Multiple<DeployHash>,
) -> Vec<Option<Deploy>> {
let response = harness.send_request(storage, move |responder| {
StorageRequest::GetDeploys {
deploy_hashes,
responder,
}
.into()
});
assert!(harness.is_idle());
response
}
fn get_deploy_and_metadata(
harness: &mut ComponentHarness<()>,
storage: &mut Storage,
deploy_hash: DeployHash,
) -> Option<(Deploy, DeployMetadata)> {
let response = harness.send_request(storage, |responder| {
StorageRequest::GetDeployAndMetadata {
deploy_hash,
responder,
}
.into()
});
assert!(harness.is_idle());
response
}
fn get_highest_block(harness: &mut ComponentHarness<()>, storage: &mut Storage) -> Option<Block> {
let response = harness.send_request(storage, |responder| {
StorageRequest::GetHighestBlock { responder }.into()
});
assert!(harness.is_idle());
response
}
fn load_state<T>(
harness: &mut ComponentHarness<()>,
storage: &mut Storage,
key: Cow<'static, [u8]>,
) -> Option<T>
where
T: DeserializeOwned,
{
let response: Option<Vec<u8>> = harness.send_request(storage, move |responder| {
StateStoreRequest::Load { key, responder }.into()
});
assert!(harness.is_idle());
response.map(|raw| bincode::deserialize(&raw).expect("deserialization failed"))
}
fn put_block(harness: &mut ComponentHarness<()>, storage: &mut Storage, block: Box<Block>) -> bool {
let response = harness.send_request(storage, move |responder| {
StorageRequest::PutBlock { block, responder }.into()
});
assert!(harness.is_idle());
response
}
fn put_chainspec(harness: &mut ComponentHarness<()>, storage: &mut Storage, chainspec: Chainspec) {
harness.send_request(storage, move |responder| {
StorageRequest::PutChainspec {
chainspec: Arc::new(chainspec),
responder,
}
.into()
});
assert!(harness.is_idle());
}
fn put_deploy(
harness: &mut ComponentHarness<()>,
storage: &mut Storage,
deploy: Box<Deploy>,
) -> bool {
let response = harness.send_request(storage, move |responder| {
StorageRequest::PutDeploy { deploy, responder }.into()
});
assert!(harness.is_idle());
response
}
fn put_execution_results(
harness: &mut ComponentHarness<()>,
storage: &mut Storage,
block_hash: BlockHash,
execution_results: HashMap<DeployHash, ExecutionResult>,
) {
let response = harness.send_request(storage, move |responder| {
StorageRequest::PutExecutionResults {
block_hash,
execution_results,
responder,
}
.into()
});
assert!(harness.is_idle());
response
}
fn save_state<T>(
harness: &mut ComponentHarness<()>,
storage: &mut Storage,
key: Cow<'static, [u8]>,
value: &T,
) where
T: Serialize,
{
let data = bincode::serialize(value).expect("serialization failed");
harness.send_request(storage, move |responder| {
StateStoreRequest::Save {
key,
responder,
data,
}
.into()
});
assert!(harness.is_idle());
}
#[test]
fn get_block_of_non_existing_block_returns_none() {
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let block_hash = BlockHash::random(&mut harness.rng);
let response = get_block(&mut harness, &mut storage, block_hash);
assert!(response.is_none());
assert!(harness.is_idle());
}
#[test]
fn can_put_and_get_block() {
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let block = Box::new(Block::random(&mut harness.rng));
let was_new = put_block(&mut harness, &mut storage, block.clone());
assert!(was_new, "putting block should have returned `true`");
let was_new_second_time = put_block(&mut harness, &mut storage, block.clone());
assert!(
was_new_second_time,
"storing block the second time should have returned `true`"
);
let response = get_block(&mut harness, &mut storage, *block.hash());
assert_eq!(response.as_ref(), Some(&*block));
let response = harness.send_request(&mut storage, |responder| {
StorageRequest::GetBlockHeader {
block_hash: *block.hash(),
responder,
}
.into()
});
assert_eq!(response.as_ref(), Some(block.header()));
}
#[test]
fn can_retrieve_block_by_height() {
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let block_33 = random_block_at_height(&mut harness.rng, 33);
let block_14 = random_block_at_height(&mut harness.rng, 14);
let block_99 = random_block_at_height(&mut harness.rng, 99);
assert!(get_block_at_height(&mut harness, &mut storage, 0).is_none());
assert!(get_highest_block(&mut harness, &mut storage).is_none());
assert!(get_block_at_height(&mut harness, &mut storage, 14).is_none());
assert!(get_block_at_height(&mut harness, &mut storage, 33).is_none());
assert!(get_block_at_height(&mut harness, &mut storage, 99).is_none());
let was_new = put_block(&mut harness, &mut storage, block_33.clone());
assert!(was_new);
assert_eq!(
get_highest_block(&mut harness, &mut storage).as_ref(),
Some(&*block_33)
);
assert!(get_block_at_height(&mut harness, &mut storage, 0).is_none());
assert!(get_block_at_height(&mut harness, &mut storage, 14).is_none());
assert_eq!(
get_block_at_height(&mut harness, &mut storage, 33).as_ref(),
Some(&*block_33)
);
assert!(get_block_at_height(&mut harness, &mut storage, 99).is_none());
let was_new = put_block(&mut harness, &mut storage, block_14.clone());
assert!(was_new);
assert_eq!(
get_highest_block(&mut harness, &mut storage).as_ref(),
Some(&*block_33)
);
assert!(get_block_at_height(&mut harness, &mut storage, 0).is_none());
assert_eq!(
get_block_at_height(&mut harness, &mut storage, 14).as_ref(),
Some(&*block_14)
);
assert_eq!(
get_block_at_height(&mut harness, &mut storage, 33).as_ref(),
Some(&*block_33)
);
assert!(get_block_at_height(&mut harness, &mut storage, 99).is_none());
let was_new = put_block(&mut harness, &mut storage, block_99.clone());
assert!(was_new);
assert_eq!(
get_highest_block(&mut harness, &mut storage).as_ref(),
Some(&*block_99)
);
assert!(get_block_at_height(&mut harness, &mut storage, 0).is_none());
assert_eq!(
get_block_at_height(&mut harness, &mut storage, 14).as_ref(),
Some(&*block_14)
);
assert_eq!(
get_block_at_height(&mut harness, &mut storage, 33).as_ref(),
Some(&*block_33)
);
assert_eq!(
get_block_at_height(&mut harness, &mut storage, 99).as_ref(),
Some(&*block_99)
);
}
#[test]
#[should_panic(expected = "duplicate entries")]
fn different_block_at_height_is_fatal() {
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let block_44_a = random_block_at_height(&mut harness.rng, 44);
let block_44_b = random_block_at_height(&mut harness.rng, 44);
let was_new = put_block(&mut harness, &mut storage, block_44_a.clone());
assert!(was_new);
let was_new = put_block(&mut harness, &mut storage, block_44_a);
assert!(was_new);
put_block(&mut harness, &mut storage, block_44_b);
}
#[test]
fn get_vec_of_non_existing_deploy_returns_nones() {
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let deploy_id = DeployHash::random(&mut harness.rng);
let response = get_deploys(&mut harness, &mut storage, smallvec![deploy_id]);
assert_eq!(response, vec![None]);
let response = get_deploys(&mut harness, &mut storage, smallvec![]);
assert!(response.is_empty());
}
#[test]
fn can_retrieve_store_and_load_deploys() {
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let deploy = Box::new(Deploy::random(&mut harness.rng));
let was_new = put_deploy(&mut harness, &mut storage, deploy.clone());
assert!(was_new, "putting deploy should have returned `true`");
let was_new_second_time = put_deploy(&mut harness, &mut storage, deploy.clone());
assert!(
!was_new_second_time,
"storing deploy the second time should have returned `false`"
);
let response = get_deploys(&mut harness, &mut storage, smallvec![*deploy.id()]);
assert_eq!(response, vec![Some(deploy.as_ref().clone())]);
let response = harness.send_request(&mut storage, |responder| {
StorageRequest::GetDeployHeaders {
deploy_hashes: smallvec![*deploy.id()],
responder,
}
.into()
});
assert_eq!(response, vec![Some(deploy.header().clone())]);
let (deploy_response, metadata_response) = harness
.send_request(&mut storage, |responder| {
StorageRequest::GetDeployAndMetadata {
deploy_hash: *deploy.id(),
responder,
}
.into()
})
.expect("no deploy with metadata returned");
assert_eq!(deploy_response, *deploy);
assert_eq!(metadata_response, DeployMetadata::default());
}
#[test]
fn storing_and_loading_a_lot_of_deploys_does_not_exhaust_handles() {
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let total = 1000;
let batch_size = 25;
let mut deploy_hashes = Vec::new();
for _ in 0..total {
let deploy = Box::new(Deploy::random(&mut harness.rng));
deploy_hashes.push(*deploy.id());
put_deploy(&mut harness, &mut storage, deploy);
}
deploy_hashes.as_mut_slice().shuffle(&mut harness.rng);
for chunk in deploy_hashes.chunks(batch_size) {
let result = get_deploys(&mut harness, &mut storage, chunk.iter().cloned().collect());
assert!(result.iter().all(Option::is_some));
}
}
#[test]
fn store_execution_results_for_two_blocks() {
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let deploy = Deploy::random(&mut harness.rng);
let block_hash_a = BlockHash::random(&mut harness.rng);
let block_hash_b = BlockHash::random(&mut harness.rng);
put_deploy(&mut harness, &mut storage, Box::new(deploy.clone()));
assert_eq!(
get_deploys(&mut harness, &mut storage, smallvec![*deploy.id()]),
vec![Some(deploy.clone())]
);
let first_result: ExecutionResult = harness.rng.gen();
let mut first_results = HashMap::new();
first_results.insert(*deploy.id(), first_result.clone());
put_execution_results(&mut harness, &mut storage, block_hash_a, first_results);
let (first_deploy, first_metadata) =
get_deploy_and_metadata(&mut harness, &mut storage, *deploy.id())
.expect("missing on first attempt");
assert_eq!(first_deploy, deploy);
let mut expected_per_block_results = HashMap::new();
expected_per_block_results.insert(block_hash_a, first_result);
assert_eq!(first_metadata.execution_results, expected_per_block_results);
let second_result: ExecutionResult = harness.rng.gen();
let mut second_results = HashMap::new();
second_results.insert(*deploy.id(), second_result.clone());
put_execution_results(&mut harness, &mut storage, block_hash_b, second_results);
let (second_deploy, second_metadata) =
get_deploy_and_metadata(&mut harness, &mut storage, *deploy.id())
.expect("missing on second attempt");
assert_eq!(second_deploy, deploy);
expected_per_block_results.insert(block_hash_b, second_result);
assert_eq!(
second_metadata.execution_results,
expected_per_block_results
);
}
#[test]
fn store_random_execution_results() {
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let block_hash_a = BlockHash::random(&mut harness.rng);
let block_hash_b = BlockHash::random(&mut harness.rng);
let shared_deploys = vec![
Deploy::random(&mut harness.rng),
Deploy::random(&mut harness.rng),
];
for deploy in &shared_deploys {
put_deploy(&mut harness, &mut storage, Box::new(deploy.clone()));
}
let mut expected_outcome = HashMap::new();
fn setup_block(
harness: &mut ComponentHarness<()>,
storage: &mut Storage,
expected_outcome: &mut HashMap<DeployHash, HashMap<BlockHash, ExecutionResult>>,
block_hash: &BlockHash,
shared_deploys: &[Deploy],
) {
let unique_count = 3;
let mut block_results = HashMap::new();
for _ in 0..unique_count {
let deploy = Deploy::random(&mut harness.rng);
put_deploy(harness, storage, Box::new(deploy.clone()));
let execution_result: ExecutionResult = harness.rng.gen();
let mut map = HashMap::new();
map.insert(*block_hash, execution_result.clone());
expected_outcome.insert(*deploy.id(), map);
block_results.insert(*deploy.id(), execution_result);
}
for shared_deploy in shared_deploys {
let execution_result: ExecutionResult = harness.rng.gen();
let result = block_results.insert(*shared_deploy.id(), execution_result.clone());
assert!(result.is_none());
let deploy_expected = expected_outcome.entry(*shared_deploy.id()).or_default();
let prev = deploy_expected.insert(*block_hash, execution_result.clone());
assert!(prev.is_none());
}
assert_eq!(block_results.len(), unique_count + shared_deploys.len());
put_execution_results(harness, storage, *block_hash, block_results);
}
setup_block(
&mut harness,
&mut storage,
&mut expected_outcome,
&block_hash_a,
&shared_deploys,
);
setup_block(
&mut harness,
&mut storage,
&mut expected_outcome,
&block_hash_b,
&shared_deploys,
);
for (deploy_hash, raw_meta) in expected_outcome.iter() {
let (deploy, metadata) = get_deploy_and_metadata(&mut harness, &mut storage, *deploy_hash)
.expect("missing deploy");
assert_eq!(deploy_hash, deploy.id());
assert_eq!(raw_meta, &metadata.execution_results);
}
}
#[test]
#[should_panic(expected = "duplicate execution result")]
fn store_execution_results_twice_for_same_block_deploy_pair() {
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let block_hash = BlockHash::random(&mut harness.rng);
let deploy_hash = DeployHash::random(&mut harness.rng);
let mut exec_result_1 = HashMap::new();
exec_result_1.insert(deploy_hash, harness.rng.gen());
let mut exec_result_2 = HashMap::new();
exec_result_2.insert(deploy_hash, harness.rng.gen());
put_execution_results(&mut harness, &mut storage, block_hash, exec_result_1);
put_execution_results(&mut harness, &mut storage, block_hash, exec_result_2);
}
#[test]
fn store_identical_execution_results() {
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let block_hash = BlockHash::random(&mut harness.rng);
let deploy_hash = DeployHash::random(&mut harness.rng);
let mut exec_result = HashMap::new();
exec_result.insert(deploy_hash, harness.rng.gen());
put_execution_results(&mut harness, &mut storage, block_hash, exec_result.clone());
put_execution_results(&mut harness, &mut storage, block_hash, exec_result);
}
#[test]
fn store_and_load_chainspec() {
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let version = Version::new(1, 2, 3);
let response = get_chainspec(&mut harness, &mut storage, version.clone());
assert!(response.is_none());
let chainspec = Chainspec::random(&mut harness.rng);
put_chainspec(&mut harness, &mut storage, chainspec.clone());
let response = get_chainspec(&mut harness, &mut storage, version);
assert_eq!(response, Some(Arc::new(chainspec)));
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
struct StateData {
a: Vec<u32>,
b: i32,
}
#[test]
fn store_and_load_state_data() {
let key1 = b"sample-key-1".to_vec();
let key2 = b"exkey-2".to_vec();
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let load1 = load_state::<StateData>(&mut harness, &mut storage, key1.clone().into());
let load2 = load_state::<StateData>(&mut harness, &mut storage, key2.clone().into());
assert!(load1.is_none());
assert!(load2.is_none());
let data1 = StateData { a: vec![1], b: -1 };
let data2 = StateData { a: vec![], b: 2 };
save_state(&mut harness, &mut storage, key1.clone().into(), &data1);
let load1 = load_state::<StateData>(&mut harness, &mut storage, key1.clone().into());
let load2 = load_state::<StateData>(&mut harness, &mut storage, key2.clone().into());
assert_eq!(load1, Some(data1.clone()));
assert!(load2.is_none());
save_state(&mut harness, &mut storage, key2.clone().into(), &data2);
let load1 = load_state::<StateData>(&mut harness, &mut storage, key1.clone().into());
let load2 = load_state::<StateData>(&mut harness, &mut storage, key2.clone().into());
assert_eq!(load1, Some(data1));
assert_eq!(load2, Some(data2.clone()));
save_state(&mut harness, &mut storage, key1.clone().into(), &data2);
let load1 = load_state::<StateData>(&mut harness, &mut storage, key1.into());
let load2 = load_state::<StateData>(&mut harness, &mut storage, key2.into());
assert_eq!(load1, Some(data2.clone()));
assert_eq!(load2, Some(data2));
}
#[test]
fn persist_state_data() {
let key = b"sample-key-1".to_vec();
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let load = load_state::<StateData>(&mut harness, &mut storage, key.clone().into());
assert!(load.is_none());
let data = StateData {
a: vec![1, 2, 3, 4, 5, 6],
b: -1,
};
save_state(&mut harness, &mut storage, key.clone().into(), &data);
let load = load_state::<StateData>(&mut harness, &mut storage, key.clone().into());
assert_eq!(load, Some(data.clone()));
let (on_disk, rng) = harness.into_parts();
let mut harness = ComponentHarness::builder()
.on_disk(on_disk)
.rng(rng)
.build();
let mut storage = storage_fixture(&mut harness);
let load = load_state::<StateData>(&mut harness, &mut storage, key.into());
assert_eq!(load, Some(data));
}
#[test]
fn test_legacy_interface() {
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let deploy = Box::new(Deploy::random(&mut harness.rng));
let was_new = put_deploy(&mut harness, &mut storage, deploy.clone());
assert!(was_new);
let result = storage.handle_legacy_direct_deploy_request(*deploy.id());
assert_eq!(result, Some(*deploy));
assert!(storage
.handle_legacy_direct_deploy_request(DeployHash::random(&mut harness.rng))
.is_none())
}
#[test]
fn persist_blocks_deploys_and_deploy_metadata_across_instantiations() {
let mut harness = ComponentHarness::default();
let mut storage = storage_fixture(&mut harness);
let deploy = Deploy::random(&mut harness.rng);
let block = random_block_at_height(&mut harness.rng, 42);
let execution_result: ExecutionResult = harness.rng.gen();
put_deploy(&mut harness, &mut storage, Box::new(deploy.clone()));
put_block(&mut harness, &mut storage, block.clone());
let mut execution_results = HashMap::new();
execution_results.insert(*deploy.id(), execution_result.clone());
put_execution_results(&mut harness, &mut storage, *block.hash(), execution_results);
assert_eq!(
get_block_at_height(&mut harness, &mut storage, 42).expect("block not indexed properly"),
*block
);
let (on_disk, rng) = harness.into_parts();
let mut harness = ComponentHarness::builder()
.on_disk(on_disk)
.rng(rng)
.build();
let mut storage = storage_fixture(&mut harness);
let actual_block = get_block(&mut harness, &mut storage, *block.hash())
.expect("missing block we stored earlier");
assert_eq!(actual_block, *block);
let actual_deploys = get_deploys(&mut harness, &mut storage, smallvec![*deploy.id()]);
assert_eq!(actual_deploys, vec![Some(deploy.clone())]);
let (_, deploy_metadata) = get_deploy_and_metadata(&mut harness, &mut storage, *deploy.id())
.expect("missing deploy we stored earlier");
let execution_results = deploy_metadata.execution_results;
assert_eq!(execution_results.len(), 1);
assert_eq!(execution_results[block.hash()], execution_result);
assert_eq!(
get_block_at_height(&mut harness, &mut storage, 42).expect("block index was not restored"),
*block
);
}