use crate::common::universal_io::MmapFs;
use rand::distr::{Distribution, Uniform};
use rand::{Rng, RngExt};
use serde::{Deserialize, Serialize};
use serde_json::Map;
use tempfile::{Builder, TempDir};
use crate::blobstore::config::{
Compression, DEFAULT_BLOCK_SIZE_BYTES, DEFAULT_REGION_SIZE_BLOCKS, GridstoreConfig,
LogstoreConfig, Mode, StorageConfig,
};
use crate::blobstore::{Blob, Blobstore};
#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
pub struct Payload(pub Map<String, serde_json::Value>);
impl Default for Payload {
fn default() -> Self {
Payload(serde_json::Map::new())
}
}
impl Blob for Payload {
fn to_bytes(&self) -> Vec<u8> {
serde_json::to_vec(self).unwrap()
}
fn from_bytes(data: &[u8]) -> Self {
serde_json::from_slice(data).unwrap()
}
}
pub fn empty_storage() -> (TempDir, Blobstore<Payload>) {
empty_storage_mode(Mode::Mutable)
}
pub fn empty_storage_append_only() -> (TempDir, Blobstore<Payload>) {
empty_storage_mode(Mode::AppendOnly)
}
pub fn empty_storage_mode(mode: Mode) -> (TempDir, Blobstore<Payload>) {
let dir = Builder::new().prefix("test-storage").tempdir().unwrap();
let storage = Blobstore::new(MmapFs, dir.path().to_path_buf(), default_config(mode)).unwrap();
(dir, storage)
}
pub fn default_config(mode: Mode) -> StorageConfig {
match mode {
Mode::Mutable => StorageConfig::Mutable(GridstoreConfig::DEFAULT),
Mode::AppendOnly => StorageConfig::AppendOnly(LogstoreConfig::DEFAULT),
}
}
pub fn empty_storage_sized(
page_size: usize,
compression: Compression,
) -> (TempDir, Blobstore<Payload>) {
let dir = Builder::new().prefix("test-storage").tempdir().unwrap();
let config = StorageConfig::Mutable(GridstoreConfig {
page_size_bytes: page_size,
block_size_bytes: DEFAULT_BLOCK_SIZE_BYTES,
region_size_blocks: DEFAULT_REGION_SIZE_BLOCKS,
compression,
});
let storage = Blobstore::new(MmapFs, dir.path().to_path_buf(), config).unwrap();
(dir, storage)
}
pub fn random_word(rng: &mut impl Rng) -> String {
let len = rng.random_range(1..10);
let mut word = String::with_capacity(len);
for _ in 0..len {
word.push(rng.random_range(b'a'..=b'z') as char);
}
word
}
pub fn random_payload(rng: &mut impl Rng, size_factor: usize) -> Payload {
let mut payload = Payload::default();
let word = random_word(rng);
let sentence = (0..rng.random_range(1..20 * size_factor))
.map(|_| random_word(rng))
.collect::<Vec<_>>()
.join(" ");
let distr = Uniform::new(0, 100000).unwrap();
let indices = (0..rng.random_range(1..100 * size_factor))
.map(|_| distr.sample(rng))
.collect::<Vec<_>>();
payload.0 = serde_json::json!(
{
"word": word, "sentence": sentence, "number": rng.random_range(0..1000), "indices": indices, "bool": rng.random_bool(0.5), "null": serde_json::Value::Null, "object": {
"bool": rng.random_bool(0.5),
}, }
)
.as_object()
.unwrap()
.clone();
payload
}
pub fn minimal_payload() -> Payload {
Payload(serde_json::json!({"a": 1}).as_object().unwrap().clone())
}
pub const HM_FIELDS: [&str; 23] = [
"article_id",
"product_code",
"prod_name",
"product_type_no",
"product_type_name",
"product_group_name",
"graphical_appearance_no",
"graphical_appearance_name",
"colour_group_code",
"colour_group_name",
"perceived_colour_value_id",
"perceived_colour_value_name",
"perceived_colour_master_id",
"perceived_colour_master_name",
"department_no",
"department_name",
"index_code,index_name",
"index_group_no",
"index_group_name",
"section_no,section_name",
"garment_group_no",
"garment_group_name",
"detail_desc",
];
#[cfg(test)]
mod tests {
use crate::blobstore::Blob;
use crate::blobstore::fixtures::Payload;
#[test]
fn test_serde_symmetry() {
let mut payload = Payload::default();
payload.0.insert(
"key".to_string(),
serde_json::Value::String("value".to_string()),
);
let bytes = payload.to_bytes();
let deserialized = Payload::from_bytes(&bytes);
assert_eq!(payload, deserialized);
}
}