use persy::{
IndexId, IndexInfo, IndexType, IndexTypeId, Persy, PersyError, PersyId, SegmentId, Transaction, Value, ValueMode,
};
use std::io;
use std::iter::Iterator;
pub type EIRes<T> = Result<T, Error>;
#[cfg(feature = "json")]
mod json;
#[cfg(feature = "json")]
pub use json::{export_json, import_json};
#[derive(Debug)]
pub enum Error {
PersyError(PersyError),
IoError(io::Error),
SerdeError(String),
}
impl From<PersyError> for Error {
fn from(err: PersyError) -> Error {
Error::PersyError(err)
}
}
impl From<io::Error> for Error {
fn from(err: io::Error) -> Error {
Error::IoError(err)
}
}
#[cfg(feature = "json")]
impl From<serde_json::error::Error> for Error {
fn from(err: serde_json::error::Error) -> Error {
Error::SerdeError(format!("{}", err))
}
}
pub struct SegmentMetadata {
pub version: u32,
pub name: String,
pub id: SegmentId,
}
pub struct Record {
pub version: u32,
pub segment: String,
pub id: PersyId,
pub content: Vec<u8>,
}
pub struct IndexMetadata {
pub version: u32,
pub name: String,
pub id: IndexId,
pub key_type: IndexTypeId,
pub value_type: IndexTypeId,
pub value_mode: ValueMode,
}
#[derive(Clone)]
pub enum IndexKey {
U8(u8),
U16(u16),
U32(u32),
U64(u64),
I8(i8),
I16(i16),
I32(i32),
I64(i64),
STRING(String),
PERSYID(PersyId),
}
#[derive(Clone)]
pub enum IndexValue {
U8(Vec<u8>),
U16(Vec<u16>),
U32(Vec<u32>),
U64(Vec<u64>),
I8(Vec<i8>),
I16(Vec<i16>),
I32(Vec<i32>),
I64(Vec<i64>),
STRING(Vec<String>),
PERSYID(Vec<PersyId>),
}
fn range<K: IndexType, V: IndexType>(persy: Persy, name: &str) -> EIRes<impl Iterator<Item = (K, Value<V>)>> {
Ok(persy.range::<K, V, _>(name, ..)?)
}
fn resolve_value_type<K: IndexType + 'static>(
persy: Persy,
name: &str,
info: IndexInfo,
) -> EIRes<Box<Iterator<Item = (K, IndexValue)>>> {
Ok(match info.value_type {
IndexTypeId::U8 => {
Box::new(range::<K, u8>(persy, name)?.map(|(k, v)| (k, IndexValue::U8(v.into_iter().collect()))))
}
IndexTypeId::U16 => {
Box::new(range::<K, u16>(persy, name)?.map(|(k, v)| (k, IndexValue::U16(v.into_iter().collect()))))
}
IndexTypeId::U32 => {
Box::new(range::<K, u32>(persy, name)?.map(|(k, v)| (k, IndexValue::U32(v.into_iter().collect()))))
}
IndexTypeId::U64 => {
Box::new(range::<K, u64>(persy, name)?.map(|(k, v)| (k, IndexValue::U64(v.into_iter().collect()))))
}
IndexTypeId::I8 => {
Box::new(range::<K, i8>(persy, name)?.map(|(k, v)| (k, IndexValue::I8(v.into_iter().collect()))))
}
IndexTypeId::I16 => {
Box::new(range::<K, i16>(persy, name)?.map(|(k, v)| (k, IndexValue::I16(v.into_iter().collect()))))
}
IndexTypeId::I32 => {
Box::new(range::<K, i32>(persy, name)?.map(|(k, v)| (k, IndexValue::I32(v.into_iter().collect()))))
}
IndexTypeId::I64 => {
Box::new(range::<K, i64>(persy, name)?.map(|(k, v)| (k, IndexValue::I64(v.into_iter().collect()))))
}
IndexTypeId::STRING => {
Box::new(range::<K, String>(persy, name)?.map(|(k, v)| (k, IndexValue::STRING(v.into_iter().collect()))))
}
IndexTypeId::PERSYID => {
Box::new(range::<K, PersyId>(persy, name)?.map(|(k, v)| (k, IndexValue::PERSYID(v.into_iter().collect()))))
}
})
}
fn browse(persy: Persy, name: &str, info: IndexInfo) -> EIRes<Box<Iterator<Item = (IndexKey, IndexValue)>>> {
Ok(match info.key_type {
IndexTypeId::U8 => Box::new(resolve_value_type::<u8>(persy, name, info)?.map(|(k, v)| (IndexKey::U8(k), v))),
IndexTypeId::U16 => Box::new(resolve_value_type::<u16>(persy, name, info)?.map(|(k, v)| (IndexKey::U16(k), v))),
IndexTypeId::U32 => Box::new(resolve_value_type::<u32>(persy, name, info)?.map(|(k, v)| (IndexKey::U32(k), v))),
IndexTypeId::U64 => Box::new(resolve_value_type::<u64>(persy, name, info)?.map(|(k, v)| (IndexKey::U64(k), v))),
IndexTypeId::I8 => Box::new(resolve_value_type::<i8>(persy, name, info)?.map(|(k, v)| (IndexKey::I8(k), v))),
IndexTypeId::I16 => Box::new(resolve_value_type::<i16>(persy, name, info)?.map(|(k, v)| (IndexKey::I16(k), v))),
IndexTypeId::I32 => Box::new(resolve_value_type::<i32>(persy, name, info)?.map(|(k, v)| (IndexKey::I32(k), v))),
IndexTypeId::I64 => Box::new(resolve_value_type::<i64>(persy, name, info)?.map(|(k, v)| (IndexKey::I64(k), v))),
IndexTypeId::STRING => {
Box::new(resolve_value_type::<String>(persy, name, info)?.map(|(k, v)| (IndexKey::STRING(k), v)))
}
IndexTypeId::PERSYID => {
Box::new(resolve_value_type::<PersyId>(persy, name, info)?.map(|(k, v)| (IndexKey::PERSYID(k), v)))
}
})
}
pub struct Entry {
pub version: u32,
pub index: String,
pub key: IndexKey,
pub value: IndexValue,
}
pub enum Info {
Segment(SegmentMetadata),
Record(Record),
Index(IndexMetadata),
Entry(Entry),
}
fn iter_keys(persy: Persy, (name, infos): (String, IndexInfo)) -> EIRes<Box<impl Iterator<Item = Info>>> {
let entries = browse(persy.clone(), &name, infos)?;
Ok(Box::new(entries.map(move |(k, v)| {
Info::Entry(Entry {
version: 0,
index: name.clone(),
key: k,
value: v,
})
})))
}
fn create_index_k<K: IndexType + 'static>(persy: &Persy, tx: &mut Transaction, info: IndexMetadata) -> EIRes<()> {
Ok(match info.value_type {
IndexTypeId::U8 => persy.create_index::<K, u8>(tx, &info.name, info.value_mode)?,
IndexTypeId::U16 => persy.create_index::<K, u16>(tx, &info.name, info.value_mode)?,
IndexTypeId::U32 => persy.create_index::<K, u32>(tx, &info.name, info.value_mode)?,
IndexTypeId::U64 => persy.create_index::<K, u64>(tx, &info.name, info.value_mode)?,
IndexTypeId::I8 => persy.create_index::<K, i8>(tx, &info.name, info.value_mode)?,
IndexTypeId::I16 => persy.create_index::<K, i16>(tx, &info.name, info.value_mode)?,
IndexTypeId::I32 => persy.create_index::<K, i32>(tx, &info.name, info.value_mode)?,
IndexTypeId::I64 => persy.create_index::<K, i64>(tx, &info.name, info.value_mode)?,
IndexTypeId::STRING => persy.create_index::<K, String>(tx, &info.name, info.value_mode)?,
IndexTypeId::PERSYID => persy.create_index::<K, PersyId>(tx, &info.name, info.value_mode)?,
})
}
fn create_index(persy: &Persy, tx: &mut Transaction, info: IndexMetadata) -> EIRes<()> {
match info.key_type {
IndexTypeId::U8 => create_index_k::<u8>(persy, tx, info),
IndexTypeId::U16 => create_index_k::<u16>(persy, tx, info),
IndexTypeId::U32 => create_index_k::<u32>(persy, tx, info),
IndexTypeId::U64 => create_index_k::<u64>(persy, tx, info),
IndexTypeId::I8 => create_index_k::<i8>(persy, tx, info),
IndexTypeId::I16 => create_index_k::<i16>(persy, tx, info),
IndexTypeId::I32 => create_index_k::<i32>(persy, tx, info),
IndexTypeId::I64 => create_index_k::<i64>(persy, tx, info),
IndexTypeId::STRING => create_index_k::<String>(persy, tx, info),
IndexTypeId::PERSYID => create_index_k::<PersyId>(persy, tx, info),
}
}
fn put_all<K: IndexType, V: IndexType>(persy: &Persy, tx: &mut Transaction, index: &str, k: K, v: Vec<V>) -> EIRes<()> {
for val in v {
persy.put(tx, index, k.clone(), val)?;
}
Ok(())
}
fn put_entry_k<K: IndexType + 'static>(persy: &Persy, tx: &mut Transaction, k: K, entry: Entry) -> EIRes<()> {
match entry.value {
IndexValue::U8(v) => put_all(persy, tx, &entry.index, k, v),
IndexValue::U16(v) => put_all(persy, tx, &entry.index, k, v),
IndexValue::U32(v) => put_all(persy, tx, &entry.index, k, v),
IndexValue::U64(v) => put_all(persy, tx, &entry.index, k, v),
IndexValue::I8(v) => put_all(persy, tx, &entry.index, k, v),
IndexValue::I16(v) => put_all(persy, tx, &entry.index, k, v),
IndexValue::I32(v) => put_all(persy, tx, &entry.index, k, v),
IndexValue::I64(v) => put_all(persy, tx, &entry.index, k, v),
IndexValue::STRING(v) => put_all(persy, tx, &entry.index, k, v),
IndexValue::PERSYID(v) => put_all(persy, tx, &entry.index, k, v),
}
}
fn put_entry(persy: &Persy, tx: &mut Transaction, entry: Entry) -> EIRes<()> {
match entry.key.clone() {
IndexKey::U8(k) => put_entry_k(persy, tx, k, entry),
IndexKey::U16(k) => put_entry_k(persy, tx, k, entry),
IndexKey::U32(k) => put_entry_k(persy, tx, k, entry),
IndexKey::U64(k) => put_entry_k(persy, tx, k, entry),
IndexKey::I8(k) => put_entry_k(persy, tx, k, entry),
IndexKey::I16(k) => put_entry_k(persy, tx, k, entry),
IndexKey::I32(k) => put_entry_k(persy, tx, k, entry),
IndexKey::I64(k) => put_entry_k(persy, tx, k, entry),
IndexKey::STRING(k) => put_entry_k(persy, tx, k, entry),
IndexKey::PERSYID(k) => put_entry_k(persy, tx, k, entry),
}
}
pub fn export(persy_p: &Persy) -> EIRes<Box<impl Iterator<Item = Info>>> {
let persy = persy_p.clone();
let segs = persy.list_segments()?.into_iter();
let indexes = persy.list_indexes()?.into_iter();
let p_keys = persy.clone();
let all_keys = indexes
.clone()
.map(move |i| iter_keys(p_keys.clone(), i))
.flatten()
.flatten();
let indexes_info = indexes.map(|(name, infos)| {
Info::Index(IndexMetadata {
version: 0,
name,
id: infos.id,
key_type: infos.key_type,
value_type: infos.value_type,
value_mode: infos.value_mode,
})
});
let all_records = segs
.clone()
.map(move |(name, _)| -> EIRes<_> {
Ok(persy.clone().scan(&name)?.map(move |(id, content)| {
Info::Record(Record {
version: 0,
segment: name.clone(),
id,
content,
})
}))
})
.flatten()
.flatten();
let segs_info = segs.map(|(name, id)| Info::Segment(SegmentMetadata { version: 0, name, id }));
Ok(Box::new(
segs_info.chain(indexes_info).chain(all_keys).chain(all_records),
))
}
pub fn import<I>(persy: &Persy, importer: I) -> EIRes<()>
where
I: Iterator<Item = Info>,
{
for val in importer {
match val {
Info::Segment(segment) => {
let mut tx = persy.begin()?;
persy.create_segment(&mut tx, &segment.name)?;
let prep = persy.prepare_commit(tx)?;
persy.commit(prep)?;
}
Info::Record(record) => {
let mut tx = persy.begin()?;
persy.insert_record(&mut tx, &record.segment, &record.content)?;
let prep = persy.prepare_commit(tx)?;
persy.commit(prep)?;
}
Info::Index(index) => {
let mut tx = persy.begin()?;
create_index(&persy, &mut tx, index)?;
let prep = persy.prepare_commit(tx)?;
persy.commit(prep)?;
}
Info::Entry(entry) => {
let mut tx = persy.begin()?;
put_entry(&persy, &mut tx, entry)?;
let prep = persy.prepare_commit(tx)?;
persy.commit(prep)?;
}
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::{export, import, Info};
use persy::{Config, IndexType, Persy, PersyId, Transaction, ValueMode};
use std::fs;
fn util<C, V>(name: &str, create: C, verify: V)
where
C: FnOnce(&Persy, &mut Transaction),
V: FnOnce(&Persy),
{
let c_name = format!("target/{}.pei", name);
let rec_name = format!("target/{}_fill.pei", name);
Persy::create(c_name.to_string()).expect("creation works fine");
let p = Persy::open(c_name.to_string(), Config::new()).expect("open fine");
let mut tx = p.begin().unwrap();
create(&p, &mut tx);
let prep = p.prepare_commit(tx).unwrap();
p.commit(prep).unwrap();
let data: Vec<Info> = export(&p).unwrap().collect();
Persy::create(rec_name.to_string()).expect("creation works fine");
let p1 = Persy::open(rec_name.to_string(), Config::new()).expect("open fine");
import(&p1, data.into_iter()).unwrap();
verify(&p1);
fs::remove_file(c_name).unwrap();
fs::remove_file(rec_name).unwrap();
}
#[test]
fn segment_metadata_export_import() {
util(
"segment",
|p, tx| {
p.create_segment(tx, "test").unwrap();
},
|p| {
assert!(p.exists_segment("test").unwrap());
},
);
}
#[test]
fn index_metadata_export_import() {
util(
"index",
|p, tx| {
p.create_index::<u8, u8>(tx, "test", ValueMode::REPLACE).unwrap();
},
|p| {
assert!(p.exists_index("test").unwrap());
},
);
}
#[test]
fn segment_data_export_import() {
util(
"segment_data",
|p, tx| {
p.create_segment(tx, "test").unwrap();
p.insert_record(tx, "test", &"test".to_string().as_bytes()).unwrap();
},
|p| {
for (_, content) in p.scan("test").unwrap() {
assert_eq!("test".to_string().as_bytes().to_vec(), content);
}
},
);
}
#[test]
fn index_data_export_import() {
util(
"index_data",
|p, tx| {
p.create_index::<u8, u8>(tx, "test", ValueMode::REPLACE).unwrap();
p.put::<u8, u8>(tx, "test", 10, 10).unwrap();
},
|p| {
assert_eq!(
10,
p.get::<u8, u8>("test", &10)
.unwrap()
.unwrap()
.into_iter()
.next()
.unwrap()
)
},
);
}
#[test]
fn all_data_export_import() {
util(
"all_data",
|p, tx| {
p.create_segment(tx, "test").unwrap();
let persy_id = p.insert_record(tx, "test", &"test".to_string().as_bytes()).unwrap();
p.create_index::<u8, u8>(tx, "test_u8", ValueMode::REPLACE).unwrap();
p.put::<u8, u8>(tx, "test_u8", 10, 10).unwrap();
p.create_index::<u16, u16>(tx, "test_u16", ValueMode::REPLACE).unwrap();
p.put::<u16, u16>(tx, "test_u16", 10, 10).unwrap();
p.create_index::<u32, u32>(tx, "test_u32", ValueMode::REPLACE).unwrap();
p.put::<u32, u32>(tx, "test_u32", 10, 10).unwrap();
p.create_index::<u64, u64>(tx, "test_u64", ValueMode::REPLACE).unwrap();
p.put::<u64, u64>(tx, "test_u64", 10, 10).unwrap();
p.create_index::<i8, i8>(tx, "test_i8", ValueMode::REPLACE).unwrap();
p.put::<i8, i8>(tx, "test_i8", 10, 10).unwrap();
p.create_index::<i16, i16>(tx, "test_i16", ValueMode::REPLACE).unwrap();
p.put::<i16, i16>(tx, "test_i16", 10, 10).unwrap();
p.create_index::<i32, i32>(tx, "test_i32", ValueMode::REPLACE).unwrap();
p.put::<i32, i32>(tx, "test_i32", 10, 10).unwrap();
p.create_index::<i64, i64>(tx, "test_i64", ValueMode::REPLACE).unwrap();
p.put::<i64, i64>(tx, "test_i64", 10, 10).unwrap();
p.create_index::<String, String>(tx, "test_string", ValueMode::REPLACE)
.unwrap();
p.put::<String, String>(tx, "test_string", "one".to_string(), "two".to_string())
.unwrap();
p.create_index::<PersyId, PersyId>(tx, "test_p", ValueMode::REPLACE)
.unwrap();
p.put::<PersyId, PersyId>(tx, "test_p", persy_id.clone(), persy_id)
.unwrap();
},
|p| {
for (_, content) in p.scan("test").unwrap() {
assert_eq!("test".to_string().as_bytes().to_vec(), content);
}
check_key_value(p, "test_u8", 10 as u8, 10 as u8);
check_key_value(p, "test_u16", 10 as u16, 10 as u16);
check_key_value(p, "test_u32", 10 as u32, 10 as u32);
check_key_value(p, "test_u64", 10 as u64, 10 as u64);
check_key_value(p, "test_i8", 10 as i8, 10 as i8);
check_key_value(p, "test_i16", 10 as i16, 10 as i16);
check_key_value(p, "test_i32", 10 as i32, 10 as i32);
check_key_value(p, "test_i64", 10 as i64, 10 as i64);
check_key_value(p, "test_string", "one".to_string(), "two".to_string());
assert_eq!(
1,
p.range::<PersyId, PersyId, _>("test_p", ..)
.unwrap()
.into_iter()
.count()
);
},
);
}
pub fn check_key_value<K: IndexType, V: IndexType + std::fmt::Debug>(p: &Persy, index: &str, k: K, v: V) {
assert_eq!(
v,
p.get::<K, V>(index, &k).unwrap().unwrap().into_iter().next().unwrap()
);
}
}