persy_expimp/

lib.rs

//! Export/Import Persy storage data to a external structure.
//!
//! ## Supported format:  
//! ### Custom
//!  Hust extract/import the data through defined structs and use a custom write/read
//! Functions:  
//!  [export](fn.export.html),  [import](fn.import.html).  
//! ### JSON
//! export/import to a JSON serialized Read/Write
//! Functions:  
//! [export_json](fn.export_json.html), [import_json](fn.import_json.html).  
//!
//! # Examples
//!
//! ## Custom Export Example
//!
//! ```
//! use persy::{Persy, Config};
//! use persy_expimp::export;
//! use std::vec::Vec;
//! # use persy_expimp::EIRes;
//! # fn foo() -> EIRes<()> {
//! let persy = Persy::open("file.persy", Config::new())?;
//! for info in export(&persy)? {
//!     // Custom logic
//! }
//! # Ok(())
//! # }
//! ```
//! ## Custom Import Example
//!
//! ```
//! use persy::{Persy, Config};
//! use persy_expimp::{import,Info};
//! # use persy_expimp::EIRes;
//! # fn foo() -> EIRes<()> {
//! Persy::create("imported.persy")?;
//! let persy = Persy::open("imported.persy", Config::new())?;
//! // Source informations from a custom source
//! let source = Vec::<Info>::new();
//! import(&persy,source.into_iter())?;
//! # Ok(())
//! # }
//! ```
//!
//!
//!

//!
use persy::{
    ByteVec, IndexId, IndexInfo, IndexType, IndexTypeId, Persy, PersyError, PersyId, SegmentId, Snapshot, Transaction,
    ValueIter, ValueMode,
};
use std::io;
use std::iter::Iterator;
use std::ops::RangeBounds;

pub type EIRes<T> = Result<T, Error>;

#[cfg(feature = "serde")]
use serde_derive::{Deserialize, Serialize};

#[cfg(feature = "serde")]
mod serde;
#[cfg(feature = "serde")]
use crate::serde::{
    bytevec_deserialize, bytevec_serialize, index_id_deserialize, index_id_serialize, index_type_id_deserialize,
    index_type_id_serialize, persy_id_deserialize, persy_id_serialize, segment_id_deserialize, segment_id_serialize,
    value_mode_deserialize, value_mode_serialize, vec_bytevec_deserialize, vec_bytevec_serialize,
    vec_persy_id_deserialize, vec_persy_id_serialize,
};

#[cfg(feature = "json")]
mod json;
#[cfg(feature = "json")]
pub use json::{export_json, export_json_from_snapshot, import_json};

#[cfg(feature = "binary")]
mod binary;
#[cfg(feature = "binary")]
pub use binary::{export_binary, export_binary_from_snapshot, import_binary};

#[derive(Debug)]
pub enum Error {
    PersyError(PersyError),
    IoError(io::Error),
    SerdeError(String),
}

impl<T: Into<PersyError>> From<persy::PE<T>> for Error {
    fn from(err: persy::PE<T>) -> Error {
        Error::PersyError(err.error().into())
    }
}

impl From<io::Error> for Error {
    fn from(err: io::Error) -> Error {
        Error::IoError(err)
    }
}

#[cfg(feature = "json")]
impl From<serde_json::error::Error> for Error {
    fn from(err: serde_json::error::Error) -> Error {
        Error::SerdeError(format!("{}", err))
    }
}

#[cfg(feature = "bincode")]
impl From<bincode::Error> for Error {
    fn from(err: bincode::Error) -> Error {
        Error::SerdeError(format!("{}", err))
    }
}

#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub struct SegmentMetadata {
    pub version: u32,
    pub name: String,
    #[cfg_attr(
        feature = "serde",
        serde(serialize_with = "segment_id_serialize", deserialize_with = "segment_id_deserialize")
    )]
    pub id: SegmentId,
}

#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub struct Record {
    pub version: u32,
    pub segment: String,
    #[cfg_attr(
        feature = "serde",
        serde(serialize_with = "persy_id_serialize", deserialize_with = "persy_id_deserialize")
    )]
    pub id: PersyId,
    pub content: Vec<u8>,
}

#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub struct IndexMetadata {
    pub version: u32,
    pub name: String,
    #[cfg_attr(
        feature = "serde",
        serde(serialize_with = "index_id_serialize", deserialize_with = "index_id_deserialize")
    )]
    pub id: IndexId,
    #[cfg_attr(
        feature = "serde",
        serde(
            serialize_with = "index_type_id_serialize",
            deserialize_with = "index_type_id_deserialize"
        )
    )]
    pub key_type: IndexTypeId,
    #[cfg_attr(
        feature = "serde",
        serde(
            serialize_with = "index_type_id_serialize",
            deserialize_with = "index_type_id_deserialize"
        )
    )]
    pub value_type: IndexTypeId,
    #[cfg_attr(
        feature = "serde",
        serde(serialize_with = "value_mode_serialize", deserialize_with = "value_mode_deserialize")
    )]
    pub value_mode: ValueMode,
}

#[derive(Clone)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub enum IndexKey {
    U8(u8),
    U16(u16),
    U32(u32),
    U64(u64),
    U128(u128),
    I8(i8),
    I16(i16),
    I32(i32),
    I64(i64),
    I128(i128),
    F32(f32),
    F64(f64),
    String(String),
    #[cfg_attr(
        feature = "serde",
        serde(serialize_with = "persy_id_serialize", deserialize_with = "persy_id_deserialize")
    )]
    PersyId(PersyId),
    #[cfg_attr(
        feature = "serde",
        serde(serialize_with = "bytevec_serialize", deserialize_with = "bytevec_deserialize")
    )]
    ByteVec(ByteVec),
}

#[derive(Clone)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub enum IndexValue {
    U8(Vec<u8>),
    U16(Vec<u16>),
    U32(Vec<u32>),
    U64(Vec<u64>),
    U128(Vec<u128>),
    I8(Vec<i8>),
    I16(Vec<i16>),
    I32(Vec<i32>),
    I64(Vec<i64>),
    I128(Vec<i128>),
    F32(Vec<f32>),
    F64(Vec<f64>),
    String(Vec<String>),
    #[cfg_attr(
        feature = "serde",
        serde(
            serialize_with = "vec_persy_id_serialize",
            deserialize_with = "vec_persy_id_deserialize"
        )
    )]
    PersyId(Vec<PersyId>),
    #[cfg_attr(
        feature = "serde",
        serde(
            serialize_with = "vec_bytevec_serialize",
            deserialize_with = "vec_bytevec_deserialize"
        )
    )]
    ByteVec(Vec<ByteVec>),
}

fn range<K: IndexType + 'static, V: IndexType + 'static, T: PersyReader>(
    persy: T,
    name: &str,
) -> EIRes<impl Iterator<Item = (K, ValueIter<V>)>> {
    persy.range::<K, V, _>(name, ..)
}

fn resolve_value_type<K: IndexType + 'static, T: PersyReader + 'static>(
    persy: T,
    name: &str,
    info: IndexInfo,
) -> EIRes<Box<dyn Iterator<Item = (K, IndexValue)>>> {
    Ok(match info.value_type {
        IndexTypeId::U8 => {
            Box::new(range::<K, u8, T>(persy, name)?.map(|(k, v)| (k, IndexValue::U8(v.into_iter().collect()))))
        }
        IndexTypeId::U16 => {
            Box::new(range::<K, u16, T>(persy, name)?.map(|(k, v)| (k, IndexValue::U16(v.into_iter().collect()))))
        }
        IndexTypeId::U32 => {
            Box::new(range::<K, u32, T>(persy, name)?.map(|(k, v)| (k, IndexValue::U32(v.into_iter().collect()))))
        }
        IndexTypeId::U64 => {
            Box::new(range::<K, u64, T>(persy, name)?.map(|(k, v)| (k, IndexValue::U64(v.into_iter().collect()))))
        }
        IndexTypeId::U128 => {
            Box::new(range::<K, u128, T>(persy, name)?.map(|(k, v)| (k, IndexValue::U128(v.into_iter().collect()))))
        }
        IndexTypeId::I8 => {
            Box::new(range::<K, i8, T>(persy, name)?.map(|(k, v)| (k, IndexValue::I8(v.into_iter().collect()))))
        }
        IndexTypeId::I16 => {
            Box::new(range::<K, i16, T>(persy, name)?.map(|(k, v)| (k, IndexValue::I16(v.into_iter().collect()))))
        }
        IndexTypeId::I32 => {
            Box::new(range::<K, i32, T>(persy, name)?.map(|(k, v)| (k, IndexValue::I32(v.into_iter().collect()))))
        }
        IndexTypeId::I64 => {
            Box::new(range::<K, i64, T>(persy, name)?.map(|(k, v)| (k, IndexValue::I64(v.into_iter().collect()))))
        }
        IndexTypeId::I128 => {
            Box::new(range::<K, i128, T>(persy, name)?.map(|(k, v)| (k, IndexValue::I128(v.into_iter().collect()))))
        }
        IndexTypeId::F32W => {
            Box::new(range::<K, f32, T>(persy, name)?.map(|(k, v)| (k, IndexValue::F32(v.into_iter().collect()))))
        }
        IndexTypeId::F64W => {
            Box::new(range::<K, f64, T>(persy, name)?.map(|(k, v)| (k, IndexValue::F64(v.into_iter().collect()))))
        }
        IndexTypeId::String => {
            Box::new(range::<K, String, T>(persy, name)?.map(|(k, v)| (k, IndexValue::String(v.into_iter().collect()))))
        }
        IndexTypeId::PersyId => Box::new(
            range::<K, PersyId, T>(persy, name)?.map(|(k, v)| (k, IndexValue::PersyId(v.into_iter().collect()))),
        ),
        IndexTypeId::ByteVec => Box::new(
            range::<K, ByteVec, T>(persy, name)?.map(|(k, v)| (k, IndexValue::ByteVec(v.into_iter().collect()))),
        ),
    })
}
fn browse<T: PersyReader + 'static>(
    persy: T,
    name: &str,
    info: IndexInfo,
) -> EIRes<Box<dyn Iterator<Item = (IndexKey, IndexValue)>>> {
    Ok(match info.key_type {
        IndexTypeId::U8 => Box::new(resolve_value_type::<u8, T>(persy, name, info)?.map(|(k, v)| (IndexKey::U8(k), v))),
        IndexTypeId::U16 => {
            Box::new(resolve_value_type::<u16, T>(persy, name, info)?.map(|(k, v)| (IndexKey::U16(k), v)))
        }
        IndexTypeId::U32 => {
            Box::new(resolve_value_type::<u32, T>(persy, name, info)?.map(|(k, v)| (IndexKey::U32(k), v)))
        }
        IndexTypeId::U64 => {
            Box::new(resolve_value_type::<u64, T>(persy, name, info)?.map(|(k, v)| (IndexKey::U64(k), v)))
        }
        IndexTypeId::U128 => {
            Box::new(resolve_value_type::<u128, T>(persy, name, info)?.map(|(k, v)| (IndexKey::U128(k), v)))
        }
        IndexTypeId::I8 => Box::new(resolve_value_type::<i8, T>(persy, name, info)?.map(|(k, v)| (IndexKey::I8(k), v))),
        IndexTypeId::I16 => {
            Box::new(resolve_value_type::<i16, T>(persy, name, info)?.map(|(k, v)| (IndexKey::I16(k), v)))
        }
        IndexTypeId::I32 => {
            Box::new(resolve_value_type::<i32, T>(persy, name, info)?.map(|(k, v)| (IndexKey::I32(k), v)))
        }
        IndexTypeId::I64 => {
            Box::new(resolve_value_type::<i64, T>(persy, name, info)?.map(|(k, v)| (IndexKey::I64(k), v)))
        }
        IndexTypeId::I128 => {
            Box::new(resolve_value_type::<i128, T>(persy, name, info)?.map(|(k, v)| (IndexKey::I128(k), v)))
        }
        IndexTypeId::F32W => {
            Box::new(resolve_value_type::<f32, T>(persy, name, info)?.map(|(k, v)| (IndexKey::F32(k), v)))
        }
        IndexTypeId::F64W => {
            Box::new(resolve_value_type::<f64, T>(persy, name, info)?.map(|(k, v)| (IndexKey::F64(k), v)))
        }
        IndexTypeId::String => {
            Box::new(resolve_value_type::<String, T>(persy, name, info)?.map(|(k, v)| (IndexKey::String(k), v)))
        }
        IndexTypeId::PersyId => {
            Box::new(resolve_value_type::<PersyId, T>(persy, name, info)?.map(|(k, v)| (IndexKey::PersyId(k), v)))
        }
        IndexTypeId::ByteVec => {
            Box::new(resolve_value_type::<ByteVec, T>(persy, name, info)?.map(|(k, v)| (IndexKey::ByteVec(k), v)))
        }
    })
}

#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub struct Entry {
    pub version: u32,
    pub index: String,
    pub key: IndexKey,
    pub value: IndexValue,
}

#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub enum Info {
    Segment(SegmentMetadata),
    Record(Record),
    Index(IndexMetadata),
    Entry(Entry),
}

fn iter_keys<T: PersyReader + 'static>(
    persy: T,
    (name, infos): (String, IndexInfo),
) -> EIRes<Box<impl Iterator<Item = Info>>> {
    let entries = browse(persy, &name, infos)?;
    Ok(Box::new(entries.map(move |(k, v)| {
        Info::Entry(Entry {
            version: 0,
            index: name.clone(),
            key: k,
            value: v,
        })
    })))
}

fn create_index_k<K: IndexType + 'static>(tx: &mut Transaction, info: IndexMetadata) -> EIRes<()> {
    Ok(match info.value_type {
        IndexTypeId::U8 => tx.create_index::<K, u8>(&info.name, info.value_mode)?,
        IndexTypeId::U16 => tx.create_index::<K, u16>(&info.name, info.value_mode)?,
        IndexTypeId::U32 => tx.create_index::<K, u32>(&info.name, info.value_mode)?,
        IndexTypeId::U64 => tx.create_index::<K, u64>(&info.name, info.value_mode)?,
        IndexTypeId::U128 => tx.create_index::<K, u128>(&info.name, info.value_mode)?,
        IndexTypeId::I8 => tx.create_index::<K, i8>(&info.name, info.value_mode)?,
        IndexTypeId::I16 => tx.create_index::<K, i16>(&info.name, info.value_mode)?,
        IndexTypeId::I32 => tx.create_index::<K, i32>(&info.name, info.value_mode)?,
        IndexTypeId::I64 => tx.create_index::<K, i64>(&info.name, info.value_mode)?,
        IndexTypeId::I128 => tx.create_index::<K, i128>(&info.name, info.value_mode)?,
        IndexTypeId::F32W => tx.create_index::<K, f32>(&info.name, info.value_mode)?,
        IndexTypeId::F64W => tx.create_index::<K, f64>(&info.name, info.value_mode)?,
        IndexTypeId::String => tx.create_index::<K, String>(&info.name, info.value_mode)?,
        IndexTypeId::PersyId => tx.create_index::<K, PersyId>(&info.name, info.value_mode)?,
        IndexTypeId::ByteVec => tx.create_index::<K, ByteVec>(&info.name, info.value_mode)?,
    })
}
fn create_index(tx: &mut Transaction, info: IndexMetadata) -> EIRes<()> {
    match info.key_type {
        IndexTypeId::U8 => create_index_k::<u8>(tx, info),
        IndexTypeId::U16 => create_index_k::<u16>(tx, info),
        IndexTypeId::U32 => create_index_k::<u32>(tx, info),
        IndexTypeId::U64 => create_index_k::<u64>(tx, info),
        IndexTypeId::U128 => create_index_k::<u128>(tx, info),
        IndexTypeId::I8 => create_index_k::<i8>(tx, info),
        IndexTypeId::I16 => create_index_k::<i16>(tx, info),
        IndexTypeId::I32 => create_index_k::<i32>(tx, info),
        IndexTypeId::I64 => create_index_k::<i64>(tx, info),
        IndexTypeId::I128 => create_index_k::<i128>(tx, info),
        IndexTypeId::F32W => create_index_k::<f32>(tx, info),
        IndexTypeId::F64W => create_index_k::<f64>(tx, info),
        IndexTypeId::String => create_index_k::<String>(tx, info),
        IndexTypeId::PersyId => create_index_k::<PersyId>(tx, info),
        IndexTypeId::ByteVec => create_index_k::<ByteVec>(tx, info),
    }
}

fn remap_persy_id(tx: &mut Transaction, orig: PersyId, new: PersyId) -> EIRes<()> {
    Ok(tx.put("xx__persy__id__mapper", orig, new)?)
}

fn create_map_persy_id_index(tx: &mut Transaction) -> EIRes<()> {
    Ok(tx.create_index::<PersyId, PersyId>("xx__persy__id__mapper", ValueMode::Replace)?)
}

fn drop_map_persy_id_index(tx: &mut Transaction) -> EIRes<()> {
    Ok(tx.drop_index("xx__persy__id__mapper")?)
}

fn map_persy_id(tx: &mut Transaction, id: PersyId) -> EIRes<PersyId> {
    Ok(tx.get("xx__persy__id__mapper", &id)?.into_iter().next().unwrap_or(id))
}

fn map_persy_id_out(persy: &Persy, id: PersyId) -> EIRes<PersyId> {
    Ok(persy
        .get("xx__persy__id__mapper", &id)?
        .into_iter()
        .next()
        .unwrap_or(id))
}

fn put_all_id<K: IndexType>(tx: &mut Transaction, index: &str, k: K, v: Vec<PersyId>) -> EIRes<()> {
    for val in v {
        let mapped = map_persy_id(tx, val)?;
        tx.put(index, k.clone(), mapped)?;
    }
    Ok(())
}

fn put_all<K: IndexType, V: IndexType>(tx: &mut Transaction, index: &str, k: K, v: Vec<V>) -> EIRes<()> {
    for val in v {
        tx.put(index, k.clone(), val)?;
    }
    Ok(())
}

fn put_entry_k<K: IndexType + 'static>(tx: &mut Transaction, k: K, entry: Entry) -> EIRes<()> {
    match entry.value {
        IndexValue::U8(v) => put_all(tx, &entry.index, k, v),
        IndexValue::U16(v) => put_all(tx, &entry.index, k, v),
        IndexValue::U32(v) => put_all(tx, &entry.index, k, v),
        IndexValue::U64(v) => put_all(tx, &entry.index, k, v),
        IndexValue::U128(v) => put_all(tx, &entry.index, k, v),
        IndexValue::I8(v) => put_all(tx, &entry.index, k, v),
        IndexValue::I16(v) => put_all(tx, &entry.index, k, v),
        IndexValue::I32(v) => put_all(tx, &entry.index, k, v),
        IndexValue::I64(v) => put_all(tx, &entry.index, k, v),
        IndexValue::I128(v) => put_all(tx, &entry.index, k, v),
        IndexValue::F32(v) => put_all(tx, &entry.index, k, v),
        IndexValue::F64(v) => put_all(tx, &entry.index, k, v),
        IndexValue::String(v) => put_all(tx, &entry.index, k, v),
        IndexValue::PersyId(v) => put_all_id(tx, &entry.index, k, v),
        IndexValue::ByteVec(v) => put_all(tx, &entry.index, k, v),
    }
}
fn put_entry(tx: &mut Transaction, entry: Entry) -> EIRes<()> {
    match entry.key.clone() {
        IndexKey::U8(k) => put_entry_k(tx, k, entry),
        IndexKey::U16(k) => put_entry_k(tx, k, entry),
        IndexKey::U32(k) => put_entry_k(tx, k, entry),
        IndexKey::U64(k) => put_entry_k(tx, k, entry),
        IndexKey::U128(k) => put_entry_k(tx, k, entry),
        IndexKey::I8(k) => put_entry_k(tx, k, entry),
        IndexKey::I16(k) => put_entry_k(tx, k, entry),
        IndexKey::I32(k) => put_entry_k(tx, k, entry),
        IndexKey::I64(k) => put_entry_k(tx, k, entry),
        IndexKey::I128(k) => put_entry_k(tx, k, entry),
        IndexKey::F32(k) => put_entry_k(tx, k, entry),
        IndexKey::F64(k) => put_entry_k(tx, k, entry),
        IndexKey::String(k) => put_entry_k(tx, k, entry),
        IndexKey::PersyId(k) => {
            let mapped_key = map_persy_id(tx, k)?;
            put_entry_k(tx, mapped_key, entry)
        }
        IndexKey::ByteVec(k) => put_entry_k(tx, k, entry),
    }
}

pub trait PersyReader: Clone + Sized {
    fn list_segments(&self) -> EIRes<Vec<(String, SegmentId)>>;

    fn list_indexes(&self) -> EIRes<Vec<(String, IndexInfo)>>;

    fn scan(&self, segment: &str) -> EIRes<Box<dyn Iterator<Item = (PersyId, Vec<u8>)>>>;

    fn range<K, V, R>(&self, index_name: &str, range: R) -> EIRes<Box<dyn Iterator<Item = (K, ValueIter<V>)>>>
    where
        K: IndexType + 'static,
        V: IndexType + 'static,
        R: RangeBounds<K>;
}

impl PersyReader for Persy {
    fn list_segments(&self) -> EIRes<Vec<(String, SegmentId)>> {
        Ok(self.list_segments()?)
    }
    fn list_indexes(&self) -> EIRes<Vec<(String, IndexInfo)>> {
        Ok(self.list_indexes()?)
    }
    fn scan(&self, segment: &str) -> EIRes<Box<dyn Iterator<Item = (PersyId, Vec<u8>)>>> {
        Ok(Box::new(self.scan(segment)?))
    }
    fn range<K, V, R>(&self, index_name: &str, range: R) -> EIRes<Box<dyn Iterator<Item = (K, ValueIter<V>)>>>
    where
        K: IndexType + 'static,
        V: IndexType + 'static,
        R: RangeBounds<K>,
    {
        Ok(Box::new(self.range::<K, V, R>(index_name, range)?))
    }
}
impl PersyReader for Snapshot {
    fn list_segments(&self) -> EIRes<Vec<(String, SegmentId)>> {
        Ok(self.list_segments()?)
    }

    fn list_indexes(&self) -> EIRes<Vec<(String, IndexInfo)>> {
        Ok(self.list_indexes()?)
    }

    fn scan(&self, segment: &str) -> EIRes<Box<dyn Iterator<Item = (PersyId, Vec<u8>)>>> {
        Ok(Box::new(self.scan(segment)?))
    }
    fn range<K, V, R>(&self, index_name: &str, range: R) -> EIRes<Box<dyn Iterator<Item = (K, ValueIter<V>)>>>
    where
        K: IndexType + 'static,
        V: IndexType + 'static,
        R: RangeBounds<K>,
    {
        Ok(Box::new(self.range::<K, V, R>(index_name, range)?))
    }
}

///
/// Export the data from a persy storage with a custom format
///
/// # Example
/// ```
/// use persy::{Persy, Config};
/// use persy_expimp::export;
/// use std::vec::Vec;
/// # use persy_expimp::EIRes;
/// # fn foo() -> EIRes<()> {
/// let persy = Persy::open("file.persy", Config::new())?;
/// for info in export(&persy)? {
///     // Custom logic
/// }
/// # Ok(())
/// # }
/// ```
pub fn export(persy_p: &Persy) -> EIRes<Box<impl Iterator<Item = Info>>> {
    export_from_reader(persy_p)
}
fn export_from_reader<T: PersyReader + 'static>(persy_p: &T) -> EIRes<Box<impl Iterator<Item = Info>>> {
    let persy = persy_p.clone();
    let segs = persy.list_segments()?.into_iter();
    let indexes = persy.list_indexes()?.into_iter();
    let p_keys = persy.clone();
    let all_keys = indexes
        .clone()
        .map(move |i| iter_keys(p_keys.clone(), i))
        .flatten()
        .flatten();
    let indexes_info = indexes.map(|(name, infos)| {
        Info::Index(IndexMetadata {
            version: 0,
            name,
            id: infos.id,
            key_type: infos.key_type,
            value_type: infos.value_type,
            value_mode: infos.value_mode,
        })
    });
    let all_records = segs
        .clone()
        .map(move |(name, _)| -> EIRes<_> {
            Ok(persy.clone().scan(&name)?.map(move |(id, content)| {
                Info::Record(Record {
                    version: 0,
                    segment: name.clone(),
                    id,
                    content,
                })
            }))
        })
        .flatten()
        .flatten();
    let segs_info = segs.map(|(name, id)| Info::Segment(SegmentMetadata { version: 0, name, id }));
    Ok(Box::new(
        segs_info.chain(indexes_info).chain(all_records).chain(all_keys),
    ))
}

///
/// Export the data from a snapshot of persy storage with a custom format
///
/// # Example
/// ```
/// use persy::{Persy, Config};
/// use persy_expimp::export_from_snapshot;
/// use std::vec::Vec;
/// # use persy_expimp::EIRes;
/// # fn foo() -> EIRes<()> {
/// let persy = Persy::open("file.persy", Config::new())?;
/// let snapshot = persy.snapshot()?;
/// for info in export_from_snapshot(&snapshot)? {
///     // Custom logic
/// }
/// # Ok(())
/// # }
/// ```
pub fn export_from_snapshot(persy_p: &Snapshot) -> EIRes<Box<impl Iterator<Item = Info>>> {
    export_from_reader(persy_p)
}

/// Import the data to a persy storage from a custom format
///
/// # Example
/// ```
/// use persy::{Persy, Config};
/// use persy_expimp::{import,Info};
/// # use persy_expimp::EIRes;
/// # fn foo() -> EIRes<()> {
/// Persy::create("imported.persy")?;
/// let persy = Persy::open("imported.persy", Config::new())?;
/// // Source informations from a custom source
/// let source = Vec::<Info>::new();
/// import(&persy,source.into_iter())?;
/// # Ok(())
/// # }
/// ```
pub fn import<I>(persy: &Persy, importer: I) -> EIRes<()>
where
    I: Iterator<Item = Info>,
{
    import_with_handler(persy, importer, NothingHandler())
}

/// Import the data to a persy storage from a custom format
///
/// # Example
/// ```
/// use persy::{Persy, Config};
/// use persy_expimp::{import_with_handler,Info, Handler, Mapper};
/// # use persy_expimp::EIRes;
/// # fn foo() -> EIRes<()> {
///
/// struct HandlerImpl{ //...
/// };
///
/// impl Handler for  HandlerImpl {
///    fn pre(&self, _persy: &Persy, _mapper: &Mapper) -> EIRes<()> {
///        //....
///        Ok(())
///    }
///    fn post(&self, _persy: &Persy, _mapper: &Mapper) -> EIRes<()> {
///        //....
///        Ok(())
///    }
/// }
/// Persy::create("imported.persy")?;
/// let persy = Persy::open("imported.persy", Config::new())?;
/// // Source informations from a custom source
/// let source = Vec::<Info>::new();
/// import_with_handler(&persy,source.into_iter(), HandlerImpl{})?;
/// # Ok(())
/// # }
/// ```
pub fn import_with_handler<I, H>(persy: &Persy, importer: I, handler: H) -> EIRes<()>
where
    I: Iterator<Item = Info>,
    H: Handler,
{
    let mut tx = persy.begin()?;
    create_map_persy_id_index(&mut tx)?;
    tx.prepare()?.commit()?;
    let mapper = Mapper::new(persy);
    handler.pre(persy, &mapper)?;
    for val in importer {
        let mut tx = persy.begin()?;
        match val {
            Info::Segment(segment) => {
                tx.create_segment(&segment.name)?;
            }
            Info::Record(record) => {
                let inserted_id = tx.insert(&record.segment, &record.content)?;
                remap_persy_id(&mut tx, record.id, inserted_id)?;
            }
            Info::Index(index) => {
                create_index(&mut tx, index)?;
            }
            Info::Entry(entry) => {
                put_entry(&mut tx, entry)?;
            }
        }
        let prep = tx.prepare()?;
        prep.commit()?;
    }
    handler.post(persy, &mapper)?;
    let mut tx = persy.begin()?;
    drop_map_persy_id_index(&mut tx)?;
    tx.prepare()?.commit()?;
    Ok(())
}

pub trait Handler {
    fn pre(&self, persy: &Persy, mapper: &Mapper) -> EIRes<()>;
    fn post(&self, persy: &Persy, mapper: &Mapper) -> EIRes<()>;
}
struct NothingHandler();

impl Handler for NothingHandler {
    fn pre(&self, _persy: &Persy, _mapper: &Mapper) -> EIRes<()> {
        Ok(())
    }
    fn post(&self, _persy: &Persy, _mapper: &Mapper) -> EIRes<()> {
        Ok(())
    }
}

pub struct Mapper {
    persy: Persy,
}
impl Mapper {
    fn new(persy: &Persy) -> Self {
        Self { persy: persy.clone() }
    }
    pub fn map(&self, id: PersyId) -> EIRes<PersyId> {
        map_persy_id_out(&self.persy, id)
    }
}

#[cfg(test)]
mod tests {
    use super::{export, export_from_snapshot, import, Info};
    use persy::{ByteVec, Config, IndexType, Persy, PersyId, Transaction, ValueMode};
    use std::fs;

    fn util<C, V>(name: &str, create: C, verify: V)
    where
        C: FnOnce(&mut Transaction),
        V: FnOnce(&Persy),
    {
        let c_name = format!("target/{}.pei", name);
        let rec_name = format!("target/{}_fill.pei", name);
        Persy::create(c_name.to_string()).expect("creation works fine");
        let p = Persy::open(c_name.to_string(), Config::new()).expect("open fine");
        let mut tx = p.begin().unwrap();
        create(&mut tx);
        let prep = tx.prepare().unwrap();
        prep.commit().unwrap();
        let data: Vec<Info> = export(&p).unwrap().collect();

        Persy::create(rec_name.to_string()).expect("creation works fine");
        let p1 = Persy::open(rec_name.to_string(), Config::new()).expect("open fine");
        import(&p1, data.into_iter()).unwrap();
        verify(&p1);
        fs::remove_file(c_name).unwrap();
        fs::remove_file(rec_name).unwrap();
    }

    #[test]
    fn base_snapshot_export_import() {
        let name = "snapshot";
        let c_name = format!("target/{}.pei", name);
        let rec_name = format!("target/{}_fill.pei", name);
        Persy::create(c_name.to_string()).expect("creation works fine");
        let p = Persy::open(c_name.to_string(), Config::new()).expect("open fine");
        let mut tx = p.begin().unwrap();
        tx.create_segment("test").unwrap();
        tx.insert("test", &"test".to_string().as_bytes()).unwrap();
        tx.create_index::<u8, u8>("test_u8", ValueMode::Replace).unwrap();
        tx.put::<u8, u8>("test_u8", 10, 10).unwrap();
        let prep = tx.prepare().unwrap();
        prep.commit().unwrap();
        let data: Vec<Info> = export_from_snapshot(&p.snapshot().unwrap()).unwrap().collect();

        Persy::create(rec_name.to_string()).expect("creation works fine");
        let p1 = Persy::open(rec_name.to_string(), Config::new()).expect("open fine");
        import(&p1, data.into_iter()).unwrap();

        for (_, content) in p.scan("test").unwrap() {
            assert_eq!("test".to_string().as_bytes().to_vec(), content);
        }
        check_key_value(&p1, "test_u8", 10 as u8, 10 as u8);
        fs::remove_file(c_name).unwrap();
        fs::remove_file(rec_name).unwrap();
    }

    #[test]
    fn segment_metadata_export_import() {
        util(
            "segment",
            |tx| {
                tx.create_segment("test").unwrap();
            },
            |p| {
                assert!(p.exists_segment("test").unwrap());
            },
        );
    }
    #[test]
    fn index_metadata_export_import() {
        util(
            "index",
            |tx| {
                tx.create_index::<u8, u8>("test", ValueMode::Replace).unwrap();
            },
            |p| {
                assert!(p.exists_index("test").unwrap());
            },
        );
    }

    #[test]
    fn segment_data_export_import() {
        util(
            "segment_data",
            |tx| {
                tx.create_segment("test").unwrap();
                tx.insert("test", &"test".to_string().as_bytes()).unwrap();
            },
            |p| {
                for (_, content) in p.scan("test").unwrap() {
                    assert_eq!("test".to_string().as_bytes().to_vec(), content);
                }
            },
        );
    }
    #[test]
    fn index_data_export_import() {
        util(
            "index_data",
            |tx| {
                tx.create_index::<u8, u8>("test", ValueMode::Replace).unwrap();
                tx.put::<u8, u8>("test", 10, 10).unwrap();
            },
            |p| assert_eq!(10, p.get::<u8, u8>("test", &10).unwrap().into_iter().next().unwrap()),
        );
    }

    #[test]
    fn all_data_export_import() {
        util(
            "all_data",
            |tx| {
                tx.create_segment("test").unwrap();
                let persy_id = tx.insert("test", &"test".to_string().as_bytes()).unwrap();
                tx.create_index::<u8, u8>("test_u8", ValueMode::Replace).unwrap();
                tx.put::<u8, u8>("test_u8", 10, 10).unwrap();
                tx.create_index::<u16, u16>("test_u16", ValueMode::Replace).unwrap();
                tx.put::<u16, u16>("test_u16", 10, 10).unwrap();
                tx.create_index::<u32, u32>("test_u32", ValueMode::Replace).unwrap();
                tx.put::<u32, u32>("test_u32", 10, 10).unwrap();
                tx.create_index::<u64, u64>("test_u64", ValueMode::Replace).unwrap();
                tx.put::<u64, u64>("test_u64", 10, 10).unwrap();
                tx.create_index::<u128, u128>("test_u128", ValueMode::Replace).unwrap();
                tx.put::<u128, u128>("test_u128", 10, 10).unwrap();
                tx.create_index::<i8, i8>("test_i8", ValueMode::Replace).unwrap();
                tx.put::<i8, i8>("test_i8", 10, 10).unwrap();
                tx.create_index::<i16, i16>("test_i16", ValueMode::Replace).unwrap();
                tx.put::<i16, i16>("test_i16", 10, 10).unwrap();
                tx.create_index::<i32, i32>("test_i32", ValueMode::Replace).unwrap();
                tx.put::<i32, i32>("test_i32", 10, 10).unwrap();
                tx.create_index::<i64, i64>("test_i64", ValueMode::Replace).unwrap();
                tx.put::<i64, i64>("test_i64", 10, 10).unwrap();
                tx.create_index::<i128, i128>("test_i128", ValueMode::Replace).unwrap();
                tx.put::<i128, i128>("test_i128", 10, 10).unwrap();
                tx.create_index::<f32, f32>("test_f32", ValueMode::Replace).unwrap();
                tx.put::<f32, f32>("test_f32", 10.0, 10.0).unwrap();
                tx.create_index::<f64, f64>("test_f64", ValueMode::Replace).unwrap();
                tx.put::<f64, f64>("test_f64", 10.0, 10.0).unwrap();
                tx.create_index::<String, String>("test_string", ValueMode::Replace)
                    .unwrap();
                tx.put::<String, String>("test_string", "one".to_string(), "two".to_string())
                    .unwrap();
                tx.create_index::<ByteVec, ByteVec>("test_bytevec", ValueMode::Replace)
                    .unwrap();
                let bv = ByteVec::new(vec![20, 10]);
                let bv1 = ByteVec::new(vec![10, 20]);
                tx.put::<ByteVec, ByteVec>("test_bytevec", bv, bv1).unwrap();
                tx.create_index::<PersyId, PersyId>("test_p", ValueMode::Replace)
                    .unwrap();
                tx.put::<PersyId, PersyId>("test_p", persy_id.clone(), persy_id)
                    .unwrap();
            },
            |p| {
                for (_, content) in p.scan("test").unwrap() {
                    assert_eq!("test".to_string().as_bytes().to_vec(), content);
                }
                check_key_value(p, "test_u8", 10 as u8, 10 as u8);
                check_key_value(p, "test_u16", 10 as u16, 10 as u16);
                check_key_value(p, "test_u32", 10 as u32, 10 as u32);
                check_key_value(p, "test_u64", 10 as u64, 10 as u64);
                check_key_value(p, "test_u128", 10 as u128, 10 as u128);
                check_key_value(p, "test_i8", 10 as i8, 10 as i8);
                check_key_value(p, "test_i16", 10 as i16, 10 as i16);
                check_key_value(p, "test_i32", 10 as i32, 10 as i32);
                check_key_value(p, "test_i64", 10 as i64, 10 as i64);
                check_key_value(p, "test_i128", 10 as i128, 10 as i128);
                check_key_value(p, "test_f32", 10 as f32, 10 as f32);
                check_key_value(p, "test_f64", 10 as f64, 10 as f64);
                check_key_value(p, "test_string", "one".to_string(), "two".to_string());
                let bv = ByteVec::new(vec![20, 10]);
                let bv1 = ByteVec::new(vec![10, 20]);
                check_key_value(p, "test_bytevec", bv, bv1);
                assert_eq!(
                    1,
                    p.range::<PersyId, PersyId, _>("test_p", ..)
                        .unwrap()
                        .into_iter()
                        .count()
                );
            },
        );
    }

    pub fn check_key_value<K: IndexType + PartialEq, V: IndexType + PartialEq + std::fmt::Debug>(
        p: &Persy,
        index: &str,
        k: K,
        v: V,
    ) {
        assert_eq!(v, p.get::<K, V>(index, &k).unwrap().into_iter().next().unwrap());
    }

    #[test]
    fn index_id_mapping() {
        util(
            "id_mapping",
            |tx| {
                tx.create_segment("test").unwrap();
                let id = tx.insert("test", "xxx".as_bytes()).unwrap();
                tx.create_index::<u8, PersyId>("test", ValueMode::Replace).unwrap();
                tx.put::<u8, PersyId>("test", 10, id).unwrap();
            },
            |p| {
                let id = p.get::<u8, PersyId>("test", &10).unwrap().next().unwrap();
                assert_eq!(Some("xxx".as_bytes().into()), p.read("test", &id).unwrap());
            },
        );
    }
}