persy_expimp 1.1.0

extern crate data_encoding;
use crate::{
    export, export_from_snapshot, import,
    serde::{index_type_id_from_string, index_type_id_to_string},
    EIRes, Entry, IndexKey, IndexMetadata, IndexValue, Info, Record, SegmentMetadata,
};
use data_encoding::BASE64;
use data_encoding::BASE64URL_NOPAD;
use persy::{ByteVec, IndexId, Persy, PersyId, SegmentId, Snapshot, ValueMode};
use serde_json::{json, Value as JsonValue};
use std::io::{BufRead, BufReader, Read, Write};
use std::ops::Deref;
use std::string::ToString;

fn parse_byte_vec(s: &str) -> Option<ByteVec> {
    BASE64URL_NOPAD.decode(s.as_bytes()).ok().map(|v| ByteVec::new(v))
}
fn byte_vec_to_string(val: ByteVec) -> String {
    BASE64URL_NOPAD.encode(val.deref())
}
fn json_key(key: IndexKey) -> JsonValue {
    match key {
        IndexKey::U8(k) => json!({"type":"u8","value":k}),
        IndexKey::U16(k) => json!({"type":"u16","value":k}),
        IndexKey::U32(k) => json!({"type":"u32","value":k}),
        IndexKey::U64(k) => json!({"type":"u64","value":k}),
        IndexKey::U128(k) => json!({"type":"u128","value":k.to_string()}),
        IndexKey::I8(k) => json!({"type":"i8","value":k}),
        IndexKey::I16(k) => json!({"type":"i16","value":k}),
        IndexKey::I32(k) => json!({"type":"i32","value":k}),
        IndexKey::I64(k) => json!({"type":"i64","value":k}),
        IndexKey::I128(k) => json!({"type":"i128","value":k.to_string()}),
        IndexKey::F32(k) => json!({"type":"f32","value":k}),
        IndexKey::F64(k) => json!({"type":"f64","value":k}),
        IndexKey::String(k) => json!({"type":"String","value":k}),
        IndexKey::PersyId(k) => json!({"type":"PersyId","value":k.to_string()}),
        IndexKey::ByteVec(k) => json!({"type":"ByteVec","value":byte_vec_to_string(k)}),
    }
}

fn json_value(key: IndexValue) -> JsonValue {
    match key {
        IndexValue::U8(k) => json!({"type":"u8","value":k.into_iter().collect::<Vec<u8>>()}),
        IndexValue::U16(k) => json!({"type":"u16","value":k.into_iter().collect::<Vec<u16>>()}),
        IndexValue::U32(k) => json!({"type":"u32","value":k.into_iter().collect::<Vec<u32>>()}),
        IndexValue::U64(k) => json!({"type":"u64","value":k.into_iter().collect::<Vec<u64>>()}),
        IndexValue::U128(k) => {
            json!({"type":"u128","value":k.into_iter().map(|x|x.to_string()).collect::<Vec<String>>()})
        }
        IndexValue::I8(k) => json!({"type":"i8","value":k.into_iter().collect::<Vec<i8>>()}),
        IndexValue::I16(k) => json!({"type":"i16","value":k.into_iter().collect::<Vec<i16>>()}),
        IndexValue::I32(k) => json!({"type":"i32","value":k.into_iter().collect::<Vec<i32>>()}),
        IndexValue::I64(k) => json!({"type":"i64","value":k.into_iter().collect::<Vec<i64>>()}),
        IndexValue::I128(k) => {
            json!({"type":"i128","value":k.into_iter().map(|x|x.to_string()).collect::<Vec<String>>()})
        }
        IndexValue::F32(k) => json!({"type":"f32","value":k.into_iter().collect::<Vec<f32>>()}),
        IndexValue::F64(k) => json!({"type":"f64","value":k.into_iter().collect::<Vec<f64>>()}),
        IndexValue::String(k) => json!({"type":"String","value":k.into_iter().collect::<Vec<String>>()}),
        IndexValue::PersyId(k) => {
            json!({"type":"PersyId","value":k.into_iter().map(|x|x.to_string()).collect::<Vec<String>>()})
        }
        IndexValue::ByteVec(k) => {
            json!({"type":"ByteVec","value":k.into_iter().map(|x|byte_vec_to_string(x)).collect::<Vec<String>>()})
        }
    }
}

///
/// Export the data from a persy storage with using JSON format
///
/// # Example
/// ```
/// use persy::{Persy, Config};
/// use persy_expimp::export_json;
/// use std::vec::Vec;
/// use std::fs::OpenOptions;
///
/// # use persy_expimp::EIRes;
/// # fn foo() -> EIRes<()> {
/// let persy = Persy::open("file.persy", Config::new())?;
/// let mut output = OpenOptions::new().write(true).create(true).open("dest.json")?;
/// export_json(&persy, &mut output)?;
/// # Ok(())
/// # }
/// ```
pub fn export_json(persy: &Persy, write: &mut dyn Write) -> EIRes<()> {
    export_json_impl(export(persy)?, write)
}

///
/// Export the data from a persy storage with using JSON format
///
/// # Example
/// ```
/// use persy::{Persy, Config};
/// use persy_expimp::export_json_from_snapshot;
/// use std::vec::Vec;
/// use std::fs::OpenOptions;
///
/// # use persy_expimp::EIRes;
/// # fn foo() -> EIRes<()> {
/// let persy = Persy::open("file.persy", Config::new())?;
/// let snapshot = persy.snapshot()?;
/// let mut output = OpenOptions::new().write(true).create(true).open("dest.json")?;
/// export_json_from_snapshot(&snapshot, &mut output)?;
/// # Ok(())
/// # }
/// ```
pub fn export_json_from_snapshot(snapshot: &Snapshot, write: &mut dyn Write) -> EIRes<()> {
    export_json_impl(export_from_snapshot(snapshot)?, write)
}

fn export_json_impl(data: Box<impl Iterator<Item = Info>>, write: &mut dyn Write) -> EIRes<()> {
    for val in data {
        let to_write = match val {
            Info::Segment(segment) => json!({
                "type":"segment",
                "version": segment.version,
                "value": {
                    "name":segment.name,
                    "id":segment.id.to_string(),
                }
            }),
            Info::Record(record) => json!({
                "type":"record",
                "version": record.version,
                "value": {
                    "segment":record.segment,
                    "id" : record.id.to_string(),
                    "content":BASE64.encode(&record.content)
                }
            }),
            Info::Index(index) => {
                let value_mode = match index.value_mode {
                    ValueMode::Replace => "replace",
                    ValueMode::Cluster => "cluster",
                    ValueMode::Exclusive => "exclusive",
                };
                json!({
                    "type":"index",
                    "version": index.version,
                    "value": {
                        "name":index.name,
                        "id":index.id.to_string(),
                        "key_type":index_type_id_to_string(index.key_type),
                        "value_type":index_type_id_to_string(index.value_type),
                        "value_mode":value_mode
                    }
                })
            }
            Info::Entry(entry) => json!({
                "type":"entry",
                "version": entry.version,
                "value": {
                    "index":entry.index,
                    "key":json_key(entry.key),
                    "value":json_value(entry.value),
                }
            }),
        };
        writeln!(write, "{}", serde_json::to_string(&to_write)?)?;
    }
    Ok(())
}

fn translate_value_mode(value: &str) -> Option<ValueMode> {
    match value {
        "replace" => Some(ValueMode::Replace),
        "cluster" => Some(ValueMode::Cluster),
        "exclusive" => Some(ValueMode::Exclusive),
        _ => None,
    }
}

fn translate_key(value: &JsonValue) -> Option<IndexKey> {
    let ob = value.as_object();
    if let Some(obj) = ob {
        let t = obj.get("type").into_iter().filter_map(|x| x.as_str()).next();
        let v = obj.get("value");
        match (t, v) {
            (Some("u8"), Some(val)) => val.as_u64().map(|uv| IndexKey::U8(uv as u8)),
            (Some("u16"), Some(val)) => val.as_u64().map(|uv| IndexKey::U16(uv as u16)),
            (Some("u32"), Some(val)) => val.as_u64().map(|uv| IndexKey::U32(uv as u32)),
            (Some("u64"), Some(val)) => val.as_u64().map(|uv| IndexKey::U64(uv as u64)),
            (Some("u128"), Some(val)) => val
                .as_str()
                .map(|s| s.parse::<u128>().ok())
                .iter()
                .flatten()
                .map(|z| *z)
                .next()
                .map(|uv| IndexKey::U128(uv as u128)),
            (Some("i8"), Some(val)) => val.as_i64().map(|uv| IndexKey::I8(uv as i8)),
            (Some("i16"), Some(val)) => val.as_i64().map(|uv| IndexKey::I16(uv as i16)),
            (Some("i32"), Some(val)) => val.as_i64().map(|uv| IndexKey::I32(uv as i32)),
            (Some("i64"), Some(val)) => val.as_i64().map(|uv| IndexKey::I64(uv as i64)),
            (Some("i128"), Some(val)) => val
                .as_str()
                .map(|s| s.parse::<i128>().ok())
                .iter()
                .flatten()
                .map(|z| *z)
                .next()
                .map(|uv| IndexKey::I128(uv as i128)),
            (Some("f32"), Some(val)) => val.as_f64().map(|uv| IndexKey::F32(uv as f32)),
            (Some("f64"), Some(val)) => val.as_f64().map(|uv| IndexKey::F64(uv as f64)),
            (Some("String"), Some(val)) => val.as_str().map(|uv| IndexKey::String(uv.to_string())),
            (Some("PersyId"), Some(val)) => val
                .as_str()
                .map(|uv| uv.parse::<PersyId>().ok().map(IndexKey::PersyId))
                .into_iter()
                .flatten()
                .next(),
            (Some("ByteVec"), Some(val)) => val
                .as_str()
                .map(|uv| parse_byte_vec(uv).map(IndexKey::ByteVec))
                .into_iter()
                .flatten()
                .next(),
            _ => None,
        }
    } else {
        None
    }
}

fn translate_value(value: &JsonValue) -> Option<IndexValue> {
    let ob = value.as_object();
    if let Some(obj) = ob {
        let t = obj.get("type").into_iter().filter_map(|x| x.as_str()).next();
        let v = obj.get("value");
        match (t, v) {
            (Some("u8"), Some(val)) => val
                .as_array()
                .map(|v| {
                    v.into_iter()
                        .filter_map(|x| x.as_u64())
                        .map(|x| x as u8)
                        .collect::<Vec<u8>>()
                })
                .map(|v| IndexValue::U8(v)),
            (Some("u16"), Some(val)) => val
                .as_array()
                .map(|v| {
                    v.into_iter()
                        .filter_map(|x| x.as_u64())
                        .map(|x| x as u16)
                        .collect::<Vec<u16>>()
                })
                .map(|v| IndexValue::U16(v)),
            (Some("u32"), Some(val)) => val
                .as_array()
                .map(|v| {
                    v.into_iter()
                        .filter_map(|x| x.as_u64())
                        .map(|x| x as u32)
                        .collect::<Vec<u32>>()
                })
                .map(|v| IndexValue::U32(v)),
            (Some("u64"), Some(val)) => val
                .as_array()
                .map(|v| {
                    v.into_iter()
                        .filter_map(|x| {
                            if x.is_string() {
                                x.as_str()
                                    .map(|s| s.parse::<u64>().ok())
                                    .iter()
                                    .flatten()
                                    .map(|z| *z)
                                    .next()
                            } else {
                                x.as_u64()
                            }
                        })
                        .collect::<Vec<u64>>()
                })
                .map(|v| IndexValue::U64(v)),
            (Some("u128"), Some(val)) => val
                .as_array()
                .map(|v| {
                    v.into_iter()
                        .filter_map(|x| {
                            x.as_str()
                                .map(|s| s.parse::<u128>().ok())
                                .iter()
                                .flatten()
                                .map(|z| *z)
                                .next()
                        })
                        .collect::<Vec<u128>>()
                })
                .map(|v| IndexValue::U128(v)),
            (Some("i8"), Some(val)) => val
                .as_array()
                .map(|v| {
                    v.into_iter()
                        .filter_map(|x| x.as_i64())
                        .map(|x| x as i8)
                        .collect::<Vec<i8>>()
                })
                .map(|v| IndexValue::I8(v)),
            (Some("i16"), Some(val)) => val
                .as_array()
                .map(|v| {
                    v.into_iter()
                        .filter_map(|x| x.as_i64())
                        .map(|x| x as i16)
                        .collect::<Vec<i16>>()
                })
                .map(|v| IndexValue::I16(v)),
            (Some("i32"), Some(val)) => val
                .as_array()
                .map(|v| {
                    v.into_iter()
                        .filter_map(|x| x.as_i64())
                        .map(|x| x as i32)
                        .collect::<Vec<i32>>()
                })
                .map(|v| IndexValue::I32(v)),
            (Some("i64"), Some(val)) => val
                .as_array()
                .map(|v| {
                    v.into_iter()
                        .filter_map(|x| {
                            if x.is_string() {
                                x.as_str()
                                    .map(|s| s.parse::<i64>().ok())
                                    .iter()
                                    .flatten()
                                    .map(|z| *z)
                                    .next()
                            } else {
                                x.as_i64()
                            }
                        })
                        .collect::<Vec<i64>>()
                })
                .map(|v| IndexValue::I64(v)),
            (Some("i128"), Some(val)) => val
                .as_array()
                .map(|v| {
                    v.into_iter()
                        .filter_map(|x| {
                            x.as_str()
                                .map(|s| s.parse::<i128>().ok())
                                .iter()
                                .flatten()
                                .map(|z| *z)
                                .next()
                        })
                        .collect::<Vec<i128>>()
                })
                .map(|v| IndexValue::I128(v)),
            (Some("f32"), Some(val)) => val
                .as_array()
                .map(|v| {
                    v.into_iter()
                        .filter_map(|x| x.as_f64())
                        .map(|x| x as f32)
                        .collect::<Vec<f32>>()
                })
                .map(|v| IndexValue::F32(v)),
            (Some("f64"), Some(val)) => val
                .as_array()
                .map(|v| v.into_iter().filter_map(|x| x.as_f64()).collect::<Vec<f64>>())
                .map(|v| IndexValue::F64(v)),
            (Some("String"), Some(val)) => val
                .as_array()
                .map(|v| {
                    v.into_iter()
                        .filter_map(|x| x.as_str())
                        .map(|x| x.to_string())
                        .collect::<Vec<String>>()
                })
                .map(|v| IndexValue::String(v)),
            (Some("PersyId"), Some(val)) => val
                .as_array()
                .map(|v| {
                    v.into_iter()
                        .filter_map(|x| x.as_str())
                        .filter_map(|uv| uv.parse::<PersyId>().ok())
                        .collect::<Vec<PersyId>>()
                })
                .map(|v| IndexValue::PersyId(v)),
            (Some("ByteVec"), Some(val)) => val
                .as_array()
                .map(|v| {
                    v.into_iter()
                        .filter_map(|x| x.as_str())
                        .filter_map(|uv| parse_byte_vec(uv))
                        .collect::<Vec<ByteVec>>()
                })
                .map(|v| IndexValue::ByteVec(v)),
            _ => None,
        }
    } else {
        None
    }
}

fn translate_content(value: &JsonValue) -> Option<Vec<u8>> {
    value
        .as_str()
        .into_iter()
        .filter_map(|v| BASE64.decode(v.as_bytes()).ok())
        .next()
}

fn translate_line(line: &str) -> Option<Info> {
    let value: Result<JsonValue, _> = serde_json::from_str(&line);
    if let Some(Some(r)) = value.map(|x| x.as_object().cloned()).ok() {
        let t = r.get("type").map(|x| x.as_str());
        let v = r.get("value").map(|x| x.as_object());
        let vers = r.get("version").map(|x| x.as_u64().map(|v| v as u32));

        match (t, v, vers) {
            (Some(Some("segment")), Some(Some(obj)), Some(Some(version))) => {
                let name = obj.get("name").into_iter().filter_map(|x| x.as_str()).next();
                let id = obj
                    .get("id")
                    .into_iter()
                    .filter_map(|x| x.as_str())
                    .filter_map(|x| x.parse::<SegmentId>().ok())
                    .next();
                match (id, name) {
                    (Some(val_id), Some(val_name)) => Some(Info::Segment(SegmentMetadata {
                        version,
                        name: val_name.to_string(),
                        id: val_id,
                    })),
                    _ => None,
                }
            }
            (Some(Some("record")), Some(Some(obj)), Some(Some(version))) => {
                let segment = obj.get("segment").into_iter().filter_map(|x| x.as_str()).next();
                let id = obj
                    .get("id")
                    .into_iter()
                    .filter_map(|x| x.as_str())
                    .filter_map(|x| x.parse::<PersyId>().ok())
                    .next();
                let content = obj.get("content").into_iter().filter_map(translate_content).next();
                match (segment, id, content) {
                    (Some(val_segment), Some(val_id), Some(val_content)) => Some(Info::Record(Record {
                        version,
                        segment: val_segment.to_string(),
                        id: val_id,
                        content: val_content,
                    })),
                    _ => None,
                }
            }
            (Some(Some("index")), Some(Some(obj)), Some(Some(version))) => {
                let name = obj.get("name").into_iter().filter_map(|x| x.as_str()).next();
                let id = obj
                    .get("id")
                    .into_iter()
                    .filter_map(|x| x.as_str())
                    .filter_map(|x| x.parse::<IndexId>().ok())
                    .next();
                let key_type = obj
                    .get("key_type")
                    .into_iter()
                    .filter_map(|x| x.as_str())
                    .filter_map(|x| index_type_id_from_string(x))
                    .next();
                let value_type = obj
                    .get("value_type")
                    .into_iter()
                    .filter_map(|x| x.as_str())
                    .filter_map(|x| index_type_id_from_string(x))
                    .next();
                let value_mode = obj
                    .get("value_mode")
                    .into_iter()
                    .filter_map(|x| x.as_str())
                    .filter_map(translate_value_mode)
                    .next();
                match (id, name, key_type, value_type, value_mode) {
                    (Some(val_id), Some(val_name), Some(val_key_type), Some(val_value_type), Some(val_value_mode)) => {
                        Some(Info::Index(IndexMetadata {
                            version,
                            name: val_name.to_string(),
                            id: val_id,
                            key_type: val_key_type,
                            value_type: val_value_type,
                            value_mode: val_value_mode,
                        }))
                    }
                    _ => None,
                }
            }
            (Some(Some("entry")), Some(Some(obj)), Some(Some(version))) => {
                let index = obj.get("index").into_iter().filter_map(|x| x.as_str()).next();
                let key = obj.get("key").into_iter().filter_map(translate_key).next();
                let value = obj.get("value").into_iter().filter_map(translate_value).next();
                match (index, key, value) {
                    (Some(val_index), Some(val_key), Some(val_value)) => Some(Info::Entry(Entry {
                        version,
                        index: val_index.to_string(),
                        key: val_key,
                        value: val_value,
                    })),
                    _ => None,
                }
            }
            _ => None,
        }
    } else {
        None
    }
}

/// Import data to a persy storage reading JSON export
///
///
/// ```
/// use persy::{Persy, Config};
/// use persy_expimp::import_json;
/// use std::fs::OpenOptions;
/// # use persy_expimp::EIRes;
///  
/// # fn foo() -> EIRes<()> {
/// Persy::create("imported.persy")?;
/// let persy = Persy::open("imported.persy", Config::new())?;
/// let mut input = OpenOptions::new().read(true).open("file_source.json")?;
/// import_json(&persy,&mut input)?;
/// # Ok(())
/// # }
/// ```
///
pub fn import_json(persy: &Persy, read: &mut dyn Read) -> EIRes<()> {
    let reader = BufReader::new(read);
    let source = reader.lines().filter_map(|l| {
        if let Ok(line) = l {
            let info = translate_line(&line);
            if info.is_none() {
                println!(" Skipping invalid line: {}", line);
            }
            info
        } else {
            None
        }
    });
    import(persy, source)?;

    Ok(())
}
#[cfg(test)]
mod tests {
    use super::{export_json, export_json_from_snapshot, import_json};
    use persy::{ByteVec, Config, IndexType, Persy, PersyId, Transaction, ValueMode};
    use std::fs;
    use std::fs::File;
    use std::io::Cursor;
    use std::io::Read;

    fn util<C, V>(name: &str, create: C, verify: V)
    where
        C: FnOnce(&mut Transaction),
        V: FnOnce(&Persy),
    {
        let c_name = format!("target/{}.pei", name);
        let rec_name = format!("target/{}_fill.pei", name);
        Persy::create(c_name.to_string()).expect("creation works fine");
        let p = Persy::open(c_name.to_string(), Config::new()).expect("open fine");
        let mut tx = p.begin().unwrap();
        create(&mut tx);
        let prep = tx.prepare().unwrap();
        prep.commit().unwrap();
        let mut data = Vec::new();
        export_json(&p, &mut data).unwrap();
        //std::fs::write("fixtures/0.8_1.0_all.json", String::from_utf8(data.clone()).unwrap()).unwrap();
        Persy::create(rec_name.to_string()).expect("creation works fine");
        let p1 = Persy::open(rec_name.to_string(), Config::new()).expect("open fine");
        import_json(&p1, &mut Cursor::new(data)).unwrap();
        verify(&p1);
        fs::remove_file(c_name).unwrap();
        fs::remove_file(rec_name).unwrap();
    }

    #[test]
    fn base_snapshot_export_import_json() {
        let name = "json_snapshot";
        let c_name = format!("target/{}.pei", name);
        let rec_name = format!("target/{}_fill.pei", name);
        Persy::create(c_name.to_string()).expect("creation works fine");
        let p = Persy::open(c_name.to_string(), Config::new()).expect("open fine");
        let mut tx = p.begin().unwrap();
        tx.create_segment("test").unwrap();
        tx.insert("test", &"test".to_string().as_bytes()).unwrap();
        tx.create_index::<u8, u8>("test_u8", ValueMode::Replace).unwrap();
        tx.put::<u8, u8>("test_u8", 10, 10).unwrap();
        let prep = tx.prepare().unwrap();
        prep.commit().unwrap();
        let mut data = Vec::new();
        export_json_from_snapshot(&p.snapshot().unwrap(), &mut data).expect("export works correctly");

        Persy::create(rec_name.to_string()).expect("creation works fine");
        let p1 = Persy::open(rec_name.to_string(), Config::new()).expect("open fine");
        import_json(&p1, &mut Cursor::new(data)).unwrap();

        for (_, content) in p.scan("test").unwrap() {
            assert_eq!("test".to_string().as_bytes().to_vec(), content);
        }
        check_key_value(&p1, "test_u8", 10 as u8, 10 as u8);
        fs::remove_file(c_name).unwrap();
        fs::remove_file(rec_name).unwrap();
    }

    #[test]
    fn all_data_export_import() {
        util(
            "json_all_data",
            |tx| {
                tx.create_segment("test").unwrap();
                let persy_id = tx.insert("test", &"test".to_string().as_bytes()).unwrap();
                tx.create_index::<u8, u8>("test_u8", ValueMode::Replace).unwrap();
                tx.put::<u8, u8>("test_u8", 10, 10).unwrap();
                tx.create_index::<u16, u16>("test_u16", ValueMode::Replace).unwrap();
                tx.put::<u16, u16>("test_u16", 10, 10).unwrap();
                tx.create_index::<u32, u32>("test_u32", ValueMode::Replace).unwrap();
                tx.put::<u32, u32>("test_u32", 10, 10).unwrap();
                tx.create_index::<u64, u64>("test_u64", ValueMode::Replace).unwrap();
                tx.put::<u64, u64>("test_u64", 10, 10).unwrap();
                tx.create_index::<u128, u128>("test_u128", ValueMode::Replace).unwrap();
                tx.put::<u128, u128>("test_u128", 10, 10).unwrap();
                tx.create_index::<i8, i8>("test_i8", ValueMode::Replace).unwrap();
                tx.put::<i8, i8>("test_i8", 10, 10).unwrap();
                tx.create_index::<i16, i16>("test_i16", ValueMode::Replace).unwrap();
                tx.put::<i16, i16>("test_i16", 10, 10).unwrap();
                tx.create_index::<i32, i32>("test_i32", ValueMode::Replace).unwrap();
                tx.put::<i32, i32>("test_i32", 10, 10).unwrap();
                tx.create_index::<i64, i64>("test_i64", ValueMode::Replace).unwrap();
                tx.put::<i64, i64>("test_i64", 10, 10).unwrap();
                tx.create_index::<i128, i128>("test_i128", ValueMode::Replace).unwrap();
                tx.put::<i128, i128>("test_i128", 10, 10).unwrap();
                tx.create_index::<f32, f32>("test_f32", ValueMode::Replace).unwrap();
                tx.put::<f32, f32>("test_f32", 10.0, 10.0).unwrap();
                tx.create_index::<f64, f64>("test_f64", ValueMode::Replace).unwrap();
                tx.put::<f64, f64>("test_f64", 10.0, 10.0).unwrap();
                tx.create_index::<String, String>("test_string", ValueMode::Replace)
                    .unwrap();
                tx.put::<String, String>("test_string", "one".to_string(), "two".to_string())
                    .unwrap();
                tx.create_index::<ByteVec, ByteVec>("test_bytevec", ValueMode::Replace)
                    .unwrap();
                let bv = ByteVec::new(vec![20, 10]);
                let bv1 = ByteVec::new(vec![10, 20]);
                tx.put::<ByteVec, ByteVec>("test_bytevec", bv, bv1).unwrap();
                tx.create_index::<PersyId, PersyId>("test_p", ValueMode::Replace)
                    .unwrap();
                tx.put::<PersyId, PersyId>("test_p", persy_id.clone(), persy_id)
                    .unwrap();
            },
            |p| {
                for (_, content) in p.scan("test").unwrap() {
                    assert_eq!("test".to_string().as_bytes().to_vec(), content);
                }
                check_key_value(p, "test_u8", 10 as u8, 10 as u8);
                check_key_value(p, "test_u16", 10 as u16, 10 as u16);
                check_key_value(p, "test_u32", 10 as u32, 10 as u32);
                check_key_value(p, "test_u64", 10 as u64, 10 as u64);
                check_key_value(p, "test_u128", 10 as u128, 10 as u128);
                check_key_value(p, "test_i8", 10 as i8, 10 as i8);
                check_key_value(p, "test_i16", 10 as i16, 10 as i16);
                check_key_value(p, "test_i32", 10 as i32, 10 as i32);
                check_key_value(p, "test_i64", 10 as i64, 10 as i64);
                check_key_value(p, "test_i128", 10 as i128, 10 as i128);
                check_key_value(p, "test_f32", 10 as f32, 10 as f32);
                check_key_value(p, "test_f64", 10 as f64, 10 as f64);
                check_key_value(p, "test_string", "one".to_string(), "two".to_string());
                let bv = ByteVec::new(vec![20, 10]);
                let bv1 = ByteVec::new(vec![10, 20]);
                check_key_value(p, "test_bytevec", bv, bv1);
                assert_eq!(
                    1,
                    p.range::<PersyId, PersyId, _>("test_p", ..)
                        .unwrap()
                        .into_iter()
                        .count()
                );
            },
        );
    }

    pub fn check_key_value<K: IndexType + PartialEq, V: IndexType + PartialEq + std::fmt::Debug>(
        p: &Persy,
        index: &str,
        k: K,
        v: V,
    ) {
        assert_eq!(v, p.get::<K, V>(index, &k).unwrap().into_iter().next().unwrap());
    }

    #[test]
    pub fn test_import_01_05() {
        Persy::create("import_from_01_05.persy").expect("creation works fine");
        let p = &Persy::open("import_from_01_05.persy", Config::new()).expect("open fine");
        let mut f = File::open("fixtures/0.1_0.5_all.json").expect("open correctly the file");
        let mut data = String::new();

        f.read_to_string(&mut data).expect("read all fine");
        import_json(p, &mut Cursor::new(data)).unwrap();
        for (_, content) in p.scan("test").unwrap() {
            assert_eq!("test".to_string().as_bytes().to_vec(), content);
        }
        check_key_value(p, "test_u8", 10 as u8, 10 as u8);
        check_key_value(p, "test_u16", 10 as u16, 10 as u16);
        check_key_value(p, "test_u32", 10 as u32, 10 as u32);
        check_key_value(p, "test_u64", 10 as u64, 10 as u64);
        check_key_value(p, "test_i8", 10 as i8, 10 as i8);
        check_key_value(p, "test_i16", 10 as i16, 10 as i16);
        check_key_value(p, "test_i32", 10 as i32, 10 as i32);
        check_key_value(p, "test_i64", 10 as i64, 10 as i64);
        check_key_value(p, "test_string", "one".to_string(), "two".to_string());
        check_key_value(p, "test_string", "one".to_string(), "two".to_string());
        assert_eq!(
            1,
            p.range::<PersyId, PersyId, _>("test_p", ..)
                .unwrap()
                .into_iter()
                .count()
        );
        fs::remove_file("import_from_01_05.persy").unwrap();
    }

    #[test]
    pub fn test_import_02_06() {
        Persy::create("import_from_02_06.persy").expect("creation works fine");
        let p = &Persy::open("import_from_02_06.persy", Config::new()).expect("open fine");
        let mut f = File::open("fixtures/0.2_0.6_all.json").expect("open correctly the file");
        let mut data = String::new();

        f.read_to_string(&mut data).expect("read all fine");
        import_json(p, &mut Cursor::new(data)).unwrap();
        for (_, content) in p.scan("test").unwrap() {
            assert_eq!("test".to_string().as_bytes().to_vec(), content);
        }
        check_key_value(p, "test_u8", 10 as u8, 10 as u8);
        check_key_value(p, "test_u16", 10 as u16, 10 as u16);
        check_key_value(p, "test_u32", 10 as u32, 10 as u32);
        check_key_value(p, "test_u64", 10 as u64, 10 as u64);
        check_key_value(p, "test_u128", 10 as u128, 10 as u128);
        check_key_value(p, "test_i8", 10 as i8, 10 as i8);
        check_key_value(p, "test_i16", 10 as i16, 10 as i16);
        check_key_value(p, "test_i32", 10 as i32, 10 as i32);
        check_key_value(p, "test_i64", 10 as i64, 10 as i64);
        check_key_value(p, "test_i128", 10 as i128, 10 as i128);
        check_key_value(p, "test_f32", 10 as f32, 10 as f32);
        check_key_value(p, "test_f64", 10 as f64, 10 as f64);
        check_key_value(p, "test_string", "one".to_string(), "two".to_string());
        assert_eq!(
            1,
            p.range::<PersyId, PersyId, _>("test_p", ..)
                .unwrap()
                .into_iter()
                .count()
        );
        fs::remove_file("import_from_02_06.persy").unwrap();
    }

    #[test]
    pub fn test_import_03_07() {
        Persy::create("import_from_03_07.persy").expect("creation works fine");
        let p = &Persy::open("import_from_03_07.persy", Config::new()).expect("open fine");
        let mut f = File::open("fixtures/0.3_0.7_all.json").expect("open correctly the file");
        let mut data = String::new();

        f.read_to_string(&mut data).expect("read all fine");
        import_json(p, &mut Cursor::new(data)).unwrap();
        for (_, content) in p.scan("test").unwrap() {
            assert_eq!("test".to_string().as_bytes().to_vec(), content);
        }
        check_key_value(p, "test_u8", 10 as u8, 10 as u8);
        check_key_value(p, "test_u16", 10 as u16, 10 as u16);
        check_key_value(p, "test_u32", 10 as u32, 10 as u32);
        check_key_value(p, "test_u64", 10 as u64, 10 as u64);
        check_key_value(p, "test_u128", 10 as u128, 10 as u128);
        check_key_value(p, "test_i8", 10 as i8, 10 as i8);
        check_key_value(p, "test_i16", 10 as i16, 10 as i16);
        check_key_value(p, "test_i32", 10 as i32, 10 as i32);
        check_key_value(p, "test_i64", 10 as i64, 10 as i64);
        check_key_value(p, "test_i128", 10 as i128, 10 as i128);
        check_key_value(p, "test_f32", 10 as f32, 10 as f32);
        check_key_value(p, "test_f64", 10 as f64, 10 as f64);
        check_key_value(p, "test_string", "one".to_string(), "two".to_string());
        let bv = ByteVec::new(vec![20, 10]);
        let bv1 = ByteVec::new(vec![10, 20]);
        check_key_value(p, "test_bytevec", bv, bv1);
        assert_eq!(
            1,
            p.range::<PersyId, PersyId, _>("test_p", ..)
                .unwrap()
                .into_iter()
                .count()
        );
        fs::remove_file("import_from_03_07.persy").unwrap();
    }

    #[test]
    pub fn test_import_04_08() {
        Persy::create("import_from_04_08.persy").expect("creation works fine");
        let p = &Persy::open("import_from_04_08.persy", Config::new()).expect("open fine");
        let mut f = File::open("fixtures/0.4_0.8_all.json").expect("open correctly the file");
        let mut data = String::new();

        f.read_to_string(&mut data).expect("read all fine");
        import_json(p, &mut Cursor::new(data)).unwrap();
        for (_, content) in p.scan("test").unwrap() {
            assert_eq!("test".to_string().as_bytes().to_vec(), content);
        }
        check_key_value(p, "test_u8", 10 as u8, 10 as u8);
        check_key_value(p, "test_u16", 10 as u16, 10 as u16);
        check_key_value(p, "test_u32", 10 as u32, 10 as u32);
        check_key_value(p, "test_u64", 10 as u64, 10 as u64);
        check_key_value(p, "test_u128", 10 as u128, 10 as u128);
        check_key_value(p, "test_i8", 10 as i8, 10 as i8);
        check_key_value(p, "test_i16", 10 as i16, 10 as i16);
        check_key_value(p, "test_i32", 10 as i32, 10 as i32);
        check_key_value(p, "test_i64", 10 as i64, 10 as i64);
        check_key_value(p, "test_i128", 10 as i128, 10 as i128);
        check_key_value(p, "test_f32", 10 as f32, 10 as f32);
        check_key_value(p, "test_f64", 10 as f64, 10 as f64);
        check_key_value(p, "test_string", "one".to_string(), "two".to_string());
        let bv = ByteVec::new(vec![20, 10]);
        let bv1 = ByteVec::new(vec![10, 20]);
        check_key_value(p, "test_bytevec", bv, bv1);
        assert_eq!(
            1,
            p.range::<PersyId, PersyId, _>("test_p", ..)
                .unwrap()
                .into_iter()
                .count()
        );
        fs::remove_file("import_from_04_08.persy").unwrap();
    }

    #[test]
    pub fn test_import_05_09() {
        Persy::create("import_from_05_09.persy").expect("creation works fine");
        let p = &Persy::open("import_from_05_09.persy", Config::new()).expect("open fine");
        let mut f = File::open("fixtures/0.5_0.9_all.json").expect("open correctly the file");
        let mut data = String::new();

        f.read_to_string(&mut data).expect("read all fine");
        import_json(p, &mut Cursor::new(data)).unwrap();
        for (_, content) in p.scan("test").unwrap() {
            assert_eq!("test".to_string().as_bytes().to_vec(), content);
        }
        check_key_value(p, "test_u8", 10 as u8, 10 as u8);
        check_key_value(p, "test_u16", 10 as u16, 10 as u16);
        check_key_value(p, "test_u32", 10 as u32, 10 as u32);
        check_key_value(p, "test_u64", 10 as u64, 10 as u64);
        check_key_value(p, "test_u128", 10 as u128, 10 as u128);
        check_key_value(p, "test_i8", 10 as i8, 10 as i8);
        check_key_value(p, "test_i16", 10 as i16, 10 as i16);
        check_key_value(p, "test_i32", 10 as i32, 10 as i32);
        check_key_value(p, "test_i64", 10 as i64, 10 as i64);
        check_key_value(p, "test_i128", 10 as i128, 10 as i128);
        check_key_value(p, "test_f32", 10 as f32, 10 as f32);
        check_key_value(p, "test_f64", 10 as f64, 10 as f64);
        check_key_value(p, "test_string", "one".to_string(), "two".to_string());
        let bv = ByteVec::new(vec![20, 10]);
        let bv1 = ByteVec::new(vec![10, 20]);
        check_key_value(p, "test_bytevec", bv, bv1);
        assert_eq!(
            1,
            p.range::<PersyId, PersyId, _>("test_p", ..)
                .unwrap()
                .into_iter()
                .count()
        );
        fs::remove_file("import_from_05_09.persy").unwrap();
    }

    #[test]
    pub fn test_import_06_10() {
        Persy::create("import_from_06_10.persy").expect("creation works fine");
        let p = &Persy::open("import_from_06_10.persy", Config::new()).expect("open fine");
        let mut f = File::open("fixtures/0.6_0.10_all.json").expect("open correctly the file");
        let mut data = String::new();

        f.read_to_string(&mut data).expect("read all fine");
        import_json(p, &mut Cursor::new(data)).unwrap();
        for (_, content) in p.scan("test").unwrap() {
            assert_eq!("test".to_string().as_bytes().to_vec(), content);
        }
        check_key_value(p, "test_u8", 10 as u8, 10 as u8);
        check_key_value(p, "test_u16", 10 as u16, 10 as u16);
        check_key_value(p, "test_u32", 10 as u32, 10 as u32);
        check_key_value(p, "test_u64", 10 as u64, 10 as u64);
        check_key_value(p, "test_u128", 10 as u128, 10 as u128);
        check_key_value(p, "test_i8", 10 as i8, 10 as i8);
        check_key_value(p, "test_i16", 10 as i16, 10 as i16);
        check_key_value(p, "test_i32", 10 as i32, 10 as i32);
        check_key_value(p, "test_i64", 10 as i64, 10 as i64);
        check_key_value(p, "test_i128", 10 as i128, 10 as i128);
        check_key_value(p, "test_f32", 10 as f32, 10 as f32);
        check_key_value(p, "test_f64", 10 as f64, 10 as f64);
        check_key_value(p, "test_string", "one".to_string(), "two".to_string());
        let bv = ByteVec::new(vec![20, 10]);
        let bv1 = ByteVec::new(vec![10, 20]);
        check_key_value(p, "test_bytevec", bv, bv1);
        assert_eq!(
            1,
            p.range::<PersyId, PersyId, _>("test_p", ..)
                .unwrap()
                .into_iter()
                .count()
        );
        fs::remove_file("import_from_06_10.persy").unwrap();
    }

    #[test]
    pub fn test_import_07_11() {
        Persy::create("import_from_07_11.persy").expect("creation works fine");
        let p = &Persy::open("import_from_07_11.persy", Config::new()).expect("open fine");
        let mut f = File::open("fixtures/0.7_0.11_all.json").expect("open correctly the file");
        let mut data = String::new();

        f.read_to_string(&mut data).expect("read all fine");
        import_json(p, &mut Cursor::new(data)).unwrap();
        for (_, content) in p.scan("test").unwrap() {
            assert_eq!("test".to_string().as_bytes().to_vec(), content);
        }
        check_key_value(p, "test_u8", 10 as u8, 10 as u8);
        check_key_value(p, "test_u16", 10 as u16, 10 as u16);
        check_key_value(p, "test_u32", 10 as u32, 10 as u32);
        check_key_value(p, "test_u64", 10 as u64, 10 as u64);
        check_key_value(p, "test_u128", 10 as u128, 10 as u128);
        check_key_value(p, "test_i8", 10 as i8, 10 as i8);
        check_key_value(p, "test_i16", 10 as i16, 10 as i16);
        check_key_value(p, "test_i32", 10 as i32, 10 as i32);
        check_key_value(p, "test_i64", 10 as i64, 10 as i64);
        check_key_value(p, "test_i128", 10 as i128, 10 as i128);
        check_key_value(p, "test_f32", 10 as f32, 10 as f32);
        check_key_value(p, "test_f64", 10 as f64, 10 as f64);
        check_key_value(p, "test_string", "one".to_string(), "two".to_string());
        let bv = ByteVec::new(vec![20, 10]);
        let bv1 = ByteVec::new(vec![10, 20]);
        check_key_value(p, "test_bytevec", bv, bv1);
        assert_eq!(
            1,
            p.range::<PersyId, PersyId, _>("test_p", ..)
                .unwrap()
                .into_iter()
                .count()
        );
        fs::remove_file("import_from_07_11.persy").unwrap();
    }

    #[test]
    pub fn test_import_08_100() {
        Persy::create("import_from_08_100.persy").expect("creation works fine");
        let p = &Persy::open("import_from_08_100.persy", Config::new()).expect("open fine");
        let mut f = File::open("fixtures/0.8_1.0_all.json").expect("open correctly the file");
        let mut data = String::new();

        f.read_to_string(&mut data).expect("read all fine");
        import_json(p, &mut Cursor::new(data)).unwrap();
        for (_, content) in p.scan("test").unwrap() {
            assert_eq!("test".to_string().as_bytes().to_vec(), content);
        }
        check_key_value(p, "test_u8", 10 as u8, 10 as u8);
        check_key_value(p, "test_u16", 10 as u16, 10 as u16);
        check_key_value(p, "test_u32", 10 as u32, 10 as u32);
        check_key_value(p, "test_u64", 10 as u64, 10 as u64);
        check_key_value(p, "test_u128", 10 as u128, 10 as u128);
        check_key_value(p, "test_i8", 10 as i8, 10 as i8);
        check_key_value(p, "test_i16", 10 as i16, 10 as i16);
        check_key_value(p, "test_i32", 10 as i32, 10 as i32);
        check_key_value(p, "test_i64", 10 as i64, 10 as i64);
        check_key_value(p, "test_i128", 10 as i128, 10 as i128);
        check_key_value(p, "test_f32", 10 as f32, 10 as f32);
        check_key_value(p, "test_f64", 10 as f64, 10 as f64);
        check_key_value(p, "test_string", "one".to_string(), "two".to_string());
        let bv = ByteVec::new(vec![20, 10]);
        let bv1 = ByteVec::new(vec![10, 20]);
        check_key_value(p, "test_bytevec", bv, bv1);
        assert_eq!(
            1,
            p.range::<PersyId, PersyId, _>("test_p", ..)
                .unwrap()
                .into_iter()
                .count()
        );
        fs::remove_file("import_from_08_100.persy").unwrap();
    }
}