tablestg 0.4.0

pub use atom_file::Data;
use std::hash::Hash;
use std::sync::Arc;

/// Value for Hash Map lookup.
pub trait SmallFixed {
    fn size() -> usize;
    fn load(bytes: &[u8]) -> Self;
    fn save(&self, bytes: &mut [u8]);
}

/// Key for Hash Map lookup.
pub trait Key<T>: Hash
where
    T: SmallFixed,
{
    /// Check that hash lookup has found correct value.
    fn ok(&self, v: &T, ps: &mut PageSet) -> bool;
}

/// [PageSet] - keeps track of changed pages that need saving.
pub mod pageset;
pub use pageset::*;

/// [HashMap] - maps keys to values using 64 bit hash.
pub mod hashmap;
pub use hashmap::*;

/// [Table] -- fixed size record storage. Divided into pages, each page stores up to 64 records.
pub mod table;
pub use table::*;

/// [VarVal] -- storage of variable length values.
pub mod varval;
pub use varval::*;

// Private modules.

/// [PageTree] - list of pages implemented as tree.
mod pagetree;
use pagetree::*;

/// [TreeVec] -- a variable length list of bytes. Used to implement [Table].
mod treevec;
use treevec::*;

/// Hash Map Bucket - maps keys to values using 64 bit hash.
mod bucket;

const PAGE_SIZE: u64 = 4612;
// pub const PAGE_SIZE: u64 =  4612;

#[test]
fn test_main() {
    use page_store::*;

    let limits = Limits::default();
    /*
    let _dt = DataType::String{};
    let _dt = DataType::Binary{};
    let _dt = DataType::Int(4);
    let _dt = DataType::Array( 5, Box::new(DataType::String) );
    let _dt = DataType::Map( Box::new(DataType::String), Box::new(DataType::String) );
    let _dt = DataType::Struct( Vec::new() );
    */

    // Construct BlockPageStg.
    let file = atom_file::MultiFileStorage::new("test.db");
    let upd = atom_file::FastFileStorage::new("test.upd");
    let af = atom_file::AtomicFile::new_with_limits(file, upd, &limits.af_lim);
    let ps = BlockPageStg::new(af, &limits);
    let is_new = ps.is_new();

    let spd = SharedPagedData::new_from_ps(ps);

    println!("max page size={}", spd.psi.max_size_page());

    let mut ps = PageSet::new(spd.new_writer());

    if false {
        let (root, len) = if is_new { (ps.new_page(), 0) } else { (3, 410) };
        test_tv(root, len, &mut ps);
    }

    if false {
        println!("Calling test_hash");
        hashmap::test_hash(&mut ps);
    }

    if false {
        test_table(&mut ps);
    }

    if false {
        test_varval(&mut ps);
    }

    test_cust(&mut ps);

    // ps.save();

    spd.shutdown();
}

#[cfg(test)]
fn tos(s: &[u8]) -> &str {
    str::from_utf8(s).unwrap()
}

/*
   For a table with very fragmented ids, you can make a new table with new compact ids, and map the old ids to the new ones using a PagedHashMap.

   Access is slightly slower, due to the extra Hash lookup.
*/

#[derive(serde::Serialize, serde::Deserialize, Clone, Hash, PartialEq, Eq, Debug)]
pub enum DataType {
    /// e.g. date
    Named(u64),

    /// e.g. `struct{ name: string, email: string, created: date }`
    Struct(Vec<(String, DataType)>),

    /// e.g. `( string, string, int )`
    Tuple(Vec<DataType>),

    /// e.g. `enum{ leaf: int, node: [int] }`
    Enum(Vec<(String, DataType)>),

    /// e.g. `string`
    String(u8),

    /// e.g. `binary(50)` - 50 is the number of bytes stored inline.
    Binary(u8),

    /// e.g. `int`
    Int(u8),

    /// e.g. `[string; 5]`
    Array(usize, Box<DataType>),

    /// e.g. `[string]`
    List(Box<DataType>),

    /// e.g. `[string->int]`
    Map(Box<DataType>, Box<DataType>),
}

impl DataType {
    pub fn to_bytes(&self) -> Vec<u8> {
        postcard::to_stdvec(self).unwrap()
    }
    pub fn from_bytes(b: &[u8]) -> Self {
        postcard::from_bytes(b).unwrap()
    }
}

use pstd::localalloc::Local;
pub type LString = pstd::StringA<Local>;
pub type LVec<T> = pstd::VecA<T, Local>;

#[derive(serde::Serialize, serde::Deserialize, Clone, Hash, PartialEq, Eq, Debug)]
pub enum Value {
    String(LString),
    Int(i64),
    Binary(LVec<u8>),
    List(LVec<Value>),
    VarVal(u64, u64),
    IList(LVec<i64>),
}

impl Value {
    pub fn to_bytes(&self) -> Vec<u8> {
        postcard::to_stdvec(self).unwrap()
    }
    pub fn from_bytes(b: &[u8]) -> Self {
        postcard::from_bytes(b).unwrap()
    }

    pub fn list(&mut self) -> &mut LVec<Value> {
        match self {
            Value::List(list) => list,
            _ => panic!(),
        }
    }

    pub fn ilist(&mut self) -> &mut LVec<i64> {
        match self {
            Value::IList(list) => list,
            _ => panic!(),
        }
    }

    pub fn string(&self) -> &LString {
        match self {
            Value::String(s) => s,
            _ => panic!(),
        }
    }

    pub fn int(&self) -> i64 {
        match self {
            Value::Int(x) => *x,
            _ => panic!(),
        }
    }
}

#[test]
fn test_value() {
    /*
        use pstd::veca;
        let v1 = Value::List(veca![
            Value::String(LString::from("George Barwood")),
            Value::String(LString::from("george.barwood@gmail.com")),
            Value::String(LString::from("33 Sandpipe Close, GL2 4LZ")),
            Value::Int(-47),
            Value::Int(99),
            Value::VarVal(100, 20),
            Value::IList(veca![1, 2, 3, 4]),
        ]);
        let bytes = v1.to_bytes();
        println!("len={} bytes={:?}", bytes.len(), bytes);

        let v2 = Value::from_bytes(&bytes);
        println!("v2={:?}", v2);
    */
}

#[test]
fn test_datatype() {
    let t1 = DataType::String(20);
    let t2 = DataType::Int(3);
    let t3 = DataType::Struct(vec![
        ("name".to_string(), t1.clone()),
        ("email".to_string(), t1.clone()),
    ]);
    let t4 = DataType::Tuple(vec![t1, t2, t3]);

    let bytes = t4.to_bytes();
    // println!("bytes={:?}", bytes);

    let t5 = DataType::from_bytes(&bytes);
    // println!("t5={:?}", t5);
    assert_eq!(t5, t4);
}

/* System tables.

   VarVal tables with diffent chunk sizes.
   Schema table
     Name : string
   Table table
     Schema, Name, DataType (varval), root/len, next_id

   DataType table
     Schema, Name, DataType (varval)

   Function table
     Schema, Name, Definition (varval)
*/

/* Next: make test that stores customer records, Name, Email, Created in a table.
   Then scans them.
*/

#[derive(Debug, PartialEq, Copy, Clone)]
pub struct IdAndVVAddr {
    rid: i64,
    id: u64,
    len: u64,
}

impl SmallFixed for IdAndVVAddr {
    fn size() -> usize {
        3 * 8
    }
    fn load(bytes: &[u8]) -> Self {
        let rid = i64::from_le_bytes(bytes[0..8].try_into().unwrap());
        let id = u64::from_le_bytes(bytes[8..16].try_into().unwrap());
        let len = u64::from_le_bytes(bytes[16..24].try_into().unwrap());
        Self { rid, id, len }
    }
    fn save(&self, bytes: &mut [u8]) {
        bytes[0..8].copy_from_slice(&self.rid.to_le_bytes());
        bytes[8..16].copy_from_slice(&self.id.to_le_bytes());
        bytes[16..24].copy_from_slice(&self.len.to_le_bytes());
    }
}

#[derive(Debug, PartialEq, Copy, Clone)]
pub struct VarValAddr {
    id: u64,
    len: u64,
}

impl SmallFixed for VarValAddr {
    fn size() -> usize {
        2 * 8
    }
    fn load(bytes: &[u8]) -> Self {
        let id = u64::from_le_bytes(bytes[0..8].try_into().unwrap());
        let len = u64::from_le_bytes(bytes[8..16].try_into().unwrap());
        Self { id, len }
    }
    fn save(&self, bytes: &mut [u8]) {
        bytes[0..8].copy_from_slice(&self.id.to_le_bytes());
        bytes[8..16].copy_from_slice(&self.len.to_le_bytes());
    }
}

#[cfg(test)]
impl Key<IdAndVVAddr> for i64 {
    fn ok(&self, v: &IdAndVVAddr, _ps: &mut PageSet) -> bool {
        *self == v.rid
    }
}

#[cfg(test)]
#[derive(Hash)]
struct CustEmailKey {
    email: String,
}

#[cfg(test)]
impl Key<VarValAddr> for CustEmailKey {
    fn ok(&self, v: &VarValAddr, ps: &mut PageSet) -> bool {
        // First get the list of ids.

        let vv = VarVal::new(ps.vvs);
        let mut buf = vec![0; v.len as usize];
        vv.get(v.id, v.len as usize, &mut buf, ps);
        ps.vvs = vv.save();
        let mut list = Value::from_bytes(&buf);
        let list = list.ilist();

        // Now check the email of first element in the list.
        let key = list[0];

        // Use the HashMap to retrieve the VarVal address.
        let hms = ps.cust_hashmap;
        let mut hm = HashMap::restore(hms, ps);
        let x = hm.get(&key).unwrap();

        // Use the VarVal address to retrieve the customer data.
        let mut buf = vec![0; x.len as usize];
        vv.get(x.id, x.len as usize, &mut buf, ps);
        let mut list = Value::from_bytes(&buf);

        // Check the email address in the customer data matches self.email
        let list = list.list();
        let email = &list[2];
        let email = email.string();

        *self.email == **email
    }
}

#[cfg(test)]
fn init(ps: &mut PageSet) {
    let hm = HashMap::<IdAndVVAddr>::new(ps, 1);
    ps.cust_hashmap = hm.save();

    let hm = HashMap::<VarValAddr>::new(ps, 1);
    ps.cust_by_email = hm.save();
}

#[cfg(test)]
fn make_cust(ps: &mut PageSet, rid: i64, name: &str, email: &str, postal: &str) {
    use pstd::veca;

    // Set up a customer record in memory.
    let v1 = Value::List(veca![
        Value::Int(rid), // Doesn't need to be stored here.
        Value::String(LString::from(name)),
        Value::String(LString::from(email)),
        Value::String(LString::from(postal)),
    ]);

    // Turn it into bytes.
    let bytes = v1.to_bytes();

    // Store the customer record ( which has variable length ).
    let mut vv = VarVal::new(ps.vvs);
    let id = vv.store(&bytes, ps);
    ps.vvs = vv.save();

    {
        // Insert into cust_hashmap.
        let mut hm = HashMap::restore(ps.cust_hashmap, ps);

        let addr = IdAndVVAddr {
            rid,
            id,
            len: bytes.len() as u64,
        };

        let key = rid;

        hm.insert(&key, addr);

        if hm.root_changed() {
            ps.cust_hashmap = hm.save();
        }
    }

    let emkey = CustEmailKey {
        email: email.to_string(),
    };

    // Check if list already exists.
    let x = {
        let mut hm = HashMap::restore(ps.cust_by_email, ps);
        let x = hm.remove(&emkey);
        if hm.root_changed() {
            ps.cust_by_email = hm.save();
        }
        x
    };

    let val = if x == None {
        // Create  list with single element and store it as VarVal.
        let list = Value::IList(veca![rid]);
        let bytes = list.to_bytes();

        let mut vv = VarVal::new(ps.vvs);
        let id = vv.store(&bytes, ps);
        ps.vvs = vv.save();

        VarValAddr {
            id,
            len: bytes.len() as u64,
        }
    } else {
        let x = x.unwrap();

        // Get existing list of customer ids.
        let mut buf = vec![0; x.len as usize];
        vv.get(x.id, x.len as usize, &mut buf, ps);

        let mut list = Value::from_bytes(&buf);
        println!("existing list of cust ids={:?}", list);

        // Append to list.
        let list1 = list.ilist();
        list1.push(rid);
        let bytes = list.to_bytes();

        // Update varval.
        let id = vv.update(x.id, x.len as usize, &bytes, ps);
        ps.vvs = vv.save();

        VarValAddr {
            id,
            len: bytes.len() as u64,
        }
    };

    // Associate val with emkey.
    let mut hm = HashMap::restore(ps.cust_by_email, ps);
    hm.insert(&emkey, val);
    if hm.root_changed() {
        ps.cust_by_email = hm.save();
    }
}

#[cfg(test)]
fn list_cust(ps: &mut PageSet, email: &str) {
    // Get set of customer ids with specified email address.

    let key = CustEmailKey {
        email: email.to_string(),
    };

    let mut hm = HashMap::restore(ps.cust_by_email, ps);
    let x = hm.get(&key);
    if x.is_some() {
        let x = x.unwrap();

        // Get the list of customer ids.
        let vv = VarVal::new(ps.vvs);
        let mut buf = vec![0; x.len as usize];
        vv.get(x.id, x.len as usize, &mut buf, ps);

        let list = Value::from_bytes(&buf);
        println!("list of cust ids for {} = {:?}", email, list);
    } else {
        println!("Email Not Found");
    }
}

#[cfg(test)]
fn test_cust(ps: &mut PageSet) {
    init(ps);
    make_cust(
        ps,
        99,
        "George Barwood",
        "george.barwood@gmail.com",
        "33 Sandpipe Close, GL2 4LZ",
    );
    // Make a duplicate
    make_cust(
        ps,
        100,
        "George Barwood",
        "george.barwood@gmail.com",
        "33 Sandpipe Close, GL2 4LZ",
    );
    // Another duplicate
    make_cust(
        ps,
        101,
        "George Barwood",
        "george.barwood@gmail.com",
        "33 Sandpipe Close, GL2 4LZ",
    );

    list_cust(ps, "george.barwood@gmail.com");

    make_cust(
        ps,
        102,
        "Marilyn Barwood",
        "maz.barwood@gmail.com",
        "33 Sandpipe Close, GL2 4LZ",
    );

    list_cust(ps, "maz.barwood@gmail.com");
}