use crate::keri::db::dbing::LMDBer;
use crate::keri::db::errors::DBError;
use heed::types::*;
use heed::Database;
use serde::{Deserialize, Serialize};
use std::fmt::Debug;
use std::marker::PhantomData;
use std::sync::Arc;
use thiserror::Error;
#[allow(dead_code)]
#[derive(Debug, Error)]
pub enum KomerError {
#[error("Database error: {0}")]
Database(#[from] heed::Error),
#[error("LMDB error: {0}")]
DBError(#[from] DBError),
#[error("Serialization error: {0}")]
Serialization(String),
#[error("Deserialization error: {0}")]
Deserialization(String),
#[error("Invalid schema: expected {expected} got {got}")]
InvalidSchema { expected: String, got: String },
#[error("Empty keys")]
EmptyKeys,
}
#[allow(dead_code)]
#[derive(Debug, Clone, Copy)]
pub enum SerialKind {
Json,
MsgPack,
Cbor,
}
pub struct KomerBase<'db, T>
where
T: Serialize + for<'de> Deserialize<'de> + Debug,
{
db: Arc<&'db LMDBer>,
pub sdb: Database<Bytes, Bytes>,
pub kind: SerialKind,
pub sep: String,
phantom: PhantomData<&'db T>,
}
impl<'db, T> KomerBase<'db, T>
where
T: Serialize + for<'de> Deserialize<'de> + Debug,
{
pub fn new(
db: Arc<&'db LMDBer>,
subkey: &str,
kind: SerialKind,
dupsort: bool,
sep: Option<&str>,
) -> Result<Self, KomerError> {
let sdb = db.create_database(Some(subkey), Some(dupsort))?;
Ok(Self {
db,
sdb,
kind,
sep: sep.unwrap_or(".").to_string(),
phantom: PhantomData,
})
}
pub fn to_key<K: AsRef<[u8]>>(&self, keys: &[K], topive: bool) -> Vec<u8> {
let mut result = Vec::new();
if keys.len() == 1 {
let key = keys[0].as_ref();
if !topive {
return key.to_vec();
}
result.extend_from_slice(key);
result.push(self.sep.as_bytes()[0]);
return result;
}
for (i, key) in keys.iter().enumerate() {
if i > 0 {
result.push(self.sep.as_bytes()[0]);
}
result.extend_from_slice(key.as_ref());
}
if topive && (!result.is_empty() && result[result.len() - 1] != self.sep.as_bytes()[0]) {
result.push(self.sep.as_bytes()[0]);
}
result
}
pub fn to_keys(&self, key: &[u8]) -> Vec<String> {
let key_str = String::from_utf8_lossy(key);
key_str.split(&self.sep).map(String::from).collect()
}
pub fn serialize(&self, val: &T) -> Result<Vec<u8>, KomerError> {
match self.kind {
SerialKind::Json => self.serialize_json(val),
SerialKind::MsgPack => self.serialize_msgpack(val),
SerialKind::Cbor => self.serialize_cbor(val),
}
}
pub fn deserialize(&self, val: &[u8]) -> Result<T, KomerError> {
match self.kind {
SerialKind::Json => self.deserialize_json(val),
SerialKind::MsgPack => self.deserialize_msgpack(val),
SerialKind::Cbor => self.deserialize_cbor(val),
}
}
fn serialize_json(&self, val: &T) -> Result<Vec<u8>, KomerError> {
serde_json::to_vec(val)
.map_err(|e| KomerError::Serialization(format!("JSON serialization error: {}", e)))
}
fn deserialize_json(&self, val: &[u8]) -> Result<T, KomerError> {
serde_json::from_slice(val)
.map_err(|e| KomerError::Deserialization(format!("JSON deserialization error: {}", e)))
}
fn serialize_msgpack(&self, val: &T) -> Result<Vec<u8>, KomerError> {
rmp_serde::to_vec(val)
.map_err(|e| KomerError::Serialization(format!("MsgPack serialization error: {}", e)))
}
fn deserialize_msgpack(&self, val: &[u8]) -> Result<T, KomerError> {
rmp_serde::from_slice(val).map_err(|e| {
KomerError::Deserialization(format!("MsgPack deserialization error: {}", e))
})
}
fn serialize_cbor(&self, val: &T) -> Result<Vec<u8>, KomerError> {
serde_cbor::to_vec(val)
.map_err(|e| KomerError::Serialization(format!("CBOR serialization error: {}", e)))
}
fn deserialize_cbor(&self, val: &[u8]) -> Result<T, KomerError> {
serde_cbor::from_slice(val)
.map_err(|e| KomerError::Deserialization(format!("CBOR deserialization error: {}", e)))
}
pub fn get_item_iter<K>(&self, keys: &[K]) -> Result<Vec<(Vec<String>, T)>, KomerError>
where
K: AsRef<[u8]>,
{
let key_prefix = self.to_key(keys, false);
let mut result = Vec::new();
self.db
.get_top_items_iter(&self.sdb, &key_prefix, |key, value| {
let key_strings = self.to_keys(&key);
let deserialized = self
.deserialize(&value)
.map_err(|_| DBError::ValueError("Failed to deserialize value".to_string()))?;
result.push((key_strings, deserialized));
Ok(true)
})?;
Ok(result)
}
pub fn get_full_item_iter<K>(&self, keys: &[K]) -> Result<Vec<(Vec<String>, T)>, KomerError>
where
K: AsRef<[u8]>,
{
self.get_item_iter(keys)
}
pub fn put<K>(&self, keys: &[K], val: &T) -> Result<bool, KomerError>
where
K: AsRef<[u8]>,
{
let key = self.to_key(keys, false);
if key.is_empty() {
Err(KomerError::EmptyKeys)
} else {
let value = self.serialize(val)?;
Ok(self.db.put_val(&self.sdb, &key, &value)?)
}
}
pub fn get<K>(&self, keys: &[K]) -> Result<Option<T>, KomerError>
where
K: AsRef<[u8]>,
{
let key = self.to_key(keys, false);
if let Some(val) = self.db.get_val(&self.sdb, &key)? {
Ok(Some(self.deserialize(&val)?))
} else {
Ok(None)
}
}
pub fn rem<K>(&self, keys: &[K]) -> Result<bool, KomerError>
where
K: AsRef<[u8]>,
{
let key = self.to_key(keys, false);
let existed = self.db.del_val(&self.sdb, &key)?;
Ok(existed)
}
pub fn cnt_all(&self) -> Result<usize, KomerError> {
Ok(self.db.len(&self.sdb)? as usize)
}
}
#[allow(dead_code)]
pub struct Komer<'db, T>
where
T: Serialize + for<'de> Deserialize<'de> + Debug,
{
base: KomerBase<'db, T>,
}
impl<'db, T> Komer<'db, T>
where
T: Serialize + for<'de> Deserialize<'de> + Debug,
{
pub fn new(db: Arc<&'db LMDBer>, subkey: &str, kind: SerialKind) -> Result<Self, KomerError> {
let base = KomerBase::new(db, subkey, kind, false, None)?;
Ok(Self { base })
}
pub fn put<K>(&self, keys: &[K], val: &T) -> Result<bool, KomerError>
where
K: AsRef<[u8]>,
{
self.base.put(keys, val)
}
pub fn pin<K>(&self, keys: &[K], val: &T) -> Result<bool, KomerError>
where
K: AsRef<[u8]>,
{
let key = self.base.to_key(keys, false);
let serialized = self.base.serialize(val)?;
match self.base.db.set_val(&self.base.sdb, &key, &serialized) {
Ok(_) => Ok(true),
Err(e) => Err(KomerError::DBError(e)),
}
}
pub fn get<K>(&self, keys: &[K]) -> Result<Option<T>, KomerError>
where
K: AsRef<[u8]>,
{
self.base.get(keys)
}
pub fn get_json<K>(&self, keys: &[K]) -> Result<Option<serde_json::Value>, KomerError>
where
K: AsRef<[u8]>,
{
if let Some(val) = self.get(keys)? {
match serde_json::to_value(&val) {
Ok(json_val) => Ok(Some(json_val)),
Err(e) => Err(KomerError::Serialization(e.to_string())),
}
} else {
Ok(None)
}
}
pub fn get_item_iter<K>(&self, keys: &[K]) -> Result<Vec<(Vec<String>, T)>, KomerError>
where
K: AsRef<[u8]>,
{
self.base.get_item_iter(keys)
}
pub fn rem<K>(&self, keys: &[K]) -> Result<bool, KomerError>
where
K: AsRef<[u8]>,
{
self.base.rem(keys)
}
pub fn trim<K>(&self, keys: &[K]) -> Result<bool, KomerError>
where
K: AsRef<[u8]>,
{
let key = self.base.to_key(keys, true);
Ok(self.base.db.del_top_val(&self.base.sdb, &key)?)
}
pub fn cnt_all(&self) -> Result<usize, KomerError> {
self.base.cnt_all()
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fmt::Debug;
use std::sync::Arc;
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
struct Record {
first: String, last: String, street: String, city: String, state: String, zip: u32, }
#[test]
fn test_kom_happy_path() -> Result<(), Box<dyn std::error::Error>> {
let jim = Record {
first: "Jim".to_string(),
last: "Black".to_string(),
street: "100 Main Street".to_string(),
city: "Riverton".to_string(),
state: "UT".to_string(),
zip: 84058,
};
let jimser = serde_json::to_string(&jim)?;
let jim_deserialized: Record = serde_json::from_str(&jimser)?;
assert_eq!(jim_deserialized, jim);
let lmdber = LMDBer::builder().name("test_db").temp(true).build()?;
let db_ref = Arc::new(&lmdber);
let mydb = Komer::<Record>::new(db_ref.clone(), "records.", SerialKind::Json)?;
let sue = Record {
first: "Susan".to_string(),
last: "Black".to_string(),
street: "100 Main Street".to_string(),
city: "Riverton".to_string(),
state: "UT".to_string(),
zip: 84058,
};
let keys = ["test_key", "0001"];
assert_eq!(mydb.base.sep, ".");
let key = mydb.base.to_key(&keys, false);
let str = std::str::from_utf8(&key)?;
assert_eq!(key, b"test_key.0001");
assert_eq!(
mydb.base.to_keys(&key),
vec!["test_key".to_string(), "0001".to_string()]
);
mydb.put(&keys, &sue)?;
let actual = mydb.get(&keys)?.unwrap();
assert_eq!(actual.first, "Susan");
assert_eq!(actual.last, "Black");
assert_eq!(actual.street, "100 Main Street");
assert_eq!(actual.city, "Riverton");
assert_eq!(actual.state, "UT");
assert_eq!(actual.zip, 84058);
mydb.rem(&keys)?;
let actual = mydb.get(&keys)?;
assert!(actual.is_none());
let keys = ["test_key", "0001"];
mydb.put(&keys, &sue)?;
let actual = mydb.get(&keys)?.unwrap();
assert_eq!(actual, sue);
let kip = Record {
first: "Kip".to_string(),
last: "Thorne".to_string(),
street: "200 Center Street".to_string(),
city: "Bluffdale".to_string(),
state: "UT".to_string(),
zip: 84043,
};
let result = mydb.put(&keys, &kip)?;
assert!(!result);
let actual = mydb.get(&keys)?.unwrap();
assert_eq!(actual, sue);
let actual_json = mydb.get_json(&keys)?.unwrap();
let expected_json = serde_json::to_value(&sue)?;
assert_eq!(actual_json, expected_json);
let result = mydb.pin(&keys, &kip)?;
assert!(result);
let actual = mydb.get(&keys)?.unwrap();
assert_eq!(actual, kip);
let keys = ["keystr"];
let bob = Record {
first: "Bob".to_string(),
last: "Brown".to_string(),
street: "100 Center Street".to_string(),
city: "Bluffdale".to_string(),
state: "UT".to_string(),
zip: 84043,
};
mydb.put(&keys, &bob)?;
let actual = mydb.get(&keys)?.unwrap();
assert_eq!(actual.first, "Bob");
assert_eq!(actual.last, "Brown");
assert_eq!(actual.street, "100 Center Street");
assert_eq!(actual.city, "Bluffdale");
assert_eq!(actual.state, "UT");
assert_eq!(actual.zip, 84043);
let actual_json = mydb.get_json(&keys)?.unwrap();
let expected_json = serde_json::to_value(&bob)?;
assert_eq!(actual_json, expected_json);
let nonexistent_keys = ["bla", "bal"];
assert!(mydb.get_json(&nonexistent_keys)?.is_none());
mydb.rem(&keys)?;
let actual = mydb.get(&keys)?;
assert!(actual.is_none());
drop(lmdber);
Ok(())
}
#[test]
fn test_komer_error_handling() -> Result<(), Box<dyn std::error::Error>> {
let lmdber = LMDBer::builder().name("test_db").temp(true).build()?;
let db_ref = Arc::new(&lmdber);
let mydb = Komer::<Record>::new(db_ref.clone(), "records.", SerialKind::Json)?;
let empty_keys: [&str; 0] = [];
let record = Record {
first: "Test".to_string(),
last: "User".to_string(),
street: "123 Test St".to_string(),
city: "Testville".to_string(),
state: "TS".to_string(),
zip: 12345,
};
let result = mydb.put(&empty_keys, &record);
assert!(matches!(result, Err(KomerError::EmptyKeys)));
Ok(())
}
#[test]
fn test_serialization_formats() -> Result<(), Box<dyn std::error::Error>> {
let lmdber = LMDBer::builder().name("test_db").temp(true).build()?;
let db_ref = Arc::new(&lmdber);
let record = Record {
first: "Test".to_string(),
last: "User".to_string(),
street: "123 Test St".to_string(),
city: "Testville".to_string(),
state: "TS".to_string(),
zip: 12345,
};
for serial_kind in [SerialKind::Json, SerialKind::MsgPack, SerialKind::Cbor] {
let mydb = Komer::<Record>::new(
db_ref.clone(),
&format!("records_{:?}.", serial_kind),
serial_kind,
)?;
let keys = ["test_key"];
mydb.put(&keys, &record)?;
let retrieved = mydb.get(&keys)?.unwrap();
assert_eq!(retrieved, record);
}
Ok(())
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
struct Stuff {
a: String,
b: String,
}
impl Stuff {
pub fn to_map(&self) -> HashMap<String, String> {
let mut map = HashMap::new();
map.insert("a".to_string(), self.a.clone());
map.insert("b".to_string(), self.b.clone());
map
}
}
#[test]
fn test_kom_get_item_iter() -> Result<(), Box<dyn std::error::Error>> {
let w = Stuff {
a: "Big".to_string(),
b: "Blue".to_string(),
};
let x = Stuff {
a: "Tall".to_string(),
b: "Red".to_string(),
};
let y = Stuff {
a: "Fat".to_string(),
b: "Green".to_string(),
};
let z = Stuff {
a: "Eat".to_string(),
b: "White".to_string(),
};
let lmdber = LMDBer::builder().name("test").temp(true).build()?;
let db_ref = Arc::new(&lmdber);
assert_eq!(lmdber.name(), "test");
assert!(lmdber.opened());
let mydb = Komer::<Stuff>::new(db_ref.clone(), "recs.", SerialKind::Json)?;
mydb.put(&["a", "1"], &w)?;
mydb.put(&["a", "2"], &x)?;
mydb.put(&["a", "3"], &y)?;
mydb.put(&["a", "4"], &z)?;
let items: Vec<(Vec<String>, HashMap<String, String>)> = mydb
.get_item_iter(&[] as &[&str])?
.into_iter()
.map(|(keys, data)| (keys, data.to_map()))
.collect();
assert_eq!(
items,
vec![
(vec!["a".to_string(), "1".to_string()], w.to_map()),
(vec!["a".to_string(), "2".to_string()], x.to_map()),
(vec!["a".to_string(), "3".to_string()], y.to_map()),
(vec!["a".to_string(), "4".to_string()], z.to_map()),
]
);
mydb.put(&["b", "1"], &w)?;
mydb.put(&["b", "2"], &x)?;
mydb.put(&["bc", "3"], &y)?;
mydb.put(&["bc", "4"], &z)?;
let topkeys = ["b", ""];
let items: Vec<(Vec<String>, HashMap<String, String>)> = mydb
.get_item_iter(&topkeys)?
.into_iter()
.map(|(keys, data)| (keys, data.to_map()))
.collect();
assert_eq!(
items,
vec![
(vec!["b".to_string(), "1".to_string()], w.to_map()),
(vec!["b".to_string(), "2".to_string()], x.to_map()),
]
);
let items: Vec<(Vec<String>, HashMap<String, String>)> = mydb
.get_item_iter(&[] as &[&str])?
.into_iter()
.map(|(keys, data)| (keys, data.to_map()))
.collect();
assert_eq!(
items,
vec![
(vec!["a".to_string(), "1".to_string()], w.to_map()),
(vec!["a".to_string(), "2".to_string()], x.to_map()),
(vec!["a".to_string(), "3".to_string()], y.to_map()),
(vec!["a".to_string(), "4".to_string()], z.to_map()),
(vec!["b".to_string(), "1".to_string()], w.to_map()),
(vec!["b".to_string(), "2".to_string()], x.to_map()),
(vec!["bc".to_string(), "3".to_string()], y.to_map()),
(vec!["bc".to_string(), "4".to_string()], z.to_map()),
]
);
assert_eq!(mydb.cnt_all()?, 8);
assert!(mydb.trim(&["b", ""])?);
let items: Vec<(Vec<String>, HashMap<String, String>)> = mydb
.get_item_iter(&[] as &[&str])?
.into_iter()
.map(|(keys, data)| (keys, data.to_map()))
.collect();
assert_eq!(
items,
vec![
(vec!["a".to_string(), "1".to_string()], w.to_map()),
(vec!["a".to_string(), "2".to_string()], x.to_map()),
(vec!["a".to_string(), "3".to_string()], y.to_map()),
(vec!["a".to_string(), "4".to_string()], z.to_map()),
(vec!["bc".to_string(), "3".to_string()], y.to_map()),
(vec!["bc".to_string(), "4".to_string()], z.to_map()),
]
);
assert!(mydb.trim::<[u8; 0]>(&[])?);
let items: Vec<(Vec<String>, Stuff)> = mydb.get_item_iter(&[] as &[&str])?;
assert_eq!(items, vec![]);
drop(lmdber);
Ok(())
}
#[test]
fn test_kom_put_get() -> Result<(), Box<dyn std::error::Error>> {
let lmdber = LMDBer::builder().name("test").temp(true).build()?;
let db_ref = Arc::new(&lmdber);
let mydb = Komer::<Stuff>::new(db_ref.clone(), "recs.", SerialKind::Json)?;
let data = Stuff {
a: "Test".to_string(),
b: "Value".to_string(),
};
assert!(mydb.put(&["test", "key"], &data)?);
let retrieved = mydb.get(&["test", "key"])?.unwrap();
assert_eq!(retrieved, data);
assert!(mydb.get(&["nonexistent", "key"])?.is_none());
let new_data = Stuff {
a: "Modified".to_string(),
b: "Data".to_string(),
};
assert!(!mydb.put(&["test", "key"], &new_data)?);
let retrieved = mydb.get(&["test", "key"])?.unwrap();
assert_eq!(retrieved, data);
assert!(mydb.pin(&["test", "key"], &new_data)?);
let retrieved = mydb.get(&["test", "key"])?.unwrap();
assert_eq!(retrieved, new_data);
assert!(mydb.rem(&["test", "key"])?);
assert!(mydb.get(&["test", "key"])?.is_none());
assert!(!mydb.rem(&["test", "key"])?);
Ok(())
}
#[test]
fn test_kom_serialization_formats() -> Result<(), Box<dyn std::error::Error>> {
let lmdber = LMDBer::builder().name("test").temp(true).build()?;
let db_ref = Arc::new(&lmdber);
let test_data = Stuff {
a: "Test".to_string(),
b: "Value".to_string(),
};
for format in [SerialKind::Json, SerialKind::MsgPack, SerialKind::Cbor].iter() {
let db_name = format!("test_{:?}", format);
let mydb = Komer::<Stuff>::new(db_ref.clone(), &db_name, *format)?;
mydb.put(&["test"], &test_data)?;
let retrieved = mydb.get(&["test"])?.unwrap();
assert_eq!(retrieved, test_data);
}
Ok(())
}
#[test]
fn test_kom_empty_keys() -> Result<(), Box<dyn std::error::Error>> {
let lmdber = LMDBer::builder().name("test").temp(true).build()?;
let db_ref = Arc::new(&lmdber);
let mydb = Komer::<Stuff>::new(db_ref.clone(), "recs.", SerialKind::Json)?;
let data = Stuff {
a: "Test".to_string(),
b: "Value".to_string(),
};
let empty_keys: [&str; 0] = [];
let result = mydb.put(&empty_keys, &data);
assert!(matches!(result, Err(KomerError::EmptyKeys)));
Ok(())
}
}