#![warn(missing_docs)]
#[macro_use] extern crate log;
extern crate rustc_hash;
#[macro_use] extern crate try_or;
extern crate failure;
#[macro_use] extern crate failure_derive;
extern crate walkdir;
extern crate typenum;
extern crate generic_array;
#[cfg(test)] mod tests;
pub mod source;
use crate::source::{PackageError, Source, SourceManager, SourceId, TrustLevel};
use rustc_hash::FxHasher;
use generic_array::{GenericArray, sequence::GenericSequence};
use generic_array::typenum::U1024;
use std::any::{Any, TypeId};
use std::cell::{RefCell, Ref, RefMut};
use std::collections::HashMap;
use std::hash::BuildHasherDefault;
use std::rc::Rc;
use std::io::{Read, Seek};
use std::marker::Sized;
pub type DataId = usize;
type DataStoreTypeMax = U1024;
#[derive(Debug, PartialEq)]
pub enum DataStoreOk
{
Loaded,
AlreadyLoaded,
}
#[derive(Debug, Fail)]
pub enum DataError
{
#[fail(display = "The package was not found")]
PackageNotFound,
#[fail(display = "The package source was not trusted")]
SourceNotTrusted,
#[fail(display = "The data object was not found")]
DataNotFound,
#[fail(display = "Invalid character in name")]
BadName,
#[fail(display = "Invalid source ID")]
BadSource,
#[fail(display = "Operation not supported")]
NotSupported,
#[fail(display = "Package source failure: {}", _0)]
PackageSourceError(PackageError),
#[fail(display = "{}", _0)]
IoError(#[cause] std::io::Error),
#[fail(display = "The data object contained invalid data: {}", _0)]
BadData(String)
}
#[derive(Debug, Fail)]
pub enum PreparedStoreError
{
#[fail(display = "The provided DataStore is already associated with a PreparedStore.")]
AlreadyConnected,
#[fail(display = "The provided DataStore is not the one used to create this PreparedStore.")]
WrongDataStore
}
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum TrustPolicy
{
UntrustedOk,
TrustRequired
}
impl From<std::io::Error> for DataError
{
fn from(error : std::io::Error) -> DataError
{
DataError::IoError(error)
}
}
#[derive(Clone)]
struct DataIdGeneration
{
index : DataId,
generation : u64
}
struct LoadedData<T : DataObject + 'static>
{
data_object : T,
index : DataIdGeneration,
pathname : String
}
pub trait ReadSeek : Read + Seek {}
impl<T: Read + Seek> ReadSeek for T {}
pub trait DataObject
{
fn folder_name() -> &'static str where Self : Sized;
fn trust_policy() -> TrustPolicy;
fn want_file(pathname : &str) -> bool where Self : Sized;
fn from_io(reader : Box<dyn ReadSeek>, full_pathname : &str, source : &mut Box<dyn Source>) -> Result<Self, DataError> where Self : Sized;
#[allow(unused_variables)]
fn write(&mut self, full_pathname : &str, source : &mut Box<dyn Source>) -> Result<(), DataError> { Err(DataError::NotSupported) }
fn generation(&self) -> u64 { 0 }
#[allow(unused_variables)]
fn set_generation(&mut self, generation : u64) {}
}
pub struct DataStore<T : DataObject + 'static>
{
source_manager : Rc<RefCell<SourceManager>>,
unloaded_packages : HashMap<String, HashMap<String, DataIdGeneration, BuildHasherDefault<FxHasher>>, BuildHasherDefault<FxHasher>>,
data_list : Vec<Option<T>>,
data_map : HashMap<String, HashMap<String, DataIdGeneration, BuildHasherDefault<FxHasher>>, BuildHasherDefault<FxHasher>>,
next_index : DataId,
prepare_info : Option<Rc<RefCell<PrepareInfo>>>
}
impl<T : DataObject + 'static> DataStore<T>
{
pub fn new(source_manager : Rc<RefCell<SourceManager>>) -> DataStore<T>
{
DataStore::<T>
{
source_manager,
unloaded_packages : HashMap::with_hasher(BuildHasherDefault::<FxHasher>::default()),
data_list : Vec::new(),
data_map : HashMap::with_hasher(BuildHasherDefault::<FxHasher>::default()),
next_index : 1,
prepare_info : None
}
}
pub fn load_package(&mut self, package_name : &str) -> Result<DataStoreOk, DataError>
{
if package_name.find('/').is_some()
{
return Err(DataError::BadName);
}
if self.package_loaded(package_name)
{
return Ok(DataStoreOk::AlreadyLoaded);
}
if let Some(mut source) = self.source_manager.borrow_mut().get_package_source(package_name)
{
if !source.has_package(package_name)
{
return Err(DataError::PackageNotFound);
}
if T::trust_policy() == TrustPolicy::TrustRequired
&& source.trust_level(package_name) != TrustLevel::TrustedSource
{
return Err(DataError::SourceNotTrusted);
}
let type_folder = T::folder_name();
if type_folder.find('/').is_some()
{
return Err(DataError::BadName);
}
let iter = source.iter_entries(package_name, type_folder);
let mut loaded_items : Vec<LoadedData<T>> = Vec::new();
let mut new_index = self.next_index;
if let Some(old_package_map) = self.unloaded_packages.get(package_name)
{
self.data_map.insert(package_name.to_string(), old_package_map.clone());
}
self.unloaded_packages.remove(package_name);
let package_map = self.data_map.entry(package_name.to_string()).or_insert(HashMap::with_hasher(BuildHasherDefault::<FxHasher>::default()));
for entry in iter
{
let pathname = try_or_else!(entry,
|error| Err(DataError::PackageSourceError(error)));
if pathname.starts_with('/') || pathname.ends_with('/')
{
return Err(DataError::BadName);
}
if !T::want_file(&pathname)
{
continue;
}
let full_pathname = format!("{}/{}/{}", package_name, type_folder, pathname);
let reader = try_or_else!(source.read_file(&full_pathname),
|error| Err(DataError::PackageSourceError(error)));
let mut data_object = T::from_io(reader, &full_pathname, &mut source)?;
let index_gen : DataIdGeneration;
if let Some(entry) = package_map.get(&pathname)
{
let mut generation = entry.generation + 1;
if generation == 0
{
warn!("Generation overflow for package item {}!", pathname);
generation = 1;
}
index_gen = DataIdGeneration
{
index: entry.index,
generation
};
}
else
{
index_gen = DataIdGeneration
{
index: new_index,
generation: 1
};
new_index += 1;
}
data_object.set_generation(index_gen.generation);
loaded_items.push(LoadedData::<T> { data_object, index : index_gen, pathname });
}
self.next_index = new_index;
while self.data_list.len() <= self.next_index
{
self.data_list.push(None);
}
for item in loaded_items.drain(..)
{
self.data_list[item.index.index] = Some(item.data_object);
package_map.insert(item.pathname, item.index.clone());
if let Some(prepare_info) = &self.prepare_info
{
prepare_info.borrow_mut().to_prepare.insert(item.index.index, PrepareType::FullPrepare);
}
}
}
else
{
return Err(DataError::PackageNotFound);
}
Ok(DataStoreOk::Loaded)
}
pub fn get_id(&self, package_name : &str, pathname : &str) -> Result<DataId, DataError>
{
if let Some(package_map) = self.data_map.get(package_name)
{
if let Some(id) = package_map.get(pathname)
{
return Ok(id.index);
}
else
{
return Err(DataError::DataNotFound);
}
}
Err(DataError::PackageNotFound)
}
pub fn get_id_mut(&mut self, package_name : &str, pathname : &str) -> Result<DataId, DataError>
{
if let Some(package_map) = self.data_map.get(package_name)
{
if let Some(id) = package_map.get(pathname)
{
return Ok(id.index);
}
else
{
return Err(DataError::DataNotFound);
}
}
else
{
self.load_package(package_name)?;
return self.get_id_mut(package_name, pathname);
}
}
pub fn get(&self, index : DataId) -> Option<&T>
{
if index >= self.data_list.len()
{
return None;
}
match &self.data_list[index]
{
Some(data) => Some(data),
None => None
}
}
pub fn get_mut(&mut self, index : DataId) -> Option<&mut T>
{
if index >= self.data_list.len()
{
return None;
}
match &mut self.data_list[index]
{
Some(data) => Some(data),
None => None
}
}
pub fn list_package_contents(&self, package_name : &str) -> Vec<String>
{
let mut listing : Vec<String> = Vec::new();
if let Some(package_map) = self.data_map.get(package_name)
{
for (name, index) in package_map.iter()
{
if self.data_list[index.index].is_some()
{
listing.push(name.clone());
}
}
listing.sort();
}
listing
}
pub fn unload_package(&mut self, package_name : &str) -> bool
{
if let Some(package_map) = self.data_map.get_mut(package_name)
{
for id in package_map.values_mut()
{
if let Some(data) = &self.data_list[id.index]
{
id.generation = data.generation();
}
self.data_list[id.index] = None;
if let Some(prepare_info) = &self.prepare_info
{
prepare_info.borrow_mut().to_prepare.insert(id.index, PrepareType::FullPrepare);
}
}
self.unloaded_packages.insert(package_name.to_string(), package_map.clone());
}
else
{
return false;
}
self.data_map.remove(package_name);
true
}
pub fn unload_all(&mut self)
{
for (package_name, package_map) in &self.data_map
{
for id in package_map.values()
{
if let Some(prepare_info) = &self.prepare_info
{
prepare_info.borrow_mut().to_prepare.insert(id.index, PrepareType::FullPrepare);
}
}
self.unloaded_packages.insert(package_name.to_string(), package_map.clone());
}
self.data_map.clear();
self.data_list.clear();
}
pub fn package_loaded(&self, package_name : &str) -> bool
{
self.data_map.contains_key(package_name)
}
pub fn save(&mut self, package_name : &str, pathname : &str, source_id : SourceId) -> Result<(), DataError>
{
let index = try_or_else!(self.get_id_mut(package_name, pathname),
|error| Err(error));
let data = try_opt_or_else!(&mut self.data_list[index],
|| Err(DataError::DataNotFound));
if let Some(source) = self.source_manager.borrow_mut().source(source_id)
{
let type_folder = T::folder_name();
let full_pathname = format!("{}/{}/{}", package_name, type_folder, pathname);
return data.write(&full_pathname, source);
}
else
{
return Err(DataError::BadSource);
}
}
pub fn reprepare(&mut self, index : DataId) -> bool
{
if let Some(prepare_info) = &self.prepare_info
{
if index >= self.data_list.len()
{
return false;
}
match &mut self.data_list[index]
{
Some(_) =>
{
if !prepare_info.borrow_mut().to_prepare.contains_key(&index)
{
prepare_info.borrow_mut().to_prepare.insert(index, PrepareType::Reprepare);
return true;
}
},
_ => ()
}
}
false
}
pub fn reload(&mut self, package_name : &str, pathname : &str) -> Result<(), DataError>
{
let index = try_or_else!(self.get_id_mut(package_name, pathname), |error| Err(error));
if let Some(mut source) = self.source_manager.borrow_mut().get_package_source(package_name)
{
let type_folder = T::folder_name();
if pathname.starts_with('/') || pathname.ends_with('/')
{
return Err(DataError::BadName);
}
if !T::want_file(&pathname)
{
return Err(DataError::DataNotFound);
}
let full_pathname = format!("{}/{}/{}", package_name, type_folder, pathname);
let reader = try_or_else!(source.read_file(&full_pathname),
|error| Err(DataError::PackageSourceError(error)));
let data_object = T::from_io(reader, &full_pathname, &mut source)?;
self.data_list[index] = Some(data_object);
}
else
{
return Err(DataError::DataNotFound);
}
self.reprepare(index);
Ok(())
}
}
struct DataMultistoreLookup
{
index_to_type_id : Vec<TypeId>,
type_id_to_index : HashMap<TypeId, usize, BuildHasherDefault<FxHasher>>
}
impl DataMultistoreLookup
{
#[inline(always)]
fn lookup(&self, type_id : &TypeId) -> Option<usize>
{
self.type_id_to_index.get(type_id).cloned()
}
}
pub struct DataMultistore
{
source_manager : Rc<RefCell<SourceManager>>,
stores : GenericArray<RefCell<Box<dyn Any>>, DataStoreTypeMax>,
lookup : RefCell<DataMultistoreLookup>,
locked : bool
}
impl DataMultistore
{
pub fn new(source_manager : Rc<RefCell<SourceManager>>) -> DataMultistore
{
let stores = GenericArray::<RefCell<Box<dyn Any>>, DataStoreTypeMax>::generate(|_| RefCell::new(Box::new(())));
let lookup = DataMultistoreLookup
{
index_to_type_id : Vec::new(),
type_id_to_index : HashMap::with_hasher(BuildHasherDefault::<FxHasher>::default())
};
DataMultistore
{
source_manager,
stores,
lookup : RefCell::new(lookup),
locked : false
}
}
pub fn register_type<D : DataObject + 'static>(&self) -> bool
{
let wanted_type = TypeId::of::<D>();
let store = DataStore::<D>::new(self.source_manager.clone());
self.register_boxed_type(wanted_type, Box::new(store))
}
fn register_boxed_type(&self, type_id : TypeId, mut store : Box<dyn Any>) -> bool
{
let mut lookup = self.lookup.try_borrow_mut().expect("DataMultistore data already in use!");
if !lookup.type_id_to_index.contains_key(&type_id)
{
if lookup.index_to_type_id.len() >= self.stores.len()
{
panic!("Too many types registered! Limit is {}", self.stores.len());
}
let insert_pos = lookup.index_to_type_id.len();
let mut placeholder = self.stores[insert_pos].try_borrow_mut().expect("Placeholder under use somehow!");
std::mem::swap(&mut *placeholder, &mut store);
lookup.type_id_to_index.insert(type_id, insert_pos);
lookup.index_to_type_id.push(type_id);
return true;
}
false
}
pub fn lock(&mut self, locked : bool)
{
self.locked = locked;
}
pub fn store<D : DataObject + 'static>(&self) -> Ref<DataStore<D>>
{
if let Some(store) = self.try_store()
{
return store;
}
panic!("DataObject type {} not registered.", std::any::type_name::<D>());
}
pub fn store_mut<D : DataObject + 'static>(&self) -> RefMut<DataStore<D>>
{
if let Some(store) = self.try_store_mut()
{
return store;
}
if self.locked
{
panic!("Mutable datastore access is currently restricted.");
}
else
{
panic!("DataObject type {} not registered.", std::any::type_name::<D>());
}
}
pub fn try_store<D : DataObject + 'static>(&self) -> Option<Ref<DataStore<D>>>
{
let wanted_type = TypeId::of::<D>();
let mut index_opt = self.lookup.try_borrow().expect("DataMultistore data already in use!").lookup(&wanted_type);
if index_opt.is_none()
{
self.register_type::<D>();
index_opt = self.lookup.try_borrow().expect("DataMultistore data already in use!").lookup(&wanted_type);
}
if let Some(index) = index_opt
{
return Some(Ref::map(self.stores[index].borrow(),
|b| b.downcast_ref::<DataStore<D>>().expect("DataMultistore borrow failed!")));
}
None
}
pub fn try_store_mut<D : DataObject + 'static>(&self) -> Option<RefMut<DataStore<D>>>
{
if self.locked
{
return None;
}
let wanted_type = TypeId::of::<D>();
let mut index_opt = self.lookup.try_borrow().expect("DataMultistore data already in use!").lookup(&wanted_type);
if index_opt.is_none()
{
self.register_type::<D>();
index_opt = self.lookup.try_borrow().expect("DataMultistore data already in use!").lookup(&wanted_type);
}
if let Some(index) = index_opt
{
return Some(RefMut::map(self.stores[index].borrow_mut(),
|b| b.downcast_mut::<DataStore<D>>().expect("DataMultistore borrow failed!")));
}
None
}
}
#[derive(Debug, Eq, PartialEq)]
enum PrepareType
{
FullPrepare,
Reprepare
}
struct PrepareInfo
{
to_prepare : HashMap<DataId, PrepareType, BuildHasherDefault<FxHasher>>,
}
impl PrepareInfo
{
fn new() -> PrepareInfo
{
PrepareInfo
{
to_prepare : HashMap::with_hasher(BuildHasherDefault::<FxHasher>::default())
}
}
}
pub trait DataPreparer<T : DataObject + 'static, U>
{
fn prepare(&mut self, data : &mut T, id : DataId) -> U;
fn unprepare(&mut self, prepared : &mut U, id : DataId);
fn reprepare(&mut self, data : &mut T, prepared : &mut U, id : DataId)
{
self.unprepare(prepared, id);
*prepared = self.prepare(data, id);
}
}
pub struct PreparedStore<T : DataObject + 'static, U>
{
data_list : Vec<Option<U>>,
data_preparer : Box<dyn DataPreparer<T, U>>,
prepare_info : Rc<RefCell<PrepareInfo>>
}
impl<T : DataObject + 'static, U> PreparedStore<T, U>
{
pub fn new(data_store : &mut DataStore<T>, data_preparer : Box<dyn DataPreparer<T, U>>) -> Result<PreparedStore<T, U>, PreparedStoreError>
{
if data_store.prepare_info.is_some()
{
return Err(PreparedStoreError::AlreadyConnected);
}
let prepare_info = Rc::new(RefCell::new(PrepareInfo::new()));
data_store.prepare_info = Some(prepare_info.clone());
Ok(PreparedStore::<T, U>
{
data_list : Vec::new(),
data_preparer,
prepare_info
})
}
pub fn get(&self, index : DataId) -> Option<&U>
{
if index >= self.data_list.len()
{
return None;
}
match &self.data_list[index]
{
Some(data) => Some(data),
None => None
}
}
pub fn get_mut(&mut self, index : DataId) -> Option<&mut U>
{
if index >= self.data_list.len()
{
return None;
}
match &mut self.data_list[index]
{
Some(data) => Some(data),
None => None
}
}
pub fn sync(&mut self, data_store : &mut DataStore<T>) -> Result<(), PreparedStoreError>
{
if let Some(prepare_info) = &mut data_store.prepare_info
{
if !Rc::ptr_eq(&prepare_info, &self.prepare_info)
{
return Err(PreparedStoreError::WrongDataStore);
}
}
else
{
return Err(PreparedStoreError::WrongDataStore);
}
for (id, prepare_type) in &self.prepare_info.borrow_mut().to_prepare
{
if let Some(data) = data_store.get_mut(*id)
{
while self.data_list.len() <= *id
{
self.data_list.push(None);
}
let mut existing = false;
if let Some(prepared) = &mut self.data_list[*id]
{
match *prepare_type
{
PrepareType::FullPrepare =>
{
self.data_preparer.unprepare(prepared, *id);
},
PrepareType::Reprepare =>
{
self.data_preparer.reprepare(data, prepared, *id);
existing = true;
}
}
}
if !existing
{
self.data_list[*id] = Some(self.data_preparer.prepare(data, *id));
}
}
else
{
if *id < self.data_list.len()
{
if let Some(prepared) = &mut self.data_list[*id]
{
self.data_preparer.unprepare(prepared, *id);
}
self.data_list[*id] = None;
}
}
}
self.prepare_info.borrow_mut().to_prepare.clear();
Ok(())
}
}