keeshond_datapack 0.13.0

#![warn(missing_docs)]

//! `keeshond_datapack` lets you easily load resources for your game and cache them in memory,
//!  mapped via pathname and accessible by handle objects. Define how they load
//!  with the [DataObject] trait, by implementing a function that takes a [Read] + [Seek] object.
//!  Define where they load from by instantiating a [source::Source].
//!
//! Datapack is used by Keeshond but can work with any engine.
//!
//! # How to use
//!
//! - Implement the [DataObject] trait for each type of object you want to be able to load in.
//! - Instantiate a [source::SourceManager] object and give it a [source::Source] such as a
//!  [source::FilesystemSource]
//! - Create a folder for each package on disk (if using [FilesystemSource](source::FilesystemSource)).
//! - Create subfolders within each package for each data type and place files within.
//! - Instantiate a [DataStore] for each data type you want to load.
//! - (Optional) use [DataStore::load_package()] to ensure the package is loaded at start.
//! - When creating objects that need resources, instantiate a [DataHandle] with a desired path.
//! - Resolve this path using a [DataStore] to access the data. Store this handle somewhere it can
//!   be used later, as resolving does involve a couple hash lookups.
//!
//! Pathnames are of the form `path/to/file.png`. They must match exactly, including the use of one
//!  forward slash as a separator, and no separators at the beginning or end. The package name and
//!  data type folders are not included (they are implied). Pathnames are also case-sensitive even on
//!  platforms with case-insensitive filesystems (Windows, macOS).

#[macro_use] extern crate log;
extern crate rustc_hash;
extern crate failure;
#[macro_use] extern crate failure_derive;
extern crate walkdir;
extern crate typenum;
extern crate generic_array;
#[macro_use] extern crate downcast_rs;
extern crate zip;
#[cfg(feature = "serde")] extern crate serde;

#[cfg(test)] mod tests;

/// The `Source` trait and implementations.
pub mod source;

use crate::source::{PackageError, Source, SourceManager, SourceId, TrustLevel};

use rustc_hash::FxHasher;
use generic_array::{GenericArray, sequence::GenericSequence};
use generic_array::typenum::U1024;
use downcast_rs::Downcast;

use std::any::TypeId;
use std::cell::{RefCell, Ref, RefMut, Cell};
use std::collections::HashMap;
use std::fmt::{Debug, Formatter};
use std::hash::{BuildHasherDefault, Hash, Hasher};
use std::rc::Rc;
use std::io::{Read, Seek};
use std::marker::{Sized, PhantomData};
use std::time::Instant;

#[cfg(feature = "serde")]
use serde::{Serialize, Deserialize};

/// A numeric ID used to refer to a [DataObject] of a specific type. Should only be used directly
///  when performing operations that need an index into an array or Vec. This value may change
///  between sessions. In most cases you probably want to use a [DataHandle] instead.
pub struct DataId<T : DataObject + 'static>
{
    index : usize,
    _phantom : PhantomData<T>
}

impl<T : DataObject + 'static> DataId<T>
{
    /// Creates a new ID using the given numerical value. Passing a value of 0 will make this a
    ///  "null" ID
    pub fn new(index : usize) -> DataId<T>
    {
        DataId
        {
            index,
            _phantom : PhantomData
        }
    }

    /// Returns the numerical value associated with this ID, for use with array indexing. This value
    ///  should not be serialized as its value may change between sessions.
    pub fn index(&self) -> usize
    {
        self.index
    }
}

impl<T : DataObject + 'static> Clone for DataId<T>
{
    fn clone(&self) -> Self
    {
        DataId
        {
            index : self.index,
            _phantom : Default::default()
        }
    }
}

impl<T : DataObject + 'static> Copy for DataId<T> {}

impl<T : DataObject + 'static> Default for DataId<T>
{
    fn default() -> Self
    {
        DataId::new(0)
    }
}

impl<T : DataObject + 'static> Debug for DataId<T>
{
    fn fmt(&self, f : &mut Formatter<'_>) -> std::fmt::Result
    {
        self.index.fmt(f)
    }
}

impl<T: DataObject + 'static> PartialEq<Self> for DataId<T>
{
    fn eq(&self, other : &Self) -> bool
    {
        self.index.eq(&other.index)
    }
}

impl<T : DataObject + 'static> Eq for DataId<T> {}

impl<T : DataObject + 'static> Hash for DataId<T>
{
    fn hash<H : Hasher>(&self, state : &mut H)
    {
        self.index.hash(state)
    }
}

type DataStoreTypeMax = U1024;

/// Return type when loading information from a [DataStore]
#[derive(Debug, PartialEq)]
pub enum DataStoreOk
{
    /// The package was loaded successfully
    Loaded,
    /// The package was already loaded, so no action was taken
    AlreadyLoaded,
}

/// Return type when failing to load a [DataObject] from a [DataStore]
#[derive(Debug, Fail)]
pub enum DataError
{
    /// The given package was not found in the [source::Source]
    #[fail(display = "The package was not found")]
    PackageNotFound,
    /// The given package is not trusted by the [source::Source]
    #[fail(display = "The package source was not trusted")]
    SourceNotTrusted,
    /// The package is loaded, but has no [DataObject] by the given pathname
    #[fail(display = "The data object was not found")]
    DataNotFound,
    /// An invalid character (forward slash) was used in a package or type folder name
    #[fail(display = "Invalid character in name")]
    BadName,
    /// An invalid source ID was specified
    #[fail(display = "Invalid source ID")]
    BadSource,
    /// The operation is not supported by this implementation
    #[fail(display = "Operation not supported")]
    NotSupported,
    /// An error occurred while accessing a package's source
    ///  from the [source::Source]
    #[fail(display = "Package source failure: {}", _0)]
    PackageSourceError(PackageError),
    /// An error of type [std::io::Error] occurred.
    #[fail(display = "{}", _0)]
    IoError(#[cause] std::io::Error),
    /// A logical error occurred while reading a [DataObject]
    #[fail(display = "The data object contained invalid data: {}", _0)]
    BadData(String)
}

impl From<PackageError> for DataError
{
    fn from(error : PackageError) -> Self
    {
        DataError::PackageSourceError(error)
    }
}

/// Return type for when [PreparedStore] operations fail
#[derive(Debug, Fail)]
pub enum PreparedStoreError
{
    /// The DataStore provided in [PreparedStore::new()] is already associated with another
    ///  [PreparedStore]
    #[fail(display = "The provided DataStore is already associated with a PreparedStore.")]
    AlreadyConnected,
    /// The DataStore provided in [PreparedStore::sync()] is not the same one passed to
    ///  [PreparedStore::new()]
    #[fail(display = "The provided DataStore is not the one used to create this PreparedStore.")]
    WrongDataStore
}

/// Behavior for when automatically loading a resource fails
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
pub enum LoadErrorMode
{
    /// The application will panic if the load fails
    Fatal,
    /// The application will print a warning if the load fails, and the data ID will resolve to null
    Warning
}

/// Trust settings for a given [DataObject] resource type
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub enum TrustPolicy
{
    /// The resource may be loaded from an untrusted source.
    UntrustedOk,
    /// The resource may only be loaded from a trusted source
    TrustRequired
}

impl From<std::io::Error> for DataError
{
    fn from(error : std::io::Error) -> DataError
    {
        DataError::IoError(error)
    }
}

struct DataIdGeneration<T : DataObject + 'static>
{
    id : DataId<T>,
    generation : u64
}

impl<T : DataObject + 'static> Clone for DataIdGeneration<T>
{
    fn clone(&self) -> Self
    {
        DataIdGeneration
        {
            id : self.id,
            generation : self.generation
        }
    }
}

struct LoadedData<T : DataObject + 'static>
{
    data_object : T,
    id_gen : DataIdGeneration<T>,
    pathname : String
}

#[cfg(feature = "serde")]
fn data_id_default<T : DataObject + 'static>() -> Cell<DataId<T>>
{
    Cell::new(DataId::new(usize::MAX))
}

#[cfg(feature = "serde")]
fn use_token_default<T : DataObject + 'static>() -> RefCell<Option<PackageUseToken<T>>>
{
    RefCell::new(None)
}

/// A path to a data resource, resolvable to a cached [DataId].
///
/// This object serves three purposes:
///
/// - Conveniently resolving a path to an ID but only when resolution is necessary
/// - Serializing data that needs to reference a DataObject
/// - Keeping a package open so it doesn't get auto-unloaded.
///
/// If placing inside a DataObject struct, take care not to have a self-referencing type
///  (i.e. a DataHandle\<T\> inside a T) or a type cycle
///  (i.e. a DataHandle\<T\> inside a U and a DataHandle\<U\> inside a T). Otherwise the auto-unload
///  mechanism may not be able to unload the package the object is from.
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub struct DataHandle<T : DataObject + 'static>
{
    #[cfg_attr(feature = "serde", serde(rename = "pkg"))]
    #[cfg_attr(feature = "serde", serde(alias = "package"))]
    package : String,
    #[cfg_attr(feature = "serde", serde(rename = "pth"))]
    #[cfg_attr(feature = "serde", serde(alias = "path"))]
    path : String,
    #[cfg_attr(feature = "serde", serde(skip, default = "data_id_default"))]
    data_id : Cell<DataId<T>>,
    #[cfg_attr(feature = "serde", serde(skip, default = "use_token_default"))]
    use_token : RefCell<Option<PackageUseToken<T>>>,
    #[cfg_attr(feature = "serde", serde(skip))]
    _phantom : PhantomData<T>
}

impl<T : DataObject + 'static> Default for DataHandle<T>
{
    fn default() -> Self
    {
        DataHandle::null()
    }
}

impl<T : DataObject + 'static> Clone for DataHandle<T>
{
    fn clone(&self) -> Self
    {
        DataHandle
        {
            package : self.package.clone(),
            path : self.path.clone(),
            data_id : self.data_id.clone(),
            use_token : self.use_token.clone(),
            _phantom : PhantomData
        }
    }
}

impl<T : DataObject + 'static> Debug for DataHandle<T>
{
    fn fmt(&self, f : &mut Formatter) -> std::fmt::Result
    {
        write!(f, "DataHandle {{ package: {:?}, path: {:?}, data_id: {:?} }}", self.package, self.path, self.data_id)
    }
}

impl<T : DataObject + 'static> DataHandle<T>
{
    /// Creates a new [DataHandle] that points to nothing. Resolving this data handle will fail
    ///  until it is given a valid path, such as through set_path().
    pub fn null() -> DataHandle<T>
    {
        DataHandle::<T>
        {
            package : String::new(),
            path : String::new(),
            data_id : Cell::new(DataId::new(usize::MAX)),
            use_token : RefCell::new(None),
            _phantom : PhantomData
        }
    }

    /// Creates a new [DataHandle] with the given resource path. The data ID will be unresolved.
    ///  You will need to call resolve() for the data ID to point to a valid resource.
    pub fn with_path(package : &str, path : &str) -> DataHandle<T>
    {
        DataHandle::<T>
        {
            package : package.to_string(),
            path : path.to_string(),
            data_id : Cell::new(DataId::new(usize::MAX)),
            use_token : RefCell::new(None),
            _phantom : PhantomData
        }
    }

    /// Clones the given [DataHandle] but only retains the ID, erasing the path. This is useful for
    ///  instances where you want to store a lot of handles but don't need the path, to cut down on
    ///  memory usage.
    pub fn clone_without_path(&self) -> DataHandle<T>
    {
        DataHandle::<T>
        {
            package : String::new(),
            path : String::new(),
            data_id : self.data_id.clone(),
            use_token : self.use_token.clone(),
            _phantom : PhantomData
        }
    }

    /// Changes the resource path. The data ID will reset to be unresolved.
    ///  You will need to call resolve() for the data ID to point to a valid resource.
    pub fn set_path(&mut self, package : &str, path : &str)
    {
        self.package.clear();
        self.package.push_str(package);
        self.path.clear();
        self.path.push_str(path);
        self.data_id = Cell::new(DataId::new(usize::MAX));
        self.use_token = RefCell::new(None);
    }

    /// Returns the package name part of the resource path
    pub fn package(&self) -> &String
    {
        &self.package
    }

    /// Returns the filepath to the resource
    pub fn path(&self) -> &String
    {
        &self.path
    }

    /// Returns the data ID.
    ///  You will need to call resolve() beforehand for the data ID to point to a valid resource.
    pub fn id(&self) -> DataId<T>
    {
        let id = self.data_id.get();

        if id.index == usize::MAX
        {
            return DataId::new(0);
        }

        id
    }

    /// Returns true if the path has been resolved via a call to resolve().
    #[inline(always)]
    pub fn resolved(&self) -> bool
    {
        self.data_id.get().index != usize::MAX
    }

    /// Resolves the data ID using the given [DataStore]. Returns the resolved data ID,
    ///  which is cached inside the object. If load_error is Fatal, this function will panic
    ///  if the load fails. Otherwise it will simply return a null data ID.
    #[inline(always)]
    pub fn resolve(&self, data_res : &mut DataStore<T>, load_error : LoadErrorMode) -> DataId<T>
    {
        if !self.resolved()
        {
            match data_res.get_id_mut(&self.package, &self.path)
            {
                Ok(data_id) => self.data_id.set(data_id),
                Err(error) =>
                {
                    if load_error == LoadErrorMode::Fatal
                    {
                        panic!("Failed to load \"{}/{}/{}\": {}",
                                self.package, T::folder_name(), self.path, error.to_string());
                    }
                    else
                    {
                        warn!("Failed to load \"{}/{}/{}\": {}",
                                self.package, T::folder_name(), self.path, error.to_string());
                        self.data_id.set(DataId::new(0));
                    }
                }
            }

            self.use_token.replace(data_res.package_use_token(&self.package));
        }

        self.data_id.get()
    }

    /// Returns a reference to data object in the store, if it exists. If the data object is not resolved,
    ///  then resolve() will be called, and then the data object reference returned, if available.
    #[inline(always)]
    pub fn get<'a, 'b : 'a>(&self, data_res : &'b mut DataStore<T>, load_error : LoadErrorMode) -> Option<&'a T>
    {
        let id = self.resolve(data_res, load_error);

        data_res.get(id)
    }

    /// Returns a mutable reference to data object in the store, if it exists. If the data object is not resolved,
    ///  then resolve() will be called, and then the data object reference returned, if available.
    #[inline(always)]
    pub fn get_mut<'a, 'b : 'a>(&self, data_res : &'b mut DataStore<T>, load_error : LoadErrorMode) -> Option<&'a mut T>
    {
        let id = self.resolve(data_res, load_error);

        data_res.get_mut(id)
    }

    /// Returns a reference to data object in the store, if it exists. If the data object is not resolved,
    ///  returns None instead of attempting to resolve.
    #[inline(always)]
    pub fn get_if_resolved<'a, 'b : 'a>(&self, data_res : &'b DataStore<T>) -> Option<&'a T>
    {
        data_res.get(self.id())
    }

    /// Returns a mutable reference to data object in the store, if it exists. If the data object is not resolved,
    ///  returns None instead of attempting to resolve.
    #[inline(always)]
    pub fn get_mut_if_resolved<'a, 'b : 'a>(&self, data_res : &'b mut DataStore<T>) -> Option<&'a mut T>
    {
        data_res.get_mut(self.id())
    }

    /// Resolves the data ID using the given [DataStore]. Returns the resolved data ID,
    ///  which is cached inside the object. If the data is not yet loaded, returns a null ID.
    #[inline(always)]
    pub fn resolve_if_loaded(&self, data_res : &DataStore<T>) -> DataId<T>
    {
        if !self.resolved()
        {
            match data_res.get_id(&self.package, &self.path)
            {
                Ok(data_id) => self.data_id.set(data_id),
                Err(_) => {}
            }

            self.use_token.replace(data_res.package_use_token(&self.package));
        }

        self.data_id.get()
    }
}

/// A token that indicates that a package is in use and should not be automatically unloaded.
pub struct PackageUseToken<T : DataObject + 'static>
{
    use_count : Rc<String>,
    _phantom : PhantomData<T>
}

impl<T : DataObject + 'static> PackageUseToken<T>
{
    fn new(package : &str) -> PackageUseToken<T>
    {
        PackageUseToken
        {
            use_count : Rc::new(package.to_string()),
            _phantom : PhantomData
        }
    }

    /// Returns the name of the package associated with this token
    pub fn package(&self) -> &String
    {
        self.use_count.as_ref()
    }

    /// Returns how many copies of the token reference this package
    pub fn uses(&self) -> usize
    {
        // Subtracting 1 because a copy of the token is stored internally and cloned out as
        //  necessary - this copy should not count toward the total use count
        Rc::strong_count(&self.use_count).saturating_sub(1)
    }
}

impl<T : DataObject + 'static> Clone for PackageUseToken<T>
{
    fn clone(&self) -> Self
    {
        PackageUseToken
        {
            use_count : Rc::clone(&self.use_count),
            _phantom : PhantomData
        }
    }
}

impl<T : DataObject + 'static> Debug for PackageUseToken<T>
{
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result
    {
        write!(f, "PackageUseToken<{}> {{ uses: {} }}", std::any::type_name::<T>(), self.uses())
    }
}

/// A boxable trait that implements both [Read] and [Seek], used by the
///  [source::Source] types.
pub trait ReadSeek : Read + Seek {}

impl<T: Read + Seek> ReadSeek for T {}

/// Represents a data item loaded from a package. Implement this trait to allow the system
///  to load new types of data.
pub trait DataObject
{
    /// The folder name that [DataObjects](DataObject) of this type are stored in
    fn folder_name() -> &'static str where Self : Sized;
    /// The [TrustPolicy] for this resource type, which determines what sources are allowed to load it.
    fn trust_policy() -> TrustPolicy;
    /// Determines whether or not a given file should be loaded while iterating through a package.
    fn want_file(pathname : &str) -> bool where Self : Sized;
    /// A constructor that returns a new [DataObject] of this type given a path and a [Source] object.
    fn from_package_source(source : &mut Box<dyn Source>, package_name : &str, pathname : &str) -> Result<Self, DataError> where Self : Sized;
    /// A function that writes the object to the given [Source] to save its data
    #[allow(unused_variables)]
    fn write(&mut self, package_name : &str, pathname : &str, source : &mut Box<dyn Source>) -> Result<(), DataError> { Err(DataError::NotSupported) }
    /// Implement this to support "generations" for detecting when the data for a given path is changed.
    fn generation(&self) -> u64 { 0 }
    /// Implement this to support "generations" for detecting when the data for a given path is changed.
    #[allow(unused_variables)]
    fn set_generation(&mut self, generation : u64) {}
}

/// Storage that allows lookup and access of [DataObjects](DataObject) of a given type
pub struct DataStore<T : DataObject + 'static>
{
    source_manager : Rc<RefCell<SourceManager>>,
    unloaded_packages : HashMap<String, HashMap<String, DataIdGeneration<T>, BuildHasherDefault<FxHasher>>, BuildHasherDefault<FxHasher>>,
    data_list : Vec<Option<T>>,
    data_map : HashMap<String, HashMap<String, DataIdGeneration<T>, BuildHasherDefault<FxHasher>>, BuildHasherDefault<FxHasher>>,
    package_use_tokens : HashMap<String, PackageUseToken<T>, BuildHasherDefault<FxHasher>>,
    next_index : usize,
    prepare_info : Option<Rc<RefCell<PrepareInfo<T>>>>
}

impl<T : DataObject + 'static> DataStore<T>
{
    /// Constructs a new [DataStore] that gets its [Sources](source::Source) from the given
    ///  [SourceManager]
    pub fn new(source_manager : Rc<RefCell<SourceManager>>) -> DataStore<T>
    {
        DataStore::<T>
        {
            source_manager,
            unloaded_packages : HashMap::with_hasher(BuildHasherDefault::<FxHasher>::default()),
            data_list : Vec::new(),
            data_map : HashMap::with_hasher(BuildHasherDefault::<FxHasher>::default()),
            package_use_tokens : HashMap::with_hasher(BuildHasherDefault::<FxHasher>::default()),
            next_index : 1,
            prepare_info : None
        }
    }

    /// Loads the package by the given name if it is not already loaded.
    pub fn load_package(&mut self, package_name : &str) -> Result<DataStoreOk, DataError>
    {
        if package_name.find('/').is_some()
        {
            return Err(DataError::BadName);
        }

        if self.package_loaded(package_name)
        {
            return Ok(DataStoreOk::AlreadyLoaded);
        }

        let load_start = Instant::now();

        if let Some(mut source) = self.source_manager.borrow_mut().package_source(package_name)
        {
            if !source.has_package(package_name)
            {
                return Err(DataError::PackageNotFound);
            }

            if T::trust_policy() == TrustPolicy::TrustRequired
                && source.trust_level(package_name) != TrustLevel::TrustedSource
            {
                return Err(DataError::SourceNotTrusted);
            }

            let type_folder = T::folder_name();

            if type_folder.find('/').is_some()
            {
                return Err(DataError::BadName);
            }

            let iter : Vec<Result<String, PackageError>> = source.iter_entries(package_name, type_folder).collect();

            let mut loaded_items : Vec<LoadedData<T>> = Vec::new();
            let mut new_index = self.next_index;

            if let Some(old_package_map) = self.unloaded_packages.get(package_name)
            {
                self.data_map.insert(package_name.to_string(), old_package_map.clone());
            }
            self.unloaded_packages.remove(package_name);

            let package_map = self.data_map.entry(package_name.to_string()).or_insert(HashMap::with_hasher(BuildHasherDefault::<FxHasher>::default()));

            for entry in iter
            {
                let pathname = entry.map_err(|error| DataError::PackageSourceError(error))?;

                if pathname.starts_with('/') || pathname.ends_with('/')
                {
                    return Err(DataError::BadName);
                }

                if !T::want_file(&pathname)
                {
                    continue;
                }

                let full_pathname = format!("{}/{}", type_folder, pathname);

                let mut data_object = T::from_package_source(&mut source, &package_name, &full_pathname)?;
                let index_gen : DataIdGeneration<T>;

                if let Some(entry) = package_map.get(&pathname)
                {
                    let mut generation = entry.generation + 1;

                    if generation == 0
                    {
                        warn!("Generation overflow for package item {}!", pathname);
                        generation = 1;
                    }

                    index_gen = DataIdGeneration
                    {
                        id: entry.id,
                        generation
                    };
                }
                else
                {
                    index_gen = DataIdGeneration
                    {
                        id: DataId::new(new_index),
                        generation : 1
                    };
                    new_index += 1;
                }

                data_object.set_generation(index_gen.generation);

                loaded_items.push(LoadedData::<T> { data_object, id_gen: index_gen, pathname });
            }

            self.next_index = new_index;

            while self.data_list.len() <= self.next_index
            {
                self.data_list.push(None);
            }

            for item in loaded_items.drain(..)
            {
                self.data_list[item.id_gen.id.index] = Some(item.data_object);
                package_map.insert(item.pathname, item.id_gen.clone());

                if let Some(prepare_info) = &self.prepare_info
                {
                    prepare_info.borrow_mut().to_prepare.insert(item.id_gen.id, PrepareType::FullPrepare);
                }
            }

            if !self.package_use_tokens.contains_key(package_name)
            {
                self.package_use_tokens.insert(package_name.to_string(), PackageUseToken::new(package_name));
            }
        }
        else
        {
            return Err(DataError::PackageNotFound);
        }

        let load_time = (Instant::now() - load_start).as_millis();

        info!("Loaded package \"{}/{}\" in {} ms", package_name, T::folder_name(), load_time);

        Ok(DataStoreOk::Loaded)
    }

    /// Gets the numeric ID of the [DataObject] from the given package at the given pathname.
    pub fn get_id(&self, package_name : &str, pathname : &str) -> Result<DataId<T>, DataError>
    {
        if let Some(package_map) = self.data_map.get(package_name)
        {
            if let Some(id) = package_map.get(pathname)
            {
                return Ok(id.id);
            }
            else
            {
                return Err(DataError::DataNotFound);
            }
        }

        Err(DataError::PackageNotFound)
    }

    /// Gets the numeric ID of the [DataObject] from the given package at the given pathname.
    ///  If the package is not loaded, it will be loaded automatically.
    pub fn get_id_mut(&mut self, package_name : &str, pathname : &str) -> Result<DataId<T>, DataError>
    {
        if let Some(package_map) = self.data_map.get(package_name)
        {
            if let Some(id) = package_map.get(pathname)
            {
                return Ok(id.id);
            }
            else
            {
                return Err(DataError::DataNotFound);
            }
        }
        else
        {
            self.load_package(package_name)?;

            return self.get_id_mut(package_name, pathname);
        }
    }

    /// Returns a reference to the [DataObject] by the given [DataId], if one exists.
    ///  Otherwise returns [None].
    pub fn get(&self, id : DataId<T>) -> Option<&T>
    {
        if id.index >= self.data_list.len()
        {
            return None;
        }

        match &self.data_list[id.index]
        {
            Some(data) => Some(data),
            None => None
        }
    }

    /// Returns a mutable reference to the [DataObject] by the given [DataId], if one exists.
    ///  Otherwise returns [None].
    pub fn get_mut(&mut self, id : DataId<T>) -> Option<&mut T>
    {
        if id.index >= self.data_list.len()
        {
            return None;
        }

        match &mut self.data_list[id.index]
        {
            Some(data) => Some(data),
            None => None
        }
    }

    /// Returns a string list of the names of each [DataObject] in the given package.
    ///  The package will need to have been loaded or this will return an empty list.
    ///  Please do not call this repeatedly.
    pub fn list_package_contents(&self, package_name : &str) -> Vec<String>
    {
        let mut listing : Vec<String> = Vec::new();

        if let Some(package_map) = self.data_map.get(package_name)
        {
            for (name, id_gen) in package_map.iter()
            {
                if self.data_list[id_gen.id.index].is_some()
                {
                    listing.push(name.clone());
                }
            }

            listing.sort();
        }

        listing
    }

    /// Unloads the given package from memory. Any [DataObjects](DataObject) will be dropped,
    ///  but pathname-id mappings will be retained in memory so that existing references will
    ///  not be invalidated. Returns true if the package was loaded.
    pub fn unload_package(&mut self, package_name : &str) -> bool
    {
        if let Some(package_map) = self.data_map.get_mut(package_name)
        {
            for id_gen in package_map.values_mut()
            {
                if let Some(data) = &self.data_list[id_gen.id.index]
                {
                    id_gen.generation = data.generation();
                }

                self.data_list[id_gen.id.index] = None;

                if let Some(prepare_info) = &self.prepare_info
                {
                    prepare_info.borrow_mut().to_prepare.insert(id_gen.id, PrepareType::FullPrepare);
                }
            }

            self.unloaded_packages.insert(package_name.to_string(), package_map.clone());
        }
        else
        {
            return false;
        }

        self.data_map.remove(package_name);

        info!("Unloaded package \"{}/{}\"", package_name, T::folder_name());

        true
    }

    /// Unloads all packages from memory. Any [DataObjects](DataObject) will be dropped,
    ///  but pathname-id mappings will be retained in memory so that existing references will
    ///  not be invalidated. Returns true if the package was loaded.
    pub fn unload_all(&mut self)
    {
        self.unload_all_internal();
        info!("Unloaded all packages for \"{}\"", T::folder_name());
    }

    /// Unloads all packages that do not have an active [PackageUseToken].
    pub fn unload_unused(&mut self)
    {
        self.unload_unused_internal()
    }

    /// Returns true if the given package is loaded.
    pub fn package_loaded(&self, package_name : &str) -> bool
    {
        self.data_map.contains_key(package_name)
    }

    /// Returns a [PackageUseToken] that can be used to keep the given package from being unloaded
    ///  automatically.
    pub fn package_use_token(&self, package_name : &str) -> Option<PackageUseToken<T>>
    {
        self.package_use_tokens.get(package_name).cloned()
    }

    /// Saves the indicated item using the [Source] given by [SourceId].
    pub fn save(&mut self, package_name : &str, pathname : &str, source_id : SourceId) -> Result<(), DataError>
    {
        let id = self.get_id_mut(package_name, pathname)?;
        let data = match &mut self.data_list[id.index]
        {
            Some(data) => { data }
            None => { return Err(DataError::DataNotFound); }
        };

        if let Some(source) = self.source_manager.borrow_mut().source(source_id)
        {
            let type_folder = T::folder_name();
            let full_pathname = format!("{}/{}", type_folder, pathname);

            return data.write(&package_name, &full_pathname, source);
        }
        else
        {
            return Err(DataError::BadSource);
        }
    }

    /// Marks the [DataObject] by the given [DataId] as needing to be "reprepared". Use this,
    ///  for example, when data has been changed and it needs to be reflected in a backend via
    ///  the [PreparedStore]. Returns true if the data was marked for reprepare, otherwise false.
    pub fn reprepare(&mut self, id : DataId<T>) -> bool
    {
        if let Some(prepare_info) = &self.prepare_info
        {
            if id.index >= self.data_list.len()
            {
                return false;
            }

            match &mut self.data_list[id.index]
            {
                Some(_) =>
                {
                    if !prepare_info.borrow_mut().to_prepare.contains_key(&id)
                    {
                        prepare_info.borrow_mut().to_prepare.insert(id, PrepareType::Reprepare);
                        return true;
                    }
                },
                _ => ()
            }
        }

        false
    }

    /// Reloads an already-loaded resource by the given name.
    pub fn reload(&mut self, package_name : &str, pathname : &str) -> Result<(), DataError>
    {
        let id = self.get_id_mut(package_name, pathname)?;

        if let Some(mut source) = self.source_manager.borrow_mut().package_source(package_name)
        {
            let type_folder = T::folder_name();

            if pathname.starts_with('/') || pathname.ends_with('/')
            {
                return Err(DataError::BadName);
            }

            if !T::want_file(&pathname)
            {
                return Err(DataError::DataNotFound);
            }

            let full_pathname = format!("{}/{}", type_folder, pathname);
            let data_object = T::from_package_source(&mut source, &package_name, &full_pathname)?;

            self.data_list[id.index] = Some(data_object);
        }
        else
        {
            return Err(DataError::DataNotFound);
        }

        self.reprepare(id);

        Ok(())
    }

    /// Loads a single [DataObject] from the given source and returns it. This function does not
    ///  index, cache, or track the loaded object in any way. Loading the object through here will
    ///  result in a separate copy in memory even if it was previously loaded through the usual
    ///  package loading mechanism. For most cases you should use `load_package()` or
    ///  `get_id_mut()` instead.
    pub fn load_direct(&self, package_name : &str, pathname : &str, source_id : SourceId) -> Result<T, DataError>
    {
        if package_name.find('/').is_some()
        {
            return Err(DataError::BadName);
        }

        if let Some(mut source) = self.source_manager.borrow_mut().source(source_id)
        {
            if T::trust_policy() == TrustPolicy::TrustRequired
                && source.trust_level(package_name) != TrustLevel::TrustedSource
            {
                return Err(DataError::SourceNotTrusted);
            }

            let type_folder = T::folder_name();

            if type_folder.find('/').is_some()
            {
                return Err(DataError::BadName);
            }

            if pathname.starts_with('/') || pathname.ends_with('/')
            {
                return Err(DataError::BadName);
            }

            let full_pathname = format!("{}/{}", type_folder, pathname);
            return Ok(T::from_package_source(&mut source, &package_name, &full_pathname)?);
        }

        Err(DataError::BadSource)
    }
}

trait DataStoreBase : Downcast
{
    fn unload_unused_internal(&mut self);
    fn unload_all_internal(&mut self);
}
impl_downcast!(DataStoreBase);

impl<T : DataObject + 'static> DataStoreBase for DataStore<T>
{
    fn unload_unused_internal(&mut self)
    {
        let mut to_unload = Vec::new();

        for (package_name, token) in &self.package_use_tokens
        {
            if token.uses() == 0
            {
                to_unload.push(package_name.clone());
            }
        }

        for package in &to_unload
        {
            self.unload_package(package);
        }
    }

    fn unload_all_internal(&mut self)
    {
        for (package_name, package_map) in &self.data_map
        {
            for id in package_map.values()
            {
                if let Some(prepare_info) = &self.prepare_info
                {
                    prepare_info.borrow_mut().to_prepare.insert(id.id, PrepareType::FullPrepare);
                }
            }

            self.unloaded_packages.insert(package_name.to_string(), package_map.clone());
        }

        self.data_map.clear();
        self.data_list.clear();
    }
}

struct NullDataStore {}

impl DataStoreBase for NullDataStore
{
    fn unload_unused_internal(&mut self) {}
    fn unload_all_internal(&mut self) {}
}

struct DataMultistoreLookup
{
    index_to_type_id : Vec<TypeId>,
    type_id_to_index : HashMap<TypeId, usize, BuildHasherDefault<FxHasher>>
}

impl DataMultistoreLookup
{
    #[inline(always)]
    fn lookup(&self, type_id : &TypeId) -> Option<usize>
    {
        self.type_id_to_index.get(type_id).cloned()
    }
}

/// Storage that allows lookup and access of [DataObjects](DataObject) of multiple types by wrapping
///  multiple [DataStores](DataStore).
pub struct DataMultistore
{
    source_manager : Rc<RefCell<SourceManager>>,
    stores : GenericArray<RefCell<Box<dyn DataStoreBase>>, DataStoreTypeMax>,
    lookup : RefCell<DataMultistoreLookup>,
    locked : bool
}

impl DataMultistore
{
    /// Constructs a new [DataMultistore] that gets its [Sources](source::Source) from the given
    ///  [SourceManager]
    pub fn new(source_manager : Rc<RefCell<SourceManager>>) -> DataMultistore
    {
        let stores = GenericArray::<RefCell<Box<dyn DataStoreBase>>, DataStoreTypeMax>::generate(|_| RefCell::new(Box::new(NullDataStore {})));

        let lookup = DataMultistoreLookup
        {
            index_to_type_id : Vec::new(),
            type_id_to_index : HashMap::with_hasher(BuildHasherDefault::<FxHasher>::default())
        };

        DataMultistore
        {
            source_manager,
            stores,
            lookup : RefCell::new(lookup),
            locked : false
        }
    }

    /// Informs the [DataMultistore] of a new [DataObject] to manage a [DataStore] for. Returns true
    ///  if successful.
    pub fn register_type<D : DataObject + 'static>(&self) -> bool
    {
        let wanted_type = TypeId::of::<D>();
        let store = DataStore::<D>::new(self.source_manager.clone());

        self.register_boxed_type(wanted_type, Box::new(store))
    }

    fn register_boxed_type(&self, type_id : TypeId, mut store : Box<dyn DataStoreBase>) -> bool
    {
        let mut lookup = self.lookup.try_borrow_mut().expect("DataMultistore data already in use!");

        if !lookup.type_id_to_index.contains_key(&type_id)
        {
            if lookup.index_to_type_id.len() >= self.stores.len()
            {
                panic!("Too many types registered! Limit is {}", self.stores.len());
            }

            let insert_pos = lookup.index_to_type_id.len();

            let mut placeholder = self.stores[insert_pos].try_borrow_mut().expect("Placeholder under use somehow!");
            std::mem::swap(&mut *placeholder, &mut store);

            lookup.type_id_to_index.insert(type_id, insert_pos);
            lookup.index_to_type_id.push(type_id);

            return true;
        }

        false
    }

    /// Sets whether mutable access to types is disallowed.
    pub fn lock(&mut self, locked : bool)
    {
        self.locked = locked;
    }

    /// Returns a reference to the [DataStore] of the given type. Panics if it cannot be retrieved.
    pub fn store<D : DataObject + 'static>(&self) -> Ref<DataStore<D>>
    {
        if let Some(store) = self.try_store()
        {
            return store;
        }

        panic!("DataObject type {} not registered.", std::any::type_name::<D>());
    }

    /// Returns a mutable reference to the [DataStore] of the given type. Panics if it cannot be retrieved.
    pub fn store_mut<D : DataObject + 'static>(&self) -> RefMut<DataStore<D>>
    {
        if let Some(store) = self.try_store_mut()
        {
            return store;
        }

        if self.locked
        {
            panic!("Mutable datastore access is currently restricted.");
        }
        else
        {
            panic!("DataObject type {} not registered.", std::any::type_name::<D>());
        }
    }

    /// Returns a reference to the [DataStore] of the given type. Returns None if it cannot be retrieved.
    pub fn try_store<D : DataObject + 'static>(&self) -> Option<Ref<DataStore<D>>>
    {
        let wanted_type = TypeId::of::<D>();
        let mut index_opt = self.lookup.try_borrow().expect("DataMultistore data already in use!").lookup(&wanted_type);

        if index_opt.is_none()
        {
            self.register_type::<D>();
            index_opt = self.lookup.try_borrow().expect("DataMultistore data already in use!").lookup(&wanted_type);
        }

        if let Some(index) = index_opt
        {
            return Some(Ref::map(self.stores[index].borrow(),
                                 |b| b.downcast_ref::<DataStore<D>>().expect("DataMultistore borrow failed!")));
        }

        None
    }

    /// Returns a mutable reference to the [DataStore] of the given type. Returns None if it cannot be retrieved.
    pub fn try_store_mut<D : DataObject + 'static>(&self) -> Option<RefMut<DataStore<D>>>
    {
        if self.locked
        {
            return None;
        }

        let wanted_type = TypeId::of::<D>();
        let mut index_opt = self.lookup.try_borrow().expect("DataMultistore data already in use!").lookup(&wanted_type);

        if index_opt.is_none()
        {
            self.register_type::<D>();
            index_opt = self.lookup.try_borrow().expect("DataMultistore data already in use!").lookup(&wanted_type);
        }

        if let Some(index) = index_opt
        {
            return Some(RefMut::map(self.stores[index].borrow_mut(),
                                    |b| b.downcast_mut::<DataStore<D>>().expect("DataMultistore borrow failed!")));
        }

        None
    }

    /// Unloads all packages from memory. Any [DataObjects](DataObject) will be dropped,
    ///  but pathname-id mappings will be retained in memory so that existing references will
    ///  not be invalidated. Returns true if the package was loaded.
    pub fn unload_all(&self)
    {
        let lookup = self.lookup.try_borrow_mut().expect("DataMultistore data already in use!");

        for index in (0..lookup.index_to_type_id.len()).rev()
        {
            self.stores[index].borrow_mut().unload_all_internal();
        }

        info!("Unloaded all packages");
    }

    /// Unloads all packages that do not have an active [PackageUseToken].
    pub fn unload_unused(&self)
    {
        let lookup = self.lookup.try_borrow_mut().expect("DataMultistore data already in use!");

        for index in (0..lookup.index_to_type_id.len()).rev()
        {
            self.stores[index].borrow_mut().unload_unused_internal();
        }
    }

    /// Returns the [SourceId] of the [Source] that has the given package, going in
    ///  reverse order of when they were added. If no suitable [Source] is available,
    ///  returns a "null" ID.
    pub fn package_source_id(&self, package_name : &str) -> SourceId
    {
        let source_manager = self.source_manager.borrow_mut();

        source_manager.package_source_id(package_name)
    }
}

#[derive(Debug, Eq, PartialEq)]
enum PrepareType
{
    FullPrepare,
    Reprepare
}

struct PrepareInfo<T : DataObject + 'static>
{
    to_prepare : HashMap<DataId<T>, PrepareType, BuildHasherDefault<FxHasher>>,
}

impl<T : DataObject + 'static> PrepareInfo<T>
{
    fn new() -> PrepareInfo<T>
    {
        PrepareInfo
        {
            to_prepare : HashMap::with_hasher(BuildHasherDefault::<FxHasher>::default())
        }
    }
}

/// Used with [PreparedStore], this allows the definition of behavior when initializing resources
///  with a backend. See [PreparedStore] for more information.
pub trait DataPreparer<T : DataObject + 'static, U>
{
    /// Called when data has been loaded but not "prepared" yet. You can use this, for example,
    ///  to load textures onto the GPU.
    fn prepare(&mut self, data : &mut T, id : DataId<T>) -> U;
    /// Called when data has been unloaded recently and should be cleared in the backend. For example,
    ///  use this to unload textures from GPU memory.
    fn unprepare(&mut self, prepared : &mut U, id : DataId<T>);
    /// Called when data is already loaded but needs to be "reprepared". You can use this, for example,
    ///  to update existing textures on the GPU.
    fn reprepare(&mut self, data : &mut T, prepared : &mut U, id : DataId<T>)
    {
        self.unprepare(prepared, id);
        *prepared = self.prepare(data, id);
    }
}

/// An optional companion to a [DataStore]. If you have data that needs initialization with a backend
///  (for example, textures you need to upload to a GPU), [PreparedStore] provides a way to handle this
///  in a two-step process that is borrow-checker-friendly.
///
/// Template parameter `T` is the type of the source DataObject, and template parameter `U` is the prepared
///  data. This system allows you to separate logical, backend-agnostic data from backend-specific resources.
///  For graphics, you might use `T` to represent the raw image data and metadata such as width and height,
///  and use `U` to represent a handle to the texture in VRAM.
///
/// In order to function, you should call [PreparedStore::sync()] once per frame before needing to use the resources in
///  question in your backend. This will "prepare" or "unprepare" any resources as needed.
///
/// The [PreparedStore] instance must be associated with a [DataStore]. Only one [PreparedStore] can be associated
///  with any given [DataStore].
pub struct PreparedStore<T : DataObject + 'static, U>
{
    data_list : Vec<Option<U>>,
    data_preparer : Box<dyn DataPreparer<T, U>>,
    prepare_info : Rc<RefCell<PrepareInfo<T>>>
}

impl<T : DataObject + 'static, U> PreparedStore<T, U>
{
    /// Creates a new [PreparedStore] that holds prepared versions `U` of [DataObjects](DataObject) `T`, and
    ///  associates with the given [DataStore].
    pub fn new(data_store : &mut DataStore<T>, data_preparer : Box<dyn DataPreparer<T, U>>) -> Result<PreparedStore<T, U>, PreparedStoreError>
    {
        if data_store.prepare_info.is_some()
        {
            return Err(PreparedStoreError::AlreadyConnected);
        }
        
        let prepare_info = Rc::new(RefCell::new(PrepareInfo::new()));
        data_store.prepare_info = Some(prepare_info.clone());
        
        Ok(PreparedStore::<T, U>
        {
            data_list : Vec::new(),
            data_preparer,
            prepare_info
        })
    }
    
    /// Returns a reference to the prepared data by the given [DataId], if one exists.
    ///  Otherwise returns [None].
    pub fn get(&self, id : DataId<T>) -> Option<&U>
    {
        if id.index >= self.data_list.len()
        {
            return None;
        }
        
        match &self.data_list[id.index]
        {
            Some(data) => Some(data),
            None => None
        }
    }
    
    /// Returns a mutable reference to the prepared data by the given [DataId], if one exists.
    ///  Otherwise returns [None].
    pub fn get_mut(&mut self, id : DataId<T>) -> Option<&mut U>
    {
        if id.index >= self.data_list.len()
        {
            return None;
        }
        
        match &mut self.data_list[id.index]
        {
            Some(data) => Some(data),
            None => None
        }
    }
    
    /// Synchronizes this store with the associated [DataStore]. If any [DataObjects](DataObject) were recently
    ///  loaded, they will be prepared by calling [DataPreparer::prepare()]. If any were recently unloaded,
    ///  they will be "unprepared" (unloaded from the backend) by calling [DataPreparer::unprepare()].
    pub fn sync(&mut self, data_store : &mut DataStore<T>) -> Result<(), PreparedStoreError>
    {
        if let Some(prepare_info) = &mut data_store.prepare_info
        {
            if !Rc::ptr_eq(&prepare_info, &self.prepare_info)
            {
                return Err(PreparedStoreError::WrongDataStore);
            }
        }
        else
        {
            return Err(PreparedStoreError::WrongDataStore);
        }

        for (id, prepare_type) in &self.prepare_info.borrow_mut().to_prepare
        {
            if let Some(data) = data_store.get_mut(*id)
            {
                // TODO: Make faster
                while self.data_list.len() <= id.index
                {
                    self.data_list.push(None);
                }

                let mut existing = false;

                if let Some(prepared) = &mut self.data_list[id.index]
                {
                    match *prepare_type
                    {
                        PrepareType::FullPrepare =>
                        {
                            // Force unprepare() and then prepare() later
                            self.data_preparer.unprepare(prepared, *id);
                        },
                        PrepareType::Reprepare =>
                        {
                            // Reprepare, don't do full prepare()
                            self.data_preparer.reprepare(data, prepared, *id);
                            existing = true;
                        }
                    }
                }

                if !existing
                {
                    self.data_list[id.index] = Some(self.data_preparer.prepare(data, *id));
                }
            }
            else
            {
                if id.index < self.data_list.len()
                {
                    if let Some(prepared) = &mut self.data_list[id.index]
                    {
                        self.data_preparer.unprepare(prepared, *id);
                    }

                    self.data_list[id.index] = None;
                }
            }
        }

        self.prepare_info.borrow_mut().to_prepare.clear();
        
        Ok(())
    }
}