ghee 0.6.1

#[macro_use]
extern crate lazy_static;

pub mod cmd;
pub mod paths;
mod tableinfo;
#[cfg(test)]
mod test_support;
pub mod walk;

pub use tableinfo::*;

use std::{
    collections::BTreeMap,
    fs::File,
    path::{Path, PathBuf},
};

use ghee_lang::{parse_value, parse_xattr, Key, Namespace, Predicate, Record, Value, Xattr};

use anyhow::Result;
use path_absolutize::Absolutize;
use thiserror::Error;
use walkdir::{DirEntry, WalkDir};
use xdg::BaseDirectories;

/// Uppercase the first character in a string
/// https://stackoverflow.com/a/38406885/5374919
fn uppercase_first(s: &str) -> String {
    let mut c = s.chars();
    match c.next() {
        None => String::new(),
        Some(f) => f.to_uppercase().collect::<String>() + c.as_str(),
    }
}

const UNINDEXED_PREDICATE_PENALTY: usize = 1000;

#[derive(Error, Debug)]
pub enum GheeErr {
    #[error("Key not found at {0}")]
    KeyNotFound(PathBuf),
    #[error("Path {0:?} wasn't absolute, but needed to be")]
    PathNotAbsolute(PathBuf),
    #[error("Path {0:?} wasn't relative, but needed to be")]
    PathNotRelative(PathBuf),
    #[error("Value in path was not a UTF-8 string")]
    PathValueNotUtf8,
    #[error("A table path was not actually a directory, therefore doesn't represent a table")]
    TablePathWasNotDirectory,
    #[error("Key must not be empty for table initialization")]
    WontInitializeToEmptyKey,
}

lazy_static! {
    pub static ref PKG_NAME: String = env!("CARGO_PKG_NAME").to_string();
    pub static ref APP_NAME: String = uppercase_first(PKG_NAME.as_str());

    static ref XATTR_GHEE: Xattr= Xattr::new(Namespace::User, "ghee");

    static ref XATTR_GHEE_V0: Xattr = XATTR_GHEE.sub("v0");

    /// Xattr representing lower bound of user.ghee range
    pub static ref XATTR_GHEE_LOWER: Xattr = Xattr::new(Namespace::User, "ghee.");

    /// Xattr representing upper bound of user.ghee range
    pub static ref XATTR_GHEE_UPPER: Xattr = Xattr::new(Namespace::User, "ghef");

    /// Xattr representing the most recent BTRFS snapshot
    ///
    /// These should be traversable backwards by accessing this xattr from each snapshot in the chain
    pub static ref XATTR_HEAD: Xattr = XATTR_GHEE.sub("HEAD");

    /// Xattr representing the message of a snapshot
    pub static ref XATTR_COMMIT_MESSAGE: Xattr = XATTR_GHEE.sub("commit-message");

    /// Where we store everything we know about a table
    pub static ref XATTR_TABLE_INFO: Xattr = XATTR_GHEE.sub("tableinfo");

    pub static ref XDG_DIRS: BaseDirectories =
        xdg::BaseDirectories::with_prefix(APP_NAME.as_str()).unwrap();

    /// The key assigned to bare (uninitialized) directories when they're treated as tables
    pub static ref DEFAULT_KEY: Key = Key::from_string("key0,key1,key2,key3,key4,key5,key6,key7,key8,key9");
}

const HIDDEN_PREFIX: &'static str = ":";

/// Whether a path should be skipped when iterating over records
pub fn is_hidden(entry: &DirEntry) -> bool {
    entry
        .file_name()
        .to_str()
        .map(|s| s.starts_with(HIDDEN_PREFIX))
        .unwrap_or(false)
}

/// The number of records recursively found in a directory
///
/// Ignores nested indexes
pub fn record_count(dir: &PathBuf) -> usize {
    WalkDir::new(dir)
        .into_iter()
        .filter_entry(|e| !is_hidden(e))
        .filter(|e| {
            if let Ok(entry) = e {
                entry.file_type().is_file()
            } else {
                false
            }
        })
        .count()
}

/// For a `path` under a `base_path`, interpret the sub-paths beneath `base_path` as
/// values of the given key; partial key matches are OK, e.g. just `age` on a key
/// of (`age`, `name`).
///
/// If `path` is the same as `base_path`, an empty map is returned since no xattrs
/// can be inferred.
pub fn xattr_values_from_path<P1: AsRef<Path>, P2: AsRef<Path>>(
    key: &Key,
    base_path: P1,
    path: P2,
) -> Result<Record> {
    // If base_path and path are the same, no xattrs can be inferred
    let base_path = base_path.as_ref();
    let path = path.as_ref();
    if base_path == path {
        return Ok(Record::new());
    }

    let base_path = base_path.absolutize().unwrap().to_path_buf();
    let path = path.absolutize().unwrap().to_path_buf();

    let base_len = base_path.components().count();

    // Get just the parts of `path` that go beyond `base_path`
    let parts = path.components().skip(base_len).map(|c| c.as_os_str());

    let mut values = Record::new();
    for (i, part) in parts.enumerate() {
        let bytes: Vec<u8> = part
            .to_str()
            .ok_or(GheeErr::PathValueNotUtf8)?
            .bytes()
            .collect();
        let subkey = &key.subkeys[i];
        let value = parse_value(&bytes).unwrap().1;
        values.insert(subkey.clone(), value);
    }

    Ok(values)
}

/// Get the index which places the predicate xattrs earliest in its primary key order
/// The idea is that this will maximally speed up traversal of records, but this may
/// depend on the cardinality / distribution of the subkey values
pub fn best_index<'a, 'b>(
    indices: &'a BTreeMap<Key, PathBuf>,
    where_: &Vec<Predicate>,
    tie_breaker_key: &'a Key,
) -> (&'a Key, &'a PathBuf) {
    let predicate_xattrs: Vec<Xattr> = where_.iter().map(|pred| pred.xattr.clone()).collect();

    let earliest_subkey_indices: Vec<Option<usize>> = indices
        .keys()
        .map(|key| {
            let x: Option<usize> = predicate_xattrs
                .iter()
                .map(|xattr| key.subkeys.iter().position(|subkey| *subkey == *xattr))
                .reduce(|a, b| {
                    if let Some(a) = a {
                        if let Some(b) = b {
                            Some(a + b)
                        } else {
                            Some(a)
                        }
                    } else {
                        if let Some(b) = b {
                            Some(b)
                        } else {
                            None
                        }
                    }
                })
                .unwrap_or(None);
            x
        })
        .collect();

    if earliest_subkey_indices.iter().all(|idx| idx.is_none()) {
        let path = &indices[tie_breaker_key];
        (tie_breaker_key, path)
    } else {
        indices
            .iter()
            .enumerate()
            .min_by_key(|(idx, (_key, _path))| {
                earliest_subkey_indices[*idx].unwrap_or(UNINDEXED_PREDICATE_PENALTY)
            })
            .unwrap()
            .1
    }
}

// Like xattr::list but parsed into our format
pub fn list_xattrs<P: AsRef<Path>>(path: P) -> Vec<Xattr> {
    let path: &Path = path.as_ref();
    xattr::list(path)
        .unwrap_or_else(|e| panic!("Could not list xattrs on {}: {}", path.display(), e))
        .map(|osstr| osstr.into_string().unwrap().into_bytes())
        .map(|field| {
            parse_xattr(field.as_slice())
                .unwrap_or_else(|e| panic!("Could not parse xattr: {}", e))
                .1
        })
        .collect()
}

#[derive(Error, Debug)]
pub enum XattrValueErr {
    #[error("An IO error occurred while getting the xattr value")]
    IoError(std::io::Error),
    #[error("No xattr value found")]
    NoValue,
    #[error("Couldn't parse value")]
    CantParseValue,
}

/// Get the specified xattr value for the given path;
/// like `xattr::get` but parsed into our format.
pub fn xattr_value<P: AsRef<Path>>(path: P, xattr: &Xattr) -> Result<Value> {
    let path = path.as_ref();
    let raw = xattr::get(path, xattr.to_osstring())
        .map_err(XattrValueErr::IoError)?
        .ok_or(XattrValueErr::NoValue)?;

    let value = parse_value(raw.as_slice())
        .map_err(|_e| XattrValueErr::CantParseValue)?
        .1;

    Ok(value)
}

/// Get all xattr values for the given path;
/// like `xattr::list` and `xattr::get` but parsed into our format.
pub fn xattr_values<P: AsRef<Path>>(path: P) -> Result<Record> {
    let path = path.as_ref();
    let xattrs = list_xattrs(path);
    let mut values = Record::new();

    for xattr in xattrs.into_iter() {
        let value = xattr_value(path, &xattr)?;
        values.insert(xattr, value);
    }

    Ok(values)
}

/** Write file extended attribute values to disk. Creates an empty file at the path if none exists. */
pub fn write_xattr_values<P: AsRef<Path>>(path: P, record: &Record) -> Result<()> {
    let path = path.as_ref();
    if !path.exists() {
        File::create(&path)?;
    }

    debug_assert!(path.exists());
    for (xattr, value) in record {
        xattr::set(&path, xattr.to_osstring(), value.as_bytes().as_slice())?;
    }
    Ok(())
}

#[derive(Error, Debug)]
pub enum GetKeyErr {
    #[error("An IO error occurred while getting the key")]
    IoError(std::io::Error),
    #[error("The key could be retrieved but not deserialized")]
    CantDeserializeJson(serde_json::Error),
    #[error("The key could not be parsed")]
    CantParseKey(String),
}

#[derive(Error, Debug)]
pub enum SetKeyErr {
    #[error("An IO error occurred while setting the key")]
    IoError(std::io::Error),
    #[error("The key could not be serialized as JSON")]
    CantSerializeJson(serde_json::Error),
}

#[derive(Error, Debug)]
pub enum IndexListPushErr {
    #[error("Table info not found at {0}; can only push to initialized tables")]
    TargetTableInfoNotFound(PathBuf),
    #[error("Table info not found at {0}; can only add initialized indices")]
    SourceTableInfoNotFound(PathBuf),
}

/// Adds `index_dir` as an alternate index of the data in `dir`
///
/// This happens assymmetrically; see `declare_indices` for a symmetric version
fn index_list_push<P1: AsRef<Path>, P2: AsRef<Path>>(dir: P1, index_dir: P2) -> Result<()> {
    let dir = dir.as_ref();
    let index_dir = index_dir.as_ref();
    let mut info =
        table_info(dir)?.ok_or_else(|| IndexListPushErr::TargetTableInfoNotFound(dir.into()))?;

    let index_dir_info = table_info(index_dir)?
        .ok_or_else(|| IndexListPushErr::SourceTableInfoNotFound(index_dir.into()))?;

    info.add_index(index_dir_info.key().clone(), index_dir.into());

    set_table_info(dir, &info)
}

/// Marks `dir1` and `dir2` as indices of each other.
pub fn declare_indices<P1: AsRef<Path>, P2: AsRef<Path>>(dir1: P1, dir2: P2) -> Result<()> {
    index_list_push(&dir1, &dir2)?;
    index_list_push(dir2, dir1)
}

/// Mark `dir1`, `dir2`, the indices of `dir1`, and the indices of `dir2`
/// all as indices of each other.
pub fn declare_closure_indices<P1: AsRef<Path>, P2: AsRef<Path>>(dir1: P1, dir2: P2) -> Result<()> {
    let info1 = table_info(&dir1)?
        .ok_or_else(|| IndexListPushErr::TargetTableInfoNotFound(dir1.as_ref().into()))?;
    let info2 = table_info(&dir2)?
        .ok_or_else(|| IndexListPushErr::TargetTableInfoNotFound(dir2.as_ref().into()))?;

    for path1 in info1.indices_abs().values() {
        for path2 in info2.indices_abs().values() {
            declare_indices(&path1, path2)?;
        }
    }

    Ok(())
}

#[cfg(test)]
mod test {
    use ghee_lang::{parse_predicate, parse_xattr, Key, Value};

    use crate::{
        best_index,
        cmd::{idx, init},
        declare_indices, index_list_push, table_info,
        test_support::{Scenario, TempDirAuto},
        write_xattr_values, xattr_values, xattr_values_from_path, Record,
    };

    #[test]
    fn test_xattr_values_from_path() {
        let s = Scenario::new("ghee-test-xattr-values-from-path");

        let path = {
            let mut path = s.dir1.clone();
            path.push("0");
            path
        };

        {
            let values = xattr_values_from_path(&s.key1, &s.dir1, &path).unwrap();

            assert_eq!(values.len(), 1);

            assert_eq!(values[&s.xattr1], Value::Number(0f64));
        }
    }

    #[test]
    fn test_xattr_values_round_trip() {
        let s = Scenario::new("xattr-values-round-trip");
        let record = {
            let mut r = Record::new();

            r.insert(s.xattr1.clone(), Value::String("lmnopqrstuv".to_string()));

            r
        };

        let path = {
            let mut p = s.dir2.clone();

            p.push("blorpity");

            p
        };

        write_xattr_values(&path, &record).unwrap();

        let record2 = xattr_values(&path).unwrap();

        assert_eq!(record, record2);
    }

    #[test]
    fn test_index_list_push() {
        let dir1 = TempDirAuto::new("ghee-test-index-list-push:1");

        let key1 = Key::from_string("name");

        init(&dir1, &key1, false).unwrap();

        {
            let info1 = table_info(&dir1).unwrap().unwrap();
            assert_eq!(info1.key(), &key1);
            assert_eq!(info1.indices_abs().len(), 1);
        }

        let dir2 = TempDirAuto::new("ghee-test-index-list-push:2");

        let key2 = Key::new(vec![parse_xattr(b"state").unwrap().1]);
        init(&dir2, &key2, false).unwrap();

        {
            let info1 = table_info(&dir1).unwrap().unwrap();
            assert_eq!(info1.key(), &key1);
            assert_eq!(info1.indices_abs().len(), 1);

            let info2 = table_info(&dir2).unwrap().unwrap();
            assert_eq!(info2.key(), &key2);
            assert_eq!(info2.indices_abs().len(), 1);
        }

        index_list_push(&dir1, &dir2).unwrap();

        {
            let info1 = table_info(&dir1).unwrap().unwrap();
            assert_eq!(info1.key(), &key1);
            assert_eq!(info1.indices_abs().len(), 2);

            let info2 = table_info(&dir2).unwrap().unwrap();
            assert_eq!(info2.key(), &key2);
            assert_eq!(info2.indices_abs().len(), 1);
        }
    }

    #[test]
    fn test_declare_indices() {
        let dir1 = TempDirAuto::new("ghee-test-index-list-push:1");

        let key1 = Key::from_string("name");

        init(&dir1, &key1, false).unwrap();

        {
            let info1 = table_info(&dir1).unwrap().unwrap();
            assert_eq!(info1.key(), &key1);
            assert_eq!(info1.indices_abs().len(), 1);
        }

        let dir2 = TempDirAuto::new("ghee-test-index-list-push:2");

        let key2 = Key::new(vec![parse_xattr(b"state").unwrap().1]);
        init(&dir2, &key2, false).unwrap();

        {
            let info1 = table_info(&dir1).unwrap().unwrap();
            assert_eq!(info1.key(), &key1);
            assert_eq!(info1.indices_abs().len(), 1);

            let info2 = table_info(&dir2).unwrap().unwrap();
            assert_eq!(info2.key(), &key2);
            assert_eq!(info2.indices_abs().len(), 1);
        }

        declare_indices(&dir1, &dir2).unwrap();

        {
            let info1 = table_info(&dir1).unwrap().unwrap();
            assert_eq!(info1.key(), &key1);
            assert_eq!(info1.indices_abs().len(), 2);

            let info2 = table_info(&dir2).unwrap().unwrap();
            assert_eq!(info2.key(), &key2);
            assert_eq!(info2.indices_abs().len(), 2);
        }
    }

    #[test]
    fn test_best_index() {
        let dir1 = TempDirAuto::new("ghee-test-best-index-dir1");

        let key1 = Key::from_string("test");

        init(&dir1, &key1, false).unwrap();

        let dir2 = TempDirAuto::new("ghee-test-best-index-dir2");

        let key2 = Key::from_string("blah,test");

        idx(&dir1, Some(&dir2), &key2, false).unwrap();

        let info = table_info(&dir1).unwrap().unwrap();
        let indices = info.indices_abs();

        {
            // No predicate; should fall back to tie breaker
            let (best_key, best_path) = best_index(&indices, &vec![], &key1);

            assert_eq!(best_key, &key1);
            assert_eq!(*best_path, *dir1);
        }

        {
            // Predicate "test" makes dir1 best
            let (best_key, best_path) = best_index(
                &indices,
                &vec![parse_predicate(b"test=5").unwrap().1],
                &key1,
            );

            assert_eq!(best_key, &key1);
            assert_eq!(*best_path, *dir1);
        }

        {
            // Predicate "blah" makes dir2 best
            let (best_key, best_path) = best_index(
                &indices,
                &vec![parse_predicate(b"blah=6").unwrap().1],
                &key1,
            );

            assert_eq!(best_key, &key2);
            assert_eq!(*best_path, *dir2);
        }
    }
}