use std::fs::File;
#[allow(unused_imports)] use std::io::ErrorKind;
use std::io::Result;
use std::path::Path;
use std::sync::Arc;
use derivative::Derivative;
use crate::plain::Cache as PlainCache;
use crate::sharded::Cache as ShardedCache;
use crate::Key;
type ConsistencyChecker = Arc<
dyn Fn(&mut File, &mut File) -> Result<()>
+ Sync
+ Send
+ std::panic::RefUnwindSafe
+ std::panic::UnwindSafe,
>;
trait ReadSide:
std::fmt::Debug + Sync + Send + std::panic::RefUnwindSafe + std::panic::UnwindSafe
{
fn get(&self, key: Key) -> Result<Option<File>>;
fn touch(&self, key: Key) -> Result<bool>;
}
impl ReadSide for PlainCache {
fn get(&self, key: Key) -> Result<Option<File>> {
PlainCache::get(self, key.name)
}
fn touch(&self, key: Key) -> Result<bool> {
PlainCache::touch(self, key.name)
}
}
impl ReadSide for ShardedCache {
fn get(&self, key: Key) -> Result<Option<File>> {
ShardedCache::get(self, key)
}
fn touch(&self, key: Key) -> Result<bool> {
ShardedCache::touch(self, key)
}
}
#[derive(Default, Derivative)]
#[derivative(Debug)]
pub struct ReadOnlyCacheBuilder {
stack: Vec<Box<dyn ReadSide>>,
#[derivative(Debug = "ignore")]
consistency_checker: Option<ConsistencyChecker>,
}
#[derive(Clone, Derivative)]
#[derivative(Debug)]
pub struct ReadOnlyCache {
stack: Arc<[Box<dyn ReadSide>]>,
#[derivative(Debug = "ignore")]
consistency_checker: Option<ConsistencyChecker>,
}
impl ReadOnlyCacheBuilder {
pub fn new() -> Self {
Self::default()
}
pub fn consistency_checker(
&mut self,
checker: impl Fn(&mut File, &mut File) -> Result<()>
+ Sync
+ Send
+ std::panic::RefUnwindSafe
+ std::panic::UnwindSafe
+ Sized
+ 'static,
) -> &mut Self {
self.arc_consistency_checker(Some(Arc::new(checker)))
}
pub fn byte_equality_checker(&mut self) -> &mut Self {
self.consistency_checker(crate::byte_equality_checker)
}
pub fn panicking_byte_equality_checker(&mut self) -> &mut Self {
self.consistency_checker(crate::panicking_byte_equality_checker)
}
pub fn clear_consistency_checker(&mut self) -> &mut Self {
self.arc_consistency_checker(None)
}
#[allow(clippy::type_complexity)] pub fn arc_consistency_checker(
&mut self,
checker: Option<
Arc<
dyn Fn(&mut File, &mut File) -> Result<()>
+ Sync
+ Send
+ std::panic::RefUnwindSafe
+ std::panic::UnwindSafe,
>,
>,
) -> &mut Self {
self.consistency_checker = checker;
self
}
pub fn cache(&mut self, path: impl AsRef<Path>, num_shards: usize) -> &mut Self {
if num_shards <= 1 {
self.plain(path)
} else {
self.sharded(path, num_shards)
}
}
pub fn plain(&mut self, path: impl AsRef<Path>) -> &mut Self {
self.stack.push(Box::new(PlainCache::new(
path.as_ref().to_owned(),
usize::MAX,
)));
self
}
pub fn plain_caches<P>(&mut self, paths: impl IntoIterator<Item = P>) -> &mut Self
where
P: AsRef<Path>,
{
for path in paths {
self.plain(path);
}
self
}
pub fn sharded(&mut self, path: impl AsRef<Path>, num_shards: usize) -> &mut Self {
self.stack.push(Box::new(ShardedCache::new(
path.as_ref().to_owned(),
num_shards,
usize::MAX,
)));
self
}
pub fn take(&mut self) -> Self {
std::mem::take(self)
}
pub fn build(self) -> ReadOnlyCache {
ReadOnlyCache::new(self.stack, self.consistency_checker)
}
}
impl Default for ReadOnlyCache {
fn default() -> ReadOnlyCache {
ReadOnlyCache::new(Default::default(), None)
}
}
impl ReadOnlyCache {
fn new(
stack: Vec<Box<dyn ReadSide>>,
consistency_checker: Option<ConsistencyChecker>,
) -> ReadOnlyCache {
ReadOnlyCache {
stack: stack.into_boxed_slice().into(),
consistency_checker,
}
}
pub fn get<'a>(&self, key: impl Into<Key<'a>>) -> Result<Option<File>> {
fn doit(
stack: &[Box<dyn ReadSide>],
checker: &Option<ConsistencyChecker>,
key: Key,
) -> Result<Option<File>> {
use std::io::Seek;
use std::io::SeekFrom;
let mut ret = None;
for cache in stack.iter() {
let mut hit = match cache.get(key)? {
Some(hit) => hit,
None => continue,
};
match checker {
None => return Ok(Some(hit)),
Some(checker) => match ret.as_mut() {
None => ret = Some(hit),
Some(prev) => {
checker(prev, &mut hit)?;
prev.seek(SeekFrom::Start(0))?;
}
},
}
}
Ok(ret)
}
if self.stack.is_empty() {
return Ok(None);
}
doit(&*self.stack, &self.consistency_checker, key.into())
}
pub fn touch<'a>(&self, key: impl Into<Key<'a>>) -> Result<bool> {
fn doit(stack: &[Box<dyn ReadSide>], key: Key) -> Result<bool> {
for cache in stack.iter() {
if cache.touch(key)? {
return Ok(true);
}
}
Ok(false)
}
if self.stack.is_empty() {
return Ok(false);
}
doit(&*self.stack, key.into())
}
}
#[cfg(test)]
mod test {
use std::fs::File;
use std::sync::atomic::AtomicU64;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use crate::plain::Cache as PlainCache;
use crate::sharded::Cache as ShardedCache;
use crate::Key;
use crate::ReadOnlyCache;
use crate::ReadOnlyCacheBuilder;
struct TestKey {
key: String,
}
impl TestKey {
fn new(key: &str) -> TestKey {
TestKey {
key: key.to_string(),
}
}
}
impl<'a> From<&'a TestKey> for Key<'a> {
fn from(x: &'a TestKey) -> Key<'a> {
Key::new(&x.key, 0, 1)
}
}
fn byte_equality_checker(
counter: Arc<AtomicU64>,
) -> impl 'static + Fn(&mut File, &mut File) -> std::io::Result<()> {
move |x: &mut File, y: &mut File| {
counter.fetch_add(1, Ordering::Relaxed);
crate::byte_equality_checker(x, y)
}
}
#[test]
fn empty() {
let ro: ReadOnlyCache = Default::default();
assert!(matches!(ro.get(Key::new("foo", 1, 2)), Ok(None)));
assert!(matches!(ro.touch(Key::new("foo", 1, 2)), Ok(false)));
}
#[test]
fn consistency_checker_success() {
use std::io::Read;
use test_dir::{DirBuilder, FileType, TestDir};
let temp = TestDir::temp()
.create("first", FileType::Dir)
.create("second", FileType::Dir)
.create("first/0", FileType::ZeroFile(2))
.create("second/0", FileType::ZeroFile(2))
.create("first/1", FileType::RandomFile(10))
.create("second/2", FileType::RandomFile(10));
let counter = Arc::new(AtomicU64::new(0));
let ro = ReadOnlyCacheBuilder::new()
.plain(temp.path("first"))
.plain(temp.path("second"))
.consistency_checker(byte_equality_checker(counter.clone()))
.take()
.build();
let mut hit = ro
.get(&TestKey::new("0"))
.expect("must succeed")
.expect("must exist");
assert_eq!(counter.load(Ordering::Relaxed), 1);
let mut contents = Vec::new();
hit.read_to_end(&mut contents).expect("read should succeed");
assert_eq!(contents, "00".as_bytes());
let _ = ro
.get(&TestKey::new("1"))
.expect("must succeed")
.expect("must exist");
assert_eq!(counter.load(Ordering::Relaxed), 1);
let _ = ro
.get(&TestKey::new("2"))
.expect("must succeed")
.expect("must exist");
assert_eq!(counter.load(Ordering::Relaxed), 1);
}
#[test]
fn consistency_checker_failure() {
use test_dir::{DirBuilder, FileType, TestDir};
let temp = TestDir::temp()
.create("first", FileType::Dir)
.create("second", FileType::Dir)
.create("first/0", FileType::ZeroFile(2))
.create("second/0", FileType::ZeroFile(3));
let counter = Arc::new(AtomicU64::new(0));
let ro = ReadOnlyCacheBuilder::new()
.plain(temp.path("first"))
.plain(temp.path("second"))
.consistency_checker(byte_equality_checker(counter))
.take()
.build();
assert!(ro.get(&TestKey::new("0")).is_err());
}
#[test]
fn consistency_checker_silent_failure() {
use test_dir::{DirBuilder, FileType, TestDir};
let temp = TestDir::temp()
.create("first", FileType::Dir)
.create("second", FileType::Dir)
.create("first/0", FileType::ZeroFile(2))
.create("second/0", FileType::ZeroFile(3));
let counter = Arc::new(AtomicU64::new(0));
let ro = ReadOnlyCacheBuilder::new()
.plain(temp.path("first"))
.plain(temp.path("second"))
.consistency_checker(byte_equality_checker(counter.clone()))
.clear_consistency_checker()
.take()
.build();
let _ = ro
.get(&TestKey::new("0"))
.expect("must succeed")
.expect("must exist");
assert_eq!(counter.load(Ordering::Relaxed), 0);
}
#[test]
fn two_plain_caches() {
use test_dir::{DirBuilder, FileType, TestDir};
let temp = TestDir::temp()
.create("first", FileType::Dir)
.create("second", FileType::Dir)
.create("first/0", FileType::ZeroFile(2))
.create("second/1", FileType::ZeroFile(3));
let ro = ReadOnlyCacheBuilder::new()
.plain_caches(["first", "second"].iter().map(|p| temp.path(p)))
.take()
.build();
let _ = ro
.get(&TestKey::new("0"))
.expect("must succeed")
.expect("must exist");
let _ = ro
.get(&TestKey::new("1"))
.expect("must succeed")
.expect("must exist");
assert!(ro.get(&TestKey::new("2")).expect("must succeed").is_none());
}
#[test]
fn test_byte_equality_checker() {
use test_dir::{DirBuilder, FileType, TestDir};
let temp = TestDir::temp()
.create("first", FileType::Dir)
.create("second", FileType::Dir)
.create("first/0", FileType::ZeroFile(2))
.create("second/0", FileType::ZeroFile(3));
let ro = ReadOnlyCacheBuilder::new()
.plain_caches(["first", "second"].iter().map(|p| temp.path(p)))
.byte_equality_checker()
.take()
.build();
assert!(ro.get(&TestKey::new("0")).is_err());
}
#[test]
#[should_panic(expected = "file contents do not match")]
fn test_panicking_byte_equality_checker() {
use test_dir::{DirBuilder, FileType, TestDir};
let temp = TestDir::temp()
.create("first", FileType::Dir)
.create("second", FileType::Dir)
.create("first/0", FileType::ZeroFile(2))
.create("second/0", FileType::ZeroFile(3));
let ro = ReadOnlyCacheBuilder::new()
.plain_caches(["first", "second"].iter().map(|p| temp.path(p)))
.panicking_byte_equality_checker()
.take()
.build();
assert!(ro.get(&TestKey::new("0")).is_ok());
}
#[test]
fn smoke_test() {
use std::io::{Read, Write};
use tempfile::NamedTempFile;
use test_dir::{DirBuilder, FileType, TestDir};
let temp = TestDir::temp()
.create("sharded", FileType::Dir)
.create("plain", FileType::Dir);
{
let cache = ShardedCache::new(temp.path("sharded"), 10, 20);
let tmp = NamedTempFile::new_in(cache.temp_dir(None).expect("temp_dir must succeed"))
.expect("new temp file must succeed");
tmp.as_file()
.write_all(b"sharded")
.expect("write must succeed");
cache
.put(Key::new("a", 0, 1), tmp.path())
.expect("put must succeed");
let tmp2 = NamedTempFile::new_in(cache.temp_dir(None).expect("temp_dir must succeed"))
.expect("new temp file must succeed");
tmp2.as_file()
.write_all(b"sharded2")
.expect("write must succeed");
cache
.put(Key::new("b", 0, 1), tmp2.path())
.expect("put must succeed");
}
{
let cache = PlainCache::new(temp.path("plain"), 10);
let tmp = NamedTempFile::new_in(cache.temp_dir().expect("temp_dir must succeed"))
.expect("new temp file must succeed");
tmp.as_file()
.write_all(b"plain")
.expect("write must succeed");
cache.put("b", tmp.path()).expect("put must succeed");
let tmp2 = NamedTempFile::new_in(cache.temp_dir().expect("temp_dir must succeed"))
.expect("new temp file must succeed");
tmp2.as_file()
.write_all(b"plain2")
.expect("write must succeed");
cache.put("c", tmp2.path()).expect("put must succeed");
}
{
let ro = ReadOnlyCacheBuilder::new()
.sharded(temp.path("sharded"), 10)
.plain(temp.path("plain"))
.take()
.build();
assert!(matches!(ro.get(&TestKey::new("Missing")), Ok(None)));
assert!(matches!(ro.touch(&TestKey::new("Missing")), Ok(false)));
assert!(matches!(ro.touch(&TestKey::new("a")), Ok(true)));
{
let mut a_file = ro
.get(&TestKey::new("a"))
.expect("must succeed")
.expect("must exist");
let mut dst = Vec::new();
a_file.read_to_end(&mut dst).expect("read must succeed");
assert_eq!(&dst, b"sharded");
}
{
let mut b_file = ro
.get(&TestKey::new("b"))
.expect("must succeed")
.expect("must exist");
let mut dst = Vec::new();
b_file.read_to_end(&mut dst).expect("read must succeed");
assert_eq!(&dst, b"sharded2");
}
{
let mut c_file = ro
.get(&TestKey::new("c"))
.expect("must succeed")
.expect("must exist");
let mut dst = Vec::new();
c_file.read_to_end(&mut dst).expect("read must succeed");
assert_eq!(&dst, b"plain2");
}
}
{
let ro = ReadOnlyCacheBuilder::new()
.cache(temp.path("plain"), 1)
.cache(temp.path("sharded"), 10)
.take()
.build();
{
let mut a_file = ro
.get(&TestKey::new("a"))
.expect("must succeed")
.expect("must exist");
let mut dst = Vec::new();
a_file.read_to_end(&mut dst).expect("read must succeed");
assert_eq!(&dst, b"sharded");
}
{
let mut b_file = ro
.get(&TestKey::new("b"))
.expect("must succeed")
.expect("must exist");
let mut dst = Vec::new();
b_file.read_to_end(&mut dst).expect("read must succeed");
assert_eq!(&dst, b"plain");
}
{
let mut c_file = ro
.get(&TestKey::new("c"))
.expect("must succeed")
.expect("must exist");
let mut dst = Vec::new();
c_file.read_to_end(&mut dst).expect("read must succeed");
assert_eq!(&dst, b"plain2");
}
}
}
}