use std::error::Error;
use std::fmt::Debug;
use std::future::Future;
#[cfg(feature = "object-store")]
mod acquiring;
#[cfg(feature = "object-store")]
pub use acquiring::AcquiringReader;
use anybytes::Bytes;
use crate::blob::encodings::UnknownBlob;
use crate::blob::{BlobEncoding, IntoBlob, TryFromBlob};
use crate::collection::{CollectionRead, CollectionRecord, CollectionStore};
use crate::inline::encodings::hash::Handle;
use crate::inline::{Inline, InlineEncoding};
use crate::repo::{
BlobInfo, BlobMetadata, BlobStoreForget, BlobStoreGet, BlobStoreList, BlobStoreMeta,
BlobStorePut, SnapshotSource, StoreChanges, StoreSnapshot,
};
#[cfg(feature = "object-store")]
use crate::repo::{BlobChildren, StorageClose};
pub trait AsyncBlobStoreGet {
type GetError<E: Error + Send + Sync + 'static>: Error + Send + Sync + 'static;
fn get<T, S>(
&self,
handle: Inline<Handle<S>>,
) -> impl Future<Output = Result<T, Self::GetError<<T as TryFromBlob<S>>::Error>>> + Send
where
S: BlobEncoding + 'static,
T: TryFromBlob<S>,
Handle<S>: InlineEncoding;
}
pub trait AsyncBlobStoreAcquire {
type AcquireError: Error + Send + Sync + 'static;
fn acquire(
&mut self,
handle: Inline<Handle<UnknownBlob>>,
) -> impl Future<Output = Result<Option<Bytes>, Self::AcquireError>> + Send;
fn acquires_remotely(&self) -> bool {
false
}
}
impl<S> AsyncBlobStoreAcquire for &mut S
where
S: AsyncBlobStoreAcquire + ?Sized,
{
type AcquireError = S::AcquireError;
fn acquire(
&mut self,
handle: Inline<Handle<UnknownBlob>>,
) -> impl Future<Output = Result<Option<Bytes>, Self::AcquireError>> + Send {
(**self).acquire(handle)
}
fn acquires_remotely(&self) -> bool {
(**self).acquires_remotely()
}
}
#[derive(Debug)]
pub struct ResidentBlobAcquireError {
operation: &'static str,
source: Box<dyn Error + Send + Sync>,
}
impl ResidentBlobAcquireError {
fn new(operation: &'static str, source: impl Error + Send + Sync + 'static) -> Self {
Self {
operation,
source: Box::new(source),
}
}
}
impl std::fmt::Display for ResidentBlobAcquireError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
formatter,
"failed to {operation} while acquiring a resident blob: {source}",
operation = self.operation,
source = self.source
)
}
}
impl Error for ResidentBlobAcquireError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
Some(self.source.as_ref())
}
}
fn acquire_resident<S>(
store: &mut S,
handle: Inline<Handle<UnknownBlob>>,
) -> Result<Option<Bytes>, ResidentBlobAcquireError>
where
S: SnapshotSource,
S::Snapshot: BlobStoreGet + BlobStoreList,
{
let snapshot = store
.snapshot()
.map_err(|error| ResidentBlobAcquireError::new("freeze a local snapshot", error))?;
if !snapshot
.contains_blob(handle)
.map_err(|error| ResidentBlobAcquireError::new("inspect local residency", error))?
{
return Ok(None);
}
snapshot
.get::<Bytes, UnknownBlob>(handle)
.map(Some)
.map_err(|error| ResidentBlobAcquireError::new("read validated local bytes", error))
}
macro_rules! impl_resident_blob_acquire {
($store:ty) => {
impl AsyncBlobStoreAcquire for $store {
type AcquireError = ResidentBlobAcquireError;
fn acquire(
&mut self,
handle: Inline<Handle<UnknownBlob>>,
) -> impl Future<Output = Result<Option<Bytes>, Self::AcquireError>> + Send {
std::future::ready(acquire_resident(self, handle))
}
}
};
}
impl_resident_blob_acquire!(crate::repo::memoryrepo::MemoryRepo);
impl_resident_blob_acquire!(crate::repo::pile::Pile);
impl_resident_blob_acquire!(crate::repo::yard::Yard);
impl<B, R> AsyncBlobStoreAcquire for crate::repo::hybridstore::HybridStore<B, R>
where
B: AsyncBlobStoreAcquire,
{
type AcquireError = B::AcquireError;
fn acquire(
&mut self,
handle: Inline<Handle<UnknownBlob>>,
) -> impl Future<Output = Result<Option<Bytes>, Self::AcquireError>> + Send {
self.blobs.acquire(handle)
}
fn acquires_remotely(&self) -> bool {
self.blobs.acquires_remotely()
}
}
pub trait AsyncBlobStorePut {
type PutError: Error + Send + Sync + 'static;
fn put<S, T>(
&mut self,
item: T,
) -> impl Future<Output = Result<Inline<Handle<S>>, Self::PutError>> + Send
where
S: BlobEncoding + 'static,
T: IntoBlob<S>,
Handle<S>: InlineEncoding;
}
pub trait AsyncBlobStoreList {
type Err: Error + Debug + Send + Sync + 'static;
fn blobs(&self) -> impl Future<Output = Vec<Result<BlobInfo, Self::Err>>> + Send;
}
pub trait AsyncSnapshotSource {
type Snapshot: StoreSnapshot;
type SnapshotError: Error + Debug + Send + Sync + 'static;
fn snapshot(
&mut self,
) -> impl Future<Output = Result<Self::Snapshot, Self::SnapshotError>> + Send;
}
pub trait AsyncBlobStore:
AsyncBlobStorePut + AsyncSnapshotSource<Snapshot: AsyncBlobStoreGet + AsyncBlobStoreList>
{
}
impl<S> AsyncBlobStore for S
where
S: AsyncBlobStorePut + AsyncSnapshotSource,
S::Snapshot: AsyncBlobStoreGet + AsyncBlobStoreList,
{
}
pub trait AsyncCollectionRead {
type RecordsError: Error + Debug + Send + Sync + 'static;
fn records(
&self,
) -> impl Future<Output = Result<Vec<CollectionRecord>, Self::RecordsError>> + Send;
fn collections(
&self,
) -> impl Future<Output = Result<Vec<crate::collection::CollectionHandle>, Self::RecordsError>> + Send;
}
pub trait AsyncCollectionStore {
type InsertError: Error + Debug + Send + Sync + 'static;
fn insert(
&mut self,
record: CollectionRecord,
) -> impl Future<Output = Result<(), Self::InsertError>> + Send;
}
pub trait AsyncBlobStoreMeta {
type MetaError: Error + Send + Sync + 'static;
fn metadata<S>(
&self,
handle: Inline<Handle<S>>,
) -> impl Future<Output = Result<Option<BlobMetadata>, Self::MetaError>> + Send
where
S: BlobEncoding + 'static,
Handle<S>: InlineEncoding;
}
pub trait AsyncBlobStoreForget {
type ForgetError: Error + Send + Sync + 'static;
fn forget<S>(
&mut self,
handle: Inline<Handle<S>>,
) -> impl Future<Output = Result<(), Self::ForgetError>> + Send
where
S: BlobEncoding + 'static,
Handle<S>: InlineEncoding;
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct SyncAsAsync<S>(pub S);
impl<S> SyncAsAsync<S> {
pub fn new(store: S) -> Self {
Self(store)
}
pub fn into_inner(self) -> S {
self.0
}
}
impl<S> StoreSnapshot for SyncAsAsync<S>
where
S: StoreSnapshot,
{
fn changes_since(&self, previous: &Self) -> StoreChanges {
self.0.changes_since(&previous.0)
}
}
impl<S> AsyncBlobStoreGet for SyncAsAsync<S>
where
S: BlobStoreGet + Sync,
{
type GetError<E: Error + Send + Sync + 'static> = S::GetError<E>;
fn get<T, Sch>(
&self,
handle: Inline<Handle<Sch>>,
) -> impl Future<Output = Result<T, Self::GetError<<T as TryFromBlob<Sch>>::Error>>> + Send
where
Sch: BlobEncoding + 'static,
T: TryFromBlob<Sch>,
Handle<Sch>: InlineEncoding,
{
let raw = handle.raw;
async move { self.0.get::<T, Sch>(Inline::new(raw)) }
}
}
impl<S> AsyncBlobStorePut for SyncAsAsync<S>
where
S: BlobStorePut + Send,
{
type PutError = S::PutError;
fn put<Sch, T>(
&mut self,
item: T,
) -> impl Future<Output = Result<Inline<Handle<Sch>>, Self::PutError>> + Send
where
Sch: BlobEncoding + 'static,
T: IntoBlob<Sch>,
Handle<Sch>: InlineEncoding,
{
let blob: crate::blob::Blob<Sch> = item.to_blob();
let raw = blob.get_handle().raw;
let bytes = blob.bytes;
async move {
self.0
.put::<Sch, crate::blob::Blob<Sch>>(crate::blob::Blob::new(bytes))
.map(|_| Inline::new(raw))
}
}
}
impl<S> AsyncBlobStoreList for SyncAsAsync<S>
where
S: BlobStoreList + Sync,
{
type Err = S::Err;
fn blobs(&self) -> impl Future<Output = Vec<Result<BlobInfo, Self::Err>>> + Send {
async move { self.0.blobs().collect() }
}
}
impl<S> AsyncSnapshotSource for SyncAsAsync<S>
where
S: SnapshotSource + Send,
{
type Snapshot = SyncAsAsync<S::Snapshot>;
type SnapshotError = S::SnapshotError;
fn snapshot(
&mut self,
) -> impl Future<Output = Result<Self::Snapshot, Self::SnapshotError>> + Send {
async move { self.0.snapshot().map(SyncAsAsync) }
}
}
impl<S> AsyncCollectionRead for SyncAsAsync<S>
where
S: CollectionRead + Sync,
{
type RecordsError = S::RecordsError;
fn records(
&self,
) -> impl Future<Output = Result<Vec<CollectionRecord>, Self::RecordsError>> + Send {
async move { self.0.records()?.collect() }
}
fn collections(
&self,
) -> impl Future<Output = Result<Vec<crate::collection::CollectionHandle>, Self::RecordsError>> + Send
{
async move { self.0.collections() }
}
}
impl<S> AsyncCollectionStore for SyncAsAsync<S>
where
S: CollectionStore + Send,
{
type InsertError = S::InsertError;
fn insert(
&mut self,
record: CollectionRecord,
) -> impl Future<Output = Result<(), Self::InsertError>> + Send {
async move { self.0.insert(record) }
}
}
impl<S> AsyncBlobStoreMeta for SyncAsAsync<S>
where
S: BlobStoreMeta + Sync,
{
type MetaError = S::MetaError;
fn metadata<Sch>(
&self,
handle: Inline<Handle<Sch>>,
) -> impl Future<Output = Result<Option<BlobMetadata>, Self::MetaError>> + Send
where
Sch: BlobEncoding + 'static,
Handle<Sch>: InlineEncoding,
{
let raw = handle.raw;
async move { self.0.metadata::<Sch>(Inline::new(raw)) }
}
}
impl<S> AsyncBlobStoreForget for SyncAsAsync<S>
where
S: BlobStoreForget + Send,
{
type ForgetError = S::ForgetError;
fn forget<Sch>(
&mut self,
handle: Inline<Handle<Sch>>,
) -> impl Future<Output = Result<(), Self::ForgetError>> + Send
where
Sch: BlobEncoding + 'static,
Handle<Sch>: InlineEncoding,
{
let raw = handle.raw;
async move { self.0.forget::<Sch>(Inline::new(raw)) }
}
}
#[cfg(feature = "object-store")]
pub struct Blocking<A> {
inner: A,
rt: std::sync::Arc<tokio::runtime::Runtime>,
}
#[cfg(feature = "object-store")]
impl<A: Clone> Clone for Blocking<A> {
fn clone(&self) -> Self {
Self {
inner: self.inner.clone(),
rt: self.rt.clone(),
}
}
}
#[cfg(feature = "object-store")]
impl<A: std::fmt::Debug> std::fmt::Debug for Blocking<A> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Blocking")
.field("inner", &self.inner)
.finish()
}
}
#[cfg(feature = "object-store")]
impl<A> StoreSnapshot for Blocking<A>
where
A: StoreSnapshot,
{
fn changes_since(&self, previous: &Self) -> StoreChanges {
self.inner.changes_since(&previous.inner)
}
}
#[cfg(feature = "object-store")]
impl<A> Blocking<A> {
pub fn new(inner: A) -> std::io::Result<Self> {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
Ok(Self {
inner,
rt: std::sync::Arc::new(rt),
})
}
pub fn with_runtime(inner: A, rt: std::sync::Arc<tokio::runtime::Runtime>) -> Self {
Self { inner, rt }
}
pub fn into_inner(self) -> A {
self.inner
}
}
#[cfg(feature = "object-store")]
impl<A> SnapshotSource for Blocking<A>
where
A: AsyncSnapshotSource,
{
type Snapshot = Blocking<A::Snapshot>;
type SnapshotError = A::SnapshotError;
fn snapshot(&mut self) -> Result<Self::Snapshot, Self::SnapshotError> {
let snapshot = self.rt.block_on(self.inner.snapshot())?;
Ok(Blocking {
inner: snapshot,
rt: self.rt.clone(),
})
}
}
#[cfg(feature = "object-store")]
impl<A: AsyncBlobStoreGet> BlobStoreGet for Blocking<A> {
type GetError<E: Error + Send + Sync + 'static> = A::GetError<E>;
fn get<T, S>(
&self,
handle: Inline<Handle<S>>,
) -> Result<T, Self::GetError<<T as TryFromBlob<S>>::Error>>
where
S: BlobEncoding + 'static,
T: TryFromBlob<S>,
Handle<S>: InlineEncoding,
{
self.rt.block_on(self.inner.get::<T, S>(handle))
}
}
#[cfg(feature = "object-store")]
impl<A: AsyncBlobStoreList> BlobStoreList for Blocking<A> {
type Iter<'a>
= std::vec::IntoIter<Result<BlobInfo, A::Err>>
where
A: 'a;
type Err = A::Err;
fn blobs<'a>(&'a self) -> Self::Iter<'a> {
self.rt.block_on(self.inner.blobs()).into_iter()
}
}
#[cfg(feature = "object-store")]
impl<A: AsyncBlobStorePut> BlobStorePut for Blocking<A> {
type PutError = A::PutError;
fn put<S, T>(&mut self, item: T) -> Result<Inline<Handle<S>>, Self::PutError>
where
S: BlobEncoding + 'static,
T: IntoBlob<S>,
Handle<S>: InlineEncoding,
{
self.rt.block_on(self.inner.put::<S, T>(item))
}
}
#[cfg(feature = "object-store")]
impl<A: AsyncCollectionRead> CollectionRead for Blocking<A> {
type RecordsError = A::RecordsError;
type RecordIter<'a>
= std::vec::IntoIter<Result<CollectionRecord, A::RecordsError>>
where
A: 'a;
fn records<'a>(&'a self) -> Result<Self::RecordIter<'a>, Self::RecordsError> {
self.rt
.block_on(self.inner.records())
.map(|records| records.into_iter().map(Ok).collect::<Vec<_>>().into_iter())
}
fn collections(&self) -> Result<Vec<crate::collection::CollectionHandle>, Self::RecordsError> {
self.rt.block_on(self.inner.collections())
}
}
#[cfg(feature = "object-store")]
impl<A: AsyncCollectionStore> CollectionStore for Blocking<A> {
type InsertError = A::InsertError;
fn insert(&mut self, record: CollectionRecord) -> Result<(), Self::InsertError> {
self.rt.block_on(self.inner.insert(record))
}
}
#[cfg(feature = "object-store")]
impl<A: AsyncBlobStoreMeta> BlobStoreMeta for Blocking<A> {
type MetaError = A::MetaError;
fn metadata<S>(
&self,
handle: Inline<Handle<S>>,
) -> Result<Option<BlobMetadata>, Self::MetaError>
where
S: BlobEncoding + 'static,
Handle<S>: InlineEncoding,
{
self.rt.block_on(self.inner.metadata::<S>(handle))
}
}
#[cfg(feature = "object-store")]
impl<A: AsyncBlobStoreForget> BlobStoreForget for Blocking<A> {
type ForgetError = A::ForgetError;
fn forget<S>(&mut self, handle: Inline<Handle<S>>) -> Result<(), Self::ForgetError>
where
S: BlobEncoding + 'static,
Handle<S>: InlineEncoding,
{
self.rt.block_on(self.inner.forget::<S>(handle))
}
}
#[cfg(feature = "object-store")]
impl<A: AsyncBlobStoreGet> BlobChildren for Blocking<A> {}
#[cfg(feature = "object-store")]
impl<A: StorageClose> StorageClose for Blocking<A> {
type Error = A::Error;
fn close(self) -> Result<(), Self::Error> {
self.inner.close()
}
}
#[cfg(feature = "object-store")]
impl crate::repo::StorageFlush for Blocking<crate::repo::objectstore::ObjectStoreRemote> {
type Error = std::convert::Infallible;
fn flush(&mut self) -> Result<(), Self::Error> {
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::blob::encodings::simplearchive::SimpleArchive;
use crate::blob::Blob;
use crate::blob::MemoryBlobStore;
use crate::collection::descriptor::{identity_for_tests, named_for_tests};
use crate::collection::{CollectionMerge, CollectionRecord, CollectionStore};
use crate::id::{ExclusiveId, Id};
use crate::macros::entity;
use crate::repo::memoryrepo::MemoryRepo;
use crate::trible::TribleSet;
use futures::executor::block_on;
fn blob(tag: u8) -> Blob<SimpleArchive> {
let e = Id::new([tag; 16]).unwrap();
let ts: TribleSet = entity! {
ExclusiveId::force_ref(&e) @
crate::metadata::tag: Id::new([tag.wrapping_add(3).max(1); 16]).unwrap(),
}
.into();
ts.to_blob()
}
fn collection_record(tag: u8) -> CollectionRecord {
let descriptor = named_for_tests(
&format!("tagged-{tag}"),
Id::new([tag.wrapping_add(1).max(1); 16]).unwrap(),
);
CollectionRecord::Merge(
CollectionMerge::sign(
&ed25519_dalek::SigningKey::from_bytes(&[7; 32]),
identity_for_tests(&descriptor),
[
Inline::new([tag.wrapping_add(3); 32]),
Inline::new([tag.wrapping_add(4); 32]),
],
Inline::new([tag.wrapping_add(5); 32]),
)
.unwrap(),
)
}
#[test]
fn async_adapter_preserves_snapshot_content_changes() {
let mut inner = MemoryBlobStore::new();
let mut store = SyncAsAsync::new(&mut inner);
let before = block_on(store.snapshot()).unwrap();
let unchanged = block_on(store.snapshot()).unwrap();
assert_eq!(before.clone().changes_since(&before), StoreChanges::NONE);
assert_eq!(unchanged.changes_since(&before), StoreChanges::NONE);
let handle = block_on(store.put::<SimpleArchive, _>(blob(1))).unwrap();
let after = block_on(store.snapshot()).unwrap();
assert_eq!(after.changes_since(&before), StoreChanges::BLOBS);
assert!(block_on(before.blobs()).is_empty());
assert_eq!(
block_on(after.blobs())[0].as_ref().unwrap().handle.raw,
handle.raw
);
}
#[test]
fn sync_store_reads_and_writes_through_async_facade() {
let mut store = SyncAsAsync::new(MemoryBlobStore::new());
let b = blob(1);
let handle = block_on(store.put::<SimpleArchive, _>(b.clone())).unwrap();
let snapshot = block_on(store.snapshot()).unwrap();
let got: Blob<SimpleArchive> = block_on(snapshot.get(handle)).unwrap();
assert_eq!(got.bytes, b.bytes);
}
#[test]
fn missing_blob_is_an_error_not_a_hang() {
let mut store = SyncAsAsync::new(MemoryBlobStore::new());
let snapshot = block_on(store.snapshot()).unwrap();
let missing = blob(9).get_handle();
let got = block_on(snapshot.get::<Blob<SimpleArchive>, SimpleArchive>(missing));
assert!(got.is_err(), "absent blob resolves to Err, immediately");
}
#[test]
fn async_list_through_facade() {
let mut store = SyncAsAsync::new(MemoryBlobStore::new());
let h1 = block_on(store.put::<SimpleArchive, _>(blob(1))).unwrap();
let h2 = block_on(store.put::<SimpleArchive, _>(blob(2))).unwrap();
let snapshot = block_on(store.snapshot()).unwrap();
let listed: Vec<_> = block_on(snapshot.blobs())
.into_iter()
.filter_map(Result::ok)
.map(|info| info.handle.raw)
.collect();
assert!(listed.contains(&h1.raw) && listed.contains(&h2.raw));
}
#[test]
fn sync_collection_store_reads_and_writes_through_async_facade() {
let mut store = SyncAsAsync::new(MemoryRepo::default());
let first = collection_record(1);
let second = collection_record(7);
block_on(AsyncCollectionStore::insert(&mut store, second)).unwrap();
block_on(AsyncCollectionStore::insert(&mut store, first)).unwrap();
block_on(AsyncCollectionStore::insert(&mut store, second)).unwrap();
let snapshot = block_on(store.snapshot()).unwrap();
let actual = block_on(AsyncCollectionRead::records(&snapshot)).unwrap();
let mut expected = vec![first, second];
expected.sort_unstable_by_key(CollectionRecord::fingerprint);
assert_eq!(actual, expected);
}
#[cfg(feature = "object-store")]
#[test]
fn blocking_adapter_preserves_snapshot_content_changes() {
let mut store = Blocking::new(SyncAsAsync::new(MemoryRepo::default())).unwrap();
let before = SnapshotSource::snapshot(&mut store).unwrap();
let unchanged = SnapshotSource::snapshot(&mut store).unwrap();
assert_eq!(before.clone().changes_since(&before), StoreChanges::NONE);
assert_eq!(unchanged.changes_since(&before), StoreChanges::NONE);
BlobStorePut::put::<SimpleArchive, _>(&mut store, blob(1)).unwrap();
let after = SnapshotSource::snapshot(&mut store).unwrap();
assert_eq!(after.changes_since(&before), StoreChanges::BLOBS);
assert_eq!(BlobStoreList::blobs(&before).count(), 0);
assert_eq!(BlobStoreList::blobs(&after).count(), 1);
}
#[cfg(feature = "object-store")]
#[test]
fn blocking_lowers_async_collection_store_back_to_sync() {
let mut store = Blocking::new(SyncAsAsync::new(MemoryRepo::default())).unwrap();
let record = collection_record(13);
CollectionStore::insert(&mut store, record).unwrap();
let snapshot = SnapshotSource::snapshot(&mut store).unwrap();
let actual = CollectionRead::records(&snapshot)
.unwrap()
.collect::<Result<Vec<_>, _>>()
.unwrap();
assert_eq!(actual, vec![record]);
}
#[cfg(feature = "object-store")]
#[test]
fn blocking_over_async_roundtrips_as_a_sync_store() {
use crate::repo::{BlobStoreGet, BlobStoreList, BlobStorePut, SnapshotSource};
let mut store = Blocking::new(SyncAsAsync::new(MemoryBlobStore::new())).unwrap();
let b = blob(5);
let h = store.put::<SimpleArchive, _>(b.clone()).unwrap();
let snapshot = store.snapshot().unwrap();
let got: Blob<SimpleArchive> = snapshot.get(h).unwrap();
assert_eq!(got.bytes, b.bytes);
let listed: Vec<_> = snapshot
.blobs()
.filter_map(Result::ok)
.map(|info| info.handle.raw)
.collect();
assert!(listed.contains(&h.raw));
}
fn _assert_send<F: Send>(_: F) {}
#[allow(dead_code)]
fn _send_proof(store: &mut SyncAsAsync<MemoryBlobStore>) {
_assert_send(store.put::<SimpleArchive, _>(blob(2)));
_assert_send(store.snapshot());
}
}