use std::collections::{BTreeSet, HashMap};
use std::convert::Infallible;
use std::error::Error;
use std::fmt;
use crate::blob::encodings::simplearchive::SimpleArchive;
use crate::blob::{Blob, BlobEncoding, Bytes, TryFromBlob};
use crate::inline::encodings::hash::Handle;
use crate::inline::{Inline, InlineEncoding};
use crate::repo::{BlobStoreGet, BlobStoreMeta, StoreChanges, StoreRead, StoreSnapshot};
use crate::trible::Fragment;
use super::observed_store::{DependencyTracker, ObservedStore};
use super::{
CollectionData, CollectionDescriptorError, CollectionEncoding, CollectionHandle,
CollectionRealizationError, Cover, CoverageRead, MapMapping, RecordDecodeError, SourceLocator,
Support,
};
pub struct CollectionSnapshot<R, E>
where
R: StoreSnapshot,
E: CollectionEncoding,
{
snapshot: R,
support: Support<E>,
cover: Cover<E>,
dependencies: DependencyTracker,
}
impl<R, E> Clone for CollectionSnapshot<R, E>
where
R: StoreSnapshot,
E: CollectionEncoding,
{
fn clone(&self) -> Self {
Self {
snapshot: self.snapshot.clone(),
support: self.support.clone(),
cover: self.cover.clone(),
dependencies: self.dependencies.clone(),
}
}
}
impl<R, E> CollectionSnapshot<R, E>
where
R: StoreSnapshot,
E: CollectionEncoding,
{
pub(crate) fn from_frontier(
snapshot: R,
support: Support<E>,
cover: Cover<E>,
dependencies: DependencyTracker,
) -> Self {
Self {
snapshot,
support,
cover,
dependencies,
}
}
pub fn snapshot(&self) -> &R {
&self.snapshot
}
pub fn is_current(&self, snapshot: &R) -> bool {
let dependencies = self
.dependencies
.lock()
.expect("dependency tracker poisoned");
snapshot.changes_for(&self.snapshot, &dependencies) == StoreChanges::NONE
}
pub fn support(&self) -> Result<&Support<E>, CollectionRealizationError>
where
R: StoreRead,
{
Ok(&self.support)
}
pub fn missing_from<RS, S>(
&self,
source: &CollectionSnapshot<RS, S>,
) -> Result<Cover<S>, CollectionRealizationError>
where
R: StoreRead,
RS: StoreRead,
S: CollectionEncoding,
{
let target = self.cover.collection().handle();
let observed =
ObservedStore::with_tracker(self.snapshot.clone(), self.dependencies.clone());
let descriptor = super::api::load_collection_descriptor(&observed, target)
.map_err(|error| CollectionRealizationError::storage("read view descriptor", error))?;
let named = super::descriptor::source(descriptor.fragment.facts()).map_err(|error| {
CollectionRealizationError::Resolution(format!("read view source: {error}"))
})?;
let source_collection = source.cover().collection();
if named != Some(source_collection.handle()) {
return Err(CollectionRealizationError::InvalidCover(format!(
"collection {} does not derive from {}",
hex::encode_upper(target.raw),
hex::encode_upper(source_collection.handle().raw),
)));
}
let coverage = observed
.coverage(&BTreeSet::from([target]))
.map_err(|error| CollectionRealizationError::storage("read view leaves", error))?;
let missing = source.support()?.data_members().filter(|foundation| {
!coverage
.leaf_outputs(target, SourceLocator::of(foundation.raw))
.into_iter()
.any(|output| {
self.support
.contains(crate::inline::Inline::new(output.raw))
})
});
Ok(Cover::from_data(source_collection, missing))
}
pub fn cover(&self) -> &Cover<E> {
&self.cover
}
pub fn view<V>(&self) -> Result<V, TryFromCoverError<R::GetError<Infallible>, V::Error>>
where
R: BlobStoreGet,
V: TryFromCover<E>,
{
let observed =
ObservedStore::with_tracker(self.snapshot.clone(), self.dependencies.clone());
let descriptor =
super::api::load_collection_descriptor(&observed, self.cover.collection().handle())
.map_err(TryFromCoverError::from)?;
V::try_from_cover(&self.cover, &descriptor.fragment, &observed)
}
pub fn into_parts(self) -> Result<(R, Support<E>, Cover<E>), CollectionRealizationError>
where
R: StoreRead,
{
Ok((self.snapshot, self.support, self.cover))
}
}
pub struct AttachedSnapshot<R, E>
where
R: StoreSnapshot,
E: CollectionEncoding,
{
snapshot: R,
cover: Cover<E>,
support: Support<SimpleArchive>,
residual: Support<SimpleArchive>,
dependencies: DependencyTracker,
}
#[cfg(feature = "object-store")]
impl<R, E> AttachedSnapshot<crate::repo::async_store::AcquiringReader<R>, E>
where
R: StoreSnapshot,
E: CollectionEncoding,
{
pub fn into_frozen(self) -> AttachedSnapshot<R, E> {
AttachedSnapshot {
snapshot: self.snapshot.into_frozen(),
cover: self.cover,
support: self.support,
residual: self.residual,
dependencies: self.dependencies,
}
}
}
impl<R, E> Clone for AttachedSnapshot<R, E>
where
R: StoreSnapshot,
E: CollectionEncoding,
{
fn clone(&self) -> Self {
Self {
snapshot: self.snapshot.clone(),
cover: self.cover.clone(),
support: self.support.clone(),
residual: self.residual.clone(),
dependencies: self.dependencies.clone(),
}
}
}
impl<R, E> AttachedSnapshot<R, E>
where
R: StoreSnapshot,
E: CollectionEncoding,
{
pub(crate) fn new(
snapshot: R,
cover: Cover<E>,
support: Support<SimpleArchive>,
residual: Support<SimpleArchive>,
dependencies: DependencyTracker,
) -> Self {
Self {
snapshot,
cover,
support,
residual,
dependencies,
}
}
pub fn snapshot(&self) -> &R {
&self.snapshot
}
pub fn is_current(&self, snapshot: &R) -> bool {
let dependencies = self
.dependencies
.lock()
.expect("dependency tracker poisoned");
snapshot.changes_for(&self.snapshot, &dependencies) == StoreChanges::NONE
}
pub fn cover(&self) -> &Cover<E> {
&self.cover
}
pub fn support(&self) -> &Support<SimpleArchive> {
&self.support
}
pub fn residual(&self) -> &Support<SimpleArchive> {
&self.residual
}
pub fn view<V>(&self) -> Result<V, TryFromCoverError<R::GetError<Infallible>, V::Error>>
where
R: BlobStoreGet,
V: TryFromCover<E>,
{
let observed =
ObservedStore::with_tracker(self.snapshot.clone(), self.dependencies.clone());
let descriptor =
super::api::load_collection_descriptor(&observed, self.cover.collection().handle())
.map_err(TryFromCoverError::from)?;
V::try_from_cover(&self.cover, &descriptor.fragment, &observed)
}
pub fn read<V>(
&self,
) -> Result<AttachedRead<V>, AttachedReadError<R::GetError<Infallible>, V::Error>>
where
R: StoreRead,
E: super::CollectionAttachment,
V: TryFromCover<E>,
{
self.read_with::<super::encoding::CanonicalAttachment<E>, V>()
}
pub fn read_acquiring<V>(
&self,
) -> Result<AttachedRead<V>, AttachedReadError<R::GetError<Infallible>, V::Error>>
where
R: StoreRead,
E: super::CollectionAttachment,
V: TryFromCover<E>,
{
self.read_with_acquiring::<super::encoding::CanonicalAttachment<E>, V>()
}
pub fn read_with_acquiring<M, V>(
&self,
) -> Result<AttachedRead<V>, AttachedReadError<R::GetError<Infallible>, V::Error>>
where
R: StoreRead,
M: MapMapping<Target = E>,
V: TryFromCover<E>,
{
self.read_mode::<M, V>(true)
}
#[cfg(feature = "object-store")]
pub fn acquiring(
&self,
runtime: tokio::runtime::Handle,
) -> AttachedSnapshot<crate::repo::async_store::AcquiringReader<R>, E> {
AttachedSnapshot {
snapshot: crate::repo::async_store::AcquiringReader::with_handle(
self.snapshot.clone(),
runtime,
),
cover: self.cover.clone(),
support: self.support.clone(),
residual: self.residual.clone(),
dependencies: self.dependencies.clone(),
}
}
pub fn read_with<M, V>(
&self,
) -> Result<AttachedRead<V>, AttachedReadError<R::GetError<Infallible>, V::Error>>
where
R: StoreRead,
M: MapMapping<Target = E>,
V: TryFromCover<E>,
{
self.read_mode::<M, V>(false)
}
fn read_mode<M, V>(
&self,
acquire: bool,
) -> Result<AttachedRead<V>, AttachedReadError<R::GetError<Infallible>, V::Error>>
where
R: StoreRead,
M: MapMapping<Target = E>,
V: TryFromCover<E>,
{
let observed =
ObservedStore::with_tracker(self.snapshot.clone(), self.dependencies.clone());
let attached = self.cover.collection();
let parent = self.support.collection();
let descriptor = super::api::load_collection_descriptor(&observed, attached.handle())
.map_err(|error| AttachedReadError::View(error.into()))?;
let parent_descriptor = super::api::load_collection_descriptor(&observed, parent.handle())
.map_err(|error| AttachedReadError::View(error.into()))?;
let mapping = M::bind(&parent_descriptor.fragment, &descriptor.fragment)
.map_err(AttachedReadError::Mapping)?;
let builds = mapping.siblings().is_empty();
let mut images: HashMap<[u8; 32], Bytes> = HashMap::new();
let mut unread = Vec::new();
for foundation in self.residual.members() {
let member = Handle::<SimpleArchive>::to_hash(foundation);
if !builds {
unread.push(member);
continue;
}
let failed = |reason: String| AttachedReadError::Residual { member, reason };
if !acquire
&& observed
.metadata(foundation)
.map_err(|error| failed(error.to_string()))?
.is_none()
{
unread.push(member);
continue;
}
let node: Blob<SimpleArchive> = match observed.get(foundation) {
Ok(node) => node,
Err(error) if acquire && crate::repo::is_missing_blob(&error) => {
unread.push(member);
continue;
}
Err(error) => return Err(failed(error.to_string())),
};
match mapping.map(&node, &[], &observed) {
Ok(image) => {
images.insert(image.get_handle().raw, image.bytes);
}
Err(super::CollectionOperationError::Fatal(reason)) if acquire => {
return Err(failed(reason));
}
Err(_) => unread.push(member),
}
}
let cover = Cover::from_members(
attached,
self.cover
.members()
.chain(images.keys().map(|raw| Inline::new(*raw))),
);
let reader = WithImages {
base: &observed,
images: &images,
};
let value = V::try_from_cover(&cover, &descriptor.fragment, &reader)
.map_err(|error| AttachedReadError::View(error.into_base()))?;
Ok(AttachedRead::new(value, Support::from_data(parent, unread)))
}
}
#[derive(Clone, Debug)]
pub struct AttachedRead<V> {
value: V,
unread: Support<SimpleArchive>,
}
impl<V> AttachedRead<V> {
pub(crate) fn new(value: V, unread: Support<SimpleArchive>) -> Self {
Self { value, unread }
}
pub fn value(&self) -> &V {
&self.value
}
pub fn into_value(self) -> V {
self.value
}
pub fn unread(&self) -> &Support<SimpleArchive> {
&self.unread
}
pub fn into_parts(self) -> (V, Support<SimpleArchive>) {
(self.value, self.unread)
}
pub fn map<W>(self, transform: impl FnOnce(V) -> W) -> AttachedRead<W> {
AttachedRead {
value: transform(self.value),
unread: self.unread,
}
}
}
#[derive(Debug)]
pub enum AttachedReadError<GetError, ViewError> {
View(TryFromCoverError<GetError, ViewError>),
Mapping(super::CollectionOperationError),
Residual {
member: CollectionData,
reason: String,
},
}
impl<GetError, ViewError> fmt::Display for AttachedReadError<GetError, ViewError>
where
GetError: fmt::Display,
ViewError: fmt::Display,
{
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::View(source) => source.fmt(formatter),
Self::Mapping(source) => write!(
formatter,
"attached descriptor does not bind the mapping read through: {source}"
),
Self::Residual { member, reason } => write!(
formatter,
"read residual foundation {}: {reason}",
hex::encode_upper(member.raw),
),
}
}
}
impl<GetError, ViewError> Error for AttachedReadError<GetError, ViewError>
where
GetError: Error + 'static,
ViewError: Error + 'static,
{
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::View(source) => Some(source),
Self::Mapping(source) => Some(source),
Self::Residual { .. } => None,
}
}
}
struct WithImages<'a, R> {
base: &'a R,
images: &'a HashMap<[u8; 32], Bytes>,
}
#[derive(Debug)]
enum WithImagesGetError<Decode, Base> {
Decode(Decode),
Base(Base),
}
impl<Decode: fmt::Display, Base: fmt::Display> fmt::Display for WithImagesGetError<Decode, Base> {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Decode(source) => write!(formatter, "decode an image built in memory: {source}"),
Self::Base(source) => source.fmt(formatter),
}
}
}
impl<Decode, Base> Error for WithImagesGetError<Decode, Base>
where
Decode: Error + 'static,
Base: Error + 'static,
{
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Decode(source) => Some(source),
Self::Base(source) => Some(source),
}
}
}
impl<R: BlobStoreGet> BlobStoreGet for WithImages<'_, R> {
type GetError<E: Error + Send + Sync + 'static> = WithImagesGetError<E, R::GetError<E>>;
fn get<T, S>(
&self,
handle: Inline<Handle<S>>,
) -> Result<T, Self::GetError<<T as TryFromBlob<S>>::Error>>
where
S: BlobEncoding + 'static,
T: TryFromBlob<S>,
Handle<S>: InlineEncoding,
{
match self.images.get(&handle.raw) {
Some(bytes) => T::try_from_blob(Blob::with_handle(bytes.clone(), handle))
.map_err(WithImagesGetError::Decode),
None => self.base.get(handle).map_err(WithImagesGetError::Base),
}
}
}
impl<Base, ViewError> TryFromCoverError<WithImagesGetError<Infallible, Base>, ViewError> {
fn into_base(self) -> TryFromCoverError<Base, ViewError> {
let base = |source: WithImagesGetError<Infallible, Base>| match source {
WithImagesGetError::Decode(never) => match never {},
WithImagesGetError::Base(source) => source,
};
match self {
Self::DescriptorGet { collection, source } => TryFromCoverError::DescriptorGet {
collection,
source: base(source),
},
Self::InvalidDescriptor { collection, source } => {
TryFromCoverError::InvalidDescriptor { collection, source }
}
Self::MemberGet { member, source } => TryFromCoverError::MemberGet {
member,
source: base(source),
},
Self::View(source) => TryFromCoverError::View(source),
}
}
}
#[derive(Debug)]
pub enum TryFromCoverError<GetError, ViewError> {
DescriptorGet {
collection: CollectionHandle,
source: GetError,
},
InvalidDescriptor {
collection: CollectionHandle,
source: RecordDecodeError,
},
MemberGet {
member: CollectionData,
source: GetError,
},
View(ViewError),
}
impl<GetError, ViewError> fmt::Display for TryFromCoverError<GetError, ViewError>
where
GetError: fmt::Display,
ViewError: fmt::Display,
{
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::DescriptorGet { collection, source } => write!(
formatter,
"failed to fetch collection descriptor {}: {source}",
hex::encode_upper(collection.raw),
),
Self::InvalidDescriptor { collection, source } => write!(
formatter,
"collection descriptor {} is invalid: {source}",
hex::encode_upper(collection.raw),
),
Self::MemberGet { member, source } => write!(
formatter,
"failed to fetch cover member {}: {source}",
hex::encode_upper(member.raw),
),
Self::View(source) => source.fmt(formatter),
}
}
}
impl<GetError, ViewError> Error for TryFromCoverError<GetError, ViewError>
where
GetError: Error + 'static,
ViewError: Error + 'static,
{
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::DescriptorGet { source, .. } => Some(source),
Self::InvalidDescriptor { source, .. } => Some(source),
Self::MemberGet { source, .. } => Some(source),
Self::View(source) => Some(source),
}
}
}
impl<GetError, ViewError> From<CollectionDescriptorError<GetError>>
for TryFromCoverError<GetError, ViewError>
{
fn from(source: CollectionDescriptorError<GetError>) -> Self {
match source {
CollectionDescriptorError::Get { collection, source } => {
Self::DescriptorGet { collection, source }
}
CollectionDescriptorError::Invalid { collection, source } => {
Self::InvalidDescriptor { collection, source }
}
}
}
}
pub trait TryFromCover<L: CollectionEncoding>: Sized {
type Error: Error + Send + Sync + 'static;
fn try_from_cover<R>(
cover: &Cover<L>,
descriptor: &Fragment,
snapshot: &R,
) -> Result<Self, TryFromCoverError<R::GetError<Infallible>, Self::Error>>
where
R: BlobStoreGet;
}
#[cfg(test)]
mod tests {
use crate::blob::encodings::succinctarchive::SuccinctArchiveBlob;
use crate::collection::observed_store::ObservedStore;
use crate::collection::{Collection, CollectionData, CollectionHandle};
use crate::repo::memoryrepo::MemoryRepo;
use crate::repo::SnapshotSource;
use super::{CollectionSnapshot, Cover, Support};
#[test]
fn snapshot_keeps_its_own_support_beside_its_cover() {
let target = Collection::<SuccinctArchiveBlob>::from_handle(CollectionHandle::new([2; 32]));
let support: Support<SuccinctArchiveBlob> =
Support::from_data(target, [CollectionData::new([3; 32])]);
let cover = Cover::from_data(target, [CollectionData::new([4; 32])]);
let mut store = MemoryRepo::default();
let store_snapshot = store.snapshot().unwrap();
let snapshot = CollectionSnapshot::from_frontier(
store_snapshot.clone(),
support.clone(),
cover.clone(),
ObservedStore::new(store_snapshot.clone()).tracker(),
);
assert!(snapshot.snapshot() == &store_snapshot);
assert_eq!(snapshot.support().unwrap(), &support);
assert_eq!(snapshot.cover(), &cover);
let (actual_store, actual_support, actual_cover) = snapshot.into_parts().unwrap();
assert!(actual_store == store_snapshot);
assert_eq!(actual_support, support);
assert_eq!(actual_cover, cover);
}
}