use crate::processed::{ProcessedData, ProcessedDataset, ProcessedDescriptor, ProcessedProvenance};
use crate::provenance::CanonicalDatasetDigests;
use crate::raw::{RawDataset, RawDescriptor, RawProvenance};
use crate::{DatasetIdentity, DatasetKind, Format, ReadWarning};
use std::path::{Path, PathBuf};
#[non_exhaustive]
#[derive(Clone, Copy, Debug)]
pub enum DescriptorRef<'a> {
Raw(&'a RawDescriptor),
Processed(&'a ProcessedDescriptor),
}
#[non_exhaustive]
#[derive(Clone, Copy, Debug)]
pub enum ProvenanceRef<'a> {
Raw(&'a RawProvenance),
Processed(&'a ProcessedProvenance),
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ReadSelection {
format: Format,
path: PathBuf,
}
impl ReadSelection {
pub fn format(&self) -> Format {
self.format
}
pub fn path(&self) -> &Path {
&self.path
}
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct DatasetMetadata {
identity: DatasetIdentity,
selection: Option<ReadSelection>,
warnings: Vec<ReadWarning>,
accepted_archive: bool,
}
impl DatasetMetadata {
pub fn accepted_archive(&self) -> bool {
self.accepted_archive
}
pub fn identity(&self) -> &DatasetIdentity {
&self.identity
}
pub fn selection(&self) -> Option<&ReadSelection> {
self.selection.as_ref()
}
pub fn warnings(&self) -> &[ReadWarning] {
&self.warnings
}
}
#[derive(Debug, PartialEq)]
#[allow(clippy::large_enum_variant)]
pub(crate) enum DatasetData {
Raw(RawDataset),
Processed(ProcessedDataset),
}
#[derive(Debug, PartialEq)]
pub struct Dataset {
data: DatasetData,
metadata: DatasetMetadata,
digests: CanonicalDatasetDigests,
}
impl Dataset {
pub fn from_raw(value: RawDataset) -> Self {
let digests = value.canonical_digests();
Self {
data: DatasetData::Raw(value),
metadata: DatasetMetadata::default(),
digests,
}
}
pub fn from_raw_with_context(
value: RawDataset,
control: &mut crate::ExecutionContext<'_>,
) -> Result<Self, crate::execution::ExecutionError> {
control.begin(crate::execution::ExecutionStage::Digest, None, None)?;
let digests = crate::canonical_digest::dataset_digests_controlled(
&value,
Some(control.cancellation()),
)?;
Ok(Self {
data: DatasetData::Raw(value),
metadata: DatasetMetadata::default(),
digests,
})
}
pub fn from_processed(value: ProcessedDataset) -> Self {
let digests = value.canonical_digests();
Self {
data: DatasetData::Processed(value),
metadata: DatasetMetadata::default(),
digests,
}
}
pub(crate) fn attach_read_context(
mut self,
format: Format,
identity: DatasetIdentity,
path: PathBuf,
warnings: Vec<ReadWarning>,
) -> Result<Self, crate::ReadError> {
if self.kind() != format.kind() {
return Err(crate::ReadError::new(
None,
crate::ReadErrorReason::Invalid {
detail: "format and dataset kinds differ".into(),
},
));
}
self.metadata = DatasetMetadata {
identity,
selection: Some(ReadSelection { format, path }),
warnings,
accepted_archive: false,
};
Ok(self)
}
pub(crate) fn derived_processed_with_secondary(
&self,
value: ProcessedDataset,
other: &Self,
) -> Self {
let mut output = self.derived_processed(value);
output.metadata.warnings.extend_from_slice(other.warnings());
output
}
pub(crate) fn derived_processed(&self, value: ProcessedDataset) -> Self {
let mut output = Self::from_processed(value);
output.metadata = self.metadata.clone();
output
}
pub fn kind(&self) -> DatasetKind {
match self.data {
DatasetData::Raw(_) => DatasetKind::Raw,
DatasetData::Processed(_) => DatasetKind::Processed,
}
}
pub fn descriptor(&self) -> DescriptorRef<'_> {
match &self.data {
DatasetData::Raw(value) => DescriptorRef::Raw(value.descriptor()),
DatasetData::Processed(value) => DescriptorRef::Processed(value.descriptor()),
}
}
pub fn metadata(&self) -> &DatasetMetadata {
&self.metadata
}
pub fn provenance(&self) -> ProvenanceRef<'_> {
match &self.data {
DatasetData::Raw(value) => ProvenanceRef::Raw(value.provenance()),
DatasetData::Processed(value) => ProvenanceRef::Processed(value.provenance()),
}
}
pub fn canonical_digests(&self) -> CanonicalDatasetDigests {
self.digests
}
pub fn as_raw(&self) -> Option<&RawDataset> {
match &self.data {
DatasetData::Raw(value) => Some(value),
_ => None,
}
}
pub fn as_processed(&self) -> Option<&ProcessedDataset> {
match &self.data {
DatasetData::Processed(value) => Some(value),
_ => None,
}
}
pub fn as_dense_processed(&self) -> Option<&ProcessedData> {
self.as_processed().map(ProcessedDataset::data)
}
#[allow(clippy::result_large_err)]
pub fn into_raw(self) -> Result<RawDatasetWithContext, Self> {
if self.as_raw().is_some() {
Ok(RawDatasetWithContext(self))
} else {
Err(self)
}
}
#[allow(clippy::result_large_err)]
pub fn into_processed(self) -> Result<ProcessedDatasetWithContext, Self> {
if self.as_processed().is_some() {
Ok(ProcessedDatasetWithContext(self))
} else {
Err(self)
}
}
#[allow(clippy::result_large_err)]
pub fn into_raw_data(self) -> Result<RawDataset, Self> {
match self.data {
DatasetData::Raw(value) => Ok(value),
_ => Err(self),
}
}
#[allow(clippy::result_large_err)]
pub fn into_processed_data(self) -> Result<ProcessedDataset, Self> {
match self.data {
DatasetData::Processed(value) => Ok(value),
_ => Err(self),
}
}
pub fn source_format(&self) -> Option<Format> {
self.metadata.selection.as_ref().map(ReadSelection::format)
}
pub fn identity(&self) -> &DatasetIdentity {
self.metadata.identity()
}
pub fn selected_path(&self) -> Option<&Path> {
self.metadata.selection.as_ref().map(ReadSelection::path)
}
pub fn warnings(&self) -> &[ReadWarning] {
self.metadata.warnings()
}
pub fn sources(&self) -> &[crate::provenance::SourceFile] {
match &self.data {
DatasetData::Raw(value) => value.provenance().sources(),
DatasetData::Processed(value) => value.provenance().sources(),
}
}
}
#[derive(Debug, PartialEq)]
pub struct RawDatasetWithContext(Dataset);
impl RawDatasetWithContext {
pub fn raw(&self) -> &RawDataset {
self.0.as_raw().expect("narrowed raw")
}
pub fn dataset(&self) -> &Dataset {
&self.0
}
pub fn into_dataset(self) -> Dataset {
self.0
}
}
#[derive(Debug, PartialEq)]
pub struct ProcessedDatasetWithContext(Dataset);
impl ProcessedDatasetWithContext {
pub fn processed(&self) -> &ProcessedDataset {
self.0.as_processed().expect("narrowed processed")
}
pub fn dataset(&self) -> &Dataset {
&self.0
}
pub fn into_dataset(self) -> Dataset {
self.0
}
}
impl From<RawDataset> for Dataset {
fn from(value: RawDataset) -> Self {
Self::from_raw(value)
}
}
impl From<ProcessedDataset> for Dataset {
fn from(value: ProcessedDataset) -> Self {
Self::from_processed(value)
}
}
impl Dataset {
pub(crate) fn accept_archived_history(&mut self) {
self.metadata.accepted_archive = true;
if let DatasetData::Processed(value) = &mut self.data {
value.accept_archived_history();
}
}
pub(crate) fn validate_recorded(
&self,
control: &mut crate::ExecutionContext<'_>,
) -> Result<(), crate::internal::ModelError> {
use crate::internal::ModelError;
control.check_cancelled()?;
let actual = match &self.data {
DatasetData::Raw(value) => {
value
.validate()
.map_err(|e| ModelError::Validation(e.to_string()))?;
crate::canonical_digest::dataset_digests_controlled(
value,
Some(control.cancellation()),
)?
}
DatasetData::Processed(value) => {
value.validate_recorded(control)?;
value.canonical_digests()
}
};
control.check_cancelled()?;
if actual != self.digests {
return Err(ModelError::DigestMismatch);
}
Ok(())
}
}
impl Dataset {
#[allow(clippy::type_complexity)]
pub(crate) fn model_parts(&self) -> (&DatasetData, &DatasetMetadata, &CanonicalDatasetDigests) {
(&self.data, &self.metadata, &self.digests)
}
#[allow(clippy::type_complexity)]
pub(crate) fn from_model_parts(
parts: (DatasetData, DatasetMetadata, CanonicalDatasetDigests),
) -> Result<Self, crate::internal::ModelError> {
let (data, metadata, digests) = parts;
let value = Self {
data,
metadata,
digests,
};
Ok(value)
}
}
impl DatasetMetadata {
#[allow(clippy::type_complexity)]
pub(crate) fn model_parts(
&self,
) -> (
&DatasetIdentity,
&Option<ReadSelection>,
&Vec<ReadWarning>,
&bool,
) {
(
&self.identity,
&self.selection,
&self.warnings,
&self.accepted_archive,
)
}
#[allow(clippy::type_complexity)]
pub(crate) fn from_model_parts(
parts: (
DatasetIdentity,
Option<ReadSelection>,
Vec<ReadWarning>,
bool,
),
) -> Result<Self, crate::internal::ModelError> {
let (identity, selection, warnings, accepted_archive) = parts;
let value = Self {
identity,
selection,
warnings,
accepted_archive,
};
Ok(value)
}
}
impl ReadSelection {
#[allow(clippy::type_complexity)]
pub(crate) fn model_parts(&self) -> (&Format, &PathBuf) {
(&self.format, &self.path)
}
#[allow(clippy::type_complexity)]
pub(crate) fn from_model_parts(
parts: (Format, PathBuf),
) -> Result<Self, crate::internal::ModelError> {
let (format, path) = parts;
let value = Self { format, path };
Ok(value)
}
}