#[cfg(feature = "std")]
use crate::io::{read_file, write_file, ApiResult, InputOutput};
use crate::prelude::*;
use crate::schema::standard::crosswalk::{self, mapping::datacite_to_dcat, CrosswalkError, FieldValue, Fields, SchemaBuilder, SchemaExtractor};
use crate::schema::standard::datacite::{self, RelationType};
use crate::schema::validate::is_url;
use crate::schema::{Date as PeriodOfTime, OneOrMany};
#[cfg(feature = "std")]
use crate::util::MimeType;
use crate::util::{Checksum, ToProse};
#[cfg(feature = "std")]
use crate::PathBuf;
#[cfg(feature = "std")]
use color_eyre::eyre::eyre;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde_with::skip_serializing_none;
use validator::Validate;
pub(crate) mod validate;
use validate::{is_document_refs_urls, is_one_or_many_urls};
pub type DatasetSeries = Dataset;
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
#[serde(untagged)]
pub enum ConformsTo {
Uri(String),
Standard(ConformsToStandard),
}
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
#[serde(untagged)]
pub enum ContactPoint {
Uri(String),
Kind(Kind),
}
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
#[serde(untagged)]
pub enum DocumentRef {
Uri(String),
Document(Document),
}
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
#[serde(untagged)]
pub enum Publisher {
Organization(UsOrganization),
Agent(Agent),
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct Agent {
pub name: Option<String>,
#[serde(default)]
#[validate(custom(function = "is_one_or_many_urls"))]
pub homepage: Option<OneOrMany<String>>,
pub email: Option<String>,
#[serde(default)]
pub identifier: Option<OneOrMany<String>>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct Catalog {
#[serde(rename = "@id")]
pub id: Option<String>,
#[serde(rename = "@type")]
pub jsonld_type: Option<String>,
#[serde(default)]
pub title: Option<OneOrMany<String>>,
#[serde(default)]
pub description: Option<OneOrMany<String>>,
#[serde(default)]
pub identifier: Option<OneOrMany<String>>,
pub issued: Option<String>,
pub modified: Option<String>,
#[serde(default)]
pub language: Option<OneOrMany<String>>,
#[validate(nested)]
pub publisher: Option<Publisher>,
#[validate(nested)]
pub creator: Option<Vec<Agent>>,
#[serde(rename = "contactPoint", default)]
#[validate(nested)]
pub contact_point: Option<OneOrMany<ContactPoint>>,
#[serde(default)]
pub keywords: Option<OneOrMany<String>>,
#[serde(default)]
pub themes: Option<OneOrMany<String>>,
pub license: Option<String>,
pub rights: Option<String>,
#[serde(rename = "accessRights")]
pub access_rights: Option<String>,
#[serde(rename = "hasPolicy")]
pub has_policy: Option<String>,
#[serde(rename = "conformsTo", default)]
#[validate(nested)]
pub conforms_to: Option<OneOrMany<ConformsTo>>,
#[serde(rename = "landingPage", default)]
#[validate(custom(function = "is_document_refs_urls"), nested)]
pub landing_page: Option<OneOrMany<DocumentRef>>,
#[serde(default)]
pub relation: Option<OneOrMany<String>>,
#[serde(rename = "type", default)]
pub type_: Option<OneOrMany<String>>,
pub version: Option<String>,
#[serde(rename = "versionNotes")]
pub version_notes: Option<String>,
#[serde(rename = "previousVersion")]
pub previous_version: Option<String>,
#[serde(rename = "hasVersion", default)]
pub has_version: Option<OneOrMany<String>>,
#[serde(rename = "hasCurrentVersion")]
pub has_current_version: Option<String>,
pub replaces: Option<String>,
pub status: Option<String>,
#[serde(rename = "isReferencedBy", default)]
pub is_referenced_by: Option<OneOrMany<String>>,
#[validate(nested)]
#[serde(rename = "qualifiedRelation")]
pub qualified_relation: Option<Vec<Relationship>>,
#[validate(nested)]
pub distribution: Option<Vec<Distribution>>,
pub frequency: Option<String>,
#[validate(nested)]
pub spatial: Option<Vec<Location>>,
#[serde(rename = "spatialResolutionInMeters")]
pub spatial_resolution_in_meters: Option<f64>,
#[validate(nested)]
pub temporal: Option<Vec<PeriodOfTime>>,
#[serde(rename = "temporalResolution")]
pub temporal_resolution: Option<String>,
#[serde(rename = "wasGeneratedBy", default)]
pub was_generated_by: Option<OneOrMany<String>>,
#[serde(default)]
#[validate(custom(function = "is_document_refs_urls"), nested)]
pub homepage: Option<OneOrMany<DocumentRef>>,
#[serde(rename = "themeTaxonomy", default)]
pub theme_taxonomy: Option<OneOrMany<String>>,
#[serde(default)]
pub dataset: Option<OneOrMany<String>>,
#[validate(nested)]
pub service: Option<Vec<DataService>>,
#[serde(default)]
pub catalog: Option<OneOrMany<String>>,
#[validate(nested)]
pub record: Option<Vec<CatalogRecord>>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct CatalogRecord {
pub title: Option<String>,
pub description: Option<String>,
pub issued: Option<String>,
pub modified: Option<String>,
#[serde(rename = "primaryTopic")]
pub primary_topic: String,
#[serde(rename = "conformsTo", default)]
pub conforms_to: Option<OneOrMany<String>>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct ConformsToStandard {
#[serde(rename = "@id")]
#[validate(custom(function = "is_url"))]
pub id: Option<String>,
#[serde(rename = "@type")]
pub jsonld_type: Option<String>,
pub title: Option<String>,
pub identifier: Option<String>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct DataService {
#[serde(rename = "@id")]
pub id: Option<String>,
#[serde(rename = "@type")]
pub jsonld_type: Option<String>,
#[serde(default)]
pub title: Option<OneOrMany<String>>,
#[serde(default)]
pub description: Option<OneOrMany<String>>,
#[serde(default)]
pub identifier: Option<OneOrMany<String>>,
pub issued: Option<String>,
pub modified: Option<String>,
#[serde(default)]
pub language: Option<OneOrMany<String>>,
#[validate(nested)]
pub publisher: Option<Publisher>,
#[validate(nested)]
pub creator: Option<Vec<Agent>>,
#[serde(rename = "contactPoint", default)]
#[validate(nested)]
pub contact_point: Option<OneOrMany<ContactPoint>>,
#[serde(default)]
pub keywords: Option<OneOrMany<String>>,
#[serde(default)]
pub themes: Option<OneOrMany<String>>,
pub license: Option<String>,
pub rights: Option<String>,
#[serde(rename = "accessRights")]
pub access_rights: Option<String>,
#[serde(rename = "hasPolicy")]
pub has_policy: Option<String>,
#[serde(rename = "conformsTo", default)]
#[validate(nested)]
pub conforms_to: Option<OneOrMany<ConformsTo>>,
#[serde(rename = "landingPage", default)]
#[validate(custom(function = "is_document_refs_urls"), nested)]
pub landing_page: Option<OneOrMany<DocumentRef>>,
#[serde(rename = "type", default)]
pub type_: Option<OneOrMany<String>>,
pub version: Option<String>,
#[serde(rename = "versionNotes")]
pub version_notes: Option<String>,
#[serde(rename = "previousVersion")]
pub previous_version: Option<String>,
#[serde(rename = "hasVersion", default)]
pub has_version: Option<OneOrMany<String>>,
#[serde(rename = "hasCurrentVersion")]
pub has_current_version: Option<String>,
pub replaces: Option<String>,
pub status: Option<String>,
#[validate(nested)]
#[serde(rename = "qualifiedRelation")]
pub qualified_relation: Option<Vec<Relationship>>,
#[serde(rename = "endpointURL")]
#[validate(custom(function = "is_one_or_many_urls"))]
pub endpoint_url: OneOrMany<String>,
#[serde(rename = "endpointDescription", default)]
pub endpoint_description: Option<OneOrMany<String>>,
#[serde(rename = "servesDataset", default)]
pub serves_dataset: Option<OneOrMany<String>>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct Dataset {
#[serde(rename = "@id")]
pub id: Option<String>,
#[serde(rename = "@type")]
pub jsonld_type: Option<String>,
#[serde(default)]
pub title: Option<OneOrMany<String>>,
#[serde(default)]
pub description: Option<OneOrMany<String>>,
#[serde(default)]
pub identifier: Option<OneOrMany<String>>,
pub issued: Option<String>,
pub modified: Option<String>,
#[serde(default)]
pub language: Option<OneOrMany<String>>,
#[validate(nested)]
pub publisher: Option<Publisher>,
#[validate(nested)]
pub creator: Option<Vec<Agent>>,
#[serde(rename = "contactPoint", default)]
#[validate(nested)]
pub contact_point: Option<OneOrMany<ContactPoint>>,
#[serde(default)]
pub keywords: Option<OneOrMany<String>>,
#[serde(default)]
pub themes: Option<OneOrMany<String>>,
pub license: Option<String>,
pub rights: Option<String>,
#[serde(rename = "accessRights")]
pub access_rights: Option<String>,
#[serde(rename = "hasPolicy")]
pub has_policy: Option<String>,
#[serde(rename = "conformsTo", default)]
#[validate(nested)]
pub conforms_to: Option<OneOrMany<ConformsTo>>,
#[serde(rename = "landingPage", default)]
#[validate(custom(function = "is_document_refs_urls"), nested)]
pub landing_page: Option<OneOrMany<DocumentRef>>,
#[serde(default)]
pub relation: Option<OneOrMany<String>>,
#[serde(rename = "type", default)]
pub type_: Option<OneOrMany<String>>,
pub version: Option<String>,
#[serde(rename = "versionNotes")]
pub version_notes: Option<String>,
#[serde(rename = "previousVersion")]
pub previous_version: Option<String>,
#[serde(rename = "hasVersion", default)]
pub has_version: Option<OneOrMany<String>>,
#[serde(rename = "hasCurrentVersion")]
pub has_current_version: Option<String>,
pub replaces: Option<String>,
pub status: Option<String>,
#[serde(rename = "isReferencedBy", default)]
pub is_referenced_by: Option<OneOrMany<String>>,
#[serde(rename = "hasPart", default)]
pub has_part: Option<OneOrMany<String>>,
#[validate(nested)]
#[serde(rename = "qualifiedRelation")]
pub qualified_relation: Option<Vec<Relationship>>,
pub first: Option<String>,
pub last: Option<String>,
pub previous: Option<String>,
#[validate(nested)]
pub distribution: Option<Vec<Distribution>>,
pub frequency: Option<String>,
#[serde(rename = "inSeries", default)]
pub in_series: Option<OneOrMany<String>>,
#[validate(nested)]
pub spatial: Option<Vec<Location>>,
#[serde(rename = "spatialResolutionInMeters")]
pub spatial_resolution_in_meters: Option<f64>,
#[validate(nested)]
pub temporal: Option<Vec<PeriodOfTime>>,
#[serde(rename = "temporalResolution")]
pub temporal_resolution: Option<String>,
#[serde(rename = "wasGeneratedBy", default)]
pub was_generated_by: Option<OneOrMany<String>>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct Distribution {
#[serde(rename = "@id")]
pub id: Option<String>,
#[serde(rename = "@type")]
pub jsonld_type: Option<String>,
#[serde(default)]
pub title: Option<OneOrMany<String>>,
#[serde(default)]
pub description: Option<OneOrMany<String>>,
pub issued: Option<String>,
pub modified: Option<String>,
pub license: Option<String>,
#[serde(rename = "accessRights")]
pub access_rights: Option<String>,
pub rights: Option<String>,
#[serde(rename = "hasPolicy")]
pub has_policy: Option<String>,
#[serde(rename = "accessRestriction")]
pub access_restriction: Option<String>,
#[serde(rename = "useRestriction")]
pub use_restriction: Option<String>,
#[serde(rename = "accessURL")]
#[validate(custom(function = "is_one_or_many_urls"))]
pub access_url: OneOrMany<String>,
#[serde(rename = "accessService", default)]
pub access_service: Option<OneOrMany<String>>,
#[serde(rename = "downloadURL", default)]
#[validate(custom(function = "is_one_or_many_urls"))]
pub download_url: Option<OneOrMany<String>>,
#[serde(rename = "byteSize")]
pub byte_size: Option<u64>,
#[serde(rename = "spatialResolutionInMeters")]
pub spatial_resolution_in_meters: Option<f64>,
#[serde(rename = "temporalResolution")]
pub temporal_resolution: Option<String>,
#[serde(rename = "conformsTo", default)]
#[validate(nested)]
pub conforms_to: Option<OneOrMany<ConformsTo>>,
#[serde(rename = "mediaType")]
pub media_type: Option<String>,
pub format: Option<String>,
#[serde(rename = "compressFormat")]
pub compress_format: Option<String>,
#[serde(rename = "packageFormat")]
pub package_format: Option<String>,
#[validate(nested)]
pub checksum: Option<Checksum>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct Document {
#[serde(rename = "@id")]
#[validate(custom(function = "is_url"))]
pub id: Option<String>,
#[serde(rename = "@type")]
pub jsonld_type: Option<String>,
pub title: Option<String>,
#[serde(rename = "accessURL")]
#[validate(custom(function = "is_url"))]
pub access_url: Option<String>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct Kind {
#[serde(rename = "@type")]
pub jsonld_type: Option<String>,
#[serde(rename = "fn")]
pub fn_: String,
#[serde(rename = "hasEmail")]
pub has_email: String,
pub tel: Option<String>,
#[serde(rename = "organization-name")]
pub organization_name: Option<String>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct Location {
pub geometry: Option<String>,
pub bbox: Option<String>,
pub centroid: Option<String>,
}
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct Relationship {
pub relation: String,
#[serde(rename = "hadRole", alias = "role")]
pub had_role: RelationType,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct UsOrganization {
#[serde(rename = "@id")]
#[validate(custom(function = "is_url"))]
pub id: Option<String>,
#[serde(rename = "@type")]
pub jsonld_type: Option<String>,
pub name: String,
#[serde(rename = "prefLabel")]
pub pref_label: Option<String>,
#[serde(rename = "altLabel")]
pub alt_label: Option<String>,
#[serde(rename = "subOrganizationOf")]
#[validate(nested)]
pub sub_organization_of: Option<Box<UsOrganization>>,
}
impl DocumentRef {
pub fn url(&self) -> Option<&str> {
match self {
| Self::Uri(value) => Some(value),
| Self::Document(document) => document.access_url.as_deref().or(document.id.as_deref()),
}
}
}
#[cfg(feature = "std")]
impl InputOutput for Dataset {
fn read(path: impl Into<PathBuf>) -> ApiResult<Dataset> {
let source = path.into();
match MimeType::from(source.display().to_string()) {
| MimeType::Json => Dataset::read_json(source),
| MimeType::Yaml => Dataset::read_yaml(source),
| _ => Err(eyre!("Unsupported DCAT data file extension")),
}
}
fn read_json(path: PathBuf) -> ApiResult<Dataset> {
read_file(path).and_then(|content| {
serde_json::from_str::<OneOrMany<Dataset>>(&content)
.map_err(|why| eyre!("Failed to parse JSON DCAT dataset — {why}"))
.and_then(|value| match value {
| OneOrMany::One(dataset) => Ok(dataset),
| OneOrMany::Many(datasets) => match datasets.len() {
| 1 => datasets
.into_iter()
.next()
.ok_or_else(|| eyre!("Expected one DCAT dataset but found none")),
| len => Err(eyre!("Expected one DCAT dataset but found {len}")),
},
})
})
}
fn read_yaml(path: PathBuf) -> ApiResult<Dataset> {
read_file(path).and_then(|content| {
serde_norway::from_str::<OneOrMany<Dataset>>(&content)
.map_err(|why| eyre!("Failed to parse YAML DCAT dataset — {why}"))
.and_then(|value| match value {
| OneOrMany::One(dataset) => Ok(dataset),
| OneOrMany::Many(datasets) => match datasets.len() {
| 1 => datasets
.into_iter()
.next()
.ok_or_else(|| eyre!("Expected one DCAT dataset but found none")),
| len => Err(eyre!("Expected one DCAT dataset but found {len}")),
},
})
})
}
fn write(&self, path: impl Into<PathBuf>) -> ApiResult<()> {
let output = path.into();
match MimeType::from(output.display().to_string()) {
| MimeType::Json => self.write_json(output),
| MimeType::Yaml => self.write_yaml(output),
| _ => Err(eyre!("Unsupported DCAT data file extension for writing")),
}
}
fn write_json(&self, path: impl Into<PathBuf>) -> ApiResult<()> {
let output = path.into().with_extension("json");
serde_json::to_string_pretty(self)
.map_err(|why| eyre!("Failed to serialize JSON DCAT dataset — {why}"))
.and_then(|content| write_file(output, content))
}
fn write_yaml(&self, path: impl Into<PathBuf>) -> ApiResult<()> {
let output = path.into().with_extension("yaml");
serde_norway::to_string(self)
.map_err(|why| eyre!("Failed to serialize YAML DCAT dataset — {why}"))
.and_then(|content| write_file(output, content))
}
}
impl Publisher {
pub fn name(&self) -> Option<&str> {
match self {
| Self::Organization(organization) => Some(organization.name.as_str()),
| Self::Agent(agent) => agent.name.as_deref(),
}
}
}
impl SchemaBuilder for Dataset {
fn build_from_fields(fields: &Fields) -> Result<Self, CrosswalkError> {
let identifier = fields.get_string_vec_opt("identifier").map(OneOrMany::Many);
let mut title = None;
if let Some(title_str) = fields.get_string_opt("title") {
title = Some(OneOrMany::Many(vec![title_str]));
}
let mut description = None;
if let Some(desc_str) = fields.get_string_opt("description") {
description = Some(OneOrMany::Many(vec![desc_str]));
}
let issued = fields.get_date_opt("issued");
let language = fields.get_string_vec_opt("language").map(OneOrMany::Many);
let keywords = fields.get_string_vec_opt("keywords").map(OneOrMany::Many);
let themes = fields.get_string_vec_opt("themes").map(OneOrMany::Many);
let version = fields.get_string_opt("version");
let mut publisher = None;
if let Some(pub_name) = fields.get_string_opt("publisher") {
publisher = Some(Publisher::Agent(Agent {
name: Some(pub_name),
homepage: None,
email: None,
identifier: None,
}));
}
let mut creator = None;
if let Some(creator_names) = fields.get_string_vec_opt("creators") {
creator = Some(
creator_names
.into_iter()
.map(|name| Agent {
name: Some(name),
homepage: None,
email: None,
identifier: None,
})
.collect(),
);
}
let license = fields.get_iri_opt("license");
let landing_page = fields
.get_string_opt("landing_page")
.or_else(|| fields.get_iri_opt("landing_page"))
.map(|value| OneOrMany::Many(vec![DocumentRef::Uri(value)]));
let mut spatial = None;
if let Some(spatial_str) = fields.get_string_opt("spatial") {
spatial = Some(vec![Location {
geometry: Some(spatial_str),
bbox: None,
centroid: None,
}]);
}
Ok(Dataset {
id: None,
jsonld_type: Some("dcat:Dataset".to_string()),
title,
description,
identifier,
issued,
modified: None,
language,
publisher,
creator,
contact_point: None,
keywords,
themes,
license,
rights: None,
access_rights: None,
has_policy: None,
conforms_to: None,
landing_page,
relation: None,
type_: None,
version,
version_notes: None,
previous_version: None,
has_version: None,
has_current_version: None,
replaces: None,
status: None,
is_referenced_by: None,
has_part: None,
qualified_relation: None,
first: None,
last: None,
previous: None,
distribution: None,
frequency: None,
in_series: None,
spatial,
spatial_resolution_in_meters: None,
temporal: None,
temporal_resolution: None,
was_generated_by: None,
})
}
}
impl SchemaExtractor for Dataset {
fn extract_fields(&self) -> Fields {
let mut fields = Fields::new();
if let Some(identifiers) = &self.identifier {
if !identifiers.is_empty() {
fields.insert("identifier", FieldValue::StringVec(identifiers.as_slice().to_vec()));
}
}
if let Some(titles) = &self.title {
if let Some(first) = titles.first() {
fields.insert("title", FieldValue::String(first.clone()));
if titles.len() > 1 {
let alt_titles: Vec<String> = titles.iter().skip(1).cloned().collect();
fields.insert("alternative-titles", FieldValue::StringVec(alt_titles));
}
}
}
if let Some(descriptions) = &self.description {
if let Some(first) = descriptions.first() {
fields.insert("description", FieldValue::String(first.clone()));
}
}
if let Some(issued) = &self.issued {
fields.insert("issued", FieldValue::Date(issued.clone()));
}
if let Some(keywords) = &self.keywords {
fields.insert("keywords", FieldValue::StringVec(keywords.as_slice().to_vec()));
}
if let Some(themes) = &self.themes {
fields.insert("themes", FieldValue::StringVec(themes.as_slice().to_vec()));
}
if let Some(language) = &self.language {
fields.insert("language", FieldValue::StringVec(language.as_slice().to_vec()));
}
if let Some(version) = &self.version {
fields.insert("version", FieldValue::String(version.clone()));
}
if let Some(publisher) = &self.publisher {
if let Some(name) = publisher.name() {
fields.insert("publisher", FieldValue::String(name.to_string()));
}
}
if let Some(creators) = &self.creator {
let creator_names: Vec<String> = creators.iter().filter_map(|c| c.name.clone()).collect();
if !creator_names.is_empty() {
fields.insert("creators", FieldValue::StringVec(creator_names));
}
}
if let Some(license) = &self.license {
fields.insert("license", FieldValue::IRI(license.clone()));
}
if let Some(spatial) = &self.spatial {
if let Some(first) = spatial.first() {
if let Some(geometry) = &first.geometry {
fields.insert("spatial", FieldValue::String(geometry.clone()));
}
}
}
fields
}
}
impl ToProse for Dataset {
fn to_prose(&self) -> String {
self.title
.iter()
.flatten()
.cloned()
.chain(self.description.iter().flatten().cloned())
.chain(self.keywords.iter().flatten().cloned())
.collect::<Vec<String>>()
.join("\n\n")
}
}
impl TryFrom<&datacite::Record> for Dataset {
type Error = CrosswalkError;
fn try_from(record: &datacite::Record) -> Result<Self, Self::Error> {
Dataset::try_from(record.clone())
}
}
impl TryFrom<datacite::Record> for Dataset {
type Error = CrosswalkError;
fn try_from(record: datacite::Record) -> Result<Self, Self::Error> {
let mapping = datacite_to_dcat();
crosswalk::convert(&record, &mapping).map(|(dataset, _)| dataset)
}
}