#[cfg(feature = "std")]
use crate::io::{read_file, write_file, ApiResult, InputOutput};
use crate::prelude::*;
use crate::schema::standard::crosswalk::{self, mapping::datacite_to_huwise, CrosswalkError, FieldValue, Fields, SchemaBuilder, SchemaExtractor};
use crate::schema::standard::datacite;
#[cfg(feature = "std")]
use crate::util::MimeType;
use crate::util::ToProse;
#[cfg(feature = "std")]
use crate::PathBuf;
#[cfg(feature = "std")]
use ammonia::clean;
#[cfg(feature = "std")]
use color_eyre::eyre::eyre;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use serde_with::skip_serializing_none;
use validator::Validate;
#[cfg(not(feature = "std"))]
fn clean(value: &str) -> String {
value.to_string()
}
pub type Catalog = Vec<Dataset>;
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)]
pub enum DublinCoreType {
#[serde(rename = "Dataset")]
Dataset,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct CustomTemplate {
pub projects: Option<String>,
#[serde(rename = "source-of-data")]
pub source_of_data: Option<String>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct Dataset {
pub dataset_id: String,
pub has_attachments: bool,
pub attachments_count: u64,
pub has_records: bool,
pub fields: Value,
#[validate(nested)]
pub metas: Meta,
pub features: Vec<Value>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct Meta {
#[validate(nested)]
pub dcat: Option<Dcat>,
#[serde(rename = "default")]
#[validate(nested)]
pub r#default: Option<DefaultMeta>,
#[serde(rename = "dublin-core")]
#[validate(nested)]
pub dublin_core: Option<DublinCore>,
#[validate(nested)]
pub dcat_ap: Option<DcatAp>,
#[serde(rename = "custom-template")]
#[validate(nested)]
pub custom_template: Option<CustomTemplate>,
#[validate(nested)]
pub datacite: Option<Datacite>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
#[serde(rename_all = "snake_case")]
pub struct Dcat {
pub created: Option<String>,
pub issued: Option<String>,
pub creator: Option<String>,
pub contributor: Option<String>,
pub contact_name: Option<String>,
#[validate(email)]
pub contact_email: Option<String>,
#[serde(rename = "accrualperiodicity")]
pub accrual_periodicity: Option<String>,
pub spatial: Option<String>,
pub temporal: Option<String>,
pub granularity: Option<String>,
#[serde(rename = "dataquality")]
pub data_quality: Option<String>,
pub publisher_type: Option<String>,
pub conforms_to: Option<String>,
pub temporal_coverage_start: Option<String>,
pub temporal_coverage_end: Option<String>,
#[serde(rename = "accessRights")]
pub access_rights: Option<String>,
pub relation: Option<String>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
#[serde(rename_all = "snake_case")]
pub struct DefaultMeta {
pub title: Option<String>,
pub description: Option<String>,
pub theme: Option<Vec<String>>,
pub keyword: Option<Vec<String>>,
pub license: Option<String>,
#[validate(url)]
pub license_url: Option<String>,
pub language: Option<String>,
pub metadata_languages: Option<Vec<String>>,
pub timezone: Option<String>,
pub modified: Option<String>,
pub modified_updates_on_metadata_change: Option<bool>,
pub modified_updates_on_data_change: Option<bool>,
pub data_processed: Option<String>,
pub metadata_processed: Option<String>,
pub geographic_reference: Option<String>,
pub geographic_reference_auto: Option<bool>,
pub territory: Option<String>,
pub geometry_types: Option<Vec<String>>,
pub bbox: Option<String>,
pub publisher: Option<String>,
pub references: Option<String>,
pub records_count: Option<u64>,
pub attributions: Option<String>,
pub source_domain: Option<String>,
pub source_domain_title: Option<String>,
pub source_domain_address: Option<String>,
pub source_dataset: Option<String>,
pub shared_catalog: Option<String>,
pub federated: Option<bool>,
pub parent_domain: Option<String>,
pub update_frequency: Option<String>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
pub struct DublinCore {
pub title: Option<String>,
pub alternative: Option<String>,
pub subject: Option<Vec<String>>,
pub description: Option<String>,
#[serde(rename = "abstract")]
pub abstract_text: Option<String>,
#[serde(rename = "tableOfContent")]
pub table_of_content: Option<String>,
#[serde(rename = "type")]
pub kind: Option<DublinCoreType>,
pub language: Option<String>,
pub coverage: Option<String>,
pub spatial: Option<String>,
#[serde(rename = "temporal_start")]
pub temporal_start: Option<String>,
#[serde(rename = "temporal_end")]
pub temporal_end: Option<String>,
pub relation: Option<String>,
pub source: Option<String>,
pub replaces: Option<String>,
pub references: Option<String>,
pub requires: Option<String>,
#[serde(rename = "conformsTo")]
pub conforms_to: Option<String>,
#[serde(rename = "hasFormat")]
pub has_format: Option<String>,
#[serde(rename = "hasPart")]
pub has_part: Option<String>,
#[serde(rename = "hasVersion")]
pub has_version: Option<String>,
#[serde(rename = "isFormatOf")]
pub is_format_of: Option<String>,
#[serde(rename = "isPartOf")]
pub is_part_of: Option<String>,
#[serde(rename = "isVersionOf")]
pub is_version_of: Option<String>,
#[serde(rename = "isReferencedBy")]
pub is_referenced_by: Option<String>,
#[serde(rename = "isReplacedBy")]
pub is_replaced_by: Option<String>,
#[serde(rename = "isRequiredBy")]
pub is_required_by: Option<String>,
pub contributor: Option<Vec<String>>,
pub creator: Option<String>,
pub publisher: Option<String>,
pub rights: Option<String>,
#[serde(rename = "accessRights")]
pub access_rights: Option<String>,
pub license: Option<String>,
#[serde(rename = "date_start")]
pub date_start: Option<String>,
#[serde(rename = "date_end")]
pub date_end: Option<String>,
#[serde(rename = "available_start")]
pub available_start: Option<String>,
#[serde(rename = "available_end")]
pub available_end: Option<String>,
pub created: Option<String>,
#[serde(rename = "dateAccepted")]
pub date_accepted: Option<String>,
#[serde(rename = "dateCopyrighted")]
pub date_copyrighted: Option<String>,
#[serde(rename = "dateSubmitted")]
pub date_submitted: Option<String>,
pub issued: Option<String>,
pub modified: Option<String>,
#[serde(rename = "valid_start")]
pub valid_start: Option<String>,
#[serde(rename = "valid_end")]
pub valid_end: Option<String>,
pub format: Option<String>,
pub extent: Option<String>,
pub medium: Option<String>,
pub identifier: Option<String>,
#[serde(rename = "bibliographicCitation")]
pub bibliographic_citation: Option<String>,
#[serde(rename = "rightsHolder")]
pub rights_holder: Option<String>,
#[serde(rename = "accrualMethod")]
pub accrual_method: Option<String>,
#[serde(rename = "accrualPeriodicity")]
pub accrual_periodicity: Option<String>,
#[serde(rename = "accrualPolicy")]
pub accrual_policy: Option<String>,
pub audience: Option<String>,
#[serde(rename = "educationLevel")]
pub education_level: Option<String>,
#[serde(rename = "instructionalMethod")]
pub instructional_method: Option<String>,
pub mediator: Option<String>,
pub provenance: Option<String>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
#[serde(rename_all = "snake_case")]
pub struct DcatAp {
pub title: Option<String>,
pub description: Option<String>,
pub theme: Option<String>,
pub keyword: Option<Vec<String>>,
pub contact_name: Option<String>,
#[validate(email)]
pub contact_email: Option<String>,
pub publisher_name: Option<String>,
pub publisher_type: Option<String>,
pub spatial_bbox: Option<String>,
pub spatial_centroid: Option<String>,
#[serde(rename = "temporal_startDate")]
pub temporal_start_date: Option<String>,
#[serde(rename = "temporal_endDate")]
pub temporal_end_date: Option<String>,
#[serde(rename = "accrualPeriodicity")]
pub accrual_periodicity: Option<String>,
}
#[skip_serializing_none]
#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, Validate)]
#[serde(rename_all = "kebab-case")]
pub struct Datacite {
pub identifier: Option<String>,
pub title: Option<String>,
pub alternative_title: Option<String>,
pub publisher: Option<String>,
pub creator: Option<Value>,
pub publication_year: Option<String>,
pub subject: Option<Vec<String>>,
pub contributor: Option<Value>,
pub date: Option<Value>,
pub language: Option<String>,
pub resource_type: Option<Value>,
pub alternate_identifier: Option<String>,
#[serde(rename = "relatedidentifier")]
pub related_identifier: Option<Value>,
pub size: Option<String>,
pub format: Option<String>,
pub version: Option<String>,
pub rights: Option<Value>,
pub description: Option<String>,
pub geolocation: Option<String>,
}
impl TryFrom<datacite::Record> for Dataset {
type Error = CrosswalkError;
fn try_from(record: datacite::Record) -> Result<Self, Self::Error> {
let mapping = datacite_to_huwise();
crosswalk::convert(&record, &mapping).map(|(dataset, _)| dataset)
}
}
impl TryFrom<&datacite::Record> for Dataset {
type Error = CrosswalkError;
fn try_from(record: &datacite::Record) -> Result<Self, Self::Error> {
Dataset::try_from(record.clone())
}
}
impl SchemaBuilder for Dataset {
fn build_from_fields(fields: &Fields) -> Result<Self, CrosswalkError> {
let dataset_id = fields.get_string("identifier")?;
let datacite_meta = Some(build_datacite_block(fields));
let meta = Meta {
datacite: datacite_meta,
r#default: None,
dublin_core: None,
dcat: None,
dcat_ap: None,
custom_template: None,
};
Ok(Dataset {
dataset_id,
has_attachments: false,
attachments_count: 0,
has_records: false,
fields: Value::Array(vec![]),
metas: meta,
features: vec![],
})
}
}
fn build_datacite_block(fields: &Fields) -> Datacite {
let title = fields.get_string_opt("title");
let identifier = fields.get_string_opt("identifier");
let creator = fields
.get_string_vec_opt("creators")
.map(|names| Value::Array(names.into_iter().map(Value::String).collect()));
let publication_year = fields.get_number_opt("publication-year").map(|y| (y as i32).to_string());
let description = fields.get_string_opt("description");
let subject = fields.get_string_vec_opt("subjects");
let language = fields.get_string_opt("language");
let publisher = fields.get_string_opt("publisher");
let resource_type = fields.get_string_opt("resource-type").map(Value::String);
let rights = fields.get_string_opt("license").map(Value::String);
let version = fields.get_string_opt("version");
Datacite {
identifier,
title,
alternative_title: None,
publisher,
creator,
publication_year,
date: None,
subject,
contributor: None,
description,
language,
resource_type,
alternate_identifier: None,
related_identifier: None,
size: None,
format: None,
version,
rights,
geolocation: None,
}
}
impl SchemaExtractor for Dataset {
fn extract_fields(&self) -> Fields {
fn extract_json_string_array(value: &Value) -> Option<Vec<String>> {
match value {
| Value::Array(arr) => {
let strings: Vec<String> = arr.iter().filter_map(|v| v.as_str().map(|s| s.to_string())).collect();
if !strings.is_empty() {
Some(strings)
} else {
None
}
}
| Value::String(s) => Some(vec![s.clone()]),
| _ => None,
}
}
let mut fields = Fields::new();
fields.insert("identifier", FieldValue::String(self.dataset_id.clone()));
if let Some(meta) = &self.metas.datacite {
if let Some(value) = &meta.title {
fields.insert("title", FieldValue::String(value.clone()));
}
if let Some(value) = &meta.identifier {
fields.insert("doi", FieldValue::String(value.clone()));
}
if let Some(value) = &meta.creator {
if let Some(creators) = extract_json_string_array(value) {
if !creators.is_empty() {
fields.insert("creators", FieldValue::StringVec(creators));
}
}
}
if let Some(value) = &meta.publication_year {
if let Ok(year) = value.parse::<f64>() {
fields.insert("publication-year", FieldValue::Number(year));
}
}
if let Some(value) = &meta.description {
fields.insert("description", FieldValue::String(value.clone()));
}
if let Some(value) = &meta.subject {
if !value.is_empty() {
fields.insert("subjects", FieldValue::StringVec(value.clone()));
}
}
if let Some(value) = &meta.language {
fields.insert("language", FieldValue::String(value.clone()));
}
if let Some(value) = &meta.publisher {
fields.insert("publisher", FieldValue::String(value.clone()));
}
if let Some(value) = &meta.resource_type {
if let Some(s) = value.as_str() {
fields.insert("resource-type", FieldValue::String(s.to_string()));
}
}
if let Some(value) = &meta.rights {
if let Some(s) = value.as_str() {
fields.insert("license", FieldValue::String(s.to_string()));
}
}
if let Some(value) = &meta.version {
fields.insert("version", FieldValue::String(value.clone()));
}
} else if let Some(meta) = &self.metas.r#default {
if let Some(title) = &meta.title {
fields.insert("title", FieldValue::String(title.clone()));
}
if let Some(value) = &meta.description {
let sanitized = clean(value);
fields.insert("description", FieldValue::String(sanitized));
}
if let Some(value) = &meta.keyword {
if !value.is_empty() {
fields.insert("subjects", FieldValue::StringVec(value.clone()));
}
}
if let Some(value) = &meta.language {
fields.insert("language", FieldValue::String(value.clone()));
}
if let Some(value) = &meta.publisher {
fields.insert("publisher", FieldValue::String(value.clone()));
}
if let Some(value) = &meta.license {
fields.insert("license", FieldValue::String(value.clone()));
}
if let Some(value) = &meta.modified {
fields.insert("updated", FieldValue::Date(value.clone()));
}
} else if let Some(meta) = &self.metas.dublin_core {
if let Some(value) = &meta.title {
fields.insert("title", FieldValue::String(value.clone()));
}
if let Some(value) = &meta.abstract_text {
let sanitized = clean(value);
fields.insert("description", FieldValue::String(sanitized));
}
if let Some(value) = &meta.creator {
fields.insert("creators", FieldValue::String(value.clone()));
}
if let Some(value) = &meta.subject {
if !value.is_empty() {
fields.insert("subjects", FieldValue::StringVec(value.clone()));
}
}
if let Some(value) = &meta.language {
fields.insert("language", FieldValue::String(value.clone()));
}
if let Some(value) = &meta.publisher {
fields.insert("publisher", FieldValue::String(value.clone()));
}
if let Some(value) = &meta.issued {
fields.insert("publication-year", FieldValue::Date(value.clone()));
}
if let Some(value) = &meta.license {
fields.insert("license", FieldValue::String(value.clone()));
}
} else if let Some(meta) = &self.metas.dcat {
if let Some(value) = &meta.creator {
fields.insert("creators", FieldValue::String(value.clone()));
}
if let Some(value) = &meta.issued {
fields.insert("publication-year", FieldValue::Date(value.clone()));
}
if let Some(value) = &meta.spatial {
fields.insert("spatial", FieldValue::String(value.clone()));
}
} else if let Some(meta) = &self.metas.dcat_ap {
if let Some(value) = &meta.title {
fields.insert("title", FieldValue::String(value.clone()));
}
if let Some(value) = &meta.description {
let sanitized = clean(value);
fields.insert("description", FieldValue::String(sanitized));
}
if let Some(value) = &meta.keyword {
if !value.is_empty() {
fields.insert("subjects", FieldValue::StringVec(value.clone()));
}
}
if let Some(value) = &meta.publisher_name {
fields.insert("publisher", FieldValue::String(value.clone()));
}
}
fields
}
}
impl ToProse for Dataset {
fn to_prose(&self) -> String {
self.metas
.datacite
.iter()
.flat_map(|meta| meta.title.iter().cloned())
.chain(self.metas.datacite.iter().flat_map(|meta| meta.description.iter().cloned()))
.chain(self.metas.datacite.iter().flat_map(|meta| meta.subject.iter().flatten().cloned()))
.chain(self.metas.r#default.iter().flat_map(|meta| meta.title.iter().cloned()))
.chain(
self.metas
.r#default
.iter()
.flat_map(|meta| meta.description.iter().map(|value| clean(value))),
)
.chain(self.metas.r#default.iter().flat_map(|meta| meta.keyword.iter().flatten().cloned()))
.chain(self.metas.dublin_core.iter().flat_map(|meta| meta.title.iter().cloned()))
.chain(
self.metas
.dublin_core
.iter()
.flat_map(|meta| meta.description.iter().map(|value| clean(value))),
)
.chain(
self.metas
.dublin_core
.iter()
.flat_map(|meta| meta.abstract_text.iter().map(|value| clean(value))),
)
.chain(self.metas.dublin_core.iter().flat_map(|meta| meta.subject.iter().flatten().cloned()))
.chain(self.metas.dcat_ap.iter().flat_map(|meta| meta.title.iter().cloned()))
.chain(
self.metas
.dcat_ap
.iter()
.flat_map(|meta| meta.description.iter().map(|value| clean(value))),
)
.chain(self.metas.dcat_ap.iter().flat_map(|meta| meta.keyword.iter().flatten().cloned()))
.collect::<Vec<String>>()
.join("\n\n")
}
}
#[cfg(feature = "std")]
impl InputOutput for Dataset {
fn read(path: impl Into<PathBuf>) -> ApiResult<Dataset> {
let source = path.into();
match MimeType::from(source.display().to_string()) {
| MimeType::Json => Dataset::read_json(source),
| MimeType::Yaml => Dataset::read_yaml(source),
| _ => Err(eyre!("Unsupported HuWise data file extension")),
}
}
fn read_json(path: PathBuf) -> ApiResult<Dataset> {
#[derive(Deserialize)]
#[serde(untagged)]
enum JsonInput {
One(Box<Dataset>),
Many(Vec<Dataset>),
}
read_file(path).and_then(|content| {
serde_json::from_str::<JsonInput>(&content)
.map_err(|why| eyre!("Failed to parse JSON HuWise dataset — {why}"))
.and_then(|value| match value {
| JsonInput::One(dataset) => Ok(*dataset),
| JsonInput::Many(datasets) => match datasets.len() {
| 1 => datasets
.into_iter()
.next()
.ok_or_else(|| eyre!("Expected one HuWise dataset but found none")),
| len => Err(eyre!("Expected one HuWise dataset but found {len}")),
},
})
})
}
fn read_yaml(path: PathBuf) -> ApiResult<Dataset> {
#[derive(Deserialize)]
#[serde(untagged)]
enum YamlInput {
One(Box<Dataset>),
Many(Vec<Dataset>),
}
read_file(path).and_then(|content| {
serde_norway::from_str::<YamlInput>(&content)
.map_err(|why| eyre!("Failed to parse YAML HuWise dataset — {why}"))
.and_then(|value| match value {
| YamlInput::One(dataset) => Ok(*dataset),
| YamlInput::Many(datasets) => match datasets.len() {
| 1 => datasets
.into_iter()
.next()
.ok_or_else(|| eyre!("Expected one HuWise dataset but found none")),
| len => Err(eyre!("Expected one HuWise dataset but found {len}")),
},
})
})
}
fn write(&self, path: impl Into<PathBuf>) -> ApiResult<()> {
let output = path.into();
match MimeType::from(output.display().to_string()) {
| MimeType::Json => self.write_json(output),
| MimeType::Yaml => self.write_yaml(output),
| _ => Err(eyre!("Unsupported HuWise data file extension for writing")),
}
}
fn write_json(&self, path: impl Into<PathBuf>) -> ApiResult<()> {
let output = path.into().with_extension("json");
serde_json::to_string_pretty(self)
.map_err(|why| eyre!("Failed to serialize JSON HuWise dataset — {why}"))
.and_then(|content| write_file(output, content))
}
fn write_yaml(&self, path: impl Into<PathBuf>) -> ApiResult<()> {
let output = path.into().with_extension("yaml");
serde_norway::to_string(self)
.map_err(|why| eyre!("Failed to serialize YAML HuWise dataset — {why}"))
.and_then(|content| write_file(output, content))
}
}