acorn/schema/research_activity/aspect/
data.rs

1use crate::prelude::*;
2#[cfg(feature = "std")]
3use crate::schema::agent::Model;
4use crate::schema::namespaces::{codemeta, schema_org};
5use crate::util::{LinkedData, StringInterpolation, ToMarkdown};
6use bon::Builder;
7use core::str::FromStr;
8use derive_more::Display;
9use schemars::JsonSchema;
10use serde::{Deserialize, Serialize};
11use serde_repr::{Deserialize_repr, Serialize_repr};
12
13/// Description of obstacles to shared access
14///
15/// Data availability is strongly related to the concept of "openness" and the "presumed open principle" for data.
16///
17/// Where openness is characterized as open, public, shared, or closed, availability takes a more direct approach to
18/// describing the actual availability of data, which may be open but still unavailable due to other factors
19/// (e.g., technical barriers, legal barriers, etc.).
20#[derive(Clone, Debug, Default, Deserialize_repr, Display, Serialize_repr, PartialEq, PartialOrd, JsonSchema)]
21#[repr(u8)]
22#[serde(deny_unknown_fields)]
23pub enum Availability {
24    /// Data is not available by any means
25    #[default]
26    Unavailable = 0,
27    /// Data is available but access is restricted (partial availability)
28    Restricted = 1,
29    /// Data is available and access is unrestricted (open source)
30    Unrestricted = 2,
31}
32/// Describe the data that a system can process, use, and depend on
33///
34/// The attribute of data that designates "real" vs "synthetic" is intended to capture the ***origin*** of the data, which can have implications for its quality, reliability, and applicability to different tasks.
35#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
36pub enum Data {
37    /// Data that has been collected from natural events
38    Real(DataDescription),
39    /// Data that has been artificially created by computer algorithms
40    Synthetic(DataDescription),
41    /// Opaque data artifact
42    #[cfg(feature = "std")]
43    Model(Box<Model>),
44}
45/// Framework for assessing the readiness of a data set
46///
47/// Adapted from the concept of "data readiness levels" as described by N. Lawrence in [Data Readiness Levels](http://arxiv.org/abs/1705.02245).
48#[derive(Clone, Debug, Deserialize, Display, Serialize, JsonSchema)]
49pub enum DataReadinessLevel {
50    /// Bottom of Band C. Hearsay data — existence is not yet verified. Signs include statements such
51    /// as "The sales department should have a record of that." Problems may include whether the data
52    /// is actually being recorded, the format in which it is recorded, privacy or legal constraints,
53    /// and topology limitations (e.g., data distributed across many devices).
54    #[display("Level E: Hearsay")]
55    E,
56    /// Top of Band C. Data is machine readable and ethical procedures for data handling have been
57    /// addressed. It is ready to be loaded into analysis software or made available via a data
58    /// repository. Reaching D often requires significant bespoke software and human understanding
59    /// of systems, ethics, and the law.
60    #[display("Level D: Accessible")]
61    D,
62    /// Beginning of Band C. Data is accessible and loaded into analysis software but its faithfulness
63    /// and representation have not yet been characterized. Missing values, noise, data entry errors,
64    /// unit correctness, and collection bias have not been assessed.
65    #[display("Level C: Loaded")]
66    C,
67    /// End of Band B. The analyst understands how faithful the data is to its original description.
68    /// A broad idea of limitations in the data is present and an intuition about what may really be
69    /// possible with the data set is beginning to form. Getting to this point is often the most
70    /// expensive part of a data project.
71    #[display("Level B: Faithful")]
72    B,
73    /// Data in context. The data set has been assessed as appropriate for a specific task or question.
74    /// Any remedial steps have been taken and the data is ready to be deployed in the given context.
75    /// A data set may be A1 for one question but only B1 for a different question; the task definition
76    /// is an important pre-requisite for this band.
77    #[display("Level A: Appropriate")]
78    A,
79}
80/// Levels adapted from NASA Earth Observing System Data and Information System ([EOSDIS]) data processing levels,
81/// which are intended to describe the level of processing that has been applied to a given data set.
82/// These levels are widely used in the Earth science community and provide a useful framework for understanding the maturity and usability of data products.
83///
84/// See <https://www.earthdata.nasa.gov/learn/earth-observation-data-basics/data-processing-levels> for more information.
85///
86/// [EOSDIS]: https://www.earthdata.nasa.gov/about/esdis/eosdis
87#[derive(Clone, Debug, Deserialize, Display, Serialize, JsonSchema)]
88pub enum GeospatialDataProcessingLevel {
89    /// Reconstructed, unprocessed ("raw") instrument and payload data at full resolution,
90    /// with any and all communications artifacts (e.g., synchronization frames,
91    /// communications headers, duplicate data) removed.
92    #[display("Level 0: Raw")]
93    L0,
94    /// Level 1A (L1A) data are reconstructed, unprocessed instrument data at full
95    /// resolution, time-referenced, and annotated with ancillary information.
96    /// This ancillary information can include radiometric and geometric calibration
97    /// coefficients and georeferencing parameters (e.g., platform ephemeris).
98    #[display("Level 1A: Annotated")]
99    L1A,
100    /// L1B data are L1A data that have been processed to instrument units
101    /// (not all instruments have L1B source data).
102    #[display("Level 1B: Processed Annotated")]
103    L1B,
104    /// L1C data are L1B data that include new variables to describe the spectra.
105    /// These variables allow the user to identify which L1C channels have been
106    /// copied directly from the L1B and which have been synthesized from L1B and why.
107    #[display("Level 1C: Spectral Variables")]
108    L1C,
109    /// Derived geophysical variables at the same resolution and location as
110    /// L1 source data.
111    #[display("Level 2: Derived Geophysical")]
112    L2,
113    /// L2A data contains information derived from the geolocated instrument data,
114    /// such as ground elevation, highest and lowest surface return elevations,
115    /// energy quantile heights ("relative height" metrics), and other
116    /// waveform-derived metrics describing the intercepted surface.
117    #[display("Level 2A: Derived Surface")]
118    L2A,
119    /// L2B data are L2A data that have been processed to instrument units
120    /// (not all instruments will have a L2B equivalent).
121    #[display("Level 2B: Processed Derived Surface")]
122    L2B,
123    /// Variables mapped on uniform space-time grid scales, usually with some
124    /// completeness and consistency.
125    #[display("Level 3: Gridded")]
126    L3,
127    /// L3A data are generally periodic summaries (weekly, 10-day, monthly)
128    /// of L2 products.
129    #[display("Level 3A: Periodic Summaries")]
130    L3A,
131    /// Model output or results from analyses of lower-level data
132    /// (e.g., variables derived from multiple measurements).
133    #[display("Level 4: Model Output")]
134    L4,
135}
136/// Specific type or form of data that a system can process and learn from
137#[derive(Clone, Debug, Default, Display, Deserialize, Serialize, JsonSchema)]
138#[serde(rename_all = "lowercase")]
139pub enum Modality {
140    /// Textual data that includes text and tabular data
141    #[default]
142    #[display("text")]
143    Text,
144    /// Audio data such as MPEG
145    #[display("audio")]
146    Audio,
147    /// Video data such as MP4
148    #[display("video")]
149    Video,
150    /// Signal data
151    #[display("signal")]
152    Signal,
153    /// Graph data or generally data with relational structure
154    #[display("graph")]
155    Graph,
156    /// Image data such as PNG, JPEG
157    #[display("image")]
158    Image,
159    /// PDF document data
160    #[display("pdf")]
161    Pdf,
162}
163/// Framework for partitioning data by quality using metaphor of mined metals
164#[derive(Clone, Debug, Display, Deserialize, Serialize, JsonSchema)]
165pub enum Quality {
166    /// Raw and unprocessed
167    Raw = 0,
168    /// Ingested
169    Bronze = 1,
170    /// Processed in some form short of being "AI ready" (e.g., large scale consumption)
171    Silver = 2,
172    /// AI-ready
173    Gold = 3,
174    /// AI-ready data that is adapted to a specific application and/or subjected to additional processing
175    Platinum = 4,
176}
177/// Common attributes of data
178#[derive(Builder, Clone, Debug, Deserialize, Serialize, JsonSchema)]
179#[builder(start_fn = init)]
180pub struct DataDescription {
181    /// Linked data (e.g., JSON-LD) context for data description
182    #[serde(rename = "@context")]
183    pub context: Option<DataDescriptionContext>,
184    /// Linked data (e.g., JSON-LD) type for data description
185    #[serde(rename = "@type")]
186    pub description_type: Option<String>,
187    /// Replaces ADEPT concept of "openness"
188    pub availability: Option<Availability>,
189    /// License that governs use of data
190    // TODO: validate with is_license
191    pub license: Option<String>,
192    /// Modality of data
193    pub modality: Option<Modality>,
194    /// Quality of data
195    pub quality: Option<Quality>,
196}
197/// Linked data (e.g., JSON-LD) context for data description
198#[derive(Builder, Clone, Debug, Deserialize, Serialize, JsonSchema)]
199#[builder(start_fn = init, on(String, into))]
200pub struct DataDescriptionContext {
201    pub(crate) availability: String,
202    pub(crate) license: String,
203    pub(crate) modality: String,
204    pub(crate) quality: String,
205}
206impl Default for DataDescription {
207    fn default() -> Self {
208        DataDescription::init().build()
209    }
210}
211impl Default for DataDescriptionContext {
212    fn default() -> Self {
213        DataDescriptionContext::init()
214            .availability(schema_org("DefinedTerm"))
215            .license(codemeta("softwareVersion"))
216            .modality(schema_org("DefinedTerm"))
217            .quality(schema_org("DefinedTerm"))
218            .build()
219    }
220}
221impl ToMarkdown for Data {
222    fn to_markdown(&self) -> String {
223        match self {
224            | Self::Real(description) => format!("- Real{}", description.to_markdown()),
225            | Self::Synthetic(description) => format!("- Synthetic{}", description.to_markdown()),
226            #[cfg(feature = "std")]
227            | Self::Model(model) => match model.as_ref() {
228                | Model::SLM(details) => format!("- Model\n  - Kind: SLM\n{}", details.to_markdown().with_indent(2)),
229                | Model::LLM(details) => format!("- Model\n  - Kind: LLM\n{}", details.to_markdown().with_indent(2)),
230            },
231        }
232    }
233}
234impl ToMarkdown for DataDescription {
235    fn to_markdown(&self) -> String {
236        let lines = [
237            self.availability.as_ref().map(|value| format!("Availability: {value}")),
238            self.license.as_ref().map(|value| format!("License: {value}")),
239            self.modality.as_ref().map(|value| format!("Modality: {value}")),
240            self.quality.as_ref().map(|value| format!("Quality: {}", value)),
241        ]
242        .into_iter()
243        .flatten()
244        .collect::<Vec<_>>();
245        if lines.is_empty() {
246            String::new()
247        } else {
248            format!("\n{}", lines.to_markdown().trim_start().with_indent(2))
249        }
250    }
251}
252impl LinkedData for DataDescription {
253    fn with_context(&self) -> Self {
254        Self {
255            context: Some(DataDescriptionContext::default()),
256            description_type: None,
257            ..self.clone()
258        }
259    }
260}
261impl FromStr for GeospatialDataProcessingLevel {
262    type Err = String;
263
264    fn from_str(value: &str) -> Result<Self, Self::Err> {
265        match value {
266            | "LV0" | "L0" | "0" => Ok(Self::L0),
267            | "LV1A" | "L1A" | "1A" => Ok(Self::L1A),
268            | "LV1B" | "L1B" | "1B" => Ok(Self::L1B),
269            | "LV1C" | "L1C" | "1C" => Ok(Self::L1C),
270            | "LV2" | "L2" | "2" => Ok(Self::L2),
271            | "LV2A" | "L2A" | "2A" => Ok(Self::L2A),
272            | "LV2B" | "L2B" | "2B" => Ok(Self::L2B),
273            | "LV3" | "L3" | "3" => Ok(Self::L3),
274            | "LV3A" | "L3A" | "3A" => Ok(Self::L3A),
275            | "LV4" | "L4" | "4" => Ok(Self::L4),
276            | _ => Err(format!("unknown GeospatialDataProcessingLevel: {value}")),
277        }
278    }
279}
acorn/schema/research_activity/aspect/data.rs

acorn/schema/research_activity/aspect/
data.rs
