dpcs 0.11.0

Reference implementation of the Data Pipeline Contract Standard (DPCS)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

//! Pipeline lineage model (SPEC Ch 14).

use serde::{Deserialize, Serialize};

use super::ExtensionMap;

/// Declared lineage information for a Pipeline Contract.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct PipelineLineage {
    /// Dataset provenance relationships.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub datasets: Vec<DatasetLineage>,
    /// Step provenance relationships.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub steps: Vec<StepLineage>,
    /// Pipeline contract provenance.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub provenance: Option<PipelineProvenance>,
    /// Optional audit metadata (non-semantic).
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub audit: Option<LineageAudit>,
    /// Extension fields.
    #[serde(default, flatten)]
    pub extensions: ExtensionMap,
}

/// Dataset provenance within a pipeline.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct DatasetLineage {
    /// Dataset identity.
    pub dataset: String,
    /// Producing pipeline step identifier.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub produced_by: Option<String>,
    /// Consuming pipeline step identifiers.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub consumed_by: Vec<String>,
    /// Associated data contract reference id.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub contract_ref: Option<String>,
    /// Associated transformation contract reference id.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub transform_ref: Option<String>,
    /// Extension fields.
    #[serde(default, flatten)]
    pub extensions: ExtensionMap,
}

/// Step provenance within a pipeline.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct StepLineage {
    /// Step identity.
    #[serde(rename = "stepId")]
    pub step_id: String,
    /// Predecessor step identifiers.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub predecessors: Vec<String>,
    /// Successor step identifiers.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub successors: Vec<String>,
    /// Dependency semantics description.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub dependency_kind: Option<String>,
    /// Associated contract reference id.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub contract_ref: Option<String>,
    /// Extension fields.
    #[serde(default, flatten)]
    pub extensions: ExtensionMap,
}

/// Provenance describing contract origin and relationships.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct PipelineProvenance {
    /// Originating pipeline contract identifier.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub originating: Option<String>,
    /// Parent pipeline contract identifiers.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub parents: Vec<String>,
    /// Nested pipeline contract identifiers.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub nested: Vec<String>,
    /// Imported pipeline contract identifiers.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub imported: Vec<String>,
    /// Version history entries.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub version_history: Vec<String>,
    /// Extension fields.
    #[serde(default, flatten)]
    pub extensions: ExtensionMap,
}

/// Non-semantic audit metadata for lineage.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))]
#[serde(rename_all = "camelCase")]
pub struct LineageAudit {
    /// Contract identifiers for audit.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub contract_ids: Vec<String>,
    /// Version identifiers for audit.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub version_ids: Vec<String>,
    /// Timestamps for audit.
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub timestamps: Vec<String>,
    /// Extension fields.
    #[serde(default, flatten)]
    pub extensions: ExtensionMap,
}