dpcs 0.13.1

Reference implementation of the Data Pipeline Contract Standard (DPCS)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

dpcsVersion: "1.0.0"
id: "valid.graph.features"
version: "0.1.0"
interface:
  inputs:
    - id: "raw"
      name: "Raw"
      contractRef: "contracts/raw.odcs.yaml"
      purpose: "Raw ingress"
  outputs:
    - id: "published"
      name: "Published"
      contractRef: "contracts/published.odcs.yaml"
      purpose: "Published egress"
graph:
  entryPoints:
    - "ingest"
  exitPoints:
    - "publish"
  metadata:
    description: "Linear ingestion graph"
  edges:
    - from: "ingest"
      to: "transform"
      kind: "dependsOn"
    - from: "transform"
      to: "publish"
steps:
  - id: "ingest"
    type: "extension:ingress"
    inputs:
      - id: "raw"
    outputs:
      - id: "raw"
  - id: "transform"
    type: "dtcs:transform"
    inputs:
      - id: "raw"
    outputs:
      - id: "clean"
  - id: "publish"
    type: "extension:egress"
    inputs:
      - id: "clean"
    outputs:
      - id: "published"
dataFlow:
  - from: "interface.inputs.raw"
    to: "steps.ingest.inputs.raw"
    dataset: "raw"
  - from: "steps.ingest.outputs.raw"
    to: "steps.transform.inputs.raw"
    dataset: "raw"
    contractRef: "raw_contract"
  - from: "steps.transform.outputs.clean"
    to: "steps.publish.inputs.clean"
    dataset: "clean"
  - from: "steps.publish.outputs.published"
    to: "interface.outputs.published"
    dataset: "published"
contractReferences:
  - id: "raw_contract"
    type: "odcs"
    location: "contracts/raw.odcs.yaml"