dpcs 0.13.1

Reference implementation of the Data Pipeline Contract Standard (DPCS)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

dpcsVersion: "1.0.0"
id: "valid.execution.model"
version: "0.1.0"
interface:
  inputs:
    - id: "customer_raw"
      name: "Customer Raw"
      contractRef: "contracts/customer_raw.odcs.yaml"
      purpose: "Raw customer dataset input"
  outputs:
    - id: "customer_clean"
      name: "Customer Clean"
      contractRef: "contracts/customer_clean.odcs.yaml"
      purpose: "Cleaned customer dataset output"
contractReferences:
  - id: "customer_raw_contract"
    type: "odcs"
    location: "contracts/customer_raw.odcs.yaml"
  - id: "normalize_customer_transform"
    type: "dtcs"
    location: "transforms/normalize_customer.dtcs.yaml"
steps:
  - id: "normalize_customer"
    type: "dtcs:transform"
    contractRef: "normalize_customer_transform"
    inputs:
      - id: "customer_raw"
    outputs:
      - id: "customer_clean"
graph:
  edges: []
dataFlow:
  - from: "interface.inputs.customer_raw"
    to: "steps.normalize_customer.inputs.customer_raw"
    dataset: "customer_raw"
  - from: "steps.normalize_customer.outputs.customer_clean"
    to: "interface.outputs.customer_clean"
    dataset: "customer_clean"
execution:
  requiredCapabilities:
    - "batch.compute"
  resources:
    memory: "4Gi"
    processor: "2"
  isolation:
    - "container"
  externalDependencies:
    - id: "warehouse"
      capability: "sql.readwrite"
      availability: "required"
scheduling:
  - mode: "scheduled"
    cron: "0 2 * * *"
    timezone: "UTC"
    constraints:
      earliest: "2026-01-01T00:00:00Z"
      latest: "2026-12-31T23:59:59Z"
qualityGates:
  - id: "pre_run_schema"
    purpose: "Validate input contract before normalize"
    category: "contractValidation"
    criteria:
      - contractRef: "customer_raw_contract"
        type: "odcs"
    onSuccess: "continue"
    onFailure: "abort"
    placement:
      kind: "beforeStep"
      stepId: "normalize_customer"
failureSemantics:
  - id: "normalize_retry"
    scope:
      kind: "step"
      stepId: "normalize_customer"
    category: "execution"
    triggers:
      - "transientExecutionFailure"
    responses:
      - "retry"
    retry:
      eligible: true
      maxAttempts: 3
      delayPolicy: "exponential"
lineage:
  datasets:
    - dataset: "customer_raw"
      consumedBy:
        - "normalize_customer"
      contractRef: "customer_raw_contract"
    - dataset: "customer_clean"
      producedBy: "normalize_customer"
  steps:
    - stepId: "normalize_customer"
      contractRef: "normalize_customer_transform"
  provenance:
    originating: "valid.execution.model"