pipeflow 0.0.4

A lightweight, configuration-driven data pipeline framework
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

pipeline:
  sources:
    - id: redis_in
      type: redis
      config:
        url: "redis://127.0.0.1:6379/0"
        key: "pipeflow:input"
        mode: poll
        interval: "1s"
        parse_json: false

  transforms:
    - id: pass_through
      inputs: [redis_in]
      outputs: [redis_out]

  sinks:
    - id: redis_out
      type: redis
      config:
        url: "redis://127.0.0.1:6379/0"
        key:
          value: "pipeflow:output"
        value:
          from: "$.value"
        ttl: "30s"