kafka-replicator 0.5.1

Application for replication data between kafka clusters.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81

clusters:
  - name: cluster_1
    hosts:
      #- replicator-kafka-1:9092
      #- replicator-kafka-1:9092
      - kafka_replicator_replicator_kafka_1_1:9092
  - name: cluster_2
    hosts:
      - kafka_replicator_replicator_kafka_2_1:9092
      #- replicator-kafka-2:9092

clients:
  - client: cl_1_client_1
    cluster: cluster_1
    config: # optional
       message.timeout.ms: 5000
       auto.offset.reset: earliest
  - client: cl_2_client_1
    cluster: cluster_2

routes:
  - upstream_client: cl_1_client_1 # Required: source client name from clients section
    downstream_client: cl_1_client_1 # Required: target client name from clients section
    upstream_topics: # Required: source topics
      - 'topic1'
    downstream_topic: 'topic2' # Required: target topics
    repartitioning_strategy: random # strict_p2p | random
    upstream_group_id: group_22 # Option: upstream consumer group id
    show_progress_interval_secs: 2 # Optional: interval between console output
    update_metrics_interval_secs: 1 # Optional: interval between prometheus metrics updates
    upstream_poll_interval_ms: 200 # Optional: interval between upstream poll calls
    name: route_name # Optional:
    limits:
      messages_per_sec: 10000
      number_of_messages:

  - upstream_client: cl_1_client_1
    downstream_client: cl_2_client_1
    upstream_topics:
      - 'topic2'
    downstream_topic: 'topic2'
    repartitioning_strategy: random # strict_p2p | random
    upstream_group_id: group_22
    show_progress_interval_secs: 10

  - upstream_client: cl_2_client_1
    downstream_client: cl_1_client_1
    upstream_topics:
      - 'topic2'
    downstream_topic: 'topic3'
    repartitioning_strategy: strict_p2p # strict_p2p | random
    default_begin_offset: earliest # optional
    upstream_group_id: group_2
    show_progress_interval_secs: 10


observers:
  - client: cl_1_client_1
    name: "Observer name"
    group_id: group_22
    topics:
      - 'topic1'
      # - 'topic2'
    fetch_timeout_secs: 5 # default: 5
    fetch_interval_secs: 60 # default:
    show_progress_interval_secs: 10 # default: 60

  - client: cl_2_client_1
    # group_id: group_2
    topics:
      - 'topic2'
    show_progress_interval_secsy: 20

  - client: cl_1_client_1
    topics: [] # process all topics

prometheus:
#  namespace: "app:observer:" # custom prometheus metrics prefix
  labels:
    label_key: label_value
    env: prod