emmett 0.1.3-alpha

a simple etl tool
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102

use std::path::Path;

/// This input will read events from a Kafka topic.
pub struct Kafka {
    pub auto_commit_interval_ms: Option<String>,
    pub auto_offset_reset: Option<String>,
    pub bootstrap_servers: Option<String>,
    pub check_crcs: Option<String>,
    pub client_id: Option<String>,
    pub connections_max_idle_ms: Option<String>,
    pub consumer_threads: Option<u64>,
    pub decorate_events: Option<bool>,
    pub enable_auto_commit: Option<String>,
    pub exclude_internal_topics: Option<String>,
    pub fetch_max_bytes: Option<String>,
    pub fetch_max_wait_ms: Option<String>,
    pub fetch_min_bytes: Option<String>,
    pub group_id: Option<String>,
    pub heartbeat_interval_ms: Option<String>,
    pub jaas_path: Option<&'static Path>,
    pub kerberos_config: Option<String>,
    pub key_deserializer_class: Option<String>,
    pub max_partition_fetch_bytes: Option<String>,
    pub max_poll_interval_ms: Option<String>,
    pub max_poll_records: Option<String>,
    pub metadata_max_age_ms: Option<String>,
    pub partition_assignment_strategy: Option<String>,
    pub poll_timeout_ms: Option<u64>,
    pub receive_buffer_bytes: Option<String>,
    pub reconnect_backoff_ms: Option<String>,
    pub request_timeout_ms: Option<String>,
    pub retry_backoff_ms: Option<String>,
    pub sasl_kerberos_service_name: Option<String>,
    pub sasl_mechanism: Option<String>,
    pub security_protocol: Option<String>,
    pub send_buffer_bytes: Option<String>,
    pub session_timeout_ms: Option<String>,
    pub ssl_endpoint_identification_algorithm: Option<String>,
    pub ssl_key_password: Option<String>,
    pub ssl_keystore_location: Option<&'static Path>,
    pub ssl_keystore_password: Option<String>,
    pub ssl_keystore_type: Option<String>,
    pub ssl_truststore_location: Option<&'static Path>,
    pub ssl_truststore_password: Option<String>,
    pub ssl_truststore_type: Option<String>,
    pub topics: Option<Vec<String>>,
    pub topics_pattern: Option<String>,
    pub value_deserializer_class: Option<String>,
}

impl Kafka {
    pub fn new() -> Self {
        Self {
            auto_commit_interval_ms: Some("5000".to_string()),
            auto_offset_reset: None,
            bootstrap_servers: Some("localhost:9092".to_string()),
            check_crcs: None,
            client_id: Some("logstash".to_string()),
            connections_max_idle_ms: None,
            consumer_threads: Some(1),
            decorate_events: Some(false),
            enable_auto_commit: Some("true".to_string()),
            exclude_internal_topics: None,
            fetch_max_bytes: None,
            fetch_max_wait_ms: None,
            fetch_min_bytes: None,
            group_id: Some("logstash".to_string()),
            heartbeat_interval_ms: None,
            jaas_path: None,
            kerberos_config: Some("org.apache.kafka.common.serialization.StringDeserializer".to_string()),
            key_deserializer_class: None,
            max_partition_fetch_bytes: None,
            max_poll_interval_ms: None,
            max_poll_records: None,
            metadata_max_age_ms: None,
            partition_assignment_strategy: None,
            poll_timeout_ms: Some(100),
            receive_buffer_bytes: None,
            reconnect_backoff_ms: None,
            request_timeout_ms: None,
            retry_backoff_ms: None,
            sasl_kerberos_service_name: None,
            sasl_mechanism: Some("GSSAPI".to_string()),
            security_protocol: Some("PLAINTEXT".to_string()),
            send_buffer_bytes: None,
            session_timeout_ms: None,
            ssl_endpoint_identification_algorithm: Some("https".to_string()),
            ssl_key_password: None,
            ssl_keystore_location: None,
            ssl_keystore_password: None,
            ssl_keystore_type: None,
            ssl_truststore_location: None,
            ssl_truststore_password: None,
            ssl_truststore_type: None,
            topics: Some(vec!["logstash".to_string()]),
            topics_pattern: None,
            value_deserializer_class: Some(
                "org.apache.kafka.common.serialization.StringDeserializer".to_string(),
            ),
        }
    }
}