1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

#[macro_use]
extern crate serde_derive;
#[macro_use]
extern crate log;
#[macro_use]
extern crate rlink_derive;
#[macro_use]
extern crate anyhow;

pub mod sink;
pub mod source;

pub mod buffer_gen {
    include!(concat!(env!("OUT_DIR"), "/buffer_gen/mod.rs"));
}

pub use sink::output_format::KafkaOutputFormat;
pub use source::input_format::KafkaInputFormat;

use rlink::core::element::Record;

use crate::buffer_gen::kafka_message;

pub const KAFKA: &str = "kafka";
pub const BOOTSTRAP_SERVERS: &str = "bootstrap.servers";
pub const GROUP_ID: &str = "group.id";

pub const TOPICS: &str = "topics";
pub const BUFFER_SIZE: &str = "buffer.size";

pub const OFFSET: &str = "offset";
pub const OFFSET_TYPE: &str = "type";
pub const OFFSET_BEGIN: &str = "begin";
pub const OFFSET_END: &str = "end";

pub const INPUT_FORMAT_FN_NAME_DEFAULT: &str = "KafkaInputFormat";
pub const OUTPUT_FORMAT_FN_NAME_DEFAULT: &str = "KafkaOutputFormat";

pub const SOURCE_CHANNEL_SIZE: usize = 50000;
pub const SINK_CHANNEL_SIZE: usize = 50000;

pub fn build_kafka_record(
    timestamp: i64,
    key: &[u8],
    payload: &[u8],
    topic: &str,
    partition: i32,
    offset: i64,
) -> Result<Record, std::io::Error> {
    let message = kafka_message::Entity {
        timestamp,
        key,
        payload,
        topic,
        partition,
        offset,
    };

    // 36 = 12(len(payload) + len(topic) + len(key)) +
    //      20(len(timestamp) + len(partition) + len(offset)) +
    //      4(place_holder)
    let capacity = payload.len() + topic.len() + key.len() + 36;
    let mut record = Record::with_capacity(capacity);

    message.to_buffer(record.as_buffer()).unwrap();

    Ok(record)
}