stream-consumer-task 0.1.0

Start asynchronous task in background to consume stream
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

use std::time::Duration;

use rdkafka::{
    consumer::{Consumer, StreamConsumer},
    producer::{FutureProducer, FutureRecord},
    ClientConfig, Message,
};
use stream_consumer_task::{KafkaStreamConsumerAdapter, StreamConsumerTask};
use tokio::{
    select,
    signal::unix::{signal, SignalKind},
    spawn,
    task::JoinSet,
    time::sleep,
};

// main

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    let brokers = "127.0.0.1:9092";
    let topic = "test";
    let csm: KafkaStreamConsumerAdapter = ClientConfig::new()
        .set("bootstrap.servers", brokers)
        .set("group.id", "test")
        .set("enable.auto.commit", "true")
        .create::<StreamConsumer>()?
        .into();
    csm.as_ref().subscribe(&[topic])?;
    let csm = StreamConsumerTask::start(csm, |msg, mut stop_rx| async move {
        match msg {
            Ok(msg) => {
                let res: anyhow::Result<()> = async {
                    let payload = msg.payload().map(Vec::from).unwrap_or_default();
                    let payload = String::from_utf8(payload)?;
                    select! {
                        _ = sleep(Duration::from_secs(5)) => {
                            println!("{payload} - processed");
                        }
                        _ = stop_rx.changed() => {
                            println!("{payload} - stopped");
                        }
                    }
                    Ok(())
                }
                .await;
                if let Err(err) = res {
                    eprintln!("failed to handle message: {err}");
                }
            }
            Err(err) => {
                eprintln!("failed to handle message: {err}");
            }
        }
    });
    let mut idx = 0;
    let prod: FutureProducer = ClientConfig::new()
        .set("bootstrap.servers", brokers)
        .create()?;
    let _task = spawn(async move {
        loop {
            let payload = format!("msg_{idx:0>3}");
            let res = prod
                .send(
                    FutureRecord::to(topic)
                        .key(&format!("{idx}"))
                        .payload(&payload),
                    Duration::from_secs(0),
                )
                .await;
            match res {
                Ok(_) => println!("{payload} - sent"),
                Err((err, _)) => eprintln!("failed to send payload: {err}"),
            }
            sleep(Duration::from_secs(1)).await;
            idx += 1;
        }
    });
    let mut sig_int = signal(SignalKind::interrupt())?;
    let mut sig_term = signal(SignalKind::terminate())?;
    let mut sigs = JoinSet::new();
    sigs.spawn(async move {
        sig_int.recv().await;
    });
    sigs.spawn(async move {
        sig_term.recv().await;
    });
    sigs.join_next().await;
    csm.stop().await;
    Ok(())
}