kineasy 0.1.0

Easy and performant crate for handling a kinesys stream
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100

#![feature(async_await, async_closure)]
use rusoto_kinesis::GetRecordsOutput;
use rusoto_core::Region;
use rusoto_kinesis::{
    Kinesis, KinesisClient,
    ListShardsInput, Record, Shard
};
use std::{sync::{Arc, RwLock}, time::Duration, collections::HashMap};
use futures::{future::{join_all}};
use tokio::{prelude::*, future, sync::mpsc};

use crate::checkpoint::Checkpointing;
use crate::shard::{ShardIterator, KinesisShard};

pub struct Kineasy
{
    running: Arc<RwLock<bool>>,
    region: Region,
    stream: String,
    cp: Checkpointing,
    client: Arc<KinesisClient>,
    iterator_type: ShardIterator,
    sender: mpsc::Sender<Record>,
    receiver: mpsc::Receiver<Record>,
    shards: HashMap<String, KinesisShard>,
}

impl Kineasy
{
    pub fn new (region: Region, stream: String, iterator_type: ShardIterator) -> Self {
        let (sender, receiver) = mpsc::channel::<Record>(100);

        Kineasy {
            client: Arc::new(KinesisClient::new(region.clone())),
            cp: Checkpointing::default(),
            shards: HashMap::new(),
            running: Arc::new(RwLock::new(false)),
            iterator_type,
            receiver,
            sender,
            region,
            stream,
        }
    }

    pub fn enable_checkpoint(&mut self, interval: Duration, file: Option<String>) {
        // TODO: move this logic to Checkpointing module
        self.cp.file = file;
        self.cp.active = true;
        self.cp.interval = interval;
    }

    fn list_shards(&mut self) -> Vec<Shard> {
        self.client.list_shards(
            ListShardsInput {
                stream_name: Some(self.stream.clone()),
                ..ListShardsInput::default()
            })
            .sync().expect("List shards failed.").shards.expect("No shards available.")
    }

    fn create_shards(&mut self) {
        for shard in self.list_shards().into_iter() {
            let s = KinesisShard::from_latest(self.region.clone(), self.stream.clone(), shard.shard_id.clone(), self.running.clone());
            self.shards.insert(shard.shard_id, s);
        }
    }

    pub async fn stream(mut self) -> mpsc::Receiver<Record> {
        *self.running.write().unwrap() = true;

        for shard in self.list_shards().into_iter() {
            let s = KinesisShard::from_latest(self.region.clone(), self.stream.clone(), shard.shard_id.clone(), self.running.clone());
            self.shards.insert(shard.shard_id, s);
        }

        let shards = self.shards.clone();
        let sender = self.sender.clone();

        let running = self.running.clone();
        tokio::spawn(async move {
            join_all(shards.values().map(|shard| {
                let s = shard.clone();

                let sender = sender.clone();
                s.take_while(|_| future::ready(*running.read().unwrap())).for_each(move |l: GetRecordsOutput| {
                    let mut sender = sender.clone();

                    for r in l.records {
                        let _ = sender.try_send(r);
                    }

                    future::ready(())
                })
            }).collect::<Vec<_>>()).await;
        });

        self.receiver
    }
}