batch-processing 0.1.17

A batch library for processing a list of items in parallel
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147

#[cfg(all(feature = "async", test))]
mod async_complex_step_test {
    use batch_processing::tokio::step::complex_step::{AsyncComplexStepBuilder, ComplexStepBuilderTrait};
    use batch_processing::tokio::step::step_builder::AsyncStepBuilderTrait;
    use batch_processing::tokio::step::AsyncStepRunner;
    use futures::{stream, Stream};
    use std::pin::Pin;
    use std::sync::Arc;
    use tokio::sync::Mutex;

    #[tokio::test]
    async fn test_build() {
        let step_builder: AsyncComplexStepBuilder<String, String> = AsyncComplexStepBuilder::get("test".to_string())
            .reader(Box::new(move ||
            {
                return Box::pin(async move {
                    let stream: Pin<Box<dyn Stream<Item=String> + Send>> =
                        Box::pin(stream::iter(vec![String::new()]));
                    stream
                }
                );
            }))
            .processor(
                Box::new(
                    move |item: String| Box::pin(
                        async move {
                            item.to_uppercase()
                        }
                    )
                )
            )
            .writer(
                Box::new(
                    move |items: Vec<String>| Box::pin(
                        async move {
                            println!("{:?}", items);
                        }
                    )
                )
            );

        let step = step_builder.build();

        step.run().await;
    }

    #[tokio::test]
    async fn test_fault_tolerant_is_false() {
        let vec = Arc::new(Mutex::new(Vec::new()));
        let vec_write = vec.clone();
        let step_builder: AsyncComplexStepBuilder<String, String> = AsyncComplexStepBuilder::get("test".to_string())
            .chunk_size(1)
            .reader(Box::new(move ||
                {
                    return Box::pin(async move {
                        let stream: Pin<Box<dyn Stream<Item=String> + Send>> =
                            Box::pin(stream::iter(vec!["test-failed".to_string(), "test".to_string()]));
                        stream
                    }
                    );
                }))
            .processor(
                Box::new(
                    move |item: String| Box::pin(
                        async move {
                            if item == "test-failed".to_string() {
                                panic!("test failed");
                            }
                            return item.to_uppercase();
                        }
                    )
                )
            )
            .writer(
                Box::new(
                    move |items: Vec<String>| {
                        let vec_write = vec_write.clone();
                        Box::pin(
                            async move {
                                let mut vec = vec_write.lock().await;
                                for item in items {
                                    vec.push(item);
                                }
                            }
                        )
                    }
                )
            );

        let step = step_builder.build();
        let step_result = step.run().await;
        let vec = vec.lock().await;
        assert_eq!(step_result.status.is_err(), true);
        assert_eq!(vec.len(), 0);
    }

    #[tokio::test]
    async fn test_fault_tolerant_is_true() {
        let vec = Arc::new(Mutex::new(Vec::new()));
        let vec_write = vec.clone();
        let step_builder: AsyncComplexStepBuilder<String, String> = AsyncComplexStepBuilder::get("test".to_string())
            .chunk_size(1)
            .throw_tolerant()
            .reader(Box::new(move ||
                {
                    return Box::pin(async move {
                        let stream: Pin<Box<dyn Stream<Item=String> + Send>> =
                            Box::pin(stream::iter(vec!["test-failed".to_string(), "test".to_string()]));
                        stream
                    }
                    );
                }))
            .processor(
                Box::new(
                    move |item: String| Box::pin(
                        async move {
                            if item == "test-failed".to_string() {
                                panic!("test failed");
                            }
                            return item.to_uppercase();
                        }
                    )
                )
            )
            .writer(
                Box::new(
                    move |items: Vec<String>| {
                        let vec_write = vec_write.clone();
                        Box::pin(
                            async move {
                                let mut vec = vec_write.lock().await;
                                for item in items {
                                    vec.push(item);
                                }
                            }
                        )
                    }
                )
            );

        let step = step_builder.build();
        let step_result = step.run().await;
        let vec = vec.lock().await;
        assert_eq!(step_result.status.is_err(), true);
        assert_eq!(vec.len(), 1);
    }
}