1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
use ;
use ;
use TypedBuilder;
/// default method used when open the `PipeLine`
async
/// default method used when close the `PipeLine`
async
/// default method used to consume extracted Entities
async
/// default method used to process parsed failure
async
/// the end of data flow, plugin that consume the extracted `Entity`, In general, the default
/// method does nothing, and customization is requird to store the data. An example:
/// ```
/// async fn process_item(items: &mut Arc<Mutex<Vec<I>>>) {
/// let itms = items.lock().unwrap();
/// for _ in itms.len() {
/// itms.pop();
/// }
/// println!("consumed {} items", itms.len() )
/// }
/// let pipeline = PipeLine::builder().process_item(&|items: &mut Arc<Mutex<Vec<I>>>|
/// process_item(items).boxed_local() );
/// ```
/// the member not specified is by default assigned as the default method
/*
* /// pipeline out the items
*#[async_trait]
*pub trait Pipeline<T, C>
*where
* T: std::fmt::Debug,
*{
* async fn open_pipeline(&self) -> &'static Option<Arc<C>>;
* async fn close_pipeline(&self);
* async fn process_item(&self, item: &mut Arc<Mutex<Vec<T>>>);
* async fn process_yielderr(&self, item: &mut Arc<Mutex<Vec<String>>>);
*}
*
*pub struct PipelineDefault<T, C> {
* _t: std::marker::PhantomData<T>,
* _c: std::marker::PhantomData<C>,
*}
*
*impl<T, C> PipelineDefault<T, C> {
* pub fn new() -> Self {
* PipelineDefault {
* _t: std::marker::PhantomData::<T>,
* _c: std::marker::PhantomData::<C>,
* }
* }
*}
*/
/*
*#[async_trait]
*impl<T> Pipeline<T, std::fs::File> for PipelineDefault<T, std::fs::File>
*where
* T: std::fmt::Debug + Send + Sync,
*{
* async fn open_pipeline(&self) -> &'static Option<Arc<std::fs::File>> {
* static INIT: Once = Once::new();
* static mut VAL: Option<Arc<std::fs::File>> = None;
* unsafe {
* INIT.call_once(|| {
* let file = std::fs::File::open("result").unwrap();
* VAL = Some(Arc::new(file));
* });
* &VAL
* }
* }
*
* async fn close_pipeline(&self) {
* drop(self);
* }
*
* async fn process_item(&self, item: &mut Arc<Mutex<Vec<T>>>)
* where
* T: Send + Sync,
* {
* let len = item.lock().unwrap().len();
* log::info!("process {} item", len);
* for _ in 0..len {
* let itm = item.lock().unwrap().pop().unwrap();
* println!("pipeline out item: {:?}", itm)
* }
* }
*
* async fn process_yielderr(&self, item: &mut Arc<Mutex<Vec<String>>>) {
* let len = item.lock().unwrap().len();
* log::info!("process {} yield_err", len);
* for _ in 0..len {
* let itm = item.lock().unwrap().pop().unwrap();
* println!("pipeline out item: {:?}", itm)
* }
* }
*}
*/