crypto-crawler 3.3.5

A rock-solid cryprocurrency crawler.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

use crypto_crawler::Message;
use crypto_market_type::MarketType;
use crypto_msg_type::MessageType;

pub(crate) fn parse(msg: Message) -> bool {
    match msg.msg_type {
        MessageType::Trade => {
            crypto_msg_parser::parse_trade(&msg.exchange, msg.market_type, &msg.json).is_ok()
        }
        MessageType::L2Event => {
            match msg.market_type {
                // crypto-msg-parser doesn't support quanto contracts
                MarketType::QuantoSwap | MarketType::QuantoFuture => true,
                _ => crypto_msg_parser::parse_l2(
                    &msg.exchange,
                    msg.market_type,
                    &msg.json,
                    Some(msg.received_at as i64),
                )
                .is_ok(),
            }
        }
        MessageType::FundingRate => {
            crypto_msg_parser::parse_funding_rate(&msg.exchange, msg.market_type, &msg.json).is_ok()
        }
        _ => true,
    }
}

#[allow(unused_macros)]
macro_rules! test_one_symbol {
    ($crawl_func:ident, $exchange:expr, $market_type:expr, $symbol:expr, $msg_type:expr) => {{
        let (tx, rx) = std::sync::mpsc::channel();
        let mut messages = Vec::new();
        let symbols = vec![$symbol.to_string()];
        $crawl_func($exchange, $market_type, Some(&symbols), tx, Some(0));

        for msg in rx {
            messages.push(msg);
        }

        assert!(!messages.is_empty());
        assert_eq!(messages[0].exchange, $exchange.to_string());
        assert_eq!(messages[0].market_type, $market_type);
        assert_eq!(messages[0].msg_type, $msg_type);
        for msg in messages {
            assert!(parse(msg));
        }
    }};
}

#[allow(unused_macros)]
macro_rules! test_all_symbols {
    ($crawl_func:ident, $exchange:expr, $market_type:expr, $msg_type:expr) => {{
        let (tx, rx) = std::sync::mpsc::channel();
        let mut messages = Vec::new();
        let symbols = if $market_type == MarketType::Spot {
            let spot_symbols = fetch_symbols_retry($exchange, $market_type);
            get_hot_spot_symbols($exchange, &spot_symbols)
        } else {
            fetch_symbols_retry($exchange, $market_type)
        };
        $crawl_func($exchange, $market_type, Some(&symbols), tx, Some(0));

        for msg in rx {
            messages.push(msg);
        }

        assert!(!messages.is_empty());
        assert_eq!(messages[0].exchange, $exchange.to_string());
        assert_eq!(messages[0].market_type, $market_type);
        assert_eq!(messages[0].msg_type, $msg_type);
        for msg in messages {
            assert!(parse(msg));
        }
    }};
}

#[allow(unused_macros)]
macro_rules! gen_test_crawl_candlestick {
    ($exchange:expr, $market_type:expr) => {{
        let (tx, rx) = std::sync::mpsc::channel();
        let mut messages = Vec::new();
        crawl_candlestick($exchange, $market_type, None, tx, Some(0));
        for msg in rx {
            messages.push(msg);
        }
        assert!(!messages.is_empty());
        assert_eq!(messages[0].exchange, EXCHANGE_NAME.to_string());
        assert_eq!(messages[0].market_type, $market_type);
        assert_eq!(messages[0].msg_type, MessageType::Candlestick);
        for msg in messages {
            assert!(parse(msg));
        }
    }};
}

#[allow(unused_macros)]
macro_rules! gen_test_subscribe_symbol {
    ($exchange:expr, $market_type:expr, $symbol:expr) => {{
        let (tx, rx) = std::sync::mpsc::channel();
        let mut messages = Vec::new();
        let msg_types = vec![MessageType::Trade, MessageType::L2Event];
        subscribe_symbol($exchange, $market_type, $symbol, &msg_types, tx, Some(0));
        for msg in rx {
            messages.push(msg);
        }
        assert!(!messages.is_empty());
    }};
}