1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144

use {
    solana_gossip::cluster_info::ClusterInfo,
    solana_sdk::{clock::Slot, pubkey::Pubkey},
    std::{
        collections::HashSet,
        sync::{
            atomic::{AtomicBool, Ordering},
            Arc,
        },
    },
};

#[derive(PartialEq, Clone, Copy, Debug)]
pub enum RpcHealthStatus {
    Ok,
    Behind { num_slots: Slot }, // Validator is behind its known validators
    Unknown,
}

pub struct RpcHealth {
    cluster_info: Arc<ClusterInfo>,
    known_validators: Option<HashSet<Pubkey>>,
    health_check_slot_distance: u64,
    override_health_check: Arc<AtomicBool>,
    #[cfg(test)]
    stub_health_status: std::sync::RwLock<Option<RpcHealthStatus>>,
}

impl RpcHealth {
    pub fn new(
        cluster_info: Arc<ClusterInfo>,
        known_validators: Option<HashSet<Pubkey>>,
        health_check_slot_distance: u64,
        override_health_check: Arc<AtomicBool>,
    ) -> Self {
        Self {
            cluster_info,
            known_validators,
            health_check_slot_distance,
            override_health_check,
            #[cfg(test)]
            stub_health_status: std::sync::RwLock::new(None),
        }
    }

    pub fn check(&self) -> RpcHealthStatus {
        #[cfg(test)]
        {
            if let Some(stub_health_status) = *self.stub_health_status.read().unwrap() {
                return stub_health_status;
            }
        }

        if self.override_health_check.load(Ordering::Relaxed) {
            RpcHealthStatus::Ok
        } else if let Some(known_validators) = &self.known_validators {
            match (
                self.cluster_info
                    .get_accounts_hash_for_node(&self.cluster_info.id(), |hashes| {
                        hashes
                            .iter()
                            .max_by(|a, b| a.0.cmp(&b.0))
                            .map(|slot_hash| slot_hash.0)
                    })
                    .flatten(),
                known_validators
                    .iter()
                    .filter_map(|known_validator| {
                        self.cluster_info
                            .get_accounts_hash_for_node(known_validator, |hashes| {
                                hashes
                                    .iter()
                                    .max_by(|a, b| a.0.cmp(&b.0))
                                    .map(|slot_hash| slot_hash.0)
                            })
                            .flatten()
                    })
                    .max(),
            ) {
                (
                    Some(latest_account_hash_slot),
                    Some(latest_known_validator_account_hash_slot),
                ) => {
                    // The validator is considered healthy if its latest account hash slot is within
                    // `health_check_slot_distance` of the latest known validator's account hash slot
                    if latest_account_hash_slot
                        > latest_known_validator_account_hash_slot
                            .saturating_sub(self.health_check_slot_distance)
                    {
                        RpcHealthStatus::Ok
                    } else {
                        let num_slots = latest_known_validator_account_hash_slot
                            .saturating_sub(latest_account_hash_slot);
                        warn!(
                            "health check: behind by {} slots: me={}, latest known_validator={}",
                            num_slots,
                            latest_account_hash_slot,
                            latest_known_validator_account_hash_slot
                        );
                        RpcHealthStatus::Behind { num_slots }
                    }
                }
                (latest_account_hash_slot, latest_known_validator_account_hash_slot) => {
                    if latest_account_hash_slot.is_none() {
                        warn!("health check: latest_account_hash_slot not available");
                    }
                    if latest_known_validator_account_hash_slot.is_none() {
                        warn!(
                            "health check: latest_known_validator_account_hash_slot not available"
                        );
                    }
                    RpcHealthStatus::Unknown
                }
            }
        } else {
            // No known validator point of reference available, so this validator is healthy
            // because it's running
            RpcHealthStatus::Ok
        }
    }

    #[cfg(test)]
    pub(crate) fn stub() -> Arc<Self> {
        use {
            solana_gossip::contact_info::ContactInfo, solana_sdk::signer::keypair::Keypair,
            solana_streamer::socket::SocketAddrSpace,
        };
        Arc::new(Self::new(
            Arc::new(ClusterInfo::new(
                ContactInfo::default(),
                Arc::new(Keypair::new()),
                SocketAddrSpace::Unspecified,
            )),
            None,
            42,
            Arc::new(AtomicBool::new(false)),
        ))
    }

    #[cfg(test)]
    pub(crate) fn stub_set_health_status(&self, stub_health_status: Option<RpcHealthStatus>) {
        *self.stub_health_status.write().unwrap() = stub_health_status;
    }
}