sharedstate 1.2.2

Sync heavily read state across many servers
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150

//! Cluster coordination: peer discovery, leader election, and the RPC
//! server/client tasks that keep nodes in sync.

pub mod leader;
pub mod node_state;
pub mod peer_connections;
pub mod peer_discovery;
pub mod rpc_server;
pub mod state_sync;

#[cfg(test)]
mod tests {
    use std::{collections::HashMap, io::Result, sync::Arc};

    use message_encoding::MessageEncoding;
    use sequenced_broadcast::SequencedBroadcastSettings;
    use tokio::sync::{mpsc, Mutex};

    use crate::{
        cluster::{
            leader::{LeaderMode, LeaderTask, LeaderTiming},
            node_state::{NodeState, PeerState},
            peer_connections::PeerConnections,
            peer_discovery::{PeerDiscoveryTask, PeerDiscoveryTiming},
            rpc_server::RpcServer,
        },
        protocol::messages::{ElectionTerm, LeaderState},
        state::{
            deterministic_state::DeterministicState, recoverable_state::RecoverableState,
            subscribable_state::SubscribableState,
        },
        transport::{
            channels::NetIoSettings,
            simulated::{SimulatedIo, SimulatedNet},
        },
    };

    #[derive(Clone, Debug, PartialEq, Eq)]
    struct TestState(u64);

    impl DeterministicState for TestState {
        type Action = u64;
        type AuthorityAction = u64;

        fn accept_seq(&self) -> u64 {
            self.0
        }

        fn authority(&self, action: Self::Action) -> Self::AuthorityAction {
            action
        }

        fn update(&mut self, _action: &Self::AuthorityAction) {
            self.0 += 1;
        }
    }

    impl MessageEncoding for TestState {
        fn write_to<T: std::io::Write>(&self, out: &mut T) -> Result<usize> {
            self.0.write_to(out)
        }

        fn read_from<T: std::io::Read>(read: &mut T) -> Result<Self> {
            Ok(Self(MessageEncoding::read_from(read)?))
        }
    }

    struct TestNode {
        state: Arc<NodeState<u64, TestState>>,
        discovery: PeerDiscoveryTask<SimulatedIo, TestState>,
        leader: LeaderTask<u64, TestState>,
        _actions_rx: mpsc::Receiver<(u64, u64)>,
    }

    impl TestNode {
        async fn start(net: &SimulatedNet, addr: u64, can_lead: bool, initial_peers: &[u64]) -> Self {
            let io = net.start_io(addr).await;
            let settings = NetIoSettings::default();

            let peers = initial_peers
                .iter()
                .map(|peer| (*peer, PeerState::empty(*peer)))
                .collect::<HashMap<_, _>>();

            let state = Arc::new(NodeState {
                my_address: addr,
                can_lead,
                peers: Mutex::new(peers),
                state: SubscribableState::new(
                    RecoverableState::new(addr, TestState(0)),
                    SequencedBroadcastSettings::default(),
                )
                .unwrap(),
                leader_state: Mutex::new(LeaderState {
                    term: ElectionTerm::from_term(0),
                    mode: LeaderMode::NoLeader,
                }),
            });

            let connections = Arc::new(PeerConnections::new(io.clone(), settings.clone(), state.clone()));
            let (actions_tx, actions_rx) = mpsc::channel(16);
            let rpc_server = Arc::new(RpcServer::new(state.clone(), actions_tx));
            rpc_server.start_listener(io, settings);

            Self {
                state: state.clone(),
                discovery: PeerDiscoveryTask::new(state.clone(), connections, PeerDiscoveryTiming::default()),
                leader: LeaderTask::new(state, LeaderTiming::default()),
                _actions_rx: actions_rx,
            }
        }

        async fn leader_mode(&self) -> LeaderMode<u64> {
            self.state.leader_state.lock().await.mode.clone()
        }
    }

    #[tokio::test]
    async fn observer_discovers_cluster_and_leader_through_single_seed_peer() {
        let net = SimulatedNet::new();

        /* voters know each other; the observer only knows voter 1 */
        let mut nodes = [
            TestNode::start(&net, 1, true, &[2]).await,
            TestNode::start(&net, 2, true, &[1]).await,
            TestNode::start(&net, 3, false, &[1]).await,
        ];

        for _ in 0..20 {
            for node in nodes.iter_mut() {
                node.discovery.tick().await;
                node.leader.tick().await;
            }
            tokio::time::sleep(std::time::Duration::from_millis(5)).await;
        }

        assert_eq!(nodes[0].leader_mode().await, LeaderMode::Leading);
        assert_eq!(nodes[1].leader_mode().await, LeaderMode::Following { leader: 1 });

        /* the observer found the leader and discovered voter 2 via gossip */
        assert_eq!(nodes[2].leader_mode().await, LeaderMode::Following { leader: 1 });
        let observer_peers = nodes[2].state.peers.lock().await;
        assert!(observer_peers.contains_key(&2), "observer never discovered voter 2");
        assert_eq!(observer_peers.get(&2).unwrap().can_lead, Some(true));

        /* and voter 2, which never had the observer configured, learned of it */
        let voter_peers = nodes[1].state.peers.lock().await;
        assert_eq!(voter_peers.get(&3).and_then(|peer| peer.can_lead), Some(false));
    }
}