near_primitives/
state_sync.rs

1use crate::hash::CryptoHash;
2use crate::merkle::MerklePath;
3use crate::sharding::{
4    ReceiptProof, ShardChunk, ShardChunkHeader, ShardChunkHeaderV1, ShardChunkV1,
5};
6use crate::types::{BlockHeight, EpochId, ShardId, StateRoot, StateRootNode};
7use borsh::{BorshDeserialize, BorshSerialize};
8use near_primitives_core::types::EpochHeight;
9use near_schema_checker_lib::ProtocolSchema;
10use std::sync::Arc;
11
12#[derive(PartialEq, Eq, Clone, Debug, BorshSerialize, BorshDeserialize, ProtocolSchema)]
13pub struct ReceiptProofResponse(pub CryptoHash, pub Arc<Vec<ReceiptProof>>);
14
15#[derive(PartialEq, Eq, Clone, Debug, BorshSerialize, BorshDeserialize, ProtocolSchema)]
16pub struct RootProof(pub CryptoHash, pub MerklePath);
17
18#[derive(PartialEq, Eq, Clone, Debug, BorshSerialize, BorshDeserialize, ProtocolSchema)]
19pub struct StateHeaderKey(pub ShardId, pub CryptoHash);
20
21#[derive(PartialEq, Eq, Clone, Debug, BorshSerialize, BorshDeserialize, ProtocolSchema)]
22pub struct StatePartKey(pub CryptoHash, pub ShardId, pub u64 /* PartId */);
23
24#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
25pub struct ShardStateSyncResponseHeaderV1 {
26    pub chunk: ShardChunkV1,
27    pub chunk_proof: MerklePath,
28    pub prev_chunk_header: Option<ShardChunkHeaderV1>,
29    pub prev_chunk_proof: Option<MerklePath>,
30    pub incoming_receipts_proofs: Vec<ReceiptProofResponse>,
31    pub root_proofs: Vec<Vec<RootProof>>,
32    pub state_root_node: StateRootNode,
33}
34
35/// Let B[h] be the block with hash h.
36/// Let shard_id be the shard ID of the shard this header is meant for
37/// As a shorthand,let B_sync = B[sync_hash], B_prev = B[B_sync.prev_hash]
38///
39/// Also let B_chunk be the block with height B_prev.chunks[shard_id].height_included
40/// that is an ancestor of B_sync. So, the last block with a new chunk before B_sync.
41/// And let B_prev_chunk = B[B_chunk.prev_hash]. So, the block before the last block with a new chunk before B_sync.
42///
43/// Given these definitions, the meaning of fields are explained below.
44#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
45pub struct ShardStateSyncResponseHeaderV2 {
46    /// The chunk whose header in included as B_prev.chunks[shard_id]
47    /// This chunk will be applied after downloading state
48    pub chunk: ShardChunk,
49    /// A merkle path for (Self::chunk.hash, Self::chunk.height_included), verifiable
50    /// against B_prev.chunk_headers_root
51    pub chunk_proof: MerklePath,
52    /// This is None if sync_hash is the genesis hash. Otherwise, it's B_prev_chunk.chunks[shard_id]
53    pub prev_chunk_header: Option<ShardChunkHeader>,
54    /// A merkle path for (Self::prev_chunk_header.hash, Self::prev_chunk_header.height_included), verifiable
55    /// against B_prev_chunk.chunk_headers_root
56    pub prev_chunk_proof: Option<MerklePath>,
57    /// This field contains the incoming receipts for shard_id for B_sync and B_prev_chunk.
58    /// So, this field has at most two elements.
59    /// These receipts are used to apply `chunk` after downloading state
60    pub incoming_receipts_proofs: Vec<ReceiptProofResponse>,
61    /// This field contains the info necessary to verify that the receipt proofs in Self::incoming_receipts_proofs
62    /// are actually the ones referenced on chain
63    ///
64    /// The length of this field is the same as the length of Self::incoming_receipts_proofs, and elements
65    /// of the two at a given index are taken together for verification. For a given index i,
66    /// root_proofs[i] is a vector of the same length as incoming_receipts_proofs[i].1 , which itself is a
67    /// vector of receipt proofs for all "from_shard_ids" that sent receipts to shard_id. root_proofs[i][j]
68    /// contains a merkle root equal to the prev_outgoing_receipts_root field of the corresponding chunk
69    /// included in the block with hash incoming_receipts_proofs[i].0, and a merkle path to verify it against
70    /// that block's prev_chunk_outgoing_receipts_root field.
71    pub root_proofs: Vec<Vec<RootProof>>,
72    /// The state root with hash equal to B_prev.chunks[shard_id].prev_state_root.
73    /// That is, the state root node of the trie before applying the chunks in B_prev
74    pub state_root_node: StateRootNode,
75}
76
77#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
78pub enum CachedParts {
79    AllParts,
80    NoParts,
81    /// Represents a subset of parts cached.
82    /// Can represent both NoParts and AllParts, but in those cases use the
83    /// corresponding enum values for efficiency.
84    BitArray(BitArray),
85}
86
87/// Represents an array of boolean values in a compact form.
88#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
89pub struct BitArray {
90    data: Vec<u8>,
91    capacity: u64,
92}
93
94impl BitArray {
95    pub fn new(capacity: u64) -> Self {
96        let num_bytes = (capacity + 7) / 8;
97        Self { data: vec![0; num_bytes as usize], capacity }
98    }
99}
100
101#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)]
102pub enum ShardStateSyncResponseHeader {
103    V1(ShardStateSyncResponseHeaderV1),
104    V2(ShardStateSyncResponseHeaderV2),
105}
106
107impl ShardStateSyncResponseHeader {
108    #[inline]
109    pub fn take_chunk(self) -> ShardChunk {
110        match self {
111            Self::V1(header) => ShardChunk::V1(header.chunk),
112            Self::V2(header) => header.chunk,
113        }
114    }
115
116    #[inline]
117    pub fn cloned_chunk(&self) -> ShardChunk {
118        match self {
119            Self::V1(header) => ShardChunk::V1(header.chunk.clone()),
120            Self::V2(header) => header.chunk.clone(),
121        }
122    }
123
124    #[inline]
125    pub fn cloned_prev_chunk_header(&self) -> Option<ShardChunkHeader> {
126        match self {
127            Self::V1(header) => header.prev_chunk_header.clone().map(ShardChunkHeader::V1),
128            Self::V2(header) => header.prev_chunk_header.clone(),
129        }
130    }
131
132    #[inline]
133    pub fn chunk_height_included(&self) -> BlockHeight {
134        match self {
135            Self::V1(header) => header.chunk.header.height_included,
136            Self::V2(header) => header.chunk.height_included(),
137        }
138    }
139
140    #[inline]
141    pub fn chunk_prev_state_root(&self) -> StateRoot {
142        match self {
143            Self::V1(header) => header.chunk.header.inner.prev_state_root,
144            Self::V2(header) => header.chunk.prev_state_root(),
145        }
146    }
147
148    #[inline]
149    pub fn chunk_proof(&self) -> &MerklePath {
150        match self {
151            Self::V1(header) => &header.chunk_proof,
152            Self::V2(header) => &header.chunk_proof,
153        }
154    }
155
156    #[inline]
157    pub fn prev_chunk_proof(&self) -> &Option<MerklePath> {
158        match self {
159            Self::V1(header) => &header.prev_chunk_proof,
160            Self::V2(header) => &header.prev_chunk_proof,
161        }
162    }
163
164    #[inline]
165    pub fn incoming_receipts_proofs(&self) -> &[ReceiptProofResponse] {
166        match self {
167            Self::V1(header) => &header.incoming_receipts_proofs,
168            Self::V2(header) => &header.incoming_receipts_proofs,
169        }
170    }
171
172    #[inline]
173    pub fn root_proofs(&self) -> &[Vec<RootProof>] {
174        match self {
175            Self::V1(header) => &header.root_proofs,
176            Self::V2(header) => &header.root_proofs,
177        }
178    }
179
180    #[inline]
181    pub fn state_root_node(&self) -> &StateRootNode {
182        match self {
183            Self::V1(header) => &header.state_root_node,
184            Self::V2(header) => &header.state_root_node,
185        }
186    }
187
188    pub fn num_state_parts(&self) -> u64 {
189        get_num_state_parts(self.state_root_node().memory_usage)
190    }
191}
192
193#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
194pub struct ShardStateSyncResponseV1 {
195    pub header: Option<ShardStateSyncResponseHeaderV1>,
196    pub part: Option<(u64, Vec<u8>)>,
197}
198
199#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
200pub struct ShardStateSyncResponseV2 {
201    pub header: Option<ShardStateSyncResponseHeaderV2>,
202    pub part: Option<(u64, Vec<u8>)>,
203}
204
205#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
206pub struct ShardStateSyncResponseV3 {
207    pub header: Option<ShardStateSyncResponseHeaderV2>,
208    pub part: Option<(u64, Vec<u8>)>,
209    // TODO(saketh): deprecate unused fields cached_parts and can_generate
210    pub cached_parts: Option<CachedParts>,
211    pub can_generate: bool,
212}
213
214#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
215pub enum ShardStateSyncResponse {
216    V1(ShardStateSyncResponseV1),
217    V2(ShardStateSyncResponseV2),
218    V3(ShardStateSyncResponseV3),
219}
220
221impl ShardStateSyncResponse {
222    pub fn part_id(&self) -> Option<u64> {
223        match self {
224            Self::V1(response) => response.part_id(),
225            Self::V2(response) => response.part.as_ref().map(|(part_id, _)| *part_id),
226            Self::V3(response) => response.part.as_ref().map(|(part_id, _)| *part_id),
227        }
228    }
229
230    pub fn take_header(self) -> Option<ShardStateSyncResponseHeader> {
231        match self {
232            Self::V1(response) => response.header.map(ShardStateSyncResponseHeader::V1),
233            Self::V2(response) => response.header.map(ShardStateSyncResponseHeader::V2),
234            Self::V3(response) => response.header.map(ShardStateSyncResponseHeader::V2),
235        }
236    }
237
238    pub fn part(&self) -> &Option<(u64, Vec<u8>)> {
239        match self {
240            Self::V1(response) => &response.part,
241            Self::V2(response) => &response.part,
242            Self::V3(response) => &response.part,
243        }
244    }
245
246    pub fn take_part(self) -> Option<(u64, Vec<u8>)> {
247        match self {
248            Self::V1(response) => response.part,
249            Self::V2(response) => response.part,
250            Self::V3(response) => response.part,
251        }
252    }
253
254    pub fn can_generate(&self) -> bool {
255        match self {
256            Self::V1(_response) => false,
257            Self::V2(_response) => false,
258            Self::V3(response) => response.can_generate,
259        }
260    }
261
262    pub fn cached_parts(&self) -> &Option<CachedParts> {
263        match self {
264            Self::V1(_response) => &None,
265            Self::V2(_response) => &None,
266            Self::V3(response) => &response.cached_parts,
267        }
268    }
269
270    pub fn has_header(&self) -> bool {
271        match self {
272            Self::V1(response) => response.header.is_some(),
273            Self::V2(response) => response.header.is_some(),
274            Self::V3(response) => response.header.is_some(),
275        }
276    }
277}
278
279impl ShardStateSyncResponseV1 {
280    pub fn part_id(&self) -> Option<u64> {
281        self.part.as_ref().map(|(part_id, _)| *part_id)
282    }
283}
284
285pub const STATE_PART_MEMORY_LIMIT: bytesize::ByteSize = bytesize::ByteSize(30 * bytesize::MIB);
286
287pub fn get_num_state_parts(memory_usage: u64) -> u64 {
288    (memory_usage + STATE_PART_MEMORY_LIMIT.as_u64() - 1) / STATE_PART_MEMORY_LIMIT.as_u64()
289}
290
291#[derive(BorshSerialize, BorshDeserialize, Debug, Clone, serde::Serialize, ProtocolSchema)]
292/// Represents the progress of dumps state of a shard.
293pub enum StateSyncDumpProgress {
294    /// Represents two cases:
295    /// * An epoch dump is complete
296    /// * The node is running its first epoch and there is nothing to dump.
297    AllDumped {
298        /// The dumped state corresponds to the state at the beginning of the specified epoch.
299        epoch_id: EpochId,
300        epoch_height: EpochHeight,
301    },
302    /// * An epoch dump is skipped in the epoch where shard layout changes
303    Skipped { epoch_id: EpochId, epoch_height: EpochHeight },
304    /// Represents the case of an epoch being partially dumped.
305    InProgress {
306        /// The dumped state corresponds to the state at the beginning of the specified epoch.
307        epoch_id: EpochId,
308        epoch_height: EpochHeight,
309        /// Block hash of the first block of the epoch.
310        /// The dumped state corresponds to the state before applying this block.
311        sync_hash: CryptoHash,
312    },
313}
314
315#[cfg(test)]
316mod tests {
317    use crate::state_sync::{STATE_PART_MEMORY_LIMIT, get_num_state_parts};
318
319    #[test]
320    fn test_get_num_state_parts() {
321        assert_eq!(get_num_state_parts(0), 0);
322        assert_eq!(get_num_state_parts(1), 1);
323        assert_eq!(get_num_state_parts(STATE_PART_MEMORY_LIMIT.as_u64()), 1);
324        assert_eq!(get_num_state_parts(STATE_PART_MEMORY_LIMIT.as_u64() + 1), 2);
325        assert_eq!(get_num_state_parts(STATE_PART_MEMORY_LIMIT.as_u64() * 100), 100);
326        assert_eq!(get_num_state_parts(STATE_PART_MEMORY_LIMIT.as_u64() * 100 + 1), 101);
327    }
328}
near_primitives/state_sync.rs

near_primitives/
state_sync.rs
