near_primitives/
state_sync.rs

1use crate::hash::CryptoHash;
2use crate::merkle::MerklePath;
3use crate::sharding::{
4    ReceiptProof, ShardChunk, ShardChunkHeader, ShardChunkHeaderV1, ShardChunkV1,
5};
6use crate::state_part::{StatePart, StatePartV0};
7use crate::types::{BlockHeight, EpochId, ShardId, StateRoot, StateRootNode};
8use borsh::{BorshDeserialize, BorshSerialize};
9use near_primitives_core::types::EpochHeight;
10use near_schema_checker_lib::ProtocolSchema;
11use std::sync::Arc;
12
13#[derive(PartialEq, Eq, Clone, Debug, BorshSerialize, BorshDeserialize, ProtocolSchema)]
14pub struct ReceiptProofResponse(pub CryptoHash, pub Arc<Vec<ReceiptProof>>);
15
16#[derive(PartialEq, Eq, Clone, Debug, BorshSerialize, BorshDeserialize, ProtocolSchema)]
17pub struct RootProof(pub CryptoHash, pub MerklePath);
18
19#[derive(PartialEq, Eq, Clone, Debug, BorshSerialize, BorshDeserialize, ProtocolSchema)]
20pub struct StateHeaderKey(pub ShardId, pub CryptoHash);
21
22#[derive(PartialEq, Eq, Clone, Debug, BorshSerialize, BorshDeserialize, ProtocolSchema)]
23pub struct StatePartKey(pub CryptoHash, pub ShardId, pub u64 /* PartId */);
24
25#[derive(
26    Copy, PartialEq, Eq, Clone, Debug, Hash, BorshSerialize, BorshDeserialize, ProtocolSchema,
27)]
28pub enum PartIdOrHeader {
29    Part { part_id: u64 },
30    Header,
31}
32
33impl Into<&'static str> for PartIdOrHeader {
34    fn into(self) -> &'static str {
35        match self {
36            PartIdOrHeader::Part { .. } => "part",
37            PartIdOrHeader::Header => "header",
38        }
39    }
40}
41
42#[derive(Copy, PartialEq, Eq, Clone, Debug, BorshSerialize, BorshDeserialize, ProtocolSchema)]
43pub enum StateRequestAckBody {
44    WillRespond,
45    Busy,
46    Error,
47}
48
49impl Into<&'static str> for StateRequestAckBody {
50    fn into(self) -> &'static str {
51        match self {
52            StateRequestAckBody::WillRespond => "will_respond",
53            StateRequestAckBody::Busy => "busy",
54            StateRequestAckBody::Error => "error",
55        }
56    }
57}
58
59#[derive(PartialEq, Eq, Clone, Debug, BorshSerialize, BorshDeserialize, ProtocolSchema)]
60pub struct StateRequestAck {
61    /// Requested shard id
62    pub shard_id: ShardId,
63    /// Sync block hash
64    pub sync_hash: CryptoHash,
65    /// Requested header or part id
66    pub part_id_or_header: PartIdOrHeader,
67    /// Ack contents
68    pub body: StateRequestAckBody,
69}
70
71#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
72pub struct ShardStateSyncResponseHeaderV1 {
73    pub chunk: ShardChunkV1,
74    pub chunk_proof: MerklePath,
75    pub prev_chunk_header: Option<ShardChunkHeaderV1>,
76    pub prev_chunk_proof: Option<MerklePath>,
77    pub incoming_receipts_proofs: Vec<ReceiptProofResponse>,
78    pub root_proofs: Vec<Vec<RootProof>>,
79    pub state_root_node: StateRootNode,
80}
81
82/// Let B[h] be the block with hash h.
83/// Let shard_id be the shard ID of the shard this header is meant for
84/// As a shorthand,let B_sync = B[sync_hash], B_prev = B[B_sync.prev_hash]
85///
86/// Also let B_chunk be the block with height B_prev.chunks[shard_id].height_included
87/// that is an ancestor of B_sync. So, the last block with a new chunk before B_sync.
88/// And let B_prev_chunk = B[B_chunk.prev_hash]. So, the block before the last block with a new chunk before B_sync.
89///
90/// Given these definitions, the meaning of fields are explained below.
91#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
92pub struct ShardStateSyncResponseHeaderV2 {
93    /// The chunk whose header in included as B_prev.chunks[shard_id]
94    /// This chunk will be applied after downloading state
95    pub chunk: ShardChunk,
96    /// A merkle path for (Self::chunk.hash, Self::chunk.height_included), verifiable
97    /// against B_prev.chunk_headers_root
98    pub chunk_proof: MerklePath,
99    /// This is None if sync_hash is the genesis hash. Otherwise, it's B_prev_chunk.chunks[shard_id]
100    pub prev_chunk_header: Option<ShardChunkHeader>,
101    /// A merkle path for (Self::prev_chunk_header.hash, Self::prev_chunk_header.height_included), verifiable
102    /// against B_prev_chunk.chunk_headers_root
103    pub prev_chunk_proof: Option<MerklePath>,
104    /// This field contains the incoming receipts for shard_id for B_sync and B_prev_chunk.
105    /// So, this field has at most two elements.
106    /// These receipts are used to apply `chunk` after downloading state
107    pub incoming_receipts_proofs: Vec<ReceiptProofResponse>,
108    /// This field contains the info necessary to verify that the receipt proofs in Self::incoming_receipts_proofs
109    /// are actually the ones referenced on chain
110    ///
111    /// The length of this field is the same as the length of Self::incoming_receipts_proofs, and elements
112    /// of the two at a given index are taken together for verification. For a given index i,
113    /// root_proofs[i] is a vector of the same length as incoming_receipts_proofs[i].1 , which itself is a
114    /// vector of receipt proofs for all "from_shard_ids" that sent receipts to shard_id. root_proofs[i][j]
115    /// contains a merkle root equal to the prev_outgoing_receipts_root field of the corresponding chunk
116    /// included in the block with hash incoming_receipts_proofs[i].0, and a merkle path to verify it against
117    /// that block's prev_chunk_outgoing_receipts_root field.
118    pub root_proofs: Vec<Vec<RootProof>>,
119    /// The state root with hash equal to B_prev.chunks[shard_id].prev_state_root.
120    /// That is, the state root node of the trie before applying the chunks in B_prev
121    pub state_root_node: StateRootNode,
122}
123
124#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
125#[borsh(use_discriminant = true)]
126#[repr(u8)]
127pub enum CachedParts {
128    AllParts = 0,
129    NoParts = 1,
130    /// Represents a subset of parts cached.
131    /// Can represent both NoParts and AllParts, but in those cases use the
132    /// corresponding enum values for efficiency.
133    BitArray(BitArray) = 2,
134}
135
136/// Represents an array of boolean values in a compact form.
137#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
138pub struct BitArray {
139    data: Vec<u8>,
140    capacity: u64,
141}
142
143impl BitArray {
144    pub fn new(capacity: u64) -> Self {
145        let num_bytes = (capacity + 7) / 8;
146        Self { data: vec![0; num_bytes as usize], capacity }
147    }
148}
149
150#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)]
151#[borsh(use_discriminant = true)]
152#[repr(u8)]
153#[allow(clippy::large_enum_variant)]
154pub enum ShardStateSyncResponseHeader {
155    V1(ShardStateSyncResponseHeaderV1) = 0,
156    V2(ShardStateSyncResponseHeaderV2) = 1,
157}
158
159impl ShardStateSyncResponseHeader {
160    #[inline]
161    pub fn take_chunk(self) -> ShardChunk {
162        match self {
163            Self::V1(header) => ShardChunk::V1(header.chunk),
164            Self::V2(header) => header.chunk,
165        }
166    }
167
168    #[inline]
169    pub fn cloned_chunk(&self) -> ShardChunk {
170        match self {
171            Self::V1(header) => ShardChunk::V1(header.chunk.clone()),
172            Self::V2(header) => header.chunk.clone(),
173        }
174    }
175
176    #[inline]
177    pub fn cloned_prev_chunk_header(&self) -> Option<ShardChunkHeader> {
178        match self {
179            Self::V1(header) => header.prev_chunk_header.clone().map(ShardChunkHeader::V1),
180            Self::V2(header) => header.prev_chunk_header.clone(),
181        }
182    }
183
184    #[inline]
185    pub fn chunk_height_included(&self) -> BlockHeight {
186        match self {
187            Self::V1(header) => header.chunk.header.height_included,
188            Self::V2(header) => header.chunk.height_included(),
189        }
190    }
191
192    #[inline]
193    pub fn chunk_prev_state_root(&self) -> StateRoot {
194        match self {
195            Self::V1(header) => header.chunk.header.inner.prev_state_root,
196            Self::V2(header) => header.chunk.prev_state_root(),
197        }
198    }
199
200    #[inline]
201    pub fn chunk_proof(&self) -> &MerklePath {
202        match self {
203            Self::V1(header) => &header.chunk_proof,
204            Self::V2(header) => &header.chunk_proof,
205        }
206    }
207
208    #[inline]
209    pub fn prev_chunk_proof(&self) -> &Option<MerklePath> {
210        match self {
211            Self::V1(header) => &header.prev_chunk_proof,
212            Self::V2(header) => &header.prev_chunk_proof,
213        }
214    }
215
216    #[inline]
217    pub fn incoming_receipts_proofs(&self) -> &[ReceiptProofResponse] {
218        match self {
219            Self::V1(header) => &header.incoming_receipts_proofs,
220            Self::V2(header) => &header.incoming_receipts_proofs,
221        }
222    }
223
224    #[inline]
225    pub fn root_proofs(&self) -> &[Vec<RootProof>] {
226        match self {
227            Self::V1(header) => &header.root_proofs,
228            Self::V2(header) => &header.root_proofs,
229        }
230    }
231
232    #[inline]
233    pub fn state_root_node(&self) -> &StateRootNode {
234        match self {
235            Self::V1(header) => &header.state_root_node,
236            Self::V2(header) => &header.state_root_node,
237        }
238    }
239
240    pub fn num_state_parts(&self) -> u64 {
241        get_num_state_parts(self.state_root_node().memory_usage)
242    }
243}
244
245#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
246pub struct ShardStateSyncResponseV1 {
247    pub header: Option<ShardStateSyncResponseHeaderV1>,
248    pub part: Option<(u64, Vec<u8>)>,
249}
250
251impl ShardStateSyncResponseV1 {
252    pub fn part_id(&self) -> Option<u64> {
253        self.part.as_ref().map(|(part_id, _)| *part_id)
254    }
255
256    pub fn payload_length(&self) -> Option<usize> {
257        self.part.as_ref().map(|(_, part)| part.len())
258    }
259}
260
261#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
262pub struct ShardStateSyncResponseV2 {
263    pub header: Option<ShardStateSyncResponseHeaderV2>,
264    pub part: Option<(u64, Vec<u8>)>,
265}
266
267#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
268pub struct ShardStateSyncResponseV3 {
269    pub header: Option<ShardStateSyncResponseHeaderV2>,
270    pub part: Option<(u64, Vec<u8>)>,
271    pub cached_parts: Option<CachedParts>,
272    pub can_generate: bool,
273}
274
275/// Between V3 to V4 we removed unused fields `cached_parts` and `can_generate` and introduced versioned `StatePart`.
276#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
277pub struct ShardStateSyncResponseV4 {
278    pub header: Option<ShardStateSyncResponseHeaderV2>,
279    pub part: Option<(u64, StatePart)>,
280}
281
282#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, ProtocolSchema)]
283#[borsh(use_discriminant = true)]
284#[repr(u8)]
285pub enum ShardStateSyncResponse {
286    V1(ShardStateSyncResponseV1) = 0,
287    V2(ShardStateSyncResponseV2) = 1,
288    V3(ShardStateSyncResponseV3) = 2,
289    V4(ShardStateSyncResponseV4) = 3,
290}
291
292impl ShardStateSyncResponse {
293    pub fn new_from_header(header: Option<ShardStateSyncResponseHeaderV2>) -> Self {
294        Self::new_from_header_or_part(header, None)
295    }
296
297    pub fn new_from_part(part: Option<(u64, StatePart)>) -> Self {
298        Self::new_from_header_or_part(None, part)
299    }
300
301    fn new_from_header_or_part(
302        header: Option<ShardStateSyncResponseHeaderV2>,
303        part: Option<(u64, StatePart)>,
304    ) -> Self {
305        Self::V4(ShardStateSyncResponseV4 { header, part })
306    }
307
308    pub fn take_header(self) -> Option<ShardStateSyncResponseHeader> {
309        match self {
310            Self::V1(response) => response.header.map(ShardStateSyncResponseHeader::V1),
311            Self::V2(response) => response.header.map(ShardStateSyncResponseHeader::V2),
312            Self::V3(response) => response.header.map(ShardStateSyncResponseHeader::V2),
313            Self::V4(response) => response.header.map(ShardStateSyncResponseHeader::V2),
314        }
315    }
316
317    pub fn part_id(&self) -> Option<u64> {
318        match self {
319            Self::V1(response) => response.part.as_ref().map(|(part_id, _)| *part_id),
320            Self::V2(response) => response.part.as_ref().map(|(part_id, _)| *part_id),
321            Self::V3(response) => response.part.as_ref().map(|(part_id, _)| *part_id),
322            Self::V4(response) => response.part.as_ref().map(|(part_id, _)| *part_id),
323        }
324    }
325
326    pub fn take_part(self) -> Option<(u64, StatePart)> {
327        match self {
328            Self::V1(response) => {
329                response.part.map(|(idx, part)| (idx, StatePart::V0(StatePartV0(part))))
330            }
331            Self::V2(response) => {
332                response.part.map(|(idx, part)| (idx, StatePart::V0(StatePartV0(part))))
333            }
334            Self::V3(response) => {
335                response.part.map(|(idx, part)| (idx, StatePart::V0(StatePartV0(part))))
336            }
337            Self::V4(response) => response.part,
338        }
339    }
340
341    pub fn payload_length(&self) -> Option<usize> {
342        match self {
343            Self::V1(response) => response.part.as_ref().map(|(_, part)| part.len()),
344            Self::V2(response) => response.part.as_ref().map(|(_, part)| part.len()),
345            Self::V3(response) => response.part.as_ref().map(|(_, part)| part.len()),
346            Self::V4(response) => response.part.as_ref().map(|(_, part)| part.payload_length()),
347        }
348    }
349}
350
351pub const STATE_PART_MEMORY_LIMIT: bytesize::ByteSize = bytesize::ByteSize(30 * bytesize::MIB);
352
353pub fn get_num_state_parts(memory_usage: u64) -> u64 {
354    (memory_usage + STATE_PART_MEMORY_LIMIT.as_u64() - 1) / STATE_PART_MEMORY_LIMIT.as_u64()
355}
356
357#[derive(BorshSerialize, BorshDeserialize, Debug, Clone, serde::Serialize, ProtocolSchema)]
358#[borsh(use_discriminant = true)]
359#[repr(u8)]
360/// Represents the progress of dumps state of a shard.
361pub enum StateSyncDumpProgress {
362    /// Represents two cases:
363    /// * An epoch dump is complete
364    /// * The node is running its first epoch and there is nothing to dump.
365    AllDumped {
366        /// The dumped state corresponds to the state at the beginning of the specified epoch.
367        epoch_id: EpochId,
368        epoch_height: EpochHeight,
369    } = 0,
370    /// * An epoch dump is skipped in the epoch where shard layout changes
371    Skipped { epoch_id: EpochId, epoch_height: EpochHeight } = 1,
372    /// Represents the case of an epoch being partially dumped.
373    InProgress {
374        /// The dumped state corresponds to the state at the beginning of the specified epoch.
375        epoch_id: EpochId,
376        epoch_height: EpochHeight,
377        /// Block hash of the first block of the epoch.
378        /// The dumped state corresponds to the state before applying this block.
379        sync_hash: CryptoHash,
380    } = 2,
381}
382
383#[cfg(test)]
384mod tests {
385    use crate::state_sync::{STATE_PART_MEMORY_LIMIT, get_num_state_parts};
386
387    #[test]
388    fn test_get_num_state_parts() {
389        assert_eq!(get_num_state_parts(0), 0);
390        assert_eq!(get_num_state_parts(1), 1);
391        assert_eq!(get_num_state_parts(STATE_PART_MEMORY_LIMIT.as_u64()), 1);
392        assert_eq!(get_num_state_parts(STATE_PART_MEMORY_LIMIT.as_u64() + 1), 2);
393        assert_eq!(get_num_state_parts(STATE_PART_MEMORY_LIMIT.as_u64() * 100), 100);
394        assert_eq!(get_num_state_parts(STATE_PART_MEMORY_LIMIT.as_u64() * 100 + 1), 101);
395    }
396}
near_primitives/state_sync.rs

near_primitives/
state_sync.rs
