stable_fs/storage/
types.rs

1use crate::{error::Error, fs::ChunkType};
2use ic_stable_structures::storable::Bound;
3use serde::{Deserialize, Serialize};
4
5pub const FILE_CHUNK_SIZE_V1: usize = 4096;
6
7pub const DEFAULT_FILE_CHUNK_SIZE_V2: usize = 16384;
8pub const MAX_FILE_CHUNK_SIZE_V2: usize = 65536;
9
10pub const MAX_FILE_NAME: usize = 255;
11
12// maximal chunk index. (reserve last 10 chunks for custom needs)
13pub const MAX_FILE_CHUNK_COUNT: u32 = u32::MAX - 10;
14
15// maximal file size supported by the file system.
16pub const MAX_FILE_SIZE: u64 = (MAX_FILE_CHUNK_COUNT as u64) * FILE_CHUNK_SIZE_V1 as u64;
17
18// maximal file entry index
19pub const MAX_FILE_ENTRY_INDEX: u32 = u32::MAX - 10;
20
21// special "." entry index
22pub const DUMMY_DOT_ENTRY_INDEX: u32 = u32::MAX - 5;
23// special ".." entry index
24pub const DUMMY_DOT_DOT_ENTRY_INDEX: u32 = u32::MAX - 4;
25
26pub const DUMMY_DOT_ENTRY: (DirEntryIndex, DirEntry) = (
27    DUMMY_DOT_ENTRY_INDEX,
28    DirEntry {
29        name: FileName {
30            length: 1,
31            bytes: {
32                let mut arr = [0u8; 255];
33                arr[0] = b'.';
34                arr
35            },
36        },
37        node: 0,
38        entry_type: None,
39    },
40);
41
42pub const DUMMY_DOT_DOT_ENTRY: (DirEntryIndex, DirEntry) = (
43    DUMMY_DOT_DOT_ENTRY_INDEX,
44    DirEntry {
45        name: FileName {
46            length: 2,
47            bytes: {
48                let mut arr = [0u8; 255];
49                arr[0] = b'.';
50                arr[1] = b'.';
51                arr
52            },
53        },
54        node: 0,
55        entry_type: None,
56    },
57);
58
59// The unique identifier of a node, which can be a file or a directory.
60// Also known as inode in WASI and other file systems.
61pub type Node = u64;
62
63// An integer type for representing file sizes and offsets.
64pub type FileSize = u64;
65
66// An index of a file chunk.
67pub type FileChunkIndex = u32;
68
69// The address in memory where the V2 chunk is stored.
70pub type FileChunkPtr = u64;
71
72// An array filled with 0 used to fill memory with 0 via copy.
73pub static ZEROES: [u8; MAX_FILE_CHUNK_SIZE_V2] = [0u8; MAX_FILE_CHUNK_SIZE_V2];
74
75// A handle used for writing files in chunks.
76#[derive(Debug, PartialEq, Eq)]
77pub(crate) struct ChunkHandle {
78    pub index: FileChunkIndex,
79    pub offset: FileSize,
80    pub len: FileSize,
81}
82
83// A file consists of multiple file chunks.
84#[derive(Clone, Debug, PartialEq)]
85pub struct FileChunk {
86    pub bytes: [u8; FILE_CHUNK_SIZE_V1],
87}
88
89impl Default for FileChunk {
90    fn default() -> Self {
91        Self {
92            bytes: [0; FILE_CHUNK_SIZE_V1],
93        }
94    }
95}
96
97impl ic_stable_structures::Storable for FileChunk {
98    fn to_bytes(&'_ self) -> std::borrow::Cow<'_, [u8]> {
99        std::borrow::Cow::Borrowed(&self.bytes)
100    }
101
102    fn into_bytes(self) -> Vec<u8> {
103        self.bytes.to_vec()
104    }
105
106    fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
107        Self {
108            bytes: bytes.as_ref().try_into().unwrap(),
109        }
110    }
111
112    const BOUND: Bound = Bound::Bounded {
113        max_size: FILE_CHUNK_SIZE_V1 as u32,
114        is_fixed_size: true,
115    };
116}
117
118#[derive(Clone, Debug, Default, Serialize, Deserialize)]
119pub struct Header {
120    pub version: u32,
121    pub next_node: Node,
122}
123
124impl ic_stable_structures::Storable for Header {
125    fn to_bytes(&'_ self) -> std::borrow::Cow<'_, [u8]> {
126        let mut buf = vec![];
127        ciborium::ser::into_writer(&self, &mut buf).unwrap();
128        std::borrow::Cow::Owned(buf)
129    }
130
131    fn into_bytes(self) -> Vec<u8> {
132        let mut buf = vec![];
133        ciborium::ser::into_writer(&self, &mut buf).unwrap();
134        buf
135    }
136
137    fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
138        ciborium::de::from_reader(bytes.as_ref()).unwrap()
139    }
140
141    const BOUND: Bound = Bound::Unbounded;
142}
143
144#[repr(C, align(8))]
145#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
146pub struct Metadata {
147    pub node: Node,
148    pub file_type: FileType,
149    pub link_count: u64,
150    pub size: FileSize,
151    pub times: Times,
152    pub first_dir_entry: Option<DirEntryIndex>, // obsolete field, must be kept for compatibility because of repr(C)
153    pub last_dir_entry: Option<DirEntryIndex>, // obsolete field, must be kept for compatibility because of repr(C)
154    pub chunk_type: Option<ChunkType>,
155    pub maximum_size_allowed: Option<FileSize>,
156}
157
158impl ic_stable_structures::Storable for Metadata {
159    fn to_bytes(&'_ self) -> std::borrow::Cow<'_, [u8]> {
160        let mut buf = vec![];
161        ciborium::ser::into_writer(&self, &mut buf).unwrap();
162        std::borrow::Cow::Owned(buf)
163    }
164
165    fn into_bytes(self) -> Vec<u8> {
166        let mut buf = vec![];
167        ciborium::ser::into_writer(&self, &mut buf).unwrap();
168        buf
169    }
170
171    fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
172        ciborium::de::from_reader(bytes.as_ref()).unwrap()
173    }
174
175    const BOUND: Bound = Bound::Unbounded;
176}
177
178// The type of a node.
179#[derive(Clone, Copy, Default, Debug, PartialEq, Eq, Serialize, Deserialize)]
180pub enum FileType {
181    Directory = 3,
182    #[default]
183    RegularFile = 4,
184    SymbolicLink = 7,
185}
186
187impl TryFrom<u8> for FileType {
188    type Error = Error;
189
190    fn try_from(value: u8) -> Result<Self, Self::Error> {
191        match value {
192            3 => Ok(FileType::Directory),
193            4 => Ok(FileType::RegularFile),
194            7 => Ok(FileType::SymbolicLink),
195            _ => Err(Error::InvalidArgument),
196        }
197    }
198}
199
200impl From<FileType> for u8 {
201    fn from(val: FileType) -> Self {
202        match val {
203            FileType::Directory => 3,
204            FileType::RegularFile => 4,
205            FileType::SymbolicLink => 7,
206        }
207    }
208}
209
210// The time stats of a node.
211#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq)]
212pub struct Times {
213    pub accessed: u64,
214    pub modified: u64,
215    pub created: u64,
216}
217
218use std::cmp::{Ord, Ordering, PartialOrd};
219
220// The name of a file or a directory. Most operating systems limit the max file
221// name length to 255.
222#[derive(Clone, Debug, Serialize, Deserialize)]
223pub struct FileName {
224    pub length: u8,
225    #[serde(
226        deserialize_with = "deserialize_file_name",
227        serialize_with = "serialize_file_name"
228    )]
229    pub bytes: [u8; MAX_FILE_NAME],
230}
231
232impl Eq for FileName {}
233
234impl PartialEq for FileName {
235    fn eq(&self, other: &Self) -> bool {
236        self.length == other.length
237            && self.bytes[..self.length as usize] == other.bytes[..other.length as usize]
238    }
239}
240
241impl PartialOrd for FileName {
242    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
243        Some(self.cmp(other))
244    }
245}
246
247impl Ord for FileName {
248    fn cmp(&self, other: &Self) -> Ordering {
249        let min_len = self.length.min(other.length) as usize;
250
251        match self.bytes[..min_len].cmp(&other.bytes[..min_len]) {
252            Ordering::Equal => self.length.cmp(&other.length),
253            ord => ord,
254        }
255    }
256}
257
258use ic_stable_structures::Storable;
259
260impl Storable for FileName {
261    fn to_bytes(&'_ self) -> std::borrow::Cow<'_, [u8]> {
262        let mut buf = [0u8; MAX_FILE_NAME + 1];
263
264        buf[0] = self.length;
265        buf[1..256].copy_from_slice(&self.bytes);
266
267        std::borrow::Cow::Owned(buf.to_vec())
268    }
269
270    fn into_bytes(self) -> Vec<u8> {
271        let mut buf = [0u8; MAX_FILE_NAME + 1];
272
273        buf[0] = self.length;
274        buf[1..256].copy_from_slice(&self.bytes);
275
276        buf.to_vec()
277    }
278
279    fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
280        let mut arr = [0u8; MAX_FILE_NAME];
281        arr.copy_from_slice(&bytes[1..MAX_FILE_NAME + 1]);
282
283        FileName {
284            length: bytes[0],
285            bytes: arr,
286        }
287    }
288
289    const BOUND: ic_stable_structures::storable::Bound =
290        ic_stable_structures::storable::Bound::Bounded {
291            max_size: 256,
292            is_fixed_size: true,
293        };
294}
295
296fn serialize_file_name<S>(bytes: &[u8; MAX_FILE_NAME], serializer: S) -> Result<S::Ok, S::Error>
297where
298    S: serde::Serializer,
299{
300    serde_bytes::Bytes::new(bytes).serialize(serializer)
301}
302
303fn deserialize_file_name<'de, D>(deserializer: D) -> Result<[u8; MAX_FILE_NAME], D::Error>
304where
305    D: serde::Deserializer<'de>,
306{
307    let bytes: Vec<u8> = serde_bytes::deserialize(deserializer).unwrap();
308    let len = bytes.len();
309    let bytes_array: [u8; MAX_FILE_NAME] = bytes
310        .try_into()
311        .map_err(|_| serde::de::Error::invalid_length(len, &"expected MAX_FILE_NAME bytes"))?;
312    Ok(bytes_array)
313}
314
315impl Default for FileName {
316    fn default() -> Self {
317        Self {
318            length: 0,
319            bytes: [0; MAX_FILE_NAME],
320        }
321    }
322}
323
324impl FileName {
325    pub fn new(name: &[u8]) -> Result<Self, Error> {
326        let len = name.len();
327        if len > MAX_FILE_NAME {
328            return Err(Error::FilenameTooLong);
329        }
330
331        let mut bytes = [0; MAX_FILE_NAME];
332        bytes[0..len].copy_from_slice(name);
333        Ok(Self {
334            length: len as u8,
335            bytes,
336        })
337    }
338}
339
340impl std::fmt::Display for FileName {
341    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
342        write!(f, "{}", unsafe {
343            std::str::from_utf8_unchecked(&self.bytes[..(self.length as usize)])
344        })
345    }
346}
347
348// An index of a directory entry.
349pub type DirEntryIndex = u32;
350
351// A directory contains a list of directory entries.
352// Each entry describes a name of a file or a directory.
353#[derive(Clone, Debug, Default, Serialize, Deserialize)]
354pub struct DirEntry {
355    pub name: FileName,
356    pub node: Node,
357    pub entry_type: Option<FileType>,
358}
359
360impl ic_stable_structures::Storable for DirEntry {
361    fn to_bytes(&'_ self) -> std::borrow::Cow<'_, [u8]> {
362        let mut buf = vec![];
363        ciborium::ser::into_writer(&self, &mut buf).unwrap();
364        std::borrow::Cow::Owned(buf)
365    }
366
367    fn into_bytes(self) -> Vec<u8> {
368        let mut buf = vec![];
369        ciborium::ser::into_writer(&self, &mut buf).unwrap();
370        buf
371    }
372
373    fn from_bytes(bytes: std::borrow::Cow<[u8]>) -> Self {
374        ciborium::de::from_reader(bytes.as_ref()).unwrap()
375    }
376
377    const BOUND: ic_stable_structures::storable::Bound = Bound::Unbounded;
378}
379
380/// Mounting policy to determine actual mounted memory file size
381pub enum MountedFileSizePolicy {
382    /// Reuse the size from a previous mount or 0 if unknown
383    PreviousOrZero,
384    /// Reuse the size from a previous mount or determine from memory pages.
385    PreviousOrMemoryPages,
386    /// Explicitly set the file size, overriding other options.
387    Explicit(FileSize),
388    /// Determine file size from the memory pages.
389    MemoryPages,
390}
391
392impl MountedFileSizePolicy {
393    pub fn get_mounted_file_size(
394        &self,
395        previous_size: Option<FileSize>,
396        current_memory_pages: FileSize,
397    ) -> FileSize {
398        let current_size = current_memory_pages * ic_cdk::stable::WASM_PAGE_SIZE_IN_BYTES;
399
400        match self {
401            MountedFileSizePolicy::PreviousOrZero => {
402                if let Some(old_size) = previous_size
403                    && old_size <= current_size
404                {
405                    old_size
406                } else {
407                    0
408                }
409            }
410            MountedFileSizePolicy::PreviousOrMemoryPages => {
411                if let Some(old_size) = previous_size
412                    && old_size <= current_size
413                {
414                    old_size
415                } else {
416                    current_size
417                }
418            }
419            MountedFileSizePolicy::Explicit(size) => *size,
420            MountedFileSizePolicy::MemoryPages => current_size,
421        }
422    }
423}
424
425#[cfg(test)]
426mod tests {
427    use crate::{fs::ChunkType, storage::types::MountedFileSizePolicy};
428
429    use super::{DirEntryIndex, FileSize, FileType, Node, Times};
430    use serde::{Deserialize, Serialize};
431
432    // Old node structure.
433    #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
434    pub struct MetadataOld {
435        pub node: Node,
436        pub file_type: FileType,
437        pub link_count: u64,
438        pub size: FileSize,
439        pub times: Times,
440        pub first_dir_entry: Option<DirEntryIndex>,
441        pub last_dir_entry: Option<DirEntryIndex>,
442    }
443
444    // New node structure.
445    #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
446    pub struct MetadataNew {
447        pub node: Node,
448        pub file_type: FileType,
449        pub link_count: u64,
450        pub size: FileSize,
451        pub times: Times,
452        pub _first_dir_entry: Option<DirEntryIndex>, // obsolete field
453        pub _last_dir_entry: Option<DirEntryIndex>,  // obsolete field
454        pub chunk_type: Option<ChunkType>,
455    }
456
457    fn meta_to_bytes(meta: &'_ MetadataOld) -> std::borrow::Cow<'_, [u8]> {
458        let mut buf = vec![];
459        ciborium::ser::into_writer(meta, &mut buf).unwrap();
460        std::borrow::Cow::Owned(buf)
461    }
462
463    fn meta_from_bytes(bytes: std::borrow::Cow<[u8]>) -> MetadataNew {
464        ciborium::de::from_reader(bytes.as_ref()).unwrap()
465    }
466
467    #[test]
468    fn store_old_load_new() {
469        let meta_old = MetadataOld {
470            node: 23,
471            file_type: FileType::RegularFile,
472            link_count: 3,
473            size: 123,
474            times: Times::default(),
475            first_dir_entry: Some(23),
476            last_dir_entry: Some(35),
477        };
478
479        let bytes = meta_to_bytes(&meta_old);
480
481        let meta_new = meta_from_bytes(bytes);
482
483        assert_eq!(meta_new.node, meta_old.node);
484        assert_eq!(meta_new.file_type, meta_old.file_type);
485        assert_eq!(meta_new.link_count, meta_old.link_count);
486        assert_eq!(meta_new.size, meta_old.size);
487        assert_eq!(meta_new.times, meta_old.times);
488        assert_eq!(meta_new.chunk_type, None);
489    }
490
491    #[test]
492    fn store_old_load_new_both_none() {
493        let meta_old = MetadataOld {
494            node: 23,
495            file_type: FileType::RegularFile,
496            link_count: 3,
497            size: 123,
498            times: Times::default(),
499            first_dir_entry: None,
500            last_dir_entry: None,
501        };
502
503        let bytes = meta_to_bytes(&meta_old);
504
505        let meta_new = meta_from_bytes(bytes);
506
507        assert_eq!(meta_new.node, meta_old.node);
508        assert_eq!(meta_new.file_type, meta_old.file_type);
509        assert_eq!(meta_new.link_count, meta_old.link_count);
510        assert_eq!(meta_new.size, meta_old.size);
511        assert_eq!(meta_new.times, meta_old.times);
512        assert_eq!(meta_new.chunk_type, None);
513    }
514
515    #[test]
516    fn store_old_load_new_first_none() {
517        let meta_old = MetadataOld {
518            node: 23,
519            file_type: FileType::RegularFile,
520            link_count: 3,
521            size: 123,
522            times: Times::default(),
523            first_dir_entry: None,
524            last_dir_entry: Some(23),
525        };
526
527        let bytes = meta_to_bytes(&meta_old);
528
529        let meta_new = meta_from_bytes(bytes);
530
531        assert_eq!(meta_new.node, meta_old.node);
532        assert_eq!(meta_new.file_type, meta_old.file_type);
533        assert_eq!(meta_new.link_count, meta_old.link_count);
534        assert_eq!(meta_new.size, meta_old.size);
535        assert_eq!(meta_new.times, meta_old.times);
536        assert_eq!(meta_new.chunk_type, None);
537    }
538
539    #[test]
540    fn size_policy_test() {
541        use ic_cdk::stable::WASM_PAGE_SIZE_IN_BYTES as PAGE_SIZE;
542
543        let p = MountedFileSizePolicy::PreviousOrZero;
544        assert_eq!(p.get_mounted_file_size(Some(100000), 1), 0);
545        assert_eq!(p.get_mounted_file_size(Some(100000), 2), 100000);
546        assert_eq!(p.get_mounted_file_size(None, 2), 0);
547
548        let p = MountedFileSizePolicy::PreviousOrMemoryPages;
549        assert_eq!(p.get_mounted_file_size(Some(100000), 1), PAGE_SIZE);
550        assert_eq!(p.get_mounted_file_size(Some(100000), 2), 100000);
551        assert_eq!(p.get_mounted_file_size(None, 2), PAGE_SIZE * 2);
552
553        let p = MountedFileSizePolicy::Explicit(3000);
554        assert_eq!(p.get_mounted_file_size(Some(100000), 1), 3000);
555        assert_eq!(p.get_mounted_file_size(Some(100000), 2), 3000);
556        assert_eq!(p.get_mounted_file_size(None, 2), 3000);
557
558        let p = MountedFileSizePolicy::MemoryPages;
559        assert_eq!(p.get_mounted_file_size(Some(100000), 1), PAGE_SIZE);
560        assert_eq!(p.get_mounted_file_size(Some(100000), 2), PAGE_SIZE * 2);
561        assert_eq!(p.get_mounted_file_size(None, 2), PAGE_SIZE * 2);
562    }
563}
stable_fs/storage/types.rs

stable_fs/storage/
types.rs
