bucky_objects/objects/
dir.rs

1use crate::*;
2
3use std::collections::HashMap;
4use std::convert::TryFrom;
5use bucky_error::{BuckyError, BuckyResult};
6
7#[derive(Clone, Debug, RawEncode, RawDecode)]
8pub struct Attributes {
9    flags: u32,
10}
11
12impl Attributes {
13    pub fn new(flags: u32) -> Self {
14        Self { flags }
15    }
16
17    pub fn flags(&self) -> u32 {
18        self.flags
19    }
20}
21
22impl Default for Attributes {
23    fn default() -> Self {
24        Self { flags: 0 }
25    }
26}
27
28#[derive(Clone, Debug, RawEncode, RawDecode)]
29pub enum InnerNode {
30    // 如果是一个子Dir，把该Dir编码放在Body，此处持有DirId
31    // 如果是一个Diff，把Diff编码放在Body，此处持有DiffId
32    // 如果是一个大File，把File编码放在Body，此处持有FileId
33    ObjId(ObjectId),
34
35    // 以下针对单Chunk的File
36
37    // 只有一个Chunk的File
38    Chunk(ChunkId),
39
40    // 只有一个Chunk的File的小文件，持有NDNObjectList::parent_chunk的buffer索引
41    IndexInParentChunk(u32, u32),
42}
43
44#[derive(Clone, Debug, RawEncode, RawDecode)]
45pub struct InnerNodeInfo {
46    attributes: Attributes,
47    node: InnerNode,
48}
49
50impl InnerNodeInfo {
51    pub fn new(attributes: Attributes, node: InnerNode) -> Self {
52        Self { attributes, node }
53    }
54    pub fn get_object_id(&self) -> ObjectId {
55        //需要计算
56        unimplemented!();
57    }
58
59    pub fn attributes(&self) -> &Attributes {
60        &self.attributes
61    }
62
63    pub fn node(&self) -> &InnerNode {
64        &self.node
65    }
66}
67
68pub type DirBodyDescObjectMap = std::collections::HashMap<String, InnerNodeInfo>;
69
70#[derive(Clone, Debug, RawEncode, RawDecode)]
71pub struct NDNObjectList {
72    // Dir内部的多个小文件的File对象，打包成一个Chunk，放在Body
73    // 此处持有该打包Chunk的一个Id
74    pub parent_chunk: Option<ChunkId>,
75
76    // 文件内部路径-文件信息字典
77    pub object_map: DirBodyDescObjectMap,
78}
79
80impl NDNObjectList {
81    pub fn new(parent_chunk: Option<ChunkId>) -> Self {
82        Self {
83            parent_chunk,
84            object_map: HashMap::new(),
85        }
86    }
87
88    pub fn parent_chunk(&self) -> Option<&ChunkId> {
89        self.parent_chunk.as_ref()
90    }
91
92    pub fn object_map(&self) -> &HashMap<String, InnerNodeInfo> {
93        &self.object_map
94    }
95}
96
97#[derive(Clone, Debug, RawEncode, RawDecode)]
98pub enum NDNObjectInfo {
99    // 要么把NDNObjectList编码成一个Chunk，并在Body内持有
100    // 此处持有ChunkId
101    Chunk(ChunkId),
102
103    // 要么直接内置对象列表
104    ObjList(NDNObjectList),
105}
106
107#[derive(Clone, Debug, RawDecode)]
108pub struct DirDescContent {
109    attributes: Attributes,
110    obj_list: NDNObjectInfo,
111}
112
113impl RawEncode for DirDescContent {
114    fn raw_measure(&self, purpose: &Option<RawEncodePurpose>) -> BuckyResult<usize> {
115        let ret = if purpose == &Some(RawEncodePurpose::Hash) {
116            self.attributes.raw_measure(purpose)? + ChunkId::raw_bytes().unwrap()
117        } else {
118            self.attributes.raw_measure(purpose)? + self.obj_list.raw_measure(purpose)?
119        };
120        Ok(ret)
121    }
122
123    fn raw_encode<'a>(
124        &self,
125        buf: &'a mut [u8],
126        purpose: &Option<RawEncodePurpose>,
127    ) -> BuckyResult<&'a mut [u8]> {
128        let ret = if purpose == &Some(RawEncodePurpose::Hash) {
129            let remain_buf = self.attributes.raw_encode(buf, purpose)?;
130            match &self.obj_list {
131                NDNObjectInfo::Chunk(chunk_id) => chunk_id.raw_encode(remain_buf, purpose).unwrap(),
132                NDNObjectInfo::ObjList(list) => {
133                    // 把list编码到chunk
134                    let size = list.raw_measure(&Some(RawEncodePurpose::Serialize))?;
135                    let mut chunk_buf = vec![0u8; size];
136                    let left_buf =
137                        list.raw_encode(&mut chunk_buf, &Some(RawEncodePurpose::Serialize))?;
138                    assert!(left_buf.len() == 0);
139
140                    let chunk_id = ChunkId::calculate_sync(&chunk_buf).unwrap();
141                    assert!(chunk_id.len() == size);
142
143                    chunk_id.raw_encode(remain_buf, purpose).unwrap()
144                }
145            }
146        } else {
147            let buf = self.attributes.raw_encode(buf, purpose)?;
148            self.obj_list.raw_encode(buf, purpose)?
149        };
150        Ok(ret)
151    }
152}
153
154impl DirDescContent {
155    pub fn new(attributes: Attributes, obj_list: NDNObjectInfo) -> Self {
156        Self {
157            attributes,
158            obj_list,
159        }
160    }
161
162    pub fn attributes(&self) -> &Attributes {
163        &self.attributes
164    }
165
166    pub fn obj_list(&self) -> &NDNObjectInfo {
167        &self.obj_list
168    }
169}
170
171impl DescContent for DirDescContent {
172    fn obj_type() -> u16 {
173        ObjectTypeCode::Dir.into()
174    }
175
176    type OwnerType = Option<ObjectId>;
177    type AreaType = SubDescNone;
178    type AuthorType = Option<ObjectId>;
179    type PublicKeyType = SubDescNone;
180}
181
182pub type DirBodyContentObjectList = HashMap<ObjectId, Vec<u8>>;
183
184#[derive(Clone, Debug)]
185pub enum DirBodyContent {
186    // 要么把ObjList压缩放chunk
187    // TODO? 但是Chunk要放在哪里？
188    Chunk(ChunkId),
189
190    // 要么直接展开持有
191    ObjList(DirBodyContentObjectList),
192}
193
194impl BodyContent for DirBodyContent {
195    fn format(&self) -> u8 {
196        OBJECT_CONTENT_CODEC_FORMAT_PROTOBUF
197    }
198}
199
200impl TryFrom<protos::DirBodyContent> for DirBodyContent {
201    type Error = BuckyError;
202
203    fn try_from(mut value: protos::DirBodyContent) -> BuckyResult<Self> {
204        let ret = match value.get_field_type() {
205            protos::DirBodyContent_Type::Chunk => {
206                Self::Chunk(ProtobufCodecHelper::decode_buf(value.take_chunk_id())?)
207            }
208            protos::DirBodyContent_Type::ObjList => {
209                let mut ret = HashMap::new();
210                for mut item in value.take_obj_list().into_iter() {
211                    let (k, _) = ObjectId::raw_decode(&item.get_obj_id())?;
212                    let v = item.take_value();
213                    if let Some(_old) = ret.insert(k, v) {
214                        error!("decode dir obj_list from protobuf got repeated key! {}", k);
215                    }
216                }
217                Self::ObjList(ret)
218            }
219        };
220
221        Ok(ret)
222    }
223}
224
225impl TryFrom<&DirBodyContent> for protos::DirBodyContent {
226    type Error = BuckyError;
227
228    fn try_from(value: &DirBodyContent) -> BuckyResult<Self> {
229        let mut ret = protos::DirBodyContent::new();
230        match value {
231            DirBodyContent::Chunk(id) => {
232                ret.set_field_type(protos::DirBodyContent_Type::Chunk);
233                ret.set_chunk_id(id.to_vec()?);
234            }
235            DirBodyContent::ObjList(list) => {
236                ret.set_field_type(protos::DirBodyContent_Type::ObjList);
237                let mut item_list = Vec::new();
238                for (k, v) in list {
239                    let mut item = protos::DirBodyContent_ObjItem::new();
240                    item.set_obj_id(k.to_vec()?);
241                    item.set_value(v.to_owned());
242                    item_list.push(item);
243                }
244
245                item_list.sort_by(|left, right| left.obj_id.partial_cmp(&right.obj_id).unwrap());
246                ret.set_obj_list(item_list.into());
247            }
248        }
249
250        Ok(ret)
251    }
252}
253
254inner_impl_default_protobuf_raw_codec!(DirBodyContent);
255
256pub type DirType = NamedObjType<DirDescContent, DirBodyContent>;
257pub type DirBuilder = NamedObjectBuilder<DirDescContent, DirBodyContent>;
258
259pub type DirDesc = NamedObjectDesc<DirDescContent>;
260pub type DirId = NamedObjectId<DirType>;
261pub type Dir = NamedObjectBase<DirType>;
262
263impl DirDesc {
264    pub fn dir_id(&self) -> DirId {
265        DirId::try_from(self.calculate_id()).unwrap()
266    }
267}
268
269impl Dir {
270    pub fn new(
271        dir_attributes: Attributes,
272        obj_desc: NDNObjectInfo,
273        obj_map: HashMap<ObjectId, Vec<u8>>,
274    ) -> DirBuilder {
275        let desc_content = DirDescContent::new(dir_attributes, obj_desc);
276        let body_content = DirBodyContent::ObjList(obj_map);
277        DirBuilder::new(desc_content, body_content)
278    }
279
280    pub fn new_with_chunk_body(
281        dir_attributes: Attributes,
282        obj_desc: NDNObjectInfo,
283        chunk_body: ChunkId,
284    ) -> DirBuilder {
285        let desc_content = DirDescContent::new(dir_attributes, obj_desc);
286        let body_content = DirBodyContent::Chunk(chunk_body);
287        DirBuilder::new(desc_content, body_content)
288    }
289
290    pub fn get_data_from_body(&self, id: &ObjectId) -> Option<&Vec<u8>> {
291        match self.body() {
292            Some(body) => {
293                match body.content() {
294                    DirBodyContent::ObjList(list) => list.get(id),
295                    DirBodyContent::Chunk(_chunk_id) => {
296                        // 不处理这种情况，上层需要对这种情况进一步展开成ObjList模式才可以进一步查询
297                        None
298                    }
299                }
300            }
301            None => None,
302        }
303    }
304
305    // desc content最大支持65535长度，如果超出此长度，需要切换为chunk模式
306    pub fn check_and_fix_desc_limit(&mut self) -> BuckyResult<()> {
307        let size = self.desc().content().raw_measure(&None)?;
308        if size > u16::MAX as usize {
309            match &self.desc_mut().content().obj_list {
310                NDNObjectInfo::ObjList(list) => {
311                    let chunk = list.to_vec()?;
312                    let chunk_id = ChunkId::calculate_sync(&chunk)?;
313                    drop(list);
314
315                    match self.body_mut() {
316                        Some(body) => match body.content_mut() {
317                            DirBodyContent::ObjList(list) => {
318                                info!("will convert dir desc content list to chunk: list len={}, chunk_id={}",
319                                    size, chunk_id);
320                                list.insert(chunk_id.object_id(), chunk);
321                                drop(list);
322
323                                self.desc_mut().content_mut().obj_list =
324                                    NDNObjectInfo::Chunk(chunk_id.clone());
325                            }
326                            DirBodyContent::Chunk(_chunk_id) => {
327                                // 不支持body的chunk模式
328                                let msg = format!(
329                                    "fix dir desc limit not support body chunk mode! dir={}",
330                                    self.desc().dir_id()
331                                );
332                                error!("{}", msg);
333                                return Err(BuckyError::new(BuckyErrorCode::NotSupport, msg));
334                            }
335                        },
336                        None => {
337                            info!("will convert dir desc content list to chunk: list len={}, chunk_id={}",
338                                    size, chunk_id);
339
340                            // 如果dir不存在body，那么要动态的创建body
341                            let mut object_list = DirBodyContentObjectList::new();
342                            object_list.insert(chunk_id.object_id(), chunk);
343
344                            let builder =
345                                ObjectMutBodyBuilder::new(DirBodyContent::ObjList(object_list));
346                            let body = builder.update_time(bucky_time_now()).build();
347
348                            *self.body_mut() = Some(body);
349
350                            self.desc_mut().content_mut().obj_list =
351                                NDNObjectInfo::Chunk(chunk_id.clone());
352                        }
353                    }
354                }
355                _ => {}
356            }
357        }
358
359        Ok(())
360    }
361}
362
363#[cfg(test)]
364mod test {
365    use crate::*;
366    use std::collections::HashMap;
367
368    #[test]
369    fn dir() {
370        let inner_node =
371            InnerNodeInfo::new(Attributes::default(), InnerNode::ObjId(ObjectId::default()));
372
373        let mut object_map = HashMap::new();
374        object_map.insert("path1".to_owned(), inner_node);
375        let list = NDNObjectList {
376            parent_chunk: None,
377            object_map,
378        };
379        // 第一种情况，构造一个普通大小的dir，内容可以放到desc里面
380        let attr = Attributes::new(0xFFFF);
381        let builder = Dir::new(
382            attr.clone(),
383            NDNObjectInfo::ObjList(list.clone()),
384            HashMap::new(),
385        );
386        let dir = builder.no_create_time().update_time(0).build();
387        let dir_id = dir.desc().calculate_id();
388        println!("dir id={}", dir_id);
389        assert_eq!(
390            dir_id.to_string(),
391            "7jMmeXZpjj4YRfshnxsTqyDbqyo9zDoDA5phG9AXDC7X"
392        );
393        let buf = dir.to_vec().unwrap();
394        let hash = hash_data(&buf);
395        info!("dir hash={}", hash);
396        // 第二种情况，对于超大内容的dir，使用chunk模式，但和上面一种模式是对等的
397        let data = list.to_vec().unwrap();
398        let chunk_id = ChunkId::calculate_sync(&data).unwrap();
399
400        // chunk可以放到body缓存里面，方便查找；也可以独立存放，但dir在解析时候需要再次查找该chunk可能会耗时久，以及查找失败等情况
401        let mut obj_map = HashMap::new();
402        obj_map.insert(chunk_id.object_id(), data);
403
404        let builder = Dir::new(attr.clone(), NDNObjectInfo::Chunk(chunk_id.clone()), obj_map.clone());
405        let dir = builder.no_create_time().update_time(0).build();
406        let dir_id2 = dir.desc().calculate_id();
407        info!("dir id2={}", dir_id2);
408        let buf = dir.to_vec().unwrap();
409        let hash = hash_data(&buf);
410        info!("dir2 hash={}", hash);
411
412        let _dir3 = AnyNamedObject::clone_from_slice(&buf).unwrap();
413
414        // 上述两种模式生成的dir_id应该是相同
415        assert_eq!(dir_id, dir_id2);
416
417        // body也可以放到chunk,由于只是影响body的结构，所以不影响dir的object_id
418        let body_data = obj_map.to_vec().unwrap();
419        let body_chunk_id = ChunkId::calculate_sync(&body_data).unwrap();
420        // 注意： body_chunk_id需要额外的保存到本地，put_data(body_chunk, body_chunk_id)
421
422        let builder = Dir::new_with_chunk_body(attr.clone(), NDNObjectInfo::Chunk(chunk_id), body_chunk_id);
423        let dir = builder.no_create_time().update_time(0).build();
424        let dir_id3 = dir.desc().calculate_id();
425        assert_eq!(dir_id, dir_id3);
426    }
427
428    #[test]
429    fn test_fix_limit() {
430        let inner_node =
431            InnerNodeInfo::new(Attributes::default(), InnerNode::ObjId(ObjectId::default()));
432
433        let mut object_map = HashMap::new();
434        for i in 0..1024 * 10 {
435            let path = format!("test dir path {}", i);
436            object_map.insert(path, inner_node.clone());
437        }
438        let list = NDNObjectList {
439            parent_chunk: None,
440            object_map,
441        };
442
443        let builder = Dir::new(
444            Attributes::default(),
445            NDNObjectInfo::ObjList(list),
446            HashMap::new(),
447        );
448        let mut dir = builder.no_create_time().update_time(0).build();
449        *dir.body_mut() = None;
450        let ret = dir.raw_measure(&None).unwrap_err();
451        assert!(ret.code() == BuckyErrorCode::OutOfLimit);
452
453        dir.check_and_fix_desc_limit().unwrap();
454
455        let size = dir.raw_measure(&None).unwrap();
456        println!("dir len: {}", size);
457    }
458}
bucky_objects/objects/dir.rs

bucky_objects/objects/
dir.rs
