nydus_builder/
merge.rs

1// Copyright (C) 2022 Nydus Developers. All rights reserved.
2//
3// SPDX-License-Identifier: Apache-2.0
4
5use std::collections::hash_map::Entry;
6use std::collections::{HashMap, HashSet};
7use std::convert::TryFrom;
8use std::path::PathBuf;
9use std::sync::Arc;
10
11use anyhow::{anyhow, bail, ensure, Context, Result};
12use hex::FromHex;
13use nydus_api::ConfigV2;
14use nydus_rafs::metadata::{RafsSuper, RafsVersion};
15use nydus_storage::device::{BlobFeatures, BlobInfo};
16use nydus_utils::crypt;
17
18use super::{
19    ArtifactStorage, BlobContext, BlobManager, Bootstrap, BootstrapContext, BuildContext,
20    BuildOutput, ChunkSource, ConversionType, Overlay, Tree,
21};
22
23/// Struct to generate the merged RAFS bootstrap for an image from per layer RAFS bootstraps.
24///
25/// A container image contains one or more layers, a RAFS bootstrap is built for each layer.
26/// Those per layer bootstraps could be mounted by overlayfs to form the container rootfs.
27/// To improve performance by avoiding overlayfs, an image level bootstrap is generated by
28/// merging per layer bootstrap with overlayfs rules applied.
29pub struct Merger {}
30
31impl Merger {
32    fn get_string_from_list(
33        original_ids: &Option<Vec<String>>,
34        idx: usize,
35    ) -> Result<Option<String>> {
36        Ok(if let Some(id) = &original_ids {
37            let id_string = id
38                .get(idx)
39                .ok_or_else(|| anyhow!("unmatched digest index {}", idx))?;
40            Some(id_string.clone())
41        } else {
42            None
43        })
44    }
45
46    fn get_digest_from_list(digests: &Option<Vec<String>>, idx: usize) -> Result<Option<[u8; 32]>> {
47        Ok(if let Some(digests) = &digests {
48            let digest = digests
49                .get(idx)
50                .ok_or_else(|| anyhow!("unmatched digest index {}", idx))?;
51            Some(<[u8; 32]>::from_hex(digest)?)
52        } else {
53            None
54        })
55    }
56
57    fn get_size_from_list(sizes: &Option<Vec<u64>>, idx: usize) -> Result<Option<u64>> {
58        Ok(if let Some(sizes) = &sizes {
59            let size = sizes
60                .get(idx)
61                .ok_or_else(|| anyhow!("unmatched size index {}", idx))?;
62            Some(*size)
63        } else {
64            None
65        })
66    }
67
68    /// Overlay multiple RAFS filesystems into a merged RAFS filesystem.
69    ///
70    /// # Arguments
71    /// - sources: contains one or more per layer bootstraps in order of lower to higher.
72    /// - chunk_dict: contain the chunk dictionary used to build per layer boostrap, or None.
73    #[allow(clippy::too_many_arguments)]
74    pub fn merge(
75        ctx: &mut BuildContext,
76        parent_bootstrap_path: Option<String>,
77        sources: Vec<PathBuf>,
78        blob_digests: Option<Vec<String>>,
79        original_blob_ids: Option<Vec<String>>,
80        blob_sizes: Option<Vec<u64>>,
81        blob_toc_digests: Option<Vec<String>>,
82        blob_toc_sizes: Option<Vec<u64>>,
83        target: ArtifactStorage,
84        chunk_dict: Option<PathBuf>,
85        config_v2: Arc<ConfigV2>,
86    ) -> Result<BuildOutput> {
87        if sources.is_empty() {
88            bail!("source bootstrap list is empty , at least one bootstrap is required");
89        }
90        if let Some(digests) = blob_digests.as_ref() {
91            ensure!(
92                digests.len() == sources.len(),
93                "number of blob digest entries {} doesn't match number of sources {}",
94                digests.len(),
95                sources.len(),
96            );
97        }
98        if let Some(original_ids) = original_blob_ids.as_ref() {
99            ensure!(
100                original_ids.len() == sources.len(),
101                "number of original blob id entries {} doesn't match number of sources {}",
102                original_ids.len(),
103                sources.len(),
104            );
105        }
106        if let Some(sizes) = blob_sizes.as_ref() {
107            ensure!(
108                sizes.len() == sources.len(),
109                "number of blob size entries {} doesn't match number of sources {}",
110                sizes.len(),
111                sources.len(),
112            );
113        }
114        if let Some(toc_digests) = blob_toc_digests.as_ref() {
115            ensure!(
116                toc_digests.len() == sources.len(),
117                "number of toc digest entries {} doesn't match number of sources {}",
118                toc_digests.len(),
119                sources.len(),
120            );
121        }
122        if let Some(sizes) = blob_toc_sizes.as_ref() {
123            ensure!(
124                sizes.len() == sources.len(),
125                "number of toc size entries {} doesn't match number of sources {}",
126                sizes.len(),
127                sources.len(),
128            );
129        }
130
131        let mut tree: Option<Tree> = None;
132        let mut blob_mgr = BlobManager::new(ctx.digester, false);
133        let mut blob_idx_map = HashMap::new();
134        let mut parent_layers = 0;
135
136        // Load parent bootstrap
137        if let Some(parent_bootstrap_path) = &parent_bootstrap_path {
138            let (rs, _) =
139                RafsSuper::load_from_file(parent_bootstrap_path, config_v2.clone(), false)
140                    .context(format!("load parent bootstrap {:?}", parent_bootstrap_path))?;
141            let blobs = rs.superblock.get_blob_infos();
142            for blob in &blobs {
143                let blob_ctx = BlobContext::from(ctx, &blob, ChunkSource::Parent)?;
144                blob_idx_map.insert(blob_ctx.blob_id.clone(), blob_mgr.len());
145                blob_mgr.add_blob(blob_ctx);
146            }
147            parent_layers = blobs.len();
148            tree = Some(Tree::from_bootstrap(&rs, &mut ())?);
149        }
150
151        // Get the blobs come from chunk dictionary.
152        let mut chunk_dict_blobs = HashSet::new();
153        let mut config = None;
154        if let Some(chunk_dict_path) = &chunk_dict {
155            let (rs, _) = RafsSuper::load_from_file(chunk_dict_path, config_v2.clone(), false)
156                .context(format!("load chunk dict bootstrap {:?}", chunk_dict_path))?;
157            config = Some(rs.meta.get_config());
158            for blob in rs.superblock.get_blob_infos() {
159                chunk_dict_blobs.insert(blob.blob_id().to_string());
160            }
161        }
162
163        let mut fs_version = RafsVersion::V6;
164        let mut chunk_size = None;
165
166        for (layer_idx, bootstrap_path) in sources.iter().enumerate() {
167            let (rs, _) = RafsSuper::load_from_file(bootstrap_path, config_v2.clone(), false)
168                .context(format!("load bootstrap {:?}", bootstrap_path))?;
169            config
170                .get_or_insert_with(|| rs.meta.get_config())
171                .check_compatibility(&rs.meta)?;
172            fs_version = RafsVersion::try_from(rs.meta.version)
173                .context("failed to get RAFS version number")?;
174            ctx.compressor = rs.meta.get_compressor();
175            ctx.digester = rs.meta.get_digester();
176            // If any RAFS filesystems are encrypted, the merged boostrap will be marked as encrypted.
177            match rs.meta.get_cipher() {
178                crypt::Algorithm::None => (),
179                crypt::Algorithm::Aes128Xts => ctx.cipher = crypt::Algorithm::Aes128Xts,
180                _ => bail!("invalid per layer bootstrap, only supports aes-128-xts"),
181            }
182            ctx.explicit_uidgid = rs.meta.explicit_uidgid();
183            if config.as_ref().unwrap().is_tarfs_mode {
184                ctx.conversion_type = ConversionType::TarToTarfs;
185                ctx.blob_features |= BlobFeatures::TARFS;
186            }
187
188            let mut parent_blob_added = false;
189            let blobs = &rs.superblock.get_blob_infos();
190            for blob in blobs {
191                let mut blob_ctx = BlobContext::from(ctx, &blob, ChunkSource::Parent)?;
192                if let Some(chunk_size) = chunk_size {
193                    ensure!(
194                        chunk_size == blob_ctx.chunk_size,
195                        "can not merge bootstraps with inconsistent chunk size, current bootstrap {:?} with chunk size {:x}, expected {:x}",
196                        bootstrap_path,
197                        blob_ctx.chunk_size,
198                        chunk_size,
199                    );
200                } else {
201                    chunk_size = Some(blob_ctx.chunk_size);
202                }
203                if !chunk_dict_blobs.contains(&blob.blob_id()) {
204                    // It is assumed that the `nydus-image create` at each layer and `nydus-image merge` commands
205                    // use the same chunk dict bootstrap. So the parent bootstrap includes multiple blobs, but
206                    // only at most one new blob, the other blobs should be from the chunk dict image.
207                    if parent_blob_added {
208                        bail!("invalid per layer bootstrap, having multiple associated data blobs");
209                    }
210                    parent_blob_added = true;
211
212                    if ctx.configuration.internal.blob_accessible()
213                        || ctx.conversion_type == ConversionType::TarToTarfs
214                    {
215                        // `blob.blob_id()` should have been fixed when loading the bootstrap.
216                        blob_ctx.blob_id = blob.blob_id();
217                    } else {
218                        // The blob id (blob sha256 hash) in parent bootstrap is invalid for nydusd
219                        // runtime, should change it to the hash of whole tar blob.
220                        if let Some(original_id) =
221                            Self::get_string_from_list(&original_blob_ids, layer_idx)?
222                        {
223                            blob_ctx.blob_id = original_id;
224                        } else {
225                            blob_ctx.blob_id =
226                                BlobInfo::get_blob_id_from_meta_path(bootstrap_path)?;
227                        }
228                    }
229                    if let Some(digest) = Self::get_digest_from_list(&blob_digests, layer_idx)? {
230                        if blob.has_feature(BlobFeatures::SEPARATE) {
231                            blob_ctx.blob_meta_digest = digest;
232                        } else {
233                            blob_ctx.blob_id = hex::encode(digest);
234                        }
235                    }
236                    if let Some(size) = Self::get_size_from_list(&blob_sizes, layer_idx)? {
237                        if blob.has_feature(BlobFeatures::SEPARATE) {
238                            blob_ctx.blob_meta_size = size;
239                        } else {
240                            blob_ctx.compressed_blob_size = size;
241                        }
242                    }
243                    if let Some(digest) = Self::get_digest_from_list(&blob_toc_digests, layer_idx)?
244                    {
245                        blob_ctx.blob_toc_digest = digest;
246                    }
247                    if let Some(size) = Self::get_size_from_list(&blob_toc_sizes, layer_idx)? {
248                        blob_ctx.blob_toc_size = size as u32;
249                    }
250                }
251
252                if let Entry::Vacant(e) = blob_idx_map.entry(blob.blob_id()) {
253                    e.insert(blob_mgr.len());
254                    blob_mgr.add_blob(blob_ctx);
255                }
256            }
257
258            let upper = Tree::from_bootstrap(&rs, &mut ())?;
259            upper.walk_bfs(true, &mut |n| {
260                let mut node = n.borrow_mut_node();
261                for chunk in &mut node.chunks {
262                    let origin_blob_index = chunk.inner.blob_index() as usize;
263                    let blob_ctx = blobs[origin_blob_index].as_ref();
264                    if let Some(blob_index) = blob_idx_map.get(&blob_ctx.blob_id()) {
265                        // Set the blob index of chunk to real index in blob table of final bootstrap.
266                        chunk.set_blob_index(*blob_index as u32);
267                    }
268                }
269                // Set node's layer index to distinguish same inode number (from bootstrap)
270                // between different layers.
271                let idx = u16::try_from(layer_idx).context(format!(
272                    "too many layers {}, limited to {}",
273                    layer_idx,
274                    u16::MAX
275                ))?;
276                if parent_layers + idx as usize > u16::MAX as usize {
277                    bail!("too many layers {}, limited to {}", layer_idx, u16::MAX);
278                }
279                node.layer_idx = idx + parent_layers as u16;
280                node.overlay = Overlay::UpperAddition;
281                Ok(())
282            })?;
283
284            if let Some(tree) = &mut tree {
285                tree.merge_overaly(ctx, upper)?;
286            } else {
287                tree = Some(upper);
288            }
289        }
290
291        if ctx.conversion_type == ConversionType::TarToTarfs {
292            if parent_layers > 0 {
293                bail!("merging RAFS in TARFS mode conflicts with `--parent-bootstrap`");
294            }
295            if !chunk_dict_blobs.is_empty() {
296                bail!("merging RAFS in TARFS mode conflicts with `--chunk-dict`");
297            }
298        }
299
300        // Safe to unwrap because there is at least one source bootstrap.
301        let tree = tree.unwrap();
302        ctx.fs_version = fs_version;
303        if let Some(chunk_size) = chunk_size {
304            ctx.chunk_size = chunk_size;
305        }
306
307        // After merging all trees, we need to re-calculate the blob index of
308        // referenced blobs, as the upper tree might have deleted some files
309        // or directories by opaques, and some blobs are dereferenced.
310        let mut used_blobs = HashMap::new(); // HashMap<blob_id, new_blob_index>
311        let mut used_blob_mgr = BlobManager::new(ctx.digester, false);
312        let origin_blobs = blob_mgr.get_blobs();
313        tree.walk_bfs(true, &mut |n| {
314            let mut node = n.borrow_mut_node();
315            for chunk in &mut node.chunks {
316                let origin_blob_index = chunk.inner.blob_index() as usize;
317                let blob_ctx = origin_blobs[origin_blob_index].clone();
318                let origin_blob_id = blob_ctx.blob_id();
319                let new_blob_index = if let Some(new_blob_index) = used_blobs.get(&origin_blob_id) {
320                    *new_blob_index
321                } else {
322                    let new_blob_index = used_blob_mgr.len();
323                    used_blobs.insert(origin_blob_id, new_blob_index);
324                    used_blob_mgr.add_blob(blob_ctx);
325                    new_blob_index
326                };
327                chunk.set_blob_index(new_blob_index as u32);
328            }
329            Ok(())
330        })?;
331
332        let mut bootstrap_ctx = BootstrapContext::new(Some(target.clone()), false)?;
333        let mut bootstrap = Bootstrap::new(tree)?;
334        bootstrap.build(ctx, &mut bootstrap_ctx)?;
335        let blob_table = used_blob_mgr.to_blob_table(ctx)?;
336        let mut bootstrap_storage = Some(target.clone());
337        bootstrap
338            .dump(ctx, &mut bootstrap_storage, &mut bootstrap_ctx, &blob_table)
339            .context(format!("dump bootstrap to {:?}", target.display()))?;
340        BuildOutput::new(&used_blob_mgr, None, &bootstrap_storage, &None)
341    }
342}
343
344#[cfg(test)]
345mod tests {
346    use nydus_utils::digest;
347    use vmm_sys_util::tempfile::TempFile;
348
349    use super::*;
350
351    #[test]
352    fn test_merger_get_string_from_list() {
353        let res = Merger::get_string_from_list(&None, 1);
354        assert!(res.is_ok());
355        assert!(res.unwrap().is_none());
356
357        let original_ids = vec!["string1".to_owned(), "string2".to_owned()];
358        let original_ids = Some(original_ids);
359
360        let res = Merger::get_string_from_list(&original_ids, 0);
361        assert!(res.is_ok());
362        assert_eq!(res.unwrap(), Some("string1".to_owned()));
363        assert!(Merger::get_string_from_list(&original_ids, 2).is_err());
364    }
365
366    #[test]
367    fn test_merger_get_digest_from_list() {
368        let res = Merger::get_digest_from_list(&None, 1);
369        assert!(res.is_ok());
370        assert!(res.unwrap().is_none());
371
372        let original_ids = vec!["string1".to_owned(), "12ab".repeat(16)];
373        let original_ids = Some(original_ids);
374
375        let res = Merger::get_digest_from_list(&original_ids, 1);
376        assert!(res.is_ok());
377        assert_eq!(
378            res.unwrap(),
379            Some([
380                18u8, 171, 18, 171, 18, 171, 18, 171, 18, 171, 18, 171, 18, 171, 18, 171, 18, 171,
381                18, 171, 18, 171, 18, 171, 18, 171, 18, 171, 18, 171, 18, 171
382            ])
383        );
384        assert!(Merger::get_digest_from_list(&original_ids, 0).is_err());
385        assert!(Merger::get_digest_from_list(&original_ids, 2).is_err());
386    }
387
388    #[test]
389    fn test_merger_get_size_from_list() {
390        let res = Merger::get_size_from_list(&None, 1);
391        assert!(res.is_ok());
392        assert!(res.unwrap().is_none());
393
394        let original_ids = vec![1u64, 2, 3, 4];
395        let original_ids = Some(original_ids);
396        let res = Merger::get_size_from_list(&original_ids, 1);
397        assert!(res.is_ok());
398        assert_eq!(res.unwrap(), Some(2u64));
399        assert!(Merger::get_size_from_list(&original_ids, 4).is_err());
400    }
401
402    #[test]
403    fn test_merger_merge() {
404        let mut ctx = BuildContext::default();
405        ctx.configuration.internal.set_blob_accessible(false);
406        ctx.digester = digest::Algorithm::Sha256;
407
408        let root_dir = &std::env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR");
409        let mut source_path1 = PathBuf::from(root_dir);
410        source_path1.push("../tests/texture/bootstrap/rafs-v6-2.2.boot");
411        let mut source_path2 = PathBuf::from(root_dir);
412        source_path2.push("../tests/texture/bootstrap/rafs-v6-2.2.boot");
413
414        let tmp_file = TempFile::new().unwrap();
415        let target = ArtifactStorage::SingleFile(tmp_file.as_path().to_path_buf());
416
417        let blob_toc_digests = Some(vec![
418            "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855".to_owned(),
419            "4cf0c409788fc1c149afbf4c81276b92427ae41e46412334ca495991b8526650".to_owned(),
420        ]);
421
422        let build_output = Merger::merge(
423            &mut ctx,
424            None,
425            vec![source_path1, source_path2],
426            Some(vec!["a70f".repeat(16), "9bd3".repeat(16)]),
427            Some(vec!["blob_id".to_owned(), "blob_id2".to_owned()]),
428            Some(vec![16u64, 32u64]),
429            blob_toc_digests,
430            Some(vec![64u64, 128]),
431            target,
432            None,
433            Arc::new(ConfigV2::new("config_v2")),
434        );
435        assert!(build_output.is_ok());
436        let build_output = build_output.unwrap();
437        println!("BuildOutput: {}", build_output);
438        assert_eq!(build_output.blob_size, Some(16));
439    }
440}
nydus_builder/merge.rs

nydus_builder/
merge.rs
